fix crashes when C++ global destructors call DDS code

- properly count invocations of os_osInit/os_osExit
- handle concurrent invocations of those
- provide a single mutex for use by dds_init
- eliminate use of atexit()
- attendant dds_init/dds_fini changes
- fix related crash in thread local storage cleanup

Signed-off-by: Erik Boasson <eb@ilities.com>

src/core/ddsc/src/dds__init.h

@@ -22,16 +22,6 @@ dds_return_t
 dds__check_domain(
         _In_ dds_domainid_t domain);
 
-/**
- *Description : Initialization function. This operation initializes all the
- *required resources that are needed for the DDSC API process lifecycle
- *(like the init mutex and os layer).
- *A function will be registered that is called at the end of the process
- *lifecycle and will destroy the resources initialized in this function.
- **/
-void
-dds__startup(void);
-
 /**
  *Description : Initialization function, called from main. This operation
  *initializes all the required DDS resources,
@@ -66,12 +56,6 @@ dds_fini(void);
 dds_domainid_t dds_domain_default (void);
 
 
-/**
- * Description : Mutex used for initialization synchronization.
- */
-extern os_mutex dds__init_mutex;
-
-
 #if defined (__cplusplus)
 }
 #endif

src/core/ddsc/src/dds_init.c

@@ -45,36 +45,6 @@ dds_globals dds_global =
 
 static struct cfgst * dds_cfgst = NULL;
 
-
-
-
-os_mutex dds__init_mutex;
-
-static void
-dds__fini_once(void)
-{
-  os_mutexDestroy(&dds__init_mutex);
-  os_osExit();
-}
-
-static void
-dds__init_once(void)
-{
-  os_osInit();
-  os_mutexInit(&dds__init_mutex);
-  os_procAtExit(dds__fini_once);
-}
-
-
-
-void
-dds__startup(void)
-{
-    static os_once_t dds__init_control = OS_ONCE_T_STATIC_INIT;
-    os_once(&dds__init_control, dds__init_once);
-}
-
-
 dds_return_t
 dds_init(void)
 {
@@ -83,13 +53,15 @@ dds_init(void)
   char progname[50];
   char hostname[64];
   uint32_t len;
+  os_mutex *init_mutex;
 
   /* Be sure the DDS lifecycle resources are initialized. */
-  dds__startup();
+  os_osInit();
+  init_mutex = os_getSingletonMutex();
 
   DDS_REPORT_STACK();
 
-  os_mutexLock(&dds__init_mutex);
+  os_mutexLock(init_mutex);
 
   dds_global.m_init_count++;
   if (dds_global.m_init_count > 1)
@@ -181,7 +153,7 @@ dds_init(void)
   gv.default_plist_pp.present |= PP_ENTITY_NAME;
 
 skip:
-  os_mutexUnlock(&dds__init_mutex);
+  os_mutexUnlock(init_mutex);
   DDS_REPORT_FLUSH(false);
   return DDS_RETCODE_OK;
 
@@ -203,8 +175,9 @@ fail_config:
   ut_handleserver_fini();
 fail_handleserver:
   dds_global.m_init_count--;
-  os_mutexUnlock(&dds__init_mutex);
+  os_mutexUnlock(init_mutex);
   DDS_REPORT_FLUSH(true);
+  os_osExit();
   return ret;
 }
 
@@ -212,7 +185,9 @@ fail_handleserver:
 
 extern void dds_fini (void)
 {
-  os_mutexLock(&dds__init_mutex);
+  os_mutex *init_mutex;
+  init_mutex = os_getSingletonMutex();
+  os_mutexLock(init_mutex);
   assert(dds_global.m_init_count > 0);
   dds_global.m_init_count--;
   if (dds_global.m_init_count == 0)
@@ -232,7 +207,8 @@ extern void dds_fini (void)
     os_mutexDestroy (&dds_global.m_mutex);
     dds_global.m_default_domain = DDS_DOMAIN_DEFAULT;
   }
-  os_mutexUnlock(&dds__init_mutex);
+  os_mutexUnlock(init_mutex);
+  os_osExit();
 }
 
 

src/core/ddsc/src/dds_participant.c

@@ -145,9 +145,6 @@ dds_create_participant(
     struct thread_state1 * thr;
     bool asleep;
 
-    /* Be sure the DDS lifecycle resources are initialized. */
-    dds__startup();
-
     /* Make sure DDS instance is initialized. */
     ret = dds_init();
     if (ret != DDS_RETCODE_OK) {
@@ -243,9 +240,11 @@ dds_lookup_participant(
         _In_        size_t size)
 {
     dds_return_t ret = 0;
+    os_mutex *init_mutex;
 
     /* Be sure the DDS lifecycle resources are initialized. */
-    dds__startup();
+    os_osInit();
+    init_mutex = os_getSingletonMutex();
 
     DDS_REPORT_STACK();
 
@@ -262,7 +261,7 @@ dds_lookup_participant(
         participants[0] = 0;
     }
 
-    os_mutexLock (&dds__init_mutex);
+    os_mutexLock (init_mutex);
 
     /* Check if dds is intialized. */
     if (dds_global.m_init_count > 0) {
@@ -281,9 +280,10 @@ dds_lookup_participant(
         os_mutexUnlock (&dds_global.m_mutex);
     }
 
-    os_mutexUnlock (&dds__init_mutex);
+    os_mutexUnlock (init_mutex);
 
 err:
     DDS_REPORT_FLUSH(ret != DDS_RETCODE_OK);
+    os_osExit();
     return ret;
 }

src/core/ddsi/src/q_thread.c

@@ -110,7 +110,7 @@ cleanup_thread_state(
     assert(ts->state == THREAD_STATE_ALIVE);
     assert(vtime_asleep_p(ts->vtime));
     reset_thread_state(ts);
-    os_reportExit(); /* FIXME: Should not be here! */
+    os_osExit();
 }
 
 _Ret_valid_ struct thread_state1 *
@@ -133,6 +133,7 @@ lookup_thread_state(
             os_mutexLock(&thread_states.lock);
             ts1 = init_thread_state(tname);
             if (ts1 != NULL) {
+                os_osInit();
                 ts1->lb = 0;
                 ts1->extTid = tid;
                 ts1->tid = tid;

src/os/CMakeLists.txt

@@ -20,7 +20,7 @@ ENDIF()
 PREPEND(srcs_platform ${platform} os_platform_errno.c os_platform_heap.c os_platform_init.c os_platform_process.c os_platform_socket.c os_platform_stdlib.c os_platform_sync.c os_platform_thread.c os_platform_time.c)
 
 include (GenerateExportHeader)
-PREPEND(srcs_os "${CMAKE_CURRENT_SOURCE_DIR}/src" os_atomics.c os_init.c os_process.c os_report.c os_socket.c os_thread.c os_time.c os_errno.c os_iter.c ${srcs_platform})
+PREPEND(srcs_os "${CMAKE_CURRENT_SOURCE_DIR}/src" os_atomics.c os_init.c os_report.c os_socket.c os_thread.c os_time.c os_errno.c os_iter.c ${srcs_platform})
 add_library(OSAPI ${srcs_os})
 
 configure_file(

src/os/include/os/os_init.h

@@ -15,4 +15,8 @@
 void os_osInit(void);
 void os_osExit(void);
 
-#endif
+/* implemented by the platform-specific code */
+void os_osPlatformInit (void);
+void os_osPlatformExit (void);
+
+#endif

src/os/include/os/os_process.h

@@ -72,22 +72,6 @@ os_procName(
     _Out_writes_z_(procNameSize) char *procName,
     _In_ size_t procNameSize);
 
-
-/** \brief Register an process exit handler
- *
- * Register an process exit handler. Multiple handlers may be
- * registered. The handlers are called in reverse order of
- * registration.
- *
- * Possible Results:
- * - os_resultSuccess: function registered
- * - os_resultFail: function could not be registered
- * - assertion failure: function = NULL
- */
-OSAPI_EXPORT os_result
-os_procAtExit(
-    _In_ void (*function)(void));
-
 #if defined (__cplusplus)
 }
 #endif

src/os/include/os/os_sync.h

@@ -360,6 +360,10 @@ extern "C" {
         _Inout_ os_once_t *control,
         _In_ os_once_fn init_fn);
 
+    OSAPI_EXPORT os_mutex *
+    os_getSingletonMutex(
+        void);
+
 #if defined (__cplusplus)
 }
 #endif

src/os/src/os_init.c

@@ -29,3 +29,53 @@ os_versionString(void)
 {
     return OSPL_VERSION_STR;
 }
+
+#define OSINIT_STATUS_OK 0x80000000u
+static os_atomic_uint32_t osinit_status = OS_ATOMIC_UINT32_INIT(0);
+static os_mutex init_mutex;
+
+void os_osInit (void)
+{
+  uint32_t v;
+  v = os_atomic_inc32_nv(&osinit_status);
+retry:
+  if (v > OSINIT_STATUS_OK)
+    return;
+  else if (v == 1) {
+    os_osPlatformInit();
+    os_mutexInit(&init_mutex);
+    os_atomic_or32(&osinit_status, OSINIT_STATUS_OK);
+  } else {
+    while (v > 1 && !(v & OSINIT_STATUS_OK)) {
+      os_nanoSleep((os_time){10000000});
+      v = os_atomic_ld32(&osinit_status);
+    }
+    goto retry;
+  }
+}
+
+void os_osExit (void)
+{
+  uint32_t v, nv;
+  do {
+    v = os_atomic_ld32(&osinit_status);
+    if (v == (OSINIT_STATUS_OK | 1)) {
+      nv = 1;
+    } else {
+      nv = v - 1;
+    }
+  } while (!os_atomic_cas32(&osinit_status, v, nv));
+  if (nv == 1)
+  {
+    os_mutexDestroy(&init_mutex);
+    os_osPlatformExit();
+    os_atomic_dec32(&osinit_status);
+  }
+}
+
+os_mutex *os_getSingletonMutex(void)
+{
+  return &init_mutex;
+}
+
+

src/os/src/os_process.c

@@ -1,42 +0,0 @@
-/*
- * Copyright(c) 2006 to 2018 ADLINK Technology Limited and others
- *
- * This program and the accompanying materials are made available under the
- * terms of the Eclipse Public License v. 2.0 which is available at
- * http://www.eclipse.org/legal/epl-2.0, or the Eclipse Distribution License
- * v. 1.0 which is available at
- * http://www.eclipse.org/org/documents/edl-v10.php.
- *
- * SPDX-License-Identifier: EPL-2.0 OR BSD-3-Clause
- */
-#include "os/os.h"
-#include <stdlib.h>
-#include <assert.h>
-
-/** \brief Register an process exit handler
- *
- * \b os_procAtExit registers an process exit
- * handler by calling \b atexit passing the \b function
- * to be called when the process exits.
- * The standard POSIX implementation guarantees the
- * required order of execution of the exit handlers.
- */
-os_result
-os_procAtExit(
-    _In_ void (*function)(void))
-{
-    int result;
-    os_result osResult;
-
-    assert (function != NULL);
-
-    result = atexit (function);
-    if(!result)
-    {
-        osResult = os_resultSuccess;
-    } else
-    {
-        osResult = os_resultFail;
-    }
-    return osResult;
-}

src/os/src/posix/os_platform_init.c

@@ -21,53 +21,28 @@
 #include <assert.h>
 #include "os/os.h"
 
-/** \brief Counter that keeps track of number of times os-layer is initialized */
-static os_atomic_uint32_t _ospl_osInitCount = OS_ATOMIC_UINT32_INIT(0);
-
 /** \brief OS layer initialization
  *
  * \b os_osInit calls:
  * - \b os_sharedMemoryInit
  * - \b os_threadInit
  */
-void os_osInit (void)
+void os_osPlatformInit (void)
 {
-  uint32_t initCount;
+  os_syncModuleInit();
-
+  os_threadModuleInit();
-  initCount = os_atomic_inc32_nv(&_ospl_osInitCount);
+  os_processModuleInit();
-
+  os_reportInit(false);
-  if (initCount == 1) {
-    os_syncModuleInit();
-    os_threadModuleInit();
-    os_processModuleInit();
-    os_reportInit(false);
-  }
-
-  return;
 }
 
 /** \brief OS layer deinitialization
  */
-void os_osExit (void)
+void os_osPlatformExit (void)
 {
-  uint32_t initCount;
+  os_reportExit();
-
+  os_processModuleExit();
-  initCount = os_atomic_dec32_nv(&_ospl_osInitCount);
+  os_threadModuleExit();
-
+  os_syncModuleExit();
-  if (initCount == 0) {
-    os_reportExit();
-    os_processModuleExit();
-    os_threadModuleExit();
-    os_syncModuleExit();
-  } else if ((initCount + 1) < initCount){
-    /* The 0 boundary is passed, so os_osExit is called more often than
-     * os_osInit. Therefore undo decrement as nothing happened and warn. */
-    os_atomic_inc32(&_ospl_osInitCount);
-    OS_WARNING("os_osExit", 1, "OS-layer not initialized");
-    /* Fail in case of DEV, as it is incorrect API usage */
-    assert(0);
-  }
-  return;
 }
 
 /* This constructor is invoked when the library is loaded into a process. */

src/os/src/windows/os_platform_init.c

@@ -21,55 +21,22 @@
 
 #include "os/os.h"
 
-/** \brief Counter that keeps track of number of times os-layer is initialized */
+void os_osPlatformInit (void)
-static os_atomic_uint32_t _ospl_osInitCount = OS_ATOMIC_UINT32_INIT(0);
-
-/** \brief OS layer initialization
- *
- * \b os_osInit calls:
- * - \b os_sharedMemoryInit
- * - \b os_threadInit
- */
-void os_osInit (void)
 {
-  uint32_t initCount;
+  os_processModuleInit();
-
+  os_threadModuleInit();
-  initCount = os_atomic_inc32_nv(&_ospl_osInitCount);
+  os_timeModuleInit();
-
+  os_reportInit(false);
-  if (initCount == 1) {
+  os_socketModuleInit();
-    os_processModuleInit();
-    os_threadModuleInit();
-    os_timeModuleInit();
-    os_reportInit(false);
-    os_socketModuleInit();
-  }
-
-  return;
 }
 
-/** \brief OS layer deinitialization
+void os_osPlatformExit (void)
- */
-void os_osExit (void)
 {
-  uint32_t initCount;
+  os_socketModuleExit();
-
+  os_reportExit();
-  initCount = os_atomic_dec32_nv(&_ospl_osInitCount);
+  os_timeModuleExit();
-
+  os_threadModuleExit();
-  if (initCount == 0) {
+  os_processModuleExit();
-    os_socketModuleExit();
-    os_reportExit();
-    os_timeModuleExit();
-    os_threadModuleExit();
-    os_processModuleExit();
-  } else if ((initCount + 1) < initCount){
-    /* The 0 boundary is passed, so os_osExit is called more often than
-     * os_osInit. Therefore undo decrement as nothing happened and warn. */
-    os_atomic_inc32(&_ospl_osInitCount);
-    OS_WARNING("os_osExit", 1, "OS-layer not initialized");
-    /* Fail in case of DEV, as it is incorrect API usage */
-    assert(0);
-  }
-  return;
 }
 
 /* We need this on windows to make sure the main thread of MFC applications