fix crashes when C++ global destructors call DDS code

- properly count invocations of os_osInit/os_osExit
- handle concurrent invocations of those
- provide a single mutex for use by dds_init
- eliminate use of atexit()
- attendant dds_init/dds_fini changes
- fix related crash in thread local storage cleanup

Signed-off-by: Erik Boasson <eb@ilities.com>

src/core/ddsc/src/dds__init.h

@@ -22,16 +22,6 @@ dds_return_t
 dds__check_domain(
         _In_ dds_domainid_t domain);
 
-/**
- *Description : Initialization function. This operation initializes all the
- *required resources that are needed for the DDSC API process lifecycle
- *(like the init mutex and os layer).
- *A function will be registered that is called at the end of the process
- *lifecycle and will destroy the resources initialized in this function.
- **/
-void
-dds__startup(void);
-
 /**
  *Description : Initialization function, called from main. This operation
  *initializes all the required DDS resources,
@@ -66,12 +56,6 @@ dds_fini(void);
 dds_domainid_t dds_domain_default (void);
 
 
-/**
- * Description : Mutex used for initialization synchronization.
- */
-extern os_mutex dds__init_mutex;
-
-
 #if defined (__cplusplus)
 }
 #endif

src/core/ddsc/src/dds_init.c

@@ -45,36 +45,6 @@ dds_globals dds_global =
 
 static struct cfgst * dds_cfgst = NULL;
 
-
-
-
-os_mutex dds__init_mutex;
-
-static void
-dds__fini_once(void)
-{
-  os_mutexDestroy(&dds__init_mutex);
-  os_osExit();
-}
-
-static void
-dds__init_once(void)
-{
-  os_osInit();
-  os_mutexInit(&dds__init_mutex);
-  os_procAtExit(dds__fini_once);
-}
-
-
-
-void
-dds__startup(void)
-{
-    static os_once_t dds__init_control = OS_ONCE_T_STATIC_INIT;
-    os_once(&dds__init_control, dds__init_once);
-}
-
-
 dds_return_t
 dds_init(void)
 {
@@ -83,13 +53,15 @@ dds_init(void)
   char progname[50];
   char hostname[64];
   uint32_t len;
+  os_mutex *init_mutex;
 
   /* Be sure the DDS lifecycle resources are initialized. */
-  dds__startup();
+  os_osInit();
+  init_mutex = os_getSingletonMutex();
 
   DDS_REPORT_STACK();
 
-  os_mutexLock(&dds__init_mutex);
+  os_mutexLock(init_mutex);
 
   dds_global.m_init_count++;
   if (dds_global.m_init_count > 1)
@@ -181,7 +153,7 @@ dds_init(void)
   gv.default_plist_pp.present |= PP_ENTITY_NAME;
 
 skip:
-  os_mutexUnlock(&dds__init_mutex);
+  os_mutexUnlock(init_mutex);
   DDS_REPORT_FLUSH(false);
   return DDS_RETCODE_OK;
 
@@ -203,8 +175,9 @@ fail_config:
   ut_handleserver_fini();
 fail_handleserver:
   dds_global.m_init_count--;
-  os_mutexUnlock(&dds__init_mutex);
+  os_mutexUnlock(init_mutex);
   DDS_REPORT_FLUSH(true);
+  os_osExit();
   return ret;
 }
 
@@ -212,7 +185,9 @@ fail_handleserver:
 
 extern void dds_fini (void)
 {
-  os_mutexLock(&dds__init_mutex);
+  os_mutex *init_mutex;
+  init_mutex = os_getSingletonMutex();
+  os_mutexLock(init_mutex);
   assert(dds_global.m_init_count > 0);
   dds_global.m_init_count--;
   if (dds_global.m_init_count == 0)
@@ -232,7 +207,8 @@ extern void dds_fini (void)
     os_mutexDestroy (&dds_global.m_mutex);
     dds_global.m_default_domain = DDS_DOMAIN_DEFAULT;
   }
-  os_mutexUnlock(&dds__init_mutex);
+  os_mutexUnlock(init_mutex);
+  os_osExit();
 }
 
 

src/core/ddsc/src/dds_participant.c

@@ -145,9 +145,6 @@ dds_create_participant(
     struct thread_state1 * thr;
     bool asleep;
 
-    /* Be sure the DDS lifecycle resources are initialized. */
-    dds__startup();
-
     /* Make sure DDS instance is initialized. */
     ret = dds_init();
     if (ret != DDS_RETCODE_OK) {
@@ -243,9 +240,11 @@ dds_lookup_participant(
         _In_        size_t size)
 {
     dds_return_t ret = 0;
+    os_mutex *init_mutex;
 
     /* Be sure the DDS lifecycle resources are initialized. */
-    dds__startup();
+    os_osInit();
+    init_mutex = os_getSingletonMutex();
 
     DDS_REPORT_STACK();
 
@@ -262,7 +261,7 @@ dds_lookup_participant(
         participants[0] = 0;
     }
 
-    os_mutexLock (&dds__init_mutex);
+    os_mutexLock (init_mutex);
 
     /* Check if dds is intialized. */
     if (dds_global.m_init_count > 0) {
@@ -281,9 +280,10 @@ dds_lookup_participant(
         os_mutexUnlock (&dds_global.m_mutex);
     }
 
-    os_mutexUnlock (&dds__init_mutex);
+    os_mutexUnlock (init_mutex);
 
 err:
     DDS_REPORT_FLUSH(ret != DDS_RETCODE_OK);
+    os_osExit();
     return ret;
 }

src/core/ddsi/src/q_thread.c

@@ -110,7 +110,7 @@ cleanup_thread_state(
     assert(ts->state == THREAD_STATE_ALIVE);
     assert(vtime_asleep_p(ts->vtime));
     reset_thread_state(ts);
-    os_reportExit(); /* FIXME: Should not be here! */
+    os_osExit();
 }
 
 _Ret_valid_ struct thread_state1 *
@@ -133,6 +133,7 @@ lookup_thread_state(
             os_mutexLock(&thread_states.lock);
             ts1 = init_thread_state(tname);
             if (ts1 != NULL) {
+                os_osInit();
                 ts1->lb = 0;
                 ts1->extTid = tid;
                 ts1->tid = tid;