Trivial changes for thread sanitizer

Thread sanitizer warns about reads and writes of variables that are
meant to be read without holding a lock:

* Global "keep_going" is now a ddsrt_atomic_uint32_t

* Thread "vtime" is now a ddsrt_atomic_uint32_t

Previously the code relied on the assumption that a 32-bit int would be
treated as atomic, now that is all wrapped in ddsrt_atomic_{ld,st}32.
These being inline functions doing exactly the same thing, there is no
functional change, but it does allow annotating the loads and stores for
via function attributes on the ddsrt_atomic_{ld,st}X.

The concurrent hashtable implementation is replaced by a locked version
of the non-concurrent implementation if thread sanitizer is used.  This
changes eliminates the scores of problems signalled by thread sanitizer
in the GUID-to-entity translation and the key-to-instance id lookups.

Other than that, this replaces a flag used in a waitset test case to be
a ddsrt_atomic_uint32_t.

Signed-off-by: Erik Boasson <eb@ilities.com>

src/core/ddsc/tests/waitset.c

@@ -20,6 +20,7 @@
 #include "dds/ddsrt/misc.h"
 #include "dds/ddsrt/process.h"
 #include "dds/ddsrt/threads.h"
+#include "dds/ddsrt/atomics.h"
 #include "dds/ddsrt/time.h"
 
 /**************************************************************************************************
@@ -36,7 +37,7 @@ typedef enum thread_state_t {
 
 typedef struct thread_arg_t {
     ddsrt_thread_t tid;
-    thread_state_t state;
+    ddsrt_atomic_uint32_t state;
     dds_entity_t   expected;
 } thread_arg_t;
 
@@ -1056,26 +1057,26 @@ waiting_thread(void *a)
     dds_attach_t triggered;
     dds_return_t ret;
 
-    arg->state = WAITING;
+    ddsrt_atomic_st32 (&arg->state, WAITING);
     /* This should block until the main test released all claims. */
     ret = dds_waitset_wait(waitset, &triggered, 1, DDS_SECS(1000));
     CU_ASSERT_EQUAL_FATAL(ret, 1);
     CU_ASSERT_EQUAL_FATAL(arg->expected, (dds_entity_t)(intptr_t)triggered);
-    arg->state = STOPPED;
+    ddsrt_atomic_st32 (&arg->state, STOPPED);
 
     return 0;
 }
 
 static dds_return_t
-thread_reached_state(thread_state_t *actual, thread_state_t expected, int32_t msec)
+thread_reached_state(ddsrt_atomic_uint32_t *actual, thread_state_t expected, int32_t msec)
 {
     /* Convenience function. */
     dds_time_t msec10 = DDS_MSECS(10);
-    while ((msec > 0) && (*actual != expected)) {
+    while ((msec > 0) && ((thread_state_t) ddsrt_atomic_ld32 (actual) != expected)) {
         dds_sleepfor(msec10);
         msec -= 10;
     }
-    return (*actual == expected) ? DDS_RETCODE_OK : DDS_RETCODE_TIMEOUT;
+    return ((thread_state_t) ddsrt_atomic_ld32 (actual) == expected) ? DDS_RETCODE_OK : DDS_RETCODE_TIMEOUT;
 }
 
 static void
@@ -1089,7 +1090,7 @@ waiting_thread_start(struct thread_arg_t *arg, dds_entity_t expected)
 
     /* Create an other thread that will blocking wait on the waitset. */
     arg->expected = expected;
-    arg->state   = STARTING;
+    ddsrt_atomic_st32 (&arg->state, STARTING);
     ddsrt_threadattr_init(&thread_attr);
     rc = ddsrt_thread_create(&thread_id, "waiting_thread", &thread_attr, waiting_thread, arg);
     CU_ASSERT_EQUAL_FATAL(rc, DDS_RETCODE_OK);

src/core/ddsi/include/dds/ddsi/q_globals.h

@@ -221,7 +221,7 @@ struct q_globals {
   struct thread_state1 *listen_ts;
 
   /* Flag cleared when stopping (receive threads). FIXME. */
-  int rtps_keepgoing;
+  ddsrt_atomic_uint32_t rtps_keepgoing;
 
   /* Start time of the DDSI2 service, for logging relative time stamps,
      should I ever so desire. */

src/core/ddsi/include/dds/ddsi/q_thread.h

@@ -57,13 +57,14 @@ enum thread_state {
 struct logbuf;
 
 /*
- * watchdog indicates progress for the service lease liveliness mechsanism, while vtime
+ * vtime indicates progress for the garbage collector and the liveliness monitoring.
- * indicates progress for the Garbage collection purposes.
+ *
- *  vtime even : thread awake
+ * vtime is updated without using atomic operations: only the owning thread updates
- *  vtime odd  : thread asleep
+ * them, and the garbage collection mechanism and the liveliness monitoring only
+ * observe the value
  */
 #define THREAD_BASE                             \
-  volatile vtime_t vtime;                       \
+  ddsrt_atomic_uint32_t vtime;                  \
   ddsrt_thread_t tid;                           \
   ddsrt_thread_t extTid;                        \
   enum thread_state state;                      \
@@ -131,17 +132,21 @@ DDS_EXPORT inline bool vtime_gt (vtime_t vtime1, vtime_t vtime0)
 
 DDS_EXPORT inline bool thread_is_awake (void)
 {
-  return vtime_awake_p (lookup_thread_state ()->vtime);
+  struct thread_state1 *ts = lookup_thread_state ();
+  vtime_t vt = ddsrt_atomic_ld32 (&ts->vtime);
+  return vtime_awake_p (vt);
 }
 
 DDS_EXPORT inline bool thread_is_asleep (void)
 {
-  return vtime_asleep_p (lookup_thread_state ()->vtime);
+  struct thread_state1 *ts = lookup_thread_state ();
+  vtime_t vt = ddsrt_atomic_ld32 (&ts->vtime);
+  return vtime_asleep_p (vt);
 }
 
 DDS_EXPORT inline void thread_state_asleep (struct thread_state1 *ts1)
 {
-  vtime_t vt = ts1->vtime;
+  vtime_t vt = ddsrt_atomic_ld32 (&ts1->vtime);
   assert (vtime_awake_p (vt));
   /* nested calls a rare and an extra fence doesn't break things */
   ddsrt_atomic_fence_rel ();
@@ -149,24 +154,24 @@ DDS_EXPORT inline void thread_state_asleep (struct thread_state1 *ts1)
     vt += (1u << VTIME_TIME_SHIFT) - 1u;
   else
     vt -= 1u;
-  ts1->vtime = vt;
+  ddsrt_atomic_st32 (&ts1->vtime, vt);
 }
 
 DDS_EXPORT inline void thread_state_awake (struct thread_state1 *ts1)
 {
-  vtime_t vt = ts1->vtime;
+  vtime_t vt = ddsrt_atomic_ld32 (&ts1->vtime);
   assert ((vt & VTIME_NEST_MASK) < VTIME_NEST_MASK);
-  ts1->vtime = vt + 1u;
+  ddsrt_atomic_st32 (&ts1->vtime, vt + 1u);
   /* nested calls a rare and an extra fence doesn't break things */
   ddsrt_atomic_fence_acq ();
 }
 
 DDS_EXPORT inline void thread_state_awake_to_awake_no_nest (struct thread_state1 *ts1)
 {
-  vtime_t vt = ts1->vtime;
+  vtime_t vt = ddsrt_atomic_ld32 (&ts1->vtime);
   assert ((vt & VTIME_NEST_MASK) == 1);
   ddsrt_atomic_fence_rel ();
-  ts1->vtime = vt + (1u << VTIME_TIME_SHIFT);
+  ddsrt_atomic_st32 (&ts1->vtime, vt + (1u << VTIME_TIME_SHIFT));
   ddsrt_atomic_fence_acq ();
 }
 

src/core/ddsi/src/ddsi_tcp.c

@@ -706,6 +706,7 @@ static ddsrt_socket_t ddsi_tcp_conn_handle (ddsi_tran_base_t base)
   return ((ddsi_tcp_conn_t) base)->m_sock;
 }
 
+ddsrt_attribute_no_sanitize (("thread"))
 static bool ddsi_tcp_supports (int32_t kind)
 {
   return kind == ddsi_tcp_factory_g.m_kind;
@@ -770,7 +771,7 @@ static ddsi_tran_conn_t ddsi_tcp_accept (ddsi_tran_listener_t listener)
     {
       rc = ddsrt_accept(tl->m_sock, NULL, NULL, &sock);
     }
-    if (! gv.rtps_keepgoing)
+    if (!ddsrt_atomic_ld32(&gv.rtps_keepgoing))
     {
       ddsi_tcp_sock_free (sock, NULL);
       return NULL;

src/core/ddsi/src/ddsi_threadmon.c

@@ -83,7 +83,7 @@ static uint32_t threadmon_thread (struct ddsi_threadmon *sl)
           n_unused++;
         else
         {
-          vtime_t vt = thread_states.ts[i].vtime;
+          vtime_t vt = ddsrt_atomic_ld32 (&thread_states.ts[i].vtime);
           bool alive = vtime_asleep_p (vt) || vtime_asleep_p (sl->av_ary[i].vt) || vtime_gt (vt, sl->av_ary[i].vt);
           n_alive += (unsigned) alive;
           DDS_TRACE(" %u(%s):%c:%"PRIx32"->%"PRIx32, i, thread_states.ts[i].name, alive ? 'a' : 'd', sl->av_ary[i].vt, vt);

src/core/ddsi/src/ddsi_tran.c

@@ -91,6 +91,7 @@ static ddsi_tran_factory_t ddsi_factory_find_with_len (const char * type, size_t
   return factory;
 }
 
+ddsrt_attribute_no_sanitize (("thread"))
 ddsi_tran_factory_t ddsi_factory_find_supported_kind (int32_t kind)
 {
   /* FIXME: MUST speed up */
@@ -124,7 +125,7 @@ void ddsi_conn_free (ddsi_tran_conn_t conn)
         for (uint32_t i = 0; i < gv.n_recv_threads; i++)
         {
           if (!gv.recv_threads[i].ts)
-            assert (!gv.rtps_keepgoing);
+            assert (!ddsrt_atomic_ld32 (&gv.rtps_keepgoing));
           else
           {
             switch (gv.recv_threads[i].arg.mode)

src/core/ddsi/src/q_gc.c

@@ -44,7 +44,7 @@ static void threads_vtime_gather_for_wait (unsigned *nivs, struct idx_vtime *ivs
   uint32_t i, j;
   for (i = j = 0; i < thread_states.nthreads; i++)
   {
-    vtime_t vtime = thread_states.ts[i].vtime;
+    vtime_t vtime = ddsrt_atomic_ld32 (&thread_states.ts[i].vtime);
     if (vtime_awake_p (vtime))
     {
       ivs[j].idx = i;
@@ -63,7 +63,7 @@ static int threads_vtime_check (uint32_t *nivs, struct idx_vtime *ivs)
   while (i < *nivs)
   {
     uint32_t thridx = ivs[i].idx;
-    vtime_t vtime = thread_states.ts[thridx].vtime;
+    vtime_t vtime = ddsrt_atomic_ld32 (&thread_states.ts[thridx].vtime);
     assert (vtime_awake_p (ivs[i].vtime));
     if (!vtime_gt (vtime, ivs[i].vtime))
       ++i;

src/core/ddsi/src/q_init.c

@@ -679,9 +679,9 @@ static void rtps_term_prep (void)
 {
   /* Stop all I/O */
   ddsrt_mutex_lock (&gv.lock);
-  if (gv.rtps_keepgoing)
+  if (ddsrt_atomic_ld32 (&gv.rtps_keepgoing))
   {
-    gv.rtps_keepgoing = 0; /* so threads will stop once they get round to checking */
+    ddsrt_atomic_st32 (&gv.rtps_keepgoing, 0); /* so threads will stop once they get round to checking */
     ddsrt_atomic_fence ();
     /* can't wake up throttle_writer, currently, but it'll check every few seconds */
     trigger_recv_threads ();
@@ -1252,7 +1252,7 @@ int rtps_init (void)
 
   gv.gcreq_queue = gcreq_queue_new ();
 
-  gv.rtps_keepgoing = 1;
+  ddsrt_atomic_st32 (&gv.rtps_keepgoing, 1);
   ddsrt_rwlock_init (&gv.qoslock);
 
   if (config.xpack_send_async)

src/core/ddsi/src/q_receive.c

@@ -3156,7 +3156,7 @@ uint32_t listen_thread (struct ddsi_tran_listener * listener)
 {
   ddsi_tran_conn_t conn;
 
-  while (gv.rtps_keepgoing)
+  while (ddsrt_atomic_ld32 (&gv.rtps_keepgoing))
   {
     /* Accept connection from listener */
 
@@ -3310,7 +3310,7 @@ uint32_t recv_thread (void *vrecv_thread_arg)
   nn_rbufpool_setowner (rbpool, ddsrt_thread_self ());
   if (waitset == NULL)
   {
-    while (gv.rtps_keepgoing)
+    while (ddsrt_atomic_ld32 (&gv.rtps_keepgoing))
     {
       LOG_THREAD_CPUTIME (next_thread_cputime);
       (void) do_packet (ts1, recv_thread_arg->u.single.conn, NULL, rbpool);
@@ -3343,7 +3343,7 @@ uint32_t recv_thread (void *vrecv_thread_arg)
       num_fixed += (unsigned)rc;
     }
 
-    while (gv.rtps_keepgoing)
+    while (ddsrt_atomic_ld32 (&gv.rtps_keepgoing))
     {
       int rebuildws;
       LOG_THREAD_CPUTIME (next_thread_cputime);

src/core/ddsi/src/q_thread.c

@@ -75,7 +75,7 @@ static void ddsrt_free_aligned (void *ptr)
 void thread_states_init_static (void)
 {
   static struct thread_state1 ts = {
-    .state = THREAD_STATE_ALIVE, .vtime = 0u, .name = "(anon)"
+    .state = THREAD_STATE_ALIVE, .vtime = DDSRT_ATOMIC_UINT32_INIT (0), .name = "(anon)"
   };
   tsd_thread_state = &ts;
 }
@@ -92,7 +92,7 @@ void thread_states_init (unsigned maxthreads)
   for (uint32_t i = 0; i < thread_states.nthreads; i++)
   {
     thread_states.ts[i].state = THREAD_STATE_ZERO;
-    thread_states.ts[i].vtime = 0u;
+    ddsrt_atomic_st32 (&thread_states.ts[i].vtime, 0);
     thread_states.ts[i].name = NULL;
   }
   DDSRT_WARNING_MSVC_ON(6386);
@@ -113,6 +113,7 @@ void thread_states_fini (void)
   thread_states.ts = NULL;
 }
 
+ddsrt_attribute_no_sanitize (("thread"))
 static struct thread_state1 *find_thread_state (ddsrt_thread_t tid)
 {
   if (thread_states.ts) {
@@ -132,7 +133,7 @@ static void cleanup_thread_state (void *data)
   if (ts)
   {
     assert(ts->state == THREAD_STATE_LAZILY_CREATED);
-    assert(vtime_asleep_p(ts->vtime));
+    assert(vtime_asleep_p(ddsrt_atomic_ld32 (&ts->vtime)));
     reset_thread_state(ts);
   }
   ddsrt_fini();
@@ -207,7 +208,7 @@ void upgrade_main_thread (void)
     abort ();
   ts1 = &thread_states.ts[cand];
   if (ts1->state == THREAD_STATE_ZERO)
-    assert (vtime_asleep_p (ts1->vtime));
+    assert (vtime_asleep_p (ddsrt_atomic_ld32 (&ts1->vtime)));
   ts1->state = THREAD_STATE_LAZILY_CREATED;
   ts1->tid = ddsrt_thread_self ();
   ts1->name = main_thread_name;
@@ -233,7 +234,7 @@ static struct thread_state1 *init_thread_state (const char *tname, enum thread_s
     return NULL;
 
   ts = &thread_states.ts[cand];
-  assert (vtime_asleep_p (ts->vtime));
+  assert (vtime_asleep_p (ddsrt_atomic_ld32 (&ts->vtime)));
   ts->name = ddsrt_strdup (tname);
   ts->state = state;
 
@@ -300,7 +301,7 @@ dds_return_t join_thread (struct thread_state1 *ts1)
   dds_return_t ret;
   assert (ts1->state == THREAD_STATE_ALIVE);
   ret = ddsrt_thread_join (ts1->extTid, NULL);
-  assert (vtime_asleep_p (ts1->vtime));
+  assert (vtime_asleep_p (ddsrt_atomic_ld32 (&ts1->vtime)));
   reap_thread_state (ts1, 1);
   return ret;
 }
@@ -317,7 +318,7 @@ void reset_thread_state (struct thread_state1 *ts1)
 void downgrade_main_thread (void)
 {
   struct thread_state1 *ts1 = lookup_thread_state ();
-  assert (vtime_asleep_p (ts1->vtime));
+  assert (vtime_asleep_p (ddsrt_atomic_ld32 (&ts1->vtime)));
   /* no need to sync with service lease: already stopped */
   reap_thread_state (ts1, 0);
   thread_states_init_static ();

src/core/ddsi/src/q_transmit.c

@@ -968,7 +968,7 @@ static dds_return_t throttle_writer (struct thread_state1 * const ts1, struct nn
     whc_get_state (wr->whc, &whcst);
   }
 
-  while (gv.rtps_keepgoing && !writer_may_continue (wr, &whcst))
+  while (ddsrt_atomic_ld32 (&gv.rtps_keepgoing) && !writer_may_continue (wr, &whcst))
   {
     int64_t reltimeout;
     tnow = now_mt ();

src/core/ddsi/src/q_xevent.c

@@ -597,8 +597,8 @@ static void handle_xevk_heartbeat (struct nn_xpack *xp, struct xevent *ev, nn_mt
     return;
   }
 
-  assert (wr->reliable);
   ddsrt_mutex_lock (&wr->e.lock);
+  assert (wr->reliable);
   whc_get_state(wr->whc, &whcst);
   if (!writer_must_have_hb_scheduled (wr, &whcst))
   {

src/ddsrt/include/dds/ddsrt/atomics/gcc.h

@@ -13,6 +13,7 @@
 #define DDSRT_ATOMICS_GCC_H
 
 #include "dds/ddsrt/misc.h"
+#include "dds/ddsrt/attributes.h"
 
 #if defined (__cplusplus)
 extern "C" {
@@ -25,19 +26,51 @@ extern "C" {
 
 /* LD, ST */
 
-inline uint32_t ddsrt_atomic_ld32(const volatile ddsrt_atomic_uint32_t *x) { return x->v; }
+ddsrt_attribute_no_sanitize (("thread"))
+inline uint32_t ddsrt_atomic_ld32(const volatile ddsrt_atomic_uint32_t *x)
+{
+  return x->v;
+}
 #if DDSRT_HAVE_ATOMIC64
-inline uint64_t ddsrt_atomic_ld64(const volatile ddsrt_atomic_uint64_t *x) { return x->v; }
+ddsrt_attribute_no_sanitize (("thread"))
+inline uint64_t ddsrt_atomic_ld64(const volatile ddsrt_atomic_uint64_t *x)
+{
+  return x->v;
+}
 #endif
-inline uintptr_t ddsrt_atomic_ldptr(const volatile ddsrt_atomic_uintptr_t *x) { return x->v; }
+ddsrt_attribute_no_sanitize (("thread"))
-inline void *ddsrt_atomic_ldvoidp(const volatile ddsrt_atomic_voidp_t *x) { return (void *) ddsrt_atomic_ldptr(x); }
+inline uintptr_t ddsrt_atomic_ldptr(const volatile ddsrt_atomic_uintptr_t *x)
+{
+  return x->v;
+}
+ddsrt_attribute_no_sanitize (("thread"))
+inline void *ddsrt_atomic_ldvoidp(const volatile ddsrt_atomic_voidp_t *x)
+{
+  return (void *) ddsrt_atomic_ldptr(x);
+}
 
-inline void ddsrt_atomic_st32(volatile ddsrt_atomic_uint32_t *x, uint32_t v) { x->v = v; }
+ddsrt_attribute_no_sanitize (("thread"))
+inline void ddsrt_atomic_st32(volatile ddsrt_atomic_uint32_t *x, uint32_t v)
+{
+  x->v = v;
+}
 #if DDSRT_HAVE_ATOMIC64
-inline void ddsrt_atomic_st64(volatile ddsrt_atomic_uint64_t *x, uint64_t v) { x->v = v; }
+ddsrt_attribute_no_sanitize (("thread"))
+inline void ddsrt_atomic_st64(volatile ddsrt_atomic_uint64_t *x, uint64_t v)
+{
+  x->v = v;
+}
 #endif
-inline void ddsrt_atomic_stptr(volatile ddsrt_atomic_uintptr_t *x, uintptr_t v) { x->v = v; }
+ddsrt_attribute_no_sanitize (("thread"))
-inline void ddsrt_atomic_stvoidp(volatile ddsrt_atomic_voidp_t *x, void *v) { ddsrt_atomic_stptr(x, (uintptr_t)v); }
+inline void ddsrt_atomic_stptr(volatile ddsrt_atomic_uintptr_t *x, uintptr_t v)
+{
+  x->v = v;
+}
+ddsrt_attribute_no_sanitize (("thread"))
+inline void ddsrt_atomic_stvoidp(volatile ddsrt_atomic_voidp_t *x, void *v)
+{
+  ddsrt_atomic_stptr(x, (uintptr_t)v);
+}
 
 /* INC */
 

src/ddsrt/include/dds/ddsrt/attributes.h

@@ -111,4 +111,16 @@
 # define ddsrt_attribute_packed
 #endif
 
+#if ddsrt_has_attribute(no_sanitize)
+# define ddsrt_attribute_no_sanitize(params) __attribute__ ((__no_sanitize__ params))
+#else
+# define ddsrt_attribute_no_sanitize(params)
+#endif
+
+#if defined(__has_feature)
+# define ddsrt_has_feature_thread_sanitizer __has_feature(thread_sanitizer)
+#else
+# define ddsrt_has_feature_thread_sanitizer 0
+#endif
+
 #endif /* DDSRT_ATTRIBUTES_H */

src/ddsrt/include/dds/ddsrt/hopscotch.h

@@ -20,15 +20,6 @@
 extern "C" {
 #endif
 
-/* Concurrent version */
-struct ddsrt_chh;
-struct ddsrt_chh_bucket;
-struct ddsrt_chh_iter {
-  struct ddsrt_chh_bucket *bs;
-  uint32_t size;
-  uint32_t cursor;
-};
-
 /*
  * The hopscotch hash table is dependent on a proper functioning hash.
  * If the hash function generates a lot of hash collisions, then it will
@@ -54,15 +45,6 @@ typedef int (*ddsrt_hh_equals_fn) (const void *, const void *);
  */
 typedef void (*ddsrt_hh_buckets_gc_fn) (void *);
 
-DDS_EXPORT struct ddsrt_chh *ddsrt_chh_new (uint32_t init_size, ddsrt_hh_hash_fn hash, ddsrt_hh_equals_fn equals, ddsrt_hh_buckets_gc_fn gc_buckets);
-DDS_EXPORT void ddsrt_chh_free (struct ddsrt_chh * __restrict hh);
-DDS_EXPORT void *ddsrt_chh_lookup (struct ddsrt_chh * __restrict rt, const void * __restrict template);
-DDS_EXPORT int ddsrt_chh_add (struct ddsrt_chh * __restrict rt, const void * __restrict data);
-DDS_EXPORT int ddsrt_chh_remove (struct ddsrt_chh * __restrict rt, const void * __restrict template);
-DDS_EXPORT void ddsrt_chh_enum_unsafe (struct ddsrt_chh * __restrict rt, void (*f) (void *a, void *f_arg), void *f_arg); /* may delete a */
-void *ddsrt_chh_iter_first (struct ddsrt_chh * __restrict rt, struct ddsrt_chh_iter *it);
-void *ddsrt_chh_iter_next (struct ddsrt_chh_iter *it);
-
 /* Sequential version */
 struct ddsrt_hh;
 
@@ -80,6 +62,31 @@ DDS_EXPORT void ddsrt_hh_enum (struct ddsrt_hh * __restrict rt, void (*f) (void
 DDS_EXPORT void *ddsrt_hh_iter_first (struct ddsrt_hh * __restrict rt, struct ddsrt_hh_iter * __restrict iter); /* may delete nodes */
 DDS_EXPORT void *ddsrt_hh_iter_next (struct ddsrt_hh_iter * __restrict iter);
 
+/* Concurrent version */
+struct ddsrt_chh;
+struct ddsrt_chh_bucket;
+
+#if ! ddsrt_has_feature_thread_sanitizer
+struct ddsrt_chh_iter {
+  struct ddsrt_chh_bucket *bs;
+  uint32_t size;
+  uint32_t cursor;
+};
+#else
+struct ddsrt_chh_iter {
+  struct ddsrt_chh *chh;
+  struct ddsrt_hh_iter it;
+};
+#endif
+
+DDS_EXPORT struct ddsrt_chh *ddsrt_chh_new (uint32_t init_size, ddsrt_hh_hash_fn hash, ddsrt_hh_equals_fn equals, ddsrt_hh_buckets_gc_fn gc_buckets);
+DDS_EXPORT void ddsrt_chh_free (struct ddsrt_chh * __restrict hh);
+DDS_EXPORT void *ddsrt_chh_lookup (struct ddsrt_chh * __restrict rt, const void * __restrict template);
+DDS_EXPORT int ddsrt_chh_add (struct ddsrt_chh * __restrict rt, const void * __restrict data);
+DDS_EXPORT int ddsrt_chh_remove (struct ddsrt_chh * __restrict rt, const void * __restrict template);
+DDS_EXPORT void ddsrt_chh_enum_unsafe (struct ddsrt_chh * __restrict rt, void (*f) (void *a, void *f_arg), void *f_arg); /* may delete a */
+DDS_EXPORT void *ddsrt_chh_iter_first (struct ddsrt_chh * __restrict rt, struct ddsrt_chh_iter *it);
+DDS_EXPORT void *ddsrt_chh_iter_next (struct ddsrt_chh_iter *it);
 /* Sequential version, embedded data */
 struct ddsrt_ehh;
 

src/ddsrt/src/hopscotch.c

@@ -12,6 +12,7 @@
 #include <assert.h>
 #include <string.h>
 
+#include "dds/ddsrt/attributes.h"
 #include "dds/ddsrt/atomics.h"
 #include "dds/ddsrt/heap.h"
 #include "dds/ddsrt/sync.h"
@@ -22,8 +23,229 @@
 
 #define NOT_A_BUCKET (~(uint32_t)0)
 
+/************* SEQUENTIAL VERSION ***************/
+
+struct ddsrt_hh_bucket {
+  uint32_t hopinfo;
+  void *data;
+};
+
+struct ddsrt_hh {
+  uint32_t size; /* power of 2 */
+  struct ddsrt_hh_bucket *buckets;
+  ddsrt_hh_hash_fn hash;
+  ddsrt_hh_equals_fn equals;
+};
+
+static void ddsrt_hh_init (struct ddsrt_hh *rt, uint32_t init_size, ddsrt_hh_hash_fn hash, ddsrt_hh_equals_fn equals)
+{
+  uint32_t size = HH_HOP_RANGE;
+  uint32_t i;
+  while (size < init_size) {
+    size *= 2;
+  }
+  rt->hash = hash;
+  rt->equals = equals;
+  rt->size = size;
+  rt->buckets = ddsrt_malloc (size * sizeof (*rt->buckets));
+  for (i = 0; i < size; i++) {
+    rt->buckets[i].hopinfo = 0;
+    rt->buckets[i].data = NULL;
+  }
+}
+
+static void ddsrt_hh_fini (struct ddsrt_hh *rt)
+{
+  ddsrt_free (rt->buckets);
+}
+
+struct ddsrt_hh *ddsrt_hh_new (uint32_t init_size, ddsrt_hh_hash_fn hash, ddsrt_hh_equals_fn equals)
+{
+  struct ddsrt_hh *hh = ddsrt_malloc (sizeof (*hh));
+  ddsrt_hh_init (hh, init_size, hash, equals);
+  return hh;
+}
+
+void ddsrt_hh_free (struct ddsrt_hh * __restrict hh)
+{
+  ddsrt_hh_fini (hh);
+  ddsrt_free (hh);
+}
+
+static void *ddsrt_hh_lookup_internal (const struct ddsrt_hh *rt, const uint32_t bucket, const void *template)
+{
+  const uint32_t idxmask = rt->size - 1;
+  uint32_t hopinfo = rt->buckets[bucket].hopinfo;
+  uint32_t idx;
+  for (idx = 0; hopinfo != 0; hopinfo >>= 1, idx++) {
+    const uint32_t bidx = (bucket + idx) & idxmask;
+    void *data = rt->buckets[bidx].data;
+    if (data && rt->equals (data, template))
+      return data;
+  }
+  return NULL;
+}
+
+void *ddsrt_hh_lookup (const struct ddsrt_hh * __restrict rt, const void * __restrict template)
+{
+  const uint32_t hash = rt->hash (template);
+  const uint32_t idxmask = rt->size - 1;
+  const uint32_t bucket = hash & idxmask;
+  return ddsrt_hh_lookup_internal (rt, bucket, template);
+}
+
+static uint32_t ddsrt_hh_find_closer_free_bucket (struct ddsrt_hh *rt, uint32_t free_bucket, uint32_t *free_distance)
+{
+  const uint32_t idxmask = rt->size - 1;
+  uint32_t move_bucket, free_dist;
+  move_bucket = (free_bucket - (HH_HOP_RANGE - 1)) & idxmask;
+  for (free_dist = HH_HOP_RANGE - 1; free_dist > 0; free_dist--) {
+    uint32_t move_free_distance = NOT_A_BUCKET;
+    uint32_t mask = 1;
+    uint32_t i;
+    for (i = 0; i < free_dist; i++, mask <<= 1) {
+      if (mask & rt->buckets[move_bucket].hopinfo) {
+        move_free_distance = i;
+        break;
+      }
+    }
+    if (move_free_distance != NOT_A_BUCKET) {
+      uint32_t new_free_bucket = (move_bucket + move_free_distance) & idxmask;
+      rt->buckets[move_bucket].hopinfo |= 1u << free_dist;
+      rt->buckets[free_bucket].data = rt->buckets[new_free_bucket].data;
+      rt->buckets[new_free_bucket].data = NULL;
+      rt->buckets[move_bucket].hopinfo &= ~(1u << move_free_distance);
+      *free_distance -= free_dist - move_free_distance;
+      return new_free_bucket;
+    }
+    move_bucket = (move_bucket + 1) & idxmask;
+  }
+  return NOT_A_BUCKET;
+}
+
+static void ddsrt_hh_resize (struct ddsrt_hh *rt)
+{
+  struct ddsrt_hh_bucket *bs1;
+  uint32_t i, idxmask0, idxmask1;
+
+  bs1 = ddsrt_malloc (2 * rt->size * sizeof (*rt->buckets));
+
+  for (i = 0; i < 2 * rt->size; i++) {
+    bs1[i].hopinfo = 0;
+    bs1[i].data = NULL;
+  }
+  idxmask0 = rt->size - 1;
+  idxmask1 = 2 * rt->size - 1;
+  for (i = 0; i < rt->size; i++) {
+    void *data = rt->buckets[i].data;
+    if (data) {
+      const uint32_t hash = rt->hash (data);
+      const uint32_t old_start_bucket = hash & idxmask0;
+      const uint32_t new_start_bucket = hash & idxmask1;
+      const uint32_t dist = (i >= old_start_bucket) ? (i - old_start_bucket) : (rt->size + i - old_start_bucket);
+      const uint32_t newb = (new_start_bucket + dist) & idxmask1;
+      assert (dist < HH_HOP_RANGE);
+      bs1[new_start_bucket].hopinfo |= 1u << dist;
+      bs1[newb].data = data;
+    }
+  }
+
+  ddsrt_free (rt->buckets);
+  rt->size *= 2;
+  rt->buckets = bs1;
+}
+
+int ddsrt_hh_add (struct ddsrt_hh * __restrict rt, const void * __restrict data)
+{
+  const uint32_t hash = rt->hash (data);
+  const uint32_t idxmask = rt->size - 1;
+  const uint32_t start_bucket = hash & idxmask;
+  uint32_t free_distance, free_bucket;
+
+  if (ddsrt_hh_lookup_internal (rt, start_bucket, data)) {
+    return 0;
+  }
+
+  free_bucket = start_bucket;
+  for (free_distance = 0; free_distance < HH_ADD_RANGE; free_distance++) {
+    if (rt->buckets[free_bucket].data == NULL)
+      break;
+    free_bucket = (free_bucket + 1) & idxmask;
+  }
+  if (free_distance < HH_ADD_RANGE) {
+    do {
+      if (free_distance < HH_HOP_RANGE) {
+        assert ((uint32_t) free_bucket == ((start_bucket + free_distance) & idxmask));
+        rt->buckets[start_bucket].hopinfo |= 1u << free_distance;
+        rt->buckets[free_bucket].data = (void *) data;
+        return 1;
+      }
+      free_bucket = ddsrt_hh_find_closer_free_bucket (rt, free_bucket, &free_distance);
+      assert (free_bucket == NOT_A_BUCKET || free_bucket <= idxmask);
+    } while (free_bucket != NOT_A_BUCKET);
+  }
+
+  ddsrt_hh_resize (rt);
+  return ddsrt_hh_add (rt, data);
+}
+
+int ddsrt_hh_remove (struct ddsrt_hh * __restrict rt, const void * __restrict template)
+{
+  const uint32_t hash = rt->hash (template);
+  const uint32_t idxmask = rt->size - 1;
+  const uint32_t bucket = hash & idxmask;
+  uint32_t hopinfo;
+  uint32_t idx;
+  hopinfo = rt->buckets[bucket].hopinfo;
+  for (idx = 0; hopinfo != 0; hopinfo >>= 1, idx++) {
+    if (hopinfo & 1) {
+      const uint32_t bidx = (bucket + idx) & idxmask;
+      void *data = rt->buckets[bidx].data;
+      if (data && rt->equals (data, template)) {
+        rt->buckets[bidx].data = NULL;
+        rt->buckets[bucket].hopinfo &= ~(1u << idx);
+        return 1;
+      }
+    }
+  }
+  return 0;
+}
+
+void ddsrt_hh_enum (struct ddsrt_hh * __restrict rt, void (*f) (void *a, void *f_arg), void *f_arg)
+{
+  uint32_t i;
+  for (i = 0; i < rt->size; i++) {
+    void *data = rt->buckets[i].data;
+    if (data) {
+      f (data, f_arg);
+    }
+  }
+}
+
+void *ddsrt_hh_iter_first (struct ddsrt_hh * __restrict rt, struct ddsrt_hh_iter * __restrict iter)
+{
+  iter->hh = rt;
+  iter->cursor = 0;
+  return ddsrt_hh_iter_next (iter);
+}
+
+void *ddsrt_hh_iter_next (struct ddsrt_hh_iter * __restrict iter)
+{
+  struct ddsrt_hh *rt = iter->hh;
+  while (iter->cursor < rt->size) {
+    void *data = rt->buckets[iter->cursor].data;
+    iter->cursor++;
+    if (data) {
+      return data;
+    }
+  }
+  return NULL;
+}
+
 /********** CONCURRENT VERSION ************/
 
+#if ! ddsrt_has_feature_thread_sanitizer
+
 #define N_BACKING_LOCKS 32
 #define N_RESIZE_LOCKS 8
 
@@ -228,7 +450,7 @@ static void *ddsrt_chh_lookup_internal (struct ddsrt_chh_bucket_array const * co
 
 #define ddsrt_atomic_rmw32_nonatomic(var_, tmp_, expr_) do {                 \
         ddsrt_atomic_uint32_t *var__ = (var_);                               \
-        uint32_t tmp_ = ddsrt_atomic_ld32 (var__);                          \
+        uint32_t tmp_ = ddsrt_atomic_ld32 (var__);                           \
         ddsrt_atomic_st32 (var__, (expr_));                                  \
     } while (0)
 
@@ -467,224 +689,78 @@ void *ddsrt_chh_iter_first (struct ddsrt_chh * __restrict rt, struct ddsrt_chh_i
   return ddsrt_chh_iter_next (it);
 }
 
-/************* SEQUENTIAL VERSION ***************/
+#else
 
-struct ddsrt_hh_bucket {
+struct ddsrt_chh {
-    uint32_t hopinfo;
+  ddsrt_mutex_t lock;
-    void *data;
+  struct ddsrt_hh rt;
 };
 
-struct ddsrt_hh {
+struct ddsrt_chh *ddsrt_chh_new (uint32_t init_size, ddsrt_hh_hash_fn hash, ddsrt_hh_equals_fn equals, ddsrt_hh_buckets_gc_fn gc)
-    uint32_t size; /* power of 2 */
-    struct ddsrt_hh_bucket *buckets;
-    ddsrt_hh_hash_fn hash;
-    ddsrt_hh_equals_fn equals;
-};
-
-static void ddsrt_hh_init (struct ddsrt_hh *rt, uint32_t init_size, ddsrt_hh_hash_fn hash, ddsrt_hh_equals_fn equals)
 {
-    uint32_t size = HH_HOP_RANGE;
+  struct ddsrt_chh *hh = ddsrt_malloc (sizeof (*hh));
-    uint32_t i;
+  (void) gc;
-    while (size < init_size) {
+  ddsrt_mutex_init (&hh->lock);
-        size *= 2;
+  ddsrt_hh_init (&hh->rt, init_size, hash, equals);
-    }
+  return hh;
-    rt->hash = hash;
-    rt->equals = equals;
-    rt->size = size;
-    rt->buckets = ddsrt_malloc (size * sizeof (*rt->buckets));
-    for (i = 0; i < size; i++) {
-        rt->buckets[i].hopinfo = 0;
-        rt->buckets[i].data = NULL;
-    }
 }
 
-static void ddsrt_hh_fini (struct ddsrt_hh *rt)
+void ddsrt_chh_free (struct ddsrt_chh * __restrict hh)
 {
-    ddsrt_free (rt->buckets);
+  ddsrt_hh_fini (&hh->rt);
+  ddsrt_mutex_destroy (&hh->lock);
+  ddsrt_free (hh);
 }
 
-struct ddsrt_hh *ddsrt_hh_new (uint32_t init_size, ddsrt_hh_hash_fn hash, ddsrt_hh_equals_fn equals)
+void *ddsrt_chh_lookup (struct ddsrt_chh * __restrict hh, const void * __restrict template)
 {
-    struct ddsrt_hh *hh = ddsrt_malloc (sizeof (*hh));
+  ddsrt_mutex_lock (&hh->lock);
-    ddsrt_hh_init (hh, init_size, hash, equals);
+  void *x = ddsrt_hh_lookup (&hh->rt, template);
-    return hh;
+  ddsrt_mutex_unlock (&hh->lock);
+  return x;
 }
 
-void ddsrt_hh_free (struct ddsrt_hh * __restrict hh)
+int ddsrt_chh_add (struct ddsrt_chh * __restrict hh, const void * __restrict data)
 {
-    ddsrt_hh_fini (hh);
+  ddsrt_mutex_lock (&hh->lock);
-    ddsrt_free (hh);
+  int x = ddsrt_hh_add (&hh->rt, data);
+  ddsrt_mutex_unlock (&hh->lock);
+  return x;
 }
 
-static void *ddsrt_hh_lookup_internal (const struct ddsrt_hh *rt, const uint32_t bucket, const void *template)
+int ddsrt_chh_remove (struct ddsrt_chh * __restrict hh, const void * __restrict template)
 {
-    const uint32_t idxmask = rt->size - 1;
+  ddsrt_mutex_lock (&hh->lock);
-    uint32_t hopinfo = rt->buckets[bucket].hopinfo;
+  int x = ddsrt_hh_remove (&hh->rt, template);
-    uint32_t idx;
+  ddsrt_mutex_unlock (&hh->lock);
-    for (idx = 0; hopinfo != 0; hopinfo >>= 1, idx++) {
+  return x;
-        const uint32_t bidx = (bucket + idx) & idxmask;
-        void *data = rt->buckets[bidx].data;
-        if (data && rt->equals (data, template))
-            return data;
-    }
-    return NULL;
 }
 
-void *ddsrt_hh_lookup (const struct ddsrt_hh * __restrict rt, const void * __restrict template)
+void ddsrt_chh_enum_unsafe (struct ddsrt_chh * __restrict hh, void (*f) (void *a, void *f_arg), void *f_arg)
 {
-    const uint32_t hash = rt->hash (template);
+  ddsrt_mutex_lock (&hh->lock);
-    const uint32_t idxmask = rt->size - 1;
+  ddsrt_hh_enum (&hh->rt, f, f_arg);
-    const uint32_t bucket = hash & idxmask;
+  ddsrt_mutex_unlock (&hh->lock);
-    return ddsrt_hh_lookup_internal (rt, bucket, template);
 }
 
-static uint32_t ddsrt_hh_find_closer_free_bucket (struct ddsrt_hh *rt, uint32_t free_bucket, uint32_t *free_distance)
+void *ddsrt_chh_iter_first (struct ddsrt_chh * __restrict hh, struct ddsrt_chh_iter *it)
 {
-    const uint32_t idxmask = rt->size - 1;
+  ddsrt_mutex_lock (&hh->lock);
-    uint32_t move_bucket, free_dist;
+  it->chh = hh;
-    move_bucket = (free_bucket - (HH_HOP_RANGE - 1)) & idxmask;
+  void *x = ddsrt_hh_iter_first (&hh->rt, &it->it);
-    for (free_dist = HH_HOP_RANGE - 1; free_dist > 0; free_dist--) {
+  ddsrt_mutex_unlock (&hh->lock);
-        uint32_t move_free_distance = NOT_A_BUCKET;
+  return x;
-        uint32_t mask = 1;
-        uint32_t i;
-        for (i = 0; i < free_dist; i++, mask <<= 1) {
-            if (mask & rt->buckets[move_bucket].hopinfo) {
-                move_free_distance = i;
-                break;
-            }
-        }
-        if (move_free_distance != NOT_A_BUCKET) {
-            uint32_t new_free_bucket = (move_bucket + move_free_distance) & idxmask;
-            rt->buckets[move_bucket].hopinfo |= 1u << free_dist;
-            rt->buckets[free_bucket].data = rt->buckets[new_free_bucket].data;
-            rt->buckets[new_free_bucket].data = NULL;
-            rt->buckets[move_bucket].hopinfo &= ~(1u << move_free_distance);
-            *free_distance -= free_dist - move_free_distance;
-            return new_free_bucket;
-        }
-        move_bucket = (move_bucket + 1) & idxmask;
-    }
-    return NOT_A_BUCKET;
 }
 
-static void ddsrt_hh_resize (struct ddsrt_hh *rt)
+void *ddsrt_chh_iter_next (struct ddsrt_chh_iter *it)
 {
-    struct ddsrt_hh_bucket *bs1;
+  ddsrt_mutex_lock (&it->chh->lock);
-    uint32_t i, idxmask0, idxmask1;
+  void *x = ddsrt_hh_iter_next (&it->it);
-
+  ddsrt_mutex_unlock (&it->chh->lock);
-    bs1 = ddsrt_malloc (2 * rt->size * sizeof (*rt->buckets));
+  return x;
-
-    for (i = 0; i < 2 * rt->size; i++) {
-        bs1[i].hopinfo = 0;
-        bs1[i].data = NULL;
-    }
-    idxmask0 = rt->size - 1;
-    idxmask1 = 2 * rt->size - 1;
-    for (i = 0; i < rt->size; i++) {
-        void *data = rt->buckets[i].data;
-        if (data) {
-            const uint32_t hash = rt->hash (data);
-            const uint32_t old_start_bucket = hash & idxmask0;
-            const uint32_t new_start_bucket = hash & idxmask1;
-            const uint32_t dist = (i >= old_start_bucket) ? (i - old_start_bucket) : (rt->size + i - old_start_bucket);
-            const uint32_t newb = (new_start_bucket + dist) & idxmask1;
-            assert (dist < HH_HOP_RANGE);
-            bs1[new_start_bucket].hopinfo |= 1u << dist;
-            bs1[newb].data = data;
-        }
-    }
-
-    ddsrt_free (rt->buckets);
-    rt->size *= 2;
-    rt->buckets = bs1;
 }
 
-int ddsrt_hh_add (struct ddsrt_hh * __restrict rt, const void * __restrict data)
+#endif
-{
-    const uint32_t hash = rt->hash (data);
-    const uint32_t idxmask = rt->size - 1;
-    const uint32_t start_bucket = hash & idxmask;
-    uint32_t free_distance, free_bucket;
-
-    if (ddsrt_hh_lookup_internal (rt, start_bucket, data)) {
-        return 0;
-    }
-
-    free_bucket = start_bucket;
-    for (free_distance = 0; free_distance < HH_ADD_RANGE; free_distance++) {
-        if (rt->buckets[free_bucket].data == NULL)
-            break;
-        free_bucket = (free_bucket + 1) & idxmask;
-    }
-    if (free_distance < HH_ADD_RANGE) {
-        do {
-            if (free_distance < HH_HOP_RANGE) {
-                assert ((uint32_t) free_bucket == ((start_bucket + free_distance) & idxmask));
-                rt->buckets[start_bucket].hopinfo |= 1u << free_distance;
-                rt->buckets[free_bucket].data = (void *) data;
-                return 1;
-            }
-            free_bucket = ddsrt_hh_find_closer_free_bucket (rt, free_bucket, &free_distance);
-            assert (free_bucket == NOT_A_BUCKET || free_bucket <= idxmask);
-        } while (free_bucket != NOT_A_BUCKET);
-    }
-
-    ddsrt_hh_resize (rt);
-    return ddsrt_hh_add (rt, data);
-}
-
-int ddsrt_hh_remove (struct ddsrt_hh * __restrict rt, const void * __restrict template)
-{
-    const uint32_t hash = rt->hash (template);
-    const uint32_t idxmask = rt->size - 1;
-    const uint32_t bucket = hash & idxmask;
-    uint32_t hopinfo;
-    uint32_t idx;
-    hopinfo = rt->buckets[bucket].hopinfo;
-    for (idx = 0; hopinfo != 0; hopinfo >>= 1, idx++) {
-        if (hopinfo & 1) {
-            const uint32_t bidx = (bucket + idx) & idxmask;
-            void *data = rt->buckets[bidx].data;
-            if (data && rt->equals (data, template)) {
-                rt->buckets[bidx].data = NULL;
-                rt->buckets[bucket].hopinfo &= ~(1u << idx);
-                return 1;
-            }
-        }
-    }
-    return 0;
-}
-
-void ddsrt_hh_enum (struct ddsrt_hh * __restrict rt, void (*f) (void *a, void *f_arg), void *f_arg)
-{
-    uint32_t i;
-    for (i = 0; i < rt->size; i++) {
-        void *data = rt->buckets[i].data;
-        if (data) {
-            f (data, f_arg);
-        }
-    }
-}
-
-void *ddsrt_hh_iter_first (struct ddsrt_hh * __restrict rt, struct ddsrt_hh_iter * __restrict iter)
-{
-    iter->hh = rt;
-    iter->cursor = 0;
-    return ddsrt_hh_iter_next (iter);
-}
-
-void *ddsrt_hh_iter_next (struct ddsrt_hh_iter * __restrict iter)
-{
-    struct ddsrt_hh *rt = iter->hh;
-    while (iter->cursor < rt->size) {
-        void *data = rt->buckets[iter->cursor].data;
-        iter->cursor++;
-        if (data) {
-            return data;
-        }
-    }
-    return NULL;
-}
 
 /************* SEQUENTIAL VERSION WITH EMBEDDED DATA ***************/
 

src/mpt/tests/qos/procs/ppud.c

@@ -98,8 +98,8 @@ MPT_ProcessEntry (ppud,
             exp = "X";
           const size_t expsz = strlen (exp);
           bool eq = (usz == expsz && (usz == 0 || memcmp (ud, exp, usz) == 0));
-          printf ("%d: expected %u %zu/%s received %zu/%s\n",
+          //printf ("%d: expected %u %zu/%s received %zu/%s\n",
-                  id, exp_index, expsz, exp, usz, ud ? (char *) ud : "(null)");
+          //        id, exp_index, expsz, exp, usz, ud ? (char *) ud : "(null)");
           MPT_ASSERT (eq, "User data mismatch: expected %u %zu/%s received %zu/%s\n",
                       exp_index, expsz, exp, usz, ud ? (char *) ud : "(null)");
           if (strcmp (exp, "X") != 0 && ++exp_index == sizeof (exp_ud) / sizeof (exp_ud[0]))
@@ -235,8 +235,8 @@ MPT_ProcessEntry (rwud,
             exp = "X";
           const size_t expsz = first ? 1 : strlen (exp);
           bool eq = (usz == expsz && (usz == 0 || memcmp (ud, exp, usz) == 0));
-          printf ("%d: expected %u %zu/%s received %zu/%s\n",
+          //printf ("%d: expected %u %zu/%s received %zu/%s\n",
-                  id, exp_index, expsz, exp, usz, ud ? (char *) ud : "(null)");
+          //        id, exp_index, expsz, exp, usz, ud ? (char *) ud : "(null)");
           MPT_ASSERT (eq, "User data mismatch: expected %u %zu/%s received %zu/%s\n",
                       exp_index, expsz, exp, usz, ud ? (char *) ud : "(null)");
           if (strcmp (exp, "X") != 0 && ++exp_index == sizeof (exp_ud) / sizeof (exp_ud[0]))
@@ -372,8 +372,8 @@ MPT_ProcessEntry (rwtd,
             exp = "X";
           const size_t expsz = first ? 1 : strlen (exp);
           bool eq = (tsz == expsz && (tsz == 0 || memcmp (td, exp, tsz) == 0));
-          printf ("%d: expected %u %zu/%s received %zu/%s\n",
+          //printf ("%d: expected %u %zu/%s received %zu/%s\n",
-                  id, exp_index, expsz, exp, tsz, td ? (char *) td : "(null)");
+          //        id, exp_index, expsz, exp, tsz, td ? (char *) td : "(null)");
           MPT_ASSERT (eq, "Topic data mismatch: expected %u %zu/%s received %zu/%s\n",
                       exp_index, expsz, exp, tsz, td ? (char *) td : "(null)");
           if (strcmp (exp, "X") != 0 && ++exp_index == sizeof (exp_ud) / sizeof (exp_ud[0]))
@@ -511,8 +511,8 @@ MPT_ProcessEntry (rwgd,
             exp = "X";
           const size_t expsz = first ? 1 : strlen (exp);
           bool eq = (gsz == expsz && (gsz == 0 || memcmp (gd, exp, gsz) == 0));
-          printf ("%d: expected %u %zu/%s received %zu/%s\n",
+          //printf ("%d: expected %u %zu/%s received %zu/%s\n",
-                  id, exp_index, expsz, exp, gsz, gd ? (char *) gd : "(null)");
+          //        id, exp_index, expsz, exp, gsz, gd ? (char *) gd : "(null)");
           MPT_ASSERT (eq, "Group data mismatch: expected %u %zu/%s received %zu/%s\n",
                       exp_index, expsz, exp, gsz, gd ? (char *) gd : "(null)");
           if (strcmp (exp, "X") != 0 && ++exp_index == sizeof (exp_ud) / sizeof (exp_ud[0]))