diff --git a/lib/isc/include/isc/thread.h b/lib/isc/include/isc/thread.h index f433d063ac..a24dfcdca5 100644 --- a/lib/isc/include/isc/thread.h +++ b/lib/isc/include/isc/thread.h @@ -35,6 +35,13 @@ ISC_LANG_BEGINDECLS typedef pthread_t isc_thread_t; typedef void *(*isc_threadfunc_t)(void *); +/*% + * like isc_thread_create(), but run the function on the current + * thread which must be the main thread. + */ +void +isc_thread_main(isc_threadfunc_t, void *); + void isc_thread_create(isc_threadfunc_t, void *, isc_thread_t *); diff --git a/lib/isc/include/isc/util.h b/lib/isc/include/isc/util.h index b688b20cad..fd286931d3 100644 --- a/lib/isc/include/isc/util.h +++ b/lib/isc/include/isc/util.h @@ -247,9 +247,19 @@ #endif /* if __has_feature(thread_sanitizer) */ #if __SANITIZE_THREAD__ +/* + * We should rather be including , but GCC 10 + * header is broken, so we just make the declarations by hand. + */ +void +__tsan_acquire(void *addr); +void +__tsan_release(void *addr); #define ISC_NO_SANITIZE_THREAD __attribute__((no_sanitize("thread"))) #else /* if __SANITIZE_THREAD__ */ #define ISC_NO_SANITIZE_THREAD +#define __tsan_acquire(addr) +#define __tsan_release(addr) #endif /* if __SANITIZE_THREAD__ */ #if __GNUC__ > 4 || (__GNUC__ == 4 && __GNUC_MINOR >= 6) diff --git a/lib/isc/loop.c b/lib/isc/loop.c index 5209172a0f..77f3dcc52e 100644 --- a/lib/isc/loop.c +++ b/lib/isc/loop.c @@ -103,6 +103,7 @@ resume_loop(isc_loop_t *loop) { (void)isc_barrier_wait(&loopmgr->resuming); loop->paused = false; + rcu_thread_online(); } @@ -268,54 +269,17 @@ static void quiescent_cb(uv_prepare_t *handle) { isc__qsbr_quiescent_cb(handle); -#ifndef RCU_QSBR - INSIST(!rcu_read_ongoing()); -#else +#if defined(RCU_QSBR) /* safe memory reclamation */ rcu_quiescent_state(); /* mark the thread offline when polling */ rcu_thread_offline(); +#else + INSIST(!rcu_read_ongoing()); #endif } -static void -loop_call_rcu_init(struct rcu_head *rcu_head __attribute__((__unused__))) { - /* Work around the jemalloc bug, see trampoline.c for details */ - void *ptr = malloc(8); - free(ptr); - - /* Initialize the random generator in the call_rcu thread */ - isc__random_initialize(); -} - -static void -loop_run(isc_loop_t *loop) { - int r = uv_prepare_start(&loop->quiescent, quiescent_cb); - UV_RUNTIME_CHECK(uv_prepare_start, r); - - isc_barrier_wait(&loop->loopmgr->starting); - - isc_async_run(loop, setup_jobs_cb, loop); - - rcu_register_thread(); - - struct call_rcu_data *crdp = create_call_rcu_data(0, -1); - set_thread_call_rcu_data(crdp); - - call_rcu(&loop->rcu_head, loop_call_rcu_init); - - r = uv_run(&loop->loop, UV_RUN_DEFAULT); - UV_RUNTIME_CHECK(uv_run, r); - - rcu_unregister_thread(); - - set_thread_call_rcu_data(NULL); - call_rcu_data_free(crdp); - - isc_barrier_wait(&loop->loopmgr->stopping); -} - static void loop_close(isc_loop_t *loop) { int r = uv_loop_close(&loop->loop); @@ -337,7 +301,17 @@ loop_thread(void *arg) { isc__tid_init(loop->tid); - loop_run(loop); + int r = uv_prepare_start(&loop->quiescent, quiescent_cb); + UV_RUNTIME_CHECK(uv_prepare_start, r); + + isc_barrier_wait(&loop->loopmgr->starting); + + isc_async_run(loop, setup_jobs_cb, loop); + + r = uv_run(&loop->loop, UV_RUN_DEFAULT); + UV_RUNTIME_CHECK(uv_run, r); + + isc_barrier_wait(&loop->loopmgr->stopping); return (NULL); } @@ -511,7 +485,7 @@ isc_loopmgr_run(isc_loopmgr_t *loopmgr) { isc_thread_setname(loop->thread, name); } - loop_thread(&loopmgr->loops[0]); + isc_thread_main(loop_thread, &loopmgr->loops[0]); } void diff --git a/lib/isc/loop_p.h b/lib/isc/loop_p.h index 373d5df574..9594a0f306 100644 --- a/lib/isc/loop_p.h +++ b/lib/isc/loop_p.h @@ -28,7 +28,6 @@ #include #include #include -#include #include #include @@ -80,8 +79,6 @@ struct isc_loop { uv_async_t wakeup_trigger; uv_prepare_t quiescent; isc_qsbr_phase_t qsbr_phase; - - struct rcu_head rcu_head; }; /* diff --git a/lib/isc/thread.c b/lib/isc/thread.c index c0a1607db4..aca7c38332 100644 --- a/lib/isc/thread.c +++ b/lib/isc/thread.c @@ -30,9 +30,12 @@ #include +#include #include #include #include +#include +#include #include #ifndef THREAD_MINSTACKSIZE @@ -46,18 +49,15 @@ */ struct thread_wrap { + struct rcu_head rcu_head; isc_threadfunc_t func; void *arg; - void (*free)(void *); }; static struct thread_wrap * thread_wrap(isc_threadfunc_t func, void *arg) { struct thread_wrap *wrap = malloc(sizeof(*wrap)); - - RUNTIME_CHECK(wrap != NULL); *wrap = (struct thread_wrap){ - .free = free, /* from stdlib */ .func = func, .arg = arg, }; @@ -66,30 +66,56 @@ thread_wrap(isc_threadfunc_t func, void *arg) { } static void * -thread_run(void *arg) { - struct thread_wrap *wrap = arg; - isc_threadfunc_t wrap_func = wrap->func; - void *wrap_arg = wrap->arg; - void *result = NULL; +thread_body(struct thread_wrap *wrap) { + isc_threadfunc_t func = wrap->func; + void *arg = wrap->arg; + void *ret = NULL; /* * Every thread starts with a malloc() call to prevent memory bloat - * caused by a jemalloc quirk. To stop an optimizing compiler from - * stripping out free(malloc(1)), we call free via a function pointer. + * caused by a jemalloc quirk. We use CMM_ACCESS_ONCE() To stop an + * optimizing compiler from stripping out free(malloc(1)). */ - wrap->free(malloc(1)); - wrap->free(wrap); + void *jemalloc_enforce_init = NULL; + CMM_ACCESS_ONCE(jemalloc_enforce_init) = malloc(1); + free(jemalloc_enforce_init); - /* Get a thread-local digest context. */ + /* Reassure Thread Sanitizer that it is safe to free the wrapper */ + __tsan_acquire(wrap); + free(wrap); + + rcu_register_thread(); + + ret = func(arg); + + rcu_unregister_thread(); + + return (ret); +} + +static void * +thread_run(void *wrap) { + /* + * Get a thread-local digest context only in new threads. + * The main thread is handled by isc__initialize(). + */ isc__iterated_hash_initialize(); - /* Run the main function */ - result = wrap_func(wrap_arg); + void *ret = thread_body(wrap); - /* Cleanup */ isc__iterated_hash_shutdown(); - return (result); + return (ret); +} + +void +isc_thread_main(isc_threadfunc_t func, void *arg) { + /* + * Either this thread has not yet been started, so it can become the + * main thread, or it has already been annointed as the chosen zero + */ + REQUIRE(isc_tid() == ISC_TID_UNKNOWN || isc_tid() == 0); + thread_body(thread_wrap(func, arg)); } void