diff --git a/CHANGES b/CHANGES index 5ddf3d2146..10706d629e 100644 --- a/CHANGES +++ b/CHANGES @@ -1,3 +1,6 @@ +4728. [func] Use C11's stdatomic.h instead of isc_atomic + where available. [RT #40668] + 4727. [bug] Retransferring an inline-signed slave using NSEC3 around the time its NSEC3 salt was changed could result in an infinite signing loop. [RT #45080] diff --git a/config.h.in b/config.h.in index 9cb97efadf..8c669e276a 100644 --- a/config.h.in +++ b/config.h.in @@ -449,6 +449,9 @@ int sigwait(const unsigned int *set, int *sig); /* Define to 1 if you have the `setresuid' function. */ #undef HAVE_SETRESUID +/* Define to 1 if you have the header file. */ +#undef HAVE_STDATOMIC_H + /* Define to 1 if you have the header file. */ #undef HAVE_STDINT_H diff --git a/configure b/configure index 4d28ad9ec1..25081d21b2 100755 --- a/configure +++ b/configure @@ -718,6 +718,7 @@ ISC_PLATFORM_HAVEATOMICSTORE ISC_PLATFORM_HAVECMPXCHG ISC_PLATFORM_HAVEXADDQ ISC_PLATFORM_HAVEXADD +ISC_PLATFORM_HAVESTDATOMIC ISC_PLATFORM_HAVEIFNAMETOINDEX ISC_PLATFORM_HAVESTRINGSH ISC_PLATFORM_BRACEPTHREADONCEINIT @@ -20126,6 +20127,21 @@ done # # Machine architecture dependent features # +for ac_header in stdatomic.h +do : + ac_fn_c_check_header_mongrel "$LINENO" "stdatomic.h" "ac_cv_header_stdatomic_h" "$ac_includes_default" +if test "x$ac_cv_header_stdatomic_h" = xyes; then : + cat >>confdefs.h <<_ACEOF +#define HAVE_STDATOMIC_H 1 +_ACEOF + ISC_PLATFORM_HAVESTDATOMIC="#define ISC_PLATFORM_HAVESTDATOMIC 1" +else + ISC_PLATFORM_HAVESTDATOMIC="#undef ISC_PLATFORM_HAVESTDATOMIC" +fi + +done + + # Check whether --enable-atomic was given. if test "${enable_atomic+set}" = set; then : enableval=$enable_atomic; enable_atomic="$enableval" @@ -20201,11 +20217,14 @@ rm -f core conftest.err conftest.$ac_objext conftest.$ac_ext esac ;; no) + ISC_PLATFORM_HAVESTDATOMIC="#undef ISC_PLATFORM_HAVESTDATOMIC" use_atomic=no arch=noatomic ;; esac + + ISC_PLATFORM_USEOSFASM="#undef ISC_PLATFORM_USEOSFASM" ISC_PLATFORM_USEGCCASM="#undef ISC_PLATFORM_USEGCCASM" ISC_PLATFORM_USESTDASM="#undef ISC_PLATFORM_USESTDASM" diff --git a/configure.in b/configure.in index f71a7fcbbf..9fa10c48e2 100644 --- a/configure.in +++ b/configure.in @@ -4006,6 +4006,10 @@ AC_CHECK_FUNCS(nanosleep usleep) # # Machine architecture dependent features # +AC_CHECK_HEADERS(stdatomic.h, + [ISC_PLATFORM_HAVESTDATOMIC="#define ISC_PLATFORM_HAVESTDATOMIC 1"], + [ISC_PLATFORM_HAVESTDATOMIC="#undef ISC_PLATFORM_HAVESTDATOMIC"]) + AC_ARG_ENABLE(atomic, [ --enable-atomic enable machine specific atomic operations [[default=autodetect]]], @@ -4048,11 +4052,14 @@ case "$enable_atomic" in esac ;; no) + ISC_PLATFORM_HAVESTDATOMIC="#undef ISC_PLATFORM_HAVESTDATOMIC" use_atomic=no arch=noatomic ;; esac +AC_SUBST(ISC_PLATFORM_HAVESTDATOMIC) + ISC_PLATFORM_USEOSFASM="#undef ISC_PLATFORM_USEOSFASM" ISC_PLATFORM_USEGCCASM="#undef ISC_PLATFORM_USEGCCASM" ISC_PLATFORM_USESTDASM="#undef ISC_PLATFORM_USESTDASM" diff --git a/lib/isc/include/isc/platform.h.in b/lib/isc/include/isc/platform.h.in index f648dffa5d..ef9b255e62 100644 --- a/lib/isc/include/isc/platform.h.in +++ b/lib/isc/include/isc/platform.h.in @@ -306,6 +306,12 @@ */ @ISC_PLATFORM_HAVECMPXCHG@ +/* + * If is available on this architecture, + * ISC_PLATFORM_HAVESTDATOMIC will be defined. + */ +@ISC_PLATFORM_HAVESTDATOMIC@ + /* * Define if gcc ASM extension is available */ diff --git a/lib/isc/include/isc/refcount.h b/lib/isc/include/isc/refcount.h index a2b7f5f9a7..8550ea3db9 100644 --- a/lib/isc/include/isc/refcount.h +++ b/lib/isc/include/isc/refcount.h @@ -18,6 +18,10 @@ #include #include +#if defined(ISC_PLATFORM_HAVESTDATOMIC) +#include +#endif + /*! \file isc/refcount.h * \brief Implements a locked reference counter. * @@ -86,17 +90,59 @@ ISC_LANG_BEGINDECLS * Sample implementations */ #ifdef ISC_PLATFORM_USETHREADS -#ifdef ISC_PLATFORM_HAVEXADD - +#if (defined(ISC_PLATFORM_HAVESTDATOMIC) && defined(ATOMIC_INT_LOCK_FREE)) || defined(ISC_PLATFORM_HAVEXADD) #define ISC_REFCOUNT_HAVEATOMIC 1 +#if (defined(ISC_PLATFORM_HAVESTDATOMIC) && defined(ATOMIC_INT_LOCK_FREE)) +#define ISC_REFCOUNT_HAVESTDATOMIC 1 +#endif typedef struct isc_refcount { +#if defined(ISC_REFCOUNT_HAVESTDATOMIC) + atomic_int_fast32_t refs; +#else isc_int32_t refs; +#endif } isc_refcount_t; #define isc_refcount_destroy(rp) REQUIRE((rp)->refs == 0) #define isc_refcount_current(rp) ((unsigned int)((rp)->refs)) +#if defined(ISC_REFCOUNT_HAVESTDATOMIC) + +#define isc_refcount_increment0(rp, tp) \ + do { \ + unsigned int *_tmp = (unsigned int *)(tp); \ + isc_int32_t prev; \ + prev = atomic_fetch_add_explicit \ + (&(rp)->refs, 1, memory_order_relaxed); \ + if (_tmp != NULL) \ + *_tmp = prev + 1; \ + } while (0) + +#define isc_refcount_increment(rp, tp) \ + do { \ + unsigned int *_tmp = (unsigned int *)(tp); \ + isc_int32_t prev; \ + prev = atomic_fetch_add_explicit \ + (&(rp)->refs, 1, memory_order_relaxed); \ + REQUIRE(prev > 0); \ + if (_tmp != NULL) \ + *_tmp = prev + 1; \ + } while (0) + +#define isc_refcount_decrement(rp, tp) \ + do { \ + unsigned int *_tmp = (unsigned int *)(tp); \ + isc_int32_t prev; \ + prev = atomic_fetch_sub_explicit \ + (&(rp)->refs, 1, memory_order_relaxed); \ + REQUIRE(prev > 0); \ + if (_tmp != NULL) \ + *_tmp = prev - 1; \ + } while (0) + +#else /* ISC_REFCOUNT_HAVESTDATOMIC */ + #define isc_refcount_increment0(rp, tp) \ do { \ unsigned int *_tmp = (unsigned int *)(tp); \ @@ -126,6 +172,8 @@ typedef struct isc_refcount { *_tmp = prev - 1; \ } while (0) +#endif /* ISC_REFCOUNT_HAVESTDATOMIC */ + #else /* ISC_PLATFORM_HAVEXADD */ typedef struct isc_refcount { @@ -176,7 +224,7 @@ typedef struct isc_refcount { UNLOCK(&(rp)->lock); \ } while (0) -#endif /* ISC_PLATFORM_HAVEXADD */ +#endif /* (defined(ISC_PLATFORM_HAVESTDATOMIC) && defined(ATOMIC_INT_LOCK_FREE)) || defined(ISC_PLATFORM_HAVEXADD) */ #else /* ISC_PLATFORM_USETHREADS */ typedef struct isc_refcount { diff --git a/lib/isc/include/isc/rwlock.h b/lib/isc/include/isc/rwlock.h index 6f8d3788bf..2917174177 100644 --- a/lib/isc/include/isc/rwlock.h +++ b/lib/isc/include/isc/rwlock.h @@ -18,6 +18,10 @@ #include #include +#if defined(ISC_PLATFORM_HAVESTDATOMIC) +#include +#endif + ISC_LANG_BEGINDECLS typedef enum { @@ -27,8 +31,11 @@ typedef enum { } isc_rwlocktype_t; #ifdef ISC_PLATFORM_USETHREADS -#if defined(ISC_PLATFORM_HAVEXADD) && defined(ISC_PLATFORM_HAVECMPXCHG) +#if (defined(ISC_PLATFORM_HAVESTDATOMIC) && defined(ATOMIC_INT_LOCK_FREE)) || (defined(ISC_PLATFORM_HAVEXADD) && defined(ISC_PLATFORM_HAVECMPXCHG)) #define ISC_RWLOCK_USEATOMIC 1 +#if (defined(ISC_PLATFORM_HAVESTDATOMIC) && defined(ATOMIC_INT_LOCK_FREE)) +#define ISC_RWLOCK_USESTDATOMIC 1 +#endif #endif struct isc_rwlock { @@ -37,7 +44,7 @@ struct isc_rwlock { isc_mutex_t lock; isc_int32_t spins; -#if defined(ISC_PLATFORM_HAVEXADD) && defined(ISC_PLATFORM_HAVECMPXCHG) +#if defined(ISC_RWLOCK_USEATOMIC) /* * When some atomic instructions with hardware assistance are * available, rwlock will use those so that concurrent readers do not @@ -52,9 +59,15 @@ struct isc_rwlock { */ /* Read or modified atomically. */ +#if defined(ISC_RWLOCK_USESTDATOMIC) + atomic_int_fast32_t write_requests; + atomic_int_fast32_t write_completions; + atomic_int_fast32_t cnt_and_flag; +#else isc_int32_t write_requests; isc_int32_t write_completions; isc_int32_t cnt_and_flag; +#endif /* Locked by lock. */ isc_condition_t readable; @@ -67,7 +80,7 @@ struct isc_rwlock { /* Unlocked. */ unsigned int write_quota; -#else /* ISC_PLATFORM_HAVEXADD && ISC_PLATFORM_HAVECMPXCHG */ +#else /* ISC_RWLOCK_USEATOMIC */ /*%< Locked by lock. */ isc_condition_t readable; @@ -89,7 +102,7 @@ struct isc_rwlock { unsigned int read_quota; unsigned int write_quota; isc_rwlocktype_t original; -#endif /* ISC_PLATFORM_HAVEXADD && ISC_PLATFORM_HAVECMPXCHG */ +#endif /* ISC_RWLOCK_USEATOMIC */ }; #else /* ISC_PLATFORM_USETHREADS */ struct isc_rwlock { diff --git a/lib/isc/refcount.c b/lib/isc/refcount.c index 3af4d19ed3..ad1fd5b945 100644 --- a/lib/isc/refcount.c +++ b/lib/isc/refcount.c @@ -21,7 +21,7 @@ isc_refcount_init(isc_refcount_t *ref, unsigned int n) { REQUIRE(ref != NULL); ref->refs = n; -#if defined(ISC_PLATFORM_USETHREADS) && !defined(ISC_PLATFORM_HAVEXADD) +#if defined(ISC_PLATFORM_USETHREADS) && !defined(ISC_REFCOUNT_HAVEATOMIC) return (isc_mutex_init(&ref->lock)); #else return (ISC_R_SUCCESS); diff --git a/lib/isc/rwlock.c b/lib/isc/rwlock.c index 57e78b39ec..e703f8426d 100644 --- a/lib/isc/rwlock.c +++ b/lib/isc/rwlock.c @@ -39,7 +39,7 @@ #define RWLOCK_MAX_ADAPTIVE_COUNT 100 #endif -#if defined(ISC_PLATFORM_HAVEXADD) && defined(ISC_PLATFORM_HAVECMPXCHG) +#if defined(ISC_RWLOCK_USEATOMIC) static isc_result_t isc__rwlock_lock(isc_rwlock_t *rwl, isc_rwlocktype_t type); #endif @@ -50,7 +50,7 @@ isc__rwlock_lock(isc_rwlock_t *rwl, isc_rwlocktype_t type); static void print_lock(const char *operation, isc_rwlock_t *rwl, isc_rwlocktype_t type) { -#if defined(ISC_PLATFORM_HAVEXADD) && defined(ISC_PLATFORM_HAVECMPXCHG) +#if defined(ISC_RWLOCK_USEATOMIC) fprintf(stderr, isc_msgcat_get(isc_msgcat, ISC_MSGSET_RWLOCK, ISC_MSG_PRINTLOCK2, @@ -105,7 +105,7 @@ isc_rwlock_init(isc_rwlock_t *rwl, unsigned int read_quota, rwl->magic = 0; rwl->spins = 0; -#if defined(ISC_PLATFORM_HAVEXADD) && defined(ISC_PLATFORM_HAVECMPXCHG) +#if defined(ISC_RWLOCK_USEATOMIC) rwl->write_requests = 0; rwl->write_completions = 0; rwl->cnt_and_flag = 0; @@ -174,7 +174,7 @@ void isc_rwlock_destroy(isc_rwlock_t *rwl) { REQUIRE(VALID_RWLOCK(rwl)); -#if defined(ISC_PLATFORM_HAVEXADD) && defined(ISC_PLATFORM_HAVECMPXCHG) +#if defined(ISC_RWLOCK_USEATOMIC) REQUIRE(rwl->write_requests == rwl->write_completions && rwl->cnt_and_flag == 0 && rwl->readers_waiting == 0); #else @@ -191,7 +191,7 @@ isc_rwlock_destroy(isc_rwlock_t *rwl) { DESTROYLOCK(&rwl->lock); } -#if defined(ISC_PLATFORM_HAVEXADD) && defined(ISC_PLATFORM_HAVECMPXCHG) +#if defined(ISC_RWLOCK_USEATOMIC) /* * When some architecture-dependent atomic operations are available, @@ -281,7 +281,13 @@ isc__rwlock_lock(isc_rwlock_t *rwl, isc_rwlocktype_t type) { UNLOCK(&rwl->lock); } +#if defined(ISC_RWLOCK_USESTDATOMIC) + cntflag = atomic_fetch_add_explicit(&rwl->cnt_and_flag, + READER_INCR, + memory_order_relaxed); +#else cntflag = isc_atomic_xadd(&rwl->cnt_and_flag, READER_INCR); +#endif POST(cntflag); while (1) { if ((rwl->cnt_and_flag & WRITER_ACTIVE) == 0) @@ -331,7 +337,12 @@ isc__rwlock_lock(isc_rwlock_t *rwl, isc_rwlocktype_t type) { isc_int32_t prev_writer; /* enter the waiting queue, and wait for our turn */ +#if defined(ISC_RWLOCK_USESTDATOMIC) + prev_writer = atomic_fetch_add_explicit(&rwl->write_requests, 1, + memory_order_relaxed); +#else prev_writer = isc_atomic_xadd(&rwl->write_requests, 1); +#endif while (rwl->write_completions != prev_writer) { LOCK(&rwl->lock); if (rwl->write_completions != prev_writer) { @@ -344,9 +355,18 @@ isc__rwlock_lock(isc_rwlock_t *rwl, isc_rwlocktype_t type) { } while (1) { - cntflag = isc_atomic_cmpxchg(&rwl->cnt_and_flag, 0, - WRITER_ACTIVE); - if (cntflag == 0) +#if defined(ISC_RWLOCK_USESTDATOMIC) + atomic_int_fast32_t cntflag2 = 0; + atomic_compare_exchange_strong_explicit + (&rwl->cnt_and_flag, &cntflag2, WRITER_ACTIVE, + memory_order_relaxed, memory_order_relaxed); +#else + isc_int32_t cntflag2; + cntflag2 = isc_atomic_cmpxchg(&rwl->cnt_and_flag, 0, + WRITER_ACTIVE); +#endif + + if (cntflag2 == 0) break; /* Another active reader or writer is working. */ @@ -409,14 +429,26 @@ isc_rwlock_trylock(isc_rwlock_t *rwl, isc_rwlocktype_t type) { return (ISC_R_LOCKBUSY); /* Otherwise, be ready for reading. */ +#if defined(ISC_RWLOCK_USESTDATOMIC) + cntflag = atomic_fetch_add_explicit(&rwl->cnt_and_flag, + READER_INCR, + memory_order_relaxed); +#else cntflag = isc_atomic_xadd(&rwl->cnt_and_flag, READER_INCR); +#endif if ((cntflag & WRITER_ACTIVE) != 0) { /* * A writer is working. We lose, and cancel the read * request. */ +#if defined(ISC_RWLOCK_USESTDATOMIC) + cntflag = atomic_fetch_sub_explicit + (&rwl->cnt_and_flag, READER_INCR, + memory_order_relaxed); +#else cntflag = isc_atomic_xadd(&rwl->cnt_and_flag, -READER_INCR); +#endif /* * If no other readers are waiting and we've suspended * new writers in this short period, wake them up. @@ -432,16 +464,29 @@ isc_rwlock_trylock(isc_rwlock_t *rwl, isc_rwlocktype_t type) { } } else { /* Try locking without entering the waiting queue. */ +#if defined(ISC_RWLOCK_USESTDATOMIC) + atomic_int_fast32_t zero = 0; + if (!atomic_compare_exchange_strong_explicit + (&rwl->cnt_and_flag, &zero, WRITER_ACTIVE, + memory_order_relaxed, memory_order_relaxed)) + return (ISC_R_LOCKBUSY); +#else cntflag = isc_atomic_cmpxchg(&rwl->cnt_and_flag, 0, WRITER_ACTIVE); if (cntflag != 0) return (ISC_R_LOCKBUSY); +#endif /* * XXXJT: jump into the queue, possibly breaking the writer * order. */ +#if defined(ISC_RWLOCK_USESTDATOMIC) + atomic_fetch_sub_explicit(&rwl->write_completions, 1, + memory_order_relaxed); +#else (void)isc_atomic_xadd(&rwl->write_completions, -1); +#endif rwl->write_granted++; } @@ -456,31 +501,60 @@ isc_rwlock_trylock(isc_rwlock_t *rwl, isc_rwlocktype_t type) { isc_result_t isc_rwlock_tryupgrade(isc_rwlock_t *rwl) { - isc_int32_t prevcnt; - REQUIRE(VALID_RWLOCK(rwl)); - /* Try to acquire write access. */ - prevcnt = isc_atomic_cmpxchg(&rwl->cnt_and_flag, - READER_INCR, WRITER_ACTIVE); - /* - * There must have been no writer, and there must have been at least - * one reader. - */ - INSIST((prevcnt & WRITER_ACTIVE) == 0 && - (prevcnt & ~WRITER_ACTIVE) != 0); +#if defined(ISC_RWLOCK_USESTDATOMIC) + { + atomic_int_fast32_t reader_incr = READER_INCR; - if (prevcnt == READER_INCR) { + /* Try to acquire write access. */ + atomic_compare_exchange_strong_explicit + (&rwl->cnt_and_flag, &reader_incr, WRITER_ACTIVE, + memory_order_relaxed, memory_order_relaxed); /* - * We are the only reader and have been upgraded. - * Now jump into the head of the writer waiting queue. + * There must have been no writer, and there must have + * been at least one reader. */ - (void)isc_atomic_xadd(&rwl->write_completions, -1); - } else - return (ISC_R_LOCKBUSY); + INSIST((reader_incr & WRITER_ACTIVE) == 0 && + (reader_incr & ~WRITER_ACTIVE) != 0); + + if (reader_incr == READER_INCR) { + /* + * We are the only reader and have been upgraded. + * Now jump into the head of the writer waiting queue. + */ + atomic_fetch_sub_explicit(&rwl->write_completions, 1, + memory_order_relaxed); + } else + return (ISC_R_LOCKBUSY); + + } +#else + { + isc_int32_t prevcnt; + + /* Try to acquire write access. */ + prevcnt = isc_atomic_cmpxchg(&rwl->cnt_and_flag, + READER_INCR, WRITER_ACTIVE); + /* + * There must have been no writer, and there must have + * been at least one reader. + */ + INSIST((prevcnt & WRITER_ACTIVE) == 0 && + (prevcnt & ~WRITER_ACTIVE) != 0); + + if (prevcnt == READER_INCR) { + /* + * We are the only reader and have been upgraded. + * Now jump into the head of the writer waiting queue. + */ + (void)isc_atomic_xadd(&rwl->write_completions, -1); + } else + return (ISC_R_LOCKBUSY); + } +#endif return (ISC_R_SUCCESS); - } void @@ -489,14 +563,33 @@ isc_rwlock_downgrade(isc_rwlock_t *rwl) { REQUIRE(VALID_RWLOCK(rwl)); - /* Become an active reader. */ - prev_readers = isc_atomic_xadd(&rwl->cnt_and_flag, READER_INCR); - /* We must have been a writer. */ - INSIST((prev_readers & WRITER_ACTIVE) != 0); +#if defined(ISC_RWLOCK_USESTDATOMIC) + { + /* Become an active reader. */ + prev_readers = atomic_fetch_add_explicit(&rwl->cnt_and_flag, + READER_INCR, + memory_order_relaxed); + /* We must have been a writer. */ + INSIST((prev_readers & WRITER_ACTIVE) != 0); - /* Complete write */ - (void)isc_atomic_xadd(&rwl->cnt_and_flag, -WRITER_ACTIVE); - (void)isc_atomic_xadd(&rwl->write_completions, 1); + /* Complete write */ + atomic_fetch_sub_explicit(&rwl->cnt_and_flag, WRITER_ACTIVE, + memory_order_relaxed); + atomic_fetch_add_explicit(&rwl->write_completions, 1, + memory_order_relaxed); + } +#else + { + /* Become an active reader. */ + prev_readers = isc_atomic_xadd(&rwl->cnt_and_flag, READER_INCR); + /* We must have been a writer. */ + INSIST((prev_readers & WRITER_ACTIVE) != 0); + + /* Complete write */ + (void)isc_atomic_xadd(&rwl->cnt_and_flag, -WRITER_ACTIVE); + (void)isc_atomic_xadd(&rwl->write_completions, 1); + } +#endif /* Resume other readers */ LOCK(&rwl->lock); @@ -517,8 +610,13 @@ isc_rwlock_unlock(isc_rwlock_t *rwl, isc_rwlocktype_t type) { #endif if (type == isc_rwlocktype_read) { +#if defined(ISC_RWLOCK_USESTDATOMIC) + prev_cnt = atomic_fetch_sub_explicit(&rwl->cnt_and_flag, + READER_INCR, + memory_order_relaxed); +#else prev_cnt = isc_atomic_xadd(&rwl->cnt_and_flag, -READER_INCR); - +#endif /* * If we're the last reader and any writers are waiting, wake * them up. We need to wake up all of them to ensure the @@ -537,8 +635,15 @@ isc_rwlock_unlock(isc_rwlock_t *rwl, isc_rwlocktype_t type) { * Reset the flag, and (implicitly) tell other writers * we are done. */ +#if defined(ISC_RWLOCK_USESTDATOMIC) + atomic_fetch_sub_explicit(&rwl->cnt_and_flag, WRITER_ACTIVE, + memory_order_relaxed); + atomic_fetch_add_explicit(&rwl->write_completions, 1, + memory_order_relaxed); +#else (void)isc_atomic_xadd(&rwl->cnt_and_flag, -WRITER_ACTIVE); (void)isc_atomic_xadd(&rwl->write_completions, 1); +#endif if (rwl->write_granted >= rwl->write_quota || rwl->write_requests == rwl->write_completions || @@ -576,7 +681,7 @@ isc_rwlock_unlock(isc_rwlock_t *rwl, isc_rwlocktype_t type) { return (ISC_R_SUCCESS); } -#else /* ISC_PLATFORM_HAVEXADD && ISC_PLATFORM_HAVECMPXCHG */ +#else /* ISC_RWLOCK_USEATOMIC */ static isc_result_t doit(isc_rwlock_t *rwl, isc_rwlocktype_t type, isc_boolean_t nonblock) { @@ -782,7 +887,7 @@ isc_rwlock_unlock(isc_rwlock_t *rwl, isc_rwlocktype_t type) { return (ISC_R_SUCCESS); } -#endif /* ISC_PLATFORM_HAVEXADD && ISC_PLATFORM_HAVECMPXCHG */ +#endif /* ISC_RWLOCK_USEATOMIC */ #else /* ISC_PLATFORM_USETHREADS */ isc_result_t diff --git a/lib/isc/stats.c b/lib/isc/stats.c index 85f485ce84..b9d74966a1 100644 --- a/lib/isc/stats.c +++ b/lib/isc/stats.c @@ -24,6 +24,10 @@ #include #include +#if defined(ISC_PLATFORM_HAVESTDATOMIC) +#include +#endif + #define ISC_STATS_MAGIC ISC_MAGIC('S', 't', 'a', 't') #define ISC_STATS_VALID(x) ISC_MAGIC_VALID(x, ISC_STATS_MAGIC) @@ -32,8 +36,12 @@ * increment and store operations, just to make * the later macros simpler */ -#if defined(ISC_PLATFORM_HAVEXADDQ) && defined(ISC_PLATFORM_HAVEATOMICSTOREQ) +#if (defined(ISC_PLATFORM_HAVESTDATOMIC) && defined(ATOMIC_LONG_LOCK_FREE)) || \ + (defined(ISC_PLATFORM_HAVEXADDQ) && defined(ISC_PLATFORM_HAVEATOMICSTOREQ)) #define ISC_STATS_HAVEATOMICQ 1 +#if (defined(ISC_PLATFORM_HAVESTDATOMIC) && defined(ATOMIC_LONG_LOCK_FREE)) +#define ISC_STATS_HAVESTDATOMICQ 1 +#endif #else #define ISC_STATS_HAVEATOMICQ 0 #endif @@ -61,20 +69,32 @@ * Otherwise, just rely on standard 64-bit data types * and operations */ -#if !ISC_STATS_HAVEATOMICQ && defined(ISC_PLATFORM_HAVEXADD) +#if !ISC_STATS_HAVEATOMICQ && ((defined(ISC_PLATFORM_HAVESTDATOMIC) && defined(ATOMIC_INT_LOCK_FREE)) || defined(ISC_PLATFORM_HAVEXADD)) #define ISC_STATS_USEMULTIFIELDS 1 +#if (defined(ISC_PLATFORM_HAVESTDATOMIC) && defined(ATOMIC_INT_LOCK_FREE)) +#define ISC_STATS_HAVESTDATOMIC 1 +#endif #else #define ISC_STATS_USEMULTIFIELDS 0 #endif #if ISC_STATS_USEMULTIFIELDS typedef struct { +#if defined(ISC_STATS_HAVESTDATOMIC) + atomic_int_fast32_t hi; + atomic_int_fast32_t lo; +#else isc_uint32_t hi; isc_uint32_t lo; +#endif } isc_stat_t; #else +#if defined(ISC_STATS_HAVESTDATOMICQ) +typedef atomic_int_fast64_t isc_stat_t; +#else typedef isc_uint64_t isc_stat_t; #endif +#endif struct isc_stats { /*% Unlocked */ @@ -232,7 +252,12 @@ incrementcounter(isc_stats_t *stats, int counter) { #endif #if ISC_STATS_USEMULTIFIELDS +#if defined(ISC_STATS_HAVESTDATOMIC) + prev = atomic_fetch_add_explicit(&stats->counters[counter].lo, 1, + memory_order_relaxed); +#else prev = isc_atomic_xadd((isc_int32_t *)&stats->counters[counter].lo, 1); +#endif /* * If the lower 32-bit field overflows, increment the higher field. * Note that it's *theoretically* possible that the lower field @@ -241,11 +266,22 @@ incrementcounter(isc_stats_t *stats, int counter) { * isc_stats_copy() is called where the whole process is protected * by the write (exclusive) lock. */ - if (prev == (isc_int32_t)0xffffffff) + if (prev == (isc_int32_t)0xffffffff) { +#if defined(ISC_STATS_HAVESTDATOMIC) + atomic_fetch_add_explicit(&stats->counters[counter].hi, 1, + memory_order_relaxed); +#else isc_atomic_xadd((isc_int32_t *)&stats->counters[counter].hi, 1); +#endif + } #elif ISC_STATS_HAVEATOMICQ UNUSED(prev); +#if defined(ISC_STATS_HAVESTDATOMICQ) + atomic_fetch_add_explicit(&stats->counters[counter], 1, + memory_order_relaxed); +#else isc_atomic_xaddq((isc_int64_t *)&stats->counters[counter], 1); +#endif #else UNUSED(prev); stats->counters[counter]++; @@ -265,13 +301,29 @@ decrementcounter(isc_stats_t *stats, int counter) { #endif #if ISC_STATS_USEMULTIFIELDS +#if defined(ISC_STATS_HAVESTDATOMIC) + prev = atomic_fetch_sub_explicit(&stats->counters[counter].lo, 1, + memory_order_relaxed); +#else prev = isc_atomic_xadd((isc_int32_t *)&stats->counters[counter].lo, -1); - if (prev == 0) +#endif + if (prev == 0) { +#if defined(ISC_STATS_HAVESTDATOMIC) + atomic_fetch_sub_explicit(&stats->counters[counter].hi, 1, + memory_order_relaxed); +#else isc_atomic_xadd((isc_int32_t *)&stats->counters[counter].hi, -1); +#endif + } #elif ISC_STATS_HAVEATOMICQ UNUSED(prev); +#if defined(ISC_STATS_HAVESTDATOMICQ) + atomic_fetch_sub_explicit(&stats->counters[counter], 1, + memory_order_relaxed); +#else isc_atomic_xaddq((isc_int64_t *)&stats->counters[counter], -1); +#endif #else UNUSED(prev); stats->counters[counter]--; @@ -300,9 +352,15 @@ copy_counters(isc_stats_t *stats) { (isc_uint64_t)(stats->counters[i].hi) << 32 | stats->counters[i].lo; #elif ISC_STATS_HAVEATOMICQ +#if defined(ISC_STATS_HAVESTDATOMICQ) + stats->copiedcounters[i] = + atomic_load_explicit(&stats->counters[i], + memory_order_relaxed); +#else /* use xaddq(..., 0) as an atomic load */ stats->copiedcounters[i] = (isc_uint64_t)isc_atomic_xaddq((isc_int64_t *)&stats->counters[i], 0); +#endif #else stats->copiedcounters[i] = stats->counters[i]; #endif @@ -373,7 +431,12 @@ isc_stats_set(isc_stats_t *stats, isc_uint64_t val, stats->counters[counter].hi = (isc_uint32_t)((val >> 32) & 0xffffffff); stats->counters[counter].lo = (isc_uint32_t)(val & 0xffffffff); #elif ISC_STATS_HAVEATOMICQ +#if defined(ISC_STATS_HAVESTDATOMICQ) + atomic_store_explicit(&stats->counters[counter], val, + memory_order_relaxed); +#else isc_atomic_storeq((isc_int64_t *)&stats->counters[counter], val); +#endif #else stats->counters[counter] = val; #endif @@ -382,4 +445,3 @@ isc_stats_set(isc_stats_t *stats, isc_uint64_t val, isc_rwlock_unlock(&stats->counterlock, isc_rwlocktype_write); #endif } -