diff --git a/lib/isc/include/isc/mutexatomic.h b/lib/isc/include/isc/mutexatomic.h index bac34cab00..6e6f49d16a 100644 --- a/lib/isc/include/isc/mutexatomic.h +++ b/lib/isc/include/isc/mutexatomic.h @@ -13,6 +13,9 @@ #include #include +#if HAVE_UCHAR_H +#include +#endif /* HAVE_UCHAR_H */ #include @@ -73,35 +76,65 @@ enum memory_order { typedef enum memory_order memory_order; -typedef struct atomic_int_fast32 { - isc_mutex_t m; - int_fast32_t v; -} atomic_int_fast32_t; +#define ___TYPEDEF(type, name, orig) \ + typedef struct name { \ + isc_mutex_t m; \ + orig v; \ + } type; -typedef struct atomic_int_fast64 { - isc_mutex_t m; - int_fast64_t v; -} atomic_int_fast64_t; +#define _TYPEDEF_S(type) ___TYPEDEF(atomic_##type, atomic_##type##_s, type) +#define _TYPEDEF_O(type, orig) \ + ___TYPEDEF(atomic_##type, atomic_##type##_s, orig) +#define _TYPEDEF_T(type) \ + ___TYPEDEF(atomic_##type##_t, atomic_##type##_s, type##_t) -typedef struct atomic_uint_fast32 { - isc_mutex_t m; - uint_fast32_t v; -} atomic_uint_fast32_t; +#ifndef HAVE_UCHAR_H +typedef uint_least16_t char16_t; +typedef uint_least32_t char32_t; +#endif /* HAVE_UCHAR_H */ -typedef struct atomic_uint_fast64 { - isc_mutex_t m; - uint_fast64_t v; -} atomic_uint_fast64_t; +_TYPEDEF_S(bool); +_TYPEDEF_S(char); +_TYPEDEF_O(schar, signed char); +_TYPEDEF_O(uchar, unsigned char); +_TYPEDEF_S(short); +_TYPEDEF_O(ushort, unsigned short); +_TYPEDEF_S(int); +_TYPEDEF_O(uint, unsigned int); +_TYPEDEF_S(long); +_TYPEDEF_O(ulong, unsigned long); +_TYPEDEF_O(llong, long long); +_TYPEDEF_O(ullong, unsigned long long); +_TYPEDEF_T(char16); +_TYPEDEF_T(char32); +_TYPEDEF_T(wchar); +_TYPEDEF_T(int_least8); +_TYPEDEF_T(uint_least8); +_TYPEDEF_T(int_least16); +_TYPEDEF_T(uint_least16); +_TYPEDEF_T(int_least32); +_TYPEDEF_T(uint_least32); +_TYPEDEF_T(int_least64); +_TYPEDEF_T(uint_least64); +_TYPEDEF_T(int_fast8); +_TYPEDEF_T(uint_fast8); +_TYPEDEF_T(int_fast16); +_TYPEDEF_T(uint_fast16); +_TYPEDEF_T(int_fast32); +_TYPEDEF_T(uint_fast32); +_TYPEDEF_T(int_fast64); +_TYPEDEF_T(uint_fast64); +_TYPEDEF_T(intptr); +_TYPEDEF_T(uintptr); +_TYPEDEF_T(size); +_TYPEDEF_T(ptrdiff); +_TYPEDEF_T(intmax); +_TYPEDEF_T(uintmax); -typedef struct atomic_uintptr { - isc_mutex_t m; - uintptr_t v; -} atomic_uintptr_t; - -typedef struct atomic_bool_s { - isc_mutex_t m; - bool v; -} atomic_bool; +#undef ___TYPEDEF +#undef _TYPEDEF_S +#undef _TYPEDEF_T +#undef _TYPEDEF_O #define ATOMIC_VAR_INIT(arg) \ { \ diff --git a/lib/isc/unix/include/isc/stdatomic.h b/lib/isc/unix/include/isc/stdatomic.h index 7339b87f91..949df07f41 100644 --- a/lib/isc/unix/include/isc/stdatomic.h +++ b/lib/isc/unix/include/isc/stdatomic.h @@ -13,6 +13,10 @@ #include #include +#include +#if HAVE_UCHAR_H +#include +#endif /* HAVE_UCHAR_H */ #if !defined(__has_feature) #define __has_feature(x) 0 @@ -73,13 +77,48 @@ enum memory_order { typedef enum memory_order memory_order; -typedef int_fast32_t atomic_int_fast32_t; -typedef uint_fast32_t atomic_uint_fast32_t; -typedef int_fast64_t atomic_int_fast64_t; -typedef uint_fast64_t atomic_uint_fast64_t; -typedef bool atomic_bool; +#ifndef HAVE_UCHAR_H +typedef uint_least16_t char16_t; +typedef uint_least32_t char32_t; +#endif /* HAVE_UCHAR_H */ -typedef uint_fast64_t atomic_uintptr_t; +typedef bool atomic_bool; +typedef char atomic_char; +typedef signed char atomic_schar; +typedef unsigned char atomic_uchar; +typedef short atomic_short; +typedef unsigned short atomic_ushort; +typedef int atomic_int; +typedef unsigned int atomic_uint; +typedef long atomic_long; +typedef unsigned long atomic_ulong; +typedef long long atomic_llong; +typedef unsigned long long atomic_ullong; +typedef char16_t atomic_char16_t; +typedef char32_t atomic_char32_t; +typedef wchar_t atomic_wchar_t; +typedef int_least8_t atomic_int_least8_t; +typedef uint_least8_t atomic_uint_least8_t; +typedef int_least16_t atomic_int_least16_t; +typedef uint_least16_t atomic_uint_least16_t; +typedef int_least32_t atomic_int_least32_t; +typedef uint_least32_t atomic_uint_least32_t; +typedef int_least64_t atomic_int_least64_t; +typedef uint_least64_t atomic_uint_least64_t; +typedef int_fast8_t atomic_int_fast8_t; +typedef uint_fast8_t atomic_uint_fast8_t; +typedef int_fast16_t atomic_int_fast16_t; +typedef uint_fast16_t atomic_uint_fast16_t; +typedef int_fast32_t atomic_int_fast32_t; +typedef uint_fast32_t atomic_uint_fast32_t; +typedef int_fast64_t atomic_int_fast64_t; +typedef uint_fast64_t atomic_uint_fast64_t; +typedef intptr_t atomic_intptr_t; +typedef uintptr_t atomic_uintptr_t; +typedef size_t atomic_size_t; +typedef ptrdiff_t atomic_ptrdiff_t; +typedef intmax_t atomic_intmax_t; +typedef uintmax_t atomic_uintmax_t; #if defined(__CLANG_ATOMICS) /* __c11_atomic builtins */ #define atomic_init(obj, desired) __c11_atomic_init(obj, desired) diff --git a/lib/isc/win32/include/isc/stdatomic.h b/lib/isc/win32/include/isc/stdatomic.h index fe19df001d..540407e1ea 100644 --- a/lib/isc/win32/include/isc/stdatomic.h +++ b/lib/isc/win32/include/isc/stdatomic.h @@ -15,6 +15,7 @@ #include #include #include +#include #include #pragma warning(disable : 4133) @@ -65,19 +66,58 @@ typedef enum memory_order memory_order; */ typedef bool volatile atomic_bool; +typedef char volatile atomic_char; +typedef signed char volatile atomic_schar; +typedef unsigned char volatile atomic_uchar; +typedef short volatile atomic_short; +typedef unsigned short volatile atomic_ushort; +typedef int volatile atomic_int; +typedef unsigned int volatile atomic_uint; +typedef long volatile atomic_long; +typedef unsigned long volatile atomic_ulong; +typedef long long volatile atomic_llong; +typedef unsigned long long volatile atomic_ullong; +typedef char16_t volatile atomic_char16_t; +typedef char32_t volatile atomic_char32_t; +typedef wchar_t volatile atomic_wchar_t; +typedef int_least8_t volatile atomic_int_least8_t; +typedef uint_least8_t volatile atomic_uint_least8_t; +typedef int_least16_t volatile atomic_int_least16_t; +typedef uint_least16_t volatile atomic_uint_least16_t; +typedef int_least32_t volatile atomic_int_least32_t; +typedef uint_least32_t volatile atomic_uint_least32_t; +typedef int_least64_t volatile atomic_int_least64_t; +typedef uint_least64_t volatile atomic_uint_least64_t; typedef int_fast8_t volatile atomic_int_fast8_t; typedef uint_fast8_t volatile atomic_uint_fast8_t; +typedef int_fast16_t volatile atomic_int_fast16_t; +typedef uint_fast16_t volatile atomic_uint_fast16_t; typedef int_fast32_t volatile atomic_int_fast32_t; typedef uint_fast32_t volatile atomic_uint_fast32_t; typedef int_fast64_t volatile atomic_int_fast64_t; typedef uint_fast64_t volatile atomic_uint_fast64_t; +typedef intptr_t volatile atomic_intptr_t; typedef uintptr_t volatile atomic_uintptr_t; +typedef size_t volatile atomic_size_t; +typedef ptrdiff_t volatile atomic_ptrdiff_t; +typedef intmax_t volatile atomic_intmax_t; +typedef uintmax_t volatile atomic_uintmax_t; #define atomic_init(obj, desired) (*(obj) = (desired)) #define atomic_store_explicit8(obj, desired, order) \ (void)InterlockedExchange8((atomic_int_fast8_t *)obj, desired) +#define atomic_store_explicit16(obj, desired, order) \ + (order == memory_order_relaxed \ + ? (void)InterlockedExchangeNoFence16((atomic_short *)obj, \ + desired) \ + : (order == memory_order_acquire \ + ? (void)InterlockedExchangeAcquire16( \ + (atomic_short *)obj, desired) \ + : (void)InterlockedExchange16((atomic_short *)obj, \ + desired))) + #define atomic_store_explicit32(obj, desired, order) \ (order == memory_order_relaxed \ ? (void)InterlockedExchangeNoFence( \ @@ -109,15 +149,19 @@ atomic_store_abort(void) { ISC_UNREACHABLE(); } -#define atomic_store_explicit(obj, desired, order) \ - (sizeof(*(obj)) == 8 \ - ? atomic_store_explicit64(obj, desired, order) \ - : (sizeof(*(obj)) == 4 \ - ? atomic_store_explicit32(obj, desired, order) \ - : (sizeof(*(obj)) == 1 \ - ? atomic_store_explicit8(obj, desired, \ - order) \ - : atomic_store_abort()))) +#define atomic_store_explicit(obj, desired, order) \ + (sizeof(*(obj)) == 8 \ + ? atomic_store_explicit64(obj, desired, order) \ + : (sizeof(*(obj)) == 4 \ + ? atomic_store_explicit32(obj, desired, order) \ + : (sizeof(*(obj)) == 2 \ + ? atomic_store_explicit16(obj, desired, \ + order) \ + : (sizeof(*(obj)) == 1 \ + ? atomic_store_explicit8( \ + obj, desired, \ + order) \ + : atomic_store_abort())))) #define atomic_store(obj, desired) \ atomic_store_explicit(obj, desired, memory_order_seq_cst) @@ -125,6 +169,9 @@ atomic_store_abort(void) { #define atomic_load_explicit8(obj, order) \ (int8_t) InterlockedOr8((atomic_int_fast8_t *)obj, 0) +#define atomic_load_explicit16(obj, order) \ + (short)InterlockedOr16((atomic_short *)obj, 0) + #define atomic_load_explicit32(obj, order) \ (order == memory_order_relaxed \ ? (int32_t)InterlockedOrNoFence((atomic_int_fast32_t *)obj, \ @@ -163,26 +210,35 @@ atomic_load_abort(void) { ISC_UNREACHABLE(); } -#define atomic_load_explicit(obj, order) \ - ((sizeof(*(obj)) == 8 \ - ? atomic_load_explicit64(obj, order) \ - : (sizeof(*(obj) == 4) \ - ? atomic_load_explicit32(obj, order) \ - : (sizeof(*(obj) == 1) \ - ? atomic_load_explicit8(obj, order) \ - : atomic_load_abort()))) & \ - (sizeof(*(obj)) == 8 \ - ? 0xffffffffffffffffULL \ - : (sizeof(*(obj)) == 4 \ - ? 0xffffffffULL \ - : (sizeof(*(obj)) == 1 ? 0xffULL \ - : atomic_load_abort())))) +#define atomic_load_explicit(obj, order) \ + ((sizeof(*(obj)) == 8 \ + ? atomic_load_explicit64(obj, order) \ + : (sizeof(*(obj) == 4) \ + ? atomic_load_explicit32(obj, order) \ + : (sizeof(*(obj) == 2) \ + ? atomic_load_explicit16(obj, order) \ + : (sizeof(*(obj) == 1) \ + ? atomic_load_explicit8( \ + obj, order) \ + : atomic_load_abort())))) & \ + (sizeof(*(obj)) == 8 \ + ? 0xffffffffffffffffULL \ + : (sizeof(*(obj)) == 4 \ + ? 0xffffffffULL \ + : (sizeof(*(obj)) == 2 \ + ? 0xffffULL \ + : (sizeof(*(obj)) == 1 \ + ? 0xffULL \ + : atomic_load_abort()))))) #define atomic_load(obj) atomic_load_explicit(obj, memory_order_seq_cst) #define atomic_fetch_add_explicit8(obj, arg, order) \ InterlockedExchangeAdd8((atomic_int_fast8_t *)obj, arg) +#define atomic_fetch_add_explicit16(obj, arg, order) \ + InterlockedExchangeAdd16((atomic_short *)obj, arg) + #define atomic_fetch_add_explicit32(obj, arg, order) \ (order == memory_order_relaxed \ ? InterlockedExchangeAddNoFence((atomic_int_fast32_t *)obj, \ @@ -224,15 +280,18 @@ atomic_add_abort(void) { ISC_UNREACHABLE(); } -#define atomic_fetch_add_explicit(obj, arg, order) \ - (sizeof(*(obj)) == 8 \ - ? atomic_fetch_add_explicit64(obj, arg, order) \ - : (sizeof(*(obj)) == 4 \ - ? atomic_fetch_add_explicit32(obj, arg, order) \ - : (sizeof(*(obj)) == 1 \ - ? atomic_fetch_add_explicit8(obj, arg, \ - order) \ - : atomic_add_abort()))) +#define atomic_fetch_add_explicit(obj, arg, order) \ + (sizeof(*(obj)) == 8 \ + ? atomic_fetch_add_explicit64(obj, arg, order) \ + : (sizeof(*(obj)) == 4 \ + ? atomic_fetch_add_explicit32(obj, arg, order) \ + : (sizeof(*(obj)) == 2 \ + ? atomic_fetch_add_explicit16(obj, arg, \ + order) \ + : (sizeof(*(obj)) == 1 \ + ? atomic_fetch_add_explicit8( \ + obj, arg, order) \ + : atomic_add_abort())))) #define atomic_fetch_add(obj, arg) \ atomic_fetch_add_explicit(obj, arg, memory_order_seq_cst) @@ -246,6 +305,9 @@ atomic_add_abort(void) { #define atomic_fetch_and_explicit8(obj, arg, order) \ InterlockedAnd8((atomic_int_fast8_t *)obj, arg) +#define atomic_fetch_and_explicit16(obj, arg, order) \ + InterlockedAnd16((atomic_short *)obj, arg) + #define atomic_fetch_and_explicit32(obj, arg, order) \ (order == memory_order_relaxed \ ? InterlockedAndNoFence((atomic_int_fast32_t *)obj, arg) \ @@ -285,15 +347,18 @@ atomic_and_abort(void) { ISC_UNREACHABLE(); } -#define atomic_fetch_and_explicit(obj, arg, order) \ - (sizeof(*(obj)) == 8 \ - ? atomic_fetch_and_explicit64(obj, arg, order) \ - : (sizeof(*(obj)) == 4 \ - ? atomic_fetch_and_explicit32(obj, arg, order) \ - : (sizeof(*(obj)) == 1 \ - ? atomic_fetch_and_explicit8(obj, arg, \ - order) \ - : atomic_and_abort()))) +#define atomic_fetch_and_explicit(obj, arg, order) \ + (sizeof(*(obj)) == 8 \ + ? atomic_fetch_and_explicit64(obj, arg, order) \ + : (sizeof(*(obj)) == 4 \ + ? atomic_fetch_and_explicit32(obj, arg, order) \ + : (sizeof(*(obj)) == 2 \ + ? atomic_fetch_and_explicit16(obj, arg, \ + order) \ + : (sizeof(*(obj)) == 1 \ + ? atomic_fetch_and_explicit8( \ + obj, arg, order) \ + : atomic_and_abort())))) #define atomic_fetch_and(obj, arg) \ atomic_fetch_and_explicit(obj, arg, memory_order_seq_cst) @@ -301,6 +366,9 @@ atomic_and_abort(void) { #define atomic_fetch_or_explicit8(obj, arg, order) \ InterlockedOr8((atomic_int_fast8_t *)obj, arg) +#define atomic_fetch_or_explicit16(obj, arg, order) \ + InterlockedOr16((atomic_short *)obj, arg) + #define atomic_fetch_or_explicit32(obj, arg, order) \ (order == memory_order_relaxed \ ? InterlockedOrNoFence((atomic_int_fast32_t *)obj, arg) \ @@ -340,15 +408,18 @@ atomic_or_abort(void) { ISC_UNREACHABLE(); } -#define atomic_fetch_or_explicit(obj, arg, order) \ - (sizeof(*(obj)) == 8 \ - ? atomic_fetch_or_explicit64(obj, arg, order) \ - : (sizeof(*(obj)) == 4 \ - ? atomic_fetch_or_explicit32(obj, arg, order) \ - : (sizeof(*(obj)) == 1 \ - ? atomic_fetch_or_explicit8(obj, arg, \ - order) \ - : atomic_or_abort()))) +#define atomic_fetch_or_explicit(obj, arg, order) \ + (sizeof(*(obj)) == 8 \ + ? atomic_fetch_or_explicit64(obj, arg, order) \ + : (sizeof(*(obj)) == 4 \ + ? atomic_fetch_or_explicit32(obj, arg, order) \ + : (sizeof(*(obj)) == 2 \ + ? atomic_fetch_or_explicit16(obj, arg, \ + order) \ + : (sizeof(*(obj)) == 1 \ + ? atomic_fetch_or_explicit8( \ + obj, arg, order) \ + : atomic_or_abort())))) #define atomic_fetch_or(obj, arg) \ atomic_fetch_or_explicit(obj, arg, memory_order_seq_cst) @@ -372,6 +443,25 @@ atomic_compare_exchange_strong_explicit8(atomic_int_fast8_t *obj, return (__r); } +static inline bool +atomic_compare_exchange_strong_explicit16(atomic_short *obj, short *expected, + short desired, memory_order succ, + memory_order fail) { + bool __r; + short __v; + + UNUSED(succ); + UNUSED(fail); + + __v = InterlockedCompareExchange16((atomic_short *)obj, desired, + *expected); + __r = (*(expected) == __v); + if (!__r) { + *(expected) = __v; + } + return (__r); +} + static inline bool atomic_compare_exchange_strong_explicit32(atomic_int_fast32_t *obj, int32_t *expected, int32_t desired, @@ -455,19 +545,24 @@ atomic_compare_exchange_abort(void) { ISC_UNREACHABLE(); } -#define atomic_compare_exchange_strong_explicit(obj, expected, desired, succ, \ - fail) \ - (sizeof(*(obj)) == 8 \ - ? atomic_compare_exchange_strong_explicit64( \ - obj, expected, desired, succ, fail) \ - : (sizeof(*(obj)) == 4 \ - ? atomic_compare_exchange_strong_explicit32( \ - obj, expected, desired, succ, fail) \ - : (sizeof(*(obj)) == 1 \ - ? atomic_compare_exchange_strong_explicit8( \ - obj, expected, desired, succ, \ - fail) \ - : atomic_compare_exchange_abort()))) +#define atomic_compare_exchange_strong_explicit(obj, expected, desired, succ, \ + fail) \ + (sizeof(*(obj)) == 8 \ + ? atomic_compare_exchange_strong_explicit64( \ + obj, expected, desired, succ, fail) \ + : (sizeof(*(obj)) == 4 \ + ? atomic_compare_exchange_strong_explicit32( \ + obj, expected, desired, succ, fail) \ + : (sizeof(*(obj)) == 2 \ + ? atomic_compare_exchange_strong_explicit16( \ + obj, expected, desired, succ, \ + fail) \ + : (sizeof(*(obj)) == 1 \ + ? atomic_compare_exchange_strong_explicit8( \ + obj, expected, \ + desired, succ, \ + fail) \ + : atomic_compare_exchange_abort())))) #define atomic_compare_exchange_strong(obj, expected, desired) \ atomic_compare_exchange_strong_explicit(obj, expected, desired, \ @@ -490,14 +585,17 @@ atomic_exchange_abort(void) { ISC_UNREACHABLE(); } -#define atomic_exchange_explicit(obj, desired, order) \ - (sizeof(*(obj)) == 8 \ - ? InterlockedExchange64(obj, desired) \ - : (sizeof(*(obj)) == 4 \ - ? InterlockedExchange(obj, desired) \ - : (sizeof(*(obj)) == 1 \ - ? InterlockedExchange8(obj, desired) \ - : atomic_exchange_abort()))) +#define atomic_exchange_explicit(obj, desired, order) \ + (sizeof(*(obj)) == 8 \ + ? InterlockedExchange64(obj, desired) \ + : (sizeof(*(obj)) == 4 \ + ? InterlockedExchange(obj, desired) \ + : (sizeof(*(obj)) == 2 \ + ? InterlockedExchange16(obj, desired) \ + : (sizeof(*(obj)) == 1 \ + ? InterlockedExchange8( \ + obj, desired) \ + : atomic_exchange_abort())))) #define atomic_exchange(obj, desired) \ atomic_exchange_explicit(obj, desired, memory_order_seq_cst)