diff --git a/lib/isc/include/isc/atomic.h b/lib/isc/include/isc/atomic.h index e75ada094b..1e0ad4b08d 100644 --- a/lib/isc/include/isc/atomic.h +++ b/lib/isc/include/isc/atomic.h @@ -35,6 +35,10 @@ atomic_fetch_add_explicit((o), (v), memory_order_relaxed) #define atomic_fetch_sub_relaxed(o, v) \ atomic_fetch_sub_explicit((o), (v), memory_order_relaxed) +#define atomic_fetch_or_relaxed(o, v) \ + atomic_fetch_or_explicit((o), (v), memory_order_relaxed) +#define atomic_fetch_and_relaxed(o, v) \ + atomic_fetch_and_explicit((o), (v), memory_order_relaxed) #define atomic_exchange_relaxed(o, v) \ atomic_exchange_explicit((o), (v), memory_order_relaxed) #define atomic_compare_exchange_weak_relaxed(o, e, d) \ diff --git a/lib/isc/unix/include/isc/stdatomic.h b/lib/isc/unix/include/isc/stdatomic.h index 96eb7f0a15..d5fbdf9409 100644 --- a/lib/isc/unix/include/isc/stdatomic.h +++ b/lib/isc/unix/include/isc/stdatomic.h @@ -108,6 +108,10 @@ typedef bool atomic_bool; __atomic_fetch_add(obj, arg, order) #define atomic_fetch_sub_explicit(obj, arg, order) \ __atomic_fetch_sub(obj, arg, order) +#define atomic_fetch_and_explicit(obj, arg, order) \ + __atomic_fetch_and(obj, arg, order) +#define atomic_fetch_or_explicit(obj, arg, order) \ + __atomic_fetch_or(obj, arg, order) #define atomic_compare_exchange_strong_explicit(obj, expected, desired, succ, fail) \ __atomic_compare_exchange_n(obj, expected, desired, 0, succ, fail) #define atomic_compare_exchange_weak_explicit(obj, expected, desired, succ, fail) \ @@ -123,10 +127,14 @@ typedef bool atomic_bool; *obj = desired; \ __sync_synchronize(); \ } while (0); -#define atomic_fetch_add_explicit(obj, arg, order) \ +#define atomic_fetch_add_explicit(obj, arg, order) \ __sync_fetch_and_add(obj, arg) -#define atomic_fetch_sub_explicit(obj, arg, order) \ +#define atomic_fetch_sub_explicit(obj, arg, order) \ __sync_fetch_and_sub(obj, arg, order) +#define atomic_fetch_and_explicit(obj, arg, order) \ + __sync_fetch_and_and(obj, arg, order) +#define atomic_fetch_or_explicit(obj, arg, order) \ + __sync_fetch_and_or(obj, arg, order) #define atomic_compare_exchange_strong_explicit(obj, expected, desired, succ, fail) \ ({ \ __typeof__(obj) __v; \ @@ -150,6 +158,10 @@ typedef bool atomic_bool; atomic_fetch_add_explicit(obj, arg, memory_order_seq_cst) #define atomic_fetch_sub(obj, arg) \ atomic_fetch_sub_explicit(obj, arg, memory_order_seq_cst) +#define atomic_fetch_and(obj, arg) \ + atomic_fetch_and_explicit(obj, arg, memory_order_seq_cst) +#define atomic_fetch_or(obj, arg) \ + atomic_fetch_or_explicit(obj, arg, memory_order_seq_cst) #define atomic_compare_exchange_strong(obj, expected, desired) \ atomic_compare_exchange_strong_explicit(obj, expected, desired, memory_order_seq_cst, memory_order_seq_cst) #define atomic_compare_exchange_weak(obj, expected, desired) \ diff --git a/lib/isc/win32/include/isc/stdatomic.h b/lib/isc/win32/include/isc/stdatomic.h index 97375da615..a5fe6a208a 100644 --- a/lib/isc/win32/include/isc/stdatomic.h +++ b/lib/isc/win32/include/isc/stdatomic.h @@ -189,7 +189,7 @@ atomic_load_abort() { : InterlockedExchangeAdd64((atomic_int_fast64_t *)obj, arg)))) #else #define atomic_fetch_add_explicit64(obj, arg, order) \ - InterlockedExchange64((atomic_int_fast64_t *)obj, arg) + InterlockedExchangeAdd64((atomic_int_fast64_t *)obj, arg) #endif static inline @@ -217,6 +217,96 @@ atomic_add_abort() { #define atomic_fetch_sub(obj, arg) \ atomic_fetch_sub_explicit(obj, arg, memory_order_seq_cst) +#define atomic_fetch_and_explicit8(obj, arg, order) \ + InterlockedAnd8((atomic_int_fast8_t)obj, arg) + +#define atomic_fetch_and_explicit32(obj, arg, order) \ + (order == memory_order_relaxed \ + ? InterlockedAndNoFence((atomic_int_fast32_t *)obj, arg) \ + : (order == memory_order_acquire \ + ? InterlockedAndAcquire((atomic_int_fast32_t *)obj, arg) \ + : (order == memory_order_release \ + ? InterlockedAndRelease((atomic_int_fast32_t *)obj, arg) \ + : InterlockedAnd((atomic_int_fast32_t *)obj, arg)))) + +#ifdef _WIN64 +#define atomic_fetch_and_explicit64(obj, arg, order) \ + (order == memory_order_relaxed \ + ? InterlockedAnd64NoFence((atomic_int_fast64_t *)obj, arg) \ + : (order == memory_order_acquire \ + ? InterlockedAnd64Acquire((atomic_int_fast64_t *)obj, arg) \ + : (order == memory_order_release \ + ? InterlockedAnd64Release((atomic_int_fast64_t *)obj, arg) \ + : InterlockedAnd64((atomic_int_fast64_t *)obj, arg)))) +#else +#define atomic_fetch_and_explicit64(obj, arg, order) \ + InterlockedAnd64((atomic_int_fast64_t *)obj, arg) +#endif + +static inline +int8_t +atomic_and_abort() { + INSIST(0); + ISC_UNREACHABLE(); +} + +#define atomic_fetch_and_explicit(obj, arg, order) \ + (sizeof(*(obj)) == 8 \ + ? atomic_fetch_and_explicit64(obj, arg, order) \ + : (sizeof(*(obj)) == 4 \ + ? atomic_fetch_and_explicit32(obj, arg, order) \ + : (sizeof(*(obj)) == 1 \ + ? atomic_fetch_and_explicit8(obj, arg, order) \ + : atomic_and_abort()))) + +#define atomic_fetch_and(obj, arg) \ + atomic_fetch_and_explicit(obj, arg, memory_order_seq_cst) + +#define atomic_fetch_or_explicit8(obj, arg, order) \ + InterlockedOr8((atomic_int_fast8_t)obj, arg) + +#define atomic_fetch_or_explicit32(obj, arg, order) \ + (order == memory_order_relaxed \ + ? InterlockedOrNoFence((atomic_int_fast32_t *)obj, arg) \ + : (order == memory_order_acquire \ + ? InterlockedOrAcquire((atomic_int_fast32_t *)obj, arg) \ + : (order == memory_order_release \ + ? InterlockedOrRelease((atomic_int_fast32_t *)obj, arg) \ + : InterlockedOr((atomic_int_fast32_t *)obj, arg)))) + +#ifdef _WIN64 +#define atomic_fetch_or_explicit64(obj, arg, order) \ + (order == memory_order_relaxed \ + ? InterlockedOr64NoFence((atomic_int_fast64_t *)obj, arg) \ + : (order == memory_order_acquire \ + ? InterlockedOr64Acquire((atomic_int_fast64_t *)obj, arg) \ + : (order == memory_order_release \ + ? InterlockedOr64Release((atomic_int_fast64_t *)obj, arg) \ + : InterlockedOr64((atomic_int_fast64_t *)obj, arg)))) +#else +#define atomic_fetch_or_explicit64(obj, arg, order) \ + InterlockedOr64((atomic_int_fast64_t *)obj, arg) +#endif + +static inline +int8_t +atomic_or_abort() { + INSIST(0); + ISC_UNREACHABLE(); +} + +#define atomic_fetch_or_explicit(obj, arg, order) \ + (sizeof(*(obj)) == 8 \ + ? atomic_fetch_or_explicit64(obj, arg, order) \ + : (sizeof(*(obj)) == 4 \ + ? atomic_fetch_or_explicit32(obj, arg, order) \ + : (sizeof(*(obj)) == 1 \ + ? atomic_fetch_or_explicit8(obj, arg, order) \ + : atomic_or_abort()))) + +#define atomic_fetch_or(obj, arg) \ + atomic_fetch_or_explicit(obj, arg, memory_order_seq_cst) + static inline bool atomic_compare_exchange_strong_explicit8(atomic_int_fast8_t *obj, int8_t *expected,