From 7d5928c3a28fab5c0b67fbf711de20bdc6d56890 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ayd=C4=B1n=20Mercan?= Date: Mon, 18 Aug 2025 14:07:38 +0300 Subject: [PATCH] add isc/bit.h and unify common bit operations The `` header is a GNU C11 compatible version of C23's ``. It currently uses either `` or the equivilent compiler builtins. However, the generic `__builtin_ctzg` and `__builtin_ctlz` builtins are not available in every compiler version and thus falls back to manually selecting from type. Furthermore, the ctz fallback has been removed since `__builtin_ctzll` has been used for a while directly without any compilation issues from users. Thus, we can also require `__builtin_ctz`. Unlike the rest of C23's bit utilities, we avoid the stdc_rotate_* functions since we don't need the rotation modulus precision. This adds a couple (admittedly cheap) unwanted instructions on some architectures. --- lib/dns/qp.c | 9 +-- lib/dns/qp_p.h | 3 +- lib/dns/rpz.c | 46 +------------ lib/isc/histo.c | 3 +- lib/isc/include/isc/bit.h | 123 ++++++++++++++++++++++++++++++++++ lib/isc/include/isc/fxhash.h | 9 +-- lib/isc/include/isc/siphash.h | 29 ++++---- meson.build | 40 ++++++++++- util/checklibs.sh | 11 +++ 9 files changed, 200 insertions(+), 73 deletions(-) create mode 100644 lib/isc/include/isc/bit.h diff --git a/lib/dns/qp.c b/lib/dns/qp.c index d5b22c0646..3f10bd04a4 100644 --- a/lib/dns/qp.c +++ b/lib/dns/qp.c @@ -27,6 +27,7 @@ #endif #include +#include #include #include #include @@ -493,11 +494,11 @@ cells_immutable(dns_qp_t *qp, dns_qpref_t ref) { static dns_qpcell_t next_capacity(uint32_t prev_capacity, uint32_t size) { /* - * Unfortunately builtin_clz is undefined for 0. We work around this - * issue by flooring the request size at 2. + * Request size was floored at 2 because builtin_clz used to be 0. + * We keep this behavior because ISC_LEADING_ZEROS(0) = 32. */ - size = ISC_MAX3(size, prev_capacity, 2u); - uint32_t log2 = 32u - __builtin_clz(size - 1u); + size = ISC_MAX3(size, prev_capacity, 2U); + uint32_t log2 = 32U - ISC_LEADING_ZEROS(size - 1U); return 1U << ISC_CLAMP(log2, QP_CHUNK_LOG_MIN, QP_CHUNK_LOG_MAX); } diff --git a/lib/dns/qp_p.h b/lib/dns/qp_p.h index efe5c5d9b0..a5dce44d84 100644 --- a/lib/dns/qp_p.h +++ b/lib/dns/qp_p.h @@ -19,6 +19,7 @@ #pragma once +#include #include #include @@ -752,7 +753,7 @@ static inline dns_qpweight_t branch_count_bitmap_before(dns_qpnode_t *n, dns_qpshift_t bit) { uint64_t mask = (1ULL << bit) - 1 - TAG_MASK; uint64_t bitmap = branch_index(n) & mask; - return (dns_qpweight_t)__builtin_popcountll(bitmap); + return (dns_qpweight_t)ISC_POPCOUNT(bitmap); } /* diff --git a/lib/dns/rpz.c b/lib/dns/rpz.c index 88ca72de29..3333478e20 100644 --- a/lib/dns/rpz.c +++ b/lib/dns/rpz.c @@ -19,6 +19,7 @@ #include #include +#include #include #include #include @@ -1074,45 +1075,6 @@ name2data(dns_rpz_zone_t *rpz, dns_rpz_type_t rpz_type, (void)dns_name_concatenate(&tmp_name, dns_rootname, trig_name); } -#ifndef HAVE_BUILTIN_CLZ -/** - * \brief Count Leading Zeros: Find the location of the left-most set - * bit. - */ -static unsigned int -clz(dns_rpz_cidr_word_t w) { - unsigned int bit; - - bit = DNS_RPZ_CIDR_WORD_BITS - 1; - - if ((w & 0xffff0000) != 0) { - w >>= 16; - bit -= 16; - } - - if ((w & 0xff00) != 0) { - w >>= 8; - bit -= 8; - } - - if ((w & 0xf0) != 0) { - w >>= 4; - bit -= 4; - } - - if ((w & 0xc) != 0) { - w >>= 2; - bit -= 2; - } - - if ((w & 2) != 0) { - --bit; - } - - return bit; -} -#endif /* ifndef HAVE_BUILTIN_CLZ */ - /* * Find the first differing bit in two keys (IP addresses). */ @@ -1132,11 +1094,7 @@ diff_keys(const dns_rpz_cidr_key_t *key1, dns_rpz_prefix_t prefix1, for (i = 0; bit < maxbit; i++, bit += DNS_RPZ_CIDR_WORD_BITS) { delta = key1->w[i] ^ key2->w[i]; if (delta != 0) { -#ifdef HAVE_BUILTIN_CLZ - bit += __builtin_clz(delta); -#else /* ifdef HAVE_BUILTIN_CLZ */ - bit += clz(delta); -#endif /* ifdef HAVE_BUILTIN_CLZ */ + bit += ISC_LEADING_ZEROS(delta); break; } } diff --git a/lib/isc/histo.c b/lib/isc/histo.c index 4fa9d83396..25febb55c9 100644 --- a/lib/isc/histo.c +++ b/lib/isc/histo.c @@ -21,6 +21,7 @@ #include #include +#include #include #include #include @@ -180,7 +181,7 @@ static inline uint value_to_key(const isc_histo_t *hg, uint64_t value) { /* ensure that denormal numbers are all in chunk zero */ uint64_t chunked = value | CHUNKSIZE(hg); - int clz = __builtin_clzll((unsigned long long)(chunked)); + int clz = ISC_LEADING_ZEROS(chunked); /* actually 1 less than the exponent except for denormals */ uint exponent = 63 - hg->sigbits - clz; /* mantissa has leading bit set except for denormals */ diff --git a/lib/isc/include/isc/bit.h b/lib/isc/include/isc/bit.h new file mode 100644 index 0000000000..78a72df3e9 --- /dev/null +++ b/lib/isc/include/isc/bit.h @@ -0,0 +1,123 @@ +/* + * Copyright (C) Internet Systems Consortium, Inc. ("ISC") + * + * SPDX-License-Identifier: MPL-2.0 + * + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, you can obtain one at https://mozilla.org/MPL/2.0/. + * + * See the COPYRIGHT file distributed with this work for additional + * information regarding copyright ownership. + */ + +#pragma once + +#include +#include + +#include +#include + +#ifndef __has_header +#define __has_header(x) 0 +#endif + +#if __has_header() + +#include + +#define ISC_POPCOUNT(x) stdc_count_zeros(x) +#define ISC_LEADING_ZEROS(x) stdc_leading_zeros(x) +#define ISC_TRAILING_ZEROS(x) stdc_trailing_zeros(x) +#define ISC_LEADING_ONES(x) stdc_leading_ones(x) +#define ISC_TRAILING_ONES(x) stdc_trailing_ones(x) + +#else /* __has_header() */ + +#ifdef HAVE_BUILTIN_POPCOUNTG +#define ISC_POPCOUNT(x) __builtin_popcountg(x) +#else /* HAVE_BUILTIN_POPCOUNTG */ +#define ISC_POPCOUNT(x) \ + _Generic((x), \ + unsigned int: __builtin_popcount, \ + unsigned long: __builtin_popcountl, \ + unsigned long long: __builtin_popcountll)(x) +#endif /* HAVE_BUILTIN_POPCOUNTG */ + +#ifdef HAVE_BUILTIN_CLZG +#define ISC_LEADING_ZEROS(x) __builtin_clzg(x, (int)(sizeof(x) * 8)) +#else /* HAVE_BUILTIN_CLZG */ +#define ISC_LEADING_ZEROS(x) \ + ((x) == 0) ? (sizeof(x) * 8) \ + : _Generic((x), \ + unsigned int: __builtin_clz, \ + unsigned long: __builtin_clzl, \ + unsigned long long: __builtin_clzll)(x) +#endif /* HAVE_BUILTIN_CLZG */ + +#ifdef HAVE_BUILTIN_CTZG +#define ISC_TRAILING_ZEROS(x) __builtin_ctzg(x, (int)sizeof(x) * 8) +#else /* HAVE_BUILTIN_CTZG */ +#define ISC_TRAILING_ZEROS(x) \ + ((x) == 0) ? (sizeof(x) * 8) \ + : _Generic((x), \ + unsigned int: __builtin_ctz, \ + unsigned long: __builtin_ctzl, \ + unsigned long long: __builtin_ctzll)(x) +#endif /* HAVE_BUILTIN_CTZG */ + +#define ISC_LEADING_ONES(x) ISC_LEADING_ZEROS(~(x)) +#define ISC_TRAILING_ONES(x) ISC_TRAILING_ZEROS(~(x)) + +#endif /* __has_header() */ + +#if SIZE_MAX == UINT64_MAX +#define isc_rotate_leftsize(x, n) isc_rotate_left64(x, n) +#define isc_rotate_rightsize(x, n) isc_rotate_right64(x, n) +#elif SIZE_MAX == UINT32_MAX +#define isc_rotate_leftsize(x, n) isc_rotate_left32(x, n) +#define isc_rotate_rightsize(x, n) isc_rotate_right32(x, n) +#else +#error "size_t must be either 32 or 64-bits" +#endif + +static inline uint8_t __attribute__((always_inline)) +isc_rotate_left8(const uint8_t x, uint32_t n) { + return (x << n) | (x >> (8 - n)); +} + +static inline uint16_t __attribute__((always_inline)) +isc_rotate_left16(const uint16_t x, uint32_t n) { + return (x << n) | (x >> (16 - n)); +} + +static inline uint32_t __attribute__((always_inline)) +isc_rotate_left32(const uint32_t x, uint32_t n) { + return (x << n) | (x >> (32 - n)); +} + +static inline uint64_t __attribute__((always_inline)) +isc_rotate_left64(const uint64_t x, uint32_t n) { + return (x << n) | (x >> (64 - n)); +} + +static inline uint8_t __attribute__((always_inline)) +isc_rotate_right8(const uint8_t x, uint32_t n) { + return (x >> n) | (x << (8 - n)); +} + +static inline uint16_t __attribute__((always_inline)) +isc_rotate_right16(const uint16_t x, uint32_t n) { + return (x >> n) | (x << (16 - n)); +} + +static inline uint32_t __attribute__((always_inline)) +isc_rotate_right32(const uint32_t x, uint32_t n) { + return (x >> n) | (x << (32 - n)); +} + +static inline uint64_t __attribute__((always_inline)) +isc_rotate_right64(const uint64_t x, uint32_t n) { + return (x >> n) | (x << (64 - n)); +} diff --git a/lib/isc/include/isc/fxhash.h b/lib/isc/include/isc/fxhash.h index b76cce4305..0ee1cb0b03 100644 --- a/lib/isc/include/isc/fxhash.h +++ b/lib/isc/include/isc/fxhash.h @@ -29,17 +29,14 @@ #include #include +#include + /* The constant K from Rust's fxhash */ #define K 0x9e3779b97f4a7c15ull -static inline size_t -rotate_left(size_t x, unsigned int n) { - return (x << n) | (x >> (sizeof(size_t) * 8 - n)); -} - static inline size_t fx_add_to_hash(size_t hash, size_t i) { - return rotate_left(hash, 5) ^ i * K; + return isc_rotate_leftsize(hash, 5) ^ i * K; } /* diff --git a/lib/isc/include/isc/siphash.h b/lib/isc/include/isc/siphash.h index c066a85f16..81cfaf30c3 100644 --- a/lib/isc/include/isc/siphash.h +++ b/lib/isc/include/isc/siphash.h @@ -30,6 +30,7 @@ #pragma once #include +#include #include #include #include @@ -43,14 +44,12 @@ #define cROUNDS 2 #define dROUNDS 4 -#define ROTATE64(x, b) (uint64_t)(((x) << (b)) | ((x) >> (64 - (b)))) - -#define HALF_ROUND64(a, b, c, d, s, t) \ - a += b; \ - c += d; \ - b = ROTATE64(b, s) ^ a; \ - d = ROTATE64(d, t) ^ c; \ - a = ROTATE64(a, 32); +#define HALF_ROUND64(a, b, c, d, s, t) \ + a += b; \ + c += d; \ + b = isc_rotate_left64(b, s) ^ a; \ + d = isc_rotate_left64(d, t) ^ c; \ + a = isc_rotate_left64(a, 32); #define FULL_ROUND64(v0, v1, v2, v3) \ HALF_ROUND64(v0, v1, v2, v3, 13, 16); \ @@ -58,14 +57,12 @@ #define SIPROUND FULL_ROUND64 -#define ROTATE32(x, b) (uint32_t)(((x) << (b)) | ((x) >> (32 - (b)))) - -#define HALF_ROUND32(a, b, c, d, s, t) \ - a += b; \ - c += d; \ - b = ROTATE32(b, s) ^ a; \ - d = ROTATE32(d, t) ^ c; \ - a = ROTATE32(a, 16); +#define HALF_ROUND32(a, b, c, d, s, t) \ + a += b; \ + c += d; \ + b = isc_rotate_left32(b, s) ^ a; \ + d = isc_rotate_left32(d, t) ^ c; \ + a = isc_rotate_left32(a, 16); #define FULL_ROUND32(v0, v1, v2, v3) \ HALF_ROUND32(v0, v1, v2, v3, 5, 8); \ diff --git a/meson.build b/meson.build index 6f45d81f63..b4a4ca75d7 100644 --- a/meson.build +++ b/meson.build @@ -312,7 +312,6 @@ endif foreach fn : [ '__builtin_add_overflow', - '__builtin_clz', '__builtin_mul_overflow', '__builtin_sub_overflow', '__builtin_unreachable', @@ -322,6 +321,45 @@ foreach fn : [ endif endforeach +# meson_version (>=1.3.0) : required in cc.has_function +if cc.has_function('__builtin_clzg') + config.set('HAVE_BUILTIN_CLZG', true) +elif not ( + cc.has_function('__builtin_clz') + and cc.has_function('__builtin_clzl') + and cc.has_function('__builtin_clzll') +) + error( + '__builtin_clzg or __builtin_clz* functions are required, please fix your toolchain', + ) +endif + +# meson_version (>=1.3.0) : required in cc.has_function +if cc.has_function('__builtin_ctzg') + config.set('HAVE_BUILTIN_CTZG', true) +elif not ( + cc.has_function('__builtin_ctz') + and cc.has_function('__builtin_ctzl') + and cc.has_function('__builtin_ctzll') +) + error( + '__builtin_ctzg or __builtin_ctz* functions are required, please fix your toolchain', + ) +endif + +# meson_version (>=1.3.0) : required in cc.has_function +if cc.has_function('__builtin_popcountg') + config.set('HAVE_BUILTIN_POPCOUNTG', true) +elif not ( + cc.has_function('__builtin_popcount') + and cc.has_function('__builtin_popcountl') + and cc.has_function('__builtin_popcountll') +) + error( + '__builtin_popcountg or __builtin_popcount* functions are required, please fix your toolchain', + ) +endif + foreach attr : ['malloc', 'returns_nonnull'] if cc.has_function_attribute(attr) config.set('HAVE_FUNC_ATTRIBUTE_@0@'.format(attr.to_upper()), 1) diff --git a/util/checklibs.sh b/util/checklibs.sh index e0f8c79d64..b47cb52fe1 100755 --- a/util/checklibs.sh +++ b/util/checklibs.sh @@ -64,4 +64,15 @@ list=$(git grep -l memory_order_.* lib bin ':(exclude)lib/isc/include/isc/atomic echo "$list" } +# +# Check for the usage of built-in bit operarators +# +list=$(git grep -l -e '__builtin_ctz' --or -e '__builtin_popcount' --or -e '__builtin_clz' lib bin ':(exclude)lib/isc/include/isc/bit\.h' \ + | grep -e '\.c$' -e '\.h$') +[ -n "$list" ] && { + status=1 + echo 'Prefer the helpers in over builtin bit utilities:' + echo "$list" +} + exit $status