Merge branch 'ondrej-implement-case-insensitive-siphash-2-4' into 'main'

Add a case-insensitive option directly to siphash 2-4 implementation

See merge request isc-projects/bind9!6789

lib/dns/resolver.c

@@ -2458,7 +2458,8 @@ compute_cc(const resquery_t *query, uint8_t *cookie, const size_t len) {
 	size_t buflen = add_serveraddr(buf, sizeof(buf), query);
 
 	uint8_t digest[ISC_SIPHASH24_TAG_LENGTH] ISC_NONSTRING = { 0 };
-	isc_siphash24(query->fctx->res->view->secret, buf, buflen, digest);
+	isc_siphash24(query->fctx->res->view->secret, buf, buflen, true,
+		      digest);
 	memmove(cookie, digest, CLIENT_COOKIE_SIZE);
 }
 

lib/isc/hash.c

@@ -86,14 +86,8 @@ isc_hash64(const void *data, const size_t length, const bool case_sensitive) {
 	RUNTIME_CHECK(isc_once_do(&isc_hash_once, isc_hash_initialize) ==
 		      ISC_R_SUCCESS);
 
-	if (case_sensitive) {
-		isc_siphash24(isc_hash_key, data, length, (uint8_t *)&hval);
-	} else {
-		uint8_t lower[1024];
-		REQUIRE(length <= sizeof(lower));
-		isc_ascii_lowercopy(lower, data, length);
-		isc_siphash24(isc_hash_key, lower, length, (uint8_t *)&hval);
-	}
+	isc_siphash24(isc_hash_key, data, length, case_sensitive,
+		      (uint8_t *)&hval);
 
 	return (hval);
 }
@@ -107,15 +101,8 @@ isc_hash32(const void *data, const size_t length, const bool case_sensitive) {
 	RUNTIME_CHECK(isc_once_do(&isc_hash_once, isc_hash_initialize) ==
 		      ISC_R_SUCCESS);
 
-	if (case_sensitive) {
-		isc_halfsiphash24(isc_hash_key, data, length, (uint8_t *)&hval);
-	} else {
-		uint8_t lower[1024];
-		REQUIRE(length <= sizeof(lower));
-		isc_ascii_lowercopy(lower, data, length);
-		isc_halfsiphash24(isc_hash_key, lower, length,
-				  (uint8_t *)&hval);
-	}
+	isc_halfsiphash24(isc_hash_key, data, length, case_sensitive,
+			  (uint8_t *)&hval);
 
 	return (hval);
 }

lib/isc/include/isc/ascii.h

@@ -68,7 +68,7 @@ isc_ascii_strtolower(char *str) {
  * memove() below) the balance might be different.
  */
 static inline uint64_t
-isc__ascii_tolower8(uint64_t octets) {
+isc_ascii_tolower8(uint64_t octets) {
 	/*
 	 * Multiply a single-byte constant by `all_bytes` to replicate
 	 * it to all eight bytes in a word.
@@ -103,6 +103,20 @@ isc__ascii_tolower8(uint64_t octets) {
 	return (octets | (is_upper >> 2));
 }
 
+/*
+ * Same, but 4 bytes at a time, used by isc_halfsiphash24()
+ */
+static inline uint32_t
+isc_ascii_tolower4(uint32_t octets) {
+	uint32_t all_bytes = 0x01010101;
+	uint32_t heptets = octets & (0x7F * all_bytes);
+	uint32_t is_ascii = ~octets & (0x80 * all_bytes);
+	uint32_t is_gt_Z = heptets + (0x7F - 'Z') * all_bytes;
+	uint32_t is_ge_A = heptets + (0x80 - 'A') * all_bytes;
+	uint32_t is_upper = (is_ge_A ^ is_gt_Z) & is_ascii;
+	return (octets | (is_upper >> 2));
+}
+
 /*
  * Helper function to do an unaligned load of 8 bytes in host byte order
  */
@@ -120,8 +134,8 @@ static inline bool
 isc_ascii_lowerequal(const uint8_t *a, const uint8_t *b, unsigned int len) {
 	uint64_t a8 = 0, b8 = 0;
 	while (len >= 8) {
-		a8 = isc__ascii_tolower8(isc__ascii_load8(a));
-		b8 = isc__ascii_tolower8(isc__ascii_load8(b));
+		a8 = isc_ascii_tolower8(isc__ascii_load8(a));
+		b8 = isc_ascii_tolower8(isc__ascii_load8(b));
 		if (a8 != b8) {
 			return (false);
 		}
@@ -147,8 +161,8 @@ static inline int
 isc_ascii_lowercmp(const uint8_t *a, const uint8_t *b, unsigned int len) {
 	uint64_t a8 = 0, b8 = 0;
 	while (len >= 8) {
-		a8 = isc__ascii_tolower8(htobe64(isc__ascii_load8(a)));
-		b8 = isc__ascii_tolower8(htobe64(isc__ascii_load8(b)));
+		a8 = isc_ascii_tolower8(htobe64(isc__ascii_load8(a)));
+		b8 = isc_ascii_tolower8(htobe64(isc__ascii_load8(b)));
 		if (a8 != b8) {
 			goto ret;
 		}

lib/isc/include/isc/siphash.h

@@ -28,9 +28,9 @@ ISC_LANG_BEGINDECLS
 
 void
 isc_siphash24(const uint8_t *key, const uint8_t *in, const size_t inlen,
-	      uint8_t *out);
+	      bool case_sensitive, uint8_t *out);
 void
 isc_halfsiphash24(const uint8_t *key, const uint8_t *in, const size_t inlen,
-		  uint8_t *out);
+		  bool case_sensitive, uint8_t *out);
 
 ISC_LANG_ENDDECLS

lib/isc/siphash.c

@@ -15,6 +15,7 @@
 #include <string.h>
 #include <unistd.h>
 
+#include <isc/ascii.h>
 #include <isc/endian.h>
 #include <isc/siphash.h>
 #include <isc/util.h>
@@ -76,6 +77,9 @@
 	(((uint32_t)((p)[0])) | ((uint32_t)((p)[1]) << 8) | \
 	 ((uint32_t)((p)[2]) << 16) | ((uint32_t)((p)[3]) << 24))
 
+#define U8TO32_ONE(case_sensitive, byte) \
+	(uint32_t)(case_sensitive ? byte : isc__ascii_tolower1(byte))
+
 #define U64TO8_LE(p, v)                  \
 	U32TO8_LE((p), (uint32_t)((v))); \
 	U32TO8_LE((p) + 4, (uint32_t)((v) >> 32));
@@ -86,9 +90,12 @@
 	 ((uint64_t)((p)[4]) << 32) | ((uint64_t)((p)[5]) << 40) | \
 	 ((uint64_t)((p)[6]) << 48) | ((uint64_t)((p)[7]) << 56))
 
+#define U8TO64_ONE(case_sensitive, byte) \
+	(uint64_t)(case_sensitive ? byte : isc__ascii_tolower1(byte))
+
 void
 isc_siphash24(const uint8_t *k, const uint8_t *in, const size_t inlen,
-	      uint8_t *out) {
+	      bool case_sensitive, uint8_t *out) {
 	REQUIRE(k != NULL);
 	REQUIRE(out != NULL);
 
@@ -106,7 +113,8 @@ isc_siphash24(const uint8_t *k, const uint8_t *in, const size_t inlen,
 	const size_t left = inlen & 7;
 
 	for (; in != end; in += 8) {
-		uint64_t m = U8TO64_LE(in);
+		uint64_t m = case_sensitive ? U8TO64_LE(in)
+					    : isc_ascii_tolower8(U8TO64_LE(in));
 
 		v3 ^= m;
 
@@ -119,25 +127,25 @@ isc_siphash24(const uint8_t *k, const uint8_t *in, const size_t inlen,
 
 	switch (left) {
 	case 7:
-		b |= ((uint64_t)in[6]) << 48;
+		b |= U8TO64_ONE(case_sensitive, in[6]) << 48;
 		FALLTHROUGH;
 	case 6:
-		b |= ((uint64_t)in[5]) << 40;
+		b |= U8TO64_ONE(case_sensitive, in[5]) << 40;
 		FALLTHROUGH;
 	case 5:
-		b |= ((uint64_t)in[4]) << 32;
+		b |= U8TO64_ONE(case_sensitive, in[4]) << 32;
 		FALLTHROUGH;
 	case 4:
-		b |= ((uint64_t)in[3]) << 24;
+		b |= U8TO64_ONE(case_sensitive, in[3]) << 24;
 		FALLTHROUGH;
 	case 3:
-		b |= ((uint64_t)in[2]) << 16;
+		b |= U8TO64_ONE(case_sensitive, in[2]) << 16;
 		FALLTHROUGH;
 	case 2:
-		b |= ((uint64_t)in[1]) << 8;
+		b |= U8TO64_ONE(case_sensitive, in[1]) << 8;
 		FALLTHROUGH;
 	case 1:
-		b |= ((uint64_t)in[0]);
+		b |= U8TO64_ONE(case_sensitive, in[0]);
 		FALLTHROUGH;
 	case 0:
 		break;
@@ -166,7 +174,7 @@ isc_siphash24(const uint8_t *k, const uint8_t *in, const size_t inlen,
 
 void
 isc_halfsiphash24(const uint8_t *k, const uint8_t *in, const size_t inlen,
-		  uint8_t *out) {
+		  bool case_sensitive, uint8_t *out) {
 	REQUIRE(k != NULL);
 	REQUIRE(out != NULL);
 
@@ -184,7 +192,9 @@ isc_halfsiphash24(const uint8_t *k, const uint8_t *in, const size_t inlen,
 	const int left = inlen & 3;
 
 	for (; in != end; in += 4) {
-		uint32_t m = U8TO32_LE(in);
+		uint32_t m = case_sensitive ? U8TO32_LE(in)
+					    : isc_ascii_tolower4(U8TO32_LE(in));
+
 		v3 ^= m;
 
 		for (size_t i = 0; i < cROUNDS; ++i) {
@@ -196,13 +206,13 @@ isc_halfsiphash24(const uint8_t *k, const uint8_t *in, const size_t inlen,
 
 	switch (left) {
 	case 3:
-		b |= ((uint32_t)in[2]) << 16;
+		b |= U8TO32_ONE(case_sensitive, in[2]) << 16;
 		FALLTHROUGH;
 	case 2:
-		b |= ((uint32_t)in[1]) << 8;
+		b |= U8TO32_ONE(case_sensitive, in[1]) << 8;
 		FALLTHROUGH;
 	case 1:
-		b |= ((uint32_t)in[0]);
+		b |= U8TO32_ONE(case_sensitive, in[0]);
 		FALLTHROUGH;
 	case 0:
 		break;

lib/ns/client.c

@@ -1196,7 +1196,7 @@ compute_cookie(ns_client_t *client, uint32_t when, uint32_t nonce,
 			UNREACHABLE();
 		}
 
-		isc_siphash24(secret, input, inputlen, digest);
+		isc_siphash24(secret, input, inputlen, true, digest);
 		isc_buffer_putmem(buf, digest, 8);
 		break;
 	}

tests/bench/.gitignore

@@ -1 +1,2 @@
 ascii
+siphash

tests/bench/Makefile.am

@@ -7,4 +7,5 @@ LDADD +=			\
 	$(LIBISC_LIBS)
 
 noinst_PROGRAMS =		\
-	ascii
+	ascii			\
+	siphash

tests/bench/siphash.c

@@ -0,0 +1,184 @@
+/*
+ * Copyright (C) Internet Systems Consortium, Inc. ("ISC")
+ *
+ * SPDX-License-Identifier: MPL-2.0
+ *
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, you can obtain one at https://mozilla.org/MPL/2.0/.
+ *
+ * See the COPYRIGHT file distributed with this work for additional
+ * information regarding copyright ownership.
+ */
+
+#include <stdbool.h>
+#include <stdint.h>
+#include <stdio.h>
+#include <string.h>
+
+#include <isc/ascii.h>
+#include <isc/random.h>
+#include <isc/siphash.h>
+#include <isc/time.h>
+
+#define SIZE (1024 * 1024)
+
+int
+main(void) {
+	static uint8_t bytes[SIZE];
+	static uint8_t key[16];
+
+	isc_random_buf(bytes, SIZE);
+	isc_random_buf(key, sizeof(key));
+
+	for (size_t len = 256; len > 0; len = len * 4 / 5) {
+		isc_time_t start, finish;
+		uint64_t count = 0;
+		uint64_t sum = 0;
+		uint64_t us;
+
+		isc_time_now_hires(&start);
+
+		for (size_t end = len; end < SIZE; end += len) {
+			uint64_t hash;
+			uint8_t lower[1024];
+			isc_ascii_lowercopy(lower, bytes + end - len, len);
+			isc_siphash24(key, lower, len, true, (void *)&hash);
+			sum += hash;
+			count++;
+		}
+
+		isc_time_now_hires(&finish);
+
+		us = isc_time_microdiff(&finish, &start);
+		printf("%f us wide-lower len %3zu, %7llu kh/s (%llx)\n",
+		       (double)us / 1000000.0, len,
+		       (unsigned long long)(count * 1000 / us),
+		       (unsigned long long)sum);
+	}
+
+	for (size_t len = 256; len > 0; len = len * 4 / 5) {
+		isc_time_t start, finish;
+		uint64_t count = 0;
+		uint64_t sum = 0;
+		uint64_t us;
+
+		isc_time_now_hires(&start);
+
+		for (size_t end = len; end < SIZE; end += len) {
+			uint64_t hash;
+			isc_siphash24(key, bytes + end - len, len, false,
+				      (void *)&hash);
+			sum += hash;
+			count++;
+		}
+
+		isc_time_now_hires(&finish);
+
+		us = isc_time_microdiff(&finish, &start);
+		printf("%f us wide-icase len %3zu, %7llu kh/s (%llx)\n",
+		       (double)us / 1000000.0, len,
+		       (unsigned long long)(count * 1000 / us),
+		       (unsigned long long)sum);
+	}
+	for (size_t len = 256; len > 0; len = len * 4 / 5) {
+		isc_time_t start, finish;
+		uint64_t count = 0;
+		uint64_t sum = 0;
+		uint64_t us;
+
+		isc_time_now_hires(&start);
+
+		for (size_t end = len; end < SIZE; end += len) {
+			uint64_t hash;
+			isc_siphash24(key, bytes + end - len, len, true,
+				      (void *)&hash);
+			sum += hash;
+			count++;
+		}
+
+		isc_time_now_hires(&finish);
+
+		us = isc_time_microdiff(&finish, &start);
+		printf("%f us wide-bytes len %3zu, %7llu kh/s (%llx)\n",
+		       (double)us / 1000000.0, len,
+		       (unsigned long long)(count * 1000 / us),
+		       (unsigned long long)sum);
+	}
+
+	for (size_t len = 256; len > 0; len = len * 4 / 5) {
+		isc_time_t start, finish;
+		uint64_t count = 0;
+		uint64_t sum = 0;
+		uint64_t us;
+
+		isc_time_now_hires(&start);
+
+		for (size_t end = len; end < SIZE; end += len) {
+			uint32_t hash;
+			uint8_t lower[1024];
+			isc_ascii_lowercopy(lower, bytes + end - len, len);
+			isc_halfsiphash24(key, lower, len, true, (void *)&hash);
+			sum += hash;
+			count++;
+		}
+
+		isc_time_now_hires(&finish);
+
+		us = isc_time_microdiff(&finish, &start);
+		printf("%f us half-lower len %3zu, %7llu kh/s (%llx)\n",
+		       (double)us / 1000000.0, len,
+		       (unsigned long long)(count * 1000 / us),
+		       (unsigned long long)sum);
+	}
+
+	for (size_t len = 256; len > 0; len = len * 4 / 5) {
+		isc_time_t start, finish;
+		uint64_t count = 0;
+		uint64_t sum = 0;
+		uint64_t us;
+
+		isc_time_now_hires(&start);
+
+		for (size_t end = len; end < SIZE; end += len) {
+			uint32_t hash;
+			isc_halfsiphash24(key, bytes + end - len, len, false,
+					  (void *)&hash);
+			sum += hash;
+			count++;
+		}
+
+		isc_time_now_hires(&finish);
+
+		us = isc_time_microdiff(&finish, &start);
+		printf("%f us half-icase len %3zu, %7llu kh/s (%llx)\n",
+		       (double)us / 1000000.0, len,
+		       (unsigned long long)(count * 1000 / us),
+		       (unsigned long long)sum);
+	}
+
+	for (size_t len = 256; len > 0; len = len * 4 / 5) {
+		isc_time_t start, finish;
+		uint64_t count = 0;
+		uint64_t sum = 0;
+		uint64_t us;
+
+		isc_time_now_hires(&start);
+
+		for (size_t end = len; end < SIZE; end += len) {
+			uint32_t hash;
+			isc_halfsiphash24(key, bytes + end - len, len, true,
+					  (void *)&hash);
+			sum += hash;
+			count++;
+		}
+
+		isc_time_now_hires(&finish);
+
+		us = isc_time_microdiff(&finish, &start);
+		printf("%f us half-bytes len %3zu, %7llu kh/s (%llx)\n",
+		       (double)us / 1000000.0, len,
+		       (unsigned long long)(count * 1000 / us),
+		       (unsigned long long)sum);
+	}
+}

tests/isc/ascii_test.c

@@ -145,14 +145,14 @@ ISC_RUN_TEST_IMPL(exhaustive) {
 		uint64_t abi = isc_ascii_tolower(a) << 8 | isc_ascii_tolower(b);
 		uint64_t ab1 = isc__ascii_tolower1(a) << 8 |
 			       isc__ascii_tolower1(b);
-		uint64_t ab8 = isc__ascii_tolower8(ab);
+		uint64_t ab8 = isc_ascii_tolower8(ab);
 		/* each byte individually matches ctype.h */
 		assert_int_equal(tolower(a), isc_ascii_tolower(a));
 		assert_int_equal(tolower(a), isc__ascii_tolower1(a));
-		assert_int_equal(tolower(a), isc__ascii_tolower8(a));
+		assert_int_equal(tolower(a), isc_ascii_tolower8(a));
 		assert_int_equal(tolower(b), isc_ascii_tolower(b));
 		assert_int_equal(tolower(b), isc__ascii_tolower1(b));
-		assert_int_equal(tolower(b), isc__ascii_tolower8(b));
+		assert_int_equal(tolower(b), isc_ascii_tolower8(b));
 		/* two lanes of SWAR match other implementations */
 		assert_int_equal(ab8, abc);
 		assert_int_equal(ab8, abi);
@@ -160,12 +160,12 @@ ISC_RUN_TEST_IMPL(exhaustive) {
 		/* check lack of overflow */
 		assert_int_equal(ab8 >> 16, 0);
 		/* all lanes of SWAR work */
-		assert_int_equal(isc__ascii_tolower8(ab << 8), abc << 8);
-		assert_int_equal(isc__ascii_tolower8(ab << 16), abc << 16);
-		assert_int_equal(isc__ascii_tolower8(ab << 24), abc << 24);
-		assert_int_equal(isc__ascii_tolower8(ab << 32), abc << 32);
-		assert_int_equal(isc__ascii_tolower8(ab << 40), abc << 40);
-		assert_int_equal(isc__ascii_tolower8(ab << 48), abc << 48);
+		assert_int_equal(isc_ascii_tolower8(ab << 8), abc << 8);
+		assert_int_equal(isc_ascii_tolower8(ab << 16), abc << 16);
+		assert_int_equal(isc_ascii_tolower8(ab << 24), abc << 24);
+		assert_int_equal(isc_ascii_tolower8(ab << 32), abc << 32);
+		assert_int_equal(isc_ascii_tolower8(ab << 40), abc << 40);
+		assert_int_equal(isc_ascii_tolower8(ab << 48), abc << 48);
 	}
 }
 

tests/isc/siphash_test.c

@@ -139,7 +139,7 @@ ISC_RUN_TEST_IMPL(isc_siphash24) {
 
 	for (size_t i = 0; i < ARRAY_SIZE(in); i++) {
 		in[i] = i;
-		isc_siphash24(key, in, i, out);
+		isc_siphash24(key, in, i, true, out);
 		assert_memory_equal(out, vectors_sip64[i], 8);
 	}
 }
@@ -154,7 +154,7 @@ ISC_RUN_TEST_IMPL(isc_halfsiphash24) {
 
 	for (size_t i = 0; i < ARRAY_SIZE(in); i++) {
 		in[i] = i;
-		isc_halfsiphash24(key, in, i, out);
+		isc_halfsiphash24(key, in, i, true, out);
 		assert_memory_equal(out, vectors_hsip32[i], 4);
 	}
 }