postfix-3.8-20220129

postfix/HISTORY

@@ -26258,6 +26258,12 @@ Apologies for any names omitted.
 
 20220128
 
-	Clenaup: standardize on FNV hash, having verified that
+	Clenaup: standardize on FNV hash, after having verified
-	collisions will depend on the hash seed value, and that the
+	that collisions will change with the hash seed value, and
-	collision rate is low. Files: util/htable.c, util/fnv_hash.[hc].
+	that the collision rate is low. Files: util/htable.c,
+	util/hash_fnv.[hc].
+
+20220129
+
+	Cleanup: factored out the non-cryptographic seeder. Files:
+	ldseed.[hc].

postfix/proto/stop.double-cc

@@ -327,3 +327,4 @@ more  more useful and more consistent
  Fatal error error opening existing file 
 XXX  XXX 
  int compar DNS_RR DNS_RR 
+NO_64_BITS  NO_64_BITS 

postfix/proto/stop.spell-cc

@@ -1770,3 +1770,14 @@ vars
 verboten
 versioning
 wiki
+DSTRICT
+FNV
+NONBLOCK
+Vo
+chongo
+fnv
+isthe
+ldseed
+softwareengineering
+stackexchange
+stdint

postfix/src/global/mail_version.h

@@ -20,7 +20,7 @@
   * Patches change both the patchlevel and the release date. Snapshots have no
   * patchlevel; they change the release date only.
   */
-#define MAIL_RELEASE_DATE	"20220128"
+#define MAIL_RELEASE_DATE	"20220129"
 #define MAIL_VERSION_NUMBER	"3.8"
 
 #ifdef SNAPSHOT

postfix/src/util/Makefile.in

@@ -43,7 +43,7 @@ SRCS	= alldig.c allprint.c argv.c argv_split.c attr_clnt.c attr_print0.c \
 	split_qnameval.c argv_attr_print.c argv_attr_scan.c dict_file.c \
 	msg_logger.c logwriter.c unix_dgram_connect.c unix_dgram_listen.c \
 	byte_mask.c known_tcp_ports.c argv_split_at.c dict_stream.c \
-	sane_strtol.c hash_fnv.c
+	sane_strtol.c hash_fnv.c ldseed.c
 OBJS	= alldig.o allprint.o argv.o argv_split.o attr_clnt.o attr_print0.o \
 	attr_print64.o attr_print_plain.o attr_scan0.o attr_scan64.o \
 	attr_scan_plain.o auto_clnt.o base64_code.o basename.o binhash.o \
@@ -88,7 +88,7 @@ OBJS	= alldig.o allprint.o argv.o argv_split.o attr_clnt.o attr_print0.o \
 	split_qnameval.o argv_attr_print.o argv_attr_scan.o dict_file.o \
 	msg_logger.o logwriter.o unix_dgram_connect.o unix_dgram_listen.o \
 	byte_mask.o known_tcp_ports.o argv_split_at.o dict_stream.o \
-	sane_strtol.o hash_fnv.o
+	sane_strtol.o hash_fnv.o ldseed.o
 # MAP_OBJ is for maps that may be dynamically loaded with dynamicmaps.cf.
 # When hard-linking these, makedefs sets NON_PLUGIN_MAP_OBJ=$(MAP_OBJ),
 # otherwise it sets the PLUGIN_* macros.
@@ -119,7 +119,7 @@ HDRS	= argv.h attr.h attr_clnt.h auto_clnt.h base64_code.h binhash.h \
 	slmdb.h compat_va_copy.h dict_pipe.h dict_random.h \
 	valid_utf8_hostname.h midna_domain.h dict_union.h dict_inline.h \
 	check_arg.h argv_attr.h msg_logger.h logwriter.h byte_mask.h \
-	known_tcp_ports.h sane_strtol.h hash_fnv.h
+	known_tcp_ports.h sane_strtol.h hash_fnv.h ldseed.h
 TESTSRC	= fifo_open.c fifo_rdwr_bug.c fifo_rdonly_bug.c select_bug.c \
 	stream_test.c dup2_pass_on_exec.c
 DEFS	= -I. -D$(SYSTYPE)
@@ -1909,6 +1909,7 @@ get_hostname.o: sys_defs.h
 get_hostname.o: valid_hostname.h
 hash_fnv.o: hash_fnv.c
 hash_fnv.o: hash_fnv.h
+hash_fnv.o: ldseed.h
 hash_fnv.o: msg.h
 hash_fnv.o: sys_defs.h
 hex_code.o: check_arg.h
@@ -2038,6 +2039,10 @@ known_tcp_ports.o: stringops.h
 known_tcp_ports.o: sys_defs.h
 known_tcp_ports.o: vbuf.h
 known_tcp_ports.o: vstring.h
+ldseed.o: iostuff.h
+ldseed.o: ldseed.c
+ldseed.o: msg.h
+ldseed.o: sys_defs.h
 line_number.o: check_arg.h
 line_number.o: line_number.c
 line_number.o: line_number.h
@@ -2541,6 +2546,7 @@ stream_trigger.o: mymalloc.h
 stream_trigger.o: stream_trigger.c
 stream_trigger.o: sys_defs.h
 stream_trigger.o: trigger.h
+sys_compat.o: iostuff.h
 sys_compat.o: sys_compat.c
 sys_compat.o: sys_defs.h
 timecmp.o: timecmp.c

postfix/src/util/hash_fnv.c

@@ -10,21 +10,27 @@
 /*	const void *src,
 /*	size_t	len)
 /* DESCRIPTION
-/*	hash_fnv() implements the FNV type 1a hash function.
+/*	hash_fnv() implements a modified FNV type 1a hash function.
 /*
 /*	To thwart collision attacks, the hash function is seeded
 /*	once from /dev/urandom, and if that is unavailable, from
 /*	wallclock time, monotonic system clocks, and the process
-/*	ID. To disable seeding in tests, specify the NORANDOMIZE
+/*	ID. To disable seeding (typically, for regression tests),
-/*	environment variable (the value does not matter).
+/*	specify the NORANDOMIZE environment variable; the value
+/*	does not matter.
 /*
-/*	By default, the function is modified to avoid a sticky state
+/*	This function implements a workaround for a "sticky state"
-/*	where a zero hash value remains zero when the next input
+/*	problem with FNV hash functions: when an input produces a
-/*	byte value is zero. Compile with -DSTRICT_FNV1A to get the
+/*	zero intermediate hash state, and the next input byte is
-/*	standard behavior.
+/*	zero, then the operations "hash ^= 0" and "hash *= FNV_prime"
+/*	would not change the hash value. To avoid this, hash_fnv()
+/*	adds 1 to each input byte. Compile with -DSTRICT_FNV1A to
+/*	get the standard behavior.
 /*
 /*	The default HASH_FNV_T result type is uint64_t. When compiled
-/*	with -DNO_64_BITS, the result type is uint32_t.
+/*	with -DNO_64_BITS, the result type is uint32_t. On ancient
+/*	systems without <stdint.h>, define HASH_FNV_T on the compiler
+/*	command line as an unsigned 32-bit or 64-bit integer type.
 /* SEE ALSO
 /*	http://www.isthe.com/chongo/tech/comp/fnv/index.html
 /*	https://softwareengineering.stackexchange.com/questions/49550/
@@ -43,17 +49,14 @@
   * System library
   */
 #include <sys_defs.h>
-#include <string.h>
-#include <sys/time.h>
-#include <time.h>
 #include <stdlib.h>
-#include <fcntl.h>
 #include <unistd.h>
 
  /*
   * Utility library.
   */
 #include <msg.h>
+#include <ldseed.h>
 #include <hash_fnv.h>
 
  /*
@@ -67,80 +70,6 @@
 #define FNV_offset_basis	0xcbf29ce484222325ULL
 #endif
 
- /*
-  * Fall back to a mix of absolute and time-since-boot information in the
-  * rare case that /dev/urandom is unavailable.
-  */
-#ifdef CLOCK_UPTIME
-#define NON_WALLTIME_CLOCK      CLOCK_UPTIME
-#elif defined(CLOCK_BOOTTIME)
-#define NON_WALLTIME_CLOCK      CLOCK_BOOTTIME
-#elif defined(CLOCK_MONOTONIC)
-#define NON_WALLTIME_CLOCK      CLOCK_MONOTONIC
-#elif defined(CLOCK_HIGHRES)
-#define NON_WALLTIME_CLOCK      CLOCK_HIGHRES
-#endif
-
-/* fnv_seed - randomize the hash function */
-
-static HASH_FNV_T fnv_seed(void)
-{
-    HASH_FNV_T result = 0;
-
-    /*
-     * Medium-quality seed, for defenses against local and remote attacks.
-     */
-    int     fd;
-    int     count;
-
-    if ((fd = open("/dev/urandom", O_RDONLY)) > 0) {
-	count = read(fd, &result, sizeof(result));
-	(void) close(fd);
-	if (count == sizeof(result) && result != 0)
-	    return (result);
-    }
-
-    /*
-     * Low-quality seed, for defenses against remote attacks. Based on 1) the
-     * time since boot (good when an attacker knows the program start time
-     * but not the system boot time), and 2) absolute time (good when an
-     * attacker does not know the program start time). Assumes a system with
-     * better than microsecond resolution, and a network stack that does not
-     * leak the time since boot, for example, through TCP or ICMP timestamps.
-     * With those caveats, this seed is good for 20-30 bits of randomness.
-     */
-#ifdef NON_WALLTIME_CLOCK
-    {
-	struct timespec ts;
-
-	if (clock_gettime(NON_WALLTIME_CLOCK, &ts) != 0)
-	    msg_fatal("clock_gettime() failed: %m");
-	result += (HASH_FNV_T) ts.tv_sec ^ (HASH_FNV_T) ts.tv_nsec;
-    }
-#elif defined(USE_GETHRTIME)
-    result += gethrtime();
-#endif
-
-#ifdef CLOCK_REALTIME
-    {
-	struct timespec ts;
-
-	if (clock_gettime(CLOCK_REALTIME, &ts) != 0)
-	    msg_fatal("clock_gettime() failed: %m");
-	result += (HASH_FNV_T) ts.tv_sec ^ (HASH_FNV_T) ts.tv_nsec;
-    }
-#else
-    {
-	struct timeval tv;
-
-	if (GETTIMEOFDAY(&tv) != 0)
-	    msg_fatal("gettimeofday() failed: %m");
-	result += (HASH_FNV_T) tv.tv_sec + (HASH_FNV_T) tv.tv_usec;
-    }
-#endif
-    return (result + getpid());
-}
-
 /* hash_fnv - modified FNV 1a hash */
 
 HASH_FNV_T hash_fnv(const void *src, size_t len)
@@ -152,30 +81,25 @@ HASH_FNV_T hash_fnv(const void *src, size_t len)
     /*
      * Initialize.
      */
-    while (randomize) {
+    if (randomize) {
-	if (getenv("NORANDOMIZE")) {
+	if (!getenv("NORANDOMIZE")) {
-	    randomize = 0;
+	    HASH_FNV_T seed;
-	} else {
+
-	    basis ^= fnv_seed();
+	    ldseed(&seed, sizeof(seed));
-	    if (basis != FNV_offset_basis)
+	    basis ^= seed;
-		randomize = 0;
 	}
+	randomize = 0;
     }
 
-    /*
-     * Add 1 to each input character, to avoid a sticky state (with hash ==
-     * 0, doing "hash ^= 0" and "hash *= FNV_prime" would not change the hash
-     * value.
-     */
 #ifdef STRICT_FNV1A
-#define FNV_NEXT_CHAR(s) ((HASH_FNV_T) * (const unsigned char *) s++)
+#define FNV_NEXT_BYTE(s) ((HASH_FNV_T) * (const unsigned char *) s++)
 #else
-#define FNV_NEXT_CHAR(s) (1 + (HASH_FNV_T) * (const unsigned char *) s++)
+#define FNV_NEXT_BYTE(s) (1 + (HASH_FNV_T) * (const unsigned char *) s++)
 #endif
 
     hash = basis;
     while (len-- > 0) {
-	hash ^= FNV_NEXT_CHAR(src);
+	hash ^= FNV_NEXT_BYTE(src);
 	hash *= FNV_prime;
     }
     return (hash);

postfix/src/util/hash_fnv.h

@@ -11,21 +11,17 @@
 /* DESCRIPTION
 /* .nf
 
- /*
-  * Systemn library.
-  */
-#ifndef NO_STDINT_H
-#include <stdint.h>
-#endif
-
  /*
   * External interface.
   */
+#ifndef HASH_FNV_T
+#include <stdint.h>
 #ifdef NO_64_BITS
 #define HASH_FNV_T	uint32_t
-#else
+#else					/* NO_64_BITS */
 #define HASH_FNV_T	uint64_t
-#endif
+#endif					/* NO_64_BITS */
+#endif					/* HASH_FNV_T */
 
 extern HASH_FNV_T hash_fnv(const void *, size_t);
 

postfix/src/util/htable.c

@@ -113,6 +113,11 @@
 /*	IBM T.J. Watson Research
 /*	P.O. Box 704
 /*	Yorktown Heights, NY 10598, USA
+/*
+/*	Wietse Venema
+/*	Google, Inc.
+/*	111 8th Avenue
+/*	New York, NY 10011, USA
 /*--*/
 
 /* C library */
@@ -124,14 +129,12 @@
 
 #include "mymalloc.h"
 #include "msg.h"
-#ifndef NO_HASH_FNV
-#include "hash_fnv.h"
-#endif
 #include "htable.h"
 
 /* htable_hash - hash a string */
 
 #ifndef NO_HASH_FNV
+#include "hash_fnv.h"
 
 #define htable_hash(s, size) (hash_fnv((s), strlen(s)) % (size))
 
@@ -155,6 +158,7 @@ static size_t htable_hash(const char *s, size_t size)
     }
     return (h % size);
 }
+
 #endif
 
 /* htable_link - insert element into table */

postfix/src/util/ldseed.c

@@ -0,0 +1,137 @@
+/*++
+/* NAME
+/*	ldseed 3
+/* SUMMARY
+/*	seed for non-cryptographic applications
+/* SYNOPSIS
+/*	#include <ldseed.h>
+/*
+/*	void	ldseed(
+/*	void	*dst,
+/*	size_t	len)
+/* DESCRIPTION
+/*	ldseed() preferably extracts pseudo-random bits from
+/*	/dev/urandom, a non-blocking device that is available on
+/*	modern systems.
+/*
+/*	On systems where /dev/urandom is unavailable or does not
+/*	immediately return the requested amount of randomness,
+/*	ldseed() falls back to a combination of wallclock time,
+/*	the time since boot, and the process ID.
+/* BUGS
+/*	With Linux "the O_NONBLOCK flag has no effect when opening
+/*	/dev/urandom", but reads "can incur an appreciable delay
+/*	when requesting large amounts of data". Apparently, "large"
+/*	means more than 256 bytes.
+/* LICENSE
+/* .ad
+/* .fi
+/*	The Secure Mailer license must be distributed with this software.
+/* AUTHOR(S)
+/*	Wietse Venema
+/*	Google, Inc.
+/*	111 8th Avenue
+/*	New York, NY 10011, USA
+/*--*/
+
+ /*
+  * System library
+  */
+#include <sys_defs.h>
+#include <string.h>
+#include <sys/time.h>
+#include <time.h>
+#include <stdlib.h>
+#include <fcntl.h>
+#include <unistd.h>
+#include <limits.h>			/* CHAR_BIT */
+
+ /*
+  * Utility library.
+  */
+#include <iostuff.h>
+#include <msg.h>
+
+ /*
+  * Different systems have different names for non-wallclock time.
+  */
+#ifdef CLOCK_UPTIME
+#define NON_WALLTIME_CLOCK      CLOCK_UPTIME
+#elif defined(CLOCK_BOOTTIME)
+#define NON_WALLTIME_CLOCK      CLOCK_BOOTTIME
+#elif defined(CLOCK_MONOTONIC)
+#define NON_WALLTIME_CLOCK      CLOCK_MONOTONIC
+#elif defined(CLOCK_HIGHRES)
+#define NON_WALLTIME_CLOCK      CLOCK_HIGHRES
+#endif
+
+/* ldseed - best-effort, low-dependency seed */
+
+void    ldseed(void *dst, size_t len)
+{
+    int     count;
+    int     fd;
+    int     n;
+    time_t  fallback = 0;
+
+    /*
+     * Medium-quality seed.
+     */
+    if ((fd = open("/dev/urandom", O_RDONLY)) > 0) {
+	non_blocking(fd, NON_BLOCKING);
+	count = read(fd, dst, len);
+	(void) close(fd);
+	if (count == len)
+	    return;
+    }
+
+    /*
+     * Low-quality seed. Based on 1) the time since boot (good when an
+     * attacker knows the program start time but not the system boot time),
+     * and 2) absolute time (good when an attacker does not know the program
+     * start time). Assumes a system with better than microsecond resolution,
+     * and a network stack that does not leak the time since boot, for
+     * example, through TCP or ICMP timestamps. With those caveats, this seed
+     * is good for 20-30 bits of randomness.
+     */
+#ifdef NON_WALLTIME_CLOCK
+    {
+	struct timespec ts;
+
+	if (clock_gettime(NON_WALLTIME_CLOCK, &ts) != 0)
+	    msg_fatal("clock_gettime() failed: %m");
+	fallback += ts.tv_sec ^ ts.tv_nsec;
+    }
+#elif defined(USE_GETHRTIME)
+    fallback += gethrtime();
+#endif
+
+#ifdef CLOCK_REALTIME
+    {
+	struct timespec ts;
+
+	if (clock_gettime(CLOCK_REALTIME, &ts) != 0)
+	    msg_fatal("clock_gettime() failed: %m");
+	fallback += ts.tv_sec ^ ts.tv_nsec;
+    }
+#else
+    {
+	struct timeval tv;
+
+	if (GETTIMEOFDAY(&tv) != 0)
+	    msg_fatal("gettimeofday() failed: %m");
+	fallback += tv.tv_sec + tv.tv_usec;
+    }
+#endif
+    fallback += getpid();
+
+    /*
+     * Copy the least significant bytes first, because those are the most
+     * volatile.
+     */
+    for (n = 0; n < sizeof(fallback) && n < len; n++) {
+	*(char *) dst++ ^= (fallback & 0xff);
+	fallback >>= CHAR_BIT;
+    }
+    return;
+}

postfix/src/util/ldseed.h

@@ -0,0 +1,30 @@
+#ifndef _LDSEED_H_INCLUDED_
+#define _LDSEED_H_INCLUDED_
+
+/*++
+/* NAME
+/*	ldseed 3h
+/* SUMMARY
+/*	seed for non-cryptographic applications
+/* SYNOPSIS
+/*	#include <ldseed.h>
+/* DESCRIPTION
+/* .nf
+
+ /*
+  * External interface.
+  */
+extern void ldseed(void *, size_t);
+
+/* LICENSE
+/* .ad
+/* .fi
+/*	The Secure Mailer license must be distributed with this software.
+/* AUTHOR(S)
+/*	Wietse Venema
+/*	Google, Inc.
+/*	111 8th Avenue
+/*	New York, NY 10011, USA
+/*--*/
+
+#endif