From ce13ee888045458116958d0733beede6a32359a9 Mon Sep 17 00:00:00 2001 From: Wietse Venema Date: Sat, 29 Jan 2022 00:00:00 -0500 Subject: [PATCH] postfix-3.7.0-RC2 --- postfix/HISTORY | 21 ++++ postfix/README_FILES/FILTER_README | 4 +- postfix/RELEASE_NOTES | 117 +++++++++++------------ postfix/html/FILTER_README.html | 2 +- postfix/proto/FILTER_README.html | 2 +- postfix/proto/stop.double-cc | 1 + postfix/proto/stop.spell-cc | 11 +++ postfix/src/global/mail_version.h | 4 +- postfix/src/master/master_monitor.c | 8 +- postfix/src/util/Makefile.in | 24 ++++- postfix/src/util/attr_scan0.ref | 12 +-- postfix/src/util/attr_scan64.ref | 12 +-- postfix/src/util/attr_scan_plain.ref | 12 +-- postfix/src/util/dict_thash.ref | 2 +- postfix/src/util/hash_fnv.c | 106 +++++++++++++++++++++ postfix/src/util/hash_fnv.h | 39 ++++++++ postfix/src/util/htable.c | 127 +++++-------------------- postfix/src/util/ldseed.c | 137 +++++++++++++++++++++++++++ postfix/src/util/ldseed.h | 30 ++++++ 19 files changed, 477 insertions(+), 194 deletions(-) create mode 100644 postfix/src/util/hash_fnv.c create mode 100644 postfix/src/util/hash_fnv.h create mode 100644 postfix/src/util/ldseed.c create mode 100644 postfix/src/util/ldseed.h diff --git a/postfix/HISTORY b/postfix/HISTORY index d53113bba..55d4d4109 100644 --- a/postfix/HISTORY +++ b/postfix/HISTORY @@ -26246,3 +26246,24 @@ Apologies for any names omitted. Documentation: added LINUX_README sections for logging in a container, and for systemd logging workarounds. File: proto/LINUX_README.hmtl. + +20220126 + + Added defensive logging while waiting for the master daemon + to initialize in the background. File: master/master_monitor.c. + +20220127 + + Cleanup: smtpprox hyperlink. File: proto/FILTER_README.html. + +20220128 + + Clenaup: standardize on FNV hash, after having verified + that collisions will change with the hash seed value, and + that the collision rate is low. Files: util/htable.c, + util/hash_fnv.[hc]. + +20220129 + + Cleanup: factored out the non-cryptographic seeder. Files: + ldseed.[hc]. diff --git a/postfix/README_FILES/FILTER_README b/postfix/README_FILES/FILTER_README index bbe2adba1..4a76bb9b5 100644 --- a/postfix/README_FILES/FILTER_README +++ b/postfix/README_FILES/FILTER_README @@ -282,8 +282,8 @@ This content filter receives unfiltered mail with SMTP on localhost port 10025, and sends filtered mail back into Postfix with SMTP on localhost port 10026. For non-SMTP capable content filtering software, Bennett Todd's SMTP proxy -implements a nice PERL/SMTP content filtering framework. See: http:// -bent.latency.net/smtpprox/. +implements a nice PERL/SMTP content filtering framework. See: https:// +web.archive.org/web/20151022025756/http://bent.latency.net/smtpprox/. In the figure below, names followed by a number represent Postfix commands or daemon programs. See the OVERVIEW document for an introduction to the Postfix diff --git a/postfix/RELEASE_NOTES b/postfix/RELEASE_NOTES index fd6b9fce6..a15bf4ed9 100644 --- a/postfix/RELEASE_NOTES +++ b/postfix/RELEASE_NOTES @@ -28,14 +28,18 @@ comfortable with the IPL can continue with that license. Major changes - configuration ----------------------------- -[Feature 20210605] Support to inline the content of small cidr, -pcre, and regexp tables. +[Feature 20210605] Support to inline the content of small cidr:, +pcre:, and regexp: tables in Postfix parameter values. Example: smtpd_forbidden_commands = CONNECT GET POST regexp:{{/^[^A-Z]/ Thrash}} +This is the new smtpd_forbidden_commands default value. It will +immediately disconnect a remote SMTP client when a command does not +start with a letter (a-z or A-Z). + The basic syntax is: /etc/postfix/main.cf: @@ -56,23 +60,25 @@ in-memory file: Postfix parses the result as if it is a file in /etc/postfix. -Note: if a rule contains $, specify $$, to keep Postfix from trying +Note: if a rule contains $, specify $$ to keep Postfix from trying to do $name expansion as it evaluates the parameter value. Major changes - lmdb support ---------------------------- -[Feature 20210605] Overhauled the LMDB client implementation, added -integration tests for future-proofing. +[Feature 20210605] Overhauled the LMDB client implementation, and +added integration tests for future-proofing. There are no visible +changes in documented behavior. Major changes - logging ----------------------- [Feature 20210815] To make the maillog_file feature more useful, the postlog(1) command is now set-gid postdrop, so that unprivileged -programs can write logging through the postlogd(8) daemon. This -required hardening the postlog(1) command against privilege escalation -attacks. +programs can use it to write logging through the postlogd(8) daemon. +This required hardening the postlog(1) command against privilege +escalation attacks. DO NOT turn on the set-gid bit with older +postlog(1) implementations. Major changes - pcre2 support ----------------------------- @@ -81,44 +87,38 @@ Major changes - pcre2 support library is no longer maintained). The Postfix build procedure automatically detects if the pcre2 library is installed, and if it is unavailable, the Postfix build procedure will detect if the -legacy pcre library is installed. See PCRE_README if you need to +legacy pcre library is installed. See PCRE_README if you need to build Postfix with a specific library. Visible differences: some error messages may have a different text, and the 'X' pattern flag is no longer supported with pcre2. -Major changes - safety ----------------------- - -[Feature 20210926] Prevent sharing of xxx_tls_session_cache_database -instances between different Postfix instances when a database is -not multi-writer safe. These databases are now opened with a permanent -lock. The tlsmgr(8) daemon will raise a fatal error when it attempts -to open an xxx_tls_session_cache_database that is already opened -by a different tlsmgr(8) process. - Major changes - security ------------------------ [Feature 20220102] Postfix programs now randomize the initial state of in-memory hash tables, to defend against hash collision attacks -involving a large number of attacker-chosen lookup keys. Presently, +involving a large number of attacker-chosen lookup keys. Presently, the only known opportunity for such attacks involves remote SMTP -client IPv6 addresses in the anvil(8) service. That would require -making hundreds of short-lived connections per second, because the -service ages out idle connections after 100s. Other tables with -attacker-chosen lookup keys are by design limited in size. The fix -is cheap, and therefore implemented for all Postfix in-memory hash -tables. Problem reported by Pascal Junod. +client IPv6 addresses in the anvil(8) service. The attack would +require making hundreds of short-lived connections per second from +thousands of different IP addresses, because the anvil(8) service +drops inactive counters after 100s. Other in-memory hash tables +with attacker-chosen lookup keys are by design limited in size. The +fix is cheap, and therefore implemented for all Postfix in-memory +hash tables. Problem reported by Pascal Junod. [Feature 20211030] The postqueue command now sanitizes non-printable -characters in strings before they are formatted as json output or -legacy output. These outputs are piped into other programs that are -run by administrative users. This closes a hypothetical opportunity -for privilege escalation. +characters (such as newlines) in strings before they are formatted +as json or as legacy output. These outputs are piped into other +programs that are run by administrative users. This closes a +hypothetical opportunity for privilege escalation. [Feature 20210815] Updated defense against remote clients or servers -that 'trickle' SMTP or LMTP traffic. +that 'trickle' SMTP or LMTP traffic, based on per-request deadlines +and minimum data rates. + +Per-request deadlines: The new {smtpd,smtp,lmtp}_per_request_deadline parameters replace {smtpd,smtp,lmtp}_per_record_deadline, with backwards compatible @@ -131,16 +131,17 @@ a response, while the new {smtp,lmtp}_per_record_deadline parameters limit the combined time for the Postfix SMTP or LMTP client to send a request and to receive a response. -Additionally, the new smtpd_min_data_rate parameter enforces a -minimum plaintext data transfer rate for DATA and BDAT requests, -but only when smtpd_per_record_deadline is enabled. After a read -operation transfers N plaintext bytes (possibly after TLS decryption), -and after the DATA or BDAT request deadline is decreased by the -elapsed time of that read operation, the DATA or BDAT request -deadline is increased by N/smtpd_min_data_rate seconds. However, -the deadline is never increased beyond the smtpd_timeout value. The -default minimum data rate is 500 (bytes/second) but is still subject -to change. +Minimum data rates: + +The new smtpd_min_data_rate parameter enforces a minimum plaintext +data transfer rate for DATA and BDAT requests, but only when +smtpd_per_record_deadline is enabled. After a read operation transfers +N plaintext bytes (possibly after TLS decryption), and after the +DATA or BDAT request deadline is decreased by the elapsed time of +that read operation, the DATA or BDAT request deadline is increased +by N/smtpd_min_data_rate seconds. However, the deadline is never +increased beyond the smtpd_timeout value. The default minimum data +rate is 500 (bytes/second) but is still subject to change. The new {smtp,lmtp}_min_data_rate parameters enforce the corresponding minimum DATA transfer rates for the Postfix SMTP and LMTP client. @@ -148,31 +149,31 @@ minimum DATA transfer rates for the Postfix SMTP and LMTP client. Major changes - tls support --------------------------- -[Incompat 20220121] Renamed tlsproxy_client_level to -tlsproxy_client_security_level, and tlsproxy_client_policy to -tlsproxy_client_policy_maps, for consistent parameter naming -(tlsproxy_client_xxx corresponds to smtp_tls_xxx). +[Cleanup 20220121] The new tlsproxy_client_security_level parameter +replaces tlsproxy_client_level, and the new tlsproxy_client_policy_maps +parameter replaces tlsproxy_client_policy. This is for consistent +parameter naming (tlsproxy_client_xxx corresponds to smtp_tls_xxx). +This change was made with backwards-compatible default settings. -This change was made with backwards-compatible default settings, -and with updated documentation. - -[Feature 20210926] Postfix was updated to support OpenSSL 3.0.0 +[Feature 20210926] Postfix was updated to support OpenSSL 3.0.0 API features, and to work around OpenSSL 3.0.0 bit-rot (avoid using -deprecated features). +deprecated API features). Other code health ----------------- [typos] Typo fixes by raf. -[pre-release checks] Added pre-release checks to detect new typos, -and missing entries in postfix-files (some documentation would not -be installed), missing postlink rules (would result in missing -hyperlinks in documentation), missing proxy_read_maps entries (the -proxymap daemon would not automatically authorize some proxied maps). +[pre-release checks] Added pre-release checks to detect a) new typos +in documentation and source-code comments, b) missing entries in +the postfix-files file (some documentation would not be installed), +c) missing rules in the postlink script (some text would not have +a hyperlink in documentation), and d) missing map-based $parameter +names in the proxy_read_maps default value (the proxymap daemon +would not automatically authorize some proxied maps). [memory stream] Improved support for memory-based streams made it -possible to eliminate ad-hoc code that converted tlsproxy(8) protocol -data to and from serialized form, and to inline small cidr:, pcre:, -and regexp: maps in main.cf. +possible to inline small cidr:, pcre:, and regexp: maps in Postfix +parameter values, and to eliminate some ad-hoc code that converted +tlsproxy(8) protocol data to or from serialized form. diff --git a/postfix/html/FILTER_README.html b/postfix/html/FILTER_README.html index 4873fc1fa..6eec9ab1d 100644 --- a/postfix/html/FILTER_README.html +++ b/postfix/html/FILTER_README.html @@ -473,7 +473,7 @@ mail back into Postfix with SMTP on localhost port 10026.

For non-SMTP capable content filtering software, Bennett Todd's SMTP proxy implements a nice PERL/SMTP content filtering framework. -See: http://bent.latency.net/smtpprox/.

+See: https://web.archive.org/web/20151022025756/http://bent.latency.net/smtpprox/.

In the figure below, names followed by a number represent Postfix commands or daemon programs. See the OVERVIEW diff --git a/postfix/proto/FILTER_README.html b/postfix/proto/FILTER_README.html index a87fbce2a..12120f080 100644 --- a/postfix/proto/FILTER_README.html +++ b/postfix/proto/FILTER_README.html @@ -473,7 +473,7 @@ mail back into Postfix with SMTP on localhost port 10026.

For non-SMTP capable content filtering software, Bennett Todd's SMTP proxy implements a nice PERL/SMTP content filtering framework. -See: http://bent.latency.net/smtpprox/.

+See: https://web.archive.org/web/20151022025756/http://bent.latency.net/smtpprox/.

In the figure below, names followed by a number represent Postfix commands or daemon programs. See the OVERVIEW diff --git a/postfix/proto/stop.double-cc b/postfix/proto/stop.double-cc index 3671205f5..9a4cc5aec 100644 --- a/postfix/proto/stop.double-cc +++ b/postfix/proto/stop.double-cc @@ -327,3 +327,4 @@ more more useful and more consistent Fatal error error opening existing file XXX XXX int compar DNS_RR DNS_RR +NO_64_BITS NO_64_BITS diff --git a/postfix/proto/stop.spell-cc b/postfix/proto/stop.spell-cc index 31863165d..3f49e1421 100644 --- a/postfix/proto/stop.spell-cc +++ b/postfix/proto/stop.spell-cc @@ -1770,3 +1770,14 @@ vars verboten versioning wiki +DSTRICT +FNV +NONBLOCK +Vo +chongo +fnv +isthe +ldseed +softwareengineering +stackexchange +stdint diff --git a/postfix/src/global/mail_version.h b/postfix/src/global/mail_version.h index 3c5bf06da..591be9b6c 100644 --- a/postfix/src/global/mail_version.h +++ b/postfix/src/global/mail_version.h @@ -20,8 +20,8 @@ * Patches change both the patchlevel and the release date. Snapshots have no * patchlevel; they change the release date only. */ -#define MAIL_RELEASE_DATE "20220123" -#define MAIL_VERSION_NUMBER "3.7.0-RC1" +#define MAIL_RELEASE_DATE "20220129" +#define MAIL_VERSION_NUMBER "3.7.0-RC2" #ifdef SNAPSHOT #define MAIL_VERSION_DATE "-" MAIL_RELEASE_DATE diff --git a/postfix/src/master/master_monitor.c b/postfix/src/master/master_monitor.c index 403d07efc..6b2ca6598 100644 --- a/postfix/src/master/master_monitor.c +++ b/postfix/src/master/master_monitor.c @@ -30,6 +30,11 @@ /* IBM T.J. Watson Research /* P.O. Box 704 /* Yorktown Heights, NY 10598, USA +/* +/* Wietse Venema +/* Google, Inc. +/* 111 8th Avenue +/* New York, NY 10011, USA /*--*/ /* System library. */ @@ -86,12 +91,13 @@ int master_monitor(int time_limit) close(pipes[1]); switch (timed_read(pipes[0], buf, 1, time_limit, (void *) 0)) { default: + msg_warn("%m while waiting for daemon initialization"); /* The child process still runs, but something is wrong. */ (void) kill(pid, SIGKILL); /* FALLTHROUGH */ case 0: /* The child process exited prematurely. */ - msg_fatal("daemon initialization failure"); + msg_fatal("daemon initialization failure -- see logs for details"); case 1: /* The child process initialized successfully. */ exit(0); diff --git a/postfix/src/util/Makefile.in b/postfix/src/util/Makefile.in index 3c4736a1d..b0bfc2fa0 100644 --- a/postfix/src/util/Makefile.in +++ b/postfix/src/util/Makefile.in @@ -43,7 +43,7 @@ SRCS = alldig.c allprint.c argv.c argv_split.c attr_clnt.c attr_print0.c \ split_qnameval.c argv_attr_print.c argv_attr_scan.c dict_file.c \ msg_logger.c logwriter.c unix_dgram_connect.c unix_dgram_listen.c \ byte_mask.c known_tcp_ports.c argv_split_at.c dict_stream.c \ - sane_strtol.c + sane_strtol.c hash_fnv.c ldseed.c OBJS = alldig.o allprint.o argv.o argv_split.o attr_clnt.o attr_print0.o \ attr_print64.o attr_print_plain.o attr_scan0.o attr_scan64.o \ attr_scan_plain.o auto_clnt.o base64_code.o basename.o binhash.o \ @@ -88,7 +88,7 @@ OBJS = alldig.o allprint.o argv.o argv_split.o attr_clnt.o attr_print0.o \ split_qnameval.o argv_attr_print.o argv_attr_scan.o dict_file.o \ msg_logger.o logwriter.o unix_dgram_connect.o unix_dgram_listen.o \ byte_mask.o known_tcp_ports.o argv_split_at.o dict_stream.o \ - sane_strtol.o + sane_strtol.o hash_fnv.o ldseed.o # MAP_OBJ is for maps that may be dynamically loaded with dynamicmaps.cf. # When hard-linking these, makedefs sets NON_PLUGIN_MAP_OBJ=$(MAP_OBJ), # otherwise it sets the PLUGIN_* macros. @@ -119,7 +119,7 @@ HDRS = argv.h attr.h attr_clnt.h auto_clnt.h base64_code.h binhash.h \ slmdb.h compat_va_copy.h dict_pipe.h dict_random.h \ valid_utf8_hostname.h midna_domain.h dict_union.h dict_inline.h \ check_arg.h argv_attr.h msg_logger.h logwriter.h byte_mask.h \ - known_tcp_ports.h sane_strtol.h + known_tcp_ports.h sane_strtol.h hash_fnv.h ldseed.h TESTSRC = fifo_open.c fifo_rdwr_bug.c fifo_rdonly_bug.c select_bug.c \ stream_test.c dup2_pass_on_exec.c DEFS = -I. -D$(SYSTYPE) @@ -843,9 +843,9 @@ base32_code_test: base32_code $(SHLIB_ENV) ${VALGRIND} ./base32_code dict_thash_test: ../postmap/postmap dict_thash.map dict_thash.in dict_thash.ref - $(SHLIB_ENV) ../postmap/postmap -fs texthash:dict_thash.map 2>&1 | \ + $(SHLIB_ENV) ${VALGRIND} ../postmap/postmap -fs texthash:dict_thash.map 2>&1 | \ LANG=C sort | diff dict_thash.map - - $(SHLIB_ENV) ../postmap/postmap -fs texthash:dict_thash.in >dict_thash.tmp 2>&1 + NORANDOMIZE=1 $(SHLIB_ENV) ${VALGRIND} ../postmap/postmap -fs texthash:dict_thash.in >dict_thash.tmp 2>&1 diff dict_thash.ref dict_thash.tmp rm -f dict_thash.tmp @@ -1177,6 +1177,7 @@ attr_scan0.o: htable.h attr_scan0.o: msg.h attr_scan0.o: mymalloc.h attr_scan0.o: nvtable.h +attr_scan0.o: stringops.h attr_scan0.o: sys_defs.h attr_scan0.o: vbuf.h attr_scan0.o: vstream.h @@ -1190,6 +1191,7 @@ attr_scan64.o: htable.h attr_scan64.o: msg.h attr_scan64.o: mymalloc.h attr_scan64.o: nvtable.h +attr_scan64.o: stringops.h attr_scan64.o: sys_defs.h attr_scan64.o: vbuf.h attr_scan64.o: vstream.h @@ -1202,6 +1204,7 @@ attr_scan_plain.o: htable.h attr_scan_plain.o: msg.h attr_scan_plain.o: mymalloc.h attr_scan_plain.o: nvtable.h +attr_scan_plain.o: stringops.h attr_scan_plain.o: sys_defs.h attr_scan_plain.o: vbuf.h attr_scan_plain.o: vstream.h @@ -1904,6 +1907,11 @@ get_hostname.o: msg.h get_hostname.o: mymalloc.h get_hostname.o: sys_defs.h get_hostname.o: valid_hostname.h +hash_fnv.o: hash_fnv.c +hash_fnv.o: hash_fnv.h +hash_fnv.o: ldseed.h +hash_fnv.o: msg.h +hash_fnv.o: sys_defs.h hex_code.o: check_arg.h hex_code.o: hex_code.c hex_code.o: hex_code.h @@ -1930,6 +1938,7 @@ host_port.o: valid_hostname.h host_port.o: valid_utf8_hostname.h host_port.o: vbuf.h host_port.o: vstring.h +htable.o: hash_fnv.h htable.o: htable.c htable.o: htable.h htable.o: msg.h @@ -2030,6 +2039,10 @@ known_tcp_ports.o: stringops.h known_tcp_ports.o: sys_defs.h known_tcp_ports.o: vbuf.h known_tcp_ports.o: vstring.h +ldseed.o: iostuff.h +ldseed.o: ldseed.c +ldseed.o: msg.h +ldseed.o: sys_defs.h line_number.o: check_arg.h line_number.o: line_number.c line_number.o: line_number.h @@ -2533,6 +2546,7 @@ stream_trigger.o: mymalloc.h stream_trigger.o: stream_trigger.c stream_trigger.o: sys_defs.h stream_trigger.o: trigger.h +sys_compat.o: iostuff.h sys_compat.o: sys_compat.c sys_compat.o: sys_defs.h timecmp.o: timecmp.c diff --git a/postfix/src/util/attr_scan0.ref b/postfix/src/util/attr_scan0.ref index 1fe09fc9f..9055d795d 100644 --- a/postfix/src/util/attr_scan0.ref +++ b/postfix/src/util/attr_scan0.ref @@ -3,8 +3,8 @@ ./attr_print0: send attr long_number = 1234 ./attr_print0: send attr string = whoopee ./attr_print0: send attr data = [data 7 bytes] -./attr_print0: send attr name foo-name value foo-value ./attr_print0: send attr name bar-name value bar-value +./attr_print0: send attr name foo-name value foo-value ./attr_print0: send attr long_number = 4321 ./attr_print0: send attr protocol = test ./attr_print0: send attr number = 4711 @@ -30,12 +30,12 @@ ./attr_scan0: unknown_stream: wanted attribute: (any attribute name or list terminator) ./attr_scan0: input attribute name: { ./attr_scan0: unknown_stream: wanted attribute: (any attribute name or '}') -./attr_scan0: input attribute name: foo-name -./attr_scan0: input attribute value: foo-value -./attr_scan0: unknown_stream: wanted attribute: (any attribute name or '}') ./attr_scan0: input attribute name: bar-name ./attr_scan0: input attribute value: bar-value ./attr_scan0: unknown_stream: wanted attribute: (any attribute name or '}') +./attr_scan0: input attribute name: foo-name +./attr_scan0: input attribute value: foo-value +./attr_scan0: unknown_stream: wanted attribute: (any attribute name or '}') ./attr_scan0: input attribute name: } ./attr_scan0: unknown_stream: wanted attribute: long_number ./attr_scan0: input attribute name: long_number @@ -67,13 +67,13 @@ number 4711 long_number 1234 string whoopee data whoopee -(hash) foo-name foo-value (hash) bar-name bar-value +(hash) foo-name foo-value long_number 4321 number 4711 long_number 1234 string whoopee data whoopee -(hash) foo-name foo-value (hash) bar-name bar-value +(hash) foo-name foo-value return: -1 diff --git a/postfix/src/util/attr_scan64.ref b/postfix/src/util/attr_scan64.ref index 9f9217afe..ccf27f12e 100644 --- a/postfix/src/util/attr_scan64.ref +++ b/postfix/src/util/attr_scan64.ref @@ -3,8 +3,8 @@ ./attr_print64: send attr long_number = 1234 ./attr_print64: send attr string = whoopee ./attr_print64: send attr data = [data 7 bytes] -./attr_print64: send attr name foo-name value foo-value ./attr_print64: send attr name bar-name value bar-value +./attr_print64: send attr name foo-name value foo-value ./attr_print64: send attr long_number = 4321 ./attr_print64: send attr protocol = test ./attr_print64: send attr number = 4711 @@ -30,12 +30,12 @@ ./attr_scan64: unknown_stream: wanted attribute: (any attribute name or list terminator) ./attr_scan64: input attribute name: { ./attr_scan64: unknown_stream: wanted attribute: (any attribute name or '}') -./attr_scan64: input attribute name: foo-name -./attr_scan64: input attribute value: foo-value -./attr_scan64: unknown_stream: wanted attribute: (any attribute name or '}') ./attr_scan64: input attribute name: bar-name ./attr_scan64: input attribute value: bar-value ./attr_scan64: unknown_stream: wanted attribute: (any attribute name or '}') +./attr_scan64: input attribute name: foo-name +./attr_scan64: input attribute value: foo-value +./attr_scan64: unknown_stream: wanted attribute: (any attribute name or '}') ./attr_scan64: input attribute name: } ./attr_scan64: unknown_stream: wanted attribute: long_number ./attr_scan64: input attribute name: long_number @@ -67,13 +67,13 @@ number 4711 long_number 1234 string whoopee data whoopee -(hash) foo-name foo-value (hash) bar-name bar-value +(hash) foo-name foo-value long_number 4321 number 4711 long_number 1234 string whoopee data whoopee -(hash) foo-name foo-value (hash) bar-name bar-value +(hash) foo-name foo-value return: -1 diff --git a/postfix/src/util/attr_scan_plain.ref b/postfix/src/util/attr_scan_plain.ref index 994ba3ce3..1c0f35809 100644 --- a/postfix/src/util/attr_scan_plain.ref +++ b/postfix/src/util/attr_scan_plain.ref @@ -3,8 +3,8 @@ ./attr_print_plain: send attr long_number = 1234 ./attr_print_plain: send attr string = whoopee ./attr_print_plain: send attr data = [data 7 bytes] -./attr_print_plain: send attr name foo-name value foo-value ./attr_print_plain: send attr name bar-name value bar-value +./attr_print_plain: send attr name foo-name value foo-value ./attr_print_plain: send attr long_number = 4321 ./attr_print_plain: send attr protocol = test ./attr_print_plain: send attr number = 4711 @@ -30,12 +30,12 @@ ./attr_scan_plain: unknown_stream: wanted attribute: (any attribute name or list terminator) ./attr_scan_plain: input attribute name: { ./attr_scan_plain: unknown_stream: wanted attribute: (any attribute name or '}') -./attr_scan_plain: input attribute name: foo-name -./attr_scan_plain: input attribute value: foo-value -./attr_scan_plain: unknown_stream: wanted attribute: (any attribute name or '}') ./attr_scan_plain: input attribute name: bar-name ./attr_scan_plain: input attribute value: bar-value ./attr_scan_plain: unknown_stream: wanted attribute: (any attribute name or '}') +./attr_scan_plain: input attribute name: foo-name +./attr_scan_plain: input attribute value: foo-value +./attr_scan_plain: unknown_stream: wanted attribute: (any attribute name or '}') ./attr_scan_plain: input attribute name: } ./attr_scan_plain: unknown_stream: wanted attribute: long_number ./attr_scan_plain: input attribute name: long_number @@ -67,13 +67,13 @@ number 4711 long_number 1234 string whoopee data whoopee -(hash) foo-name foo-value (hash) bar-name bar-value +(hash) foo-name foo-value long_number 4321 number 4711 long_number 1234 string whoopee data whoopee -(hash) foo-name foo-value (hash) bar-name bar-value +(hash) foo-name foo-value return: -1 diff --git a/postfix/src/util/dict_thash.ref b/postfix/src/util/dict_thash.ref index 26ca0dd9a..efdc20799 100644 --- a/postfix/src/util/dict_thash.ref +++ b/postfix/src/util/dict_thash.ref @@ -2,5 +2,5 @@ postmap: warning: dict_thash.in, line 1: unbalanced '"' in '"the answer is 42' - postmap: warning: dict_thash.in, line 2: record is in "key: value" format; is this an alias file? postmap: warning: dict_thash.in, line 3: expected format: key whitespace value -- ignoring this line postmap: warning: dict_thash.in, line 5: duplicate entry: "aaa" -xxx: yyy aaa bbb +xxx: yyy diff --git a/postfix/src/util/hash_fnv.c b/postfix/src/util/hash_fnv.c new file mode 100644 index 000000000..d8baac549 --- /dev/null +++ b/postfix/src/util/hash_fnv.c @@ -0,0 +1,106 @@ +/*++ +/* NAME +/* hash_fnv 3 +/* SUMMARY +/* Fowler/Noll/Vo hash function +/* SYNOPSIS +/* #include +/* +/* HASH_FNV_T hash_fnv( +/* const void *src, +/* size_t len) +/* DESCRIPTION +/* hash_fnv() implements a modified FNV type 1a hash function. +/* +/* To thwart collision attacks, the hash function is seeded +/* once from /dev/urandom, and if that is unavailable, from +/* wallclock time, monotonic system clocks, and the process +/* ID. To disable seeding (typically, for regression tests), +/* specify the NORANDOMIZE environment variable; the value +/* does not matter. +/* +/* This function implements a workaround for a "sticky state" +/* problem with FNV hash functions: when an input produces a +/* zero intermediate hash state, and the next input byte is +/* zero, then the operations "hash ^= 0" and "hash *= FNV_prime" +/* would not change the hash value. To avoid this, hash_fnv() +/* adds 1 to each input byte. Compile with -DSTRICT_FNV1A to +/* get the standard behavior. +/* +/* The default HASH_FNV_T result type is uint64_t. When compiled +/* with -DNO_64_BITS, the result type is uint32_t. On ancient +/* systems without , define HASH_FNV_T on the compiler +/* command line as an unsigned 32-bit or 64-bit integer type. +/* SEE ALSO +/* http://www.isthe.com/chongo/tech/comp/fnv/index.html +/* https://softwareengineering.stackexchange.com/questions/49550/ +/* LICENSE +/* .ad +/* .fi +/* The Secure Mailer license must be distributed with this software. +/* AUTHOR(S) +/* Wietse Venema +/* Google, Inc. +/* 111 8th Avenue +/* New York, NY 10011, USA +/*--*/ + + /* + * System library + */ +#include +#include +#include + + /* + * Utility library. + */ +#include +#include +#include + + /* + * Application-specific. + */ +#ifdef NO_64_BITS +#define FNV_prime 0x01000193UL +#define FNV_offset_basis 0x811c9dc5UL +#else +#define FNV_prime 0x00000100000001B3ULL +#define FNV_offset_basis 0xcbf29ce484222325ULL +#endif + +/* hash_fnv - modified FNV 1a hash */ + +HASH_FNV_T hash_fnv(const void *src, size_t len) +{ + static HASH_FNV_T basis = FNV_offset_basis; + static int randomize = 1; + HASH_FNV_T hash; + + /* + * Initialize. + */ + if (randomize) { + if (!getenv("NORANDOMIZE")) { + HASH_FNV_T seed; + + ldseed(&seed, sizeof(seed)); + basis ^= seed; + } + randomize = 0; + } + +#ifdef STRICT_FNV1A +#define FNV_NEXT_BYTE(s) ((HASH_FNV_T) * (const unsigned char *) s++) +#else +#define FNV_NEXT_BYTE(s) (1 + (HASH_FNV_T) * (const unsigned char *) s++) +#endif + + hash = basis; + while (len-- > 0) { + hash ^= FNV_NEXT_BYTE(src); + hash *= FNV_prime; + } + return (hash); +} diff --git a/postfix/src/util/hash_fnv.h b/postfix/src/util/hash_fnv.h new file mode 100644 index 000000000..19122aeef --- /dev/null +++ b/postfix/src/util/hash_fnv.h @@ -0,0 +1,39 @@ +#ifndef _HASH_FNV_H_INCLUDED_ +#define _HASH_FNV_H_INCLUDED_ + +/*++ +/* NAME +/* hash_fnv 3h +/* SUMMARY +/* Fowler/Noll/Vo hash function +/* SYNOPSIS +/* #include +/* DESCRIPTION +/* .nf + + /* + * External interface. + */ +#ifndef HASH_FNV_T +#include +#ifdef NO_64_BITS +#define HASH_FNV_T uint32_t +#else /* NO_64_BITS */ +#define HASH_FNV_T uint64_t +#endif /* NO_64_BITS */ +#endif /* HASH_FNV_T */ + +extern HASH_FNV_T hash_fnv(const void *, size_t); + +/* LICENSE +/* .ad +/* .fi +/* The Secure Mailer license must be distributed with this software. +/* AUTHOR(S) +/* Wietse Venema +/* Google, Inc. +/* 111 8th Avenue +/* New York, NY 10011, USA +/*--*/ + +#endif diff --git a/postfix/src/util/htable.c b/postfix/src/util/htable.c index 984203267..940d68699 100644 --- a/postfix/src/util/htable.c +++ b/postfix/src/util/htable.c @@ -58,12 +58,6 @@ /* values to be remembered are not character pointers, proper casts /* should be used or the code will not be portable. /* -/* To thwart collision attacks, the hash function is seeded -/* once from /dev/urandom, and if that is unavailable, from -/* wallclock-time and monotonic system clocks. To disable -/* seeding for tests, specify NORANDOMIZE in the environment -/* (the value does not matter). -/* /* htable_create() creates a table of the specified size and returns a /* pointer to the result. The lookup keys are saved with mystrdup(). /* htable_enter() stores a (key, value) pair into the specified table @@ -109,6 +103,7 @@ /* terminate immediately: memory allocation failure; an attempt /* to delete a non-existent entry. /* SEE ALSO +/* hash_fnv() Fowler/Noll/Vo hash function /* mymalloc(3) memory management wrapper /* LICENSE /* .ad @@ -119,18 +114,17 @@ /* IBM T.J. Watson Research /* P.O. Box 704 /* Yorktown Heights, NY 10598, USA +/* +/* Wietse Venema +/* Google, Inc. +/* 111 8th Avenue +/* New York, NY 10011, USA /*--*/ /* C library */ #include #include -#include -#include -#include -#include -#include -#include /* Local stuff */ @@ -138,112 +132,36 @@ #include "msg.h" #include "htable.h" - /* - * Fall back to a mix of absolute and time-since-boot information in the - * rare case that /dev/urandom is unavailable. - */ -#ifdef CLOCK_UPTIME -#define NON_WALLTIME_CLOCK CLOCK_UPTIME -#elif defined(CLOCK_BOOTTIME) -#define NON_WALLTIME_CLOCK CLOCK_BOOTTIME -#elif defined(CLOCK_MONOTONIC) -#define NON_WALLTIME_CLOCK CLOCK_MONOTONIC -#elif defined(CLOCK_HIGHRES) -#define NON_WALLTIME_CLOCK CLOCK_HIGHRES -#endif - -/* htable_seed - randomize the hash function */ - -static size_t htable_seed(void) -{ - uint32_t result = 0; - - /* - * Medium-quality seed, for defenses against local and remote attacks. - */ - int fd; - int count; - - if ((fd = open("/dev/urandom", O_RDONLY)) > 0) { - count = read(fd, &result, sizeof(result)); - (void) close(fd); - if (count == sizeof(result) && result != 0) - return (result); - } - - /* - * Low-quality seed, for defenses against remote attacks. Based on 1) the - * time since boot (good when an attacker knows the program start time - * but not the system boot time), and 2) absolute time (good when an - * attacker does not know the program start time). Assumes a system with - * better than microsecond resolution, and a network stack that does not - * leak the time since boot, for example, through TCP or ICMP timestamps. - * With those caveats, this seed is good for 20-30 bits of randomness. - */ -#ifdef NON_WALLTIME_CLOCK - { - struct timespec ts; - - if (clock_gettime(NON_WALLTIME_CLOCK, &ts) != 0) - msg_fatal("clock_gettime() failed: %m"); - result += (size_t) ts.tv_sec ^ (size_t) ts.tv_nsec; - } -#elif defined(USE_GETHRTIME) - result += gethrtime(); -#endif - -#ifdef CLOCK_REALTIME - { - struct timespec ts; - - if (clock_gettime(CLOCK_REALTIME, &ts) != 0) - msg_fatal("clock_gettime() failed: %m"); - result += (size_t) ts.tv_sec ^ (size_t) ts.tv_nsec; - } -#else - { - struct timeval tv; - - if (GETTIMEOFDAY(&tv) != 0) - msg_fatal("gettimeofday() failed: %m"); - result += (size_t) tv.tv_sec + (size_t) tv.tv_usec; - } -#endif - return (result + getpid()); -} - /* htable_hash - hash a string */ +#ifndef NO_HASH_FNV +#include "hash_fnv.h" + +#define htable_hash(s, size) (hash_fnv((s), strlen(s)) % (size)) + +#else + static size_t htable_hash(const char *s, size_t size) { - static size_t seed = 0; - static int randomize = 1; - size_t h; + size_t h = 0; size_t g; /* - * Initialize. + * From the "Dragon" book by Aho, Sethi and Ullman. */ - while (seed == 0 && randomize) { - if (getenv("NORANDOMIZE")) - randomize = 0; - else - seed = htable_seed(); - } - /* - * Heavily mutilated code based on the "Dragon" book by Aho, Sethi and - * Ullman. Updated to use a seed, to maintain 32+ bit state, and to make - * the distance between colliding inputs seed-dependent. - */ - h = seed; while (*s) { - g = h & 0xf0000000; - h = (h << 4U) ^ (((g >> 28U) + 1) * (*(unsigned const char *) s++) + 1); + h = (h << 4U) + *(unsigned const char *) s++; + if ((g = (h & 0xf0000000)) != 0) { + h ^= (g >> 24U); + h ^= g; + } } return (h % size); } +#endif + /* htable_link - insert element into table */ #define htable_link(table, element) { \ @@ -489,7 +407,6 @@ int main(int unused_argc, char **unused_argv) /* * Load a large number of strings and delete them in a random order. */ - msg_verbose = 1; hash = htable_create(10); while (vstring_get(buf, VSTREAM_IN) != VSTREAM_EOF) htable_enter(hash, vstring_str(buf), CAST_INT_TO_VOID_PTR(count++)); diff --git a/postfix/src/util/ldseed.c b/postfix/src/util/ldseed.c new file mode 100644 index 000000000..7bce66831 --- /dev/null +++ b/postfix/src/util/ldseed.c @@ -0,0 +1,137 @@ +/*++ +/* NAME +/* ldseed 3 +/* SUMMARY +/* seed for non-cryptographic applications +/* SYNOPSIS +/* #include +/* +/* void ldseed( +/* void *dst, +/* size_t len) +/* DESCRIPTION +/* ldseed() preferably extracts pseudo-random bits from +/* /dev/urandom, a non-blocking device that is available on +/* modern systems. +/* +/* On systems where /dev/urandom is unavailable or does not +/* immediately return the requested amount of randomness, +/* ldseed() falls back to a combination of wallclock time, +/* the time since boot, and the process ID. +/* BUGS +/* With Linux "the O_NONBLOCK flag has no effect when opening +/* /dev/urandom", but reads "can incur an appreciable delay +/* when requesting large amounts of data". Apparently, "large" +/* means more than 256 bytes. +/* LICENSE +/* .ad +/* .fi +/* The Secure Mailer license must be distributed with this software. +/* AUTHOR(S) +/* Wietse Venema +/* Google, Inc. +/* 111 8th Avenue +/* New York, NY 10011, USA +/*--*/ + + /* + * System library + */ +#include +#include +#include +#include +#include +#include +#include +#include /* CHAR_BIT */ + + /* + * Utility library. + */ +#include +#include + + /* + * Different systems have different names for non-wallclock time. + */ +#ifdef CLOCK_UPTIME +#define NON_WALLTIME_CLOCK CLOCK_UPTIME +#elif defined(CLOCK_BOOTTIME) +#define NON_WALLTIME_CLOCK CLOCK_BOOTTIME +#elif defined(CLOCK_MONOTONIC) +#define NON_WALLTIME_CLOCK CLOCK_MONOTONIC +#elif defined(CLOCK_HIGHRES) +#define NON_WALLTIME_CLOCK CLOCK_HIGHRES +#endif + +/* ldseed - best-effort, low-dependency seed */ + +void ldseed(void *dst, size_t len) +{ + int count; + int fd; + int n; + time_t fallback = 0; + + /* + * Medium-quality seed. + */ + if ((fd = open("/dev/urandom", O_RDONLY)) > 0) { + non_blocking(fd, NON_BLOCKING); + count = read(fd, dst, len); + (void) close(fd); + if (count == len) + return; + } + + /* + * Low-quality seed. Based on 1) the time since boot (good when an + * attacker knows the program start time but not the system boot time), + * and 2) absolute time (good when an attacker does not know the program + * start time). Assumes a system with better than microsecond resolution, + * and a network stack that does not leak the time since boot, for + * example, through TCP or ICMP timestamps. With those caveats, this seed + * is good for 20-30 bits of randomness. + */ +#ifdef NON_WALLTIME_CLOCK + { + struct timespec ts; + + if (clock_gettime(NON_WALLTIME_CLOCK, &ts) != 0) + msg_fatal("clock_gettime() failed: %m"); + fallback += ts.tv_sec ^ ts.tv_nsec; + } +#elif defined(USE_GETHRTIME) + fallback += gethrtime(); +#endif + +#ifdef CLOCK_REALTIME + { + struct timespec ts; + + if (clock_gettime(CLOCK_REALTIME, &ts) != 0) + msg_fatal("clock_gettime() failed: %m"); + fallback += ts.tv_sec ^ ts.tv_nsec; + } +#else + { + struct timeval tv; + + if (GETTIMEOFDAY(&tv) != 0) + msg_fatal("gettimeofday() failed: %m"); + fallback += tv.tv_sec + tv.tv_usec; + } +#endif + fallback += getpid(); + + /* + * Copy the least significant bytes first, because those are the most + * volatile. + */ + for (n = 0; n < sizeof(fallback) && n < len; n++) { + *(char *) dst++ ^= (fallback & 0xff); + fallback >>= CHAR_BIT; + } + return; +} diff --git a/postfix/src/util/ldseed.h b/postfix/src/util/ldseed.h new file mode 100644 index 000000000..891986e90 --- /dev/null +++ b/postfix/src/util/ldseed.h @@ -0,0 +1,30 @@ +#ifndef _LDSEED_H_INCLUDED_ +#define _LDSEED_H_INCLUDED_ + +/*++ +/* NAME +/* ldseed 3h +/* SUMMARY +/* seed for non-cryptographic applications +/* SYNOPSIS +/* #include +/* DESCRIPTION +/* .nf + + /* + * External interface. + */ +extern void ldseed(void *, size_t); + +/* LICENSE +/* .ad +/* .fi +/* The Secure Mailer license must be distributed with this software. +/* AUTHOR(S) +/* Wietse Venema +/* Google, Inc. +/* 111 8th Avenue +/* New York, NY 10011, USA +/*--*/ + +#endif