From 2bf7cc99908b18fc2018ae90c3c8e284f417df74 Mon Sep 17 00:00:00 2001 From: Wietse Venema Date: Fri, 27 Sep 2013 00:00:00 -0500 Subject: [PATCH] postfix-2.11-20130927 --- postfix/.indent.pro | 8 +- postfix/HISTORY | 39 +- postfix/README_FILES/INSTALL | 60 +-- postfix/README_FILES/LMDB_README | 104 +--- postfix/README_FILES/SASL_README | 3 +- postfix/README_FILES/STRESS_README | 156 +++--- postfix/RELEASE_NOTES | 8 + postfix/WISHLIST | 2 + postfix/html/INSTALL.html | 7 +- postfix/html/LMDB_README.html | 78 ++- postfix/html/SASL_README.html | 3 +- postfix/html/STRESS_README.html | 204 ++++---- postfix/html/ldap_table.5.html | 6 +- postfix/html/postconf.5.html | 15 +- postfix/html/postscreen.8.html | 10 +- postfix/man/man5/ldap_table.5 | 2 +- postfix/man/man5/postconf.5 | 14 +- postfix/man/man8/postscreen.8 | 10 +- postfix/mantools/postlink | 1 + postfix/proto/INSTALL.html | 7 +- postfix/proto/LMDB_README.html | 78 ++- postfix/proto/SASL_README.html | 3 +- postfix/proto/STRESS_README.html | 204 ++++---- postfix/proto/ldap_table | 2 +- postfix/proto/postconf.proto | 15 +- postfix/src/global/mail_params.h | 2 + postfix/src/global/mail_version.h | 2 +- postfix/src/postalias/postalias.c | 137 ++--- postfix/src/postmap/postmap.c | 80 +-- postfix/src/postscreen/Makefile.in | 5 +- postfix/src/postscreen/postscreen.c | 10 +- postfix/src/smtpd/Makefile.in | 5 +- postfix/src/smtpd/smtpd_check.c | 24 +- postfix/src/tlsmgr/Makefile.in | 5 +- postfix/src/util/Makefile.in | 2 + postfix/src/util/dict.h | 33 ++ postfix/src/util/dict_alloc.c | 24 +- postfix/src/util/dict_lmdb.c | 754 +++++++++++++++++++--------- postfix/src/util/dict_open.c | 26 + postfix/src/util/dict_test.c | 6 +- postfix/src/util/sys_defs.h | 100 ++-- 41 files changed, 1374 insertions(+), 880 deletions(-) diff --git a/postfix/.indent.pro b/postfix/.indent.pro index 8e82ff87a..61201ac29 100644 --- a/postfix/.indent.pro +++ b/postfix/.indent.pro @@ -1,3 +1,4 @@ +-TMDB_val -TABOUNCE -TADDR_MATCH_LIST -TADDR_PATTERN @@ -119,8 +120,8 @@ -TDSN_BUF -TDSN_SPLIT -TDSN_STAT --TEC_KEY -TEC_GROUP +-TEC_KEY -TEDIT_FILE -TEVENT_MASK -TEVP_PKEY @@ -176,6 +177,7 @@ -TMATCH_OPS -TMBLOCK -TMBOX +-TMDB_txn -TMILTER -TMILTER8 -TMILTERS @@ -330,10 +332,10 @@ -TWATCHDOG -TWATCH_FD -TX509 +-TX509V3_CTX -TX509_EXTENSION -TX509_NAME -TX509_STORE_CTX --TX509V3_CTX -TXSASL_CLIENT -TXSASL_CLIENT_CREATE_ARGS -TXSASL_CLIENT_IMPL @@ -358,9 +360,9 @@ -Tsasl_secret_t -Tsfsistat -Tsize_t +-Tssize_t -Tssl_cipher_stack_t -Tssl_comp_stack_t --Tssize_t -Ttime_t -Tx509_extension_stack_t -Tx509_stack_t diff --git a/postfix/HISTORY b/postfix/HISTORY index 187107bd4..1e58fa1c9 100644 --- a/postfix/HISTORY +++ b/postfix/HISTORY @@ -18719,10 +18719,11 @@ Apologies for any names omitted. 20130615 - Interoperability: turn on SHA-2XX digests by force. This - improves interoperability with clients and servers with - ancient OpenSSL versions and that that prematurely deploy - SHA-2 certificates. Viktor Dukhovni. File: tls/tls_misc.c + TLS Interoperability: turn on SHA-2 digests by force. This + improves interoperability with clients and servers that + deploy SHA-2 digests without the required support for + TLSv1.2-style digest negotiation. Based on patch by Viktor + Dukhovni. Files: tls/tls_client.c, tls/tls_server.c. 20130616 @@ -18912,3 +18913,33 @@ Apologies for any names omitted. src/smtp/smtp_addr.h, src/smtp/smtp_connect.c, src/smtp/smtp_params.c, src/smtp/smtp_tls_policy.c, src/tls/tls.h, src/tls/tls_dane.c. + +20130826 + + Documentation: re-ordered STRESS_README, now that all + supported releases have stress-adaptive behavior built in. + File: proto/STRESS_README.html. + +20130903 + + Cleanup: made the default_database_type compile-time + configurable. Files: util/sys_defs.h, makedefs, proto/INSTALL. + +20130916 + + Feature: reject_known_sender_login_mismatch, which applies + reject_sender_login_mismatch only to MAIL FROM addresses + that are known in $smtpd_sender_login_maps. Viktor & Wietse. + Files: mantools/postlink, proto/SASL_README.html, + proto/postconf.proto, global/mail_params.h, smtpd/smtpd_check.c. + +20130927 + + Cleanup: no more LMDB "database full" errors. Postfix now + requires LMDB >= 0.9.8 which supports on-the-fly database + resizing. When a database becomes full, its size limit is + automatically doubled, and other processes automatically + pick up the new database size limit. Files: util/dict.h, + util/dict_open.c, util/dict_alloc.c, util/dict_lmdb.c, + postmap/postmap.c, postalias/postalias.c, proto/LMDB_README.html, + proto/postconf.proto. diff --git a/postfix/README_FILES/INSTALL b/postfix/README_FILES/INSTALL index 9ed6a0541..b04f27d7c 100644 --- a/postfix/README_FILES/INSTALL +++ b/postfix/README_FILES/INSTALL @@ -194,31 +194,33 @@ IMPORTANT: Be sure to get the quotes right. These details matter a lot. Parameters whose defaults can be specified in this way are: - _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ - |MMaaccrroo nnaammee |ddeeffaauulltt vvaalluuee ffoorr|ttyyppiiccaall ddeeffaauulltt | - |_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _|_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _|_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ | - |DEF_COMMAND_DIR |command_directory|/usr/sbin | - |_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _|_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _|_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ | - |DEF_CONFIG_DIR |config_directory |/etc/postfix | - |_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _|_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _|_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ | - |DEF_DAEMON_DIR |daemon_directory |/usr/libexec/postfix| - |_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _|_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _|_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ | - |DEF_DATA_DIR |data_directory |/var/lib/postfix | - |_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _|_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _|_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ | - |DEF_MAILQ_PATH |mailq_path |/usr/bin/mailq | - |_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _|_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _|_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ | - |DEF_HTML_DIR |html_directory |no | - |_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _|_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _|_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ | - |DEF_MANPAGE_DIR |manpage_directory|/usr/local/man | - |_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _|_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _|_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ | - |DEF_NEWALIAS_PATH|newaliases_path |/usr/bin/newaliases | - |_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _|_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _|_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ | - |DEF_QUEUE_DIR |queue_directory |/var/spool/postfix | - |_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _|_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _|_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ | - |DEF_README_DIR |readme_directory |no | - |_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _|_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _|_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ | - |DEF_SENDMAIL_PATH|sendmail_path |/usr/sbin/sendmail | - |_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _|_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _|_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ | + _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ + |MMaaccrroo nnaammee |ddeeffaauulltt vvaalluuee ffoorr |ttyyppiiccaall ddeeffaauulltt | + |_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _|_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _|_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ | + |DEF_COMMAND_DIR |command_directory |/usr/sbin | + |_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _|_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _|_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ | + |DEF_CONFIG_DIR |config_directory |/etc/postfix | + |_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _|_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _|_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ | + |DEF_DB_TYPE |default_database_type|hash | + |_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _|_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _|_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ | + |DEF_DAEMON_DIR |daemon_directory |/usr/libexec/postfix| + |_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _|_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _|_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ | + |DEF_DATA_DIR |data_directory |/var/lib/postfix | + |_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _|_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _|_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ | + |DEF_MAILQ_PATH |mailq_path |/usr/bin/mailq | + |_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _|_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _|_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ | + |DEF_HTML_DIR |html_directory |no | + |_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _|_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _|_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ | + |DEF_MANPAGE_DIR |manpage_directory |/usr/local/man | + |_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _|_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _|_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ | + |DEF_NEWALIAS_PATH|newaliases_path |/usr/bin/newaliases | + |_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _|_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _|_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ | + |DEF_QUEUE_DIR |queue_directory |/var/spool/postfix | + |_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _|_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _|_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ | + |DEF_README_DIR |readme_directory |no | + |_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _|_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _|_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ | + |DEF_SENDMAIL_PATH|sendmail_path |/usr/sbin/sendmail | + |_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _|_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _|_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ | Note: the data_directory parameter (for caches and pseudo-random numbers) was introduced with Postfix version 2.5. @@ -247,9 +249,11 @@ The following is an extensive list of names and values. | |at compile time: | |_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _|_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ | || |Do not build with Berkeley DB support. By | -||-DNO_DB |default, Berkeley DB support is compiled in on| -|| |platforms that are known to support this | -|| |feature. | +|| |default, Berkeley DB support is compiled in on| +||-DNO_DB |platforms that are known to support this | +|| |feature. If you override this, then you | +|| |probably should also override DEF_DB_TYPE as | +|| |described in section 4.4. | |_|_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _|_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ | || |Do not build with Solaris /dev/poll support. | ||-DNO_DEVPOLL |By default, /dev/poll support is compiled in | diff --git a/postfix/README_FILES/LMDB_README b/postfix/README_FILES/LMDB_README index fa7e80eb1..5d3baeb17 100644 --- a/postfix/README_FILES/LMDB_README +++ b/postfix/README_FILES/LMDB_README @@ -4,14 +4,6 @@ PPoossttffiixx OOppeennLLDDAAPP LLMMDDBB HHoowwttoo IInnttrroodduuccttiioonn - WWaarrnniinngg:: LLMMDDBB ddaattaabbaasseess hhaavvee aa sshhooww--ssttooppppeerr bbuugg:: tthheeyy ddoo nnoott ggrrooww bbeeyyoonndd aa - ssppeecciiffiieedd lliimmiitt,, aanndd iinnttrroodduuccee aa ""ddaattaabbaassee ffuullll"" hhaarrdd eerrrroorr ccoonnddiittiioonn tthhaatt - ddooeess nnoott eexxiisstt wwiitthh aannyy ootthheerr PPoossttffiixx ddaattaabbaassee ttyyppee.. TThhiiss iiss aa pprroobblleemm ffoorr - pprrooggrraammss wwhhoossee ddaattaabbaassee ggrroowwss wwiitthh ssyysstteemm llooaadd.. EExxaammpplleess aarree ppoossttssccrreeeenn((88)),, - ggrreeyylliissttiinngg,, vveerriiffyy((88)) aanndd ttllssmmggrr((88)).. - - YYoouu hhaavvee bbeeeenn wwaarrnneedd.. - Postfix uses databases of various kinds to store and look up information. Postfix databases are specified as "type:name". OpenLDAP LMDB implements the Postfix database type "lmdb". The name of a Postfix OpenLDAP LMDB database is @@ -30,14 +22,14 @@ This document describes: BBuuiillddiinngg PPoossttffiixx wwiitthh OOppeennLLDDAAPP LLMMDDBB ssuuppppoorrtt -Postfix normally does not enable OpenLDAP LMDB support. To build Postfix after -you installed OpenLDAP LMDB from source code, use something like: +Postfix normally does not enable OpenLDAP LMDB support. To build Postfix with +OpenLDAP LMDB support, use something like: % make makefiles CCARGS="-DHAS_LMDB -I/usr/local/include" \ AUXLIBS="-L/usr/local/lib -llmdb" % make -Solaris needs this: +Solaris may need this: % make makefiles CCARGS="-DHAS_LMDB -I/usr/local/include" \ AUXLIBS="-R/usr/local/lib -L/usr/local/lib -llmdb" @@ -47,12 +39,12 @@ The exact pathnames depend on how OpenLDAP LMDB was installed. CCoonnffiigguurree LLMMDDBB sseettttiinnggss -Postfix provides a configuration parameter that controls how large an OpenLDAP -LMDB database may grow. +Postfix provides one configuration parameter that controls OpenLDAP LMDB +database behavior. - * lmdb_map_size (default: 16 MBytes per table). This setting controls how - large any OpenLDAP LMDB database may grow. It must be set large enough to - accommodate the largest table that Postfix will use. + * lmdb_map_size (default: 16777216). This setting specifies the initial + OpenLDAP LMDB database size limit in bytes. Each time a database becomes + full, its size limit is doubled. MMiissssiinngg pptthhrreeaadd lliibbrraarryy ttrroouubbllee @@ -72,81 +64,17 @@ information is available at http://highlandsun.com/hyc/mdb/. UUnneexxppeecctteedd ffaaiilluurree mmooddeess ooff PPoossttffiixx LLMMDDBB ddaattaabbaasseess.. As documented below, conversion to LMDB introduces a number of failure modes -that don't exist with other Postfix databases. - -UUnneexxppeecctteedd ppoossttmmaapp((11))//ppoossttaalliiaass((11)) ""ddaattaabbaassee ffuullll"" eerrrroorrss.. - -Problem: - The "postmap lmdb:filename" command fails with an MDB_MAP_FULL error. This - problem does not exist with other Postfix databases. - -Background: - LMDB databases have a hard size limit (configured with the lmdb_map_size - configuration parameter). - - When executing "postmap lmdb:filename", the Postfix LMDB database client - stores the new data in a transaction which takes up space in addition to - the existing data, and commits the transaction when it closes the database. - Only then can the space for old data be reused. - -Impact: - This failure does not affect Postfix availability, because the old data - still exists in the database. - -Mitigation: - When the postmap(1) or postalias(1) command fails with an MDB_MAP_FULL - error, it expands the database file size to the current LMDB map size limit - before terminating. - - When the postmap(1) or postalias(1) command opens an LMDB file larger than - lmdb_map_size/3, it logs a warning and uses a larger size limit instead: - - warning: filename.lmdb: file size 15024128 >= (lmdb map size limit - 16777216)/3 -- using a larger map size limit - - The two steps above can be used to automate recovery and avoid the need for - human intervention. Just repeat "postmap lmdb:filename" (up to some limit). - After each failure it will use a 3x larger size limit, and eventually the - "database full" error will disappear. - -Prevention: - Monitor your LMDB files and make sure that lmdb_map_size > 3x the largest - LMDB file size. - -UUnneexxppeecctteedd PPoossttffiixx ddaaeemmoonn ""ddaattaabbaassee ffuullll"" eerrrroorrss.. - -Problem: - Postfix daemon programs fail with "database full" errors, such as - postscreen(8), tlsmgr(8) or verify(8). This problem does not exist with - other Postfix databases. - -Impact: - This failure temporarily affects Postfix availability. The daemon restarts - automatically and tries to open the database again as described next. - -Mitigation: - When a Postfix daemon opens an LMDB file larger than lmdb_map_size/3, it - logs a warning and uses a larger size limit instead: - - warning: filename.lmdb: file size 15024128 >= (lmdb map size limit - 16777216)/3 -- using a larger map size limit - - This can be used to automate recovery and avoid the need for human - intervention. Each time the daemon runs into a "database full" error, it - restarts and uses a 3x larger size limit. The "database full" error will - disappear, at least for a while. - -Prevention: - Monitor your LMDB files and make sure that lmdb_map_size > 3x the largest - LMDB file size. +that don't exist with other Postfix databases. Some failure modes have been +eliminated on the course of time. The writeup below reflects the status as of +of LMDB 0.9.8. NNoonn--oobbvviioouuss rreeccoovveerryy wwiitthh ppoossttmmaapp((11))//ppoossttaalliiaass((11))//ttllssmmggrr((88)) ffrroomm aa ccoorrrruupptteedd ddaattaabbaassee.. Problem: - You cannot rebuild a corrupted LMDB database simply by running postmap(1) - or postalias(1), or by waiting until the tlsmgr(8) daemon restarts - automatically. This problem does not exist with other Postfix databases. + You cannot rebuild a corrupted LMDB database simply by re-running postmap + (1) or postalias(1), or by waiting until the tlsmgr(8) daemon restarts. + This problem does not exist with other Postfix databases. Background: The Postfix LMDB database client does not truncate the database file. @@ -158,9 +86,9 @@ Impact: Postfix does not process mail until someone fixes the problem. Recovery: - First delete the ".lmdb" file by hand, then rebuild the file with the + First delete the ".lmdb" file by hand. Then, rebuild the file with the postmap(1) or postalias(1) command, or wait until the tlsmgr(8) daemon - restarts automatically. + restarts. Prevention: Arrange your file systems such that they never run out of free space. diff --git a/postfix/README_FILES/SASL_README b/postfix/README_FILES/SASL_README index 309ee9ac4..ffd51132e 100644 --- a/postfix/README_FILES/SASL_README +++ b/postfix/README_FILES/SASL_README @@ -896,7 +896,8 @@ With this, the reject_sender_login_mismatch restriction above will reject the sender address in the MAIL FROM command if smtpd_sender_login_maps does not specify the SMTP client's login name as an owner of that address. -See also reject_authenticated_sender_login_mismatch and +See also reject_authenticated_sender_login_mismatch, +reject_known_sender_login_mismatch, and reject_unauthenticated_sender_login_mismatch for additional control over the SASL login name and the envelope sender. diff --git a/postfix/README_FILES/STRESS_README b/postfix/README_FILES/STRESS_README index 8e61d3318..7f407a064 100644 --- a/postfix/README_FILES/STRESS_README +++ b/postfix/README_FILES/STRESS_README @@ -13,11 +13,11 @@ stress-adaptive behavior, and for earlier Postfix versions that don't. Topics covered in this document: * Symptoms of Postfix SMTP server overload + * Automatic stress-adaptive behavior * Service more SMTP clients at the same time * Spend less time per SMTP client * Disconnect suspicious SMTP clients * Temporary measures for older Postfix releases - * Automatic stress-adaptive behavior * Detecting support for stress-adaptive behavior * Forcing stress-adaptive behavior on or off * Other measures to off-load zombies @@ -65,13 +65,101 @@ Symptoms of Postfix SMTP server overload are: Oct 3 20:39:27 spike postfix/master[28905]: warning: to avoid this condition, increase the process count in master.cf or reduce the service time per client + Oct 3 20:39:27 spike postfix/master[28905]: warning: see + http://www.postfix.org/STRESS_README.html for examples of + stress-adapting configuration settings Legitimate mail that doesn't get through during an episode of Postfix SMTP server overload is not necessarily lost. It should still arrive once the situation returns to normal, as long as the overload condition is temporary. +AAuuttoommaattiicc ssttrreessss--aaddaappttiivvee bbeehhaavviioorr + +Postfix version 2.5 introduces automatic stress-adaptive behavior. It works as +follows. When a "public" network service such as the SMTP server runs into an +"all server ports are busy" condition, the Postfix master(8) daemon logs a +warning, restarts the service (without interrupting existing network sessions), +and runs the service with "-o stress=yes" on the server process command line: + + 80821 ?? S 0:00.24 smtpd -n smtp -t inet -u -c -o stress=yes + +Normally, the Postfix master(8) daemon runs such a service with "-o stress=" on +the command line (i.e. with an empty parameter value): + + 83326 ?? S 0:00.28 smtpd -n smtp -t inet -u -c -o stress= + +Services that have local access only never have "-o stress" parameters on the +command line. This includes services internal to Postfix such as the queue +manager, and services that listen on a loopback interface only, such as after- +filter SMTP services. + +The "stress" parameter value is the key to making main.cf parameter settings +stress adaptive. The following settings are the default with Postfix 2.6 and +later. + + 1 smtpd_timeout = ${stress?10}${stress:300}s + 2 smtpd_hard_error_limit = ${stress?1}${stress:20} + 3 smtpd_junk_command_limit = ${stress?1}${stress:100} + 4 # Parameters added after Postfix 2.6: + 5 smtpd_per_record_deadline = ${stress?yes}${stress:no} + 6 smtpd_starttls_timeout = ${stress?10}${stress:300}s + 7 address_verify_poll_count = ${stress?1}${stress:3} + +Translation: + + * Line 1: under conditions of stress, use an smtpd_timeout value of 10 + seconds instead of the default 300 seconds. Experience on the postfix-users + list from a variety of sysadmins shows that reducing the "normal" + smtpd_timeout to 60s is unlikely to affect legitimate clients. However, it + is unlikely to become the Postfix default because it's not RFC compliant. + Setting smtpd_timeout to 10s or even 5s under stress will still allow most + legitimate clients to connect and send mail, but may delay mail from some + clients. No mail should be lost, as long as this measure is used only + temporarily. + + * Line 2: under conditions of stress, use an smtpd_hard_error_limit of 1 + instead of the default 20. This helps by disconnecting clients after a + single error, giving other clients a chance to connect. However, this may + cause significant delays with legitimate mail, such as a mailing list that + contains a few no-longer-active user names that didn't bother to + unsubscribe. No mail should be lost, as long as this measure is used only + temporarily. + + * Line 3: under conditions of stress, use an smtpd_junk_command_limit of 1 + instead of the default 100. This prevents clients from keeping connections + open by repeatedly sending HELO, EHLO, NOOP, RSET, VRFY or ETRN commands. + + * Line 5: under conditions of stress, change the behavior of smtpd_timeout + and smtpd_starttls_timeout, from a time limit per read or write system + call, to a time limit to send or receive a complete record (an SMTP command + line, SMTP response line, SMTP message content line, or TLS protocol + message). + + * Line 6: under conditions of stress, reduce the time limit for TLS protocol + handshake messages to 10 seconds, from the default value of 300 seconds. + See also the smtpd_timeout discussion above. + + * Line 7: under conditions of stress, do not wait up to 6 seconds for the + completion of an address verification probe. If the result is not already + in the address verification cache, reply immediately with + $unverified_recipient_tempfail_action or + $unverified_sender_tempfail_action. No mail should be lost, as long as this + measure is used only temporarily. + +The syntax of ${name?value} and ${name:value} is explained at the beginning of +the postconf(5) manual page. + +NOTE: Please keep in mind that the stress-adaptive feature is a fairly +desperate measure to keep ssoommee legitimate mail flowing under overload +conditions. If a site is reaching the SMTP server process limit when there +isn't an attack or bot flood occurring, then either the process limit needs to +be raised or more hardware needs to be added. + SSeerrvviiccee mmoorree SSMMTTPP cclliieennttss aatt tthhee ssaammee ttiimmee +This section and the ones that follow discuss permanent measures against mail +server overload. + One measure to avoid the "all server processes busy" condition is to service more SMTP clients simultaneously. For this you need to increase the number of Postfix SMTP server processes. This will improve the responsiveness for remote @@ -256,72 +344,6 @@ With these measures, no mail should be lost, as long as these measures are used only temporarily. The next section of this document introduces a way to automate this process. -AAuuttoommaattiicc ssttrreessss--aaddaappttiivvee bbeehhaavviioorr - -Postfix version 2.5 introduces automatic stress-adaptive behavior. This is also -available as a source code patch for Postfix versions 2.4 and 2.3 from the -mirrors listed at http://www.postfix.org/download.html. - -It works as follows. When a "public" network service such as the SMTP server -runs into an "all server ports are busy" condition, the Postfix master(8) -daemon logs a warning, restarts the service (without interrupting existing -network sessions), and runs the service with "-o stress=yes" on the server -process command line: - - 80821 ?? S 0:00.24 smtpd -n smtp -t inet -u -c -o stress=yes - -Normally, the Postfix master(8) daemon runs such a service with "-o stress=" on -the command line (i.e. with an empty parameter value): - - 83326 ?? S 0:00.28 smtpd -n smtp -t inet -u -c -o stress= - -Services that have local access only never have "-o stress" parameters on the -command line. This includes services internal to Postfix such as the queue -manager, and services that listen on a loopback interface only, such as after- -filter SMTP services. - -The "stress" parameter value is the key to making main.cf parameter settings -stress adaptive. The following settings are the default with Postfix 2.6 and -later. With earlier Postfix versions that have stress-adaptive support, append -the lines below to the main.cf file and issue a "postfix reload" command: - - 1 smtpd_timeout = ${stress?10}${stress:300}s - 2 smtpd_hard_error_limit = ${stress?1}${stress:20} - 3 smtpd_junk_command_limit = ${stress?1}${stress:100} - -Translation: - - * Line 1: under conditions of stress, use an smtpd_timeout value of 10 - seconds instead of the default 300 seconds. Experience on the postfix-users - list from a variety of sysadmins shows that reducing the "normal" - smtpd_timeout to 60s is unlikely to affect legitimate clients. However, it - is unlikely to become the Postfix default because it's not RFC compliant. - Setting smtpd_timeout to 10s (line 2 below) or even 5s under stress will - still allow most legitimate clients to connect and send mail, but may delay - mail from some clients. No mail should be lost, as long as this measure is - used only temporarily. - - * Line 2: under conditions of stress, use an smtpd_hard_error_limit of 1 - instead of the default 20. This helps by disconnecting clients after a - single error, giving other clients a chance to connect. However, this may - cause significant delays with legitimate mail, such as a mailing list that - contains a few no-longer-active user names that didn't bother to - unsubscribe. No mail should be lost, as long as this measure is used only - temporarily. - - * Line 3: under conditions of stress, use an smtpd_junk_command_limit of 1 - instead of the default 100. This prevents clients from keeping idle - connections open by repeatedly sending NOOP or RSET commands. - -The syntax of ${name?value} and ${name:value} is explained at the beginning of -the postconf(5) manual page. - -NOTE: Please keep in mind that the stress-adaptive feature is a fairly -desperate measure to keep ssoommee legitimate mail flowing under overload -conditions. If a site is reaching the SMTP server process limit when there -isn't an attack or bot flood occurring, then either the process limit needs to -be raised or more hardware needs to be added. - DDeetteeccttiinngg ssuuppppoorrtt ffoorr ssttrreessss--aaddaappttiivvee bbeehhaavviioorr To find out if your Postfix installation supports stress-adaptive behavior, use diff --git a/postfix/RELEASE_NOTES b/postfix/RELEASE_NOTES index 4e204ae0d..36443c293 100644 --- a/postfix/RELEASE_NOTES +++ b/postfix/RELEASE_NOTES @@ -14,6 +14,14 @@ specifies the release date of a stable release or snapshot release. If you upgrade from Postfix 2.9 or earlier, read RELEASE_NOTES-2.10 before proceeding. +Major changes with snapshot 20130927 +==================================== + +Postfix now handles LMDB "database full" errors automatically. When +a database becomes full, its size limit is doubled, and other +processes automatically pick up the new size limit. The lmdb_map_size +parameter is now mostly irrelevant, and may be removed in the future. + Major changes with snapshot 20130602 ==================================== diff --git a/postfix/WISHLIST b/postfix/WISHLIST index 927544ef0..a40d6cf50 100644 --- a/postfix/WISHLIST +++ b/postfix/WISHLIST @@ -16,6 +16,8 @@ Wish list: Begin code revision, after DANE support stabilizes. This should be one pass that changes only names and no code. + recipient_delimiters = $recipient_delimiter for BC + All source code must specify its original author and license statement. Some code modules specify Lutz Jaenicke as the original author and fall under his liberal license. diff --git a/postfix/html/INSTALL.html b/postfix/html/INSTALL.html index d4fd3a287..22c8d06d4 100644 --- a/postfix/html/INSTALL.html +++ b/postfix/html/INSTALL.html @@ -310,6 +310,9 @@ default DEF_CONFIG_DIR config_directory /etc/postfix + DEF_DB_TYPE default_database_type +hash + DEF_DAEMON_DIR daemon_directory /usr/libexec/postfix @@ -376,7 +379,9 @@ off Postfix features at compile time: -DNO_DB Do not build with Berkeley DB support. By default, Berkeley DB support is compiled in on -platforms that are known to support this feature. +platforms that are known to support this feature. If you override +this, then you probably should also override DEF_DB_TYPE as described +in section 4.4. -DNO_DEVPOLL Do not build with Solaris /dev/poll support. By default, /dev/poll diff --git a/postfix/html/LMDB_README.html b/postfix/html/LMDB_README.html index 81d1484db..2c6dbb597 100644 --- a/postfix/html/LMDB_README.html +++ b/postfix/html/LMDB_README.html @@ -19,14 +19,6 @@

Introduction

-

Warning: LMDB databases have a show-stopper -bug: they do not grow beyond a specified limit, and introduce a -"database full" hard error condition that does not exist with any -other Postfix database type. This is a problem for programs whose -database grows with system load. Examples are postscreen(8), -greylisting, verify(8) and tlsmgr(8).

-You have been warned.

-

Postfix uses databases of various kinds to store and look up information. Postfix databases are specified as "type:name". OpenLDAP LMDB implements the Postfix database type "lmdb". @@ -52,9 +44,8 @@ don't exist with other Postfix databases.

Building Postfix with OpenLDAP LMDB support

-

Postfix normally does not enable OpenLDAP LMDB support. -To build Postfix after you installed OpenLDAP LMDB from -source code, use something like:

+

Postfix normally does not enable OpenLDAP LMDB support. To +build Postfix with OpenLDAP LMDB support, use something like: 

@@ -64,7 +55,7 @@ source code, use something like:
-

Solaris needs this:

+

Solaris may need this: 

@@ -78,15 +69,14 @@ source code, use something like: 

 
 
Configure LMDB settings

 
-
 Postfix provides a configuration parameter that controls how
-large an OpenLDAP LMDB database may grow. 

+
 Postfix provides one configuration parameter that controls 
+OpenLDAP LMDB database behavior. 

 
 
    
 
-
  •  
     lmdb_map_size (default: 16 MBytes per table).  This setting
-controls how large any OpenLDAP LMDB database may grow. It must be
-set large enough to accommodate the largest table that Postfix will
-use. 
    
+
  •  
     lmdb_map_size (default: 16777216).  This setting specifies
+the initial OpenLDAP LMDB database size limit in bytes.  Each time
+a database becomes full, its size limit is doubled.  
    
 
 

 
@@ -119,15 +109,17 @@ More information is available at
 databases.   
 
 
 As documented below, conversion to LMDB introduces a number of
-failure modes that don't exist with other Postfix databases. 

+failure modes that don't exist with other Postfix databases.  Some
+failure modes have been eliminated on the course of time.
+The writeup below reflects the status as of of LMDB 0.9.8. 

+
+
-
 
 Problem: 
 
 
 The "postmap lmdb:filename" command
 fails with an MDB_MAP_FULL error.  This problem does not exist with
 other Postfix databases. 
 

@@ -177,23 +167,25 @@ availability, because the old data still exists in the database.
 MDB_MAP_FULL error, it expands the database file size to the current
 LMDB map size limit before terminating.  

 
-
 When the postmap(1) or postalias(1) command opens an LMDB file
-larger than lmdb_map_size/3, it logs a warning and uses a larger
-size limit instead: 

+
 Next, when you re-run the postmap(1) or postalias(1) command,
+it discovers that the LMDB file is larger than lmdb_map_size/3,
+logs a warning, and uses a larger LMDB map size limit instead: 

 
 
  warning: filename.lmdb: file size 15024128 ≥
-(lmdb map size limit 16777216)/3 -- using a larger map size limit
-

+(lmdb map size limit 16777216)/3
 warning: filename.lmdb:
+using map size limit 45072384 

 
-
 The two steps above can be used to automate recovery and avoid
-the need for human intervention. Just repeat "postmap lmdb:filename"
-(up to some limit).  After each failure it will use a 3x larger
-size limit, and eventually the "database full" error will disappear.
-

+
 By repeating the two steps above you can automate recovery and
+avoid the need for human intervention. Just repeat "postmap
+lmdb:filename" (up to some limit).  After each failure it will use
+a 3x larger size limit, and eventually the "database full" error
+should disappear. This fails only when the disk is full or when
+the LMDB map size limit would exceed the memory address space size
+limit.  

 
 
 Prevention: 
 
 
 Monitor your LMDB files and make
-sure that lmdb_map_size > 3x the largest LMDB file size. 

-
 
+sure that in main.cf, lmdb_map_size > 3x the largest LMDB file
+size. 
  
 
 
 Unexpected Postfix daemon "database full" errors.
 

@@ -214,8 +206,8 @@ file larger than lmdb_map_size/3, it
 larger size limit instead: 

 
 
  warning: filename.lmdb: file size 15024128 ≥
-(lmdb map size limit 16777216)/3 -- using a larger map size limit
-

+(lmdb map size limit 16777216)/3 
warning: filename.lmdb:
+using map size limit 45072384 

 
 
 This can be used to automate recovery and avoid the need for
 human intervention. Each time the daemon runs into a "database full"
@@ -226,14 +218,16 @@ full" error will disappear, at least for a while.  

 sure that lmdb_map_size > 3x the largest LMDB file size. 

  
 
+-->
+
 
 Non-obvious recovery with postmap(1)/postalias(1)/tlsmgr(8)
 from a corrupted database.  

 
 

 
 
 Problem: 
 
 
 You cannot rebuild a corrupted LMDB
-database simply by running postmap(1) or postalias(1), or by waiting
-until the tlsmgr(8) daemon restarts automatically.  This problem
+database simply by re-running postmap(1) or postalias(1), or by
+waiting until the tlsmgr(8) daemon restarts.  This problem
 does not exist with other Postfix databases.  
 

 
 
 Background: 
 
 
 The Postfix LMDB database client
@@ -245,9 +239,9 @@ That is obviously not possible with a corrupted database file. 

 
 Impact: 
 
 
 Postfix does not process mail until
 someone fixes the problem.  
 

 
-
 Recovery: 
 
 
 First delete the ".lmdb" file by hand,
-then rebuild the file with the postmap(1) or postalias(1) command,
-or wait until the tlsmgr(8) daemon restarts automatically.  

+
 Recovery: 
 
 
 First delete the ".lmdb" file by hand.
+Then, rebuild the file with the postmap(1) or postalias(1) command,
+or wait until the tlsmgr(8) daemon restarts.  

 

 
 
 Prevention: 
 
 
diff --git a/postfix/html/SASL_README.html b/postfix/html/SASL_README.html
index ec5cb84ca..408ad2e4f 100644
--- a/postfix/html/SASL_README.html
+++ b/postfix/html/SASL_README.html
@@ -1450,7 +1450,8 @@ restriction above will reject the sender address in the MAIL FROM
 command if smtpd_sender_login_maps does not specify
 the SMTP client's login name as an owner of that address. 

 
-
 See also reject_authenticated_sender_login_mismatch and
+
 See also reject_authenticated_sender_login_mismatch,
+reject_known_sender_login_mismatch, and
 reject_unauthenticated_sender_login_mismatch for additional
 control over the SASL login name and the envelope sender. 

 
diff --git a/postfix/html/STRESS_README.html b/postfix/html/STRESS_README.html
index bd4c1b833..8dbfa3f7d 100644
--- a/postfix/html/STRESS_README.html
+++ b/postfix/html/STRESS_README.html
@@ -33,6 +33,8 @@ and for earlier Postfix versions that don't.  

 
 
  •  Symptoms of Postfix SMTP server overload  
 
+
  •  Automatic stress-adaptive behavior 
+
 
  •  Service more SMTP clients at the same time  
 
 
  •  Spend less time per SMTP client 
@@ -41,8 +43,6 @@ and for earlier Postfix versions that don't.  
    
 
 
  •  Temporary measures for older Postfix releases 
 
-
  •  Automatic stress-adaptive behavior 
-
 
  •  Detecting support for stress-adaptive behavior 
 
 
  •  Forcing stress-adaptive behavior on or off 
@@ -109,6 +109,9 @@ Oct  3 20:39:27 spike postfix/master[28905]: warning: service "smtp"
 Oct  3 20:39:27 spike postfix/master[28905]: warning: to avoid this
  condition, increase the process count in master.cf or reduce the
  service time per client
+Oct  3 20:39:27 spike postfix/master[28905]: warning: see
+  http://www.postfix.org/STRESS_README.html for examples of
+  stress-adapting configuration settings
    • @@ -118,8 +121,116 @@ Postfix SMTP server overload is not necessarily lost. It should still arrive once the situation returns to normal, as long as the overload condition is temporary.

    +

    Automatic stress-adaptive behavior

    + +

    Postfix version 2.5 introduces automatic stress-adaptive behavior. +It works as follows. When a "public" network service such as the +SMTP server runs into an "all server ports are busy" condition, the +Postfix master(8) daemon logs a warning, restarts the service +(without interrupting existing network sessions), and runs the +service with "-o stress=yes" on the server process command line: +

    + +
    +
    +80821  ??  S      0:00.24 smtpd -n smtp -t inet -u -c -o stress=yes
+
    +
    + +

    Normally, the Postfix master(8) daemon runs such a service with +"-o stress=" on the command line (i.e. with an empty parameter +value):

    + +
    +
    +83326  ??  S      0:00.28 smtpd -n smtp -t inet -u -c -o stress=
+
    +
    + +

    Services that have local access only never have "-o stress" +parameters on the command line. This includes services internal to +Postfix such as the queue manager, and services that listen on a +loopback interface only, such as after-filter SMTP services.

    + +

    The "stress" parameter value is the key to making main.cf +parameter settings stress adaptive. The following settings are the +default with Postfix 2.6 and later.

    + +
    +
    +1 smtpd_timeout = ${stress?10}${stress:300}s
+2 smtpd_hard_error_limit = ${stress?1}${stress:20}
+3 smtpd_junk_command_limit = ${stress?1}${stress:100}
+4 # Parameters added after Postfix 2.6:
+5 smtpd_per_record_deadline = ${stress?yes}${stress:no}
+6 smtpd_starttls_timeout = ${stress?10}${stress:300}s
+7 address_verify_poll_count = ${stress?1}${stress:3}
+
    +
    + +

    Translation:

    + +

    + +

    The syntax of ${name?value} and ${name:value} is explained at +the beginning of the postconf(5) manual page.

    + +

    NOTE: Please keep in mind that the stress-adaptive feature is +a fairly desperate measure to keep some legitimate mail +flowing under overload conditions. If a site is reaching the SMTP +server process limit when there isn't an attack or bot flood +occurring, then either the process limit needs to be raised or more +hardware needs to be added.

    +

    Service more SMTP clients at the same time

    +

    This section and the ones that follow discuss permanent measures +against mail server overload.

    +

    One measure to avoid the "all server processes busy" condition is to service more SMTP clients simultaneously. For this you need to increase the number of Postfix SMTP server processes. This will @@ -349,95 +460,6 @@ repeatedly sending NOOP or RSET commands.

    as these measures are used only temporarily. The next section of this document introduces a way to automate this process.

    -

    Automatic stress-adaptive behavior

    - -

    Postfix version 2.5 introduces automatic stress-adaptive behavior. -This is also available as a source code patch for Postfix versions -2.4 and 2.3 from the mirrors listed at -http://www.postfix.org/download.html.

    - -

    It works as follows. When a "public" network service such as -the SMTP server runs into an "all server ports are busy" condition, -the Postfix master(8) daemon logs a warning, restarts the service -(without interrupting existing network sessions), and runs the -service with "-o stress=yes" on the server process command line: -

    - -
    -
    -80821  ??  S      0:00.24 smtpd -n smtp -t inet -u -c -o stress=yes
-
    -
    - -

    Normally, the Postfix master(8) daemon runs such a service with -"-o stress=" on the command line (i.e. with an empty parameter -value):

    - -
    -
    -83326  ??  S      0:00.28 smtpd -n smtp -t inet -u -c -o stress=
-
    -
    - -

    Services that have local access only never have "-o stress" -parameters on the command line. This includes services internal to -Postfix such as the queue manager, and services that listen on a -loopback interface only, such as after-filter SMTP services.

    - -

    The "stress" parameter value is the key to making main.cf -parameter settings stress adaptive. The following settings are the -default with Postfix 2.6 and later. With earlier Postfix versions -that have stress-adaptive support, append the lines below to the -main.cf file and issue a "postfix reload" command:

    - -
    -
    -1 smtpd_timeout = ${stress?10}${stress:300}s
-2 smtpd_hard_error_limit = ${stress?1}${stress:20}
-3 smtpd_junk_command_limit = ${stress?1}${stress:100}
-
    -
    - -

    Translation:

    - -

    - -

    The syntax of ${name?value} and ${name:value} is explained at -the beginning of the postconf(5) manual page.

    - -

    NOTE: Please keep in mind that the stress-adaptive feature is -a fairly desperate measure to keep some legitimate mail -flowing under overload conditions. If a site is reaching the SMTP -server process limit when there isn't an attack or bot flood -occurring, then either the process limit needs to be raised or more -hardware needs to be added.

    -

    Detecting support for stress-adaptive behavior

    To find out if your Postfix installation supports stress-adaptive diff --git a/postfix/html/ldap_table.5.html b/postfix/html/ldap_table.5.html index 8d8c43039..dcb3a9ee7 100644 --- a/postfix/html/ldap_table.5.html +++ b/postfix/html/ldap_table.5.html @@ -10,9 +10,9 @@ LDAP_TABLE(5) LDAP_TABLE(5) ldap_table - Postfix LDAP client configuration SYNOPSIS - postmap -q "string" ldap:/etc/postfix/filename + postmap -q "string" ldap:/etc/postfix/filename - postmap -q - ldap:/etc/postfix/filename <inputfile + postmap -q - ldap:/etc/postfix/filename <inputfile DESCRIPTION The Postfix mail system uses optional tables for address @@ -716,7 +716,7 @@ LDAP_TABLE(5) LDAP_TABLE(5) the demand value of TLS_REQCERT in LDAP client con- figuration files. - The "try" and "never" values of TLS_REQCERT have no + The "try" and "allow" values of TLS_REQCERT have no equivalents here. They are not available with OpenLDAP 2.0, and in any case have questionable security properties. Either you want TLS verified diff --git a/postfix/html/postconf.5.html b/postfix/html/postconf.5.html index a173c8557..a9ca17824 100644 --- a/postfix/html/postconf.5.html +++ b/postfix/html/postconf.5.html @@ -3782,11 +3782,11 @@ this length; upon delivery, long lines are reconstructed.

    lmdb_map_size -(default: 10485760)
    +(default: 16777216)

    -The per-table size limit for programs that create OpenLDAP LMDB -tables. Specify a byte count. +The initial OpenLDAP LMDB database size limit in bytes. Each time +a database becomes full, its size limit is doubled.

    @@ -13683,7 +13683,8 @@ DNS lookup and increases the maximal inbound delivery rate.

    smtpd_per_record_deadline (default: normal: no, overload: yes)
    -

    Change the behavior of the smtpd_timeout time limit, from a +

    Change the behavior of the smtpd_timeout and smtpd_starttls_timeout +time limits, from a time limit per read or write system call, to a time limit to send or receive a complete record (an SMTP command line, SMTP response line, SMTP message content line, or TLS protocol message). This @@ -14706,6 +14707,12 @@ feature is available in Postfix 2.1 and later.

    authenticated clients only. This feature is available in Postfix version 2.1 and later. +
    reject_known_sender_login_mismatch
    + +
    Apply the reject_sender_login_mismatch restriction only to MAIL +FROM addresses that are known in $smtpd_sender_login_maps. This +feature is available in Postfix version 2.11 and later.
    +
    reject_non_fqdn_sender
    Reject the request when the MAIL FROM address is not in diff --git a/postfix/html/postscreen.8.html b/postfix/html/postscreen.8.html index f746a4650..898f9d707 100644 --- a/postfix/html/postscreen.8.html +++ b/postfix/html/postscreen.8.html @@ -189,7 +189,7 @@ POSTSCREEN(8) POSTSCREEN(8) where a non-whitelisted remote SMTP client can obtain postscreen(8)'s temporary whitelist status. -BEFORE-GREETING TESTS +BEFORE 220 GREETING TESTS These tests are executed before the remote SMTP client receives the "220 servername" greeting. If no tests remain after the successful completion of this phase, the client @@ -253,13 +253,13 @@ POSTSCREEN(8) POSTSCREEN(8) combined DNSBL score as defined with the postscreen_dnsbl_sites parameter. -AFTER-GREETING TESTS +AFTER 220 GREETING TESTS These tests are executed after the remote SMTP client receives the "220 servername" greeting. If a client passes all tests during this phase, it will receive a 4XX - response to RCPT TO commands until the client hangs up. - After this, the client will be allowed to talk directly to - a Postfix SMTP server process. + response to all RCPT TO commands. After the client recon- + nects, it will be allowed to talk directly to a Postfix + SMTP server process. postscreen_bare_newline_action (ignore) The action that postscreen(8) takes when a remote diff --git a/postfix/man/man5/ldap_table.5 b/postfix/man/man5/ldap_table.5 index 0b650edeb..67fa8244e 100644 --- a/postfix/man/man5/ldap_table.5 +++ b/postfix/man/man5/ldap_table.5 @@ -658,7 +658,7 @@ version dependent behavior). The \fByes\fR setting corresponds to the \fBdemand\fR value of \fBTLS_REQCERT\fR in LDAP client configuration files. .sp -The "try" and "never" values of \fBTLS_REQCERT\fR have no equivalents +The "try" and "allow" values of \fBTLS_REQCERT\fR have no equivalents here. They are not available with OpenLDAP 2.0, and in any case have questionable security properties. Either you want TLS verified LDAP connections, or you don't. diff --git a/postfix/man/man5/postconf.5 b/postfix/man/man5/postconf.5 index 02d7d8824..d9d7dfea9 100644 --- a/postfix/man/man5/postconf.5 +++ b/postfix/man/man5/postconf.5 @@ -2242,9 +2242,9 @@ This feature is available in Postfix 2.1 and later. .SH line_length_limit (default: 2048) Upon input, long lines are chopped up into pieces of at most this length; upon delivery, long lines are reconstructed. -.SH lmdb_map_size (default: 10485760) -The per-table size limit for programs that create OpenLDAP LMDB -tables. Specify a byte count. +.SH lmdb_map_size (default: 16777216) +The initial OpenLDAP LMDB database size limit in bytes. Each time +a database becomes full, its size limit is doubled. .PP This feature is available in Postfix 2.11 and later. .SH lmtp_address_preference (default: ipv6) @@ -9034,7 +9034,8 @@ DNS lookup and increases the maximal inbound delivery rate. .PP This feature is available in Postfix 2.3 and later. .SH smtpd_per_record_deadline (default: normal: no, overload: yes) -Change the behavior of the smtpd_timeout time limit, from a +Change the behavior of the smtpd_timeout and smtpd_starttls_timeout +time limits, from a time limit per read or write system call, to a time limit to send or receive a complete record (an SMTP command line, SMTP response line, SMTP message content line, or TLS protocol message). This @@ -9797,6 +9798,11 @@ Enforces the reject_sender_login_mismatch restriction for authenticated clients only. This feature is available in Postfix version 2.1 and later. .br +.IP "\fBreject_known_sender_login_mismatch\fR" +Apply the reject_sender_login_mismatch restriction only to MAIL +FROM addresses that are known in $smtpd_sender_login_maps. This +feature is available in Postfix version 2.11 and later. +.br .IP "\fBreject_non_fqdn_sender\fR" Reject the request when the MAIL FROM address is not in fully-qualified domain form, as required by the RFC. diff --git a/postfix/man/man8/postscreen.8 b/postfix/man/man8/postscreen.8 index 2aa1aacb2..de8bb2401 100644 --- a/postfix/man/man8/postscreen.8 +++ b/postfix/man/man8/postscreen.8 @@ -196,7 +196,7 @@ which would introduce a common point of failure. A list of local \fBpostscreen\fR(8) server IP addresses where a non-whitelisted remote SMTP client can obtain \fBpostscreen\fR(8)'s temporary whitelist status. -.SH "BEFORE-GREETING TESTS" +.SH "BEFORE 220 GREETING TESTS" .na .nf .ad @@ -248,7 +248,7 @@ Available in Postfix version 2.11 and later: Allow a remote SMTP client to skip "before" and "after 220 greeting" protocol tests, based on its combined DNSBL score as defined with the postscreen_dnsbl_sites parameter. -.SH "AFTER-GREETING TESTS" +.SH "AFTER 220 GREETING TESTS" .na .nf .ad @@ -256,9 +256,9 @@ defined with the postscreen_dnsbl_sites parameter. These tests are executed after the remote SMTP client receives the "220 servername" greeting. If a client passes all tests during this phase, it will receive a 4XX response -to RCPT TO commands until the client hangs up. After this, -the client will be allowed to talk directly to a Postfix -SMTP server process. +to all RCPT TO commands. After the client reconnects, it +will be allowed to talk directly to a Postfix SMTP server +process. .IP "\fBpostscreen_bare_newline_action (ignore)\fR" The action that \fBpostscreen\fR(8) takes when a remote SMTP client sends a bare newline character, that is, a newline not preceded by carriage diff --git a/postfix/mantools/postlink b/postfix/mantools/postlink index 6da58ae74..f038fb620 100755 --- a/postfix/mantools/postlink +++ b/postfix/mantools/postlink @@ -897,6 +897,7 @@ while (<>) { s;\bcheck_sender_mx_access\b;$&;g; s;\bcheck_sender_ns_access\b;$&;g; s;\b(reject_authenti)([-]*\n*[ ]*)(cated_sender_login_mismatch)\b;$1<\/a>$2$3;g; + s;\breject_known_sender_login_mismatch\b;$&;g; s;\breject_non_fqdn_sender\b;$&;g; s;\breject_rhsbl_sender\b;$&;g; s;\breject_sender_login_mis[-]*\n*[ ]*match\b;$&;g; diff --git a/postfix/proto/INSTALL.html b/postfix/proto/INSTALL.html index a4bb6ac46..93200249c 100644 --- a/postfix/proto/INSTALL.html +++ b/postfix/proto/INSTALL.html @@ -310,6 +310,9 @@ default DEF_CONFIG_DIR config_directory /etc/postfix + DEF_DB_TYPE default_database_type +hash + DEF_DAEMON_DIR daemon_directory /usr/libexec/postfix @@ -376,7 +379,9 @@ off Postfix features at compile time: -DNO_DB Do not build with Berkeley DB support. By default, Berkeley DB support is compiled in on -platforms that are known to support this feature. +platforms that are known to support this feature. If you override +this, then you probably should also override DEF_DB_TYPE as described +in section 4.4. -DNO_DEVPOLL Do not build with Solaris /dev/poll support. By default, /dev/poll diff --git a/postfix/proto/LMDB_README.html b/postfix/proto/LMDB_README.html index 724512819..5f810dcc5 100644 --- a/postfix/proto/LMDB_README.html +++ b/postfix/proto/LMDB_README.html @@ -19,14 +19,6 @@

    Introduction

    -

    Warning: LMDB databases have a show-stopper -bug: they do not grow beyond a specified limit, and introduce a -"database full" hard error condition that does not exist with any -other Postfix database type. This is a problem for programs whose -database grows with system load. Examples are postscreen(8), -greylisting, verify(8) and tlsmgr(8).

    -You have been warned.

    -

    Postfix uses databases of various kinds to store and look up information. Postfix databases are specified as "type:name". OpenLDAP LMDB implements the Postfix database type "lmdb". @@ -52,9 +44,8 @@ don't exist with other Postfix databases.

    Building Postfix with OpenLDAP LMDB support

    -

    Postfix normally does not enable OpenLDAP LMDB support. -To build Postfix after you installed OpenLDAP LMDB from -source code, use something like:

    +

    Postfix normally does not enable OpenLDAP LMDB support. To +build Postfix with OpenLDAP LMDB support, use something like: 

    @@ -64,7 +55,7 @@ source code, use something like:
    -

    Solaris needs this:

    +

    Solaris may need this: 

    @@ -78,15 +69,14 @@ source code, use something like: 
    
 
 
    Configure LMDB settings
    
 
-
     Postfix provides a configuration parameter that controls how
-large an OpenLDAP LMDB database may grow. 
    
+
     Postfix provides one configuration parameter that controls 
+OpenLDAP LMDB database behavior. 
    
 
 
      
 
-
    •  
       lmdb_map_size (default: 16 MBytes per table).  This setting
-controls how large any OpenLDAP LMDB database may grow. It must be
-set large enough to accommodate the largest table that Postfix will
-use. 
      
+
    •  
       lmdb_map_size (default: 16777216).  This setting specifies
+the initial OpenLDAP LMDB database size limit in bytes.  Each time
+a database becomes full, its size limit is doubled.  
      
 
 
    
 
@@ -119,15 +109,17 @@ http://highlandsun.com/hyc/mdb/. 
    
 databases.   
 
 
     As documented below, conversion to LMDB introduces a number of
-failure modes that don't exist with other Postfix databases. 
    
+failure modes that don't exist with other Postfix databases.  Some
+failure modes have been eliminated on the course of time.
+The writeup below reflects the status as of of LMDB 0.9.8. 
    
+
+
-
 
     Problem: 
     
     
     The "postmap lmdb:filename" command
 fails with an MDB_MAP_FULL error.  This problem does not exist with
 other Postfix databases. 
     
    
@@ -177,23 +167,25 @@ availability, because the old data still exists in the database.
 MDB_MAP_FULL error, it expands the database file size to the current
 LMDB map size limit before terminating.  
    
 
-
     When the postmap(1) or postalias(1) command opens an LMDB file
-larger than lmdb_map_size/3, it logs a warning and uses a larger
-size limit instead: 
    
+
     Next, when you re-run the postmap(1) or postalias(1) command,
+it discovers that the LMDB file is larger than lmdb_map_size/3,
+logs a warning, and uses a larger LMDB map size limit instead: 
    
 
 
      warning: filename.lmdb: file size 15024128 ≥
-(lmdb map size limit 16777216)/3 -- using a larger map size limit
-
    
+(lmdb map size limit 16777216)/3
     warning: filename.lmdb:
+using map size limit 45072384 
    
 
-
     The two steps above can be used to automate recovery and avoid
-the need for human intervention. Just repeat "postmap lmdb:filename"
-(up to some limit).  After each failure it will use a 3x larger
-size limit, and eventually the "database full" error will disappear.
-
    
+
     By repeating the two steps above you can automate recovery and
+avoid the need for human intervention. Just repeat "postmap
+lmdb:filename" (up to some limit).  After each failure it will use
+a 3x larger size limit, and eventually the "database full" error
+should disappear. This fails only when the disk is full or when
+the LMDB map size limit would exceed the memory address space size
+limit.  
    
 
 
     Prevention: 
     
     
     Monitor your LMDB files and make
-sure that lmdb_map_size > 3x the largest LMDB file size. 
    
-
     
+sure that in main.cf, lmdb_map_size > 3x the largest LMDB file
+size.

    Unexpected Postfix daemon "database full" errors.

    @@ -214,8 +206,8 @@ file larger than lmdb_map_size/3, it logs a warning and uses a larger size limit instead:

    warning: filename.lmdb: file size 15024128 ≥ -(lmdb map size limit 16777216)/3 -- using a larger map size limit -

    +(lmdb map size limit 16777216)/3
    warning: filename.lmdb: +using map size limit 45072384

    This can be used to automate recovery and avoid the need for human intervention. Each time the daemon runs into a "database full" @@ -226,14 +218,16 @@ full" error will disappear, at least for a while.

    sure that lmdb_map_size > 3x the largest LMDB file size.

    +--> +

    Non-obvious recovery with postmap(1)/postalias(1)/tlsmgr(8) from a corrupted database.

    Problem:

    You cannot rebuild a corrupted LMDB -database simply by running postmap(1) or postalias(1), or by waiting -until the tlsmgr(8) daemon restarts automatically. This problem +database simply by re-running postmap(1) or postalias(1), or by +waiting until the tlsmgr(8) daemon restarts. This problem does not exist with other Postfix databases.

    Background:

    The Postfix LMDB database client @@ -245,9 +239,9 @@ That is obviously not possible with a corrupted database file.

    Impact:

    Postfix does not process mail until someone fixes the problem.

    -
    Recovery:

    First delete the ".lmdb" file by hand, -then rebuild the file with the postmap(1) or postalias(1) command, -or wait until the tlsmgr(8) daemon restarts automatically.

    +
    Recovery:

    First delete the ".lmdb" file by hand. +Then, rebuild the file with the postmap(1) or postalias(1) command, +or wait until the tlsmgr(8) daemon restarts.

    Prevention:
    diff --git a/postfix/proto/SASL_README.html b/postfix/proto/SASL_README.html index 49e7fb7cf..b9104e2a9 100644 --- a/postfix/proto/SASL_README.html +++ b/postfix/proto/SASL_README.html @@ -1450,7 +1450,8 @@ restriction above will reject the sender address in the MAIL FROM command if smtpd_sender_login_maps does not specify the SMTP client's login name as an owner of that address.

    -

    See also reject_authenticated_sender_login_mismatch and +

    See also reject_authenticated_sender_login_mismatch, +reject_known_sender_login_mismatch, and reject_unauthenticated_sender_login_mismatch for additional control over the SASL login name and the envelope sender.

    diff --git a/postfix/proto/STRESS_README.html b/postfix/proto/STRESS_README.html index 00f849187..77caf4886 100644 --- a/postfix/proto/STRESS_README.html +++ b/postfix/proto/STRESS_README.html @@ -33,6 +33,8 @@ and for earlier Postfix versions that don't.

  • Symptoms of Postfix SMTP server overload +
  • Automatic stress-adaptive behavior +
  • Service more SMTP clients at the same time
  • Spend less time per SMTP client @@ -41,8 +43,6 @@ and for earlier Postfix versions that don't.

  • Temporary measures for older Postfix releases -
  • Automatic stress-adaptive behavior -
  • Detecting support for stress-adaptive behavior
  • Forcing stress-adaptive behavior on or off @@ -109,6 +109,9 @@ Oct 3 20:39:27 spike postfix/master[28905]: warning: service "smtp" Oct 3 20:39:27 spike postfix/master[28905]: warning: to avoid this condition, increase the process count in master.cf or reduce the service time per client +Oct 3 20:39:27 spike postfix/master[28905]: warning: see + http://www.postfix.org/STRESS_README.html for examples of + stress-adapting configuration settings @@ -118,8 +121,116 @@ Postfix SMTP server overload is not necessarily lost. It should still arrive once the situation returns to normal, as long as the overload condition is temporary.

    +

    Automatic stress-adaptive behavior

    + +

    Postfix version 2.5 introduces automatic stress-adaptive behavior. +It works as follows. When a "public" network service such as the +SMTP server runs into an "all server ports are busy" condition, the +Postfix master(8) daemon logs a warning, restarts the service +(without interrupting existing network sessions), and runs the +service with "-o stress=yes" on the server process command line: +

    + +
    +
    +80821  ??  S      0:00.24 smtpd -n smtp -t inet -u -c -o stress=yes
+
    +
    + +

    Normally, the Postfix master(8) daemon runs such a service with +"-o stress=" on the command line (i.e. with an empty parameter +value):

    + +
    +
    +83326  ??  S      0:00.28 smtpd -n smtp -t inet -u -c -o stress=
+
    +
    + +

    Services that have local access only never have "-o stress" +parameters on the command line. This includes services internal to +Postfix such as the queue manager, and services that listen on a +loopback interface only, such as after-filter SMTP services.

    + +

    The "stress" parameter value is the key to making main.cf +parameter settings stress adaptive. The following settings are the +default with Postfix 2.6 and later.

    + +
    +
    +1 smtpd_timeout = ${stress?10}${stress:300}s
+2 smtpd_hard_error_limit = ${stress?1}${stress:20}
+3 smtpd_junk_command_limit = ${stress?1}${stress:100}
+4 # Parameters added after Postfix 2.6:
+5 smtpd_per_record_deadline = ${stress?yes}${stress:no}
+6 smtpd_starttls_timeout = ${stress?10}${stress:300}s
+7 address_verify_poll_count = ${stress?1}${stress:3}
+
    +
    + +

    Translation:

    + +

      + +

    • Line 1: under conditions of stress, use an smtpd_timeout +value of 10 seconds instead of the default 300 seconds. Experience +on the postfix-users list from a variety of sysadmins shows that +reducing the "normal" smtpd_timeout to 60s is unlikely to affect +legitimate clients. However, it is unlikely to become the Postfix +default because it's not RFC compliant. Setting smtpd_timeout to +10s or even 5s under stress will still allow most +legitimate clients to connect and send mail, but may delay mail +from some clients. No mail should be lost, as long as this measure +is used only temporarily.

      + +

    • Line 2: under conditions of stress, use an smtpd_hard_error_limit +of 1 instead of the default 20. This helps by disconnecting clients +after a single error, giving other clients a chance to connect. +However, this may cause significant delays with legitimate mail, +such as a mailing list that contains a few no-longer-active user +names that didn't bother to unsubscribe. No mail should be lost, +as long as this measure is used only temporarily.

      + +

    • Line 3: under conditions of stress, use an +smtpd_junk_command_limit of 1 instead of the default 100. This +prevents clients from keeping connections open by repeatedly +sending HELO, EHLO, NOOP, RSET, VRFY or ETRN commands.

      + +

    • Line 5: under conditions of stress, change the behavior +of smtpd_timeout and smtpd_starttls_timeout, from a time limit per +read or write system call, to a time limit to send or receive a +complete record (an SMTP command line, SMTP response line, SMTP +message content line, or TLS protocol message).

      + +

    • Line 6: under conditions of stress, reduce the time limit +for TLS protocol handshake messages to 10 seconds, from the default +value of 300 seconds. See also the smtpd_timeout discussion above. +

      + +

    • Line 7: under conditions of stress, do not wait up to 6 +seconds for the completion of an address verification probe. If the +result is not already in the address verification cache, reply +immediately with $unverified_recipient_tempfail_action or +$unverified_sender_tempfail_action. No mail should be lost, as long +as this measure is used only temporarily.

      + +
    + +

    The syntax of ${name?value} and ${name:value} is explained at +the beginning of the postconf(5) manual page.

    + +

    NOTE: Please keep in mind that the stress-adaptive feature is +a fairly desperate measure to keep some legitimate mail +flowing under overload conditions. If a site is reaching the SMTP +server process limit when there isn't an attack or bot flood +occurring, then either the process limit needs to be raised or more +hardware needs to be added.

    +

    Service more SMTP clients at the same time

    +

    This section and the ones that follow discuss permanent measures +against mail server overload.

    +

    One measure to avoid the "all server processes busy" condition is to service more SMTP clients simultaneously. For this you need to increase the number of Postfix SMTP server processes. This will @@ -349,95 +460,6 @@ repeatedly sending NOOP or RSET commands.

    as these measures are used only temporarily. The next section of this document introduces a way to automate this process.

    -

    Automatic stress-adaptive behavior

    - -

    Postfix version 2.5 introduces automatic stress-adaptive behavior. -This is also available as a source code patch for Postfix versions -2.4 and 2.3 from the mirrors listed at -http://www.postfix.org/download.html.

    - -

    It works as follows. When a "public" network service such as -the SMTP server runs into an "all server ports are busy" condition, -the Postfix master(8) daemon logs a warning, restarts the service -(without interrupting existing network sessions), and runs the -service with "-o stress=yes" on the server process command line: -

    - -
    -
    -80821  ??  S      0:00.24 smtpd -n smtp -t inet -u -c -o stress=yes
-
    -
    - -

    Normally, the Postfix master(8) daemon runs such a service with -"-o stress=" on the command line (i.e. with an empty parameter -value):

    - -
    -
    -83326  ??  S      0:00.28 smtpd -n smtp -t inet -u -c -o stress=
-
    -
    - -

    Services that have local access only never have "-o stress" -parameters on the command line. This includes services internal to -Postfix such as the queue manager, and services that listen on a -loopback interface only, such as after-filter SMTP services.

    - -

    The "stress" parameter value is the key to making main.cf -parameter settings stress adaptive. The following settings are the -default with Postfix 2.6 and later. With earlier Postfix versions -that have stress-adaptive support, append the lines below to the -main.cf file and issue a "postfix reload" command:

    - -
    -
    -1 smtpd_timeout = ${stress?10}${stress:300}s
-2 smtpd_hard_error_limit = ${stress?1}${stress:20}
-3 smtpd_junk_command_limit = ${stress?1}${stress:100}
-
    -
    - -

    Translation:

    - -

      - -

    • Line 1: under conditions of stress, use an smtpd_timeout -value of 10 seconds instead of the default 300 seconds. Experience -on the postfix-users list from a variety of sysadmins shows that -reducing the "normal" smtpd_timeout to 60s is unlikely to affect -legitimate clients. However, it is unlikely to become the Postfix -default because it's not RFC compliant. Setting smtpd_timeout to -10s (line 2 below) or even 5s under stress will still allow most -legitimate clients to connect and send mail, but may delay mail -from some clients. No mail should be lost, as long as this measure -is used only temporarily.

      - -

    • Line 2: under conditions of stress, use an smtpd_hard_error_limit -of 1 instead of the default 20. This helps by disconnecting clients -after a single error, giving other clients a chance to connect. -However, this may cause significant delays with legitimate mail, -such as a mailing list that contains a few no-longer-active user -names that didn't bother to unsubscribe. No mail should be lost, -as long as this measure is used only temporarily.

      - -

    • Line 3: under conditions of stress, use an -smtpd_junk_command_limit of 1 instead of the default 100. This -prevents clients from keeping idle connections open by repeatedly -sending NOOP or RSET commands.

      - -
    - -

    The syntax of ${name?value} and ${name:value} is explained at -the beginning of the postconf(5) manual page.

    - -

    NOTE: Please keep in mind that the stress-adaptive feature is -a fairly desperate measure to keep some legitimate mail -flowing under overload conditions. If a site is reaching the SMTP -server process limit when there isn't an attack or bot flood -occurring, then either the process limit needs to be raised or more -hardware needs to be added.

    -

    Detecting support for stress-adaptive behavior

    To find out if your Postfix installation supports stress-adaptive diff --git a/postfix/proto/ldap_table b/postfix/proto/ldap_table index 666aa28ab..7b0e564f1 100644 --- a/postfix/proto/ldap_table +++ b/postfix/proto/ldap_table @@ -642,7 +642,7 @@ # \fBdemand\fR value of \fBTLS_REQCERT\fR in LDAP client configuration # files. # .sp -# The "try" and "never" values of \fBTLS_REQCERT\fR have no equivalents +# The "try" and "allow" values of \fBTLS_REQCERT\fR have no equivalents # here. They are not available with OpenLDAP 2.0, and in any case have # questionable security properties. Either you want TLS verified LDAP # connections, or you don't. diff --git a/postfix/proto/postconf.proto b/postfix/proto/postconf.proto index c4b6c53a1..ef2e709c6 100644 --- a/postfix/proto/postconf.proto +++ b/postfix/proto/postconf.proto @@ -2837,11 +2837,11 @@ The default time unit is d (days). Specify 0 when mail delivery should be tried only once.

    -%PARAM lmdb_map_size 10485760 +%PARAM lmdb_map_size 16777216

    -The per-table size limit for programs that create OpenLDAP LMDB -tables. Specify a byte count. +The initial OpenLDAP LMDB database size limit in bytes. Each time +a database becomes full, its size limit is doubled.

    @@ -6292,6 +6292,12 @@ feature is available in Postfix 2.1 and later.

    • authenticated clients only. This feature is available in Postfix version 2.1 and later. +
    reject_known_sender_login_mismatch
    + +
    Apply the reject_sender_login_mismatch restriction only to MAIL +FROM addresses that are known in $smtpd_sender_login_maps. This +feature is available in Postfix version 2.11 and later.
    +
    reject_non_fqdn_sender
    Reject the request when the MAIL FROM address is not in @@ -14754,7 +14760,8 @@ service performs plaintext <=> TLS ciphertext conversion.

    %PARAM smtpd_per_record_deadline normal: no, overload: yes -

    Change the behavior of the smtpd_timeout time limit, from a +

    Change the behavior of the smtpd_timeout and smtpd_starttls_timeout +time limits, from a time limit per read or write system call, to a time limit to send or receive a complete record (an SMTP command line, SMTP response line, SMTP message content line, or TLS protocol message). This diff --git a/postfix/src/global/mail_params.h b/postfix/src/global/mail_params.h index 93d2c3569..871fcc50d 100644 --- a/postfix/src/global/mail_params.h +++ b/postfix/src/global/mail_params.h @@ -1591,6 +1591,8 @@ extern char *var_smtpd_snd_auth_maps; #define REJECT_SENDER_LOGIN_MISMATCH "reject_sender_login_mismatch" #define REJECT_AUTH_SENDER_LOGIN_MISMATCH \ "reject_authenticated_sender_login_mismatch" +#define REJECT_KNOWN_SENDER_LOGIN_MISMATCH \ + "reject_known_sender_login_mismatch" #define REJECT_UNAUTH_SENDER_LOGIN_MISMATCH \ "reject_unauthenticated_sender_login_mismatch" diff --git a/postfix/src/global/mail_version.h b/postfix/src/global/mail_version.h index 2dc2e54da..1ee311c13 100644 --- a/postfix/src/global/mail_version.h +++ b/postfix/src/global/mail_version.h @@ -20,7 +20,7 @@ * Patches change both the patchlevel and the release date. Snapshots have no * patchlevel; they change the release date only. */ -#define MAIL_RELEASE_DATE "20130825" +#define MAIL_RELEASE_DATE "20130927" #define MAIL_VERSION_NUMBER "2.11" #ifdef SNAPSHOT diff --git a/postfix/src/postalias/postalias.c b/postfix/src/postalias/postalias.c index 750b263c8..430c15641 100644 --- a/postfix/src/postalias/postalias.c +++ b/postfix/src/postalias/postalias.c @@ -259,7 +259,7 @@ static void postalias(char *map_type, char *path_name, int postalias_flags, int open_flags, int dict_flags) { - VSTREAM *source_fp; + VSTREAM *NOCLOBBER source_fp; VSTRING *line_buffer; MKMAP *mkmap; int lineno; @@ -286,10 +286,10 @@ static void postalias(char *map_type, char *path_name, int postalias_flags, /* Create database. */ if (strcmp(map_type, DICT_TYPE_PROXY) == 0) msg_fatal("can't create maps via the proxy service"); + dict_flags |= DICT_FLAG_BULK_UPDATE; if ((source_fp = vstream_fopen(path_name, O_RDONLY, 0)) == 0) msg_fatal("open %s: %m", path_name); } - dict_flags |= DICT_FLAG_BULK_UPDATE; if (fstat(vstream_fileno(source_fp), &st) < 0) msg_fatal("fstat %s: %m", path_name); @@ -322,72 +322,85 @@ static void postalias(char *map_type, char *path_name, int postalias_flags, umask(saved_mask); /* - * Add records to the database. + * Trap "exceptions" so that we can restart a bulk-mode update after a + * recoverable error. */ - lineno = 0; - while (readlline(line_buffer, source_fp, &lineno)) { + for (;;) { + if (dict_isjmp(mkmap->dict) != 0 + && dict_setjmp(mkmap->dict) != 0 + && vstream_fseek(source_fp, SEEK_SET, 0) < 0) + msg_fatal("seek %s: %m", VSTREAM_PATH(source_fp)); /* - * Tokenize the input, so that we do the right thing when a quoted - * localpart contains special characters such as "@", ":" and so on. + * Add records to the database. */ - if ((tok_list = tok822_scan(STR(line_buffer), (TOK822 **) 0)) == 0) - continue; + lineno = 0; + while (readlline(line_buffer, source_fp, &lineno)) { - /* - * Enforce the key:value format. Disallow missing keys, multi-address - * keys, or missing values. In order to specify an empty string or - * value, enclose it in double quotes. - */ - if ((colon = tok822_find_type(tok_list, ':')) == 0 - || colon->prev == 0 || colon->next == 0 - || tok822_rfind_type(colon, ',')) { - msg_warn("%s, line %d: need name:value pair", - VSTREAM_PATH(source_fp), lineno); - tok822_free_tree(tok_list); - continue; + /* + * Tokenize the input, so that we do the right thing when a + * quoted localpart contains special characters such as "@", ":" + * and so on. + */ + if ((tok_list = tok822_scan(STR(line_buffer), (TOK822 **) 0)) == 0) + continue; + + /* + * Enforce the key:value format. Disallow missing keys, + * multi-address keys, or missing values. In order to specify an + * empty string or value, enclose it in double quotes. + */ + if ((colon = tok822_find_type(tok_list, ':')) == 0 + || colon->prev == 0 || colon->next == 0 + || tok822_rfind_type(colon, ',')) { + msg_warn("%s, line %d: need name:value pair", + VSTREAM_PATH(source_fp), lineno); + tok822_free_tree(tok_list); + continue; + } + + /* + * Key must be local. XXX We should use the Postfix rewriting and + * resolving services to handle all address forms correctly. + * However, we can't count on the mail system being up when the + * alias database is being built, so we're guessing a bit. + */ + if (tok822_rfind_type(colon, '@') || tok822_rfind_type(colon, '%')) { + msg_warn("%s, line %d: name must be local", + VSTREAM_PATH(source_fp), lineno); + tok822_free_tree(tok_list); + continue; + } + + /* + * Split the input into key and value parts, and convert from + * token representation back to string representation. Convert + * the key to internal (unquoted) form, because the resolver + * produces addresses in internal form. Convert the value to + * external (quoted) form, because it will have to be re-parsed + * upon lookup. Discard the token representation when done. + */ + key_list = tok_list; + tok_list = 0; + value_list = tok822_cut_after(colon); + tok822_unlink(colon); + tok822_free(colon); + + tok822_internalize(key_buffer, key_list, TOK822_STR_DEFL); + tok822_free_tree(key_list); + + tok822_externalize(value_buffer, value_list, TOK822_STR_DEFL); + tok822_free_tree(value_list); + + /* + * Store the value under a case-insensitive key. + */ + mkmap_append(mkmap, STR(key_buffer), STR(value_buffer)); + if (mkmap->dict->error) + msg_fatal("table %s:%s: write error: %m", + mkmap->dict->type, mkmap->dict->name); } - - /* - * Key must be local. XXX We should use the Postfix rewriting and - * resolving services to handle all address forms correctly. However, - * we can't count on the mail system being up when the alias database - * is being built, so we're guessing a bit. - */ - if (tok822_rfind_type(colon, '@') || tok822_rfind_type(colon, '%')) { - msg_warn("%s, line %d: name must be local", - VSTREAM_PATH(source_fp), lineno); - tok822_free_tree(tok_list); - continue; - } - - /* - * Split the input into key and value parts, and convert from token - * representation back to string representation. Convert the key to - * internal (unquoted) form, because the resolver produces addresses - * in internal form. Convert the value to external (quoted) form, - * because it will have to be re-parsed upon lookup. Discard the - * token representation when done. - */ - key_list = tok_list; - tok_list = 0; - value_list = tok822_cut_after(colon); - tok822_unlink(colon); - tok822_free(colon); - - tok822_internalize(key_buffer, key_list, TOK822_STR_DEFL); - tok822_free_tree(key_list); - - tok822_externalize(value_buffer, value_list, TOK822_STR_DEFL); - tok822_free_tree(value_list); - - /* - * Store the value under a case-insensitive key. - */ - mkmap_append(mkmap, STR(key_buffer), STR(value_buffer)); - if (mkmap->dict->error) - msg_fatal("table %s:%s: write error: %m", - mkmap->dict->type, mkmap->dict->name); + break; } /* diff --git a/postfix/src/postmap/postmap.c b/postfix/src/postmap/postmap.c index 5d891aa6d..e10ac1669 100644 --- a/postfix/src/postmap/postmap.c +++ b/postfix/src/postmap/postmap.c @@ -328,7 +328,7 @@ typedef struct { static void postmap(char *map_type, char *path_name, int postmap_flags, int open_flags, int dict_flags) { - VSTREAM *source_fp; + VSTREAM *NOCLOBBER source_fp; VSTRING *line_buffer; MKMAP *mkmap; int lineno; @@ -349,10 +349,10 @@ static void postmap(char *map_type, char *path_name, int postmap_flags, /* Create database. */ if (strcmp(map_type, DICT_TYPE_PROXY) == 0) msg_fatal("can't create maps via the proxy service"); + dict_flags |= DICT_FLAG_BULK_UPDATE; if ((source_fp = vstream_fopen(path_name, O_RDONLY, 0)) == 0) msg_fatal("open %s: %m", path_name); } - dict_flags |= DICT_FLAG_BULK_UPDATE; if (fstat(vstream_fileno(source_fp), &st) < 0) msg_fatal("fstat %s: %m", path_name); @@ -385,44 +385,56 @@ static void postmap(char *map_type, char *path_name, int postmap_flags, umask(saved_mask); /* - * Add records to the database. + * Trap "exceptions" so that we can restart a bulk-mode update after a + * recoverable error. */ - lineno = 0; - while (readlline(line_buffer, source_fp, &lineno)) { + for (;;) { + if (dict_isjmp(mkmap->dict) != 0 + && dict_setjmp(mkmap->dict) != 0 + && vstream_fseek(source_fp, SEEK_SET, 0) < 0) + msg_fatal("seek %s: %m", VSTREAM_PATH(source_fp)); /* - * Split on the first whitespace character, then trim leading and - * trailing whitespace from key and value. + * Add records to the database. */ - key = STR(line_buffer); - value = key + strcspn(key, " \t\r\n"); - if (*value) - *value++ = 0; - while (ISSPACE(*value)) - value++; - trimblanks(key, 0)[0] = 0; - trimblanks(value, 0)[0] = 0; + lineno = 0; + while (readlline(line_buffer, source_fp, &lineno)) { - /* - * Enforce the "key whitespace value" format. Disallow missing keys - * or missing values. - */ - if (*key == 0 || *value == 0) { - msg_warn("%s, line %d: expected format: key whitespace value", - VSTREAM_PATH(source_fp), lineno); - continue; + /* + * Split on the first whitespace character, then trim leading and + * trailing whitespace from key and value. + */ + key = STR(line_buffer); + value = key + strcspn(key, " \t\r\n"); + if (*value) + *value++ = 0; + while (ISSPACE(*value)) + value++; + trimblanks(key, 0)[0] = 0; + trimblanks(value, 0)[0] = 0; + + /* + * Enforce the "key whitespace value" format. Disallow missing + * keys or missing values. + */ + if (*key == 0 || *value == 0) { + msg_warn("%s, line %d: expected format: key whitespace value", + VSTREAM_PATH(source_fp), lineno); + continue; + } + if (key[strlen(key) - 1] == ':') + msg_warn("%s, line %d: record is in \"key: value\" format; is this an alias file?", + VSTREAM_PATH(source_fp), lineno); + + /* + * Store the value under a case-insensitive key. + */ + mkmap_append(mkmap, key, value); + if (mkmap->dict->error) + msg_fatal("table %s:%s: write error: %m", + mkmap->dict->type, mkmap->dict->name); } - if (key[strlen(key) - 1] == ':') - msg_warn("%s, line %d: record is in \"key: value\" format; is this an alias file?", - VSTREAM_PATH(source_fp), lineno); - - /* - * Store the value under a case-insensitive key. - */ - mkmap_append(mkmap, key, value); - if (mkmap->dict->error) - msg_fatal("table %s:%s: write error: %m", - mkmap->dict->type, mkmap->dict->name); + break; } /* diff --git a/postfix/src/postscreen/Makefile.in b/postfix/src/postscreen/Makefile.in index 9c29a3d55..ecbf3da13 100644 --- a/postfix/src/postscreen/Makefile.in +++ b/postfix/src/postscreen/Makefile.in @@ -16,8 +16,9 @@ CFLAGS = $(DEBUG) $(OPT) $(DEFS) TESTPROG= PROG = postscreen INC_DIR = ../../include -LIBS = ../../lib/libmaster.a ../../lib/libtls.a ../../lib/libglobal.a \ - ../../lib/libutil.a +# Fake libdns dependency, for early-binding shared-library builds. +LIBS = ../../lib/libmaster.a ../../lib/libtls.a ../../lib/libdns.a \ + ../../lib/libglobal.a ../../lib/libutil.a .c.o:; $(CC) $(CFLAGS) -c $*.c diff --git a/postfix/src/postscreen/postscreen.c b/postfix/src/postscreen/postscreen.c index deff9abfa..58ecc40d4 100644 --- a/postfix/src/postscreen/postscreen.c +++ b/postfix/src/postscreen/postscreen.c @@ -170,7 +170,7 @@ /* A list of local \fBpostscreen\fR(8) server IP addresses where a /* non-whitelisted remote SMTP client can obtain \fBpostscreen\fR(8)'s temporary /* whitelist status. -/* BEFORE-GREETING TESTS +/* BEFORE 220 GREETING TESTS /* .ad /* .fi /* These tests are executed before the remote SMTP client @@ -220,15 +220,15 @@ /* Allow a remote SMTP client to skip "before" and "after 220 /* greeting" protocol tests, based on its combined DNSBL score as /* defined with the postscreen_dnsbl_sites parameter. -/* AFTER-GREETING TESTS +/* AFTER 220 GREETING TESTS /* .ad /* .fi /* These tests are executed after the remote SMTP client /* receives the "220 servername" greeting. If a client passes /* all tests during this phase, it will receive a 4XX response -/* to RCPT TO commands until the client hangs up. After this, -/* the client will be allowed to talk directly to a Postfix -/* SMTP server process. +/* to all RCPT TO commands. After the client reconnects, it +/* will be allowed to talk directly to a Postfix SMTP server +/* process. /* .IP "\fBpostscreen_bare_newline_action (ignore)\fR" /* The action that \fBpostscreen\fR(8) takes when a remote SMTP client sends /* a bare newline character, that is, a newline not preceded by carriage diff --git a/postfix/src/smtpd/Makefile.in b/postfix/src/smtpd/Makefile.in index 01b4a545f..7376b7b39 100644 --- a/postfix/src/smtpd/Makefile.in +++ b/postfix/src/smtpd/Makefile.in @@ -16,9 +16,10 @@ CFLAGS = $(DEBUG) $(OPT) $(DEFS) TESTPROG= smtpd_token smtpd_check PROG = smtpd INC_DIR = ../../include +# Fake libdns dependency, for early-binding shared-library builds. LIBS = ../../lib/libmaster.a ../../lib/libtls.a ../../lib/libdns.a \ - ../../lib/libxsasl.a ../../lib/libmilter.a ../../lib/libglobal.a \ - ../../lib/libutil.a + ../../lib/libxsasl.a ../../lib/libmilter.a ../../lib/libdns.a \ + ../../lib/libglobal.a ../../lib/libutil.a .c.o:; $(CC) $(CFLAGS) -c $*.c diff --git a/postfix/src/smtpd/smtpd_check.c b/postfix/src/smtpd/smtpd_check.c index a27dc7012..3b9508328 100644 --- a/postfix/src/smtpd/smtpd_check.c +++ b/postfix/src/smtpd/smtpd_check.c @@ -3461,7 +3461,7 @@ static int reject_maps_rbl(SMTPD_STATE *state) /* reject_auth_sender_login_mismatch - logged in client must own sender address */ -static int reject_auth_sender_login_mismatch(SMTPD_STATE *state, const char *sender) +static int reject_auth_sender_login_mismatch(SMTPD_STATE *state, const char *sender, int allow_unknown_sender) { const RESOLVE_REPLY *reply; const char *owners; @@ -3470,6 +3470,9 @@ static int reject_auth_sender_login_mismatch(SMTPD_STATE *state, const char *sen char *name; int found = 0; +#define ALLOW_UNKNOWN_SENDER 1 +#define FORBID_UNKNOWN_SENDER 0 + /* * Reject if the client is logged in and does not own the sender address. */ @@ -3487,7 +3490,8 @@ static int reject_auth_sender_login_mismatch(SMTPD_STATE *state, const char *sen } } myfree(saved_owners); - } + } else if (allow_unknown_sender) + return (SMTPD_CHECK_DUNNO); if (!found) return (smtpd_check_reject(state, MAIL_ERROR_POLICY, 553, "5.7.1", "<%s>: Sender address rejected: not owned by user %s", @@ -4017,7 +4021,21 @@ static int generic_checks(SMTPD_STATE *state, ARGV *restrictions, #ifdef USE_SASL_AUTH if (var_smtpd_sasl_enable) { if (state->sender && *state->sender) - status = reject_auth_sender_login_mismatch(state, state->sender); + status = reject_auth_sender_login_mismatch(state, + state->sender, FORBID_UNKNOWN_SENDER); + } else +#endif + msg_warn("restriction `%s' ignored: no SASL support", name); + } else if (strcasecmp(name, REJECT_KNOWN_SENDER_LOGIN_MISMATCH) == 0) { +#ifdef USE_SASL_AUTH + if (var_smtpd_sasl_enable) { + if (state->sender && *state->sender) { + if (state->sasl_username) + status = reject_auth_sender_login_mismatch(state, + state->sender, ALLOW_UNKNOWN_SENDER); + else + status = reject_unauth_sender_login_mismatch(state, state->sender); + } } else #endif msg_warn("restriction `%s' ignored: no SASL support", name); diff --git a/postfix/src/tlsmgr/Makefile.in b/postfix/src/tlsmgr/Makefile.in index c40c91fe7..ced0b5404 100644 --- a/postfix/src/tlsmgr/Makefile.in +++ b/postfix/src/tlsmgr/Makefile.in @@ -8,8 +8,9 @@ CFLAGS = $(DEBUG) $(OPT) $(DEFS) TESTPROG= PROG = tlsmgr INC_DIR = ../../include -LIBS = ../../lib/libmaster.a ../../lib/libtls.a ../../lib/libglobal.a \ - ../../lib/libutil.a +# Fake libdns dependency, for early-binding shared-library builds. +LIBS = ../../lib/libmaster.a ../../lib/libtls.a ../../lib/libdns.a \ + ../../lib/libglobal.a ../../lib/libutil.a .c.o:; $(CC) $(CFLAGS) -c $*.c diff --git a/postfix/src/util/Makefile.in b/postfix/src/util/Makefile.in index 74323ffa5..0dacf13b8 100644 --- a/postfix/src/util/Makefile.in +++ b/postfix/src/util/Makefile.in @@ -1833,6 +1833,8 @@ stream_trigger.o: sys_defs.h stream_trigger.o: trigger.h sys_compat.o: sys_compat.c sys_compat.o: sys_defs.h +timecmp.o: timecmp.c +timecmp.o: timecmp.h timed_connect.o: iostuff.h timed_connect.o: msg.h timed_connect.o: sane_connect.h diff --git a/postfix/src/util/dict.h b/postfix/src/util/dict.h index c47c20657..d255aac3b 100644 --- a/postfix/src/util/dict.h +++ b/postfix/src/util/dict.h @@ -15,6 +15,13 @@ * System library. */ #include +#include + +#ifdef NO_SIGSETJMP +#define DICT_JMP_BUF jmp_buf +#else +#define DICT_JMP_BUF sigjmp_buf +#endif /* * Utility library. @@ -56,6 +63,7 @@ typedef struct DICT { VSTRING *fold_buf; /* key folding buffer */ DICT_OWNER owner; /* provenance */ int error; /* last operation only */ + DICT_JMP_BUF *jbuf; /* exception handling */ } DICT; extern DICT *dict_alloc(const char *, const char *, ssize_t); @@ -206,6 +214,31 @@ extern DICT *dict_surrogate(const char *, const char *, int, int, const char *,. */ #define DICT_TYPE_NOFILE "non-existent" + /* + * Duplicated from vstream(3). This should probably be abstracted out. + * + * Exception handling. We use pointer to jmp_buf to avoid a lot of unused + * baggage for streams that don't need this functionality. + * + * XXX sigsetjmp()/siglongjmp() save and restore the signal mask which can + * avoid surprises in code that manipulates signals, but unfortunately some + * systems have bugs in their implementation. + */ +#ifdef NO_SIGSETJMP +#define dict_setjmp(stream) setjmp((stream)->jbuf[0]) +#define dict_longjmp(stream, val) longjmp((stream)->jbuf[0], (val)) +#else +#define dict_setjmp(stream) sigsetjmp((stream)->jbuf[0], 1) +#define dict_longjmp(stream, val) siglongjmp((stream)->jbuf[0], (val)) +#endif +#define dict_isjmp(stream) ((stream)->jbuf != 0) + + /* + * Temporary API. If exception handling proves to be useful, + * dict_jmp_alloc() should be integrated into dict_alloc(). + */ +extern void dict_jmp_alloc(DICT *); + /* LICENSE /* .ad /* .fi diff --git a/postfix/src/util/dict_alloc.c b/postfix/src/util/dict_alloc.c index a5cbd204c..fc35dc0a8 100644 --- a/postfix/src/util/dict_alloc.c +++ b/postfix/src/util/dict_alloc.c @@ -13,6 +13,9 @@ /* /* void dict_free(dict) /* DICT *ptr; +/* +/* void dict_jmp_alloc(dict) +/* DICT *ptr; /* DESCRIPTION /* dict_alloc() allocates memory for a dictionary structure of /* \fIsize\fR bytes, initializes all generic dictionary @@ -35,6 +38,9 @@ /* It is up to the caller to dispose of any memory that was allocated /* by the caller. /* +/* dict_jmp_alloc() implements preliminary support for exception +/* handling. This will eventually be built into dict_alloc(). +/* /* Arguments: /* .IP dict_type /* The official name for this type of dictionary, as used by @@ -114,7 +120,7 @@ static int dict_default_lock(DICT *dict, int operation) return (0); } } - + /* dict_default_close - trap unimplemented operation */ static void dict_default_close(DICT *dict) @@ -145,6 +151,7 @@ DICT *dict_alloc(const char *dict_type, const char *dict_name, ssize_t size) dict->owner.status = DICT_OWNER_UNKNOWN; dict->owner.uid = ~0; dict->error = DICT_ERR_NONE; + dict->jbuf = 0; return dict; } @@ -154,5 +161,20 @@ void dict_free(DICT *dict) { myfree(dict->type); myfree(dict->name); + if (dict->jbuf) + myfree((char *) dict->jbuf); myfree((char *) dict); } + + /* + * TODO: add a dict_flags() argument to dict_alloc() and handle jump buffer + * allocation there. + */ + +/* dict_jmp_alloc - enable exception handling */ + +void dict_jmp_alloc(DICT *dict) +{ + if (dict->jbuf == 0) + dict->jbuf = (DICT_JMP_BUF *) mymalloc(sizeof(DICT_JMP_BUF)); +} diff --git a/postfix/src/util/dict_lmdb.c b/postfix/src/util/dict_lmdb.c index e9d07458c..c4c206e03 100644 --- a/postfix/src/util/dict_lmdb.c +++ b/postfix/src/util/dict_lmdb.c @@ -14,27 +14,20 @@ /* int open_flags; /* int dict_flags; /* DESCRIPTION -/* dict_lmdb_open() opens the named LMDB database and makes it available -/* via the generic interface described in dict_open(3). +/* dict_lmdb_open() opens the named LMDB database and makes +/* it available via the generic interface described in +/* dict_open(3). /* -/* The dict_lmdb_map_size variable specifies a non-default -/* per-table memory map size. The map size is also the maximum -/* size the table can grow to, so it must be set large enough -/* to accomodate the largest tables in use. -/* -/* As a safety measure, when Postfix opens an LMDB database -/* it will set the memory map size to at least 3x the ".lmdb" -/* file size, so that there is room for the file to grow. This -/* ensures that a process can recover from a "table full" error -/* with a simple terminate-and-restart. -/* -/* As a second safety measure, when an update or delete operation -/* runs into an MDB_MAP_FULL error, Postfix will extend the -/* database file to the current ".lmdb" file size, so that the -/* above workaround will be triggered the next time the database -/* is opened. +/* The dict_lmdb_map_size variable specifies the initial +/* database memory map size. When a map becomes full its size +/* is doubled, and other programs pick up the size change. /* DIAGNOSTICS -/* Fatal errors: cannot open file, file write error, out of memory. +/* Fatal errors: cannot open file, file write error, out of +/* memory. +/* BUGS +/* The on-the-fly map resize operations require no concurrent +/* activity in the same database by other threads in the same +/* process. /* SEE ALSO /* dict(3) generic dictionary manager /* LICENSE @@ -44,9 +37,14 @@ /* AUTHOR(S) /* Howard Chu /* Symas Corporation +/* +/* Wietse Venema +/* IBM T.J. Watson Research +/* P.O. Box 704 +/* Yorktown Heights, NY 10598, USA /*--*/ -#include "sys_defs.h" +#include #ifdef HAS_LMDB @@ -61,20 +59,29 @@ #include PATH_LMDB_H #else #include +#endif + + /* + * As of LMDB 0.9.8 the database size limit can be updated on-the-fly. The + * only limit that remains is imposed by the hardware address space. Earlier + * LMDB versions are not suitable for use with Postfix. + */ +#if MDB_VERSION_FULL < MDB_VERINT(0, 9, 8) +#error "Build with LMDB version 0.9.8 or later" #endif /* Utility library. */ -#include "msg.h" -#include "mymalloc.h" -#include "htable.h" -#include "iostuff.h" -#include "vstring.h" -#include "myflock.h" -#include "stringops.h" -#include "dict.h" -#include "dict_lmdb.h" -#include "warn_stat.h" +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include /* Application-specific. */ @@ -84,55 +91,423 @@ typedef struct { MDB_dbi dbi; /* database handle */ MDB_txn *txn; /* bulk update transaction */ MDB_cursor *cursor; /* for sequence ops */ + size_t map_size; /* per-database size limit */ VSTRING *key_buf; /* key buffer */ - VSTRING *val_buf; /* result buffer */ + VSTRING *val_buf; /* value buffer */ + /* The following facilitate LMDB quirk workarounds. */ + int dict_api_retries; /* workarounds per dict(3) call */ + int bulk_mode_retries; /* workarounds per bulk transaction */ + int open_flags; /* dict(3) open flags */ + int env_flags; /* LMDB open flags */ } DICT_LMDB; + /* + * The LMDB database filename suffix happens to equal our DICT_TYPE_LMDB + * prefix, but that doesn't mean it is kosher to use DICT_TYPE_LMDB where a + * suffix is needed, so we define an explicit suffix here. + */ +#define DICT_LMDB_SUFFIX "lmdb" + + /* + * Make a safe string copy that is guaranteed to be null-terminated. + */ #define SCOPY(buf, data, size) \ vstring_str(vstring_strncpy(buf ? buf : (buf = vstring_alloc(10)), data, size)) -size_t dict_lmdb_map_size = (10 * 1024 * 1024); /* 10MB default mmap - * size */ + /* + * Postfix writers recover from a "map full" error by increasing the memory + * map size with a factor DICT_LMDB_SIZE_INCR (up to some limit) and + * retrying the transaction. + * + * Each dict(3) API call is retried no more than a few times. For bulk-mode + * transactions the number of retries is proportional to the size of the + * address space. + */ +#ifndef SSIZE_T_MAX /* The maximum map size */ +#define SSIZE_T_MAX __MAXINT__(ssize_t) /* XXX Assumes two's complement */ +#endif + +#define DICT_LMDB_SIZE_INCR 2 /* Increase size by 1 bit on retry */ +#define DICT_LMDB_SIZE_MAX SSIZE_T_MAX + +#define DICT_LMDB_API_RETRY_LIMIT 2 /* Retries per dict(3) API call */ +#define DICT_LMDB_BULK_RETRY_LIMIT \ + (2 * sizeof(size_t) * CHAR_BIT) /* Retries per bulk-mode transaction */ + + /* + * XXX Should dict_lmdb_max_readers be configurable? Is this a per-database + * property? Per-process? Does it need to be the same for all processes? + */ +size_t dict_lmdb_map_size = 8192; /* Minimum size without SIGSEGV */ unsigned int dict_lmdb_max_readers = 216; /* 200 postfix processes, * plus some extra */ +/* #define msg_verbose 1 */ + /* - * Out-of-space safety net. When an update or delete operation fails with - * MDB_MAP_FULL, extend the database file size so that the next - * dict_lmdb_open() call will force a 3x over-allocation. + * The purpose of the error-recovering functions below is to hide LMDB + * quirks (MAP_FULL, MAP_CHANGED), so that the dict(3) API routines can + * pretend that those quirks don't exist, and focus on their own job. * - * XXX This strategy assumes that a bogus file size will not affect LMDB - * operation. In private communication on August 18, 2013, Howard Chu - * confirmed this as follows: "It will have no effect. LMDB internally - * accounts for the last used page#, the filesystem's notion of filesize - * isn't used for any purpose." + * - To recover from a single-transaction LMDB error, each wrapper function + * uses tail recursion instead of goto. Since LMDB errors are rare, code + * clarity is more important than speed. * - * We make no assumptions about which LMDB operations may fail with - * MDB_MAP_FULL. Instead we wrap all LMDB operations inside a Postfix - * function that may change a database. + * - To recover from a bulk-mode transaction LMDB error, the error-recovery + * code jumps back into the caller to some pre-arranged point (the closest + * thing that C has to exception handling). With postmap, this means that + * bulk-mode LMDB error recovery is limited to input that is seekable. */ -#define DICT_LMDB_WRAPPER(dict, status, operation) \ - ((status = operation) == MDB_MAP_FULL ? \ - (dict_lmdb_grow(dict), status) : status) -/* dict_lmdb_grow - grow the DB file if the last txn failed to grow it */ +/* dict_lmdb_prepare - LMDB-specific (re)initialization before actual access */ -static void dict_lmdb_grow(DICT *dict) +static void dict_lmdb_prepare(DICT_LMDB *dict_lmdb) { - struct stat st; - char *mdb_path = concatenate(dict->name, "." DICT_TYPE_LMDB, (char *) 0); + int status; /* - * After MDB_MAP_FULL error, expand the file size to trigger the 3x size - * limit workaround on the next open() attempt. + * This is called before accessing the database, or after recovery from + * an LMDB error. dict_lmdb->txn is either the database open() + * transaction or a freshly-created bulk-mode transaction. + * + * - With O_TRUNC we make a "drop" request before populating the database. + * + * - With DICT_FLAG_BULK_UPDATE we commit a bulk-mode transaction when the + * database is closed. */ - if (stat(mdb_path, &st) == 0 && st.st_size < dict_lmdb_map_size - && truncate(mdb_path, dict_lmdb_map_size) < 0) - msg_warn("dict_lmdb_grow: cannot grow database file %s:%s: %m", - dict->type, dict->name); - myfree(mdb_path); + if (dict_lmdb->open_flags & O_TRUNC) { + if ((status = mdb_drop(dict_lmdb->txn, dict_lmdb->dbi, 0)) != 0) + msg_fatal("truncate %s:%s: %s", + dict_lmdb->dict.type, dict_lmdb->dict.name, + mdb_strerror(status)); + if ((dict_lmdb->dict.flags & DICT_FLAG_BULK_UPDATE) == 0) { + if ((status = mdb_txn_commit(dict_lmdb->txn))) + msg_fatal("truncate %s:%s: %s", + dict_lmdb->dict.type, dict_lmdb->dict.name, + mdb_strerror(status)); + dict_lmdb->txn = NULL; + } + } else if ((dict_lmdb->env_flags & MDB_RDONLY) != 0 + || (dict_lmdb->dict.flags & DICT_FLAG_BULK_UPDATE) == 0) { + mdb_txn_abort(dict_lmdb->txn); + dict_lmdb->txn = NULL; + } } +/* dict_lmdb_recover - recover from LMDB errors */ + +static int dict_lmdb_recover(DICT_LMDB *dict_lmdb, int status) +{ + const char *myname = "dict_lmdb_recover"; + MDB_envinfo info; + + /* + * Limit the number of recovery attempts per dict(3) API request. + */ + if ((dict_lmdb->dict_api_retries += 1) > DICT_LMDB_API_RETRY_LIMIT) { + if (msg_verbose) + msg_info("%s: %s:%s too many recovery attempts %d", + myname, dict_lmdb->dict.type, dict_lmdb->dict.name, + dict_lmdb->dict_api_retries); + return (status); + } + + /* + * If we can recover from the error, we clear the error condition and the + * caller should retry the failed operation immediately. Otherwise, the + * caller should terminate with a fatal run-time error and the program + * should be re-run later. + * + * dict_lmdb->txn is either null (non-bulk transaction error), or an aborted + * bulk-mode transaction. If we want to make this wrapper layer suitable + * for general use, then the bulk/non-bulk distinction should be made + * less specific to Postfix. + */ + switch (status) { + + /* + * As of LMDB 0.9.8 when a non-bulk update runs into a "map full" + * error, we can resize the environment's memory map and clear the + * error condition. The caller should retry immediately. + */ + case MDB_MAP_FULL: + /* Can we increase the memory map? Give up if we can't. */ + if (dict_lmdb->map_size < DICT_LMDB_SIZE_MAX / DICT_LMDB_SIZE_INCR) { + dict_lmdb->map_size = dict_lmdb->map_size * DICT_LMDB_SIZE_INCR; + } else if (dict_lmdb->map_size < DICT_LMDB_SIZE_MAX) { + dict_lmdb->map_size = DICT_LMDB_SIZE_MAX; + } else { + /* Sorry, but we are already maxed out. */ + break; + } + /* Resize the memory map. */ + if (msg_verbose) + msg_info("updating database %s:%s size limit to %lu", + dict_lmdb->dict.type, dict_lmdb->dict.name, + (unsigned long) dict_lmdb->map_size); + if ((status = mdb_env_set_mapsize(dict_lmdb->env, + dict_lmdb->map_size)) != 0) + msg_fatal("env_set_mapsize %s:%s to %lu: %s", + dict_lmdb->dict.type, dict_lmdb->dict.name, + (unsigned long) dict_lmdb->map_size, + mdb_strerror(status)); + break; + + /* + * When a writer resizes the database, read-only applications must + * increase their LMDB memory map size limit, too. Otherwise, they + * won't be able to read a table after it grows. + * + * As of LMDB 0.9.8 we can import the new memory map size limit into the + * database environment by calling mdb_env_set_mapsize() with a zero + * size argument. Then we extract the map size limit for later use. + * The caller should retry immediately. + */ + case MDB_MAP_RESIZED: + if ((status = mdb_env_set_mapsize(dict_lmdb->env, 0)) != 0) + msg_fatal("env_set_mapsize %s:%s to 0: %s", + dict_lmdb->dict.type, dict_lmdb->dict.name, + mdb_strerror(status)); + /* Do not panic. Maps may shrink after bulk update. */ + mdb_env_info(dict_lmdb->env, &info); + dict_lmdb->map_size = info.me_mapsize; + if (msg_verbose) + msg_info("importing database %s:%s new size limit %lu", + dict_lmdb->dict.type, dict_lmdb->dict.name, + (unsigned long) dict_lmdb->map_size); + break; + + /* + * We can't solve this problem. The application should terminate with + * a fatal run-time error and the program should be re-run later. + */ + default: + break; + } + + /* + * If a bulk-mode transaction error is recoverable, build a new bulk-mode + * transaction from scratch, by making a long jump back into the caller + * at some pre-arranged point. + */ + if (dict_lmdb->txn != 0 && status == 0 + && (dict_lmdb->bulk_mode_retries += 1) <= DICT_LMDB_BULK_RETRY_LIMIT) { + status = mdb_txn_begin(dict_lmdb->env, NULL, + dict_lmdb->env_flags & MDB_RDONLY, + &dict_lmdb->txn); + if (status != 0) + msg_fatal("txn_begin %s:%s: %s", + dict_lmdb->dict.type, dict_lmdb->dict.name, + mdb_strerror(status)); + dict_lmdb_prepare(dict_lmdb); + dict_longjmp(&dict_lmdb->dict, 1); + } + return (status); +} + +/* dict_lmdb_txn_begin - mdb_txn_begin() wrapper with LMDB error recovery */ + +static void dict_lmdb_txn_begin(DICT_LMDB *dict_lmdb, int rdonly, MDB_txn **txn) +{ + int status; + + if ((status = mdb_txn_begin(dict_lmdb->env, NULL, rdonly, txn)) != 0) { + if ((status = dict_lmdb_recover(dict_lmdb, status)) == 0) { + dict_lmdb_txn_begin(dict_lmdb, rdonly, txn); + return; + } + msg_fatal("%s:%s: error starting %s transaction: %s", + dict_lmdb->dict.type, dict_lmdb->dict.name, + rdonly ? "read" : "write", mdb_strerror(status)); + } +} + +/* dict_lmdb_get - mdb_get() wrapper with LMDB error recovery */ + +static int dict_lmdb_get(DICT_LMDB *dict_lmdb, MDB_val *mdb_key, + MDB_val *mdb_value) +{ + MDB_txn *txn; + int status; + + /* + * Start a read transaction if there's no bulk-mode txn. + */ + if (dict_lmdb->txn) + txn = dict_lmdb->txn; + else + dict_lmdb_txn_begin(dict_lmdb, MDB_RDONLY, &txn); + + /* + * Do the lookup. + */ + if ((status = mdb_get(txn, dict_lmdb->dbi, mdb_key, mdb_value)) != 0 + && status != MDB_NOTFOUND) { + mdb_txn_abort(txn); + if (dict_lmdb->txn == 0) + txn = 0; + if ((status = dict_lmdb_recover(dict_lmdb, status)) == 0) + return (dict_lmdb_get(dict_lmdb, mdb_key, mdb_value)); + } + + /* + * Close the read txn if it's not the bulk-mode txn. + */ + if (txn && dict_lmdb->txn == 0) + mdb_txn_abort(txn); + + return (status); +} + +/* dict_lmdb_put - mdb_put() wrapper with LMDB error recovery */ + +static int dict_lmdb_put(DICT_LMDB *dict_lmdb, MDB_val *mdb_key, + MDB_val *mdb_value, int flags) +{ + MDB_txn *txn; + int status; + + /* + * Start a write transaction if there's no bulk-mode txn. + */ + if (dict_lmdb->txn) + txn = dict_lmdb->txn; + else + dict_lmdb_txn_begin(dict_lmdb, 0, &txn); + + /* + * Do the update. + */ + if ((status = mdb_put(txn, dict_lmdb->dbi, mdb_key, mdb_value, flags)) != 0 + && status != MDB_KEYEXIST) { + mdb_txn_abort(txn); + if (dict_lmdb->txn == 0) + txn = 0; + if ((status = dict_lmdb_recover(dict_lmdb, status)) == 0) + return (dict_lmdb_put(dict_lmdb, mdb_key, mdb_value, flags)); + } + + /* + * Commit the transaction if it's not the bulk-mode txn. + */ + if (txn && dict_lmdb->txn == 0 && (status = mdb_txn_commit(txn)) != 0) { + if ((status = dict_lmdb_recover(dict_lmdb, status)) == 0) + return (dict_lmdb_put(dict_lmdb, mdb_key, mdb_value, flags)); + msg_fatal("error committing database %s:%s: %s", + dict_lmdb->dict.type, dict_lmdb->dict.name, + mdb_strerror(status)); + } + return (status); +} + +/* dict_lmdb_del - mdb_del() wrapper with LMDB error recovery */ + +static int dict_lmdb_del(DICT_LMDB *dict_lmdb, MDB_val *mdb_key) +{ + MDB_txn *txn; + int status; + + /* + * Start a write transaction if there's no bulk-mode txn. + */ + if (dict_lmdb->txn) + txn = dict_lmdb->txn; + else + dict_lmdb_txn_begin(dict_lmdb, 0, &txn); + + /* + * Do the update. + */ + if ((status = mdb_del(txn, dict_lmdb->dbi, mdb_key, NULL)) != 0 + && status != MDB_NOTFOUND) { + mdb_txn_abort(txn); + if (dict_lmdb->txn == 0) + txn = 0; + if ((status = dict_lmdb_recover(dict_lmdb, status)) == 0) + return (dict_lmdb_del(dict_lmdb, mdb_key)); + } + + /* + * Commit the transaction if it's not the bulk-mode txn. + */ + if (txn && dict_lmdb->txn == 0 && (status = mdb_txn_commit(txn)) != 0) { + if ((status = dict_lmdb_recover(dict_lmdb, status)) == 0) + return (dict_lmdb_del(dict_lmdb, mdb_key)); + msg_fatal("error committing database %s:%s: %s", + dict_lmdb->dict.type, dict_lmdb->dict.name, + mdb_strerror(status)); + } + return (status); +} + +/* dict_lmdb_cursor_get - mdb_cursor_get() wrapper with LMDB error recovery */ + +static int dict_lmdb_cursor_get(DICT_LMDB *dict_lmdb, MDB_val *mdb_key, + MDB_val *mdb_value, MDB_cursor_op op) +{ + MDB_txn *txn; + int status; + + /* + * Open a read transaction and cursor if needed. + */ + if (dict_lmdb->cursor == 0) { + dict_lmdb_txn_begin(dict_lmdb, MDB_RDONLY, &txn); + if ((status = mdb_cursor_open(txn, dict_lmdb->dbi, &dict_lmdb->cursor))) { + if ((status = dict_lmdb_recover(dict_lmdb, status)) == 0) + return (dict_lmdb_cursor_get(dict_lmdb, mdb_key, mdb_value, op)); + msg_fatal("%s:%s: cursor_open database: %s", + dict_lmdb->dict.type, dict_lmdb->dict.name, + mdb_strerror(status)); + } + } + + /* + * Database lookup. + */ + status = mdb_cursor_get(dict_lmdb->cursor, mdb_key, mdb_value, op); + if (status != 0 && status != MDB_NOTFOUND) + if ((status = dict_lmdb_recover(dict_lmdb, status)) == 0) + return (dict_lmdb_cursor_get(dict_lmdb, mdb_key, mdb_value, op)); + return (status); +} + +/* dict_lmdb_finish - wrapper with LMDB error recovery */ + +static void dict_lmdb_finish(DICT_LMDB *dict_lmdb) +{ + int status; + + /* + * Finish the bulk-mode transaction. If dict_lmdb_recover() returns after + * a bulk-mode transaction error, then it was unable to recover. + */ + if (dict_lmdb->txn) { + if ((status = mdb_txn_commit(dict_lmdb->txn)) != 0) { + (void) dict_lmdb_recover(dict_lmdb, status); + msg_fatal("%s:%s: closing dictionary: %s", + dict_lmdb->dict.type, dict_lmdb->dict.name, + mdb_strerror(status)); + } + } + + /* + * Clean up after an unfinished sequence() operation. + */ + if (dict_lmdb->cursor) { + MDB_txn *txn = mdb_cursor_txn(dict_lmdb->cursor); + + mdb_cursor_close(dict_lmdb->cursor); + mdb_txn_abort(txn); + } +} + + /* + * With all recovery from LMDB quirks encapsulated in the routines above, + * the dict(3) API routines below can pretend that LMDB quirks don't exist + * and focus on their own job: accessing or updating the database. + */ + /* dict_lmdb_lookup - find database entry */ static const char *dict_lmdb_lookup(DICT *dict, const char *name) @@ -140,11 +515,11 @@ static const char *dict_lmdb_lookup(DICT *dict, const char *name) DICT_LMDB *dict_lmdb = (DICT_LMDB *) dict; MDB_val mdb_key; MDB_val mdb_value; - MDB_txn *txn; const char *result = 0; int status, klen; dict->error = 0; + dict_lmdb->dict_api_retries = 0; klen = strlen(name); /* @@ -163,14 +538,6 @@ static const char *dict_lmdb_lookup(DICT *dict, const char *name) name = lowercase(vstring_str(dict->fold_buf)); } - /* - * Start a read transaction if there's no global txn. - */ - if (dict_lmdb->txn) - txn = dict_lmdb->txn; - else if ((status = mdb_txn_begin(dict_lmdb->env, NULL, MDB_RDONLY, &txn))) - msg_fatal("%s: txn_begin(read) dictionary: %s", dict_lmdb->dict.name, mdb_strerror(status)); - /* * See if this LMDB file was written with one null byte appended to key * and value. @@ -178,10 +545,15 @@ static const char *dict_lmdb_lookup(DICT *dict, const char *name) if (dict->flags & DICT_FLAG_TRY1NULL) { mdb_key.mv_data = (void *) name; mdb_key.mv_size = klen + 1; - status = mdb_get(txn, dict_lmdb->dbi, &mdb_key, &mdb_value); - if (!status) { + status = dict_lmdb_get(dict_lmdb, &mdb_key, &mdb_value); + if (status == 0) { dict->flags &= ~DICT_FLAG_TRY0NULL; - result = SCOPY(dict_lmdb->val_buf, mdb_value.mv_data, mdb_value.mv_size); + result = SCOPY(dict_lmdb->val_buf, mdb_value.mv_data, + mdb_value.mv_size); + } else if (status != MDB_NOTFOUND) { + msg_fatal("error reading %s:%s: %s", + dict_lmdb->dict.type, dict_lmdb->dict.name, + mdb_strerror(status)); } } @@ -192,19 +564,17 @@ static const char *dict_lmdb_lookup(DICT *dict, const char *name) if (result == 0 && (dict->flags & DICT_FLAG_TRY0NULL)) { mdb_key.mv_data = (void *) name; mdb_key.mv_size = klen; - status = mdb_get(txn, dict_lmdb->dbi, &mdb_key, &mdb_value); - if (!status) { + status = dict_lmdb_get(dict_lmdb, &mdb_key, &mdb_value); + if (status == 0) { dict->flags &= ~DICT_FLAG_TRY1NULL; - result = SCOPY(dict_lmdb->val_buf, mdb_value.mv_data, mdb_value.mv_size); + result = SCOPY(dict_lmdb->val_buf, mdb_value.mv_data, + mdb_value.mv_size); + } else if (status != MDB_NOTFOUND) { + msg_fatal("error reading %s:%s: %s", + dict_lmdb->dict.type, dict_lmdb->dict.name, + mdb_strerror(status)); } } - - /* - * Close the read txn if it's not the global txn. - */ - if (!dict_lmdb->txn) - mdb_txn_abort(txn); - return (result); } @@ -215,10 +585,10 @@ static int dict_lmdb_update(DICT *dict, const char *name, const char *value) DICT_LMDB *dict_lmdb = (DICT_LMDB *) dict; MDB_val mdb_key; MDB_val mdb_value; - MDB_txn *txn; int status; dict->error = 0; + dict_lmdb->dict_api_retries = 0; /* * Sanity check. @@ -236,6 +606,7 @@ static int dict_lmdb_update(DICT *dict, const char *name, const char *value) name = lowercase(vstring_str(dict->fold_buf)); } mdb_key.mv_data = (void *) name; + mdb_value.mv_data = (void *) value; mdb_key.mv_size = strlen(name); mdb_value.mv_size = strlen(value); @@ -261,40 +632,27 @@ static int dict_lmdb_update(DICT *dict, const char *name, const char *value) mdb_value.mv_size++; } - /* - * Start a write transaction if there's no global txn. - */ - if (dict_lmdb->txn) - txn = dict_lmdb->txn; - else if (DICT_LMDB_WRAPPER(dict, status, - mdb_txn_begin(dict_lmdb->env, NULL, 0, &txn))) - msg_fatal("%s: txn_begin(write) dictionary: %s", dict_lmdb->dict.name, mdb_strerror(status)); - /* * Do the update. */ - (void) DICT_LMDB_WRAPPER(dict, status, - mdb_put(txn, dict_lmdb->dbi, &mdb_key, &mdb_value, - (dict->flags & DICT_FLAG_DUP_REPLACE) ? 0 : MDB_NOOVERWRITE)); - if (status) { + status = dict_lmdb_put(dict_lmdb, &mdb_key, &mdb_value, + (dict->flags & DICT_FLAG_DUP_REPLACE) ? 0 : MDB_NOOVERWRITE); + if (status != 0) { if (status == MDB_KEYEXIST) { if (dict->flags & DICT_FLAG_DUP_IGNORE) /* void */ ; else if (dict->flags & DICT_FLAG_DUP_WARN) - msg_warn("%s: duplicate entry: \"%s\"", dict_lmdb->dict.name, name); + msg_warn("%s:%s: duplicate entry: \"%s\"", + dict_lmdb->dict.type, dict_lmdb->dict.name, name); else - msg_fatal("%s: duplicate entry: \"%s\"", dict_lmdb->dict.name, name); + msg_fatal("%s:%s: duplicate entry: \"%s\"", + dict_lmdb->dict.type, dict_lmdb->dict.name, name); } else { - msg_fatal("error writing LMDB database %s: %s", dict_lmdb->dict.name, mdb_strerror(status)); + msg_fatal("error updating %s:%s: %s", + dict_lmdb->dict.type, dict_lmdb->dict.name, + mdb_strerror(status)); } } - - /* - * Commit the transaction if it's not the global txn. - */ - if (!dict_lmdb->txn && DICT_LMDB_WRAPPER(dict, status, mdb_txn_commit(txn))) - msg_fatal("error committing LMDB database %s: %s", dict_lmdb->dict.name, mdb_strerror(status)); - return (status); } @@ -304,10 +662,10 @@ static int dict_lmdb_delete(DICT *dict, const char *name) { DICT_LMDB *dict_lmdb = (DICT_LMDB *) dict; MDB_val mdb_key; - MDB_txn *txn; - int status = 1, klen, rc; + int status = 1, klen; dict->error = 0; + dict_lmdb->dict_api_retries = 0; klen = strlen(name); /* @@ -326,15 +684,6 @@ static int dict_lmdb_delete(DICT *dict, const char *name) name = lowercase(vstring_str(dict->fold_buf)); } - /* - * Start a write transaction if there's no global txn. - */ - if (dict_lmdb->txn) - txn = dict_lmdb->txn; - else if (DICT_LMDB_WRAPPER(dict, status, - mdb_txn_begin(dict_lmdb->env, NULL, 0, &txn))) - msg_fatal("%s: txn_begin(write) dictionary: %s", dict_lmdb->dict.name, mdb_strerror(status)); - /* * See if this LMDB file was written with one null byte appended to key * and value. @@ -342,13 +691,14 @@ static int dict_lmdb_delete(DICT *dict, const char *name) if (dict->flags & DICT_FLAG_TRY1NULL) { mdb_key.mv_data = (void *) name; mdb_key.mv_size = klen + 1; - (void) DICT_LMDB_WRAPPER(dict, status, - mdb_del(txn, dict_lmdb->dbi, &mdb_key, NULL)); - if (status) { + status = dict_lmdb_del(dict_lmdb, &mdb_key); + if (status != 0) { if (status == MDB_NOTFOUND) status = 1; else - msg_fatal("error deleting from %s: %s", dict_lmdb->dict.name, mdb_strerror(status)); + msg_fatal("error deleting from %s:%s: %s", + dict_lmdb->dict.type, dict_lmdb->dict.name, + mdb_strerror(status)); } else { dict->flags &= ~DICT_FLAG_TRY0NULL; /* found */ } @@ -361,28 +711,22 @@ static int dict_lmdb_delete(DICT *dict, const char *name) if (status > 0 && (dict->flags & DICT_FLAG_TRY0NULL)) { mdb_key.mv_data = (void *) name; mdb_key.mv_size = klen; - (void) DICT_LMDB_WRAPPER(dict, status, - mdb_del(txn, dict_lmdb->dbi, &mdb_key, NULL)); - if (status) { + status = dict_lmdb_del(dict_lmdb, &mdb_key); + if (status != 0) { if (status == MDB_NOTFOUND) status = 1; else - msg_fatal("error deleting from %s: %s", dict_lmdb->dict.name, mdb_strerror(status)); + msg_fatal("error deleting from %s:%s: %s", + dict_lmdb->dict.type, dict_lmdb->dict.name, + mdb_strerror(status)); } else { dict->flags &= ~DICT_FLAG_TRY1NULL; /* found */ } } - - /* - * Commit the transaction if it's not the global txn. - */ - if (!dict_lmdb->txn && DICT_LMDB_WRAPPER(dict, rc, mdb_txn_commit(txn))) - msg_fatal("error committing LMDB database %s: %s", dict_lmdb->dict.name, mdb_strerror(rc)); - return (status); } -/* traverse the dictionary */ +/* dict_lmdb_sequence - traverse the dictionary */ static int dict_lmdb_sequence(DICT *dict, int function, const char **key, const char **value) @@ -396,6 +740,7 @@ static int dict_lmdb_sequence(DICT *dict, int function, int status; dict->error = 0; + dict_lmdb->dict_api_retries = 0; /* * Determine the seek function. @@ -411,50 +756,42 @@ static int dict_lmdb_sequence(DICT *dict, int function, msg_panic("%s: invalid function: %d", myname, function); } - /* - * Open a read transaction and cursor if needed. - */ - if (dict_lmdb->cursor == 0) { - if ((status = mdb_txn_begin(dict_lmdb->env, NULL, MDB_RDONLY, &txn))) - msg_fatal("%s: txn_begin(read) dictionary: %s", dict_lmdb->dict.name, mdb_strerror(status)); - if ((status = mdb_cursor_open(txn, dict_lmdb->dbi, &dict_lmdb->cursor))) - msg_fatal("%s: cursor_open dictionary: %s", dict_lmdb->dict.name, mdb_strerror(status)); - } - /* * Database lookup. */ - status = mdb_cursor_get(dict_lmdb->cursor, &mdb_key, &mdb_value, op); - if (status && status != MDB_NOTFOUND) - msg_fatal("%s: seeking dictionary: %s", dict_lmdb->dict.name, mdb_strerror(status)); + status = dict_lmdb_cursor_get(dict_lmdb, &mdb_key, &mdb_value, op); - if (status == MDB_NOTFOUND) { + switch (status) { /* - * Caller must read to end, to ensure cursor gets closed. + * Copy the key and value so they are guaranteed null terminated. */ + case 0: + *key = SCOPY(dict_lmdb->key_buf, mdb_key.mv_data, mdb_key.mv_size); + if (mdb_value.mv_data != 0 && mdb_value.mv_size > 0) + *value = SCOPY(dict_lmdb->val_buf, mdb_value.mv_data, + mdb_value.mv_size); + break; + + /* + * Destroy cursor and read transaction. + */ + case MDB_NOTFOUND: status = 1; txn = mdb_cursor_txn(dict_lmdb->cursor); mdb_cursor_close(dict_lmdb->cursor); mdb_txn_abort(txn); dict_lmdb->cursor = 0; - } else { + break; /* - * Copy the key so that it is guaranteed null terminated. + * Bust. */ - *key = SCOPY(dict_lmdb->key_buf, mdb_key.mv_data, mdb_key.mv_size); - - if (mdb_value.mv_data != 0 && mdb_value.mv_size > 0) { - - /* - * Copy the value so that it is guaranteed null terminated. - */ - *value = SCOPY(dict_lmdb->val_buf, mdb_value.mv_data, mdb_value.mv_size); - status = 0; - } + default: + msg_fatal("error seeking %s:%s: %s", + dict_lmdb->dict.type, dict_lmdb->dict.name, + mdb_strerror(status)); } - return (status); } @@ -472,20 +809,8 @@ static void dict_lmdb_close(DICT *dict) { DICT_LMDB *dict_lmdb = (DICT_LMDB *) dict; - if (dict_lmdb->txn) { - int status; - - (void) DICT_LMDB_WRAPPER(dict, status, mdb_txn_commit(dict_lmdb->txn)); - if (status) - msg_fatal("%s: closing dictionary: %s", dict_lmdb->dict.name, mdb_strerror(status)); - dict_lmdb->cursor = NULL; - } - if (dict_lmdb->cursor) { - MDB_txn *txn = mdb_cursor_txn(dict_lmdb->cursor); - - mdb_cursor_close(dict_lmdb->cursor); - mdb_txn_abort(txn); - } + dict_lmdb->dict_api_retries = 0; + dict_lmdb_finish(dict_lmdb); if (dict_lmdb->dict.stat_fd >= 0) close(dict_lmdb->dict.stat_fd); mdb_env_close(dict_lmdb->env); @@ -509,6 +834,7 @@ DICT *dict_lmdb_open(const char *path, int open_flags, int dict_flags) MDB_dbi dbi; char *mdb_path; int env_flags, status; + size_t map_size = dict_lmdb_map_size; mdb_path = concatenate(path, "." DICT_TYPE_LMDB, (char *) 0); @@ -519,73 +845,33 @@ DICT *dict_lmdb_open(const char *path, int open_flags, int dict_flags) if ((status = mdb_env_create(&env))) msg_fatal("env_create %s: %s", mdb_path, mdb_strerror(status)); - /* - * Try to ensure that the LMDB size limit is at least 3x the current LMDB - * file size. This ensures that Postfix daemon processes can recover from - * a "table full" error with a simple terminate-and-restart. - * - * Note: read-only applications must increase their LMDB size limit, too, - * otherwise they won't be able to read a table after it grows. - */ -#ifndef SIZE_T_MAX -#define SIZE_T_MAX __MAXINT__(size_t) -#endif -#define LMDB_SIZE_FUDGE_FACTOR 3 - - if (stat(mdb_path, &st) == 0 - && st.st_size >= dict_lmdb_map_size / LMDB_SIZE_FUDGE_FACTOR) { - msg_warn("%s: file size %lu >= (%s map size limit %ld)/%d -- " - "using a larger map size limit", - mdb_path, (unsigned long) st.st_size, - DICT_TYPE_LMDB, (long) dict_lmdb_map_size, - LMDB_SIZE_FUDGE_FACTOR); - /* Defense against naive optimizers. */ - if (st.st_size < SIZE_T_MAX / LMDB_SIZE_FUDGE_FACTOR) - dict_lmdb_map_size = st.st_size * LMDB_SIZE_FUDGE_FACTOR; - else - dict_lmdb_map_size = SIZE_T_MAX; - } - if ((status = mdb_env_set_mapsize(env, dict_lmdb_map_size))) + if (stat(mdb_path, &st) == 0 && st.st_size > map_size) + map_size = st.st_size; + if ((status = mdb_env_set_mapsize(env, map_size))) msg_fatal("env_set_mapsize %s: %s", mdb_path, mdb_strerror(status)); if ((status = mdb_env_set_maxreaders(env, dict_lmdb_max_readers))) msg_fatal("env_set_maxreaders %s: %s", mdb_path, mdb_strerror(status)); - if ((status = mdb_env_open(env, mdb_path, env_flags, 0644))) - msg_fatal("env_open %s: %s", mdb_path, mdb_strerror(status)); - + /* + * Gracefully handle the most common mistake. + */ + if ((status = mdb_env_open(env, mdb_path, env_flags, 0644))) { + mdb_env_close(env); + return (dict_surrogate(DICT_TYPE_LMDB, path, open_flags, dict_flags, + "open database %s: %m", mdb_path)); + } if ((status = mdb_txn_begin(env, NULL, env_flags & MDB_RDONLY, &txn))) msg_fatal("txn_begin %s: %s", mdb_path, mdb_strerror(status)); /* - * mdb_open requires a txn, but since the default DB always exists in an - * LMDB environment, we usually don't need to do anything else with the - * txn. + * mdb_open() requires a txn, but since the default DB always exists in + * an LMDB environment, we usually don't need to do anything else with + * the txn. It is currently used for bulk transactions. */ if ((status = mdb_open(txn, NULL, 0, &dbi))) msg_fatal("mdb_open %s: %s", mdb_path, mdb_strerror(status)); - /* - * Cases where we use the mdb_open transaction: - * - * - With O_TRUNC we make the "drop" request before populating the database. - * - * - With DICT_FLAG_BULK_UPDATE we commit the transaction when the database - * is closed. - */ - if (open_flags & O_TRUNC) { - if ((status = mdb_drop(txn, dbi, 0))) - msg_fatal("truncate %s: %s", mdb_path, mdb_strerror(status)); - if ((dict_flags & DICT_FLAG_BULK_UPDATE) == 0) { - if ((status = mdb_txn_commit(txn))) - msg_fatal("truncate %s: %s", mdb_path, mdb_strerror(status)); - txn = NULL; - } - } else if ((env_flags & MDB_RDONLY) != 0 - || (dict_flags & DICT_FLAG_BULK_UPDATE) == 0) { - mdb_txn_abort(txn); - txn = NULL; - } dict_lmdb = (DICT_LMDB *) dict_alloc(DICT_TYPE_LMDB, path, sizeof(*dict_lmdb)); dict_lmdb->dict.lookup = dict_lmdb_lookup; dict_lmdb->dict.update = dict_lmdb_update; @@ -619,14 +905,22 @@ DICT *dict_lmdb_open(const char *path, int open_flags, int dict_flags) dict_lmdb->dict.fold_buf = vstring_alloc(10); dict_lmdb->env = env; dict_lmdb->dbi = dbi; - - /* Save the write txn if we opened with O_TRUNC */ - dict_lmdb->txn = txn; + dict_lmdb->map_size = map_size; dict_lmdb->cursor = 0; dict_lmdb->key_buf = 0; dict_lmdb->val_buf = 0; + /* The following facilitate transparent error recovery. */ + dict_lmdb->dict_api_retries = 0; + dict_lmdb->bulk_mode_retries = 0; + dict_lmdb->open_flags = open_flags; + dict_lmdb->env_flags = env_flags; + dict_lmdb->txn = txn; + dict_lmdb_prepare(dict_lmdb); + if (dict_flags & DICT_FLAG_BULK_UPDATE) + dict_jmp_alloc(&dict_lmdb->dict); /* build into dict_alloc() */ + myfree(mdb_path); return (DICT_DEBUG (&dict_lmdb->dict)); diff --git a/postfix/src/util/dict_open.c b/postfix/src/util/dict_open.c index 580d2b3a9..aee1f8ddd 100644 --- a/postfix/src/util/dict_open.c +++ b/postfix/src/util/dict_open.c @@ -44,6 +44,16 @@ /* DICT *(*open) (const char *, int, int); /* /* ARGV *dict_mapnames() +/* +/* int dict_isjmp(dict) +/* DICT *dict; +/* +/* int dict_setjmp(dict) +/* DICT *dict; +/* +/* int dict_longjmp(dict, val) +/* DICT *dict; +/* int val; /* DESCRIPTION /* This module implements a low-level interface to multiple /* physical dictionary types. @@ -112,6 +122,10 @@ /* .IP DICT_FLAG_PARANOID /* A combination of all the paranoia flags: DICT_FLAG_NO_REGSUB, /* DICT_FLAG_NO_PROXY and DICT_FLAG_NO_UNAUTH. +/* .IP DICT_FLAG_BULK_UPDATE +/* Enable preliminary code for bulk-mode database updates. +/* The caller must create an exception handler with dict_jmp_alloc() +/* and must trap exceptions from the database client with dict_setjmp(). /* .IP DICT_FLAG_DEBUG /* Enable additional logging. /* .PP @@ -169,6 +183,18 @@ /* /* dict_mapnames() returns a sorted list with the names of all available /* dictionary types. +/* +/* dict_setjmp() saves processing context and makes that context +/* available for use with dict_longjmp(). Normally, dict_setjmp() +/* returns zero. A non-zero result means that dict_setjmp() +/* returned through a dict_longjmp() call; the result is the +/* \fIval\fR argment given to dict_longjmp(). dict_isjmp() +/* returns non-zero when dict_setjmp() and dict_longjmp() +/* are enabled for a given dictionary. +/* +/* NB: non-local jumps such as dict_longjmp() are not safe for +/* jumping out of any routine that manipulates DICT data. +/* longjmp() like calls are best avoided in signal handlers. /* DIAGNOSTICS /* Fatal error: open error, unsupported dictionary type, attempt to /* update non-writable dictionary. diff --git a/postfix/src/util/dict_test.c b/postfix/src/util/dict_test.c index b7a3df586..409b634b9 100644 --- a/postfix/src/util/dict_test.c +++ b/postfix/src/util/dict_test.c @@ -45,6 +45,8 @@ void dict_test(int argc, char **argv) int n; int rc; +#define USAGE "verbose|del key|get key|put key=value|first|next|masks|flags" + signal(SIGPIPE, SIG_IGN); msg_vstream_init(argv[0], VSTREAM_ERR); @@ -88,7 +90,7 @@ void dict_test(int argc, char **argv) if (*bufp == '#') continue; if ((cmd = mystrtok(&bufp, " ")) == 0) { - vstream_printf("usage: verbose|del key|get key|put key=value|first|next|masks|flags\n"); + vstream_printf("usage: %s\n", USAGE); vstream_fflush(VSTREAM_OUT); continue; } @@ -143,7 +145,7 @@ void dict_test(int argc, char **argv) vstream_printf("DICT_FLAG_INST_MASK %s\n", dict_flags_str(DICT_FLAG_INST_MASK)); } else { - vstream_printf("usage: del key|get key|put key=value|first|next|masks|flags\n"); + vstream_printf("usage: %s\n", USAGE); } vstream_fflush(VSTREAM_OUT); } diff --git a/postfix/src/util/sys_defs.h b/postfix/src/util/sys_defs.h index 2bb01d792..04814a537 100644 --- a/postfix/src/util/sys_defs.h +++ b/postfix/src/util/sys_defs.h @@ -46,15 +46,15 @@ #define HAS_FSYNC #define HAS_DB #define HAS_SA_LEN -#define DEF_DB_TYPE "hash" +#define NATIVE_DB_TYPE "hash" #if (defined(__NetBSD_Version__) && __NetBSD_Version__ >= 104250000) -#define ALIAS_DB_MAP "hash:/etc/mail/aliases" /* sendmail 8.10 */ +#define ALIAS_DB_MAP DEF_DB_TYPE ":/etc/mail/aliases" /* sendmail 8.10 */ #endif #if (defined(OpenBSD) && OpenBSD >= 200006) -#define ALIAS_DB_MAP "hash:/etc/mail/aliases" /* OpenBSD 2.7 */ +#define ALIAS_DB_MAP DEF_DB_TYPE ":/etc/mail/aliases" /* OpenBSD 2.7 */ #endif #ifndef ALIAS_DB_MAP -#define ALIAS_DB_MAP "hash:/etc/aliases" +#define ALIAS_DB_MAP DEF_DB_TYPE ":/etc/aliases" #endif #define GETTIMEOFDAY(t) gettimeofday(t,(struct timezone *) 0) #define ROOT_PATH "/bin:/usr/bin:/sbin:/usr/sbin" @@ -227,8 +227,8 @@ #define HAS_FSYNC #define HAS_DB #define HAS_SA_LEN -#define DEF_DB_TYPE "hash" -#define ALIAS_DB_MAP "hash:/etc/aliases" +#define NATIVE_DB_TYPE "hash" +#define ALIAS_DB_MAP DEF_DB_TYPE ":/etc/aliases" #define GETTIMEOFDAY(t) gettimeofday(t,(struct timezone *) 0) #define ROOT_PATH "/bin:/usr/bin:/sbin:/usr/sbin" #define USE_STATFS @@ -287,12 +287,12 @@ #define HAS_FSYNC /* might be set by makedef */ #ifdef HAS_DB -#define DEF_DB_TYPE "hash" -#define ALIAS_DB_MAP "hash:/etc/aliases" +#define NATIVE_DB_TYPE "hash" +#define ALIAS_DB_MAP DEF_DB_TYPE ":/etc/aliases" #else #define HAS_DBM -#define DEF_DB_TYPE "dbm" -#define ALIAS_DB_MAP "dbm:/etc/aliases" +#define NATIVE_DB_TYPE "dbm" +#define ALIAS_DB_MAP DEF_DB_TYPE ":/etc/aliases" #endif extern int optind; extern char *optarg; @@ -336,8 +336,8 @@ extern int h_errno; #define HAS_FSYNC #define HAVE_BASENAME #define HAS_DBM -#define DEF_DB_TYPE "dbm" -#define ALIAS_DB_MAP "dbm:/var/adm/sendmail/aliases" +#define NATIVE_DB_TYPE "dbm" +#define ALIAS_DB_MAP DEF_DB_TYPE ":/var/adm/sendmail/aliases" extern int optind; /* XXX use */ extern char *optarg; /* XXX use */ extern int opterr; /* XXX use */ @@ -383,8 +383,8 @@ extern int opterr; /* XXX use */ #define DEF_MAILBOX_LOCK "flock, dotlock" #define HAS_FSYNC #define HAS_DBM -#define DEF_DB_TYPE "dbm" -#define ALIAS_DB_MAP "dbm:/etc/aliases" +#define NATIVE_DB_TYPE "dbm" +#define ALIAS_DB_MAP DEF_DB_TYPE ":/etc/aliases" extern int optind; extern char *optarg; extern int opterr; @@ -429,8 +429,8 @@ extern int opterr; #define DEF_MAILBOX_LOCK "fcntl, dotlock" #define HAS_FSYNC #define HAS_DBM -#define DEF_DB_TYPE "dbm" -#define ALIAS_DB_MAP "dbm:/etc/mail/aliases" +#define NATIVE_DB_TYPE "dbm" +#define ALIAS_DB_MAP DEF_DB_TYPE ":/etc/mail/aliases" #ifndef NO_NIS #define HAS_NIS #define HAS_NISPLUS @@ -506,8 +506,8 @@ extern int opterr; #define DEF_MAILBOX_LOCK "fcntl, dotlock" #define HAS_FSYNC #define HAS_DBM -#define DEF_DB_TYPE "dbm" -#define ALIAS_DB_MAP "dbm:/etc/mail/aliases" +#define NATIVE_DB_TYPE "dbm" +#define ALIAS_DB_MAP DEF_DB_TYPE ":/etc/mail/aliases" #ifndef NO_NIS #define HAS_NIS #endif @@ -537,8 +537,8 @@ extern int opterr; #define DEF_MAILBOX_LOCK "fcntl, dotlock" #define HAS_FSYNC #define HAS_DBM -#define DEF_DB_TYPE "dbm" -#define ALIAS_DB_MAP "dbm:/etc/mail/aliases" +#define NATIVE_DB_TYPE "dbm" +#define ALIAS_DB_MAP DEF_DB_TYPE ":/etc/mail/aliases" #ifndef NO_NIS #define HAS_NIS */ #endif @@ -580,8 +580,8 @@ extern int opterr; #define USE_SYS_SELECT_H #define HAS_FSYNC #define HAS_DBM -#define DEF_DB_TYPE "dbm" -#define ALIAS_DB_MAP "dbm:/etc/aliases" +#define NATIVE_DB_TYPE "dbm" +#define ALIAS_DB_MAP DEF_DB_TYPE ":/etc/aliases" #ifndef NO_NIS #define HAS_NIS #endif @@ -634,8 +634,8 @@ extern int opterr; #define USE_SYS_SELECT_H #define HAS_FSYNC #define HAS_DBM -#define DEF_DB_TYPE "dbm" -#define ALIAS_DB_MAP "dbm:/etc/aliases" +#define NATIVE_DB_TYPE "dbm" +#define ALIAS_DB_MAP DEF_DB_TYPE ":/etc/aliases" #ifndef NO_NIS #define HAS_NIS #endif @@ -680,8 +680,8 @@ extern int initgroups(const char *, int); #define USE_SYS_SELECT_H #define HAS_FSYNC #define HAS_DBM -#define DEF_DB_TYPE "dbm" -#define ALIAS_DB_MAP "dbm:/etc/aliases" +#define NATIVE_DB_TYPE "dbm" +#define ALIAS_DB_MAP DEF_DB_TYPE ":/etc/aliases" #ifndef NO_NIS #define HAS_NIS #endif @@ -722,8 +722,8 @@ extern int initgroups(const char *, int); #define DEF_MAILBOX_LOCK "fcntl, dotlock" #define HAS_FSYNC #define HAS_DBM -#define DEF_DB_TYPE "dbm" -#define ALIAS_DB_MAP "dbm:/etc/aliases" +#define NATIVE_DB_TYPE "dbm" +#define ALIAS_DB_MAP DEF_DB_TYPE ":/etc/aliases" #ifndef NO_NIS #define HAS_NIS #endif @@ -766,8 +766,8 @@ extern int initgroups(const char *, int); #define DEF_MAILBOX_LOCK "fcntl, dotlock" /* RedHat >= 4.x */ #define HAS_FSYNC #define HAS_DB -#define DEF_DB_TYPE "hash" -#define ALIAS_DB_MAP "hash:/etc/aliases" +#define NATIVE_DB_TYPE "hash" +#define ALIAS_DB_MAP DEF_DB_TYPE ":/etc/aliases" #ifndef NO_NIS #define HAS_NIS #endif @@ -843,8 +843,8 @@ extern int initgroups(const char *, int); #define DEF_MAILBOX_LOCK "dotlock" /* verified RedHat 3.03 */ #define HAS_FSYNC #define HAS_DB -#define DEF_DB_TYPE "hash" -#define ALIAS_DB_MAP "hash:/etc/aliases" +#define NATIVE_DB_TYPE "hash" +#define ALIAS_DB_MAP DEF_DB_TYPE ":/etc/aliases" #ifndef NO_NIS #define HAS_NIS #endif @@ -877,8 +877,8 @@ extern int initgroups(const char *, int); #define DEF_MAILBOX_LOCK "fcntl, dotlock" /* RedHat >= 4.x */ #define HAS_FSYNC #define HAS_DB -#define DEF_DB_TYPE "hash" -#define ALIAS_DB_MAP "hash:/etc/aliases" +#define NATIVE_DB_TYPE "hash" +#define ALIAS_DB_MAP DEF_DB_TYPE ":/etc/aliases" #ifndef NO_NIS #define HAS_NIS #endif @@ -943,8 +943,8 @@ extern int initgroups(const char *, int); #define INTERNAL_LOCK MYFLOCK_STYLE_FCNTL #define DEF_MAILBOX_LOCK "fcntl, dotlock" #define HAS_FSYNC -#define DEF_DB_TYPE "dbm" -#define ALIAS_DB_MAP "dbm:/etc/mail/aliases" +#define NATIVE_DB_TYPE "dbm" +#define ALIAS_DB_MAP DEF_DB_TYPE ":/etc/mail/aliases" #define ROOT_PATH "/usr/bin:/sbin:/usr/sbin" #define MISSING_SETENV #ifndef NO_NIS @@ -982,8 +982,8 @@ extern int h_errno; /* imports too much stuff */ #define INTERNAL_LOCK MYFLOCK_STYLE_FCNTL #define DEF_MAILBOX_LOCK "fcntl, dotlock" #define HAS_FSYNC -#define DEF_DB_TYPE "dbm" -#define ALIAS_DB_MAP "dbm:/etc/mail/aliases" +#define NATIVE_DB_TYPE "dbm" +#define ALIAS_DB_MAP DEF_DB_TYPE ":/etc/mail/aliases" #define ROOT_PATH "/usr/bin:/sbin:/usr/sbin" #define MISSING_SETENV #ifndef NO_NIS @@ -1027,8 +1027,8 @@ extern int h_errno; /* imports too much stuff */ #define MISSING_SETENV #define MISSING_RLIMIT_FSIZE #define GETTIMEOFDAY(t) gettimeofday(t,(struct timezone *) 0) -#define DEF_DB_TYPE "dbm" -#define ALIAS_DB_MAP "dbm:/usr/lib/aliases" +#define NATIVE_DB_TYPE "dbm" +#define ALIAS_DB_MAP DEF_DB_TYPE ":/usr/lib/aliases" #define ROOT_PATH "/bin:/usr/bin:/etc" #define _PATH_BSHELL "/bin/sh" #define _PATH_MAILDIR "/usr/mail" @@ -1087,7 +1087,7 @@ extern int h_errno; #define _PATH_DEFPATH "/bin:/usr/bin:/usr/ucb" #define _PATH_STDPATH "/bin:/usr/bin:/usr/ucb" #define ROOT_PATH "/bin:/usr/bin:/usr/etc:/usr/ucb" -#define DEF_DB_TYPE "dbm" +#define NATIVE_DB_TYPE "dbm" #define ALIAS_DB_MAP "netinfo:/aliases" #include #define MISSING_POSIX_S_IS @@ -1142,7 +1142,7 @@ typedef unsigned short mode_t; #define _PATH_DEFPATH "/bin:/usr/bin:/usr/ucb" #define _PATH_STDPATH "/bin:/usr/bin:/usr/ucb" #define ROOT_PATH "/bin:/usr/bin:/usr/etc:/usr/ucb" -#define DEF_DB_TYPE "dbm" +#define NATIVE_DB_TYPE "dbm" #define ALIAS_DB_MAP "netinfo:/aliases" #include #define MISSING_POSIX_S_IS @@ -1179,8 +1179,8 @@ typedef unsigned short mode_t; #define FIONREAD_IN_SYS_FILIO_H #define USE_SYS_SOCKIO_H #define HAS_DBM -#define DEF_DB_TYPE "dbm" -#define ALIAS_DB_MAP "dbm:/var/adm/sendmail/aliases" +#define NATIVE_DB_TYPE "dbm" +#define ALIAS_DB_MAP DEF_DB_TYPE ":/var/adm/sendmail/aliases" extern int optind; /* XXX use */ extern char *optarg; /* XXX use */ extern int opterr; /* XXX use */ @@ -1209,8 +1209,8 @@ extern int opterr; /* XXX use */ #define INTERNAL_LOCK MYFLOCK_STYLE_FCNTL #define DEF_MAILBOX_LOCK "fcntl, dotlock" #define HAS_FSYNC -#define DEF_DB_TYPE "hash" -#define ALIAS_DB_MAP "hash:/etc/aliases" +#define NATIVE_DB_TYPE "hash" +#define ALIAS_DB_MAP DEF_DB_TYPE ":/etc/aliases" /* Uncomment the following line if you have NIS package installed */ /* #define HAS_NIS */ #define USE_SYS_SOCKIO_H @@ -1242,8 +1242,8 @@ extern int h_errno; #define DEF_MAILBOX_LOCK "fcntl, dotlock" #define HAS_FSYNC #define HAS_DBM -#define DEF_DB_TYPE "dbm" -#define ALIAS_DB_MAP "dbm:/etc/mail/aliases" +#define NATIVE_DB_TYPE "dbm" +#define ALIAS_DB_MAP DEF_DB_TYPE ":/etc/mail/aliases" #define DBM_NO_TRAILING_NULL #ifndef NO_NIS #define HAS_NIS @@ -1303,6 +1303,10 @@ extern int h_errno; #endif #endif +#ifndef DEF_DB_TYPE +#define DEF_DB_TYPE NATIVE_DB_TYPE +#endif + #define CAST_CHAR_PTR_TO_INT(cptr) ((int) (long) (cptr)) #define CAST_INT_TO_CHAR_PTR(ival) ((char *) (long) (ival))