From 2ccf57f4ac4711daec4548a7571e58aa8c0d177e Mon Sep 17 00:00:00 2001 From: Wietse Venema Date: Tue, 29 Dec 2009 00:00:00 -0500 Subject: [PATCH] postfix-2.7-20091229 --- postfix/.indent.pro | 1 + postfix/HISTORY | 41 +- postfix/README_FILES/SMTPD_PROXY_README | 31 +- postfix/RELEASE_NOTES | 34 +- postfix/WISHLIST | 14 + postfix/html/SMTPD_PROXY_README.html | 15 +- postfix/html/cleanup.8.html | 2 + postfix/html/postconf.5.html | 106 ++++- postfix/html/postscreen.8.html | 217 +++++---- postfix/html/smtpd.8.html | 179 ++++---- postfix/html/verify.8.html | 51 ++- postfix/man/man5/postconf.5 | 82 +++- postfix/man/man8/cleanup.8 | 2 + postfix/man/man8/postscreen.8 | 86 ++-- postfix/man/man8/smtpd.8 | 7 +- postfix/man/man8/verify.8 | 14 + postfix/mantools/postlink | 3 + postfix/proto/SMTPD_PROXY_README.html | 15 +- postfix/proto/postconf.proto | 96 +++- postfix/src/cleanup/cleanup.c | 2 + postfix/src/global/mail_params.h | 16 +- postfix/src/global/mail_version.h | 2 +- postfix/src/postscreen/Makefile.in | 2 + postfix/src/postscreen/postscreen.c | 171 +++++-- postfix/src/smtpd/smtpd.c | 9 +- postfix/src/util/Makefile.in | 18 +- postfix/src/util/dict.c | 7 +- postfix/src/util/dict_cache.c | 577 ++++++++++++++++++++++++ postfix/src/util/dict_cache.h | 49 ++ postfix/src/util/dict_dbm.c | 8 +- postfix/src/util/dict_ht.c | 24 + postfix/src/util/dict_open.c | 6 +- postfix/src/util/events.c | 62 ++- postfix/src/util/htable.c | 47 +- postfix/src/util/htable.h | 7 + postfix/src/verify/Makefile.in | 2 + postfix/src/verify/verify.c | 95 +++- 37 files changed, 1692 insertions(+), 408 deletions(-) create mode 100644 postfix/src/util/dict_cache.c create mode 100644 postfix/src/util/dict_cache.h diff --git a/postfix/.indent.pro b/postfix/.indent.pro index c229c2ce6..5951928c3 100644 --- a/postfix/.indent.pro +++ b/postfix/.indent.pro @@ -57,6 +57,7 @@ -TDELIVER_REQUEST -TDELTA_TIME -TDICT +-TDICT_CACHE -TDICT_CDBM -TDICT_CDBQ -TDICT_CIDR diff --git a/postfix/HISTORY b/postfix/HISTORY index 2cee54e8d..3911a4883 100644 --- a/postfix/HISTORY +++ b/postfix/HISTORY @@ -15453,7 +15453,7 @@ Apologies for any names omitted. 20091023 - Feature: specify "smtp_command_filter = pcre:/file/name" + Feature: specify "smtpd_command_filter = pcre:/file/name" to replace remote SMTP client commands before they are executed by the Postfix SMTP server. This a last-resort tool to fix inter-operability problems. See examples in @@ -15563,3 +15563,42 @@ Apologies for any names omitted. Cleanup: the postscreen daemon now applies the permanent whitelist first. It is a safety feature that prevents mail from being blocked. File: postscreeb/postscreen.c. + +20091224 + + Bugfix (introduced 20041215): dict_dbm_sequence() did not + release the shared lock when the end of the sequence was + reached. File: util/dict_dbm.c. + +20091227 + + Cleanup: postscreen and verify periodic cache cleanup + (default: 12 hours after the previous cache cleanup run). + This is based on a new dict_cache(3) module that implements + a generalized version of the tlsmgr(8) cache maintenance + code. Once the new dict_cache(3) code is burned in, the + tlsmgr(8) will be migrated to it. See the RELEASE_NOTES for + user interface details. Files: util/htable.[hc], util/dict_ht.c, + util/dict_cache.[hc], postscreen/postscreen.c, verify/verify.c. + + Bugfix: the event handler starved I/O events when a timer + call-back routine scheduled a zero-delay timer request. + This bug was exposed when adding the new dict_cache(3) + module for cache expiration. File: util/events.c. + +20091228 + + Cleanup: postscreen and verify periodic cache cleanup is + now optional (specify a null time interval between cache + cleanup runs). + +20091229 + + Cleanup: the address_verify_poll_count default parameter + value is now stress-dependent, so that the Postfix SMTP + server will not wait (up to 6 seconds) for the address + verification result. File: global/mail_params.h. + + Final slution for the I/O event starvation problem when a + timer call-back schedules a zero-delay timer request. File: + util/events.c. diff --git a/postfix/README_FILES/SMTPD_PROXY_README b/postfix/README_FILES/SMTPD_PROXY_README index 27e3eb762..8aadb4a9a 100644 --- a/postfix/README_FILES/SMTPD_PROXY_README +++ b/postfix/README_FILES/SMTPD_PROXY_README @@ -36,23 +36,28 @@ This document describes the following topics: PPrriinncciipplleess ooff ooppeerraattiioonn -The before-filter Postfix SMTP server accepts connections from the Internet and -does the usual relay access control, SASL authentication, TLS negotiation, RBL -lookups, rejecting non-existent sender or recipient addresses, etc. The before- -queue filter receives unfiltered mail content from Postfix and does one of the -following: +As shown in the diagram above, the before-queue filter sits between two Postfix +SMTP server processes. - 1. Re-inject the mail back into Postfix via SMTP, perhaps after changing its - content and/or destination. + * The before-filter Postfix SMTP server accepts connections from the Internet + and does the usual relay access control, SASL authentication, TLS + negotiation, RBL lookups, rejecting non-existent sender or recipient + addresses, etc. - 2. Discard or quarantine the mail. + * The before-queue filter receives unfiltered mail content from Postfix and + does one of the following: - 3. Reject the mail by sending a suitable SMTP status code back to Postfix. - Postfix passes the status back to the remote SMTP client. This way, Postfix - does not have to send a bounce message. + 1. Re-inject the mail back into Postfix via SMTP, perhaps after changing + its content and/or destination. -The after-filter Postfix SMTP server receives mail from the content filter. -From then on Postfix processes the mail as usual. + 2. Discard or quarantine the mail. + + 3. Reject the mail by sending a suitable SMTP status code back to Postfix. + Postfix passes the status back to the remote SMTP client. This way, + Postfix does not have to send a bounce message. + + * The after-filter Postfix SMTP server receives mail from the content filter. + From then on Postfix processes the mail as usual. The before-queue content filter described here works just like the after-queue content filter described in the FILTER_README document. In many cases you can diff --git a/postfix/RELEASE_NOTES b/postfix/RELEASE_NOTES index a7cba8714..95f1d77df 100644 --- a/postfix/RELEASE_NOTES +++ b/postfix/RELEASE_NOTES @@ -14,6 +14,33 @@ specifies the release date of a stable release or snapshot release. If you upgrade from Postfix 2.5 or earlier, read RELEASE_NOTES-2.6 before proceeding. +Incompatibility with snapshot 20091229 +====================================== + +The verify(8) service now uses a persistent cache by default +(address_verify_map = btree:$data_directory/verify_cache). To +disable, specify "address_verify_map =" in main.cf. + +When periodic cache cleanup is enabled (the default), the postscreen(8) +and verify(8) servers now require that their cache databases support +the "delete" and "sequence" operations. To disable periodic cache +cleanup specify a zero xxx_cache_cleanup_interval. + +Major changes with snapshot 20091229 +==================================== + +Periodic cache cleanup for the postscreen(8) and verify(8) cache +databases. The time between cache cleanup runs is controlled with +the address_verify_cache_cleanup_interval (default: 12h) and +postscreen_cache_cleanup_interval (default: 12h) parameters. Cache +cleanup increases the database access latency, so this should not +be run more often than necessary. + +In addition, the postscreen_cache_retention_time (default: 1d) +parameter specifies how long to keep an expired entry in the cache. +This prevents a client from being logged as "NEW" after its record +expired only a little while ago. + Incompatibility with snapshot 20091209 ====================================== @@ -112,11 +139,12 @@ without blocking mail: 1 - Comment out the "smtp inet ... smtpd" service in master.cf, including any "-o parameter=value" entries that follow. -2 - Uncomment the new "smtpd pass ... smtpd" service in master.cf. +2 - Uncomment the new "smtpd pass ... smtpd" service in master.cf, + and duplicate any "-o parameter=value" entries from the smtpd + service that was commented out in step 1. 3 - Uncomment the the new "smtp inet ... postscreen" service in - master.cf, and duplicate any "-o parameter=value" entries from - the smtpd service that was commented out in step 1. + master.cf. 4 - Uncomment the new "dnsblog unix ... dnsblog" service in master.cf. This service does DNSBL lookups for postscreen(8) diff --git a/postfix/WISHLIST b/postfix/WISHLIST index 6430c0a72..38337c5cf 100644 --- a/postfix/WISHLIST +++ b/postfix/WISHLIST @@ -2,11 +2,25 @@ Wish list: Remove this file from the stable release. + It would be nice if the generic dict_cache(3) cache manager + could postpone process suicide until cache cleanup is + completed (but that is not possible when postscreen forks + into the background to finish already-accepted connections). + + When postscreen drops a connection, a 521 "greeting" should + be of the form "521 servername..." and not have an enhanced + status code. The "521 5.7.1" form can be used after EHLO. + Of course no spammer is going to complain about Postfix + SMTP compliance. + Find a place to document all the mail routing mechanisms in one place so people can figure out how Postfix works. owner-listname does not work for shell commands. + Investigate viability of Sendmail socket maps (the moral + equivalent of tcp_table(5)), and dns maps. + The BCC action is marked "not stable", perhaps because people would also expect BCC actions in header/body_checks. How much would it take to make the queue file editing code diff --git a/postfix/html/SMTPD_PROXY_README.html b/postfix/html/SMTPD_PROXY_README.html index 6fc463c22..52702112e 100644 --- a/postfix/html/SMTPD_PROXY_README.html +++ b/postfix/html/SMTPD_PROXY_README.html @@ -108,11 +108,18 @@ filter

Principles of operation

-

The before-filter Postfix SMTP server accepts connections from the +

As shown in the diagram above, the before-queue filter sits +between two Postfix SMTP server processes.

+ + +

The before-queue content filter described here works just like the after-queue content filter described in the FILTER_README document. In many cases you can use the same software, within the diff --git a/postfix/html/cleanup.8.html b/postfix/html/cleanup.8.html index ff8859c1f..70e5e4098 100644 --- a/postfix/html/cleanup.8.html +++ b/postfix/html/cleanup.8.html @@ -58,8 +58,10 @@ CLEANUP(8) CLEANUP(8) RFC 822 (ARPA Internet Text Messages) RFC 2045 (MIME: Format of Internet Message Bodies) RFC 2046 (MIME: Media Types) + RFC 2822 (Internet Message Format) RFC 3463 (Enhanced Status Codes) RFC 3464 (Delivery status notifications) + RFC 5322 (Internet Message Format) DIAGNOSTICS Problems and transactions are logged to syslogd(8). diff --git a/postfix/html/postconf.5.html b/postfix/html/postconf.5.html index 9465bd00c..83c9a2abe 100644 --- a/postfix/html/postconf.5.html +++ b/postfix/html/postconf.5.html @@ -119,6 +119,23 @@ Do not change this unless you have a complete understanding of address_verify_cache_cleanup_interval +(default: 12h)

+ +

The amount of time between verify(8) cache cleanup runs. Cache +cleanup increases the load on the cache database and should therefore +not be run frequently. This feature requires that the cache database +supports the "delete" and "sequence" operators. Specify a zero +interval to disable cache cleanup.

+ +

Time units: s (seconds), m (minutes), h (hours), d (days), w +(weeks).

+ +

This feature is available in Postfix 2.7.

+ +
address_verify_default_transport @@ -249,7 +266,7 @@ This feature is available in Postfix 2.1 and later.
address_verify_poll_count -(default: 3)
+(default: see "postconf -d" output)

How many times to query the verify(8) service for the completion @@ -257,12 +274,16 @@ of an address verification request in progress.

-The default poll count is 3. +With Postfix version 2.7 and later, the SMTP server polls the +verify(8) service up to three times under non-overload conditions, +and only once when under overload. With earlier Postfix versions, +the SMTP server always polls the verify(8) service up to three +times.

Specify 1 to implement a crude form of greylisting, that is, always -defer the first delivery request for a never seen before address. +defer the first delivery request for a new address.

@@ -6561,6 +6582,23 @@ never uses the remote SMTP client hostname.

This feature is available in Postfix 2.7.

+
+ +
postscreen_cache_cleanup_interval +(default: 12h)
+ +

The amount of time between postscreen(8) cache cleanup runs. +Cache cleanup increases the load on the cache database and should +therefore not be run frequently. This feature requires that the +cache database supports the "delete" and "sequence" operators. +Specify a zero interval to disable cache cleanup.

+ +

Time units: s (seconds), m (minutes), h (hours), d (days), w +(weeks).

+ +

This feature is available in Postfix 2.7.

+ +
postscreen_cache_map @@ -6571,6 +6609,22 @@ never uses the remote SMTP client hostname.

This feature is available in Postfix 2.7.

+ + +
postscreen_cache_retention_time +(default: 1d)
+ +

The amount of time that postscreen(8) will cache an expired +temporary whitelist entry before it is removed. This prevents clients +from being logged as "NEW" just because their cache entry expired +an hour ago.

+ +

Time units: s (seconds), m (minutes), h (hours), d (days), w +(weeks).

+ +

This feature is available in Postfix 2.7.

+ +
postscreen_cache_ttl @@ -6579,9 +6633,9 @@ never uses the remote SMTP client hostname.

The amount of time that postscreen(8) will cache a decision for a specific SMTP client IP address. During this time, the client IP address is excluded from tests. If possible, expired decisions are -renewed silently. Specify a non-zero time value (an integral value -plus an optional one-letter suffix that specifies the time unit). -

+renewed automatically. Specify a non-zero time value (an integral +value plus an optional one-letter suffix that specifies the time +unit).

Time units: s (seconds), m (minutes), h (hours), d (days), w (weeks).

@@ -6661,7 +6715,8 @@ IP address.

postscreen_greet_banner (default: $smtpd_banner)
-

The text in the optional "220-text..." server response that +

The text in the optional "220-text..." server +response that postscreen(8) sends ahead of the real Postfix SMTP server's "220 text..." response, in an attempt to confuse bad SMTP clients so that they speak before their turn (pre-greet). Specify an empty @@ -8905,7 +8960,7 @@ invalid responses.

  • In the case of a multi-line reply, the Postfix SMTP client -uses the last reply line's numerical SMTP reply code and enhanced +uses the final reply line's numerical SMTP reply code and enhanced status code.

  • The numerical SMTP reply code (XYZ) takes precedence over @@ -8924,16 +8979,16 @@ server, except that the trailing <CR><LF> are removed.

     /etc/postfix/main.cf:
    -    smtp_reply_filter = pcre:/etc/postfix/command_filter
    +    smtp_reply_filter = pcre:/etc/postfix/reply_filter
     
     /etc/postfix/reply_filter:
    -    # Transform garbage into part of a multi-line reply. Note
    -    # that the Postfix SMTP client uses only the last numerical
    -    # SMTP reply code and enhanced status code from a multi-line
    -    # reply, so it does not matter what we substitute here as
    -    # long as it has the right syntax.
    +    # Transform garbage into "250-filler..." so that it looks like
    +    # one line from a multi-line reply. It does not matter what we
    +    # substitute here as long it has the right syntax.  The Postfix
    +    # SMTP client will use the final line's numerical SMTP reply
    +    # code and enhanced status code.
         !/^([2-5][0-9][0-9]($|[- ]))/ 250-filler for garbage
     
    @@ -11226,6 +11281,20 @@ except that initial whitespace and the trailing <CR><LF> are removed. The result value is executed by the Postfix SMTP server.

    +

    Postfix already implements a number of workarounds for malformed +client commands.

    + + +

    Examples:

    @@ -11557,8 +11626,9 @@ it changes under overload to just 1 with Postfix 2.6 and later.
     (default: no)

    -Require that a remote SMTP client introduces itself at the beginning -of an SMTP session with the HELO or EHLO command. +Require that a remote SMTP client introduces itself with the HELO +or EHLO command before sending the MAIL command or other commands +that require EHLO negotiation.

    @@ -12800,12 +12870,12 @@ inside the chroot jail.

    By default (see smtpd_tls_ask_ccert), client certificates are not requested, and smtpd_tls_CApath should remain empty. In contrast -to smtp_tls_CAfile, DNs of certificate authorities installed +to smtpd_tls_CAfile, DNs of certificate authorities installed in $smtpd_tls_CApath are not included in the client certificate request message. MUAs with multiple client certificates may use the list of preferred certificate authorities to select the correct client certificate. You may want to put your "preferred" CA or -CAs in $smtp_tls_CAfile, and install the remaining trusted CAs in +CAs in $smtpd_tls_CAfile, and install the remaining trusted CAs in $smtpd_tls_CApath.

    Example:

    diff --git a/postfix/html/postscreen.8.html b/postfix/html/postscreen.8.html index 836d3f92d..a0abf42cc 100644 --- a/postfix/html/postscreen.8.html +++ b/postfix/html/postscreen.8.html @@ -14,26 +14,34 @@ POSTSCREEN(8) POSTSCREEN(8) DESCRIPTION The Postfix postscreen(8) server performs triage on multi- - ple inbound SMTP connections in parallel. The program can - run in two basic modes. + ple inbound SMTP connections in parallel. By running + time-consuming tests in parallel in postscreen(8), zombies + and other bogus clients can be kept away from Postfix SMTP + server processes. Thus, more Postfix SMTP server processes + remain available for legitimate clients. - The purpose of observation mode is to collect statistics - without actually blocking mail. postscreen(8) runs a num- - ber of tests before it forwards a connection to a real - SMTP server process. These tests introduce a delay of a - few seconds; once a client passes the tests as "clean", - its IP address is temporarily whitelisted and subsequent - connections incur no delays until the temporary whitelist - entry expires. + This triage process involves a number of tests, documented + below. The tests introduce a delay of a few seconds; once + a client passes the tests, its IP address is temporarily + whitelisted, typically for 24 hours. - The purpose of enforcement mode is to block mail without - using up one Postfix SMTP server process for every connec- - tion. Here, postscreen(8) terminates connections from - SMTP clients that fail the above tests, and forwards only - the remaining connections to a real SMTP server process. - By running time-consuming spam tests in parallel in - postscreen(8), more Postfix SMTP server processes remain - available for legitimate clients. + The program can run in two basic modes. + + Observation mode + postscreen(8) reports the results of the tests, and + forwards all connections to a real Postfix SMTP + server process. + + Enforcement mode + postscreen(8) reports the results of the tests, but + forwards only connections to a real SMTP server + process from clients that passed the tests. + + postscreen(8) disconnects clients that fail the + tests, after sending a 521 status message (a future + version may pass the connection to a dummy SMTP + protocol engine that logs sender and recipient + information). Note: postscreen(8) is not an SMTP proxy; this is inten- tional. The purpose is to prioritize legitimate clients @@ -44,144 +52,145 @@ POSTSCREEN(8) POSTSCREEN(8) 1. PERMANENT WHITELIST TEST The postscreen_whitelist_networks parameter (default: $mynetworks) specifies a permanent whitelist for SMTP - client IP addresses. This feature is not used for - addresses that appear on the permanent blacklist. + client IP addresses. - When the SMTP client address matches the permanent + When the SMTP client address matches the permanent whitelist, this is logged as: WHITELISTED address - The action is not configurable: immediately forward the + The action is not configurable: immediately forward the connection to a real SMTP server process. 2. PERMANENT BLACKLIST TEST - The postscreen_blacklist_networks parameter (default: - empty) specifies a permanent blacklist for SMTP client IP + The postscreen_blacklist_networks parameter (default: + empty) specifies a permanent blacklist for SMTP client IP addresses. The address syntax is as with mynetworks. - When the SMTP client address matches the permanent black- + When the SMTP client address matches the permanent black- list, this is logged as: BLACKLISTED address - The postscreen_blacklist_action parameter specifies the + The postscreen_blacklist_action parameter specifies the action that is taken next: continue (default, observation mode) - Continue with the SMTP GREETING PHASE TESTS below. + Continue with the SMTP GREETING PHASE TESTS below. drop (enforcement mode) - Drop the connection immediately with a 521 SMTP - reply. In a future implementation, the connection - may instead be passed to a dummy SMTP protocol - engine that logs sender and recipient information. + Drop the connection immediately with a 521 SMTP + reply. In a future implementation, the connection + may instead be passed to a dummy SMTP protocol + engine that logs sender and recipient information. 3. TEMPORARY WHITELIST TEST - The postscreen(8) daemon maintains a temporary whitelist - for SMTP client IP addresses that have passed all the - tests described below. The postscreen_cache_map parameter - specifies the location of the temporary whitelist. The - temporary whitelist is not used for SMTP client addresses + The postscreen(8) daemon maintains a temporary whitelist + for SMTP client IP addresses that have passed all the + tests described below. The postscreen_cache_map parameter + specifies the location of the temporary whitelist. The + temporary whitelist is not used for SMTP client addresses that appear on the permanent blacklist or whitelist. - When the SMTP client address appears on the temporary + When the SMTP client address appears on the temporary whitelist, this is logged as: PASS OLD address - The action is not configurable: immediately forward the - connection to a real SMTP server process. The client is - excluded from further tests until its temporary whitelist + The action is not configurable: immediately forward the + connection to a real SMTP server process. The client is + excluded from further tests until its temporary whitelist entry expires, as controlled with the postscreen_cache_ttl parameter. Expired entries are silently renewed if possi- ble. 4. SMTP GREETING PHASE TESTS - The postscreen_greet_wait parameter specifies a time + The postscreen_greet_wait parameter specifies a time interval during which postscreen(8) runs a number of tests - as described below. These tests run before the client may - see the real SMTP server's "220 text..." server greeting. + in parallel. These tests are described below, and are run + before the client may see the real SMTP server's "220 + text..." server greeting. - When the SMTP client passes all the tests, this is logged - as: + When the SMTP client passes all greeting-phase tests, this + is logged as: PASS NEW address - The action is to forward the connection to a real SMTP - server process and to create a temporary whitelist entry - that excludes the client IP address from further tests + The action is to forward the connection to a real SMTP + server process and to create a temporary whitelist entry + that excludes the client IP address from further tests until the temporary whitelist entry expires, as controlled with the postscreen_cache_ttl parameter. - In a future implementation, the connection may first be - passed to a dummy SMTP protocol engine that implements - more protocol tests including greylisting, before the + In a future implementation, the connection may first be + passed to a dummy SMTP protocol engine that implements + more protocol tests including greylisting, before the client is allowed to talk to a real SMTP server process. 4A. PREGREET TEST - The postscreen_greet_banner parameter specifies the text - for a "220-text..." teaser banner (default: $smtpd_ban- - ner). The postscreen(8) daemon sends this before the - postscreen_greet_wait timer is started. The purpose of - the teaser banner is to confuse SPAM clients so that they - speak before their turn. It has no effect on SMTP clients - that correctly implement the protocol. + The postscreen_greet_banner parameter specifies the text + portion of a "220-text..." teaser banner (default: + $smtpd_banner). The postscreen(8) daemon sends this + before the postscreen_greet_wait timer is started. The + purpose of the teaser banner is to confuse SPAM clients so + that they speak before their turn. It has no effect on + SMTP clients that correctly implement the protocol. - To avoid problems with broken SMTP engines in network - appliances, either exclude them from all tests with the - postscreen_whitelist_networks feature or else specify an - empty postscreen_greet_banner value to disable the + To avoid problems with broken SMTP engines in network + appliances, either exclude them from all tests with the + postscreen_whitelist_networks feature or else specify an + empty postscreen_greet_banner value to disable the "220-text..." teaser banner. - When an SMTP client sends a command before the + When an SMTP client sends a command before the postscreen_greet_wait time has elapsed, this is logged as: PREGREET count after time from address text... Translation: the client at address sent count bytes before - its turn to speak, and this happened time seconds after - the postscreen_greet_wait timer was started. The text is - what the client sent (truncated to 100 bytes, and with + its turn to speak, and this happened time seconds after + the postscreen_greet_wait timer was started. The text is + what the client sent (truncated to 100 bytes, and with non-printable characters replaced with "?"). The postscreen_greet_action parameter specifies the action that is taken next: continue (default, observation mode) - Wait until the postscreen_greet_wait time has + Wait until the postscreen_greet_wait time has elapsed, then report DNSBL lookup results if appli- cable. Either perform DNSBL-related actions or for- - ward the connection to a real SMTP server process. + ward the connection to a real SMTP server process. drop (enforcement mode) - Drop the connection immediately with a 521 SMTP - reply. In a future implementation, the connection - may instead be passed to a dummy SMTP protocol - engine that logs sender and recipient information. + Drop the connection immediately with a 521 SMTP + reply. In a future implementation, the connection + may instead be passed to a dummy SMTP protocol + engine that logs sender and recipient information. 4B. HANGUP TEST - When the SMTP client hangs up without sending any data + When the SMTP client hangs up without sending any data before the postscreen_greet_wait time has elapsed, this is logged as: HANGUP after time from address - The postscreen_hangup_action specifies the action that is + The postscreen_hangup_action specifies the action that is taken next: continue (default, observation mode) - Wait until the postscreen_greet_wait time has + Wait until the postscreen_greet_wait time has elapsed, then report DNSBL lookup results if appli- - cable. Do not forward the broken connection to a + cable. Do not forward the broken connection to a real SMTP server process. drop (enforcement mode) Drop the connection immediately. 4C. DNS BLOCKLIST TEST - The postscreen_dnsbl_sites parameter (default: empty) - specifies a list of DNS blocklist servers. + The postscreen_dnsbl_sites parameter (default: empty) + specifies a list of DNS blocklist servers. These lookups + are made in parallel. When the postscreen_greet_wait time has elapsed, and the SMTP client address is listed with at least one of these @@ -237,14 +246,6 @@ POSTSCREEN(8) POSTSCREEN(8) see the postscreen_blacklist_action parameter for possible actions. - postscreen_cache_map (btree:$data_directory/ps_whitelist) - Persistent storage for the postscreen(8) server - decisions. - - postscreen_cache_ttl (1d) - The amount of time that postscreen(8) will cache a - decision for a specific SMTP client IP address. - postscreen_dnsbl_action (continue) The action that postscreen(8) takes when an SMTP client is listed at the DNS blocklist domains spec- @@ -259,7 +260,7 @@ POSTSCREEN(8) POSTSCREEN(8) ified with the postscreen_greet_wait parameter. postscreen_greet_banner ($smtpd_banner) - The text in the optional "220-text..." server + The text in the optional "220-text..." server response that postscreen(8) sends ahead of the real Postfix SMTP server's "220 text..." response, in an attempt to confuse bad SMTP clients so that they @@ -294,22 +295,40 @@ POSTSCREEN(8) POSTSCREEN(8) The internal service that postscreen(8) forwards allowed connections to. +CACHE CONTROLS + postscreen_cache_cleanup_interval (12h) + The amount of time between postscreen(8) cache + cleanup runs. + + postscreen_cache_map (btree:$data_directory/ps_whitelist) + Persistent storage for the postscreen(8) server + decisions. + + postscreen_cache_retention_time (1d) + The amount of time that postscreen(8) will cache an + expired temporary whitelist entry before it is + removed. + + postscreen_cache_ttl (1d) + The amount of time that postscreen(8) will cache a + decision for a specific SMTP client IP address. + MISCELLANEOUS CONTROLS config_directory (see 'postconf -d' output) - The default location of the Postfix main.cf and + The default location of the Postfix main.cf and master.cf configuration files. daemon_timeout (18000s) - How much time a Postfix daemon process may take to - handle a request before it is terminated by a + How much time a Postfix daemon process may take to + handle a request before it is terminated by a built-in watchdog timer. delay_logging_resolution_limit (2) - The maximal number of digits after the decimal + The maximal number of digits after the decimal point when logging sub-second delay values. command_directory (see 'postconf -d' output) - The location of all postfix administrative com- + The location of all postfix administrative com- mands. ipc_timeout (3600s) @@ -317,24 +336,24 @@ POSTSCREEN(8) POSTSCREEN(8) over an internal communication channel. max_idle (100s) - The maximum amount of time that an idle Postfix - daemon process waits for an incoming connection + The maximum amount of time that an idle Postfix + daemon process waits for an incoming connection before terminating voluntarily. process_id (read-only) - The process ID of a Postfix command or daemon + The process ID of a Postfix command or daemon process. process_name (read-only) - The process name of a Postfix command or daemon + The process name of a Postfix command or daemon process. syslog_facility (mail) The syslog facility of Postfix logging. syslog_name (see 'postconf -d' output) - The mail system name that is prepended to the - process name in syslog records, so that "smtpd" + The mail system name that is prepended to the + process name in syslog records, so that "smtpd" becomes, for example, "postfix/smtpd". SEE ALSO @@ -343,7 +362,7 @@ POSTSCREEN(8) POSTSCREEN(8) syslogd(8), system logging LICENSE - The Secure Mailer license must be distributed with this + The Secure Mailer license must be distributed with this software. AUTHOR(S) diff --git a/postfix/html/smtpd.8.html b/postfix/html/smtpd.8.html index 9685afcc5..c5bf71c69 100644 --- a/postfix/html/smtpd.8.html +++ b/postfix/html/smtpd.8.html @@ -906,123 +906,124 @@ SMTPD(8) SMTPD(8) smtpd_helo_required (no) Require that a remote SMTP client introduces itself - at the beginning of an SMTP session with the HELO - or EHLO command. + with the HELO or EHLO command before sending the + MAIL command or other commands that require EHLO + negotiation. smtpd_helo_restrictions (empty) - Optional restrictions that the Postfix SMTP server + Optional restrictions that the Postfix SMTP server applies in the context of the SMTP HELO command. smtpd_sender_restrictions (empty) - Optional restrictions that the Postfix SMTP server + Optional restrictions that the Postfix SMTP server applies in the context of the MAIL FROM command. smtpd_recipient_restrictions (permit_mynetworks, reject_unauth_destination) The access restrictions that the Postfix SMTP - server applies in the context of the RCPT TO com- + server applies in the context of the RCPT TO com- mand. smtpd_etrn_restrictions (empty) - Optional SMTP server access restrictions in the + Optional SMTP server access restrictions in the context of a client ETRN request. allow_untrusted_routing (no) - Forward mail with sender-specified routing - (user[@%!]remote[@%!]site) from untrusted clients + Forward mail with sender-specified routing + (user[@%!]remote[@%!]site) from untrusted clients to destinations matching $relay_domains. smtpd_restriction_classes (empty) - User-defined aliases for groups of access restric- + User-defined aliases for groups of access restric- tions. smtpd_null_access_lookup_key (<>) - The lookup key to be used in SMTP access(5) tables + The lookup key to be used in SMTP access(5) tables instead of the null sender address. permit_mx_backup_networks (empty) Restrict the use of the permit_mx_backup SMTP - access feature to only domains whose primary MX + access feature to only domains whose primary MX hosts match the listed networks. Available in Postfix version 2.0 and later: smtpd_data_restrictions (empty) - Optional access restrictions that the Postfix SMTP + Optional access restrictions that the Postfix SMTP server applies in the context of the SMTP DATA com- mand. smtpd_expansion_filter (see 'postconf -d' output) - What characters are allowed in $name expansions of + What characters are allowed in $name expansions of RBL reply templates. Available in Postfix version 2.1 and later: smtpd_reject_unlisted_sender (no) - Request that the Postfix SMTP server rejects mail - from unknown sender addresses, even when no - explicit reject_unlisted_sender access restriction + Request that the Postfix SMTP server rejects mail + from unknown sender addresses, even when no + explicit reject_unlisted_sender access restriction is specified. smtpd_reject_unlisted_recipient (yes) - Request that the Postfix SMTP server rejects mail + Request that the Postfix SMTP server rejects mail for unknown recipient addresses, even when no - explicit reject_unlisted_recipient access restric- + explicit reject_unlisted_recipient access restric- tion is specified. Available in Postfix version 2.2 and later: smtpd_end_of_data_restrictions (empty) - Optional access restrictions that the Postfix SMTP - server applies in the context of the SMTP END-OF- + Optional access restrictions that the Postfix SMTP + server applies in the context of the SMTP END-OF- DATA command. SENDER AND RECIPIENT ADDRESS VERIFICATION CONTROLS - Postfix version 2.1 introduces sender and recipient - address verification. This feature is implemented by - sending probe email messages that are not actually deliv- - ered. This feature is requested via the reject_unveri- - fied_sender and reject_unverified_recipient access - restrictions. The status of verification probes is main- + Postfix version 2.1 introduces sender and recipient + address verification. This feature is implemented by + sending probe email messages that are not actually deliv- + ered. This feature is requested via the reject_unveri- + fied_sender and reject_unverified_recipient access + restrictions. The status of verification probes is main- tained by the verify(8) server. See the file ADDRESS_VER- - IFICATION_README for information about how to configure + IFICATION_README for information about how to configure and operate the Postfix sender/recipient address verifica- tion service. - address_verify_poll_count (3) - How many times to query the verify(8) service for - the completion of an address verification request + address_verify_poll_count (see 'postconf -d' output) + How many times to query the verify(8) service for + the completion of an address verification request in progress. address_verify_poll_delay (3s) - The delay between queries for the completion of an + The delay between queries for the completion of an address verification request in progress. address_verify_sender ($double_bounce_sender) - The sender address to use in address verification + The sender address to use in address verification probes; prior to Postfix 2.5 the default was "post- master". unverified_sender_reject_code (450) - The numerical Postfix SMTP server response code - when a recipient address is rejected by the + The numerical Postfix SMTP server response code + when a recipient address is rejected by the reject_unverified_sender restriction. unverified_recipient_reject_code (450) - The numerical Postfix SMTP server response when a + The numerical Postfix SMTP server response when a recipient address is rejected by the reject_unveri- fied_recipient restriction. Available in Postfix version 2.6 and later: unverified_sender_defer_code (450) - The numerical Postfix SMTP server response code - when a sender address probe fails due to a tempo- + The numerical Postfix SMTP server response code + when a sender address probe fails due to a tempo- rary error condition. unverified_recipient_defer_code (450) - The numerical Postfix SMTP server response when a - recipient address probe fails due to a temporary + The numerical Postfix SMTP server response when a + recipient address probe fails due to a temporary error condition. unverified_sender_reject_reason (empty) @@ -1036,7 +1037,7 @@ SMTPD(8) SMTPD(8) unverified_sender_tempfail_action ($reject_temp- fail_action) The Postfix SMTP server's action when reject_unver- - ified_sender fails due to a temporary error condi- + ified_sender fails due to a temporary error condi- tion. unverified_recipient_tempfail_action ($reject_temp- @@ -1046,7 +1047,7 @@ SMTPD(8) SMTPD(8) dition. ACCESS CONTROL RESPONSES - The following parameters control numerical SMTP reply + The following parameters control numerical SMTP reply codes and/or text responses. access_map_reject_code (554) @@ -1054,18 +1055,18 @@ SMTPD(8) SMTPD(8) an access(5) map "reject" action. defer_code (450) - The numerical Postfix SMTP server response code - when a remote SMTP client request is rejected by + The numerical Postfix SMTP server response code + when a remote SMTP client request is rejected by the "defer" restriction. invalid_hostname_reject_code (501) - The numerical Postfix SMTP server response code - when the client HELO or EHLO command parameter is - rejected by the reject_invalid_helo_hostname + The numerical Postfix SMTP server response code + when the client HELO or EHLO command parameter is + rejected by the reject_invalid_helo_hostname restriction. maps_rbl_reject_code (554) - The numerical Postfix SMTP server response code + The numerical Postfix SMTP server response code when a remote SMTP client request is blocked by the reject_rbl_client, reject_rhsbl_client, reject_rhsbl_sender or reject_rhsbl_recipient @@ -1073,53 +1074,53 @@ SMTPD(8) SMTPD(8) non_fqdn_reject_code (504) The numerical Postfix SMTP server reply code when a - client request is rejected by the + client request is rejected by the reject_non_fqdn_helo_hostname, reject_non_fqdn_sender or reject_non_fqdn_recipient restriction. plaintext_reject_code (450) - The numerical Postfix SMTP server response code - when a request is rejected by the reject_plain- + The numerical Postfix SMTP server response code + when a request is rejected by the reject_plain- text_session restriction. reject_code (554) - The numerical Postfix SMTP server response code - when a remote SMTP client request is rejected by + The numerical Postfix SMTP server response code + when a remote SMTP client request is rejected by the "reject" restriction. relay_domains_reject_code (554) - The numerical Postfix SMTP server response code - when a client request is rejected by the + The numerical Postfix SMTP server response code + when a client request is rejected by the reject_unauth_destination recipient restriction. unknown_address_reject_code (450) - The numerical Postfix SMTP server response code - when a sender or recipient address is rejected by + The numerical Postfix SMTP server response code + when a sender or recipient address is rejected by the reject_unknown_sender_domain or reject_unknown_recipient_domain restriction. unknown_client_reject_code (450) - The numerical Postfix SMTP server response code - when a client without valid address <=> name map- + The numerical Postfix SMTP server response code + when a client without valid address <=> name map- ping is rejected by the reject_unknown_client_host- name restriction. unknown_hostname_reject_code (450) - The numerical Postfix SMTP server response code - when the hostname specified with the HELO or EHLO - command is rejected by the + The numerical Postfix SMTP server response code + when the hostname specified with the HELO or EHLO + command is rejected by the reject_unknown_helo_hostname restriction. Available in Postfix version 2.0 and later: default_rbl_reply (see 'postconf -d' output) - The default SMTP server response template for a - request that is rejected by an RBL-based restric- + The default SMTP server response template for a + request that is rejected by an RBL-based restric- tion. multi_recipient_bounce_reject_code (550) - The numerical Postfix SMTP server response code + The numerical Postfix SMTP server response code when a remote SMTP client request is blocked by the reject_multi_recipient_bounce restriction. @@ -1130,38 +1131,38 @@ SMTPD(8) SMTPD(8) access_map_defer_code (450) The numerical Postfix SMTP server response code for - an access(5) map "defer" action, including + an access(5) map "defer" action, including "defer_if_permit" or "defer_if_reject". reject_tempfail_action (defer_if_permit) The Postfix SMTP server's action when a reject-type - restriction fails due to a temporary error condi- + restriction fails due to a temporary error condi- tion. unknown_helo_hostname_tempfail_action ($reject_temp- fail_action) - The Postfix SMTP server's action when + The Postfix SMTP server's action when reject_unknown_helo_hostname fails due to an tempo- rary error condition. unknown_address_tempfail_action ($reject_tempfail_action) - The Postfix SMTP server's action when + The Postfix SMTP server's action when reject_unknown_sender_domain or - reject_unknown_recipient_domain fail due to a tem- + reject_unknown_recipient_domain fail due to a tem- porary error condition. MISCELLANEOUS CONTROLS config_directory (see 'postconf -d' output) - The default location of the Postfix main.cf and + The default location of the Postfix main.cf and master.cf configuration files. daemon_timeout (18000s) - How much time a Postfix daemon process may take to - handle a request before it is terminated by a + How much time a Postfix daemon process may take to + handle a request before it is terminated by a built-in watchdog timer. command_directory (see 'postconf -d' output) - The location of all postfix administrative com- + The location of all postfix administrative com- mands. double_bounce_sender (double-bounce) @@ -1182,37 +1183,37 @@ SMTPD(8) SMTPD(8) and most Postfix daemon processes. max_idle (100s) - The maximum amount of time that an idle Postfix - daemon process waits for an incoming connection + The maximum amount of time that an idle Postfix + daemon process waits for an incoming connection before terminating voluntarily. max_use (100) - The maximal number of incoming connections that a - Postfix daemon process will service before termi- + The maximal number of incoming connections that a + Postfix daemon process will service before termi- nating voluntarily. myhostname (see 'postconf -d' output) The internet hostname of this mail system. mynetworks (see 'postconf -d' output) - The list of "trusted" SMTP clients that have more + The list of "trusted" SMTP clients that have more privileges than "strangers". myorigin ($myhostname) The domain name that locally-posted mail appears to - come from, and that locally posted mail is deliv- + come from, and that locally posted mail is deliv- ered to. process_id (read-only) - The process ID of a Postfix command or daemon + The process ID of a Postfix command or daemon process. process_name (read-only) - The process name of a Postfix command or daemon + The process name of a Postfix command or daemon process. queue_directory (see 'postconf -d' output) - The location of the Postfix top-level queue direc- + The location of the Postfix top-level queue direc- tory. recipient_delimiter (empty) @@ -1220,28 +1221,28 @@ SMTPD(8) SMTPD(8) sions (user+foo). smtpd_banner ($myhostname ESMTP $mail_name) - The text that follows the 220 status code in the + The text that follows the 220 status code in the SMTP greeting banner. syslog_facility (mail) The syslog facility of Postfix logging. syslog_name (see 'postconf -d' output) - The mail system name that is prepended to the - process name in syslog records, so that "smtpd" + The mail system name that is prepended to the + process name in syslog records, so that "smtpd" becomes, for example, "postfix/smtpd". Available in Postfix version 2.2 and later: smtpd_forbidden_commands (CONNECT, GET, POST) - List of commands that causes the Postfix SMTP - server to immediately terminate the session with a + List of commands that causes the Postfix SMTP + server to immediately terminate the session with a 221 code. Available in Postfix version 2.5 and later: smtpd_client_port_logging (no) - Enable logging of the remote SMTP client port in + Enable logging of the remote SMTP client port in addition to the hostname and IP address. SEE ALSO @@ -1271,7 +1272,7 @@ SMTPD(8) SMTPD(8) XFORWARD_README, Postfix XFORWARD extension LICENSE - The Secure Mailer license must be distributed with this + The Secure Mailer license must be distributed with this software. AUTHOR(S) diff --git a/postfix/html/verify.8.html b/postfix/html/verify.8.html index 38db746f5..7143c1cb0 100644 --- a/postfix/html/verify.8.html +++ b/postfix/html/verify.8.html @@ -115,17 +115,23 @@ VERIFY(8) VERIFY(8) The time after which a failed address verification probe needs to be refreshed. + Available with Postfix 2.7 and later: + + address_verify_cache_cleanup_interval (12h) + The amount of time between verify(8) cache cleanup + runs. + PROBE MESSAGE ROUTING CONTROLS - By default, probe messages are delivered via the same - route as regular messages. The following parameters can + By default, probe messages are delivered via the same + route as regular messages. The following parameters can be used to override specific message routing mechanisms. address_verify_relayhost ($relayhost) - Overrides the relayhost parameter setting for + Overrides the relayhost parameter setting for address verification probes. address_verify_transport_maps ($transport_maps) - Overrides the transport_maps parameter setting for + Overrides the transport_maps parameter setting for address verification probes. address_verify_local_transport ($local_transport) @@ -133,7 +139,7 @@ VERIFY(8) VERIFY(8) address verification probes. address_verify_virtual_transport ($virtual_transport) - Overrides the virtual_transport parameter setting + Overrides the virtual_transport parameter setting for address verification probes. address_verify_relay_transport ($relay_transport) @@ -141,17 +147,32 @@ VERIFY(8) VERIFY(8) address verification probes. address_verify_default_transport ($default_transport) - Overrides the default_transport parameter setting + Overrides the default_transport parameter setting for address verification probes. + Available in Postfix 2.3 and later: + + address_verify_sender_dependent_relayhost_maps + ($sender_dependent_relayhost_maps) + Overrides the sender_dependent_relayhost_maps + parameter setting for address verification probes. + + Available in Postfix 2.7 and later: + + address_verify_sender_dependent_default_transport_maps + ($sender_dependent_default_transport_maps) + Overrides the sender_dependent_default_trans- + port_maps parameter setting for address verifica- + tion probes. + MISCELLANEOUS CONTROLS config_directory (see 'postconf -d' output) - The default location of the Postfix main.cf and + The default location of the Postfix main.cf and master.cf configuration files. daemon_timeout (18000s) - How much time a Postfix daemon process may take to - handle a request before it is terminated by a + How much time a Postfix daemon process may take to + handle a request before it is terminated by a built-in watchdog timer. ipc_timeout (3600s) @@ -159,23 +180,23 @@ VERIFY(8) VERIFY(8) over an internal communication channel. process_id (read-only) - The process ID of a Postfix command or daemon + The process ID of a Postfix command or daemon process. process_name (read-only) - The process name of a Postfix command or daemon + The process name of a Postfix command or daemon process. queue_directory (see 'postconf -d' output) - The location of the Postfix top-level queue direc- + The location of the Postfix top-level queue direc- tory. syslog_facility (mail) The syslog facility of Postfix logging. syslog_name (see 'postconf -d' output) - The mail system name that is prepended to the - process name in syslog records, so that "smtpd" + The mail system name that is prepended to the + process name in syslog records, so that "smtpd" becomes, for example, "postfix/smtpd". SEE ALSO @@ -188,7 +209,7 @@ VERIFY(8) VERIFY(8) ADDRESS_VERIFICATION_README, address verification howto LICENSE - The Secure Mailer license must be distributed with this + The Secure Mailer license must be distributed with this software. HISTORY diff --git a/postfix/man/man5/postconf.5 b/postfix/man/man5/postconf.5 index 1b31a4186..5d4c9ddad 100644 --- a/postfix/man/man5/postconf.5 +++ b/postfix/man/man5/postconf.5 @@ -77,6 +77,17 @@ The numerical Postfix SMTP server response code for an \fBaccess\fR(5) map "reject" action. .PP Do not change this unless you have a complete understanding of RFC 2821. +.SH address_verify_cache_cleanup_interval (default: 12h) +The amount of time between \fBverify\fR(8) cache cleanup runs. Cache +cleanup increases the load on the cache database and should therefore +not be run frequently. This feature requires that the cache database +supports the "delete" and "sequence" operators. Specify a zero +interval to disable cache cleanup. +.PP +Time units: s (seconds), m (minutes), h (hours), d (days), w +(weeks). +.PP +This feature is available in Postfix 2.7. .SH address_verify_default_transport (default: $default_transport) Overrides the default_transport parameter setting for address verification probes. @@ -138,14 +149,18 @@ be refreshed. Time units: s (seconds), m (minutes), h (hours), d (days), w (weeks). .PP This feature is available in Postfix 2.1 and later. -.SH address_verify_poll_count (default: 3) +.SH address_verify_poll_count (default: see "postconf -d" output) How many times to query the \fBverify\fR(8) service for the completion of an address verification request in progress. .PP -The default poll count is 3. +With Postfix version 2.7 and later, the SMTP server polls the +\fBverify\fR(8) service up to three times under non-overload conditions, +and only once when under overload. With earlier Postfix versions, +the SMTP server always polls the \fBverify\fR(8) service up to three +times. .PP Specify 1 to implement a crude form of greylisting, that is, always -defer the first delivery request for a never seen before address. +defer the first delivery request for a new address. .PP Example: .PP @@ -3673,16 +3688,38 @@ The blacklist has higher precedence than whitelists. This feature never uses the remote SMTP client hostname. .PP This feature is available in Postfix 2.7. +.SH postscreen_cache_cleanup_interval (default: 12h) +The amount of time between \fBpostscreen\fR(8) cache cleanup runs. +Cache cleanup increases the load on the cache database and should +therefore not be run frequently. This feature requires that the +cache database supports the "delete" and "sequence" operators. +Specify a zero interval to disable cache cleanup. +.PP +Time units: s (seconds), m (minutes), h (hours), d (days), w +(weeks). +.PP +This feature is available in Postfix 2.7. .SH postscreen_cache_map (default: btree:$data_directory/ps_whitelist) Persistent storage for the \fBpostscreen\fR(8) server decisions. .PP This feature is available in Postfix 2.7. +.SH postscreen_cache_retention_time (default: 1d) +The amount of time that \fBpostscreen\fR(8) will cache an expired +temporary whitelist entry before it is removed. This prevents clients +from being logged as "NEW" just because their cache entry expired +an hour ago. +.PP +Time units: s (seconds), m (minutes), h (hours), d (days), w +(weeks). +.PP +This feature is available in Postfix 2.7. .SH postscreen_cache_ttl (default: 1d) The amount of time that \fBpostscreen\fR(8) will cache a decision for a specific SMTP client IP address. During this time, the client IP address is excluded from tests. If possible, expired decisions are -renewed silently. Specify a non-zero time value (an integral value -plus an optional one-letter suffix that specifies the time unit). +renewed automatically. Specify a non-zero time value (an integral +value plus an optional one-letter suffix that specifies the time +unit). .PP Time units: s (seconds), m (minutes), h (hours), d (days), w (weeks). @@ -3722,7 +3759,8 @@ IP address. .PP This feature is available in Postfix 2.7. .SH postscreen_greet_banner (default: $smtpd_banner) -The text in the optional "220-text..." server response that +The \fItext\fR in the optional "220-\fItext\fR..." server +response that \fBpostscreen\fR(8) sends ahead of the real Postfix SMTP server's "220 text..." response, in an attempt to confuse bad SMTP clients so that they speak before their turn (pre-greet). Specify an empty @@ -5028,7 +5066,7 @@ invalid responses. Notes: .IP \(bu In the case of a multi-line reply, the Postfix SMTP client -uses the last reply line's numerical SMTP reply code and enhanced +uses the final reply line's numerical SMTP reply code and enhanced status code. .IP \(bu The numerical SMTP reply code (XYZ) takes precedence over @@ -5047,7 +5085,7 @@ Examples: .na .ft C /etc/postfix/main.cf: - smtp_reply_filter = pcre:/etc/postfix/command_filter + smtp_reply_filter = pcre:/etc/postfix/reply_filter .fi .ad .ft R @@ -5056,11 +5094,11 @@ Examples: .na .ft C /etc/postfix/reply_filter: - # Transform garbage into part of a multi-line reply. Note - # that the Postfix SMTP client uses only the last numerical - # SMTP reply code and enhanced status code from a multi-line - # reply, so it does not matter what we substitute here as - # long as it has the right syntax. + # Transform garbage into "250-filler..." so that it looks like + # one line from a multi-line reply. It does not matter what we + # substitute here as long it has the right syntax. The Postfix + # SMTP client will use the final line's numerical SMTP reply + # code and enhanced status code. !/^([2-5][0-9][0-9]($|[- ]))/ 250-filler for garbage .fi .ad @@ -6890,6 +6928,15 @@ except that initial whitespace and the trailing are removed. The result value is executed by the Postfix SMTP server. .PP +Postfix already implements a number of workarounds for malformed +client commands. +.IP \(bu +Use "resolve_numeric_domain = yes" to accept "\fIuser@ipaddress\fR" +Postfix already accepts the correct form "\fIuser@[ipaddress]\fR". +.IP \(bu +Use "strict_rfc821_envelopes = no" to accept "\fIUser Name +\fR". +.PP Examples: .PP .nf @@ -7104,8 +7151,9 @@ make without delivering mail. The Postfix SMTP server disconnects when the limit is exceeded. Normally the default limit is 20, but it changes under overload to just 1 with Postfix 2.6 and later. .SH smtpd_helo_required (default: no) -Require that a remote SMTP client introduces itself at the beginning -of an SMTP session with the HELO or EHLO command. +Require that a remote SMTP client introduces itself with the HELO +or EHLO command before sending the MAIL command or other commands +that require EHLO negotiation. .PP Example: .PP @@ -7927,12 +7975,12 @@ inside the chroot jail. .PP By default (see smtpd_tls_ask_ccert), client certificates are not requested, and smtpd_tls_CApath should remain empty. In contrast -to smtp_tls_CAfile, DNs of certificate authorities installed +to smtpd_tls_CAfile, DNs of certificate authorities installed in $smtpd_tls_CApath are not included in the client certificate request message. MUAs with multiple client certificates may use the list of preferred certificate authorities to select the correct client certificate. You may want to put your "preferred" CA or -CAs in $smtp_tls_CAfile, and install the remaining trusted CAs in +CAs in $smtpd_tls_CAfile, and install the remaining trusted CAs in $smtpd_tls_CApath. .PP Example: diff --git a/postfix/man/man8/cleanup.8 b/postfix/man/man8/cleanup.8 index a6b678fed..5407ab075 100644 --- a/postfix/man/man8/cleanup.8 +++ b/postfix/man/man8/cleanup.8 @@ -54,8 +54,10 @@ in case of trouble. RFC 822 (ARPA Internet Text Messages) RFC 2045 (MIME: Format of Internet Message Bodies) RFC 2046 (MIME: Media Types) +RFC 2822 (Internet Message Format) RFC 3463 (Enhanced Status Codes) RFC 3464 (Delivery status notifications) +RFC 5322 (Internet Message Format) .SH DIAGNOSTICS .ad .fi diff --git a/postfix/man/man8/postscreen.8 b/postfix/man/man8/postscreen.8 index 380f1cf11..6f08bea79 100644 --- a/postfix/man/man8/postscreen.8 +++ b/postfix/man/man8/postscreen.8 @@ -13,26 +13,31 @@ Postfix SMTP triage server .ad .fi The Postfix \fBpostscreen\fR(8) server performs triage on -multiple inbound SMTP connections in parallel. The program -can run in two basic modes. +multiple inbound SMTP connections in parallel. By running +time-consuming tests in parallel in \fBpostscreen\fR(8), +zombies and other bogus clients can be kept away from Postfix +SMTP server processes. Thus, more Postfix SMTP server +processes remain available for legitimate clients. -The purpose of \fBobservation mode\fR is to collect statistics -without actually blocking mail. \fBpostscreen\fR(8) runs a -number of tests before it forwards a connection to a real -SMTP server process. These tests introduce a delay of a -few seconds; once a client passes the tests as "clean", its -IP address is temporarily whitelisted and subsequent -connections incur no delays until the temporary whitelist -entry expires. +This triage process involves a number of tests, documented +below. The tests introduce a delay of a few seconds; once +a client passes the tests, its IP address is temporarily +whitelisted, typically for 24 hours. -The purpose of \fBenforcement mode\fR is to block mail -without using up one Postfix SMTP server process for every -connection. Here, \fBpostscreen\fR(8) terminates connections -from SMTP clients that fail the above tests, and forwards -only the remaining connections to a real SMTP server process. -By running time-consuming spam tests in parallel in -\fBpostscreen\fR(8), more Postfix SMTP server processes -remain available for legitimate clients. +The program can run in two basic modes. +.IP "\fBObservation mode\fR" +\fBpostscreen\fR(8) reports the results of the tests, and +forwards all connections to a real Postfix SMTP server +process. +.IP "\fBEnforcement mode\fR" +\fBpostscreen\fR(8) reports the results of the tests, but +forwards only connections to a real SMTP server process +from clients that passed the tests. +.sp +\fBpostscreen\fR(8) disconnects clients that fail the tests, +after sending a 521 status message (a future version may +pass the connection to a dummy SMTP protocol engine that +logs sender and recipient information). .PP Note: \fBpostscreen\fR(8) is not an SMTP proxy; this is intentional. The purpose is to prioritize legitimate clients @@ -44,8 +49,7 @@ with as little overhead as possible. .fi The postscreen_whitelist_networks parameter (default: $mynetworks) specifies a permanent whitelist for SMTP client -IP addresses. This feature is not used for addresses that -appear on the permanent blacklist. +IP addresses. When the SMTP client address matches the permanent whitelist, this is logged as: @@ -105,12 +109,13 @@ parameter. Expired entries are silently renewed if possible. .ad .fi The postscreen_greet_wait parameter specifies a time interval -during which \fBpostscreen\fR(8) runs a number of tests as -described below. These tests run before the client may -see the real SMTP server's "220 text..." server greeting. +during which \fBpostscreen\fR(8) runs a number of tests in +parallel. These tests are described below, and are run +before the client may see the real SMTP server's "220 +text..." server greeting. -When the SMTP client passes all the tests, this is logged -as: +When the SMTP client passes all greeting-phase tests, this +is logged as: .sp .nf \fBPASS NEW \fIaddress\fR @@ -129,8 +134,9 @@ to talk to a real SMTP server process. .SH 4A. PREGREET TEST .ad .fi -The postscreen_greet_banner parameter specifies the text -for a "220-text..." teaser banner (default: $smtpd_banner). +The postscreen_greet_banner parameter specifies the \fItext\fR +portion of a "220-\fItext\fR..." teaser banner (default: +$smtpd_banner). The \fBpostscreen\fR(8) daemon sends this before the postscreen_greet_wait timer is started. The purpose of the teaser banner is to confuse SPAM clients so that they speak @@ -191,7 +197,8 @@ Drop the connection immediately. .ad .fi The postscreen_dnsbl_sites parameter (default: empty) -specifies a list of DNS blocklist servers. +specifies a list of DNS blocklist servers. These lookups +are made in parallel. When the postscreen_greet_wait time has elapsed, and the SMTP client address is listed with at least one of these @@ -256,11 +263,6 @@ parameter. .IP "\fBpostscreen_blacklist_networks (empty)\fR" Network addresses that are permanently blacklisted; see the postscreen_blacklist_action parameter for possible actions. -.IP "\fBpostscreen_cache_map (btree:$data_directory/ps_whitelist)\fR" -Persistent storage for the \fBpostscreen\fR(8) server decisions. -.IP "\fBpostscreen_cache_ttl (1d)\fR" -The amount of time that \fBpostscreen\fR(8) will cache a decision for -a specific SMTP client IP address. .IP "\fBpostscreen_dnsbl_action (continue)\fR" The action that \fBpostscreen\fR(8) takes when an SMTP client is listed at the DNS blocklist domains specified with the postscreen_dnsbl_sites @@ -272,7 +274,8 @@ The action that \fBpostscreen\fR(8) takes when an SMTP client speaks before its turn within the time specified with the postscreen_greet_wait parameter. .IP "\fBpostscreen_greet_banner ($smtpd_banner)\fR" -The text in the optional "220-text..." server response that +The \fItext\fR in the optional "220-\fItext\fR..." server +response that \fBpostscreen\fR(8) sends ahead of the real Postfix SMTP server's "220 text..." response, in an attempt to confuse bad SMTP clients so that they speak before their turn (pre-greet). @@ -297,6 +300,21 @@ will not be subjected to \fBpostscreen\fR(8) checks. .IP "\fBsmtpd_service (smtpd)\fR" The internal service that \fBpostscreen\fR(8) forwards allowed connections to. +.SH "CACHE CONTROLS" +.na +.nf +.ad +.fi +.IP "\fBpostscreen_cache_cleanup_interval (12h)\fR" +The amount of time between \fBpostscreen\fR(8) cache cleanup runs. +.IP "\fBpostscreen_cache_map (btree:$data_directory/ps_whitelist)\fR" +Persistent storage for the \fBpostscreen\fR(8) server decisions. +.IP "\fBpostscreen_cache_retention_time (1d)\fR" +The amount of time that \fBpostscreen\fR(8) will cache an expired +temporary whitelist entry before it is removed. +.IP "\fBpostscreen_cache_ttl (1d)\fR" +The amount of time that \fBpostscreen\fR(8) will cache a decision for +a specific SMTP client IP address. .SH "MISCELLANEOUS CONTROLS" .na .nf diff --git a/postfix/man/man8/smtpd.8 b/postfix/man/man8/smtpd.8 index 61bce2244..326500ed3 100644 --- a/postfix/man/man8/smtpd.8 +++ b/postfix/man/man8/smtpd.8 @@ -729,8 +729,9 @@ instead of requiring an explicit ".domain.tld" pattern. Optional SMTP server access restrictions in the context of a client SMTP connection request. .IP "\fBsmtpd_helo_required (no)\fR" -Require that a remote SMTP client introduces itself at the beginning -of an SMTP session with the HELO or EHLO command. +Require that a remote SMTP client introduces itself with the HELO +or EHLO command before sending the MAIL command or other commands +that require EHLO negotiation. .IP "\fBsmtpd_helo_restrictions (empty)\fR" Optional restrictions that the Postfix SMTP server applies in the context of the SMTP HELO command. @@ -791,7 +792,7 @@ verification probes is maintained by the \fBverify\fR(8) server. See the file ADDRESS_VERIFICATION_README for information about how to configure and operate the Postfix sender/recipient address verification service. -.IP "\fBaddress_verify_poll_count (3)\fR" +.IP "\fBaddress_verify_poll_count (see 'postconf -d' output)\fR" How many times to query the \fBverify\fR(8) service for the completion of an address verification request in progress. .IP "\fBaddress_verify_poll_delay (3s)\fR" diff --git a/postfix/man/man8/verify.8 b/postfix/man/man8/verify.8 index 0d9a852d7..a73d8a6b6 100644 --- a/postfix/man/man8/verify.8 +++ b/postfix/man/man8/verify.8 @@ -113,6 +113,10 @@ verification cache. .IP "\fBaddress_verify_negative_refresh_time (3h)\fR" The time after which a failed address verification probe needs to be refreshed. +.PP +Available with Postfix 2.7 and later: +.IP "\fBaddress_verify_cache_cleanup_interval (12h)\fR" +The amount of time between \fBverify\fR(8) cache cleanup runs. .SH "PROBE MESSAGE ROUTING CONTROLS" .na .nf @@ -139,6 +143,16 @@ verification probes. .IP "\fBaddress_verify_default_transport ($default_transport)\fR" Overrides the default_transport parameter setting for address verification probes. +.PP +Available in Postfix 2.3 and later: +.IP "\fBaddress_verify_sender_dependent_relayhost_maps ($sender_dependent_relayhost_maps)\fR" +Overrides the sender_dependent_relayhost_maps parameter setting for address +verification probes. +.PP +Available in Postfix 2.7 and later: +.IP "\fBaddress_verify_sender_dependent_default_transport_maps ($sender_dependent_default_transport_maps)\fR" +Overrides the sender_dependent_default_transport_maps parameter +setting for address verification probes. .SH "MISCELLANEOUS CONTROLS" .na .nf diff --git a/postfix/mantools/postlink b/postfix/mantools/postlink index 78fc376c5..ad64fb78b 100755 --- a/postfix/mantools/postlink +++ b/postfix/mantools/postlink @@ -76,6 +76,7 @@ while (<>) { s;\baddress_verify_negative_cache\b;$&;g; s;\baddress_verify_negative_expire_time\b;$&;g; s;\baddress_verify_negative_refresh_time\b;$&;g; + s;\baddress_verify_cache_cleanup_interval\b;$&;g; s;\baddress_verify_poll_count\b;$&;g; s;\baddress_verify_poll_delay\b;$&;g; s;\baddress_verify_positive_expire_time\b;$&;g; @@ -899,6 +900,8 @@ while (<>) { # postscreen s;\bpostscreen_cache_map\b;$&;g; s;\bpostscreen_cache_ttl\b;$&;g; + s;\bpostscreen_cache_cleanup_interval\b;$&;g; + s;\bpostscreen_cache_retention_time\b;$&;g; s;\bsmtpd_service\b;$&;g; s;\bpostscreen_post_queue_limit\b;$&;g; s;\bpostscreen_pre_queue_limit\b;$&;g; diff --git a/postfix/proto/SMTPD_PROXY_README.html b/postfix/proto/SMTPD_PROXY_README.html index b0aebba84..b76ff82e8 100644 --- a/postfix/proto/SMTPD_PROXY_README.html +++ b/postfix/proto/SMTPD_PROXY_README.html @@ -108,11 +108,18 @@ filter

    Principles of operation

    -

    The before-filter Postfix SMTP server accepts connections from the +

    As shown in the diagram above, the before-queue filter sits +between two Postfix SMTP server processes.

    + +
      + +
    • The before-filter Postfix SMTP server accepts connections from the Internet and does the usual relay access control, SASL authentication, TLS negotiation, RBL lookups, rejecting non-existent sender or recipient addresses, -etc. The before-queue filter receives unfiltered mail content from +etc.

      + +
    • The before-queue filter receives unfiltered mail content from Postfix and does one of the following:

        @@ -129,9 +136,11 @@ Postfix and does one of the following:

      -

      The after-filter Postfix SMTP server receives mail from the +

    • The after-filter Postfix SMTP server receives mail from the content filter. From then on Postfix processes the mail as usual.

      +
    +

    The before-queue content filter described here works just like the after-queue content filter described in the FILTER_README document. In many cases you can use the same software, within the diff --git a/postfix/proto/postconf.proto b/postfix/proto/postconf.proto index 62281a169..5cb6237b2 100644 --- a/postfix/proto/postconf.proto +++ b/postfix/proto/postconf.proto @@ -199,7 +199,7 @@ verification probes. This feature is available in Postfix 2.1 and later.

    -%PARAM address_verify_map +%PARAM address_verify_map

    Optional lookup table for persistent address verification status @@ -280,7 +280,20 @@ Time units: s (seconds), m (minutes), h (hours), d (days), w (weeks). This feature is available in Postfix 2.1 and later.

    -%PARAM address_verify_poll_count 3 +%PARAM address_verify_cache_cleanup_interval 12h + +

    The amount of time between verify(8) cache cleanup runs. Cache +cleanup increases the load on the cache database and should therefore +not be run frequently. This feature requires that the cache database +supports the "delete" and "sequence" operators. Specify a zero +interval to disable cache cleanup.

    + +

    Time units: s (seconds), m (minutes), h (hours), d (days), w +(weeks).

    + +

    This feature is available in Postfix 2.7.

    + +%PARAM address_verify_poll_count see "postconf -d" output

    How many times to query the verify(8) service for the completion @@ -288,12 +301,16 @@ of an address verification request in progress.

    -The default poll count is 3. +With Postfix version 2.7 and later, the SMTP server polls the +verify(8) service up to three times under non-overload conditions, +and only once when under overload. With earlier Postfix versions, +the SMTP server always polls the verify(8) service up to three +times.

    Specify 1 to implement a crude form of greylisting, that is, always -defer the first delivery request for a never seen before address. +defer the first delivery request for a new address.

    @@ -5215,8 +5232,9 @@ This feature is available in Postfix 2.2 and later. %PARAM smtpd_helo_required no

    -Require that a remote SMTP client introduces itself at the beginning -of an SMTP session with the HELO or EHLO command. +Require that a remote SMTP client introduces itself with the HELO +or EHLO command before sending the MAIL command or other commands +that require EHLO negotiation.

    @@ -8631,12 +8649,12 @@ inside the chroot jail.

    By default (see smtpd_tls_ask_ccert), client certificates are not requested, and smtpd_tls_CApath should remain empty. In contrast -to smtp_tls_CAfile, DNs of certificate authorities installed +to smtpd_tls_CAfile, DNs of certificate authorities installed in $smtpd_tls_CApath are not included in the client certificate request message. MUAs with multiple client certificates may use the list of preferred certificate authorities to select the correct client certificate. You may want to put your "preferred" CA or -CAs in $smtp_tls_CAfile, and install the remaining trusted CAs in +CAs in $smtpd_tls_CAfile, and install the remaining trusted CAs in $smtpd_tls_CApath.

    Example:

    @@ -12436,9 +12454,34 @@ receive a 421 reponse.

    The amount of time that postscreen(8) will cache a decision for a specific SMTP client IP address. During this time, the client IP address is excluded from tests. If possible, expired decisions are -renewed silently. Specify a non-zero time value (an integral value -plus an optional one-letter suffix that specifies the time unit). -

    +renewed automatically. Specify a non-zero time value (an integral +value plus an optional one-letter suffix that specifies the time +unit).

    + +

    Time units: s (seconds), m (minutes), h (hours), d (days), w +(weeks).

    + +

    This feature is available in Postfix 2.7.

    + +%PARAM postscreen_cache_retention_time 1d + +

    The amount of time that postscreen(8) will cache an expired +temporary whitelist entry before it is removed. This prevents clients +from being logged as "NEW" just because their cache entry expired +an hour ago.

    + +

    Time units: s (seconds), m (minutes), h (hours), d (days), w +(weeks).

    + +

    This feature is available in Postfix 2.7.

    + +%PARAM postscreen_cache_cleanup_interval 12h + +

    The amount of time between postscreen(8) cache cleanup runs. +Cache cleanup increases the load on the cache database and should +therefore not be run frequently. This feature requires that the +cache database supports the "delete" and "sequence" operators. +Specify a zero interval to disable cache cleanup.

    Time units: s (seconds), m (minutes), h (hours), d (days), w (weeks).

    @@ -12560,7 +12603,8 @@ never uses the remote SMTP client hostname.

    %PARAM postscreen_greet_banner $smtpd_banner -

    The text in the optional "220-text..." server response that +

    The text in the optional "220-text..." server +response that postscreen(8) sends ahead of the real Postfix SMTP server's "220 text..." response, in an attempt to confuse bad SMTP clients so that they speak before their turn (pre-greet). Specify an empty @@ -12608,6 +12652,20 @@ except that initial whitespace and the trailing <CR><LF> are removed. The result value is executed by the Postfix SMTP server.

    +

    Postfix already implements a number of workarounds for malformed +client commands.

    + +
      + +
    • Use "resolve_numeric_domain = yes" to accept "user@ipaddress" +Postfix already accepts the correct form "user@[ipaddress]". +

      + +
    • Use "strict_rfc821_envelopes = no" to accept "User Name +<user@example.com>".

      + +
    +

    Examples:

    @@ -12647,7 +12705,7 @@ invalid responses. 

    • In the case of a multi-line reply, the Postfix SMTP client -uses the last reply line's numerical SMTP reply code and enhanced +uses the final reply line's numerical SMTP reply code and enhanced status code.

    • The numerical SMTP reply code (XYZ) takes precedence over @@ -12666,16 +12724,16 @@ server, except that the trailing <CR><LF> are removed.

       /etc/postfix/main.cf:
      -    smtp_reply_filter = pcre:/etc/postfix/command_filter
      +    smtp_reply_filter = pcre:/etc/postfix/reply_filter
       
       /etc/postfix/reply_filter:
      -    # Transform garbage into part of a multi-line reply. Note
      -    # that the Postfix SMTP client uses only the last numerical
      -    # SMTP reply code and enhanced status code from a multi-line
      -    # reply, so it does not matter what we substitute here as
      -    # long as it has the right syntax.
      +    # Transform garbage into "250-filler..." so that it looks like
      +    # one line from a multi-line reply. It does not matter what we
      +    # substitute here as long it has the right syntax.  The Postfix
      +    # SMTP client will use the final line's numerical SMTP reply
      +    # code and enhanced status code.
           !/^([2-5][0-9][0-9]($|[- ]))/ 250-filler for garbage
       
      diff --git a/postfix/src/cleanup/cleanup.c b/postfix/src/cleanup/cleanup.c index 461ebf84e..c5b71e9d8 100644 --- a/postfix/src/cleanup/cleanup.c +++ b/postfix/src/cleanup/cleanup.c @@ -46,8 +46,10 @@ /* RFC 822 (ARPA Internet Text Messages) /* RFC 2045 (MIME: Format of Internet Message Bodies) /* RFC 2046 (MIME: Media Types) +/* RFC 2822 (Internet Message Format) /* RFC 3463 (Enhanced Status Codes) /* RFC 3464 (Delivery status notifications) +/* RFC 5322 (Internet Message Format) /* DIAGNOSTICS /* Problems and transactions are logged to \fBsyslogd\fR(8). /* BUGS diff --git a/postfix/src/global/mail_params.h b/postfix/src/global/mail_params.h index a440152e1..31dd2c25c 100644 --- a/postfix/src/global/mail_params.h +++ b/postfix/src/global/mail_params.h @@ -2552,7 +2552,7 @@ extern int var_scache_stat_time; extern char *var_verify_service; #define VAR_VERIFY_MAP "address_verify_map" -#define DEF_VERIFY_MAP "" +#define DEF_VERIFY_MAP "btree:$data_directory/verify_cache" extern char *var_verify_map; #define VAR_VERIFY_POS_EXP "address_verify_positive_expire_time" @@ -2575,12 +2575,16 @@ extern int var_verify_neg_try; #define DEF_VERIFY_NEG_CACHE 1 extern bool var_verify_neg_cache; +#define VAR_VERIFY_SCAN_CACHE "address_verify_cache_cleanup_interval" +#define DEF_VERIFY_SCAN_CACHE "12h" +extern int var_verify_scan_cache; + #define VAR_VERIFY_SENDER "address_verify_sender" #define DEF_VERIFY_SENDER "$" VAR_DOUBLE_BOUNCE extern char *var_verify_sender; #define VAR_VERIFY_POLL_COUNT "address_verify_poll_count" -#define DEF_VERIFY_POLL_COUNT 3 +#define DEF_VERIFY_POLL_COUNT "${stress?1}${stress:3}" extern int var_verify_poll_count; #define VAR_VERIFY_POLL_DELAY "address_verify_poll_delay" @@ -3186,6 +3190,14 @@ extern int var_ps_pre_queue_limit; #define DEF_PS_CACHE_TTL "1d" extern int var_ps_cache_ttl; +#define VAR_PS_CACHE_RET "postscreen_cache_retention_time" +#define DEF_PS_CACHE_RET "1d" +extern int var_ps_cache_ret; + +#define VAR_PS_CACHE_SCAN "postscreen_cache_cleanup_interval" +#define DEF_PS_CACHE_SCAN "12h" +extern int var_ps_cache_scan; + #define VAR_PS_GREET_WAIT "postscreen_greet_wait" #define DEF_PS_GREET_WAIT "4s" extern int var_ps_greet_wait; diff --git a/postfix/src/global/mail_version.h b/postfix/src/global/mail_version.h index e187c4eeb..5da201516 100644 --- a/postfix/src/global/mail_version.h +++ b/postfix/src/global/mail_version.h @@ -20,7 +20,7 @@ * Patches change both the patchlevel and the release date. Snapshots have no * patchlevel; they change the release date only. */ -#define MAIL_RELEASE_DATE "20091209" +#define MAIL_RELEASE_DATE "20091229" #define MAIL_VERSION_NUMBER "2.7" #ifdef SNAPSHOT diff --git a/postfix/src/postscreen/Makefile.in b/postfix/src/postscreen/Makefile.in index 3ea518a54..dc3403e6c 100644 --- a/postfix/src/postscreen/Makefile.in +++ b/postfix/src/postscreen/Makefile.in @@ -61,7 +61,9 @@ postscreen.o: ../../include/addr_match_list.h postscreen.o: ../../include/argv.h postscreen.o: ../../include/attr.h postscreen.o: ../../include/connect.h +postscreen.o: ../../include/data_redirect.h postscreen.o: ../../include/dict.h +postscreen.o: ../../include/dict_cache.h postscreen.o: ../../include/events.h postscreen.o: ../../include/format_tv.h postscreen.o: ../../include/htable.h diff --git a/postfix/src/postscreen/postscreen.c b/postfix/src/postscreen/postscreen.c index 2e9d4b8c4..c0a6ede27 100644 --- a/postfix/src/postscreen/postscreen.c +++ b/postfix/src/postscreen/postscreen.c @@ -7,26 +7,31 @@ /* \fBpostscreen\fR [generic Postfix daemon options] /* DESCRIPTION /* The Postfix \fBpostscreen\fR(8) server performs triage on -/* multiple inbound SMTP connections in parallel. The program -/* can run in two basic modes. +/* multiple inbound SMTP connections in parallel. By running +/* time-consuming tests in parallel in \fBpostscreen\fR(8), +/* zombies and other bogus clients can be kept away from Postfix +/* SMTP server processes. Thus, more Postfix SMTP server +/* processes remain available for legitimate clients. /* -/* The purpose of \fBobservation mode\fR is to collect statistics -/* without actually blocking mail. \fBpostscreen\fR(8) runs a -/* number of tests before it forwards a connection to a real -/* SMTP server process. These tests introduce a delay of a -/* few seconds; once a client passes the tests as "clean", its -/* IP address is temporarily whitelisted and subsequent -/* connections incur no delays until the temporary whitelist -/* entry expires. +/* This triage process involves a number of tests, documented +/* below. The tests introduce a delay of a few seconds; once +/* a client passes the tests, its IP address is temporarily +/* whitelisted, typically for 24 hours. /* -/* The purpose of \fBenforcement mode\fR is to block mail -/* without using up one Postfix SMTP server process for every -/* connection. Here, \fBpostscreen\fR(8) terminates connections -/* from SMTP clients that fail the above tests, and forwards -/* only the remaining connections to a real SMTP server process. -/* By running time-consuming spam tests in parallel in -/* \fBpostscreen\fR(8), more Postfix SMTP server processes -/* remain available for legitimate clients. +/* The program can run in two basic modes. +/* .IP "\fBObservation mode\fR" +/* \fBpostscreen\fR(8) reports the results of the tests, and +/* forwards all connections to a real Postfix SMTP server +/* process. +/* .IP "\fBEnforcement mode\fR" +/* \fBpostscreen\fR(8) reports the results of the tests, but +/* forwards only connections to a real SMTP server process +/* from clients that passed the tests. +/* .sp +/* \fBpostscreen\fR(8) disconnects clients that fail the tests, +/* after sending a 521 status message (a future version may +/* pass the connection to a dummy SMTP protocol engine that +/* logs sender and recipient information). /* .PP /* Note: \fBpostscreen\fR(8) is not an SMTP proxy; this is /* intentional. The purpose is to prioritize legitimate clients @@ -38,8 +43,7 @@ /* .fi /* The postscreen_whitelist_networks parameter (default: /* $mynetworks) specifies a permanent whitelist for SMTP client -/* IP addresses. This feature is not used for addresses that -/* appear on the permanent blacklist. +/* IP addresses. /* /* When the SMTP client address matches the permanent whitelist, /* this is logged as: @@ -99,12 +103,13 @@ /* .ad /* .fi /* The postscreen_greet_wait parameter specifies a time interval -/* during which \fBpostscreen\fR(8) runs a number of tests as -/* described below. These tests run before the client may -/* see the real SMTP server's "220 text..." server greeting. +/* during which \fBpostscreen\fR(8) runs a number of tests in +/* parallel. These tests are described below, and are run +/* before the client may see the real SMTP server's "220 +/* text..." server greeting. /* -/* When the SMTP client passes all the tests, this is logged -/* as: +/* When the SMTP client passes all greeting-phase tests, this +/* is logged as: /* .sp /* .nf /* \fBPASS NEW \fIaddress\fR @@ -123,8 +128,9 @@ /* .SH 4A. PREGREET TEST /* .ad /* .fi -/* The postscreen_greet_banner parameter specifies the text -/* for a "220-text..." teaser banner (default: $smtpd_banner). +/* The postscreen_greet_banner parameter specifies the \fItext\fR +/* portion of a "220-\fItext\fR..." teaser banner (default: +/* $smtpd_banner). /* The \fBpostscreen\fR(8) daemon sends this before the /* postscreen_greet_wait timer is started. The purpose of the /* teaser banner is to confuse SPAM clients so that they speak @@ -185,7 +191,8 @@ /* .ad /* .fi /* The postscreen_dnsbl_sites parameter (default: empty) -/* specifies a list of DNS blocklist servers. +/* specifies a list of DNS blocklist servers. These lookups +/* are made in parallel. /* /* When the postscreen_greet_wait time has elapsed, and the /* SMTP client address is listed with at least one of these @@ -240,11 +247,6 @@ /* .IP "\fBpostscreen_blacklist_networks (empty)\fR" /* Network addresses that are permanently blacklisted; see the /* postscreen_blacklist_action parameter for possible actions. -/* .IP "\fBpostscreen_cache_map (btree:$data_directory/ps_whitelist)\fR" -/* Persistent storage for the \fBpostscreen\fR(8) server decisions. -/* .IP "\fBpostscreen_cache_ttl (1d)\fR" -/* The amount of time that \fBpostscreen\fR(8) will cache a decision for -/* a specific SMTP client IP address. /* .IP "\fBpostscreen_dnsbl_action (continue)\fR" /* The action that \fBpostscreen\fR(8) takes when an SMTP client is listed /* at the DNS blocklist domains specified with the postscreen_dnsbl_sites @@ -256,7 +258,8 @@ /* before its turn within the time specified with the postscreen_greet_wait /* parameter. /* .IP "\fBpostscreen_greet_banner ($smtpd_banner)\fR" -/* The text in the optional "220-text..." server response that +/* The \fItext\fR in the optional "220-\fItext\fR..." server +/* response that /* \fBpostscreen\fR(8) sends ahead of the real Postfix SMTP server's "220 /* text..." response, in an attempt to confuse bad SMTP clients so /* that they speak before their turn (pre-greet). @@ -281,6 +284,19 @@ /* .IP "\fBsmtpd_service (smtpd)\fR" /* The internal service that \fBpostscreen\fR(8) forwards allowed /* connections to. +/* CACHE CONTROLS +/* .ad +/* .fi +/* .IP "\fBpostscreen_cache_cleanup_interval (12h)\fR" +/* The amount of time between \fBpostscreen\fR(8) cache cleanup runs. +/* .IP "\fBpostscreen_cache_map (btree:$data_directory/ps_whitelist)\fR" +/* Persistent storage for the \fBpostscreen\fR(8) server decisions. +/* .IP "\fBpostscreen_cache_retention_time (1d)\fR" +/* The amount of time that \fBpostscreen\fR(8) will cache an expired +/* temporary whitelist entry before it is removed. +/* .IP "\fBpostscreen_cache_ttl (1d)\fR" +/* The amount of time that \fBpostscreen\fR(8) will cache a decision for +/* a specific SMTP client IP address. /* MISCELLANEOUS CONTROLS /* .ad /* .fi @@ -347,7 +363,7 @@ #include #include #include -#include +#include #include #include #include @@ -363,6 +379,7 @@ #include #include #include +#include /* Master server protocols. */ @@ -378,6 +395,8 @@ int var_ps_post_queue_limit; int var_ps_pre_queue_limit; int var_proc_limit; int var_ps_cache_ttl; +int var_ps_cache_ret; +int var_ps_cache_scan; int var_ps_greet_wait; char *var_ps_dnsbl_sites; char *var_ps_dnsbl_action; @@ -425,7 +444,7 @@ typedef struct { static int check_queue_length; /* connections being checked */ static int post_queue_length; /* being sent to real SMTPD */ -static DICT *cache_map; /* cache table handle */ +static DICT_CACHE *cache_map; /* cache table handle */ static VSTRING *temp; /* scratchpad */ static char *smtp_service_name; /* path to real SMTPD */ static char *teaser_greeting; /* spamware teaser banner */ @@ -520,26 +539,26 @@ static int ps_addr_match_list_match(ADDR_MATCH_LIST *addr_list, /* ps_dict_get - time-critical table lookup */ -static const char *ps_dict_get(DICT *dict, const char *key) +static const char *ps_dict_get(DICT_CACHE *cache, const char *key) { const char *myname = "ps_dict_get"; const char *result; PS_GET_TIME_BEFORE_LOOKUP; - result = dict_get(dict, key); - PS_CHECK_TIME_AFTER_LOOKUP(dict->name, "lookup"); + result = dict_cache_lookup(cache, key); + PS_CHECK_TIME_AFTER_LOOKUP(dict_cache_name(cache), "lookup"); return (result); } /* ps_dict_put - table dictionary update */ -static void ps_dict_put(DICT *dict, const char *key, const char *value) +static void ps_dict_put(DICT_CACHE *cache, const char *key, const char *value) { const char *myname = "ps_dict_put"; PS_GET_TIME_BEFORE_LOOKUP; - dict_put(dict, key, value); - PS_CHECK_TIME_AFTER_LOOKUP(dict->name, "update"); + dict_cache_update(cache, key, value); + PS_CHECK_TIME_AFTER_LOOKUP(dict_cache_name(cache), "update"); } /* @@ -965,6 +984,23 @@ static void smtp_read_event(int event, char *context) } } +/* postscreen_dump - dump some statistics before exit */ + +static void postscreen_dump(void) +{ + + /* + * Dump preliminary cache cleanup statistics when the process commits + * suicide while a cache cleanup run is in progress. We can't currently + * distinguish between "postfix reload" (we should restart) or "maximal + * idle time reached" (we could finish the cache cleanup first). + */ + if (cache_map) { + dict_cache_close(cache_map); + cache_map = 0; + } +} + /* postscreen_drain - delayed exit after "postfix reload" */ static void postscreen_drain(char *unused_service, char **unused_argv) @@ -989,7 +1025,7 @@ static void postscreen_drain(char *unused_service, char **unused_argv) * version is an improvement over its predecessor. */ if (cache_map != 0) { - dict_close(cache_map); + dict_cache_close(cache_map); cache_map = 0; } for (count = 0; /* see below */ ; count++) { @@ -1181,10 +1217,31 @@ static void postscreen_service(VSTREAM *smtp_client_stream, postscreen_dnsbl_query(smtp_client_addr.buf); } +/* postscreen_cache_validator - validate one cache entry */ + +static int postscreen_cache_validator(const char *client_addr, + const char *stamp_str, + char *unused_context) +{ + time_t stamp_time; + + /* + * This function is called by the cache cleanup pseudo thread. + * + * XX Eliminate code duplication and abstract the parser into a separate + * routine. + * + * Don't report a client as "NEW" just because their cache entry expired. + */ + stamp_time = strtoul(stamp_str, 0, 10); + return (event_time() < stamp_time + var_ps_cache_ttl + var_ps_cache_ret); +} + /* pre_jail_init - pre-jail initialization */ static void pre_jail_init(char *unused_name, char **unused_argv) { + VSTRING *redirect; /* * Open read-only maps as before dropping privilege, for consistency with @@ -1192,7 +1249,7 @@ static void pre_jail_init(char *unused_name, char **unused_argv) */ if (*var_ps_wlist_nets) wlist_nets = - addr_match_list_init(MATCH_FLAG_NONE, var_ps_wlist_nets); + addr_match_list_init(MATCH_FLAG_NONE, var_ps_wlist_nets); if (*var_ps_blist_nets) blist_nets = @@ -1213,22 +1270,26 @@ static void pre_jail_init(char *unused_name, char **unused_argv) * to jail, temporarily drop root privileges. */ SAVE_AND_SET_EUGID(var_owner_uid, var_owner_gid); + redirect = vstring_alloc(100); /* * Keep state in persistent external map. As a safety measure we sync the * database on each update. This hurts on LINUX systems that sync all * their dirty disk blocks whenever any application invokes fsync(). + * + * Start the cache maintenance pseudo thread after dropping privileges. */ #define PS_DICT_OPEN_FLAGS (DICT_FLAG_DUP_REPLACE | DICT_FLAG_SYNC_UPDATE) if (*var_ps_cache_map) - cache_map = dict_open(var_ps_cache_map, - O_CREAT | O_RDWR, - PS_DICT_OPEN_FLAGS); + cache_map = + dict_cache_open(data_redirect_map(redirect, var_ps_cache_map), + O_CREAT | O_RDWR, PS_DICT_OPEN_FLAGS); /* * Clean up and restore privilege. */ + vstring_free(redirect); RESTORE_SAVED_EUGID(); } @@ -1241,6 +1302,7 @@ static void post_jail_init(char *unused_name, char **unused_argv) "continue", PS_ACT_CONT, 0, -1, }; + int expire_flags; /* * This routine runs after the skeleton code has entered the chroot jail. @@ -1275,6 +1337,18 @@ static void post_jail_init(char *unused_name, char **unused_argv) if ((hangup_action = name_code(actions, NAME_CODE_FLAG_NONE, var_ps_hangup_action)) < 0) msg_fatal("bad %s value: %s", VAR_PS_HUP_ACTION, var_ps_hangup_action); + + /* + * Start the cache maintenance pseudo thread last. Early cleanup makes + * verbose logging more informative (we get positive confirmation that + * the cleanup thread runs). + */ + expire_flags = DICT_CACHE_FLAG_EXP_SUMMARY; + if (msg_verbose) + expire_flags |= DICT_CACHE_FLAG_EXP_VERBOSE; + if (cache_map != 0 && var_ps_cache_scan > 0) + dict_cache_expire(cache_map, expire_flags, var_ps_cache_scan, + postscreen_cache_validator, (char *) 0); } MAIL_VERSION_STAMP_DECLARE; @@ -1309,6 +1383,8 @@ int main(int argc, char **argv) static const CONFIG_TIME_TABLE time_table[] = { VAR_PS_CACHE_TTL, DEF_PS_CACHE_TTL, &var_ps_cache_ttl, 1, 0, VAR_PS_GREET_WAIT, DEF_PS_GREET_WAIT, &var_ps_greet_wait, 1, 0, + VAR_PS_CACHE_RET, DEF_PS_CACHE_RET, &var_ps_cache_ret, 0, 0, + VAR_PS_CACHE_SCAN, DEF_PS_CACHE_SCAN, &var_ps_cache_scan, 0, 0, 0, }; @@ -1326,5 +1402,6 @@ int main(int argc, char **argv) MAIL_SERVER_POST_INIT, post_jail_init, MAIL_SERVER_SOLITARY, MAIL_SERVER_SLOW_EXIT, postscreen_drain, + MAIL_SERVER_EXIT, postscreen_dump, 0); } diff --git a/postfix/src/smtpd/smtpd.c b/postfix/src/smtpd/smtpd.c index cb855005f..57acbedac 100644 --- a/postfix/src/smtpd/smtpd.c +++ b/postfix/src/smtpd/smtpd.c @@ -681,8 +681,9 @@ /* Optional SMTP server access restrictions in the context of a client /* SMTP connection request. /* .IP "\fBsmtpd_helo_required (no)\fR" -/* Require that a remote SMTP client introduces itself at the beginning -/* of an SMTP session with the HELO or EHLO command. +/* Require that a remote SMTP client introduces itself with the HELO +/* or EHLO command before sending the MAIL command or other commands +/* that require EHLO negotiation. /* .IP "\fBsmtpd_helo_restrictions (empty)\fR" /* Optional restrictions that the Postfix SMTP server applies in the /* context of the SMTP HELO command. @@ -741,7 +742,7 @@ /* See the file ADDRESS_VERIFICATION_README for information /* about how to configure and operate the Postfix sender/recipient /* address verification service. -/* .IP "\fBaddress_verify_poll_count (3)\fR" +/* .IP "\fBaddress_verify_poll_count (see 'postconf -d' output)\fR" /* How many times to query the \fBverify\fR(8) service for the completion /* of an address verification request in progress. /* .IP "\fBaddress_verify_poll_delay (3s)\fR" @@ -4942,6 +4943,7 @@ int main(int argc, char **argv) VAR_SMTPD_SOFT_ERLIM, DEF_SMTPD_SOFT_ERLIM, &var_smtpd_soft_erlim, 1, 0, VAR_SMTPD_HARD_ERLIM, DEF_SMTPD_HARD_ERLIM, &var_smtpd_hard_erlim, 1, 0, VAR_SMTPD_JUNK_CMD, DEF_SMTPD_JUNK_CMD, &var_smtpd_junk_cmd_limit, 1, 0, + VAR_VERIFY_POLL_COUNT, DEF_VERIFY_POLL_COUNT, &var_verify_poll_count, 1, 0, 0, }; static const CONFIG_INT_TABLE int_table[] = { @@ -4970,7 +4972,6 @@ int main(int argc, char **argv) VAR_VIRT_MAILBOX_CODE, DEF_VIRT_MAILBOX_CODE, &var_virt_mailbox_code, 0, 0, VAR_RELAY_RCPT_CODE, DEF_RELAY_RCPT_CODE, &var_relay_rcpt_code, 0, 0, VAR_PLAINTEXT_CODE, DEF_PLAINTEXT_CODE, &var_plaintext_code, 0, 0, - VAR_VERIFY_POLL_COUNT, DEF_VERIFY_POLL_COUNT, &var_verify_poll_count, 1, 0, VAR_SMTPD_CRATE_LIMIT, DEF_SMTPD_CRATE_LIMIT, &var_smtpd_crate_limit, 0, 0, VAR_SMTPD_CCONN_LIMIT, DEF_SMTPD_CCONN_LIMIT, &var_smtpd_cconn_limit, 0, 0, VAR_SMTPD_CMAIL_LIMIT, DEF_SMTPD_CMAIL_LIMIT, &var_smtpd_cmail_limit, 0, 0, diff --git a/postfix/src/util/Makefile.in b/postfix/src/util/Makefile.in index 96ca508b8..48f9fae22 100644 --- a/postfix/src/util/Makefile.in +++ b/postfix/src/util/Makefile.in @@ -32,7 +32,7 @@ SRCS = alldig.c allprint.c argv.c argv_split.c attr_clnt.c attr_print0.c \ write_buf.c write_wait.c sane_basename.c format_tv.c allspace.c \ allascii.c load_file.c killme_after.c vstream_tweak.c upass_connect.c \ upass_listen.c upass_trigger.c edit_file.c inet_windowsize.c \ - unix_pass_fd_fix.c + unix_pass_fd_fix.c dict_cache.c OBJS = alldig.o allprint.o argv.o argv_split.o attr_clnt.o attr_print0.o \ attr_print64.o attr_print_plain.o attr_scan0.o attr_scan64.o \ attr_scan_plain.o auto_clnt.o base64_code.o basename.o binhash.o \ @@ -66,7 +66,7 @@ OBJS = alldig.o allprint.o argv.o argv_split.o attr_clnt.o attr_print0.o \ write_buf.o write_wait.o sane_basename.o format_tv.o allspace.o \ allascii.o load_file.o killme_after.o vstream_tweak.o upass_connect.o \ upass_listen.o upass_trigger.o edit_file.o inet_windowsize.o \ - unix_pass_fd_fix.o + unix_pass_fd_fix.o dict_cache.o HDRS = argv.h attr.h attr_clnt.h auto_clnt.h base64_code.h binhash.h \ chroot_uid.h cidr_match.h clean_env.h connect.h ctable.h dict.h \ dict_cdb.h dict_cidr.h dict_db.h dict_dbm.h dict_env.h dict_ht.h \ @@ -86,7 +86,7 @@ HDRS = argv.h attr.h attr_clnt.h auto_clnt.h base64_code.h binhash.h \ stringops.h sys_defs.h timed_connect.h timed_wait.h trigger.h \ username.h valid_hostname.h vbuf.h vbuf_print.h vstream.h vstring.h \ vstring_vstream.h watchdog.h format_tv.h load_file.h killme_after.h \ - edit_file.h + edit_file.h dict_cache.h TESTSRC = fifo_open.c fifo_rdwr_bug.c fifo_rdonly_bug.c select_bug.c \ stream_test.c dup2_pass_on_exec.c test_send_fd test_recv_fd DEFS = -I. -D$(SYSTYPE) @@ -740,6 +740,17 @@ dict_alloc.o: sys_defs.h dict_alloc.o: vbuf.h dict_alloc.o: vstream.h dict_alloc.o: vstring.h +dict_cache.o: argv.h +dict_cache.o: dict.h +dict_cache.o: dict_cache.c +dict_cache.o: dict_cache.h +dict_cache.o: events.h +dict_cache.o: msg.h +dict_cache.o: mymalloc.h +dict_cache.o: sys_defs.h +dict_cache.o: vbuf.h +dict_cache.o: vstream.h +dict_cache.o: vstring.h dict_cdb.o: argv.h dict_cdb.o: dict.h dict_cdb.o: dict_cdb.c @@ -1505,7 +1516,6 @@ stream_trigger.o: mymalloc.h stream_trigger.o: stream_trigger.c stream_trigger.o: sys_defs.h stream_trigger.o: trigger.h -sys_compat.o: iostuff.h sys_compat.o: sys_compat.c sys_compat.o: sys_defs.h timed_connect.o: iostuff.h diff --git a/postfix/src/util/dict.c b/postfix/src/util/dict.c index 4747d33e7..68645d401 100644 --- a/postfix/src/util/dict.c +++ b/postfix/src/util/dict.c @@ -107,15 +107,16 @@ /* modified, or if the result is to survive multiple dict_lookup() calls. /* /* dict_delete() removes the named member from the named dictionary. -/* The result is non-zero when the member does not exist. +/* The result value is zero when the member was found. /* -/* dict_sequence() steps throuh the named dictionary and returns +/* dict_sequence() steps through the named dictionary and returns /* keys and values in some implementation-defined order. The func /* argument is DICT_SEQ_FUN_FIRST to set the cursor to the first /* entry or DICT_SEQ_FUN_NEXT to select the next entry. The result /* is owned by the underlying dictionary method. Make a copy if the /* result is to be modified, or if the result is to survive multiple -/* dict_sequence() calls. +/* dict_sequence() calls. The result value is zero when a member +/* was found. /* /* dict_eval() expands macro references in the specified string. /* The result is owned by the dictionary manager. Make a copy if the diff --git a/postfix/src/util/dict_cache.c b/postfix/src/util/dict_cache.c new file mode 100644 index 000000000..16e17037d --- /dev/null +++ b/postfix/src/util/dict_cache.c @@ -0,0 +1,577 @@ +/*++ +/* NAME +/* dict_cache 3 +/* SUMMARY +/* External cache manager +/* SYNOPSIS +/* #include +/* +/* DICT_CACHE *dict_cache_open(dbname, open_flags, dict_flags) +/* const char *dbname; +/* int open_flags; +/* int dict_flags; +/* +/* void dict_cache_close(cache) +/* DICT_CACHE *cache; +/* +/* const char *dict_cache_lookup(cache, cache_key) +/* DICT_CACHE *cache; +/* const char *cache_key; +/* +/* int dict_cache_update(cache, cache_key, cache_val) +/* DICT_CACHE *cache; +/* const char *cache_key; +/* const char *cache_val; +/* +/* int dict_cache_delete(cache, cache_key) +/* DICT_CACHE *cache; +/* const char *cache_key; +/* +/* int dict_cache_sequence(cache, first_next, cache_key, cache_val) +/* DICT_CACHE *cache; +/* int first_next; +/* const char **cache_key; +/* const char **cache_val; +/* +/* void dict_cache_expire(cache, flags, interval, validator, context) +/* DICT_CACHE *cache; +/* int flags; +/* int interval; +/* int (*validator)(const char *cache_key, const char *cache_val, +/* char *context); +/* char *context; +/* AUXILIARY FUNCTIONS +/* const char *dict_cache_name(cache) +/* DICT_CACHE *cache; +/* +/* DICT_CACHE *dict_cache_import(table) +/* DICT *table; +/* DESCRIPTION +/* This module maintains external cache files with support +/* for expiration. The underlying table must implement the +/* "lookup", "update", "delete" and "sequence" operations. +/* +/* Although this API is similar to the one documented in +/* dict_open(3), there are subtle differences in the interaction +/* between the iterators that access all cache elements, and +/* other operations that access individual cache elements. +/* +/* In particular, when a "sequence" or "expire" operation is +/* in progress the cache intercepts requests to delete the +/* "current" entry, as this would cause some databases to +/* mis-behave. Instead, the cache implements a "delete behind" +/* strategy, and deletes such an entry after the "sequence" +/* or "expire" operation moves on to the next cache element. +/* The "delete behind" strategy also affects the cache lookup +/* and update operations as detailed below. +/* +/* dict_cache_open() opens the specified cache and returns a +/* handle that must be used for subsequent access. This function +/* does not return in case of error. +/* +/* dict_cache_close() closes the specified cache and releases +/* memory that was allocated by dict_cache_open(), and terminates +/* any thread that was started with dict_cache_expire(). +/* +/* dict_cache_lookup() looks up the specified cache entry. +/* The result value is a null pointer when the cache entry was +/* not found, or when the entry is scheduled for "delete +/* behind". +/* +/* dict_cache_update() updates the specified cache entry. If +/* the entry is scheduled for "delete behind", the delete +/* operation is canceled (meaning that the cache must be opened +/* with DICT_FLAG_DUP_REPLACE). This function does not return +/* in case of error. +/* +/* dict_cache_delete() removes the specified cache entry. If +/* this is the "current" entry of a "sequence" operation, the +/* entry is scheduled for "delete behind". The result value +/* is zero when the entry was found. +/* +/* dict_cache_sequence() iterates over the specified cache and +/* returns each entry in an implementation-defined order. The +/* result value is zero when a cache entry was found. Programs +/* must not use both dict_cache_sequence() and dict_cache_expire(). +/* +/* dict_cache_expire() schedules a thread that expires cache +/* entries periodically. Specify a null validator argument to +/* cancel the thread. It is an error to schedule a cache +/* cleanup thread when one already exists. Programs must not +/* use both dict_cache_sequence() and dict_cache_expire(). +/* +/* dict_cache_name() returns the name of the specified cache. +/* +/* dict_cache_import() encapsulates a pre-opened database +/* handle and adds the above features. +/* +/* Arguments: +/* .IP "dbname, open_flags, dict_flags" +/* These are passed unchanged to dict_open(). +/* .IP cache +/* Cache handle created with dict_cache_open()or dict_cache_import(). +/* .IP cache_key +/* Cache lookup key. +/* .IP cache_val +/* Information that is stored under a cache lookup key. +/* .IP first_next +/* One of DICT_SEQ_FUN_FIRST (first cache element) or +/* DICT_SEQ_FUN_NEXT (next cache element). +/* .sp +/* Note: there is no "stop" request. To ensure that the "delete +/* behind" strategy does not interfere with database access, +/* allow dict_cache_sequence() to run to completion. +/* .IP flags +/* Bit-wise OR of zero or more of the following: +/* .RS +/* .IP DICT_CACHE_FLAG_EXP_VERBOSE +/* Log each cache entry's status during a cache cleanup run. +/* .IP DICT_CACHE_FLAG_EXP_SUMMARY +/* Log the number of cache entries retained and dropped after +/* a cache cleaning run. +/* .RE +/* .IP interval +/* The non-zero time between scans for expired cache entries. +/* The interval timer starts after a scan completes. +/* .IP validator +/* Application call-back routine that returns non-zero when a +/* cache entry should be kept. The validator must not modify +/* or close the cache. +/* .IP context +/* Application-specific context. +/* .IP table +/* A bare dictonary handle. +/* DIAGNOSTICS +/* These routines terminate with a fatal run-time error +/* for unrecoverable database errors. This allows the +/* program to restart and reset the database to an +/* empty initial state. +/* BUGS +/* There should be a way to suspend automatic program suicide +/* until a cache cleanup run is completed. Some entries may +/* never be removed when the process max_idle time is less +/* than the time needed to make a full pass over the cache. +/* LICENSE +/* .ad +/* .fi +/* The Secure Mailer license must be distributed with this software. +/* HISTORY +/* .ad +/* .fi +/* A predecessor of this code was written first for the Postfix +/* tlsmgr(8) daemon. +/* AUTHOR(S) +/* Wietse Venema +/* IBM T.J. Watson Research +/* P.O. Box 704 +/* Yorktown Heights, NY 10598, USA +/*--*/ + +/* System library. */ + +#include +#include +#include + +/* Utility library. */ + +#include +#include +#include +#include +#include + +/* Application-specific. */ + + /* + * XXX Deleting entries while enumerating a map can he tricky. Some map + * types have a concept of cursor and support a "delete the current element" + * operation. Some map types without cursors don't behave well when the + * current first/next entry is deleted (example: with Berkeley DB < 2, the + * "next" operation produces garbage). To avoid trouble, we delete an entry + * after advancing the current first/next position beyond it; we use the + * same strategy with application requests to delete the current entry. + */ + + /* + * Opaque data structure. Use dict_cache_name() to access the name of the + * underlying database. + */ +struct DICT_CACHE { + int flags; /* see below */ + DICT *db; /* database handle */ + + /* Iterator support. */ + char *saved_curr_key; /* "current" cache lookup key */ + char *saved_curr_val; /* "current" cache lookup result */ + + /* Cleanup support. */ + int exp_flags; /* logging */ + int exp_interval; /* time between cleanup runs */ + DICT_CACHE_VALIDATOR_FN exp_validator; /* expiration call-back */ + char *exp_context; /* call-back context */ + int retained; /* entries retained in cleanup run */ + int dropped; /* entries removed in cleanup run */ +}; + +#define DC_FLAG_DEL_SAVED_CURRENT_KEY (1<<0) /* delete-behind is scheduled */ + + /* + * Macros to make obscure code more readable. + */ +#define DC_SCHEDULE_FOR_DELETE_BEHIND(cp) \ + ((cp)->flags |= DC_FLAG_DEL_SAVED_CURRENT_KEY) + +#define DC_MATCH_SAVED_CURRENT_KEY(cp, cache_key) \ + ((cp)->saved_curr_key && strcmp((cp)->saved_curr_key, (cache_key)) == 0) + +#define DC_IS_SCHEDULED_FOR_DELETE_BEHIND(cp) \ + (/* NOT: (cp)->saved_curr_key && */ \ + ((cp)->flags & DC_FLAG_DEL_SAVED_CURRENT_KEY) != 0) + +#define DC_CANCEL_DELETE_BEHIND(cp) \ + ((cp)->flags &= ~DC_FLAG_DEL_SAVED_CURRENT_KEY) + + /* + * Special key to store the time of last cache cleanup run completion. + */ +#define DC_LAST_CACHE_CLEANUP_COMPLETED "_LAST_CACHE_CLEANUP_COMPLETED_" + +/* dict_cache_lookup - load entry from cache */ + +const char *dict_cache_lookup(DICT_CACHE *cp, const char *cache_key) +{ + + /* + * Search for the cache entry. Don't return an entry that was scheduled + * for deletion. + */ + if (DC_IS_SCHEDULED_FOR_DELETE_BEHIND(cp) + && DC_MATCH_SAVED_CURRENT_KEY(cp, cache_key)) { + return (0); + } else { + return (dict_get(cp->db, cache_key)); + } +} + +/* dict_cache_update - save entry to cache */ + +void dict_cache_update(DICT_CACHE *cp, const char *cache_key, + const char *cache_val) +{ + + /* + * Store the cache entry and cancel a scheduled delete-behind operation. + */ + if (DC_IS_SCHEDULED_FOR_DELETE_BEHIND(cp) + && DC_MATCH_SAVED_CURRENT_KEY(cp, cache_key)) + DC_CANCEL_DELETE_BEHIND(cp); + dict_put(cp->db, cache_key, cache_val); +} + +/* dict_cache_delete - delete entry from cache */ + +int dict_cache_delete(DICT_CACHE *cp, const char *cache_key) +{ + int zero_means_found; + + /* + * Delete the entry, unless we would delete the current first/next entry. + * Instead, schedule the "current" entry for delete-behind to avoid + * mis-behavior by some databases. + */ + if (DC_MATCH_SAVED_CURRENT_KEY(cp, cache_key)) { + DC_SCHEDULE_FOR_DELETE_BEHIND(cp); + zero_means_found = 0; + } else { + zero_means_found = dict_del(cp->db, cache_key); + } + return (zero_means_found); +} + +/* dict_cache_sequence - look up the first/next cache entry */ + +int dict_cache_sequence(DICT_CACHE *cp, int first_next, + const char **cache_key, + const char **cache_val) +{ + int zero_means_found; + const char *raw_cache_key; + const char *raw_cache_val; + char *previous_curr_key; + char *previous_curr_val; + + /* + * Find the first or next database entry. Hide the record with the cache + * cleanup completion time stamp. + */ + zero_means_found = + dict_seq(cp->db, first_next, &raw_cache_key, &raw_cache_val); + if (zero_means_found == 0 + && strcmp(raw_cache_key, DC_LAST_CACHE_CLEANUP_COMPLETED) == 0) + zero_means_found = + dict_seq(cp->db, DICT_SEQ_FUN_NEXT, &raw_cache_key, &raw_cache_val); + + /* + * Save the current cache_key and cache_val before they are clobbered by + * our own delete operation below. This also prevents surprises when the + * application accesses the database after this function returns. + * + * We also use the saved cache_key to protect the current entry against + * application delete requests. + */ + previous_curr_key = cp->saved_curr_key; + previous_curr_val = cp->saved_curr_val; + if (zero_means_found == 0) { + cp->saved_curr_key = mystrdup(raw_cache_key); + cp->saved_curr_val = mystrdup(raw_cache_val); + } else { + cp->saved_curr_key = 0; + cp->saved_curr_val = 0; + } + + /* + * Delete behind. + */ + if (DC_IS_SCHEDULED_FOR_DELETE_BEHIND(cp)) { + DC_CANCEL_DELETE_BEHIND(cp); + if (dict_del(cp->db, previous_curr_key) != 0) + msg_warn("database %s: could not delete entry for %s", + cp->db->name, previous_curr_key); + } + + /* + * Clean up previous iteration key and value. + */ + if (previous_curr_key) + myfree(previous_curr_key); + if (previous_curr_val) + myfree(previous_curr_val); + + /* + * Return the result. + */ + *cache_key = (cp)->saved_curr_key; + *cache_val = (cp)->saved_curr_val; + return (zero_means_found); +} + +/* dict_cache_delete_behind_reset - reset "delete behind" state */ + +static void dict_cache_delete_behind_reset(DICT_CACHE *cp) +{ +#define FREE_AND_WIPE(s) do { if (s) { myfree(s); (s) = 0; } } while (0) + + DC_CANCEL_DELETE_BEHIND(cp); + FREE_AND_WIPE(cp->saved_curr_key); + FREE_AND_WIPE(cp->saved_curr_val); +} + +/* dict_cache_clean_stat_log_reset - log and reset cache cleanup statistics */ + +static void dict_cache_clean_stat_log_reset(DICT_CACHE *cp, + const char *full_partial) +{ + if (cp->flags & DICT_CACHE_FLAG_EXP_SUMMARY) + msg_info("cache %s %s cleanup: retained=%d dropped=%d entries", + cp->db->name, full_partial, cp->retained, cp->dropped); + cp->retained = cp->dropped = 0; +} + +/* dict_cache_expire_event - examine one cache entry */ + +static void dict_cache_expire_event(int unused_event, char *cache_context) +{ + DICT_CACHE *cp = (DICT_CACHE *) cache_context; + const char *cache_key; + const char *cache_val; + int next_interval; + VSTRING *stamp_buf; + int first_next; + + /* + * We interleave cache cleanup with other processing, so that the + * application's service remains available, with perhaps increased + * latency. + */ + + /* + * Start a new cache cleanup run. + */ + if (cp->saved_curr_key == 0) { + cp->retained = cp->dropped = 0; + first_next = DICT_SEQ_FUN_FIRST; + if (cp->exp_flags & DICT_CACHE_FLAG_EXP_VERBOSE) + msg_info("start %s cache cleanup", cp->db->name); + } + + /* + * Continue a cache cleanup run in progress. + */ + else { + first_next = DICT_SEQ_FUN_NEXT; + } + + /* + * Examine one cache entry. + */ + if (dict_cache_sequence(cp, first_next, &cache_key, &cache_val) == 0) { + if (cp->exp_validator(cache_key, cache_val, cp->exp_context) == 0) { + DC_SCHEDULE_FOR_DELETE_BEHIND(cp); + cp->dropped++; + if (cp->exp_flags & DICT_CACHE_FLAG_EXP_VERBOSE) + msg_info("drop %s cache entry for %s", cp->db->name, cache_key); + } else { + cp->retained++; + if (cp->exp_flags & DICT_CACHE_FLAG_EXP_VERBOSE) + msg_info("keep %s cache entry for %s", cp->db->name, cache_key); + } + next_interval = 0; + } + + /* + * Cache cleanup completed. Report vital statistics. + */ + else { + if (cp->exp_flags & DICT_CACHE_FLAG_EXP_VERBOSE) + msg_info("done %s cache cleanup scan", cp->db->name); + dict_cache_clean_stat_log_reset(cp, "full"); + stamp_buf = vstring_alloc(100); + vstring_sprintf(stamp_buf, "%ld", (long) event_time()); + dict_put(cp->db, DC_LAST_CACHE_CLEANUP_COMPLETED, + vstring_str(stamp_buf)); + vstring_free(stamp_buf); + next_interval = cp->exp_interval; + } + event_request_timer(dict_cache_expire_event, cache_context, next_interval); +} + +/* dict_cache_expire - schedule or stop the cache cleanup thread */ + +void dict_cache_expire(DICT_CACHE *cp, int flags, int interval, + DICT_CACHE_VALIDATOR_FN validator, + char *context) +{ + const char *myname = "dict_cache_expire"; + const char *last_done; + time_t next_interval; + + /* + * Schedule the cache cleanup thread. + */ + if (validator != 0) { + + /* + * Sanity checks. + */ + if (cp->exp_validator != 0) + msg_panic("%s: %s cache cleanup is already scheduled", + myname, cp->db->name); + if (interval <= 0) + msg_panic("%s: bad %s cache cleanup interval %d", + myname, cp->db->name, interval); + cp->exp_flags = flags; + cp->exp_interval = interval; + cp->exp_validator = validator; + cp->exp_context = context; + + /* + * The next start time depends on the last completion time. + */ +#define NEXT_START(last, delta) ((delta) + (unsigned long) atol(last)) +#define NOW (time((time_t *) 0)) /* NOT: event_time() */ + + if ((last_done = dict_get(cp->db, DC_LAST_CACHE_CLEANUP_COMPLETED)) == 0 + || (next_interval = (NEXT_START(last_done, interval) - NOW)) < 0) + next_interval = 0; + if (next_interval > interval) + next_interval = interval; + if ((cp->exp_flags & DICT_CACHE_FLAG_EXP_VERBOSE) && next_interval > 0) + msg_info("%s cache cleanup will start after %ds", + cp->db->name, (int) next_interval); + event_request_timer(dict_cache_expire_event, (char *) cp, + (int) next_interval); + } + + /* + * Cancel the cache cleanup thread. + */ + else if (cp->exp_validator) { + if (cp->retained || cp->dropped) + dict_cache_clean_stat_log_reset(cp, "partial"); + dict_cache_delete_behind_reset(cp); + cp->exp_interval = 0; + cp->exp_validator = 0; + cp->exp_context = 0; + event_cancel_timer(dict_cache_expire_event, (char *) cp); + } +} + +/* dict_cache_open - open cache file */ + +DICT_CACHE *dict_cache_open(const char *dbname, int open_flags, int dict_flags) +{ + DICT *dict; + + /* + * Open the database as requested. Don't attempt to second-guess the + * application. + */ + dict = dict_open(dbname, open_flags, dict_flags); + return (dict_cache_import(dict)); +} + +/* dict_cache_import - encapsulate pre-opened database */ + +DICT_CACHE *dict_cache_import(DICT *dict) +{ + DICT_CACHE *cp; + + /* + * Create the DICT_CACHE object. + */ + cp = (DICT_CACHE *) mymalloc(sizeof(*cp)); + cp->flags = 0; + cp->db = dict; + cp->saved_curr_key = 0; + cp->saved_curr_val = 0; + cp->exp_interval = 0; + cp->exp_validator = 0; + cp->exp_context = 0; + cp->retained = 0; + cp->dropped = 0; + + return (cp); +} + +/* dict_cache_close - close cache file */ + +void dict_cache_close(DICT_CACHE *cp) +{ + + /* + * Destroy the DICT_CACHE object. + */ + if (cp->exp_validator) + dict_cache_expire(cp, 0, 0, (DICT_CACHE_VALIDATOR_FN) 0, (char *) 0); + dict_close(cp->db); + if (cp->saved_curr_key) + myfree(cp->saved_curr_key); + if (cp->saved_curr_val) + myfree(cp->saved_curr_val); + myfree((char *) cp); +} + +/* dict_cache_name - get the cache name */ + +const char *dict_cache_name(DICT_CACHE *cp) +{ + + /* + * This is used for verbose logging or warning messages, so the cost of + * call is only made where needed (well sort off - code that does not + * execute still presents overhead for the processor pipeline, processor + * cache, etc). + */ + return (cp->db->name); +} diff --git a/postfix/src/util/dict_cache.h b/postfix/src/util/dict_cache.h new file mode 100644 index 000000000..08523c412 --- /dev/null +++ b/postfix/src/util/dict_cache.h @@ -0,0 +1,49 @@ +#ifndef _DICT_CACHE_H_INCLUDED_ +#define _DICT_CACHE_H_INCLUDED_ + +/*++ +/* NAME +/* dict_cache 3h +/* SUMMARY +/* External cache manager +/* SYNOPSIS +/* #include +/* DESCRIPTION +/* .nf + + /* + * Utility library. + */ +#include + + /* + * External interface. + */ +typedef struct DICT_CACHE DICT_CACHE; +typedef int (*DICT_CACHE_VALIDATOR_FN) (const char *, const char *, char *); + +extern DICT_CACHE *dict_cache_open(const char *, int, int); +extern void dict_cache_close(DICT_CACHE *); +extern const char *dict_cache_lookup(DICT_CACHE *, const char *); +extern void dict_cache_update(DICT_CACHE *, const char *, const char *); +extern int dict_cache_delete(DICT_CACHE *, const char *); +extern int dict_cache_sequence(DICT_CACHE *, int, const char **, const char **); +extern void dict_cache_expire(DICT_CACHE *, int, int, DICT_CACHE_VALIDATOR_FN, char *); +extern const char *dict_cache_name(DICT_CACHE *); +extern DICT_CACHE *dict_cache_import(DICT *); + +#define DICT_CACHE_FLAG_EXP_VERBOSE (1<<0) +#define DICT_CACHE_FLAG_EXP_SUMMARY (1<<1) + +/* LICENSE +/* .ad +/* .fi +/* The Secure Mailer license must be distributed with this software. +/* AUTHOR(S) +/* Wietse Venema +/* IBM T.J. Watson Research +/* P.O. Box 704 +/* Yorktown Heights, NY 10598, USA +/*--*/ + +#endif diff --git a/postfix/src/util/dict_dbm.c b/postfix/src/util/dict_dbm.c index e034aecf6..3603e44de 100644 --- a/postfix/src/util/dict_dbm.c +++ b/postfix/src/util/dict_dbm.c @@ -310,6 +310,7 @@ static int dict_dbm_sequence(DICT *dict, int function, DICT_DBM *dict_dbm = (DICT_DBM *) dict; datum dbm_key; datum dbm_value; + int status; /* * Acquire a shared lock. @@ -350,6 +351,7 @@ static int dict_dbm_sequence(DICT *dict, int function, * Copy the value so that it is guaranteed null terminated. */ *value = SCOPY(dict_dbm->val_buf, dbm_value.dptr, dbm_value.dsize); + status = 0; } else { /* @@ -358,7 +360,7 @@ static int dict_dbm_sequence(DICT *dict, int function, */ if (dbm_error(dict_dbm->dbm)) msg_fatal("error seeking %s: %m", dict_dbm->dict.name); - return (1); /* no error: eof/not found + status = 1; /* no error: eof/not found * (should not happen!) */ } } else { @@ -368,7 +370,7 @@ static int dict_dbm_sequence(DICT *dict, int function, */ if (dbm_error(dict_dbm->dbm)) msg_fatal("error seeking %s: %m", dict_dbm->dict.name); - return (1); /* no error: eof/not found */ + status = 1; /* no error: eof/not found */ } /* @@ -378,7 +380,7 @@ static int dict_dbm_sequence(DICT *dict, int function, && myflock(dict->lock_fd, INTERNAL_LOCK, MYFLOCK_OP_NONE) < 0) msg_fatal("%s: unlock dictionary: %m", dict_dbm->dict.name); - return (0); + return (status); } /* dict_dbm_close - disassociate from data base */ diff --git a/postfix/src/util/dict_ht.c b/postfix/src/util/dict_ht.c index 38625daad..55b7c0d62 100644 --- a/postfix/src/util/dict_ht.c +++ b/postfix/src/util/dict_ht.c @@ -75,6 +75,29 @@ static void dict_ht_update(DICT *dict, const char *name, const char *value) ht->value = mystrdup(value); } +/* dict_ht_sequence - first/next iterator */ + +static int dict_ht_sequence(DICT *dict, int how, const char **name, + const char **value) +{ + DICT_HT *dict_ht = (DICT_HT *) dict; + HTABLE_INFO *ht; + + ht = htable_sequence(dict_ht->table, + how == DICT_SEQ_FUN_FIRST ? HTABLE_SEQ_FIRST : + how == DICT_SEQ_FUN_NEXT ? HTABLE_SEQ_NEXT : + HTABLE_SEQ_STOP); + if (ht != 0) { + *name = ht->key; + *value = ht->value; + return (0); + } else { + *name = 0; + *value = 0; + return (1); + } +} + /* dict_ht_close - disassociate from hash table */ static void dict_ht_close(DICT *dict) @@ -95,6 +118,7 @@ DICT *dict_ht_open(const char *name, HTABLE *table, void (*remove) (char *)) dict_ht = (DICT_HT *) dict_alloc(DICT_TYPE_HT, name, sizeof(*dict_ht)); dict_ht->dict.lookup = dict_ht_lookup; dict_ht->dict.update = dict_ht_update; + dict_ht->dict.sequence = dict_ht_sequence; dict_ht->dict.close = dict_ht_close; dict_ht->table = table; dict_ht->remove = remove; diff --git a/postfix/src/util/dict_open.c b/postfix/src/util/dict_open.c index 3e0beb7d6..beef2e6d2 100644 --- a/postfix/src/util/dict_open.c +++ b/postfix/src/util/dict_open.c @@ -144,13 +144,13 @@ /* dict_put() stores the specified key and value into the named /* dictionary. /* -/* dict_del() removes a dictionary entry, and returns non-zero +/* dict_del() removes a dictionary entry, and returns zero /* in case of success. /* /* dict_seq() iterates over all members in the named dictionary. /* func is define DICT_SEQ_FUN_FIRST (select first member) or -/* DICT_SEQ_FUN_NEXT (select next member). A null result means -/* there is more. +/* DICT_SEQ_FUN_NEXT (select next member). A zero result means +/* that an entry was found. /* /* dict_close() closes the specified dictionary and cleans up the /* associated data structures. diff --git a/postfix/src/util/events.c b/postfix/src/util/events.c index a293d5c65..c87dc5ce4 100644 --- a/postfix/src/util/events.c +++ b/postfix/src/util/events.c @@ -496,6 +496,11 @@ typedef struct epoll_event EVENT_BUFFER; /* * Timer events. Timer requests are kept sorted, in a circular list. We use * the RING abstraction, so we get to use a couple ugly macros. + * + * When a call-back function adds a timer request, we label the request with + * the event_loop() call instance that invoked the call-back. We use this to + * prevent zero-delay timer requests from running in a tight loop and + * starving I/O events. */ typedef struct EVENT_TIMER EVENT_TIMER; @@ -503,10 +508,12 @@ struct EVENT_TIMER { time_t when; /* when event is wanted */ EVENT_NOTIFY_TIME callback; /* callback function */ char *context; /* callback context */ + long loop_instance; /* event_loop() call instance */ RING ring; /* linkage */ }; static RING event_timer_head; /* timer queue head */ +static long event_loop_instance; /* event_loop() call instance */ #define RING_TO_TIMER(r) \ ((EVENT_TIMER *) ((char *) (r) - offsetof(EVENT_TIMER, ring))) @@ -921,18 +928,24 @@ time_t event_request_timer(EVENT_NOTIFY_TIME callback, char *context, int delay timer->when = event_present + delay; timer->callback = callback; timer->context = context; + timer->loop_instance = event_loop_instance; if (msg_verbose > 2) msg_info("%s: set 0x%lx 0x%lx %d", myname, (long) callback, (long) context, delay); } /* - * Insert the request at the right place. Timer requests are kept sorted - * to reduce lookup overhead in the event loop. + * Timer requests are kept sorted to reduce lookup overhead in the event + * loop. + * + * XXX Append the new request after existing requests for the same time + * slot. The event_loop() routine depends on this to avoid starving I/O + * events when a call-back function schedules a zero-delay timer request. */ - FOREACH_QUEUE_ENTRY(ring, &event_timer_head) + FOREACH_QUEUE_ENTRY(ring, &event_timer_head) { if (timer->when < RING_TO_TIMER(ring)->when) - break; + break; + } ring_prepend(ring, &timer->ring); return (timer->when); @@ -1079,17 +1092,35 @@ void event_loop(int delay) msg_panic("event_loop: recursive call"); /* - * Deliver timer events. Requests are sorted: we can stop when we reach - * the future or the list end. Allow the application to update the timer - * queue while it is being called back. To this end, we repeatedly pop - * the first request off the timer queue before delivering the event to - * the application. + * Deliver timer events. Allow the application to add/delete timer queue + * requests while it is being called back. Requests are sorted: we keep + * running over the timer request queue from the start, and stop when we + * reach the future or the list end. We also stop when we reach a timer + * request that was added by a call-back that was invoked from this + * event_loop() call instance, for reasons that are explained below. + * + * To avoid dangling pointer problems 1) we must remove a request from the + * timer queue before delivering its event to the application and 2) we + * must look up the next timer request *after* calling the application. + * The latter complicates the handling of zero-delay timer requests that + * are added by event_loop() call-back functions. + * + * XXX When a timer event call-back function adds a new timer request, + * event_request_timer() labels the request with the event_loop() call + * instance that invoked the timer event call-back. We use this instance + * label here to prevent zero-delay timer requests from running in a + * tight loop and starving I/O events. To make this solution work, + * event_request_timer() appends a new request after existing requests + * for the same time slot. */ event_present = time((time_t *) 0); + event_loop_instance += 1; while ((timer = FIRST_TIMER(&event_timer_head)) != 0) { if (timer->when > event_present) break; + if (timer->loop_instance == event_loop_instance) + break; ring_detach(&timer->ring); /* first this */ if (msg_verbose > 2) msg_info("%s: timer 0x%lx 0x%lx", myname, @@ -1192,10 +1223,10 @@ static void echo(int unused_event, char *unused_context) printf("Result: %s", buf); } -int main(int argc, char **argv) +/* request - request a bunch of timer events */ + +static void request(int unused_event, char *unused_context) { - if (argv[1]) - msg_verbose = atoi(argv[1]); event_request_timer(timer_event, "3 first", 3); event_request_timer(timer_event, "3 second", 3); event_request_timer(timer_event, "4 first", 4); @@ -1206,6 +1237,13 @@ int main(int argc, char **argv) event_request_timer(timer_event, "1 second", 1); event_request_timer(timer_event, "0 first", 0); event_request_timer(timer_event, "0 second", 0); +} + +int main(int argc, char **argv) +{ + if (argv[1]) + msg_verbose = atoi(argv[1]); + event_request_timer(request, (char *) 0, 0); event_enable_read(fileno(stdin), echo, (char *) 0); event_drain(10); exit(0); diff --git a/postfix/src/util/htable.c b/postfix/src/util/htable.c index 1fbbfcf24..aa87a3edf 100644 --- a/postfix/src/util/htable.c +++ b/postfix/src/util/htable.c @@ -46,6 +46,10 @@ /* /* HTABLE_INFO **htable_list(table) /* HTABLE *table; +/* +/* HTABLE_INFO *htable_sequence(table, how) +/* HTABLE *table; +/* int how; /* DESCRIPTION /* This module maintains one or more hash tables. Each table entry /* consists of a unique string-valued lookup key and a generic @@ -85,6 +89,12 @@ /* htable_list() returns a null-terminated list of pointers to /* all elements in the named table. The list should be passed to /* myfree(). +/* +/* htable_sequence() returns the first or next element depending +/* on the value of the "how" argument. Specify HTABLE_SEQ_FIRST +/* to start a new sequence, HTABLE_SEQ_NEXT to continue, and +/* HTABLE_SEQ_STOP to terminate a sequence early. The caller +/* must not delete the current element. /* RESTRICTIONS /* A callback function should not modify the hash table that is /* specified to its caller. @@ -172,6 +182,7 @@ HTABLE *htable_create(int size) table = (HTABLE *) mymalloc(sizeof(HTABLE)); htable_size(table, size < 13 ? 13 : size); + table->seq_element = 0; return (table); } @@ -202,7 +213,7 @@ HTABLE_INFO *htable_enter(HTABLE *table, const char *key, char *value) { HTABLE_INFO *ht; - if (table->used >= table->size) + if (table->used >= table->size && table->seq_element == 0) htable_grow(table); ht = (HTABLE_INFO *) mymalloc(sizeof(HTABLE_INFO)); ht->key = mystrdup(key); @@ -332,6 +343,34 @@ HTABLE_INFO **htable_list(HTABLE *table) return (list); } +/* htable_sequence - dict_cache(3) compatibility iterator */ + +HTABLE_INFO *htable_sequence(HTABLE *table, int how) +{ + if (table == 0) + return (0); + + switch (how) { + case HTABLE_SEQ_FIRST: /* start new sequence */ + table->seq_bucket = table->data; + table->seq_element = table->seq_bucket[0]; + break; + case HTABLE_SEQ_NEXT: /* next element */ + if (table->seq_element) { + table->seq_element = table->seq_element->next; + break; + } + /* FALLTHROUGH */ + default: /* terminate sequence */ + return (table->seq_element = 0); + } + + while (table->seq_element == 0 + && ++(table->seq_bucket) < table->data + table->size) + table->seq_element = table->seq_bucket[0]; + return (table->seq_element); +} + #ifdef TEST #include #include @@ -346,6 +385,7 @@ int main(int unused_argc, char **unused_argv) HTABLE_INFO *info; int i; int r; + int op; /* * Load a large number of strings and delete them in a random order. @@ -353,6 +393,11 @@ int main(int unused_argc, char **unused_argv) hash = htable_create(10); while (vstring_get(buf, VSTREAM_IN) != VSTREAM_EOF) htable_enter(hash, vstring_str(buf), CAST_INT_TO_CHAR_PTR(count++)); + for (i = 0, op = HTABLE_SEQ_FIRST; htable_sequence(hash, op) != 0; + i++, op = HTABLE_SEQ_NEXT) + /* void */ ; + if (i != hash->used) + msg_panic("%d entries found, but %d entries exist", i, hash->used); ht_info = htable_list(hash); for (i = 0; i < hash->used; i++) { r = myrand() % hash->used; diff --git a/postfix/src/util/htable.h b/postfix/src/util/htable.h index 72eaa98f1..23507b1a2 100644 --- a/postfix/src/util/htable.h +++ b/postfix/src/util/htable.h @@ -26,6 +26,8 @@ typedef struct HTABLE { int size; /* length of entries array */ int used; /* number of entries in table */ HTABLE_INFO **data; /* entries array, auto-resized */ + HTABLE_INFO **seq_bucket; /* current sequence bucket */ + HTABLE_INFO *seq_element; /* current sequence element */ } HTABLE; extern HTABLE *htable_create(int); @@ -36,6 +38,11 @@ extern void htable_delete(HTABLE *, const char *, void (*) (char *)); extern void htable_free(HTABLE *, void (*) (char *)); extern void htable_walk(HTABLE *, void (*) (HTABLE_INFO *, char *), char *); extern HTABLE_INFO **htable_list(HTABLE *); +extern HTABLE_INFO *htable_sequence(HTABLE *, int); + +#define HTABLE_SEQ_FIRST 0 +#define HTABLE_SEQ_NEXT 1 +#define HTABLE_SEQ_STOP (-1) /* LICENSE /* .ad diff --git a/postfix/src/verify/Makefile.in b/postfix/src/verify/Makefile.in index e5fa293c7..ba7d81e82 100644 --- a/postfix/src/verify/Makefile.in +++ b/postfix/src/verify/Makefile.in @@ -63,8 +63,10 @@ verify.o: ../../include/cleanup_user.h verify.o: ../../include/data_redirect.h verify.o: ../../include/deliver_request.h verify.o: ../../include/dict.h +verify.o: ../../include/dict_cache.h verify.o: ../../include/dict_ht.h verify.o: ../../include/dsn.h +verify.o: ../../include/events.h verify.o: ../../include/htable.h verify.o: ../../include/int_filt.h verify.o: ../../include/iostuff.h diff --git a/postfix/src/verify/verify.c b/postfix/src/verify/verify.c index 322f63661..70a5a7158 100644 --- a/postfix/src/verify/verify.c +++ b/postfix/src/verify/verify.c @@ -97,6 +97,10 @@ /* .IP "\fBaddress_verify_negative_refresh_time (3h)\fR" /* The time after which a failed address verification probe needs to /* be refreshed. +/* .PP +/* Available with Postfix 2.7 and later: +/* .IP "\fBaddress_verify_cache_cleanup_interval (12h)\fR" +/* The amount of time between \fBverify\fR(8) cache cleanup runs. /* PROBE MESSAGE ROUTING CONTROLS /* .ad /* .fi @@ -121,6 +125,16 @@ /* .IP "\fBaddress_verify_default_transport ($default_transport)\fR" /* Overrides the default_transport parameter setting for address /* verification probes. +/* .PP +/* Available in Postfix 2.3 and later: +/* .IP "\fBaddress_verify_sender_dependent_relayhost_maps ($sender_dependent_relayhost_maps)\fR" +/* Overrides the sender_dependent_relayhost_maps parameter setting for address +/* verification probes. +/* .PP +/* Available in Postfix 2.7 and later: +/* .IP "\fBaddress_verify_sender_dependent_default_transport_maps ($sender_dependent_default_transport_maps)\fR" +/* Overrides the sender_dependent_default_transport_maps parameter +/* setting for address verification probes. /* MISCELLANEOUS CONTROLS /* .ad /* .fi @@ -187,10 +201,11 @@ #include #include #include -#include +#include #include #include #include +#include /* Global library. */ @@ -216,12 +231,13 @@ int var_verify_pos_exp; int var_verify_pos_try; int var_verify_neg_exp; int var_verify_neg_try; +int var_verify_scan_cache; char *var_verify_sender; /* * State. */ -static DICT *verify_map; +static DICT_CACHE *verify_map; /* * Silly little macros. @@ -344,7 +360,7 @@ static void verify_update_service(VSTREAM *client_stream) * some probes succeed the address will remain cached as OK. */ if (addr_status == DEL_RCPT_STAT_OK - || (raw_data = dict_get(verify_map, STR(addr))) == 0 + || (raw_data = dict_cache_lookup(verify_map, STR(addr))) == 0 || STATUS_FROM_RAW_ENTRY(raw_data) != DEL_RCPT_STAT_OK) { probed = 0; updated = (long) time((time_t *) 0); @@ -352,7 +368,7 @@ static void verify_update_service(VSTREAM *client_stream) if (msg_verbose) msg_info("PUT %s status=%d probed=%ld updated=%ld text=%s", STR(addr), addr_status, probed, updated, STR(text)); - dict_put(verify_map, STR(addr), STR(buf)); + dict_cache_update(verify_map, STR(addr), STR(buf)); } attr_print(client_stream, ATTR_FLAG_NONE, ATTR_TYPE_INT, MAIL_ATTR_STATUS, VRFY_STAT_OK, @@ -419,7 +435,7 @@ static void verify_query_service(VSTREAM *client_stream) /* FIX 200501 IPv6 patch did not neuter ":" in address literals. */ translit(STR(addr), ":", "_"); - if ((raw_data = dict_get(verify_map, STR(addr))) == 0 /* not found */ + if ((raw_data = dict_cache_lookup(verify_map, STR(addr))) == 0 /* not found */ || ((get_buf = vstring_alloc(10)), vstring_strcpy(get_buf, raw_data), /* malformed */ verify_parse_entry(STR(get_buf), &addr_status, &probed, @@ -432,7 +448,7 @@ static void verify_query_service(VSTREAM *client_stream) updated = 0; text = "Address verification in progress"; if (raw_data != 0 && var_verify_neg_cache == 0) - dict_del(verify_map, STR(addr)); + dict_cache_delete(verify_map, STR(addr)); } if (msg_verbose) msg_info("GOT %s status=%d probed=%ld updated=%ld text=%s", @@ -482,7 +498,7 @@ static void verify_query_service(VSTREAM *client_stream) if (msg_verbose) msg_info("PUT %s status=%d probed=%ld updated=%ld text=%s", STR(addr), addr_status, now, updated, text); - dict_put(verify_map, STR(addr), STR(put_buf)); + dict_cache_update(verify_map, STR(addr), STR(put_buf)); } } } @@ -493,6 +509,29 @@ static void verify_query_service(VSTREAM *client_stream) vstring_free(put_buf); } +/* verify_cache_validator - cache cleanup validator */ + +static int verify_cache_validator(const char *addr, const char *raw_data, + char *context) +{ + VSTRING *get_buf = (VSTRING *) context; + int addr_status; + long probed; + long updated; + char *text; + long now = (long) event_time(); + +#define POS_OR_NEG_ENTRY_EXPIRED(stat, stamp) \ + (POSITIVE_ENTRY_EXPIRED((stat), (stamp)) \ + || NEGATIVE_ENTRY_EXPIRED((stat), (stamp))) + + vstring_strcpy(get_buf, raw_data); + return (verify_parse_entry(STR(get_buf), &addr_status, /* syntax OK */ + &probed, &updated, &text) == 0 + && (now - probed < PROBE_TTL /* probe in progress */ + || !POS_OR_NEG_ENTRY_EXPIRED(addr_status, updated))); +} + /* verify_service - perform service for client */ static void verify_service(VSTREAM *client_stream, char *unused_service, @@ -530,6 +569,21 @@ static void verify_service(VSTREAM *client_stream, char *unused_service, vstring_free(request); } +/* verify_dump - dump some statistics */ + +static void verify_dump(void) +{ + + /* + * Dump preliminary cache cleanup statistics when the process commits + * suicide while a cache cleanup run is in progress. We can't currently + * distinguish between "postfix reload" (we should restart) or "maximal + * idle time reached" (we could finish the cache cleanup first). + */ + dict_cache_close(verify_map); + verify_map = 0; +} + /* post_jail_init - post-jail initialization */ static void post_jail_init(char *unused_name, char **unused_argv) @@ -544,6 +598,20 @@ static void post_jail_init(char *unused_name, char **unused_argv) var_use_limit = 0; var_idle_limit = 0; } + + /* + * Start the cache cleanup thread. + */ + if (var_verify_scan_cache > 0) { + int expire_flags; + + expire_flags = DICT_CACHE_FLAG_EXP_SUMMARY; + if (msg_verbose) + expire_flags |= DICT_CACHE_FLAG_EXP_VERBOSE; + dict_cache_expire(verify_map, expire_flags, var_verify_scan_cache, + verify_cache_validator, + (char *) vstring_alloc(100)); + } } /* pre_jail_init - pre-jail initialization */ @@ -585,17 +653,20 @@ static void pre_jail_init(char *unused_name, char **unused_argv) /* * Keep state in persistent (external) or volatile (internal) map. + * + * Start the cache cleanup thread after permanently dropping privileges. */ #define VERIFY_DICT_OPEN_FLAGS (DICT_FLAG_DUP_REPLACE | DICT_FLAG_SYNC_UPDATE) if (*var_verify_map) { saved_mask = umask(022); - verify_map = dict_open(data_redirect_map(redirect, var_verify_map), - O_CREAT | O_RDWR, - VERIFY_DICT_OPEN_FLAGS); + verify_map = + dict_cache_open(data_redirect_map(redirect, var_verify_map), + O_CREAT | O_RDWR, VERIFY_DICT_OPEN_FLAGS); (void) umask(saved_mask); } else { - verify_map = dict_ht_open("verify", htable_create(0), myfree); + verify_map = + dict_cache_import(dict_ht_open("verify", htable_create(0), myfree)); } /* @@ -621,6 +692,7 @@ int main(int argc, char **argv) VAR_VERIFY_POS_TRY, DEF_VERIFY_POS_TRY, &var_verify_pos_try, 1, 0, VAR_VERIFY_NEG_EXP, DEF_VERIFY_NEG_EXP, &var_verify_neg_exp, 1, 0, VAR_VERIFY_NEG_TRY, DEF_VERIFY_NEG_TRY, &var_verify_neg_try, 1, 0, + VAR_VERIFY_SCAN_CACHE, DEF_VERIFY_SCAN_CACHE, &var_verify_scan_cache, 0, 0, 0, }; @@ -635,5 +707,6 @@ int main(int argc, char **argv) MAIL_SERVER_PRE_INIT, pre_jail_init, MAIL_SERVER_POST_INIT, post_jail_init, MAIL_SERVER_SOLITARY, + MAIL_SERVER_EXIT, verify_dump, 0); }