From afb3b97707bbead26f122db51a05e81744a9a2b0 Mon Sep 17 00:00:00 2001 From: Wietse Venema Date: Mon, 22 Dec 2003 00:00:00 -0500 Subject: [PATCH] postfix-2.0.16-20031222 --- postfix/HISTORY | 16 ++++++++- postfix/RELEASE_NOTES | 39 +++++++++++++++++---- postfix/conf/master.cf | 1 + postfix/conf/sample-smtp.cf | 10 +++--- postfix/html/smtp.8.html | 14 ++++---- postfix/man/man1/smtp-sink.1 | 3 ++ postfix/man/man8/smtp.8 | 11 +++--- postfix/src/cleanup/cleanup_out_recipient.c | 8 +++-- postfix/src/global/mail_version.h | 2 +- postfix/src/smtp/smtp.c | 11 +++--- postfix/src/smtp/smtp.h | 32 ++++++++++------- postfix/src/smtp/smtp_addr.c | 22 +++++++++--- postfix/src/smtp/smtp_connect.c | 10 ++++-- postfix/src/smtp/smtp_rcpt.c | 5 ++- postfix/src/smtpstone/smtp-sink.c | 13 +++++-- 15 files changed, 143 insertions(+), 54 deletions(-) diff --git a/postfix/HISTORY b/postfix/HISTORY index cccdfed8b..4be4a4853 100644 --- a/postfix/HISTORY +++ b/postfix/HISTORY @@ -8871,7 +8871,7 @@ Apologies for any names omitted. between short queue ID and message status). File: showq/showq.c. -20031216-20 +20031216-21 Cleanup: the SMTP client now moves on to the next MX host or fallback relay when delivery fails in the middle of an @@ -8884,14 +8884,28 @@ Apologies for any names omitted. (limit the number of actual SMTP sessions per delivery attempt, ignoring unusable MX IP addresses). + The new code centers around a mark-and-sweep algorithm + (replacing code that twiddled the rcpt->offset structure + member), with paranoid sanity checks to ensure that every + recipient is explicitly accounted for. + 20031217 Update: LDAP client logging (Liviu Daia) and LDAP client documentation (Victor Duchovni). Files: util/dict_ldap.c, conf/sample-ldap.cf, README_FILES/LDAP_README. +20031222 + + Cleanup: shaved half the worst-case bits off the cleanup + duplicate address filter footprint. After discussion with + Victor Duchovni. File: cleanup/cleanup_out_recipient.c. + Open problems: + Low: in the SMTP client, pass the session, request and + state structures as separate arguments. + High: when virtual aliasing is turned off after content filtering, local submissions may escape virtual aliasing. diff --git a/postfix/RELEASE_NOTES b/postfix/RELEASE_NOTES index 162572466..56328ca31 100644 --- a/postfix/RELEASE_NOTES +++ b/postfix/RELEASE_NOTES @@ -22,13 +22,40 @@ snapshot release). Patches change the patchlevel and the release date. Snapshots change only the release date, unless they include the same bugfixes as a patch release. -Major changes with Postfix snapshot 2.0.16-20031221 +Incompatible changes with Postfix snapshot 2.0.16-20031222 +========================================================== + +In mailq (queue listing) output, there no longer is space between +a short queue ID and the "*" (delivery in progress) or ! (mail on +hold) status indicator. This makes the output easier to parse. + +The SMTP client now tries to connect to an alternate MX address +when a delivery attempt fails **after the initial SMTP handshake**. +This includes both broken connections and 4XX SMTP replies. To +get the old behavior, specify "smtp_mx_session_limit = 1" in main.cf. + +After delivery failure due to a temporary error condition, the SMTP +client now always connects to the fall-back relay if specified. + +Major changes with Postfix snapshot 2.0.16-20031222 =================================================== -The SMTP client now moves on to the next MX host (or fallback relay) -when delivery fails in the middle of a session. This includes both -broken connections as well as 4XX replies to SMTP commands. Finally, -fallback_relay works as expected. +The SMTP client now tries to connect to an alternate MX address +when a delivery attempt fails **after the initial SMTP handshake**. +This includes both broken connections and 4XX SMTP replies. + +Finally, fallback_relay works as promised. + +The new SMTP client connection management is controlled by two new +configuration parameters: + +- smtp_mx_address_limit (default unlimited): the number of MX (mail +exchanger) IP addresses that can result from mail exchanger lookups. + +- smtp_mx_session_limit (default 2): the number of SMTP sessions +per delivery request before giving up or delivering to a fall-back +relay, ignoring IP addresses that fail to complete the SMTP initial +handshake. Incompatible changes with Postfix snapshot 2.0.16-20031215 ========================================================== @@ -62,7 +89,7 @@ Incompatible changes with Postfix snapshot 2.0.16-20031203 ========================================================== Many SMTPD reject logfile entries now show NOQUEUE instead of a -queue ID. This is because Postfix no longer creates queue file +queue ID. This is because Postfix no longer creates a queue file before the SMTP server has received a valid recipient. The experimental XADDR and XLOGINFO extensions to SMTP are now diff --git a/postfix/conf/master.cf b/postfix/conf/master.cf index da5ef7698..765561969 100644 --- a/postfix/conf/master.cf +++ b/postfix/conf/master.cf @@ -113,6 +113,7 @@ maildrop unix - n n - - pipe old-cyrus unix - n n - - pipe flags=R user=cyrus argv=/cyrus/bin/deliver -e -m ${extension} ${user} # Cyrus 2.1.5 (Amos Gouaux) +# Also specify in main.cf: cyrus_destination_recipient_limit=1 cyrus unix - n n - - pipe user=cyrus argv=/cyrus/bin/deliver -e -r ${sender} -m ${extension} ${user} uucp unix - n n - - pipe diff --git a/postfix/conf/sample-smtp.cf b/postfix/conf/sample-smtp.cf index 5ea6fcffa..23986df51 100644 --- a/postfix/conf/sample-smtp.cf +++ b/postfix/conf/sample-smtp.cf @@ -77,17 +77,19 @@ smtp_never_send_ehlo = no smtp_defer_if_no_mx_address_found = no # The smtp_mx_address_limit parameter limits the number of MX (mail -# exchanger) IP addresses that can result from DNS or host lookups. +# exchanger) IP addresses that can result from mail exchanger lookups. # # Specify zero to disable the limit. This is also the default. # smtp_mx_address_limit = 0 # The smtp_mx_session_limit parameter limits the number of SMTP -# sessions that the client will engage in, skipping over MX IP -# addresses that fail to complete the SMTP initial handshake. +# sessions per delivery request before giving up or delivering to a +# fall-back relay host, ignoring IP addresses that fail to complete +# the SMTP initial handshake. # -# By default, Postfix SMTP client gives up after two SMTP sessions. +# By default, Postfix SMTP client gives up or delivers to fall-back +# relay after two SMTP sessions. # # Specify zero to disable the limit. # diff --git a/postfix/html/smtp.8.html b/postfix/html/smtp.8.html index 698d49086..442e63eec 100644 --- a/postfix/html/smtp.8.html +++ b/postfix/html/smtp.8.html @@ -243,9 +243,9 @@ SMTP(8) SMTP(8) default_destination_recipient_limit parameter. smtp_mx_address_limit - An upper bound on the number of MX (mail exchanger) - IP addresses that that can result from DNS or host - lookups. + An upper bound on the total number of MX (mail + exchanger) IP addresses that that can result from + mail exchanger lookups. Specify zero to disable the limit. @@ -253,10 +253,10 @@ SMTP(8) SMTP(8) sorted into random order. smtp_mx_session_limit - An upper bound on the number of SMTP sessions that - the SMTP client will engage in per message delivery - (ignoring MX IP addresses that fail to complete the - SMTP initial handshake). + An upper bound on the number of SMTP sessions per + delivery request before giving up or delivering to + a fall-back relay host (ignoring IP addresses that + fail to complete the SMTP initial handshake). Specify zero to disable the limit. diff --git a/postfix/man/man1/smtp-sink.1 b/postfix/man/man1/smtp-sink.1 index ad8c34936..dc27e5e26 100644 --- a/postfix/man/man1/smtp-sink.1 +++ b/postfix/man/man1/smtp-sink.1 @@ -51,6 +51,9 @@ Do not announce support for ESMTP command pipelining. .IP \fB-P\fR Change the server greeting so that it appears to come through a CISCO PIX system. Implies \fB-e\fR. +.IP "\fB-q \fIcommand,command,...\fR" +Disconnect (without replying) after receiving one of the +specified commands. .IP "\fB-r \fIcommand,command,...\fR" Reject the specified commands with a soft (4xx) error code. .IP "\fB-s \fIcommand,command,...\fR" diff --git a/postfix/man/man8/smtp.8 b/postfix/man/man8/smtp.8 index d06067d7d..80df1a34a 100644 --- a/postfix/man/man8/smtp.8 +++ b/postfix/man/man8/smtp.8 @@ -204,17 +204,18 @@ Limit the number of recipients per message delivery. The default limit is taken from the \fBdefault_destination_recipient_limit\fR parameter. .IP \fBsmtp_mx_address_limit\fR -An upper bound on the number of MX (mail exchanger) IP addresses -that that can result from DNS or host lookups. +An upper bound on the total number of MX (mail exchanger) IP +addresses that that can result from mail exchanger lookups. .sp Specify zero to disable the limit. .sp Note: by default, equal preference MX addresses are sorted into random order. .IP \fBsmtp_mx_session_limit\fR -An upper bound on the number of SMTP sessions that the SMTP -client will engage in per message delivery (ignoring MX IP -addresses that fail to complete the SMTP initial handshake). +An upper bound on the number of SMTP sessions per delivery request +before giving up or delivering to a fall-back relay host +(ignoring IP addresses that fail to complete the SMTP initial +handshake). .sp Specify zero to disable the limit. .SH "Timeout controls" diff --git a/postfix/src/cleanup/cleanup_out_recipient.c b/postfix/src/cleanup/cleanup_out_recipient.c index f05feb708..cdb39a7f0 100644 --- a/postfix/src/cleanup/cleanup_out_recipient.c +++ b/postfix/src/cleanup/cleanup_out_recipient.c @@ -80,9 +80,12 @@ void cleanup_out_recipient(CLEANUP_STATE *state, const char *orcpt, * onto the same mailbox. The recipient will use our original recipient * message header to figure things out. */ +#define STREQ(x, y) (strcmp((x), (y)) == 0) + if ((state->flags & CLEANUP_FLAG_MAP_OK) == 0 || cleanup_virt_alias_maps == 0) { - if (been_here(state->dups, "%s\n%s", orcpt, recip) == 0) { + if ((STREQ(orcpt, recip) ? been_here(state->dups, "%s", orcpt) : + been_here(state->dups, "%s\n%s", orcpt, recip)) == 0) { cleanup_out_string(state, REC_TYPE_ORCP, orcpt); cleanup_out_string(state, REC_TYPE_RCPT, recip); state->rcpt_count++; @@ -91,7 +94,8 @@ void cleanup_out_recipient(CLEANUP_STATE *state, const char *orcpt, argv = cleanup_map1n_internal(state, recip, cleanup_virt_alias_maps, cleanup_ext_prop_mask & EXT_PROP_VIRTUAL); for (cpp = argv->argv; *cpp; cpp++) { - if (been_here(state->dups, "%s\n%s", orcpt, *cpp) == 0) { + if ((STREQ(orcpt, *cpp) ? been_here(state->dups, "%s", orcpt) : + been_here(state->dups, "%s\n%s", orcpt, *cpp)) == 0) { cleanup_out_string(state, REC_TYPE_ORCP, orcpt); cleanup_out_string(state, REC_TYPE_RCPT, *cpp); state->rcpt_count++; diff --git a/postfix/src/global/mail_version.h b/postfix/src/global/mail_version.h index 6b875a530..44a4fbf6e 100644 --- a/postfix/src/global/mail_version.h +++ b/postfix/src/global/mail_version.h @@ -20,7 +20,7 @@ * Patches change the patchlevel and the release date. Snapshots change the * release date only, unless they include the same bugfix as a patch release. */ -#define MAIL_RELEASE_DATE "20031221" +#define MAIL_RELEASE_DATE "20031222" #define VAR_MAIL_VERSION "mail_version" #define DEF_MAIL_VERSION "2.0.16-" MAIL_RELEASE_DATE diff --git a/postfix/src/smtp/smtp.c b/postfix/src/smtp/smtp.c index 87f5262f8..d52471c6a 100644 --- a/postfix/src/smtp/smtp.c +++ b/postfix/src/smtp/smtp.c @@ -188,17 +188,18 @@ /* The default limit is taken from the /* \fBdefault_destination_recipient_limit\fR parameter. /* .IP \fBsmtp_mx_address_limit\fR -/* An upper bound on the number of MX (mail exchanger) IP addresses -/* that that can result from DNS or host lookups. +/* An upper bound on the total number of MX (mail exchanger) IP +/* addresses that that can result from mail exchanger lookups. /* .sp /* Specify zero to disable the limit. /* .sp /* Note: by default, equal preference MX addresses are sorted into /* random order. /* .IP \fBsmtp_mx_session_limit\fR -/* An upper bound on the number of SMTP sessions that the SMTP -/* client will engage in per message delivery (ignoring MX IP -/* addresses that fail to complete the SMTP initial handshake). +/* An upper bound on the number of SMTP sessions per delivery request +/* before giving up or delivering to a fall-back relay host +/* (ignoring IP addresses that fail to complete the SMTP initial +/* handshake). /* .sp /* Specify zero to disable the limit. /* .SH "Timeout controls" diff --git a/postfix/src/smtp/smtp.h b/postfix/src/smtp/smtp.h index 6dc6268b6..74850f9b8 100644 --- a/postfix/src/smtp/smtp.h +++ b/postfix/src/smtp/smtp.h @@ -97,15 +97,6 @@ extern int smtp_host_lookup_mask; /* host lookup methods to use */ #define SMTP_MASK_DNS (1<<0) #define SMTP_MASK_NATIVE (1<<1) - /* - * What soft errors qualify for going to a backup host. - */ -extern int smtp_backup_mask; /* when to try backup host */ - -#define SMTP_BACKUP_SESSION_FAILURE (1<<0) -#define SMTP_BACKUP_MESSAGE_FAILURE (1<<1) -#define SMTP_BACKUP_RECIPIENT_FAILURE (1<<2) - /* * smtp_session.c */ @@ -147,11 +138,26 @@ extern void smtp_chat_reset(SMTP_STATE *); extern void smtp_chat_notify(SMTP_STATE *); /* - * These operations are extensively documented in smtp_rcpt.c + * These operations implement a redundant mark-and-sweep algorithm that + * explicitly accounts for the fate of every recipient. The interface is + * documented in smtp_rcpt.c, which also implements the sweeping. The + * smtp_trouble.c module does most of the marking after failure. + * + * When a delivery fails or succeeds, take one of the following actions: + * + * - Mark the recipient as KEEP (deliver to alternate MTA) and do not update + * the delivery request status. + * + * - Mark the recipient as DROP (remove from delivery request), log whether + * delivery succeeded or failed, delete the recipient from the queue file + * and/or update defer or bounce logfiles, and update the delivery request + * status. + * + * At the end of a delivery attempt, all recipients must be marked one way or + * the other. Failure to do so will trigger a panic. */ -#define SMTP_RCPT_STATE_KEEP 1 /* send to backup host */ -#define SMTP_RCPT_STATE_DROP 2 /* remove from request */ - +#define SMTP_RCPT_STATE_KEEP 1 /* send to backup host */ +#define SMTP_RCPT_STATE_DROP 2 /* remove from request */ #define SMTP_RCPT_INIT(state) do { \ (state)->rcpt_drop = (state)->rcpt_keep = 0; \ (state)->rcpt_left = state->request->rcpt_list.len; \ diff --git a/postfix/src/smtp/smtp_addr.c b/postfix/src/smtp/smtp_addr.c index da701da67..54c90ea6c 100644 --- a/postfix/src/smtp/smtp_addr.c +++ b/postfix/src/smtp/smtp_addr.c @@ -184,10 +184,12 @@ static DNS_RR *smtp_addr_one(DNS_RR *addr_list, char *host, unsigned pref, VSTRI smtp_errno = SMTP_RETRY; return (addr_list); case DNS_FAIL: - smtp_errno = SMTP_FAIL; + if (smtp_errno != SMTP_RETRY) + smtp_errno = SMTP_FAIL; return (addr_list); case DNS_NOTFOUND: - smtp_errno = SMTP_FAIL; + if (smtp_errno != SMTP_RETRY) + smtp_errno = SMTP_FAIL; /* maybe gethostbyname() will succeed */ break; } @@ -200,12 +202,14 @@ static DNS_RR *smtp_addr_one(DNS_RR *addr_list, char *host, unsigned pref, VSTRI memset((char *) &fixed, 0, sizeof(fixed)); if ((hp = gethostbyname(host)) == 0) { vstring_sprintf(why, "%s: %s", host, HSTRERROR(h_errno)); - smtp_errno = (h_errno == TRY_AGAIN ? SMTP_RETRY : SMTP_FAIL); + if (smtp_errno != SMTP_RETRY) + smtp_errno = (h_errno == TRY_AGAIN ? SMTP_RETRY : SMTP_FAIL); } else if (hp->h_addrtype != AF_INET) { vstring_sprintf(why, "%s: host not found", host); msg_warn("%s: unknown address family %d for %s", myname, hp->h_addrtype, host); - smtp_errno = SMTP_FAIL; + if (smtp_errno != SMTP_RETRY) + smtp_errno = SMTP_FAIL; } else { while (hp->h_addr_list[0]) { addr_list = dns_rr_append(addr_list, @@ -335,6 +339,8 @@ DNS_RR *smtp_domain_addr(char *name, VSTRING *why) unsigned best_pref; unsigned best_found; + smtp_errno = SMTP_NONE; /* Paranoia */ + /* * Preferences from DNS use 0..32767, fall-backs use 32768+. */ @@ -449,12 +455,20 @@ DNS_RR *smtp_host_addr(char *host, VSTRING *why) { DNS_RR *addr_list; + smtp_errno = SMTP_NONE; /* Paranoia */ + /* * If the host is specified by numerical address, just convert the * address to internal form. Otherwise, the host is specified by name. */ #define PREF0 0 addr_list = smtp_addr_one((DNS_RR *) 0, host, PREF0, why); + if (addr_list && smtp_find_self(addr_list) != 0) { + dns_rr_free(addr_list); + vstring_sprintf(why, "mail for %s loops back to myself", host); + smtp_errno = SMTP_LOOP; + return (0); + } if (addr_list && addr_list->next && var_smtp_rand_addr) addr_list = dns_rr_shuffle(addr_list); if (msg_verbose) diff --git a/postfix/src/smtp/smtp_connect.c b/postfix/src/smtp/smtp_connect.c index 4f5a43d23..f628651b2 100644 --- a/postfix/src/smtp/smtp_connect.c +++ b/postfix/src/smtp/smtp_connect.c @@ -15,7 +15,9 @@ /* smtp_connect() attempts to establish an SMTP session with a host /* that represents the destination domain, or with an optional fallback /* relay when the destination cannot be found, or when all the -/* destination servers are unavailable. +/* destination servers are unavailable. It skips over IP addresses +/* that fail to complete the SMTP handshake and tries to find +/* an alternate server when an SMTP session fails to deliver. /* /* The destination is either a host (or domain) name or a numeric /* address. Symbolic or numeric service port information may be @@ -109,6 +111,8 @@ static SMTP_SESSION *smtp_connect_addr(DNS_RR *addr, unsigned port, int ch; unsigned long inaddr; + smtp_errno = SMTP_NONE; /* Paranoia */ + /* * Sanity checks. */ @@ -318,7 +322,6 @@ int smtp_connect(SMTP_STATE *state) */ for (cpp = sites->argv; SMTP_RCPT_LEFT(state) > 0 && (dest = *cpp) != 0; cpp++) { state->final_server = (cpp[1] == 0); - smtp_errno = SMTP_NONE; /* * Parse the destination. Default is to use the SMTP port. Look up @@ -378,6 +381,7 @@ int smtp_connect(SMTP_STATE *state) smtp_chat_notify(state); /* XXX smtp_xfer() may abort in the middle of DATA. */ smtp_session_free(state->session); + state->session = 0; debug_peer_restore(); smtp_rcpt_cleanup(state); } else { @@ -440,7 +444,7 @@ int smtp_connect(SMTP_STATE *state) * We still need to bounce or defer some left-over recipients: * either mail loops or some backup mail server was unavailable. */ - state->final_server = 1; + state->final_server = 1; /* XXX */ smtp_site_fail(state, smtp_errno == SMTP_RETRY ? 450 : 550, "%s", vstring_str(why)); diff --git a/postfix/src/smtp/smtp_rcpt.c b/postfix/src/smtp/smtp_rcpt.c index 8aa732edc..3450e0153 100644 --- a/postfix/src/smtp/smtp_rcpt.c +++ b/postfix/src/smtp/smtp_rcpt.c @@ -41,6 +41,9 @@ /* After a delivery attempt any recipients marked DROP are deleted /* from the request, and the left-over recipients are unmarked. /* .PP +/* The mark/sweep algorithm is implemented in a redundant manner, +/* and ensures that all recipients are explicitly accounted for. +/* /* Operations with upper case names are implemented by macros /* whose arguments may be evaluated more than once. /* @@ -170,7 +173,7 @@ void smtp_rcpt_cleanup(SMTP_STATE *state) */ if (state->rcpt_drop > 0 && state->rcpt_keep > 0) qsort((void *) rcpt_list->info, state->rcpt_left, - sizeof(rcpt_list->info), smtp_rcpt_cleanup_callback); + sizeof(rcpt_list->info[0]), smtp_rcpt_cleanup_callback); /* * Truncate the recipient list and unmark the left-over recipients. diff --git a/postfix/src/smtpstone/smtp-sink.c b/postfix/src/smtpstone/smtp-sink.c index 0abfd78d2..f038bf9f4 100644 --- a/postfix/src/smtpstone/smtp-sink.c +++ b/postfix/src/smtpstone/smtp-sink.c @@ -45,6 +45,9 @@ /* .IP \fB-P\fR /* Change the server greeting so that it appears to come through /* a CISCO PIX system. Implies \fB-e\fR. +/* .IP "\fB-q \fIcommand,command,...\fR" +/* Disconnect (without replying) after receiving one of the +/* specified commands. /* .IP "\fB-r \fIcommand,command,...\fR" /* Reject the specified commands with a soft (4xx) error code. /* .IP "\fB-s \fIcommand,command,...\fR" @@ -320,6 +323,7 @@ typedef struct SINK_COMMAND { #define FLAG_SYSLOG (1<<1) /* log the command */ #define FLAG_HARD_ERR (1<<2) /* report hard error */ #define FLAG_SOFT_ERR (1<<3) /* report soft error */ +#define FLAG_DISCONNECT (1<<4) /* disconnect */ static SINK_COMMAND command_table[] = { "helo", helo_response, 0, @@ -476,6 +480,8 @@ static int command_read(SINK_STATE *state) smtp_flush(state->stream); return (0); } + if (cmdp->flags & FLAG_DISCONNECT) + return (-1); if (cmdp->flags & FLAG_HARD_ERR) { smtp_printf(state->stream, "500 Error: command failed"); smtp_flush(state->stream); @@ -588,7 +594,7 @@ static void connect_event(int unused_event, char *context) static void usage(char *myname) { - msg_fatal("usage: %s [-ceLpPv8] [-h hostname] [-n count] [-s commands] [-w delay] [host]:port backlog", myname); + msg_fatal("usage: %s [-acCeFLpPv8] [-f commands] [-h hostname] [-n count] [-q commands] [-r commands] [-s commands] [-w delay] [host]:port backlog", myname); } int main(int argc, char **argv) @@ -605,7 +611,7 @@ int main(int argc, char **argv) /* * Parse JCL. */ - while ((ch = GETOPT(argc, argv, "acCef:Fh:Ln:pPr:s:vw:8")) > 0) { + while ((ch = GETOPT(argc, argv, "acCef:Fh:Ln:pPq:r:s:vw:8")) > 0) { switch (ch) { case 'a': disable_saslauth = 1; @@ -644,6 +650,9 @@ int main(int argc, char **argv) pretend_pix = 1; disable_esmtp = 1; break; + case 'q': + set_cmds_flags(optarg, FLAG_DISCONNECT); + break; case 'r': set_cmds_flags(optarg, FLAG_SOFT_ERR); break;