postfix-2.0.16-20031222

postfix/HISTORY

@@ -8871,7 +8871,7 @@ Apologies for any names omitted.
 	between short queue ID and message status).  File:
 	showq/showq.c.
 
-20031216-20
+20031216-21
 
 	Cleanup: the SMTP client now moves on to the next MX host
 	or fallback relay when delivery fails in the middle of an
@@ -8884,14 +8884,28 @@ Apologies for any names omitted.
 	(limit the number of actual SMTP sessions per delivery
 	attempt, ignoring unusable MX IP addresses).
 
+	The new code centers around a mark-and-sweep algorithm
+	(replacing code that twiddled the rcpt->offset structure
+	member), with paranoid sanity checks to ensure that every
+	recipient is explicitly accounted for.
+
 20031217
 
 	Update: LDAP client logging (Liviu Daia) and LDAP client
 	documentation (Victor Duchovni). Files: util/dict_ldap.c,
 	conf/sample-ldap.cf, README_FILES/LDAP_README.
 
+20031222
+
+	Cleanup: shaved half the worst-case bits off the cleanup
+	duplicate address filter footprint. After discussion with
+	Victor Duchovni. File: cleanup/cleanup_out_recipient.c.
+
 Open problems:
 
+	Low: in the SMTP client, pass the session, request and
+	state structures as separate arguments.
+
 	High: when virtual aliasing is turned off after content
 	filtering, local submissions may escape virtual aliasing.
 

postfix/RELEASE_NOTES

@@ -22,13 +22,40 @@ snapshot release).  Patches change the patchlevel and the release
 date. Snapshots change only the release date, unless they include
 the same bugfixes as a patch release.
 
-Major changes with Postfix snapshot 2.0.16-20031221
+Incompatible changes with Postfix snapshot 2.0.16-20031222
+==========================================================
+
+In mailq (queue listing) output, there no longer is space between
+a short queue ID and the "*" (delivery in progress) or ! (mail on
+hold) status indicator. This makes the output easier to parse.
+
+The SMTP client now tries to connect to an alternate MX address
+when a delivery attempt fails **after the initial SMTP handshake**.
+This includes both broken connections and 4XX SMTP replies.  To
+get the old behavior, specify "smtp_mx_session_limit = 1" in main.cf.
+
+After delivery failure due to a temporary error condition, the SMTP
+client now always connects to the fall-back relay if specified.
+
+Major changes with Postfix snapshot 2.0.16-20031222
 ===================================================
 
-The SMTP client now moves on to the next MX host (or fallback relay)
+The SMTP client now tries to connect to an alternate MX address
-when delivery fails in the middle of a session. This includes both
+when a delivery attempt fails **after the initial SMTP handshake**.
-broken connections as well as 4XX replies to SMTP commands.  Finally,
+This includes both broken connections and 4XX SMTP replies.
-fallback_relay works as expected.
+
+Finally, fallback_relay works as promised.
+
+The new SMTP client connection management is controlled by two new
+configuration parameters:
+
+- smtp_mx_address_limit (default unlimited): the number of MX (mail
+exchanger) IP addresses that can result from mail exchanger lookups.
+
+- smtp_mx_session_limit (default 2):  the number of SMTP sessions
+per delivery request before giving up or delivering to a fall-back
+relay, ignoring IP addresses that fail to complete the SMTP initial
+handshake.
 
 Incompatible changes with Postfix snapshot 2.0.16-20031215
 ==========================================================
@@ -62,7 +89,7 @@ Incompatible changes with Postfix snapshot 2.0.16-20031203
 ==========================================================
 
 Many SMTPD reject logfile entries now show NOQUEUE instead of a
-queue ID.  This is because Postfix no longer creates queue file
+queue ID.  This is because Postfix no longer creates a queue file
 before the SMTP server has received a valid recipient.
 
 The experimental XADDR and XLOGINFO extensions to SMTP are now

postfix/conf/master.cf

@@ -113,6 +113,7 @@ maildrop  unix  -       n       n       -       -       pipe
 old-cyrus unix  -       n       n       -       -       pipe
   flags=R user=cyrus argv=/cyrus/bin/deliver -e -m ${extension} ${user}
 # Cyrus 2.1.5 (Amos Gouaux)
+# Also specify in main.cf: cyrus_destination_recipient_limit=1
 cyrus     unix  -       n       n       -       -       pipe
   user=cyrus argv=/cyrus/bin/deliver -e -r ${sender} -m ${extension} ${user}
 uucp      unix  -       n       n       -       -       pipe

postfix/conf/sample-smtp.cf

@@ -77,17 +77,19 @@ smtp_never_send_ehlo = no
 smtp_defer_if_no_mx_address_found = no
 
 # The smtp_mx_address_limit parameter limits the number of MX (mail
-# exchanger) IP addresses that can result from DNS or host lookups.
+# exchanger) IP addresses that can result from mail exchanger lookups.
 #
 # Specify zero to disable the limit. This is also the default.
 #
 smtp_mx_address_limit = 0
 
 # The smtp_mx_session_limit parameter limits the number of SMTP
-# sessions that the client will engage in, skipping over MX IP
+# sessions per delivery request before giving up or delivering to a
-# addresses that fail to complete the SMTP initial handshake.
+# fall-back relay host, ignoring IP addresses that fail to complete
+# the SMTP initial handshake.
 # 
-# By default, Postfix SMTP client gives up after two SMTP sessions.
+# By default, Postfix SMTP client gives up or delivers to fall-back
+# relay after two SMTP sessions.  
 #
 # Specify zero to disable the limit.
 #

postfix/html/smtp.8.html

@@ -243,9 +243,9 @@ SMTP(8)                                                   SMTP(8)
               <b>default_destination_recipient_limit</b> parameter.
 
        <b>smtp_mx_address_limit</b>
-              An upper bound on the number of MX (mail exchanger)
+              An upper bound on the  total  number  of  MX  (mail
-              IP  addresses that that can result from DNS or host
+              exchanger)  IP  addresses that that can result from
-              lookups.
+              mail exchanger lookups.
 
               Specify zero to disable the limit.
 
@@ -253,10 +253,10 @@ SMTP(8)                                                   SMTP(8)
               sorted into random order.
 
        <b>smtp_mx_session_limit</b>
-              An  upper bound on the number of SMTP sessions that
+              An  upper  bound on the number of SMTP sessions per
-              the SMTP client will engage in per message delivery
+              delivery request before giving up or delivering  to
-              (ignoring MX IP addresses that fail to complete the
+              a  fall-back relay host (ignoring IP addresses that
-              SMTP initial handshake).
+              fail to complete the SMTP initial handshake).
 
               Specify zero to disable the limit.
 

postfix/man/man1/smtp-sink.1

@@ -51,6 +51,9 @@ Do not announce support for ESMTP command pipelining.
 .IP \fB-P\fR
 Change the server greeting so that it appears to come through
 a CISCO PIX system. Implies \fB-e\fR.
+.IP "\fB-q  \fIcommand,command,...\fR"
+Disconnect (without replying) after receiving one of the
+specified commands.
 .IP "\fB-r  \fIcommand,command,...\fR"
 Reject the specified commands with a soft (4xx) error code.
 .IP "\fB-s \fIcommand,command,...\fR"

postfix/man/man8/smtp.8

@@ -204,17 +204,18 @@ Limit the number of recipients per message delivery.
 The default limit is taken from the
 \fBdefault_destination_recipient_limit\fR parameter.
 .IP \fBsmtp_mx_address_limit\fR
-An upper bound on the number of MX (mail exchanger) IP addresses
+An upper bound on the total number of MX (mail exchanger) IP
-that that can result from DNS or host lookups.
+addresses that that can result from mail exchanger lookups.
 .sp
 Specify zero to disable the limit.
 .sp
 Note: by default, equal preference MX addresses are sorted into
 random order.
 .IP \fBsmtp_mx_session_limit\fR
-An upper bound on the number of SMTP sessions that the SMTP
+An upper bound on the number of SMTP sessions per delivery request
-client will engage in per message delivery (ignoring MX IP
+before giving up or delivering to a fall-back relay host
-addresses that fail to complete the SMTP initial handshake).
+(ignoring IP addresses that fail to complete the SMTP initial
+handshake).
 .sp
 Specify zero to disable the limit.
 .SH "Timeout controls"

postfix/src/cleanup/cleanup_out_recipient.c

@@ -80,9 +80,12 @@ void    cleanup_out_recipient(CLEANUP_STATE *state, const char *orcpt,
      * onto the same mailbox. The recipient will use our original recipient
      * message header to figure things out.
      */
+#define STREQ(x, y) (strcmp((x), (y)) == 0)
+
     if ((state->flags & CLEANUP_FLAG_MAP_OK) == 0
 	|| cleanup_virt_alias_maps == 0) {
-	if (been_here(state->dups, "%s\n%s", orcpt, recip) == 0) {
+	if ((STREQ(orcpt, recip) ? been_here(state->dups, "%s", orcpt) :
+	     been_here(state->dups, "%s\n%s", orcpt, recip)) == 0) {
 	    cleanup_out_string(state, REC_TYPE_ORCP, orcpt);
 	    cleanup_out_string(state, REC_TYPE_RCPT, recip);
 	    state->rcpt_count++;
@@ -91,7 +94,8 @@ void    cleanup_out_recipient(CLEANUP_STATE *state, const char *orcpt,
 	argv = cleanup_map1n_internal(state, recip, cleanup_virt_alias_maps,
 				  cleanup_ext_prop_mask & EXT_PROP_VIRTUAL);
 	for (cpp = argv->argv; *cpp; cpp++) {
-	    if (been_here(state->dups, "%s\n%s", orcpt, *cpp) == 0) {
+	    if ((STREQ(orcpt, *cpp) ? been_here(state->dups, "%s", orcpt) :
+		 been_here(state->dups, "%s\n%s", orcpt, *cpp)) == 0) {
 		cleanup_out_string(state, REC_TYPE_ORCP, orcpt);
 		cleanup_out_string(state, REC_TYPE_RCPT, *cpp);
 		state->rcpt_count++;

postfix/src/global/mail_version.h

@@ -20,7 +20,7 @@
   * Patches change the patchlevel and the release date. Snapshots change the
   * release date only, unless they include the same bugfix as a patch release.
   */
-#define MAIL_RELEASE_DATE	"20031221"
+#define MAIL_RELEASE_DATE	"20031222"
 
 #define VAR_MAIL_VERSION	"mail_version"
 #define DEF_MAIL_VERSION	"2.0.16-" MAIL_RELEASE_DATE

postfix/src/smtp/smtp.c

@@ -188,17 +188,18 @@
 /*	The default limit is taken from the
 /*	\fBdefault_destination_recipient_limit\fR parameter.
 /* .IP \fBsmtp_mx_address_limit\fR
-/*	An upper bound on the number of MX (mail exchanger) IP addresses
+/*	An upper bound on the total number of MX (mail exchanger) IP
-/*	that that can result from DNS or host lookups.
+/*	addresses that that can result from mail exchanger lookups.
 /* .sp
 /*	Specify zero to disable the limit.
 /* .sp
 /*	Note: by default, equal preference MX addresses are sorted into 
 /*	random order.
 /* .IP \fBsmtp_mx_session_limit\fR
-/*      An upper bound on the number of SMTP sessions that the SMTP
+/*      An upper bound on the number of SMTP sessions per delivery request
-/*      client will engage in per message delivery (ignoring MX IP 
+/*	before giving up or delivering to a fall-back relay host
-/*	addresses that fail to complete the SMTP initial handshake).
+/*	(ignoring IP addresses that fail to complete the SMTP initial 
+/*	handshake).
 /* .sp
 /*	Specify zero to disable the limit.
 /* .SH "Timeout controls"

postfix/src/smtp/smtp.h

@@ -97,15 +97,6 @@ extern int smtp_host_lookup_mask;	/* host lookup methods to use */
 #define SMTP_MASK_DNS		(1<<0)
 #define SMTP_MASK_NATIVE	(1<<1)
 
- /*
-  * What soft errors qualify for going to a backup host.
-  */
-extern int smtp_backup_mask;		/* when to try backup host */
-
-#define SMTP_BACKUP_SESSION_FAILURE	(1<<0)
-#define SMTP_BACKUP_MESSAGE_FAILURE	(1<<1)
-#define SMTP_BACKUP_RECIPIENT_FAILURE	(1<<2)
-
  /*
   * smtp_session.c
   */
@@ -147,11 +138,26 @@ extern void smtp_chat_reset(SMTP_STATE *);
 extern void smtp_chat_notify(SMTP_STATE *);
 
  /*
-  * These operations are extensively documented in smtp_rcpt.c
+  * These operations implement a redundant mark-and-sweep algorithm that
+  * explicitly accounts for the fate of every recipient. The interface is
+  * documented in smtp_rcpt.c, which also implements the sweeping. The
+  * smtp_trouble.c module does most of the marking after failure.
+  * 
+  * When a delivery fails or succeeds, take one of the following actions:
+  * 
+  * - Mark the recipient as KEEP (deliver to alternate MTA) and do not update
+  * the delivery request status.
+  * 
+  * - Mark the recipient as DROP (remove from delivery request), log whether
+  * delivery succeeded or failed, delete the recipient from the queue file
+  * and/or update defer or bounce logfiles, and update the delivery request
+  * status.
+  * 
+  * At the end of a delivery attempt, all recipients must be marked one way or
+  * the other. Failure to do so will trigger a panic.
   */
 #define SMTP_RCPT_STATE_KEEP	1	/* send to backup host */
 #define SMTP_RCPT_STATE_DROP	2	/* remove from request */
-
 #define SMTP_RCPT_INIT(state) do { \
 	    (state)->rcpt_drop = (state)->rcpt_keep = 0; \
 	    (state)->rcpt_left = state->request->rcpt_list.len; \

postfix/src/smtp/smtp_addr.c

@@ -184,9 +184,11 @@ static DNS_RR *smtp_addr_one(DNS_RR *addr_list, char *host, unsigned pref, VSTRI
 	    smtp_errno = SMTP_RETRY;
 	    return (addr_list);
 	case DNS_FAIL:
+	    if (smtp_errno != SMTP_RETRY)
 		smtp_errno = SMTP_FAIL;
 	    return (addr_list);
 	case DNS_NOTFOUND:
+	    if (smtp_errno != SMTP_RETRY)
 		smtp_errno = SMTP_FAIL;
 	    /* maybe gethostbyname() will succeed */
 	    break;
@@ -200,11 +202,13 @@ static DNS_RR *smtp_addr_one(DNS_RR *addr_list, char *host, unsigned pref, VSTRI
 	memset((char *) &fixed, 0, sizeof(fixed));
 	if ((hp = gethostbyname(host)) == 0) {
 	    vstring_sprintf(why, "%s: %s", host, HSTRERROR(h_errno));
+	    if (smtp_errno != SMTP_RETRY)
 		smtp_errno = (h_errno == TRY_AGAIN ? SMTP_RETRY : SMTP_FAIL);
 	} else if (hp->h_addrtype != AF_INET) {
 	    vstring_sprintf(why, "%s: host not found", host);
 	    msg_warn("%s: unknown address family %d for %s",
 		     myname, hp->h_addrtype, host);
+	    if (smtp_errno != SMTP_RETRY)
 		smtp_errno = SMTP_FAIL;
 	} else {
 	    while (hp->h_addr_list[0]) {
@@ -335,6 +339,8 @@ DNS_RR *smtp_domain_addr(char *name, VSTRING *why)
     unsigned best_pref;
     unsigned best_found;
 
+    smtp_errno = SMTP_NONE;			/* Paranoia */
+
     /*
      * Preferences from DNS use 0..32767, fall-backs use 32768+.
      */
@@ -449,12 +455,20 @@ DNS_RR *smtp_host_addr(char *host, VSTRING *why)
 {
     DNS_RR *addr_list;
 
+    smtp_errno = SMTP_NONE;			/* Paranoia */
+
     /*
      * If the host is specified by numerical address, just convert the
      * address to internal form. Otherwise, the host is specified by name.
      */
 #define PREF0	0
     addr_list = smtp_addr_one((DNS_RR *) 0, host, PREF0, why);
+    if (addr_list && smtp_find_self(addr_list) != 0) {
+	dns_rr_free(addr_list);
+	vstring_sprintf(why, "mail for %s loops back to myself", host);
+	smtp_errno = SMTP_LOOP;
+	return (0);
+    }
     if (addr_list && addr_list->next && var_smtp_rand_addr)
 	addr_list = dns_rr_shuffle(addr_list);
     if (msg_verbose)

postfix/src/smtp/smtp_connect.c

@@ -15,7 +15,9 @@
 /*	smtp_connect() attempts to establish an SMTP session with a host
 /*	that represents the destination domain, or with an optional fallback
 /*	relay when the destination cannot be found, or when all the
-/*	destination servers are unavailable.
+/*	destination servers are unavailable. It skips over IP addresses
+/*	that fail to complete the SMTP handshake and tries to find
+/*	an alternate server when an SMTP session fails to deliver.
 /*
 /*	The destination is either a host (or domain) name or a numeric
 /*	address. Symbolic or numeric service port information may be
@@ -109,6 +111,8 @@ static SMTP_SESSION *smtp_connect_addr(DNS_RR *addr, unsigned port,
     int     ch;
     unsigned long inaddr;
 
+    smtp_errno = SMTP_NONE;			/* Paranoia */
+
     /*
      * Sanity checks.
      */
@@ -318,7 +322,6 @@ int     smtp_connect(SMTP_STATE *state)
      */
     for (cpp = sites->argv; SMTP_RCPT_LEFT(state) > 0 && (dest = *cpp) != 0; cpp++) {
 	state->final_server = (cpp[1] == 0);
-	smtp_errno = SMTP_NONE;
 
 	/*
 	 * Parse the destination. Default is to use the SMTP port. Look up
@@ -378,6 +381,7 @@ int     smtp_connect(SMTP_STATE *state)
 		    smtp_chat_notify(state);
 		/* XXX smtp_xfer() may abort in the middle of DATA. */
 		smtp_session_free(state->session);
+		state->session = 0;
 		debug_peer_restore();
 		smtp_rcpt_cleanup(state);
 	    } else {
@@ -440,7 +444,7 @@ int     smtp_connect(SMTP_STATE *state)
 	     * We still need to bounce or defer some left-over recipients:
 	     * either mail loops or some backup mail server was unavailable.
 	     */
-	    state->final_server = 1;
+	    state->final_server = 1;		/* XXX */
 	    smtp_site_fail(state, smtp_errno == SMTP_RETRY ? 450 : 550,
 			   "%s", vstring_str(why));
 

postfix/src/smtp/smtp_rcpt.c

@@ -41,6 +41,9 @@
 /*	After a delivery attempt any recipients marked DROP are deleted 
 /*	from the request, and the left-over recipients are unmarked.
 /* .PP
+/*	The mark/sweep algorithm is implemented in a redundant manner,
+/*	and ensures that all recipients are explicitly accounted for.
+/*
 /*	Operations with upper case names are implemented by macros
 /*	whose arguments may be evaluated more than once.
 /*
@@ -170,7 +173,7 @@ void    smtp_rcpt_cleanup(SMTP_STATE *state)
      */
     if (state->rcpt_drop > 0 && state->rcpt_keep > 0)
 	qsort((void *) rcpt_list->info, state->rcpt_left,
-	      sizeof(rcpt_list->info), smtp_rcpt_cleanup_callback);
+	      sizeof(rcpt_list->info[0]), smtp_rcpt_cleanup_callback);
 
     /*
      * Truncate the recipient list and unmark the left-over recipients.

postfix/src/smtpstone/smtp-sink.c

@@ -45,6 +45,9 @@
 /* .IP \fB-P\fR
 /*	Change the server greeting so that it appears to come through
 /*	a CISCO PIX system. Implies \fB-e\fR.
+/* .IP "\fB-q  \fIcommand,command,...\fR"
+/*	Disconnect (without replying) after receiving one of the
+/*	specified commands.
 /* .IP "\fB-r  \fIcommand,command,...\fR"
 /*	Reject the specified commands with a soft (4xx) error code.
 /* .IP "\fB-s \fIcommand,command,...\fR"
@@ -320,6 +323,7 @@ typedef struct SINK_COMMAND {
 #define FLAG_SYSLOG	(1<<1)		/* log the command */
 #define FLAG_HARD_ERR	(1<<2)		/* report hard error */
 #define FLAG_SOFT_ERR	(1<<3)		/* report soft error */
+#define FLAG_DISCONNECT	(1<<4)		/* disconnect */
 
 static SINK_COMMAND command_table[] = {
     "helo", helo_response, 0,
@@ -476,6 +480,8 @@ static int command_read(SINK_STATE *state)
 	smtp_flush(state->stream);
 	return (0);
     }
+    if (cmdp->flags & FLAG_DISCONNECT) 
+	return (-1);
     if (cmdp->flags & FLAG_HARD_ERR) {
 	smtp_printf(state->stream, "500 Error: command failed");
 	smtp_flush(state->stream);
@@ -588,7 +594,7 @@ static void connect_event(int unused_event, char *context)
 
 static void usage(char *myname)
 {
-    msg_fatal("usage: %s [-ceLpPv8] [-h hostname] [-n count] [-s commands] [-w delay] [host]:port backlog", myname);
+    msg_fatal("usage: %s [-acCeFLpPv8] [-f commands] [-h hostname] [-n count] [-q commands] [-r commands] [-s commands] [-w delay] [host]:port backlog", myname);
 }
 
 int     main(int argc, char **argv)
@@ -605,7 +611,7 @@ int     main(int argc, char **argv)
     /*
      * Parse JCL.
      */
-    while ((ch = GETOPT(argc, argv, "acCef:Fh:Ln:pPr:s:vw:8")) > 0) {
+    while ((ch = GETOPT(argc, argv, "acCef:Fh:Ln:pPq:r:s:vw:8")) > 0) {
 	switch (ch) {
 	case 'a':
 	    disable_saslauth = 1;
@@ -644,6 +650,9 @@ int     main(int argc, char **argv)
 	    pretend_pix = 1;
 	    disable_esmtp = 1;
 	    break;
+	case 'q':
+	    set_cmds_flags(optarg, FLAG_DISCONNECT);
+	    break;
 	case 'r':
 	    set_cmds_flags(optarg, FLAG_SOFT_ERR);
 	    break;