postfix-2.5-20071213

postfix/HISTORY

@@ -13950,13 +13950,13 @@ Apologies for any names omitted.
 
 	All changes up to this point should be ready for Postfix 2.5.
 
-	Documentation: updated nqgmr preemptive scheduler documentation
+	Documentation: updated nqmgr preemptive scheduler documentation
 	by Patrik Rak. File: proto/SCHEDULER_README.html.
 
 20071211
 
-	Bugfix: memory leak when the first write on a bi-directional
+	Bugfix (introduced 19980315): the "write" equivalent of
-	VSTREAM fails. File: util/vstream.c.
+	bugfix 20030104. File: util/vstream.c.
 
 20071212
 
@@ -13967,3 +13967,25 @@ Apologies for any names omitted.
 	starting with '-' at SMTP session time. To make this possible
 	the feature was moved from qmgr(8) to trivial-rewrite(8).
 	Files: *qmgr/qmgr_message.c, trivial-rewrite/resolve.c.
+
+20071213:
+
+	Cleanup: the queue manager and SMTP client now distinguish
+	between connection cache store and retrieve hints. Once the
+	queue manager enables enables connection caching (store and
+	load) hints on a per-destination queue, it keeps sending
+	connection cache retrieve hints to the delivery agent even
+	after it stops sending connection cache store hints.  This
+	prevents the SMTP client from making a new connection without
+	checking the connection cache first. Victor Duchovni.  Files:
+	*qmgr/qmgr_entry.c, smtp/smtp_connect.c.
+
+	Bugfix (introduced Postfix 2.3): the SMTP client never
+	marked corrupt files as corrupt. Victor Duchovni. File:
+	smtp/smtp_proto.c.
+
+	Cleanup: the SMTP client won't mark a destination as
+	unavailable when at least one SMTP session was completed
+	without connect or handshake error. Victor Duchovni. Files:
+	smtp/smtp_connect.c, smtp/smtp_session.c, smtp/smtp_proto.c,
+	smtp/smtp_trouble.c.

postfix/src/global/deliver_request.h

@@ -69,7 +69,14 @@ typedef struct DELIVER_REQUEST {
 #define DEL_REQ_FLAG_MTA_VRFY	(1<<8)	/* MTA-requested address probe */
 #define DEL_REQ_FLAG_USR_VRFY	(1<<9)	/* user-requested address probe */
 #define DEL_REQ_FLAG_RECORD	(1<<10)	/* record and deliver */
-#define DEL_REQ_FLAG_SCACHE	(1<<11)	/* opportunistic caching */
+#define DEL_REQ_FLAG_SCACHE_LD	(1<<11)	/* Consult opportunistic cache */
+#define DEL_REQ_FLAG_SCACHE_ST	(1<<12)	/* Update opportunistic cache */
+
+ /*
+  * Cache Load and Store as value or mask. Use explicit names for multi-bit
+  * values.
+  */
+#define DEL_REQ_FLAG_SCACHE_MASK (DEL_REQ_FLAG_SCACHE_LD|DEL_REQ_FLAG_SCACHE_ST)
 
  /*
   * For compatibility, the old confusing names.

postfix/src/global/mail_version.h

@@ -20,7 +20,7 @@
   * Patches change both the patchlevel and the release date. Snapshots have no
   * patchlevel; they change the release date only.
   */
-#define MAIL_RELEASE_DATE	"20071212"
+#define MAIL_RELEASE_DATE	"20071213"
 #define MAIL_VERSION_NUMBER	"2.5"
 
 #ifdef SNAPSHOT

postfix/src/oqmgr/qmgr_entry.c

@@ -123,9 +123,8 @@ QMGR_ENTRY *qmgr_entry_select(QMGR_QUEUE *queue)
 	 * (we need to recognize back-to-back deliveries for transports with
 	 * concurrency 1).
 	 * 
-	 * XXX It would be nice if we could say "try to reuse a cached
+	 * If caching has previously been enabled, but is not now, fetch any
-	 * connection, but don't bother saving it when you're done". As long
+	 * existing entries from the cache, but don't add new ones.
-	 * as we can't, we must not turn off session caching too early.
 	 */
 #define CONCURRENT_OR_BACK_TO_BACK_DELIVERY() \
 	    (queue->busy_refcount > 1 || BACK_TO_BACK_DELIVERY())
@@ -139,12 +138,12 @@ QMGR_ENTRY *qmgr_entry_select(QMGR_QUEUE *queue)
 	 * prevents unnecessary session caching when we have a burst of mail
 	 * <= the initial concurrency limit.
 	 */
-	if ((queue->dflags & DEL_REQ_FLAG_SCACHE) == 0) {
+	if ((queue->dflags & DEL_REQ_FLAG_SCACHE_ST) == 0) {
 	    if (BACK_TO_BACK_DELIVERY()) {
 		if (msg_verbose)
 		    msg_info("%s: allowing on-demand session caching for %s",
 			     myname, queue->name);
-		queue->dflags |= DEL_REQ_FLAG_SCACHE;
+		queue->dflags |= DEL_REQ_FLAG_SCACHE_MASK;
 	    }
 	}
 
@@ -159,7 +158,7 @@ QMGR_ENTRY *qmgr_entry_select(QMGR_QUEUE *queue)
 		if (msg_verbose)
 		    msg_info("%s: disallowing on-demand session caching for %s",
 			     myname, queue->name);
-		queue->dflags &= ~DEL_REQ_FLAG_SCACHE;
+		queue->dflags &= ~DEL_REQ_FLAG_SCACHE_ST;
 	    }
 	}
     }

postfix/src/qmgr/qmgr_entry.c

@@ -135,9 +135,8 @@ QMGR_ENTRY *qmgr_entry_select(QMGR_PEER *peer)
 	 * (we need to recognize back-to-back deliveries for transports with
 	 * concurrency 1).
 	 * 
-	 * XXX It would be nice if we could say "try to reuse a cached
+	 * If caching has previously been enabled, but is not now, fetch any
-	 * connection, but don't bother saving it when you're done". As long
+	 * existing entries from the cache, but don't add new ones.
-	 * as we can't, we must not turn off session caching too early.
 	 */
 #define CONCURRENT_OR_BACK_TO_BACK_DELIVERY() \
 	    (queue->busy_refcount > 1 || BACK_TO_BACK_DELIVERY())
@@ -151,12 +150,12 @@ QMGR_ENTRY *qmgr_entry_select(QMGR_PEER *peer)
 	 * prevents unnecessary session caching when we have a burst of mail
 	 * <= the initial concurrency limit.
 	 */
-	if ((queue->dflags & DEL_REQ_FLAG_SCACHE) == 0) {
+	if ((queue->dflags & DEL_REQ_FLAG_SCACHE_ST) == 0) {
 	    if (BACK_TO_BACK_DELIVERY()) {
 		if (msg_verbose)
 		    msg_info("%s: allowing on-demand session caching for %s",
 			     myname, queue->name);
-		queue->dflags |= DEL_REQ_FLAG_SCACHE;
+		queue->dflags |= DEL_REQ_FLAG_SCACHE_MASK;
 	    }
 	}
 
@@ -171,7 +170,7 @@ QMGR_ENTRY *qmgr_entry_select(QMGR_PEER *peer)
 		if (msg_verbose)
 		    msg_info("%s: disallowing on-demand session caching for %s",
 			     myname, queue->name);
-		queue->dflags &= ~DEL_REQ_FLAG_SCACHE;
+		queue->dflags &= ~DEL_REQ_FLAG_SCACHE_ST;
 	    }
 	}
     }

postfix/src/smtp/smtp.h

@@ -141,7 +141,12 @@ typedef struct SMTP_STATE {
 #define SMTP_MISC_FLAG_FIRST_NEXTHOP	(1<<3)
 #define SMTP_MISC_FLAG_FINAL_NEXTHOP	(1<<4)
 #define SMTP_MISC_FLAG_FINAL_SERVER	(1<<5)
-#define SMTP_MISC_FLAG_CONN_CACHE	(1<<6)
+#define SMTP_MISC_FLAG_CONN_LOAD	(1<<6)
+#define SMTP_MISC_FLAG_CONN_STORE	(1<<7)
+#define SMTP_MISC_FLAG_COMPLETE_SESSION	(1<<8)
+
+#define SMTP_MISC_FLAG_CONN_CACHE_MASK \
+	(SMTP_MISC_FLAG_CONN_LOAD | SMTP_MISC_FLAG_CONN_STORE)
 
  /*
   * smtp.c

postfix/src/smtp/smtp_addr.c

@@ -100,7 +100,7 @@
 
 /* smtp_print_addr - print address list */
 
-static void smtp_print_addr(char *what, DNS_RR *addr_list)
+static void smtp_print_addr(const char *what, DNS_RR *addr_list)
 {
     DNS_RR *addr;
     MAI_HOSTADDR_STR hostaddr;
@@ -120,8 +120,8 @@ static void smtp_print_addr(char *what, DNS_RR *addr_list)
 
 /* smtp_addr_one - address lookup for one host name */
 
-static DNS_RR *smtp_addr_one(DNS_RR *addr_list, char *host, unsigned pref,
+static DNS_RR *smtp_addr_one(DNS_RR *addr_list, const char *host,
-			             DSN_BUF *why)
+			             unsigned pref, DSN_BUF *why)
 {
     const char *myname = "smtp_addr_one";
     DNS_RR *addr = 0;
@@ -469,7 +469,7 @@ DNS_RR *smtp_domain_addr(char *name, int misc_flags, DSN_BUF *why,
 
 /* smtp_host_addr - direct host lookup */
 
-DNS_RR *smtp_host_addr(char *host, int misc_flags, DSN_BUF *why)
+DNS_RR *smtp_host_addr(const char *host, int misc_flags, DSN_BUF *why)
 {
     DNS_RR *addr_list;
 

postfix/src/smtp/smtp_addr.h

@@ -16,7 +16,7 @@
  /*
   * Internal interfaces.
   */
-extern DNS_RR *smtp_host_addr(char *, int, DSN_BUF *);
+extern DNS_RR *smtp_host_addr(const char *, int, DSN_BUF *);
 extern DNS_RR *smtp_domain_addr(char *, int, DSN_BUF *, int *);
 
 /* LICENSE

postfix/src/smtp/smtp_connect.c

@@ -394,7 +394,7 @@ static void smtp_cleanup_session(SMTP_STATE *state)
     if (THIS_SESSION_IS_EXPIRED)
 	smtp_quit(state);			/* also disables caching */
     if (THIS_SESSION_IS_CACHED
-	/* Redundant tests for safety... */
+    /* Redundant tests for safety... */
 	&& vstream_ferror(session->stream) == 0
 	&& vstream_feof(session->stream) == 0) {
 	smtp_save_session(state);
@@ -439,12 +439,36 @@ static void smtp_cleanup_session(SMTP_STATE *state)
     request->msg_stats.reuse_count = 0;
 }
 
+static void smtp_cache_policy(SMTP_STATE *state, const char *dest)
+{
+    DELIVER_REQUEST *request = state->request;
+
+    state->misc_flags &= ~SMTP_MISC_FLAG_CONN_CACHE_MASK;
+
+    /*
+     * XXX Disable connection caching when sender-dependent authentication is
+     * enabled. We must not send someone elses mail over an authenticated
+     * connection, and we must not send mail that requires authentication
+     * over a connection that wasn't authenticated.
+     */
+    if (var_smtp_sender_auth)
+	return;
+
+    if (smtp_cache_dest && string_list_match(smtp_cache_dest, dest)) {
+	state->misc_flags |= SMTP_MISC_FLAG_CONN_CACHE_MASK;
+    } else if (var_smtp_cache_demand) {
+	if (request->flags & DEL_REQ_FLAG_SCACHE_LD)
+	    state->misc_flags |= SMTP_MISC_FLAG_CONN_LOAD;
+	if (request->flags & DEL_REQ_FLAG_SCACHE_ST)
+	    state->misc_flags |= SMTP_MISC_FLAG_CONN_STORE;
+    }
+}
+
 /* smtp_connect_local - connect to local server */
 
 static void smtp_connect_local(SMTP_STATE *state, const char *path)
 {
     const char *myname = "smtp_connect_local";
-    DELIVER_REQUEST *request = state->request;
     SMTP_SESSION *session;
     DSN_BUF *why = state->why;
 
@@ -454,19 +478,8 @@ static void smtp_connect_local(SMTP_STATE *state, const char *path)
      * 
      * Connection cache management is based on the UNIX-domain pathname, without
      * the "unix:" prefix.
-     * 
-     * XXX Disable connection caching when sender-dependent authentication is
-     * enabled. We must not send someone elses mail over an authenticated
-     * connection, and we must not send mail that requires authentication
-     * over a connection that wasn't authenticated.
      */
-#define CAN_ENABLE_CONN_CACHE(request, dest) \
+    smtp_cache_policy(state, path);
-    (!var_smtp_sender_auth \
-     && ((var_smtp_cache_demand && (request->flags & DEL_REQ_FLAG_SCACHE)) \
-	 || (smtp_cache_dest && string_list_match(smtp_cache_dest, dest))))
-
-    if (CAN_ENABLE_CONN_CACHE(request, path))
-	state->misc_flags |= SMTP_MISC_FLAG_CONN_CACHE;
 
     /*
      * XXX We assume that the session->addr member refers to a copy of the
@@ -486,7 +499,7 @@ static void smtp_connect_local(SMTP_STATE *state, const char *path)
      * available, "encrypt" may be a sensible policy. Otherwise, we also
      * downgrade "encrypt" to "none", this time just to avoid waste.
      */
-    if ((state->misc_flags & SMTP_MISC_FLAG_CONN_CACHE) == 0
+    if ((state->misc_flags & SMTP_MISC_FLAG_CONN_LOAD) == 0
 	|| (session = smtp_reuse_addr(state, path, NO_PORT)) == 0)
 	session = smtp_connect_unix(path, why, state->misc_flags);
     if ((state->session = session) != 0) {
@@ -796,10 +809,10 @@ static void smtp_connect_remote(SMTP_STATE *state, const char *nexthop,
 	 * authenticated connection, and we must not send mail that requires
 	 * authentication over a connection that wasn't authenticated.
 	 */
-	if (addr_list && (state->misc_flags & SMTP_MISC_FLAG_FIRST_NEXTHOP)
+	if (addr_list && (state->misc_flags & SMTP_MISC_FLAG_FIRST_NEXTHOP)) {
-	    && CAN_ENABLE_CONN_CACHE(request, domain)) {
+	    smtp_cache_policy(state, domain);
-	    state->misc_flags |= SMTP_MISC_FLAG_CONN_CACHE;
+	    if (state->misc_flags & SMTP_MISC_FLAG_CONN_STORE)
-	    SET_NEXTHOP_STATE(state, lookup_mx, domain, port);
+		SET_NEXTHOP_STATE(state, lookup_mx, domain, port);
 	}
 
 	/*
@@ -812,7 +825,7 @@ static void smtp_connect_remote(SMTP_STATE *state, const char *nexthop,
 	 * fall-back destination. smtp_reuse_session() will truncate the
 	 * address list when either limit is reached.
 	 */
-	if (addr_list && state->misc_flags & SMTP_MISC_FLAG_CONN_CACHE) {
+	if (addr_list && (state->misc_flags & SMTP_MISC_FLAG_CONN_LOAD)) {
 	    if (state->cache_used->used > 0)
 		smtp_scrub_addr_list(state->cache_used, &addr_list);
 	    sess_count = addr_count =
@@ -843,7 +856,7 @@ static void smtp_connect_remote(SMTP_STATE *state, const char *nexthop,
 	    next = addr->next;
 	    if (++addr_count == var_smtp_mxaddr_limit)
 		next = 0;
-	    if ((state->misc_flags & SMTP_MISC_FLAG_CONN_CACHE) == 0
+	    if ((state->misc_flags & SMTP_MISC_FLAG_CONN_LOAD) == 0
 		|| addr->pref == domain_best_pref
 		|| dns_rr_to_pa(addr, &hostaddr) == 0
 		|| !(session = smtp_reuse_addr(state, hostaddr.buf, port)))

postfix/src/smtp/smtp_proto.c

@@ -1847,8 +1847,13 @@ static int smtp_loop(SMTP_STATE *state, NOCLOBBER int send_state,
 		    fail_status = smtp_mesg_fail(state, DSN_BY_LOCAL_MTA,
 					     SMTP_RESP_FAKE(&fake, "5.3.0"),
 					     "unreadable mail queue entry");
-		    if (fail_status == 0)
+		    /* Bailing out, abort stream with prejudice */
+		    (void) vstream_fpurge(session->stream, VSTREAM_PURGE_BOTH);
+		    DONT_USE_DEAD_SESSION;
+		    /* If bounce_append() succeeded, status is still 0 */
+		    if (state->status == 0)
 			(void) mark_corrupt(state->src);
+		    /* Don't override smtp_mesg_fail() here. */
 		    RETURN(fail_status);
 		}
 	    } else {
@@ -1899,6 +1904,7 @@ int     smtp_xfer(SMTP_STATE *state)
     int     send_state;
     int     recv_state;
     int     send_name_addr;
+    int     result;
 
     /*
      * Sanity check. Recipients should be unmarked at this point.
@@ -1921,6 +1927,8 @@ int     smtp_xfer(SMTP_STATE *state)
 		    "message size %lu exceeds size limit %.0f of server %s",
 		       request->data_size, (double) session->size_limit,
 		       session->namaddr);
+	/* Redundant. We abort this delivery attempt. */
+	state->misc_flags |= SMTP_MISC_FLAG_COMPLETE_SESSION;
 	return (0);
     }
 
@@ -1947,7 +1955,20 @@ int     smtp_xfer(SMTP_STATE *state)
     else
 	recv_state = send_state = SMTP_STATE_MAIL;
 
-    return (smtp_loop(state, send_state, recv_state));
+    /*
+     * Remember this session's "normal completion", even if the server 4xx-ed
+     * some or all recipients. Connection or handshake errors with a later MX
+     * host should not cause this destination be marked as unreachable.
+     */
+    result = smtp_loop(state, send_state, recv_state);
+
+    if (result == 0
+    /* Just in case */
+	&& vstream_ferror(session->stream) == 0
+	&& vstream_feof(session->stream) == 0)
+	state->misc_flags |= SMTP_MISC_FLAG_COMPLETE_SESSION;
+
+    return (result);
 }
 
 /* smtp_rset - send a lone RSET command */

postfix/src/smtp/smtp_session.c

@@ -66,9 +66,12 @@
 /* .IP flags
 /*	Zero or more of the following:
 /* .RS
-/* .IP SMTP_MISC_FLAG_CONN_CACHE
+/* .IP SMTP_MISC_FLAG_CONN_LOAD
-/*	Enable SMTP or LMTP connection caching.
+/*	Enable re-use of cached SMTP or LMTP connections.
+/* .IP SMTP_MISC_FLAG_CONN_STORE
+/*	Enable saving of cached SMTP or LMTP connections.
 /* .RE
+/*	SMTP_MISC_FLAG_CONN_MASK corresponds with both _LOAD and _STORE.
 /* .IP dest_prop
 /*	Destination specific session properties: the server is the
 /*	best MX host for the current logical destination.
@@ -545,7 +548,7 @@ SMTP_SESSION *smtp_session_alloc(VSTREAM *stream, const char *dest,
     session->sndbufsize = 0;
     session->send_proto_helo = 0;
 
-    if (flags & SMTP_MISC_FLAG_CONN_CACHE)
+    if (flags & SMTP_MISC_FLAG_CONN_STORE)
 	CACHE_THIS_SESSION_UNTIL(start + var_smtp_reuse_time);
     else
 	DONT_CACHE_THIS_SESSION;

postfix/src/smtp/smtp_trouble.c

@@ -244,7 +244,8 @@ static int smtp_bulk_fail(SMTP_STATE *state, int throttle_queue)
 	    SMTP_RCPT_DROP(state, rcpt);
 	    state->status |= status;
 	}
-	if (throttle_queue && soft_error && request->hop_status == 0)
+	if ((state->misc_flags & SMTP_MISC_FLAG_COMPLETE_SESSION) == 0
+	    && throttle_queue && soft_error && request->hop_status == 0)
 	    request->hop_status = DSN_COPY(&why->dsn);
     }