snapshot-19991211

postfix/HISTORY

@@ -3327,17 +3327,11 @@ Apologies for any names omitted.
 
 	Performance: when delivering mail to a huge list of
 	recipients, the queue manager now reads more recipients
-	from the queue file before delivery concurrency starts
+	from the queue file before delivery concurrency drops too
-	to drop.  Files:  qmgr/qmgr_entry.c, qmgr/qmgr_message.c.
+	low.  Files:  qmgr/qmgr_entry.c, qmgr/qmgr_message.c.
 
 19991208
 
-	Performance: improved worst-case behavior. A fully loaded
-	Postfix inflicts the same delay to messages with any number
-	of recipients (up to qmgr_message_recipient_limit.) Inspired
-	by discussions with Patrik Rak (although he disagrees with
-	the strategy).  File: qmgr/qmgr_message.c.
-
 	Updated LDAP client code by John Hensley with escape
 	sequences as per RFC 2254. File: util/dict_ldap.c.
 
@@ -3357,3 +3351,21 @@ Apologies for any names omitted.
 	Robustness: attempt to deliver all addresses in the expansion
 	of an alias or .forward file, even when some addresses must
 	be deferred.  File: local/token.c.
+
+19991211
+
+	Performance: qmgr_fudge_factor controls what percentage of
+	delivery resources Postfix will devote to one message.
+	With 100%, delivery of one message does not begin before
+	delivery of the previous message is completed. This is good
+	for list performance, bad for one-to-one mail. With 10%,
+	response time for one-to-one mail improves much, but list
+	performance suffers. In the worst case, people near the
+	start of a mailing list get a burst of postings today,
+	while people near the end of the list get that same burst
+	of postings a whole day later.  Files: qmgr/qmgr_message.c,
+	qmgr/qmgr_entry.c.
+
+	Bugfix: address rewriting would panic on a lone \ at the
+	end of a line where an address was expected. Jason Hoos @
+	thwack.net. File: global/rewrite_clnt.c.

postfix/Makefile.in

@@ -11,7 +11,7 @@ default: update
 makefiles Makefiles:
 	set -e; for i in $(DIRS); do \
 	 (set -e; echo "[$$i]"; cd $$i; rm -f Makefile; \
-	 $(MAKE) -f Makefile.in Makefile); \
+	 $(MAKE) -f Makefile.in Makefile MAKELEVEL=); \
 	done;
 	rm -f Makefile; (set -e; $(SHELL) makedefs; cat Makefile.in) >Makefile
 

postfix/RELEASE_NOTES

@@ -1,10 +1,13 @@
-Incompatible changes with snapshot 19991209
+Incompatible changes with snapshot 19991211
 ===========================================
 
 - In an SMTPD access map, an all-numeric right-hand side now means
 OK.  This is for better cooperation with out-of-band authentication
 mechanisms such as POP before SMTP etc.
 
+- You can no longer use an empty right-hand side in SMTPD access
+maps.
+
 - Recipient addresses may no longer begin with `-'. In order to
 reinstate the old behavior, specify "allow_min_user = yes" in
 main.cf.
@@ -13,14 +16,30 @@ main.cf.
 SMTPD access control tables. Use the permit_recipient_map feature
 instead. The loss is compensated for (see below).
 
-- transport_maps entries override mydestination.  If any of the
+- transport_maps entries override mydestination.  For every
-$mydestination domains matches a transport specification, you also
+$mydestination domain that matches a transport map entry, or a
-need to add a "domain.name local:" entry in your transport_maps.
+parent domain of a transport map entry, you must now add a
+corresponding "domain.name local:" entry in your transport_maps.
 See the html/faq.html sections for firewalls and intranets.
 
-Major changes with snapshot 19991209
+Major changes with snapshot 19991211
 ====================================
 
+- Updated LDAP client code (John Hensley).
+
+- Updated mysql client code (Scott Cotton).
+
+- New "qmgr_fudge_factor" parameter allows you to balance mailing
+list performance against response time for one-to-one mail.  The
+fudge factor controls what percentage of delivery resources Postfix
+will devote to one message.  With 100%, delivery of one message
+does not begin before delivery of the previous message is completed.
+This is good for list performance, bad for one-to-one mail. With
+10%, response time for one-to-one mail improves much, but list
+performance suffers: in the worst case, people near the start of a
+mailing list get a burst of postings today, while people near the
+end of the list get that same burst of postings a whole day later.
+
 - It is now relatively safe to configure 550 status codes for the
 main.cf unknown_address_reject_code or unknown_client_reject_code
 parameters.  The SMTP server now always sends a 450 (try again)

postfix/global/mail_params.h

@@ -215,7 +215,8 @@ extern char *var_db_type;
 extern char *var_always_bcc;
 
  /*
-  * Standards violation: allow/permit RFC 822-style addresses in SMTP commands.
+  * Standards violation: allow/permit RFC 822-style addresses in SMTP
+  * commands.
   */
 #define VAR_STRICT_RFC821_ENV	"strict_rfc821_envelopes"
 #define DEF_STRICT_RFC821_ENV	0
@@ -417,6 +418,10 @@ extern int var_delay_warn_time;
 #define DEF_QMGR_RCPT_LIMIT	10000
 extern int var_qmgr_rcpt_limit;
 
+#define VAR_QMGR_FUDGE		"qmgr_fudge_factor"
+#define DEF_QMGR_FUDGE		100
+extern int var_qmgr_fudge;
+
  /*
   * Queue manager: default destination concurrency levels.
   */

postfix/global/mail_version.h

@@ -15,7 +15,7 @@
   * Version of this program.
   */
 #define VAR_MAIL_VERSION	"mail_version"
-#define DEF_MAIL_VERSION	"Snapshot-19991209"
+#define DEF_MAIL_VERSION	"Snapshot-19991211"
 extern char *var_mail_version;
 
 /* LICENSE

postfix/global/rewrite_clnt.c

@@ -96,7 +96,7 @@ VSTRING *rewrite_clnt(const char *rule, const char *addr, VSTRING *result)
 #define STR vstring_str
 
     if (*addr == 0)
-	msg_panic("rewrite_clnt: empty address");
+	addr = "";
     if (addr == STR(result))
 	msg_panic("rewrite_clnt: result clobbers input");
 

postfix/qmgr/qmgr.c

@@ -145,6 +145,8 @@
 /* .SH Miscellaneous
 /* .ad
 /* .fi
+/* .IP \fBallow_min_user\fR
+/*	Do not bounce recipient addresses that begin with '-'.
 /* .IP \fBrelocated_maps\fR
 /*	Tables with contact information for users, hosts or domains
 /*	that no longer exist. See \fBrelocated\fR(5).
@@ -186,6 +188,17 @@
 /* .fi
 /*	In the text below, \fItransport\fR is the first field in a
 /*	\fBmaster.cf\fR entry.
+/* .IP "\fBqmgr_fudge_factor\fR (valid range: 10..100)"
+/*	The percentage of delivery resources that a busy mail system will
+/*	use up for delivery of a large mailing list message.
+/*	With 100%, delivery of one message does not begin before the previous
+/*	message has been delivered. This results in good performance for large
+/*	mailing lists, but results in poor response time for one-to-one mail. 
+/*	With less than 100%, response time for one-to-one mail improves,
+/*	but large mailing list delivery performance suffers. In the worst
+/*	case, recipients near the beginning of a large list receive a burst
+/*	of messages immediately, while recipients near the end of that list
+/*	receive that same burst of messages a whole day later.
 /* .IP \fBinitial_destination_concurrency\fR
 /*	Initial per-destination concurrency level for parallel delivery
 /*	to the same destination.
@@ -267,6 +280,7 @@ char   *var_relocated_maps;
 char   *var_virtual_maps;
 char   *var_defer_xports;
 bool    var_allow_min_user;
+bool    var_qmgr_fudge;
 
 static QMGR_SCAN *qmgr_incoming;
 static QMGR_SCAN *qmgr_deferred;
@@ -459,6 +473,7 @@ int     main(int argc, char **argv)
 	VAR_XPORT_RETRY_TIME, DEF_XPORT_RETRY_TIME, &var_transport_retry_time, 1, 0,
 	VAR_DEST_CON_LIMIT, DEF_DEST_CON_LIMIT, &var_dest_con_limit, 0, 0,
 	VAR_DEST_RCPT_LIMIT, DEF_DEST_RCPT_LIMIT, &var_dest_rcpt_limit, 0, 0,
+	VAR_QMGR_FUDGE, DEF_QMGR_FUDGE, &var_qmgr_fudge, 10, 100,
 	0,
     };
     static CONFIG_BOOL_TABLE bool_table[] = {

postfix/qmgr/qmgr_entry.c

@@ -162,16 +162,25 @@ void    qmgr_entry_done(QMGR_ENTRY *entry, int which)
      * Update the in-core message reference count. When the in-core message
      * structure has no more references, dispose of the message.
      * 
-     * When the in-core recipient count falls below some threshold and this
+     * When the in-core recipient count falls below a threshold, and this
-     * message has more recipients, read more recipients before concurrency
+     * message has more recipients, read more recipients now. If we read more
-     * starts to drop.
+     * recipients as soon as the recipient count falls below the in-core
+     * recipient limit, we do not give other messages a chance until this
+     * message is delivered. That's good for mailing list deliveries, bad for
+     * one-to-one mail. If we wait until the in-core recipient count drops
+     * well below the in-core recipient limit, we give other mail a chance,
+     * but we also allow list deliveries to become interleaved. In the worst
+     * case, people near the start of a mailing list get a burst of postings
+     * today, while people near the end of the list get that same burst of
+     * postings a whole day later.
      */
+#define FUDGE(x)	((x) * (var_qmgr_fudge / 100.0))
     message->refcount--;
+    if (message->rcpt_offset > 0
+	     && qmgr_recipient_count < FUDGE(var_qmgr_rcpt_limit))
+	qmgr_message_realloc(message);
     if (message->refcount == 0)
 	qmgr_active_done(message);
-    else if (message->rcpt_offset > 0
-	     && qmgr_recipient_count < var_qmgr_rcpt_limit / 2)
-	qmgr_message_realloc(message);
 }
 
 /* qmgr_entry_create - create queue todo entry */

postfix/qmgr/qmgr_message.c

@@ -252,9 +252,10 @@ static int qmgr_message_read(QMGR_MESSAGE *message)
 		       message->data_size, "queue %s", message->queue_name);
 	    }
 	} else if (rec_type == REC_TYPE_RCPT) {
-	    if (message->rcpt_list.len < var_qmgr_rcpt_limit) {
+#define FUDGE(x)	((x) * (var_qmgr_fudge / 100.0))
+	    if (message->rcpt_list.len < FUDGE(var_qmgr_rcpt_limit)) {
 		qmgr_rcpt_list_add(&message->rcpt_list, curr_offset, start);
-		if (message->rcpt_list.len >= var_qmgr_rcpt_limit) {
+		if (message->rcpt_list.len >= FUDGE(var_qmgr_rcpt_limit)) {
 		    if ((message->rcpt_offset = vstream_ftell(message->fp)) < 0)
 			msg_fatal("vstream_ftell %s: %m",
 				  VSTREAM_PATH(message->fp));
@@ -325,9 +326,9 @@ static int qmgr_message_read(QMGR_MESSAGE *message)
 	|| message->data_offset == 0
 	|| (message->rcpt_offset == 0 && rec_type != REC_TYPE_END)) {
 	msg_warn("%s: envelope records out of order", message->queue_id);
+	message->rcpt_offset = save_offset;	/* restore flag */
 	return (-1);
     } else {
-	message->rcpt_offset = save_offset;	/* restore flag */
 	return (0);
     }
 }
@@ -741,8 +742,6 @@ QMGR_MESSAGE *qmgr_message_realloc(QMGR_MESSAGE *message)
      */
     if (message->rcpt_offset <= 0)
 	msg_panic("%s: invalid offset: %ld", myname, message->rcpt_offset);
-    if (message->refcount != 0)
-	msg_panic("%s: bad refcount: %d", myname, message->refcount);
     if (msg_verbose)
 	msg_info("%s: %s %s offset %ld", myname, message->queue_name,
 		 message->queue_id, message->rcpt_offset);