postfix-3.3-20171223

postfix/HISTORY

@@ -23211,3 +23211,16 @@ Apologies for any names omitted.
 	Feature: preliminary support to run Postfix in the foreground.
 	This requires that multi-instance support is disabled.
 	Files: conf/postfix-script, postfix/postfix.c.
+
+20171223
+
+	Feature: Milters can now send RET and ENVID arguments in
+	SMFI_CHGFROM requests. Files: cleanup/Makefile.in,
+	cleanup/cleanup.h, cleanup/cleanup_envelope.c,
+	cleanup/cleanup_milter.c, cleanup/cleanup_milter.in13h,
+	cleanup/cleanup_milter.in13i, cleanup/cleanup_milter.ref13c,
+	cleanup/cleanup_milter.ref13d, cleanup/cleanup_milter.ref13f,
+	cleanup/cleanup_milter.ref13g, cleanup/cleanup_milter.ref13h,
+	cleanup/cleanup_milter.ref13i, cleanup/cleanup_state.c,
+	cleanup/test-queue-file13h, cleanup/test-queue-file13i,
+	oqmgr/qmgr_message.c, qmgr/qmgr_message.c.

postfix/WISHLIST

@@ -9,14 +9,12 @@ Wish list:
 	After I/O error, store errno in VSTREAM object before errno
 	may be overwritten.
 
-	Is it possible for the Milter client to 'chgfrom' the sender's
-	DSN attributes?  That is, keep existing ones or set new ones?
-
 	Add $smtpd_sender_login_maps to proxy_read_maps.
 
 	Add some tips for logging from container:
 	https://www.projectatomic.io/blog/2016/10/playing-with-docker-logging/;
-	syslog_name = $myhostname/postfix; mkdir + postfix check
+	syslog_name = $myhostname/postfix; mkdir queue and data
+	dir; postfix check to create queue subdirectories.
 
 	Add postwhite as a postscreen-related project.
 	https://github.com/stevejenkins/postwhite/blob/master/README.md

postfix/src/cleanup/Makefile.in

@@ -79,7 +79,7 @@ milter_tests: cleanup_milter_test bug_tests \
 	cleanup_milter_test10e cleanup_milter_test11 cleanup_milter_test12 \
 	cleanup_milter_test13a cleanup_milter_test13b cleanup_milter_test13c \
 	cleanup_milter_test13d cleanup_milter_test13e cleanup_milter_test13f \
-	cleanup_milter_test13g \
+	cleanup_milter_test13g cleanup_milter_test13h cleanup_milter_test13i \
 	cleanup_milter_test14a cleanup_milter_test14b cleanup_milter_test14c \
 	cleanup_milter_test14d cleanup_milter_test14e cleanup_milter_test14f \
 	cleanup_milter_test14g \
@@ -376,6 +376,24 @@ cleanup_milter_test13g: cleanup_milter test-queue-file13g cleanup_milter.in13g \
 	diff cleanup_milter.ref13g cleanup_milter.tmp
 	rm -f test-queue-file13g.tmp cleanup_milter.tmp
 
+cleanup_milter_test13h: cleanup_milter test-queue-file13h cleanup_milter.in13h \
+	cleanup_milter.ref13h ../postcat/postcat
+	cp test-queue-file13h test-queue-file13h.tmp
+	chmod u+w test-queue-file13h.tmp
+	$(SHLIB_ENV) ./cleanup_milter <cleanup_milter.in13h
+	$(SHLIB_ENV) ../postcat/postcat -ov test-queue-file13h.tmp 2>/dev/null >cleanup_milter.tmp
+	diff cleanup_milter.ref13h cleanup_milter.tmp
+	rm -f test-queue-file13h.tmp cleanup_milter.tmp
+
+cleanup_milter_test13i: cleanup_milter test-queue-file13i cleanup_milter.in13i \
+	cleanup_milter.ref13i ../postcat/postcat
+	cp test-queue-file13i test-queue-file13i.tmp
+	chmod u+w test-queue-file13i.tmp
+	$(SHLIB_ENV) ./cleanup_milter <cleanup_milter.in13i
+	$(SHLIB_ENV) ../postcat/postcat -ov test-queue-file13i.tmp 2>/dev/null >cleanup_milter.tmp
+	diff cleanup_milter.ref13i cleanup_milter.tmp
+	rm -f test-queue-file13i.tmp cleanup_milter.tmp
+
 cleanup_milter_test13f: cleanup_milter test-queue-file13f cleanup_milter.in13f \
 	cleanup_milter.ref13f ../postcat/postcat
 	cp test-queue-file13f test-queue-file13f.tmp

postfix/src/cleanup/cleanup.h

@@ -119,6 +119,7 @@ typedef struct CLEANUP_STATE {
     VSTRING *milter_err_text;		/* milter call-back reply */
     HBC_CHECKS *milter_hbc_checks;	/* Milter header checks */
     VSTRING *milter_hbc_reply;		/* Milter header checks reply */
+    VSTRING *milter_dsn_buf;		/* Milter DSN parsing buffer */
 
     /*
      * Support for Milter body replacement requests.

postfix/src/cleanup/cleanup_envelope.c

@@ -406,31 +406,21 @@ static void cleanup_envelope_process(CLEANUP_STATE *state, int type,
 	return;
     }
     if (mapped_type == REC_TYPE_DSN_ENVID) {
-	/* Allow only one instance. */
+	/* Don't break "postsuper -r" after Milter overrides ENVID. */
-	if (state->dsn_envid != 0) {
-	    msg_warn("%s: message rejected: multiple DSN envelope ID records",
-		     state->queue_id);
-	    state->errs |= CLEANUP_STAT_BAD;
-	    return;
-	}
 	if (!allprint(mapped_buf)) {
 	    msg_warn("%s: message rejected: bad DSN envelope ID record",
 		     state->queue_id);
 	    state->errs |= CLEANUP_STAT_BAD;
 	    return;
 	}
+	if (state->dsn_envid != 0)
+	    myfree(state->dsn_envid);
 	state->dsn_envid = mystrdup(mapped_buf);
 	cleanup_out(state, type, buf, len);
 	return;
     }
     if (mapped_type == REC_TYPE_DSN_RET) {
-	/* Allow only one instance. */
+	/* Don't break "postsuper -r" after Milter overrides RET. */
-	if (state->dsn_ret != 0) {
-	    msg_warn("%s: message rejected: multiple DSN RET records",
-		     state->queue_id);
-	    state->errs |= CLEANUP_STAT_BAD;
-	    return;
-	}
 	if (!alldig(mapped_buf) || (junk = atoi(mapped_buf)) == 0
 	    || DSN_RET_OK(junk) == 0) {
 	    msg_warn("%s: message rejected: bad DSN RET record <%.200s>",

postfix/src/cleanup/cleanup_milter.c

@@ -1330,22 +1330,61 @@ static const char *cleanup_chg_from(void *context, const char *ext_from,
 {
     const char *myname = "cleanup_chg_from";
     CLEANUP_STATE *state = (CLEANUP_STATE *) context;
+    off_t   new_offset;
     off_t   new_sender_offset;
     off_t   after_sender_offs;
     int     addr_count;
     TOK822 *tree;
     TOK822 *tp;
     VSTRING *int_sender_buf;
+    int     dsn_envid = 0;
+    int     dsn_ret = 0;
 
     if (msg_verbose)
 	msg_info("%s: \"%s\" \"%s\"", myname, ext_from, esmtp_args);
 
-    if (esmtp_args[0])
+    /*
-	msg_warn("%s: %s: ignoring ESMTP arguments \"%.100s\"",
+     * ESMTP support is limited to RET and ENVID, i.e. things that are stored
-		 state->queue_id, myname, esmtp_args);
+     * together with the sender queue file record.
+     */
+    if (esmtp_args[0]) {
+	ARGV   *esmtp_argv;
+	int     i;
+	const char *arg;
+
+	esmtp_argv = argv_split(esmtp_args, " ");
+	for (i = 0; i < esmtp_argv->argc; ++i) {
+	    arg = esmtp_argv->argv[i];
+	    if (strncasecmp(arg, "RET=", 4) == 0) {
+		if ((dsn_ret = dsn_ret_code(arg + 4)) == 0) {
+		    msg_warn("Ignoring bad ESMTP parameter \"%s\" in "
+			     "SMFI_CHGFROM request", arg);
+		} else {
+		    state->dsn_ret = dsn_ret;
+		}
+	    } else if (strncasecmp(arg, "ENVID=", 6) == 0) {
+		if (state->milter_dsn_buf == 0)
+		    state->milter_dsn_buf = vstring_alloc(20);
+		dsn_envid = (xtext_unquote(state->milter_dsn_buf, arg + 6)
+			     && allprint(STR(state->milter_dsn_buf)));
+		if (!dsn_envid) {
+		    msg_warn("Ignoring bad ESMTP parameter \"%s\" in "
+			     "SMFI_CHGFROM request", arg);
+		} else {
+		    if (state->dsn_envid)
+			myfree(state->dsn_envid);
+		    state->dsn_envid = mystrdup(STR(state->milter_dsn_buf));
+		}
+	    } else {
+		msg_warn("Ignoring bad ESMTP parameter \"%s\" in "
+			 "SMFI_CHGFROM request", arg);
+	    }
+	}
+	argv_free(esmtp_argv);
+    }
 
     /*
-     * The cleanup server remembers the location of the the original sender
+     * The cleanup server remembers the file offset of the current sender
      * address record (offset in sender_pt_offset) and the file offset of the
      * record that follows the sender address (offset in sender_pt_target).
      * Short original sender records are padded, so that they can safely be
@@ -1357,23 +1396,37 @@ static const char *cleanup_chg_from(void *context, const char *ext_from,
 	msg_panic("%s: no post-sender record offset", myname);
 
     /*
-     * Allocate space after the end of the queue file, and write the new
+     * Allocate space after the end of the queue file, and write the new {DSN
-     * sender record, followed by a reverse pointer record that points to the
+     * envid, DSN ret, sender address, sender BCC} records, followed by a
-     * record that follows the original sender address record. No padding is
+     * reverse pointer record that points to the record that follows the
-     * needed for a "new" short sender record, since the record is not meant
+     * original sender record.
-     * to be overwritten. When the "new" sender is replaced, we allocate a
-     * new record at the end of the queue file.
      * 
      * We update the queue file in a safe manner: save the new sender after the
      * end of the queue file, write the reverse pointer, and only then
      * overwrite the old sender record with the forward pointer to the new
      * sender.
      */
-    if ((new_sender_offset = vstream_fseek(state->dst, (off_t) 0, SEEK_END)) < 0) {
+    if ((new_offset = vstream_fseek(state->dst, (off_t) 0, SEEK_END)) < 0) {
 	msg_warn("%s: seek file %s: %m", myname, cleanup_path);
 	return (cleanup_milter_error(state, errno));
     }
 
+    /*
+     * Sender DSN attribute records precede the sender record.
+     */
+    if (dsn_envid)
+	rec_fprintf(state->dst, REC_TYPE_ATTR, "%s=%s",
+		    MAIL_ATTR_DSN_ENVID, STR(state->milter_dsn_buf));
+    if (dsn_ret)
+	rec_fprintf(state->dst, REC_TYPE_ATTR, "%s=%d",
+		    MAIL_ATTR_DSN_RET, dsn_ret);
+    if (dsn_envid == 0 && dsn_ret == 0) {
+	new_sender_offset = new_offset;
+    } else if ((new_sender_offset = vstream_ftell(state->dst)) < 0) {
+	msg_warn("%s: vstream_ftell file %s: %m", myname, cleanup_path);
+	return (cleanup_milter_error(state, errno));
+    }
+
     /*
      * Transform the address from external form to internal form. This also
      * removes the enclosing <>, if present.
@@ -1402,15 +1455,20 @@ static const char *cleanup_chg_from(void *context, const char *ext_from,
     state->sender_pt_target = after_sender_offs;
 
     /*
-     * Overwrite the original sender record with the pointer to the new
+     * Overwrite the current sender record with the pointer to the new {DSN
-     * sender address record.
+     * envid, DSN ret, sender address, sender BCC} records.
      */
     if (vstream_fseek(state->dst, state->sender_pt_offset, SEEK_SET) < 0) {
 	msg_warn("%s: seek file %s: %m", myname, cleanup_path);
 	return (cleanup_milter_error(state, errno));
     }
     cleanup_out_format(state, REC_TYPE_PTR, REC_TYPE_PTR_FORMAT,
-		       (long) new_sender_offset);
+		       (long) new_offset);
+
+    /*
+     * Remember the location of the new current sender record.
+     */
+    state->sender_pt_offset = new_sender_offset;
 
     /*
      * In case of error while doing record output.
@@ -2427,6 +2485,7 @@ int     main(int unused_argc, char **argv)
     char   *bufp;
     int     istty = isatty(vstream_fileno(VSTREAM_IN));
     CLEANUP_STATE *state = cleanup_state_alloc((VSTREAM *) 0);
+    const char *parens = "{}";
 
     state->queue_id = mystrdup("NOQUEUE");
     state->sender = mystrdup("sender");
@@ -2459,7 +2518,7 @@ int     main(int unused_argc, char **argv)
 	}
 	if (*bufp == '#' || *bufp == 0 || allspace(bufp))
 	    continue;
-	argv = argv_split(bufp, " ");
+	argv = argv_splitq(bufp, " ", parens);
 	if (argv->argc == 0) {
 	    msg_warn("missing command");
 	} else if (strcmp(argv->argv[0], "?") == 0) {
@@ -2539,7 +2598,15 @@ int     main(int unused_argc, char **argv)
 	    if (argv->argc != 3) {
 		msg_warn("bad chg_from argument count: %ld", (long) argv->argc);
 	    } else {
-		cleanup_chg_from(state, argv->argv[1], argv->argv[2]);
+		char   *arg = argv->argv[2];
+		const char *err;
+
+		if (*arg == parens[0]
+		    && (err = extpar(&arg, parens, EXTPAR_FLAG_NONE)) != 0) {
+		    msg_warn("%s in \"%s\"", err, arg);
+		} else {
+		    cleanup_chg_from(state, argv->argv[1], arg);
+		}
 	    }
 	} else if (strcmp(argv->argv[0], "add_rcpt") == 0) {
 	    if (argv->argc != 2) {

postfix/src/cleanup/cleanup_milter.in13h

@@ -0,0 +1,8 @@
+#verbose on
+open test-queue-file13h.tmp
+
+# Change the sender.
+
+chg_from m@porcupine.org { ret=hdrs envid=env-for-m }
+
+close

postfix/src/cleanup/cleanup_milter.in13i

@@ -0,0 +1,9 @@
+#verbose on
+open test-queue-file13i.tmp
+
+# Change the sender.
+
+chg_from m@porcupine.org { ret=hdrs envid=env-for-m }
+chg_from n@porcupine.org { ret=full envid=env-for-n }
+
+close

postfix/src/cleanup/cleanup_milter.ref13c

@@ -4,7 +4,8 @@
       100 create_time: Sun Jan 21 13:33:08 2007
       124 named_attribute: rewrite_context=local
       147 sender_fullname: Wietse Venema
-      162 pointer_record:             607
+      162 pointer_record:             573
+      573 pointer_record:             607
       607 sender: n@porcupine.org
       624 pointer_record:             590
       590 pointer_record:             180

postfix/src/cleanup/cleanup_milter.ref13d

@@ -3,7 +3,8 @@
        81 message_arrival_time: Mon Apr 27 20:41:30 2009
       100 create_time: Mon Apr 27 20:41:41 2009
       124 named_attribute: rewrite_context=local
-      147 pointer_record:            1009
+      147 pointer_record:             975
+      975 pointer_record:            1009
      1009 sender: n@porcupine.org
      1026 pointer_record:             992
       992 pointer_record:             164

postfix/src/cleanup/cleanup_milter.ref13f

@@ -4,7 +4,8 @@
       100 create_time: Sun Jan 21 13:33:08 2007
       124 named_attribute: rewrite_context=local
       147 sender_fullname: Wietse Venema
-      162 pointer_record:             657
+      162 pointer_record:             573
+      573 pointer_record:             657
       657 sender: n@porcupine.org
       674 pointer_record:             590
       590 named_attribute: notify_flags=1

postfix/src/cleanup/cleanup_milter.ref13g

@@ -4,7 +4,9 @@
       100 create_time: Sun Jan 21 13:33:08 2007
       124 named_attribute: rewrite_context=local
       147 sender_fullname: Wietse Venema
-      162 pointer_record:             691
+      162 pointer_record:             573
+      573 pointer_record:             657
+      657 pointer_record:             691
       691 sender: o@porcupine.org
       708 pointer_record:             674
       674 pointer_record:             590

postfix/src/cleanup/cleanup_milter.ref13h

@@ -0,0 +1,29 @@
+*** ENVELOPE RECORDS test-queue-file13h.tmp ***
+        0 message_size:             332             182               1               0             332
+       81 message_arrival_time: Sun Jan 21 13:32:59 2007
+      100 create_time: Sun Jan 21 13:33:08 2007
+      124 named_attribute: rewrite_context=local
+      147 sender_fullname: Wietse Venema
+      162 pointer_record:             573
+      573 named_attribute: envelope_id=env-for-m
+      596 named_attribute: ret_flags=2
+      609 sender: m@porcupine.org
+      626 pointer_record:             180
+      180 *** MESSAGE CONTENTS test-queue-file13h.tmp ***
+      182 regular_text: Received: by hades.porcupine.org (Postfix, from userid 1001)
+      244 regular_text: 	id DE040290405; Sun, 21 Jan 2007 13:33:08 -0500 (EST)
+      300 regular_text: From: me@porcupine.org
+      324 regular_text: To: you@porcupine.org
+      347 regular_text: Message-Id: <20060725192735.5EC2D29013F@hades.porcupine.org>
+      409 regular_text: Date: Tue, 25 Jul 2006 15:27:19 -0400 (EDT)
+      454 regular_text: Subject: hey!
+      469 padding: 0
+      472 pointer_record:               0
+      489 regular_text: 
+      491 regular_text: text
+      497 pointer_record:               0
+      514 *** HEADER EXTRACTED test-queue-file13h.tmp ***
+      516 original_recipient: you@porcupine.org
+      535 recipient: you@porcupine.org
+      554 pointer_record:               0
+      571 *** MESSAGE FILE END test-queue-file13h.tmp ***

postfix/src/cleanup/cleanup_milter.ref13i

@@ -0,0 +1,33 @@
+*** ENVELOPE RECORDS test-queue-file13i.tmp ***
+        0 message_size:             332             182               1               0             332
+       81 message_arrival_time: Sun Jan 21 13:32:59 2007
+      100 create_time: Sun Jan 21 13:33:08 2007
+      124 named_attribute: rewrite_context=local
+      147 sender_fullname: Wietse Venema
+      162 pointer_record:             573
+      573 named_attribute: envelope_id=env-for-m
+      596 named_attribute: ret_flags=2
+      609 pointer_record:             643
+      643 named_attribute: envelope_id=env-for-n
+      666 named_attribute: ret_flags=1
+      679 sender: n@porcupine.org
+      696 pointer_record:             626
+      626 pointer_record:             180
+      180 *** MESSAGE CONTENTS test-queue-file13i.tmp ***
+      182 regular_text: Received: by hades.porcupine.org (Postfix, from userid 1001)
+      244 regular_text: 	id DE040290405; Sun, 21 Jan 2007 13:33:08 -0500 (EST)
+      300 regular_text: From: me@porcupine.org
+      324 regular_text: To: you@porcupine.org
+      347 regular_text: Message-Id: <20060725192735.5EC2D29013F@hades.porcupine.org>
+      409 regular_text: Date: Tue, 25 Jul 2006 15:27:19 -0400 (EDT)
+      454 regular_text: Subject: hey!
+      469 padding: 0
+      472 pointer_record:               0
+      489 regular_text: 
+      491 regular_text: text
+      497 pointer_record:               0
+      514 *** HEADER EXTRACTED test-queue-file13i.tmp ***
+      516 original_recipient: you@porcupine.org
+      535 recipient: you@porcupine.org
+      554 pointer_record:               0
+      571 *** MESSAGE FILE END test-queue-file13i.tmp ***

postfix/src/cleanup/cleanup_state.c

@@ -130,6 +130,7 @@ CLEANUP_STATE *cleanup_state_alloc(VSTREAM *src)
     state->milter_ext_from = 0;
     state->milter_ext_rcpt = 0;
     state->milter_err_text = 0;
+    state->milter_dsn_buf = 0;
     state->free_regions = state->body_regions = state->curr_body_region = 0;
     state->smtputf8 = 0;
     return (state);
@@ -189,6 +190,8 @@ void    cleanup_state_free(CLEANUP_STATE *state)
 	vstring_free(state->milter_ext_rcpt);
     if (state->milter_err_text)
 	vstring_free(state->milter_err_text);
+    if (state->milter_dsn_buf)
+	vstring_free(state->milter_dsn_buf);
     cleanup_region_done(state);
     myfree((void *) state);
 }

postfix/src/global/mail_version.h

@@ -20,7 +20,7 @@
   * Patches change both the patchlevel and the release date. Snapshots have no
   * patchlevel; they change the release date only.
   */
-#define MAIL_RELEASE_DATE	"20171221"
+#define MAIL_RELEASE_DATE	"20171223"
 #define MAIL_VERSION_NUMBER	"3.3"
 
 #ifdef SNAPSHOT

postfix/src/oqmgr/qmgr_message.c

@@ -600,18 +600,19 @@ static int qmgr_message_read(QMGR_MESSAGE *message)
 	    continue;
 	}
 	if (rec_type == REC_TYPE_DSN_ENVID) {
-	    if (message->dsn_envid == 0)
+	    /* Allow Milter override. */
+	    if (message->dsn_envid != 0)
+		myfree(message->dsn_envid);
 	    message->dsn_envid = mystrdup(start);
 	}
 	if (rec_type == REC_TYPE_DSN_RET) {
-	    if (message->dsn_ret == 0) {
+	    /* Allow Milter override. */
 	    if (!alldig(start) || (n = atoi(start)) == 0 || !DSN_RET_OK(n))
 		msg_warn("%s: ignoring malformed DSN RET flags in queue file record:%.100s",
 			 message->queue_id, start);
 	    else
 		message->dsn_ret = n;
 	}
-	}
 	if (rec_type == REC_TYPE_ATTR) {
 	    /* Allow extra segment to override envelope segment info. */
 	    if (strcmp(name, MAIL_ATTR_ENCODING) == 0) {

postfix/src/qmgr/qmgr_message.c

@@ -641,18 +641,19 @@ static int qmgr_message_read(QMGR_MESSAGE *message)
 	    continue;
 	}
 	if (rec_type == REC_TYPE_DSN_ENVID) {
-	    if (message->dsn_envid == 0)
+	    /* Allow Milter override. */
+	    if (message->dsn_envid != 0)
+		myfree(message->dsn_envid);
 	    message->dsn_envid = mystrdup(start);
 	}
 	if (rec_type == REC_TYPE_DSN_RET) {
-	    if (message->dsn_ret == 0) {
+	    /* Allow Milter override. */
 	    if (!alldig(start) || (n = atoi(start)) == 0 || !DSN_RET_OK(n))
 		msg_warn("%s: ignoring malformed DSN RET flags in queue file record:%.100s",
 			 message->queue_id, start);
 	    else
 		message->dsn_ret = n;
 	}
-	}
 	if (rec_type == REC_TYPE_ATTR) {
 	    /* Allow extra segment to override envelope segment info. */
 	    if (strcmp(name, MAIL_ATTR_ENCODING) == 0) {