postfix-3.9-20231213

postfix/HISTORY

@@ -27610,3 +27610,23 @@ Apologies for any names omitted.
 	all TLS security levels, including warnings for levels that
 	don't implement certificate matching. Viktor Dukhovni.
 	File: posttls-finger.c.
+
+20231213
+
+	Bugfix (defect introduced: Postfix 2.3): after prepending
+	a message header with a Postfix access table PREPEND action,
+	a Milter request to delete or update an existing header
+	could have no effect, or it could target the wrong instance
+	of an existing header. Root cause: the fix dated 20141018
+	for the Postfix Milter client was incomplete. The client
+	did correctly hide the first, Postfix-generated, Received:
+	header when sending message header information to a Milter
+	with the smfi_header() application callback function, but
+	it was still hiding the first header (instead of the first
+	Received: header) when handling requests from a Milter to
+	delete or update an existing header. Problem report by
+	Carlos Velasco. This change was verified to have no effect
+	on requests from a Milter to add or insert a header. Files:
+	cleanup/cleanup_milter.c, cleanup/Makefile.in,
+	cleanup/test-queue-file18, cleanup/cleanup_milter.in18[a-d],
+	cleanup/cleanup_milter.ref18[a-d][12].

postfix/WISHLIST

@@ -8,6 +8,9 @@ Wish list:
 
 	postfix-install should mention makedefs.out.
 
+	Remove .printfck directories, and remove printfck targets
+	from Makefiles.
+
 	In documentation and configuration file examples, replace
 	IPv4 address prefixes from Cloud9 with 192.168.* from RFC
 	1918, and replace IPv6 address prefixes with unique local

postfix/src/cleanup/Makefile.in

@@ -88,7 +88,9 @@ milter_tests: cleanup_milter_test bug_tests \
 	cleanup_milter_test15g cleanup_milter_test15h cleanup_milter_test15i \
 	cleanup_milter_test16a cleanup_milter_test16b cleanup_milter_test17a \
 	cleanup_milter_test17b cleanup_milter_test17c cleanup_milter_test17d \
-	cleanup_milter_test17e cleanup_milter_test17f cleanup_milter_test17g
+	cleanup_milter_test17e cleanup_milter_test17f cleanup_milter_test17g \
+	cleanup_milter_test18a cleanup_milter_test18b cleanup_milter_test18c \
+	cleanup_milter_test18d
 
 root_tests:
 
@@ -671,6 +673,46 @@ cleanup_milter_test17g: cleanup_milter test-queue-file17 cleanup_milter.in17g \
 	diff cleanup_milter.ref17g2 cleanup_milter.tmp2
 	rm -f test-queue-file17g.tmp cleanup_milter.tmp1 cleanup_milter.tmp2
 
+cleanup_milter_test18a: cleanup_milter test-queue-file18 cleanup_milter.in18a \
+	cleanup_milter.ref18a1 ../postcat/postcat cleanup_milter.ref18a2
+	cp test-queue-file18 test-queue-file18a.tmp
+	chmod u+w test-queue-file18a.tmp
+	$(SHLIB_ENV) $(VALGRIND) ./cleanup_milter <cleanup_milter.in18a 2>cleanup_milter.tmp1
+	diff cleanup_milter.ref18a1 cleanup_milter.tmp1
+	$(SHLIB_ENV) $(VALGRIND) ../postcat/postcat -ov test-queue-file18a.tmp 2>/dev/null >cleanup_milter.tmp2
+	diff cleanup_milter.ref18a2 cleanup_milter.tmp2
+	rm -f test-queue-file18a.tmp cleanup_milter.tmp1 cleanup_milter.tmp2
+
+cleanup_milter_test18b: cleanup_milter test-queue-file18 cleanup_milter.in18b \
+	cleanup_milter.ref18b1 ../postcat/postcat cleanup_milter.ref18b2
+	cp test-queue-file18 test-queue-file18b.tmp
+	chmod u+w test-queue-file18b.tmp
+	$(SHLIB_ENV) $(VALGRIND) ./cleanup_milter <cleanup_milter.in18b 2>cleanup_milter.tmp1
+	diff cleanup_milter.ref18b1 cleanup_milter.tmp1
+	$(SHLIB_ENV) $(VALGRIND) ../postcat/postcat -ov test-queue-file18b.tmp 2>/dev/null >cleanup_milter.tmp2
+	diff cleanup_milter.ref18b2 cleanup_milter.tmp2
+	rm -f test-queue-file18b.tmp cleanup_milter.tmp1 cleanup_milter.tmp2
+
+cleanup_milter_test18c: cleanup_milter test-queue-file18 cleanup_milter.in18c \
+	cleanup_milter.ref18c1 ../postcat/postcat cleanup_milter.ref18c2
+	cp test-queue-file18 test-queue-file18c.tmp
+	chmod u+w test-queue-file18c.tmp
+	$(SHLIB_ENV) $(VALGRIND) ./cleanup_milter <cleanup_milter.in18c 2>cleanup_milter.tmp1
+	diff cleanup_milter.ref18c1 cleanup_milter.tmp1
+	$(SHLIB_ENV) $(VALGRIND) ../postcat/postcat -ov test-queue-file18c.tmp 2>/dev/null >cleanup_milter.tmp2
+	diff cleanup_milter.ref18c2 cleanup_milter.tmp2
+	rm -f test-queue-file18c.tmp cleanup_milter.tmp1 cleanup_milter.tmp2
+
+cleanup_milter_test18d: cleanup_milter test-queue-file18 cleanup_milter.in18d \
+	cleanup_milter.ref18d1 ../postcat/postcat cleanup_milter.ref18d2
+	cp test-queue-file18 test-queue-file18d.tmp
+	chmod u+w test-queue-file18d.tmp
+	$(SHLIB_ENV) $(VALGRIND) ./cleanup_milter <cleanup_milter.in18d 2>cleanup_milter.tmp1
+	diff cleanup_milter.ref18d1 cleanup_milter.tmp1
+	$(SHLIB_ENV) $(VALGRIND) ../postcat/postcat -ov test-queue-file18d.tmp 2>/dev/null >cleanup_milter.tmp2
+	diff cleanup_milter.ref18d2 cleanup_milter.tmp2
+	rm -f test-queue-file18d.tmp cleanup_milter.tmp1 cleanup_milter.tmp2
+
 depend: $(MAKES)
 	(sed '1,/^# do not edit/!d' Makefile.in; \
 	set -e; for i in [a-z][a-z0-9]*.c; do \

postfix/src/cleanup/cleanup_milter.c

@@ -119,6 +119,7 @@
 #include <dsn_util.h>
 #include <xtext.h>
 #include <info_log_addr_form.h>
+#include <header_opts.h>
 
 /* Application-specific. */
 
@@ -754,14 +755,26 @@ static const char *cleanup_add_header(void *context, const char *name,
      */
 }
 
+/* hidden_header - respect milter header hiding protocol */
+
+static int hidden_header(VSTRING *buf, ARGV *auto_hdrs, int *hide_done)
+{
+    char  **cpp;
+    int     mask;
+
+    for (cpp = auto_hdrs->argv, mask = 1; *cpp; cpp++, mask <<= 1)
+	if ((*hide_done & mask) == 0 && strncmp(*cpp, STR(buf), LEN(buf)) == 0)
+	    return (*hide_done |= mask);
+    return (0);
+}
+
 /* cleanup_find_header_start - find specific header instance */
 
 static off_t cleanup_find_header_start(CLEANUP_STATE *state, ssize_t index,
 				               const char *header_label,
 				               VSTRING *buf,
 				               int *prec_type,
-				               int allow_ptr_backup,
+				               int allow_ptr_backup)
-				               int skip_headers)
 {
     const char *myname = "cleanup_find_header_start";
     off_t   curr_offset;		/* offset after found record */
@@ -770,7 +783,7 @@ static off_t cleanup_find_header_start(CLEANUP_STATE *state, ssize_t index,
     int     rec_type = REC_TYPE_ERROR;
     int     last_type;
     ssize_t len;
-    int     hdr_count = 0;
+    int     hide_done = 0;
 
     if (msg_verbose)
 	msg_info("%s: index %ld name \"%s\"",
@@ -912,11 +925,10 @@ static off_t cleanup_find_header_start(CLEANUP_STATE *state, ssize_t index,
 	    break;
 	}
 	/* This the start of a message header. */
-	else if (hdr_count++ < skip_headers)
-	     /* Reset the saved PTR record and update last_type. */ ;
 	else if ((header_label == 0
 		  || (strncasecmp(header_label, STR(buf), len) == 0
-		      && (strlen(header_label) == len)))
+		      && strlen(header_label) == len
+		      && !hidden_header(buf, state->auto_hdrs, &hide_done)))
 		 && --index == 0) {
 	    /* If we have a saved PTR record, it points to start of header. */
 	    break;
@@ -1182,15 +1194,12 @@ static const char *cleanup_ins_header(void *context, ssize_t index,
      */
 #define NO_HEADER_NAME	((char *) 0)
 #define ALLOW_PTR_BACKUP	1
-#define SKIP_ONE_HEADER		1
-#define DONT_SKIP_HEADERS	0
 
     if (index < 1)
 	index = 1;
     old_rec_offset = cleanup_find_header_start(state, index, NO_HEADER_NAME,
 					       old_rec_buf, &old_rec_type,
-					       ALLOW_PTR_BACKUP,
+					       ALLOW_PTR_BACKUP);
-					       DONT_SKIP_HEADERS);
     if (old_rec_offset == CLEANUP_FIND_HEADER_IOERROR)
 	/* Warning and errno->error mapping are done elsewhere. */
 	CLEANUP_INS_HEADER_RETURN(cleanup_milter_error(state, 0));
@@ -1270,8 +1279,7 @@ static const char *cleanup_upd_header(void *context, ssize_t index,
     rec_buf = vstring_alloc(100);
     old_rec_offset = cleanup_find_header_start(state, index, new_hdr_name,
 					       rec_buf, &last_type,
-					       NO_PTR_BACKUP,
+					       NO_PTR_BACKUP);
-					       SKIP_ONE_HEADER);
     if (old_rec_offset == CLEANUP_FIND_HEADER_IOERROR)
 	/* Warning and errno->error mapping are done elsewhere. */
 	CLEANUP_UPD_HEADER_RETURN(cleanup_milter_error(state, 0));
@@ -1333,8 +1341,7 @@ static const char *cleanup_del_header(void *context, ssize_t index,
 
     rec_buf = vstring_alloc(100);
     header_offset = cleanup_find_header_start(state, index, hdr_name, rec_buf,
-					      &last_type, NO_PTR_BACKUP,
+					      &last_type, NO_PTR_BACKUP);
-					      SKIP_ONE_HEADER);
     if (header_offset == CLEANUP_FIND_HEADER_IOERROR)
 	/* Warning and errno->error mapping are done elsewhere. */
 	CLEANUP_DEL_HEADER_RETURN(cleanup_milter_error(state, 0));
@@ -2437,6 +2444,7 @@ static void open_queue_file(CLEANUP_STATE *state, const char *path)
     long    data_offset;
     long    rcpt_count;
     long    qmgr_opts;
+    const HEADER_OPTS *opts;
 
     if (state->dst != 0) {
 	msg_warn("closing %s", cleanup_path);
@@ -2448,6 +2456,7 @@ static void open_queue_file(CLEANUP_STATE *state, const char *path)
     if ((state->dst = vstream_fopen(path, O_RDWR, 0)) == 0) {
 	msg_warn("open %s: %m", path);
     } else {
+	var_drop_hdrs = "";
 	cleanup_path = mystrdup(path);
 	for (;;) {
 	    if ((curr_offset = vstream_ftell(state->dst)) < 0)
@@ -2504,9 +2513,16 @@ static void open_queue_file(CLEANUP_STATE *state, const char *path)
 			    msg_fatal("file %s: vstream_ftell: %m", cleanup_path);
 		    }
 		}
+	    } else if (rec_type == REC_TYPE_NORM && state->hop_count == 0
+		       && (opts = header_opts_find(STR(buf))) != 0
+		       && opts->type == HDR_RECEIVED) {
+		state->hop_count += 1;
+		/* XXX Only the first line of the first Received: header. */
+		argv_add(state->auto_hdrs, STR(buf), ARGV_END);
 	    }
 	    if (state->append_rcpt_pt_offset > 0
 		&& state->append_hdr_pt_offset > 0
+		&& state->hop_count > 0
 		&& (rec_type == REC_TYPE_END
 		    || state->append_meta_pt_offset > 0))
 		break;

postfix/src/cleanup/cleanup_milter.in18a

@@ -0,0 +1,8 @@
+#verbose on
+open test-queue-file18a.tmp
+#
+# Update a prepended header. 
+#
+upd_header 1 Header-Label new-header-value
+
+close

postfix/src/cleanup/cleanup_milter.in18b

@@ -0,0 +1,8 @@
+#verbose on
+open test-queue-file18b.tmp
+#
+# Delete a prepended header. 
+#
+del_header 1 Header-Label
+
+close

postfix/src/cleanup/cleanup_milter.in18c

@@ -0,0 +1,9 @@
+#verbose on
+open test-queue-file18c.tmp
+#
+# Update the first Received: header.  This adds a new header, because
+# there is no header that was exposed to the Milter.
+#
+upd_header 1 Received whatever
+
+close

postfix/src/cleanup/cleanup_milter.in18d

@@ -0,0 +1,8 @@
+#verbose on
+open test-queue-file18d.tmp
+#
+# Delete our Received: header.  This should do nothing.
+#
+del_header 1 Received
+
+close

postfix/src/cleanup/cleanup_milter.ref18a1

@@ -0,0 +1 @@
+./cleanup_milter: flags = enable_header_body_filter enable_milters

postfix/src/cleanup/cleanup_milter.ref18a2

@@ -0,0 +1,29 @@
+*** ENVELOPE RECORDS test-queue-file18a.tmp ***
+        0 message_size:             342             290               1               0             342               0
+       97 message_arrival_time: Tue Dec 12 14:29:04 2023
+      116 create_time: Tue Dec 12 14:29:04 2023
+      140 named_attribute: rewrite_context=local
+      163 sender_fullname: Wietse Venema
+      178 sender: user@example.com
+      196 named_attribute: dsn_orig_rcpt=rfc822;user@example.com
+      235 original_recipient: user@example.com
+      253 recipient: user@example.com
+      271 pointer_record:               0
+      288 *** MESSAGE CONTENTS test-queue-file18a.tmp ***
+      290 pointer_record:             653
+      653 regular_text: Header-Label: new-header-value
+      685 pointer_record:             318
+      318 regular_text: Received: by wzv.porcupine.org (Postfix, from userid 1000)
+      378 regular_text: 	id 4SqTFD6TVpz4w4n; Tue, 12 Dec 2023 14:29:04 -0500 (EST)
+      438 regular_text: Subject: test
+      453 padding: 0
+      456 regular_text: Message-Id: <4SqTFD6TVpz4w4n@wzv.porcupine.org>
+      505 regular_text: Date: Tue, 12 Dec 2023 14:29:04 -0500 (EST)
+      550 regular_text: From: Wietse Venema <user@example.com>
+      590 pointer_record:               0
+      607 regular_text: 
+      609 regular_text: test
+      615 pointer_record:               0
+      632 *** HEADER EXTRACTED test-queue-file18a.tmp ***
+      634 pointer_record:               0
+      651 *** MESSAGE FILE END test-queue-file18a.tmp ***

postfix/src/cleanup/cleanup_milter.ref18b1

@@ -0,0 +1 @@
+./cleanup_milter: flags = enable_header_body_filter enable_milters

postfix/src/cleanup/cleanup_milter.ref18b2

@@ -0,0 +1,27 @@
+*** ENVELOPE RECORDS test-queue-file18b.tmp ***
+        0 message_size:             342             290               1               0             342               0
+       97 message_arrival_time: Tue Dec 12 14:29:04 2023
+      116 create_time: Tue Dec 12 14:29:04 2023
+      140 named_attribute: rewrite_context=local
+      163 sender_fullname: Wietse Venema
+      178 sender: user@example.com
+      196 named_attribute: dsn_orig_rcpt=rfc822;user@example.com
+      235 original_recipient: user@example.com
+      253 recipient: user@example.com
+      271 pointer_record:               0
+      288 *** MESSAGE CONTENTS test-queue-file18b.tmp ***
+      290 pointer_record:             318
+      318 regular_text: Received: by wzv.porcupine.org (Postfix, from userid 1000)
+      378 regular_text: 	id 4SqTFD6TVpz4w4n; Tue, 12 Dec 2023 14:29:04 -0500 (EST)
+      438 regular_text: Subject: test
+      453 padding: 0
+      456 regular_text: Message-Id: <4SqTFD6TVpz4w4n@wzv.porcupine.org>
+      505 regular_text: Date: Tue, 12 Dec 2023 14:29:04 -0500 (EST)
+      550 regular_text: From: Wietse Venema <user@example.com>
+      590 pointer_record:               0
+      607 regular_text: 
+      609 regular_text: test
+      615 pointer_record:               0
+      632 *** HEADER EXTRACTED test-queue-file18b.tmp ***
+      634 pointer_record:               0
+      651 *** MESSAGE FILE END test-queue-file18b.tmp ***

postfix/src/cleanup/cleanup_milter.ref18c1

@@ -0,0 +1 @@
+./cleanup_milter: flags = enable_header_body_filter enable_milters

postfix/src/cleanup/cleanup_milter.ref18c2

@@ -0,0 +1,29 @@
+*** ENVELOPE RECORDS test-queue-file18c.tmp ***
+        0 message_size:             342             290               1               0             342               0
+       97 message_arrival_time: Tue Dec 12 14:29:04 2023
+      116 create_time: Tue Dec 12 14:29:04 2023
+      140 named_attribute: rewrite_context=local
+      163 sender_fullname: Wietse Venema
+      178 sender: user@example.com
+      196 named_attribute: dsn_orig_rcpt=rfc822;user@example.com
+      235 original_recipient: user@example.com
+      253 recipient: user@example.com
+      271 pointer_record:               0
+      288 *** MESSAGE CONTENTS test-queue-file18c.tmp ***
+      290 regular_text: Header-Label: header-value
+      318 regular_text: Received: by wzv.porcupine.org (Postfix, from userid 1000)
+      378 regular_text: 	id 4SqTFD6TVpz4w4n; Tue, 12 Dec 2023 14:29:04 -0500 (EST)
+      438 regular_text: Subject: test
+      453 padding: 0
+      456 regular_text: Message-Id: <4SqTFD6TVpz4w4n@wzv.porcupine.org>
+      505 regular_text: Date: Tue, 12 Dec 2023 14:29:04 -0500 (EST)
+      550 regular_text: From: Wietse Venema <user@example.com>
+      590 pointer_record:             653
+      653 regular_text: Received: whatever
+      673 pointer_record:             607
+      607 regular_text: 
+      609 regular_text: test
+      615 pointer_record:               0
+      632 *** HEADER EXTRACTED test-queue-file18c.tmp ***
+      634 pointer_record:               0
+      651 *** MESSAGE FILE END test-queue-file18c.tmp ***

postfix/src/cleanup/cleanup_milter.ref18d1

@@ -0,0 +1 @@
+./cleanup_milter: flags = enable_header_body_filter enable_milters

postfix/src/cleanup/cleanup_milter.ref18d2

@@ -0,0 +1,27 @@
+*** ENVELOPE RECORDS test-queue-file18d.tmp ***
+        0 message_size:             342             290               1               0             342               0
+       97 message_arrival_time: Tue Dec 12 14:29:04 2023
+      116 create_time: Tue Dec 12 14:29:04 2023
+      140 named_attribute: rewrite_context=local
+      163 sender_fullname: Wietse Venema
+      178 sender: user@example.com
+      196 named_attribute: dsn_orig_rcpt=rfc822;user@example.com
+      235 original_recipient: user@example.com
+      253 recipient: user@example.com
+      271 pointer_record:               0
+      288 *** MESSAGE CONTENTS test-queue-file18d.tmp ***
+      290 regular_text: Header-Label: header-value
+      318 regular_text: Received: by wzv.porcupine.org (Postfix, from userid 1000)
+      378 regular_text: 	id 4SqTFD6TVpz4w4n; Tue, 12 Dec 2023 14:29:04 -0500 (EST)
+      438 regular_text: Subject: test
+      453 padding: 0
+      456 regular_text: Message-Id: <4SqTFD6TVpz4w4n@wzv.porcupine.org>
+      505 regular_text: Date: Tue, 12 Dec 2023 14:29:04 -0500 (EST)
+      550 regular_text: From: Wietse Venema <user@example.com>
+      590 pointer_record:               0
+      607 regular_text: 
+      609 regular_text: test
+      615 pointer_record:               0
+      632 *** HEADER EXTRACTED test-queue-file18d.tmp ***
+      634 pointer_record:               0
+      651 *** MESSAGE FILE END test-queue-file18d.tmp ***

postfix/src/global/mail_version.h

@@ -20,7 +20,7 @@
   * Patches change both the patchlevel and the release date. Snapshots have no
   * patchlevel; they change the release date only.
   */
-#define MAIL_RELEASE_DATE	"20231212"
+#define MAIL_RELEASE_DATE	"20231213"
 #define MAIL_VERSION_NUMBER	"3.9"
 
 #ifdef SNAPSHOT