postfix-3.10-20250113-nonprod

postfix/HISTORY

@@ -28828,32 +28828,73 @@ Apologies for any names omitted.
 
 	No intentional behavior change.
 
-	Code cleanup: in SMTPUTF8 support, set and read the sender
+20250110
+
+	Completed: in SMTPUTF8 support, set and read the sender
 	options flags more selectively, instead of assuming that
 	all bits are used exclusively for SMTPUTF8 support. Files:
 	bounce/bounce_notify_util.c, cleanup/cleanup_api.c,
 	cleanup/cleanup_envelope.c, smtp/smtp_proto.c.
 
-	TODO: in smtp_proto.c, add valid_utf8_stringz() check when
+20240111
-	sending a DSN ORCPT address. Better: reuse IS_UTF8_ADDRESS()
+
+	Completed: in smtp_proto.c, add valid_utf8_stringz() check
+	when sending a DSN ORCPT address. Better: reuse IS_UTF8_ADDRESS()
 	(it's currently defined in bounce_notify_util.c).
 
-	TODO: take a cue from Arnt Gulbrandsen's qmail patch. In the SMTP
+	Completed: in stable releases in the queue managers "and"
-	client, enforce SMTPUTF8 if the message headers or envelope
+	the queue file's smtputf8 value with SMTPUTF8_FLAG_ALL, for
-	require SMTPUTF8, or if the sender requested it. Ditto for
+	forward compatibility. Files: *qmgr/qmgr_message.c.
-	bounces. The qmail patch does not look for UTF8 text in headers,
-	but Exim will reject UTF8 headers unless the client requests
-	SMTPUTF8.
 
-	TODO: If we don't auto-generate the "SMTPUTF8 requested"
+	Completed: convert post_mail.c to use sendopts instead of
-	flag we may need to update the bounce daemon. Maybe it can
+	smtputf8.
-	use similar logic as the SMTP client.
 
-	TODO: in the bounce daemon consider replacing
+20250113
-	(bounce_info->smtputf8 & SMTPUTF8_FLAG_REQUESTED) with
-	(bounce_info->smtputf8 & SMTPUTF8_FLAG_ALL): message/global
-	should be selected if a message has 8bit header or envelope.
 
-	TODO: in stable releases in the queue managers "and" the
+	Baseline is postfix-3.10-20250109.
-	queue file's smtputf8 value with SMTPUTF8_FLAG_ALL, for
+
-	forward compatibility.
+	Postfix needs "postfix reload" after upgrade, because of a
+	change in the delivery agent protocol. if this step is
+	skipped, Postfix delivery agents will log a warning:
+	"unexpected attribute smtputf8 from xxx socket (expecting:
+	sendopts)" where xxx is the delivery agent service name.
+
+TODO:
+
+	Behavior change (request SMTPUTF8 more often): In Arnt
+	Gulbrandsen's qmail patch, the SMTP client enforces SMTPUTF8
+	if the message envelope requires SMTPUTF8, or (instead of
+	AND) if the remote SMTP client requested SMTPUTF8 in the
+	MAIL FROM command.
+
+	(Background: the qmail patch does not look for UTF8 text
+	in headers; Exim rejects UTF8 headers unless the client
+	requests SMTPUTF8; Postfix detects UTF8 in headers.)
+
+	Changing Postfix SMTP client from "request SMTPUTF8 if UTF8
+	was detected AND origin requested SMTPUTF8" to "request
+	SMTPUTF8 if ... OR ..." is like Postfix auto-requesting
+	SMTPUTF8 unconditionally. Thus the Postfix SMTP client would
+	request SMTPUTF8 more often than it does now.
+
+	With Postfix, this means don't store the "origin requested
+	SMTPUTF8" flag while receiving mail, and don't set that
+	flag on-the-fly. During delivery, just look at the derived
+	flags.
+
+	Behavior change (request SMTPUTF8 less often): In the bounce
+	daemon, look at derived flags instead of "origin requested
+	SMTPUTF8". Generate message/global-delivery-status only if
+	the return address requires SMTPUTF8 or some returned message
+	header requires SMTPUTF8.
+
+	Verify: the bounce daemon in non-VERP mode should not generate
+	message/global-delivery-status just because some recipient was
+	flagged as UTF8. (With VERP, the failed recipient becomes a
+	suffix of the bounce recipient return address.)
+
+	Verify: re-queue or generate a message with a SIZE record
+	that contains SMTPUTF8_{REQUESTED,SENDER,HEADER} and verify
+	that cleanup_envelope_process() strips the derived flags.
+
+	Verify: all instances of smtputf8_autodetect() calls.

postfix/RELEASE_NOTES

@@ -26,6 +26,16 @@ now also distributed with the more recent Eclipse Public License
 license of their choice. Those who are more comfortable with the
 IPL can continue with that license.
 
+[Incompat 20250109]
+
+Postfix needs "postfix reload" after upgrade, because of a change in
+the delivery agent protocol. If this step is skipped, Postfix delivery
+agents will log a warning:
+
+    unexpected attribute smtputf8 from xxx socket (expecting: sendopts)
+
+where xxx is the delivery agent service name.
+
 [Incompat 20250106]
 
 The logging of the Milter 'quarantine' action has changed. Instead

postfix/src/cleanup/cleanup_api.c

@@ -210,7 +210,7 @@ void    cleanup_control(CLEANUP_STATE *state, int flags)
     }
     if (state->flags & CLEANUP_FLAG_SMTPUTF8)
 	state->sendopts |= SMTPUTF8_FLAG_REQUESTED;
-    /* TODO: REQUIRETLS */
+    /* TODO(wietse) REQUIRETLS. */
 }
 
 /* cleanup_flush - finish queue file */

postfix/src/global/deliver_request.c

@@ -292,7 +292,7 @@ static int deliver_request_get(VSTREAM *stream, DELIVER_REQUEST *request)
     request->queue_id = mystrdup(vstring_str(queue_id));
     request->nexthop = mystrdup(vstring_str(nexthop));
     request->encoding = mystrdup(vstring_str(encoding));
-    /* Fix 20140708: dedicated sendopts attribute with its own flags. */
+    /* Fix 20140708: dedicated attribute for SMTPUTF8 etc. flags. */
     request->sendopts = sendopts;
     request->sender = mystrdup(vstring_str(address));
     request->client_name = mystrdup(vstring_str(client_name));

postfix/src/global/mail_version.h

@@ -20,7 +20,7 @@
   * Patches change both the patchlevel and the release date. Snapshots have no
   * patchlevel; they change the release date only.
   */
-#define MAIL_RELEASE_DATE	"20250109"
+#define MAIL_RELEASE_DATE	"20250113"
 #define MAIL_VERSION_NUMBER	"3.10"
 
 #ifdef SNAPSHOT

postfix/src/global/post_mail.c

@@ -7,31 +7,31 @@
 /*	#include <post_mail.h>
 /*
 /*	VSTREAM	*post_mail_fopen(sender, recipient, source_class, trace_flags,
-/*		utf8_flags, queue_id)
+/*		sendopts, queue_id)
 /*	const char *sender;
 /*	const char *recipient;
 /*	int	source_class;
 /*	int	trace_flags;
-/*	int	utf8_flags;
+/*	int	sendopts;
 /*	VSTRING *queue_id;
 /*
 /*	VSTREAM	*post_mail_fopen_nowait(sender, recipient, source_class,
-/*					trace_flags, utf8_flags, queue_id)
+/*					trace_flags, sendopts, queue_id)
 /*	const char *sender;
 /*	const char *recipient;
 /*	int	source_class;
 /*	int	trace_flags;
-/*	int	utf8_flags;
+/*	int	sendopts;
 /*	VSTRING *queue_id;
 /*
 /*	void	post_mail_fopen_async(sender, recipient, source_class,
-/*					trace_flags, utf8_flags,
+/*					trace_flags, sendopts,
 /*					queue_id, notify, context)
 /*	const char *sender;
 /*	const char *recipient;
 /*	int	source_class;
 /*	int	trace_flags;
-/*	int	utf8_flags;
+/*	int	sendopts;
 /*	VSTRING *queue_id;
 /*	void	(*notify)(VSTREAM *stream, void *context);
 /*	void	*context;
@@ -116,9 +116,9 @@
 /*	autodetection.
 /* .IP trace_flags
 /*	Message tracing flags as specified in \fB<deliver_request.h>\fR.
-/* .IP utf8_flags
+/* .IP sendopts
-/*	Flags defined in <smtputf8.h>. Flags other than
+/*	Flags defined in <sendopts.h>. This ignores flags based on
-/*	SMTPUTF8_FLAG_REQUESTED are ignored.
+/*	message header content, or envelope email addresses.
 /* .IP queue_id
 /*	Null pointer, or pointer to buffer that receives the queue
 /*	ID of the new message.
@@ -195,7 +195,7 @@ typedef struct {
     char   *recipient;
     int     source_class;
     int     trace_flags;
-    int     utf8_flags;
+    int     sendopts;
     POST_MAIL_NOTIFY notify;
     void   *context;
     VSTREAM *stream;
@@ -217,7 +217,7 @@ typedef struct {
 static void post_mail_init(VSTREAM *stream, const char *sender,
 			           const char *recipient,
 			           int source_class, int trace_flags,
-			           int utf8_flags, VSTRING *queue_id)
+			           int sendopts, VSTRING *queue_id)
 {
     VSTRING *id = queue_id ? queue_id : vstring_alloc(100);
     struct timeval now;
@@ -225,7 +225,8 @@ static void post_mail_init(VSTREAM *stream, const char *sender,
     int     cleanup_flags =
     int_filt_flags(source_class) | CLEANUP_FLAG_MASK_INTERNAL
     | smtputf8_autodetect(source_class)
-    | ((utf8_flags & SMTPUTF8_FLAG_REQUESTED) ? CLEANUP_FLAG_SMTPUTF8 : 0);
+    | ((sendopts & SMTPUTF8_FLAG_REQUESTED) ? CLEANUP_FLAG_SMTPUTF8 : 0);
+    /* TODO(wietse) REQUIRETLS. */
 
     GETTIMEOFDAY(&now);
     date = mail_date(now.tv_sec);
@@ -289,13 +290,13 @@ static void post_mail_init(VSTREAM *stream, const char *sender,
 
 VSTREAM *post_mail_fopen(const char *sender, const char *recipient,
 			         int source_class, int trace_flags,
-			         int utf8_flags, VSTRING *queue_id)
+			         int sendopts, VSTRING *queue_id)
 {
     VSTREAM *stream;
 
     stream = mail_connect_wait(MAIL_CLASS_PUBLIC, var_cleanup_service);
     post_mail_init(stream, sender, recipient, source_class, trace_flags,
-		   utf8_flags, queue_id);
+		   sendopts, queue_id);
     return (stream);
 }
 
@@ -303,14 +304,14 @@ VSTREAM *post_mail_fopen(const char *sender, const char *recipient,
 
 VSTREAM *post_mail_fopen_nowait(const char *sender, const char *recipient,
 				        int source_class, int trace_flags,
-				        int utf8_flags, VSTRING *queue_id)
+				        int sendopts, VSTRING *queue_id)
 {
     VSTREAM *stream;
 
     if ((stream = mail_connect(MAIL_CLASS_PUBLIC, var_cleanup_service,
 			       BLOCKING)) != 0)
 	post_mail_init(stream, sender, recipient, source_class, trace_flags,
-		       utf8_flags, queue_id);
+		       sendopts, queue_id);
     else
 	msg_warn("connect to %s/%s: %m",
 		 MAIL_CLASS_PUBLIC, var_cleanup_service);
@@ -339,7 +340,7 @@ static void post_mail_open_event(int event, void *context)
 	non_blocking(vstream_fileno(state->stream), BLOCKING);
 	post_mail_init(state->stream, state->sender,
 		       state->recipient, state->source_class,
-		       state->trace_flags, state->utf8_flags,
+		       state->trace_flags, state->sendopts,
 		       state->queue_id);
 	myfree(state->sender);
 	myfree(state->recipient);
@@ -391,7 +392,7 @@ static void post_mail_open_event(int event, void *context)
 
 void    post_mail_fopen_async(const char *sender, const char *recipient,
 			              int source_class, int trace_flags,
-			              int utf8_flags, VSTRING *queue_id,
+			              int sendopts, VSTRING *queue_id,
 			              void (*notify) (VSTREAM *, void *),
 			              void *context)
 {
@@ -404,7 +405,7 @@ void    post_mail_fopen_async(const char *sender, const char *recipient,
     state->recipient = mystrdup(recipient);
     state->source_class = source_class;
     state->trace_flags = trace_flags;
-    state->utf8_flags = utf8_flags;
+    state->sendopts = sendopts;
     state->notify = notify;
     state->context = context;
     state->stream = stream;

postfix/src/global/smtputf8.h

@@ -93,10 +93,6 @@ extern int smtputf8_autodetect(int);
   * delivering messages to non-SMTPUTF8 servers. Delivery agents may then
   * pass the flag to the bounce daemon.
   * 
-  * XXXSENDOPTS Verify that the flag is sent for only one recipient. What
-  * problem would that solve? The recipient becomes message content in the
-  * delivery status notification; it is just data.
-  * 
   * If a delivery request has none of the flags SMTPUTF8_FLAG_RECIPIENT,
   * SMTPUTF8_FLAG_SENDER, or SMTPUTF8_FLAG_HEADER, then the message can
   * safely be delivered to a non-SMTPUTF8 server (DSN original recipients

postfix/src/local/forward.c

@@ -165,6 +165,7 @@ static FORWARD_INFO *forward_open(DELIVER_REQUEST *request, const char *sender)
 	| smtputf8_autodetect(MAIL_SRC_MASK_FORWARD) \
 	| ((request->sendopts & SMTPUTF8_FLAG_REQUESTED) ? \
 	CLEANUP_FLAG_SMTPUTF8 : 0))
+	/* TODO(wietse) REQUIRETLS. */
 
     attr_print(cleanup, ATTR_FLAG_NONE,
 	       SEND_ATTR_INT(MAIL_ATTR_FLAGS, FORWARD_CLEANUP_FLAGS),

postfix/src/pickup/pickup.c

@@ -486,6 +486,7 @@ static int pickup_file(PICKUP_INFO *info)
 	cleanup_flags &= ~CLEANUP_FLAG_MILTER;
     else
 	cleanup_flags |= smtputf8_autodetect(MAIL_SRC_MASK_SENDMAIL);
+    /* TODO(wietse) REQUIRETLS? */
 
     cleanup = mail_connect_wait(MAIL_CLASS_PUBLIC, var_cleanup_service);
     if (attr_scan(cleanup, ATTR_FLAG_STRICT,

postfix/src/qmqpd/qmqpd.c

@@ -271,6 +271,7 @@ static void qmqpd_open_file(QMQPD_STATE *state)
     cleanup_flags = input_transp_cleanup(CLEANUP_FLAG_MASK_EXTERNAL,
 					 qmqpd_input_transp_mask);
     cleanup_flags |= smtputf8_autodetect(MAIL_SRC_MASK_QMQPD);
+    /* TODO(wietse) REQUIRETLS? */
     state->dest = mail_stream_service(MAIL_CLASS_PUBLIC, var_cleanup_service);
     if (state->dest == 0
 	|| attr_print(state->dest->stream, ATTR_FLAG_NONE,

postfix/src/smtp/smtp_proto.c

@@ -1778,6 +1778,7 @@ static int smtp_loop(SMTP_STATE *state, NOCLOBBER int send_state,
 	    if ((session->features & SMTP_FEATURE_SMTPUTF8) != 0
 		&& (request->sendopts & SMTPUTF8_FLAG_REQUESTED) != 0)
 		vstring_strcat(next_command, " SMTPUTF8");
+	    /* TODO(wietse) REQUIRETLS. */
 
 	    /*
 	     * We authenticate the local MTA only, but not the sender.
@@ -1842,7 +1843,8 @@ static int smtp_loop(SMTP_STATE *state, NOCLOBBER int send_state,
 		    vstring_sprintf(session->scratch2, "%s;%s",
 		    /* Fix 20140707: sender must request SMTPUTF8. */
 				    ((request->sendopts & SMTPUTF8_FLAG_ALL)
-			      && !allascii(vstring_str(session->scratch))) ?
+				 && !allascii(vstring_str(session->scratch))
+		     && valid_utf8_stringz(vstring_str(session->scratch))) ?
 				    "utf-8" : "rfc822",
 				    vstring_str(session->scratch));
 		    orcpt_type_addr = vstring_str(session->scratch2);

postfix/src/smtpd/smtpd.c

@@ -2213,6 +2213,7 @@ static int mail_open_stream(SMTPD_STATE *state)
 	    cleanup_flags |= CLEANUP_FLAG_SMTPUTF8;
 	else
 	    cleanup_flags |= smtputf8_autodetect(MAIL_SRC_MASK_SMTPD);
+	/* TODO(wietse) REQUIRETLS. */
 	state->dest = mail_stream_service(MAIL_CLASS_PUBLIC,
 					  var_cleanup_service);
 	if (state->dest == 0