diff --git a/postfix/HISTORY b/postfix/HISTORY index 50729f18c..4bb16481d 100644 --- a/postfix/HISTORY +++ b/postfix/HISTORY @@ -28828,32 +28828,73 @@ Apologies for any names omitted. No intentional behavior change. - Code cleanup: in SMTPUTF8 support, set and read the sender +20250110 + + Completed: in SMTPUTF8 support, set and read the sender options flags more selectively, instead of assuming that all bits are used exclusively for SMTPUTF8 support. Files: bounce/bounce_notify_util.c, cleanup/cleanup_api.c, cleanup/cleanup_envelope.c, smtp/smtp_proto.c. - TODO: in smtp_proto.c, add valid_utf8_stringz() check when - sending a DSN ORCPT address. Better: reuse IS_UTF8_ADDRESS() +20240111 + + Completed: in smtp_proto.c, add valid_utf8_stringz() check + when sending a DSN ORCPT address. Better: reuse IS_UTF8_ADDRESS() (it's currently defined in bounce_notify_util.c). - TODO: take a cue from Arnt Gulbrandsen's qmail patch. In the SMTP - client, enforce SMTPUTF8 if the message headers or envelope - require SMTPUTF8, or if the sender requested it. Ditto for - bounces. The qmail patch does not look for UTF8 text in headers, - but Exim will reject UTF8 headers unless the client requests - SMTPUTF8. + Completed: in stable releases in the queue managers "and" + the queue file's smtputf8 value with SMTPUTF8_FLAG_ALL, for + forward compatibility. Files: *qmgr/qmgr_message.c. - TODO: If we don't auto-generate the "SMTPUTF8 requested" - flag we may need to update the bounce daemon. Maybe it can - use similar logic as the SMTP client. + Completed: convert post_mail.c to use sendopts instead of + smtputf8. - TODO: in the bounce daemon consider replacing - (bounce_info->smtputf8 & SMTPUTF8_FLAG_REQUESTED) with - (bounce_info->smtputf8 & SMTPUTF8_FLAG_ALL): message/global - should be selected if a message has 8bit header or envelope. +20250113 - TODO: in stable releases in the queue managers "and" the - queue file's smtputf8 value with SMTPUTF8_FLAG_ALL, for - forward compatibility. + Baseline is postfix-3.10-20250109. + + Postfix needs "postfix reload" after upgrade, because of a + change in the delivery agent protocol. if this step is + skipped, Postfix delivery agents will log a warning: + "unexpected attribute smtputf8 from xxx socket (expecting: + sendopts)" where xxx is the delivery agent service name. + +TODO: + + Behavior change (request SMTPUTF8 more often): In Arnt + Gulbrandsen's qmail patch, the SMTP client enforces SMTPUTF8 + if the message envelope requires SMTPUTF8, or (instead of + AND) if the remote SMTP client requested SMTPUTF8 in the + MAIL FROM command. + + (Background: the qmail patch does not look for UTF8 text + in headers; Exim rejects UTF8 headers unless the client + requests SMTPUTF8; Postfix detects UTF8 in headers.) + + Changing Postfix SMTP client from "request SMTPUTF8 if UTF8 + was detected AND origin requested SMTPUTF8" to "request + SMTPUTF8 if ... OR ..." is like Postfix auto-requesting + SMTPUTF8 unconditionally. Thus the Postfix SMTP client would + request SMTPUTF8 more often than it does now. + + With Postfix, this means don't store the "origin requested + SMTPUTF8" flag while receiving mail, and don't set that + flag on-the-fly. During delivery, just look at the derived + flags. + + Behavior change (request SMTPUTF8 less often): In the bounce + daemon, look at derived flags instead of "origin requested + SMTPUTF8". Generate message/global-delivery-status only if + the return address requires SMTPUTF8 or some returned message + header requires SMTPUTF8. + + Verify: the bounce daemon in non-VERP mode should not generate + message/global-delivery-status just because some recipient was + flagged as UTF8. (With VERP, the failed recipient becomes a + suffix of the bounce recipient return address.) + + Verify: re-queue or generate a message with a SIZE record + that contains SMTPUTF8_{REQUESTED,SENDER,HEADER} and verify + that cleanup_envelope_process() strips the derived flags. + + Verify: all instances of smtputf8_autodetect() calls. diff --git a/postfix/RELEASE_NOTES b/postfix/RELEASE_NOTES index 9316a4acf..d6c952a99 100644 --- a/postfix/RELEASE_NOTES +++ b/postfix/RELEASE_NOTES @@ -26,6 +26,16 @@ now also distributed with the more recent Eclipse Public License license of their choice. Those who are more comfortable with the IPL can continue with that license. +[Incompat 20250109] + +Postfix needs "postfix reload" after upgrade, because of a change in +the delivery agent protocol. If this step is skipped, Postfix delivery +agents will log a warning: + + unexpected attribute smtputf8 from xxx socket (expecting: sendopts) + +where xxx is the delivery agent service name. + [Incompat 20250106] The logging of the Milter 'quarantine' action has changed. Instead diff --git a/postfix/src/cleanup/cleanup_api.c b/postfix/src/cleanup/cleanup_api.c index a51269735..60fb49a59 100644 --- a/postfix/src/cleanup/cleanup_api.c +++ b/postfix/src/cleanup/cleanup_api.c @@ -210,7 +210,7 @@ void cleanup_control(CLEANUP_STATE *state, int flags) } if (state->flags & CLEANUP_FLAG_SMTPUTF8) state->sendopts |= SMTPUTF8_FLAG_REQUESTED; - /* TODO: REQUIRETLS */ + /* TODO(wietse) REQUIRETLS. */ } /* cleanup_flush - finish queue file */ diff --git a/postfix/src/global/deliver_request.c b/postfix/src/global/deliver_request.c index 196dcbfa1..b69353b45 100644 --- a/postfix/src/global/deliver_request.c +++ b/postfix/src/global/deliver_request.c @@ -292,7 +292,7 @@ static int deliver_request_get(VSTREAM *stream, DELIVER_REQUEST *request) request->queue_id = mystrdup(vstring_str(queue_id)); request->nexthop = mystrdup(vstring_str(nexthop)); request->encoding = mystrdup(vstring_str(encoding)); - /* Fix 20140708: dedicated sendopts attribute with its own flags. */ + /* Fix 20140708: dedicated attribute for SMTPUTF8 etc. flags. */ request->sendopts = sendopts; request->sender = mystrdup(vstring_str(address)); request->client_name = mystrdup(vstring_str(client_name)); diff --git a/postfix/src/global/mail_version.h b/postfix/src/global/mail_version.h index 08d6ed264..67dcf2469 100644 --- a/postfix/src/global/mail_version.h +++ b/postfix/src/global/mail_version.h @@ -20,7 +20,7 @@ * Patches change both the patchlevel and the release date. Snapshots have no * patchlevel; they change the release date only. */ -#define MAIL_RELEASE_DATE "20250109" +#define MAIL_RELEASE_DATE "20250113" #define MAIL_VERSION_NUMBER "3.10" #ifdef SNAPSHOT diff --git a/postfix/src/global/post_mail.c b/postfix/src/global/post_mail.c index e7a9a6746..1a41dfd6c 100644 --- a/postfix/src/global/post_mail.c +++ b/postfix/src/global/post_mail.c @@ -7,31 +7,31 @@ /* #include /* /* VSTREAM *post_mail_fopen(sender, recipient, source_class, trace_flags, -/* utf8_flags, queue_id) +/* sendopts, queue_id) /* const char *sender; /* const char *recipient; /* int source_class; /* int trace_flags; -/* int utf8_flags; +/* int sendopts; /* VSTRING *queue_id; /* /* VSTREAM *post_mail_fopen_nowait(sender, recipient, source_class, -/* trace_flags, utf8_flags, queue_id) +/* trace_flags, sendopts, queue_id) /* const char *sender; /* const char *recipient; /* int source_class; /* int trace_flags; -/* int utf8_flags; +/* int sendopts; /* VSTRING *queue_id; /* /* void post_mail_fopen_async(sender, recipient, source_class, -/* trace_flags, utf8_flags, +/* trace_flags, sendopts, /* queue_id, notify, context) /* const char *sender; /* const char *recipient; /* int source_class; /* int trace_flags; -/* int utf8_flags; +/* int sendopts; /* VSTRING *queue_id; /* void (*notify)(VSTREAM *stream, void *context); /* void *context; @@ -116,9 +116,9 @@ /* autodetection. /* .IP trace_flags /* Message tracing flags as specified in \fB\fR. -/* .IP utf8_flags -/* Flags defined in . Flags other than -/* SMTPUTF8_FLAG_REQUESTED are ignored. +/* .IP sendopts +/* Flags defined in . This ignores flags based on +/* message header content, or envelope email addresses. /* .IP queue_id /* Null pointer, or pointer to buffer that receives the queue /* ID of the new message. @@ -195,7 +195,7 @@ typedef struct { char *recipient; int source_class; int trace_flags; - int utf8_flags; + int sendopts; POST_MAIL_NOTIFY notify; void *context; VSTREAM *stream; @@ -217,7 +217,7 @@ typedef struct { static void post_mail_init(VSTREAM *stream, const char *sender, const char *recipient, int source_class, int trace_flags, - int utf8_flags, VSTRING *queue_id) + int sendopts, VSTRING *queue_id) { VSTRING *id = queue_id ? queue_id : vstring_alloc(100); struct timeval now; @@ -225,7 +225,8 @@ static void post_mail_init(VSTREAM *stream, const char *sender, int cleanup_flags = int_filt_flags(source_class) | CLEANUP_FLAG_MASK_INTERNAL | smtputf8_autodetect(source_class) - | ((utf8_flags & SMTPUTF8_FLAG_REQUESTED) ? CLEANUP_FLAG_SMTPUTF8 : 0); + | ((sendopts & SMTPUTF8_FLAG_REQUESTED) ? CLEANUP_FLAG_SMTPUTF8 : 0); + /* TODO(wietse) REQUIRETLS. */ GETTIMEOFDAY(&now); date = mail_date(now.tv_sec); @@ -289,13 +290,13 @@ static void post_mail_init(VSTREAM *stream, const char *sender, VSTREAM *post_mail_fopen(const char *sender, const char *recipient, int source_class, int trace_flags, - int utf8_flags, VSTRING *queue_id) + int sendopts, VSTRING *queue_id) { VSTREAM *stream; stream = mail_connect_wait(MAIL_CLASS_PUBLIC, var_cleanup_service); post_mail_init(stream, sender, recipient, source_class, trace_flags, - utf8_flags, queue_id); + sendopts, queue_id); return (stream); } @@ -303,14 +304,14 @@ VSTREAM *post_mail_fopen(const char *sender, const char *recipient, VSTREAM *post_mail_fopen_nowait(const char *sender, const char *recipient, int source_class, int trace_flags, - int utf8_flags, VSTRING *queue_id) + int sendopts, VSTRING *queue_id) { VSTREAM *stream; if ((stream = mail_connect(MAIL_CLASS_PUBLIC, var_cleanup_service, BLOCKING)) != 0) post_mail_init(stream, sender, recipient, source_class, trace_flags, - utf8_flags, queue_id); + sendopts, queue_id); else msg_warn("connect to %s/%s: %m", MAIL_CLASS_PUBLIC, var_cleanup_service); @@ -339,7 +340,7 @@ static void post_mail_open_event(int event, void *context) non_blocking(vstream_fileno(state->stream), BLOCKING); post_mail_init(state->stream, state->sender, state->recipient, state->source_class, - state->trace_flags, state->utf8_flags, + state->trace_flags, state->sendopts, state->queue_id); myfree(state->sender); myfree(state->recipient); @@ -391,7 +392,7 @@ static void post_mail_open_event(int event, void *context) void post_mail_fopen_async(const char *sender, const char *recipient, int source_class, int trace_flags, - int utf8_flags, VSTRING *queue_id, + int sendopts, VSTRING *queue_id, void (*notify) (VSTREAM *, void *), void *context) { @@ -404,7 +405,7 @@ void post_mail_fopen_async(const char *sender, const char *recipient, state->recipient = mystrdup(recipient); state->source_class = source_class; state->trace_flags = trace_flags; - state->utf8_flags = utf8_flags; + state->sendopts = sendopts; state->notify = notify; state->context = context; state->stream = stream; diff --git a/postfix/src/global/smtputf8.h b/postfix/src/global/smtputf8.h index 4aa721ba9..4f4a6c434 100644 --- a/postfix/src/global/smtputf8.h +++ b/postfix/src/global/smtputf8.h @@ -93,10 +93,6 @@ extern int smtputf8_autodetect(int); * delivering messages to non-SMTPUTF8 servers. Delivery agents may then * pass the flag to the bounce daemon. * - * XXXSENDOPTS Verify that the flag is sent for only one recipient. What - * problem would that solve? The recipient becomes message content in the - * delivery status notification; it is just data. - * * If a delivery request has none of the flags SMTPUTF8_FLAG_RECIPIENT, * SMTPUTF8_FLAG_SENDER, or SMTPUTF8_FLAG_HEADER, then the message can * safely be delivered to a non-SMTPUTF8 server (DSN original recipients diff --git a/postfix/src/local/forward.c b/postfix/src/local/forward.c index 3d5c316cb..7e553ffa3 100644 --- a/postfix/src/local/forward.c +++ b/postfix/src/local/forward.c @@ -165,6 +165,7 @@ static FORWARD_INFO *forward_open(DELIVER_REQUEST *request, const char *sender) | smtputf8_autodetect(MAIL_SRC_MASK_FORWARD) \ | ((request->sendopts & SMTPUTF8_FLAG_REQUESTED) ? \ CLEANUP_FLAG_SMTPUTF8 : 0)) + /* TODO(wietse) REQUIRETLS. */ attr_print(cleanup, ATTR_FLAG_NONE, SEND_ATTR_INT(MAIL_ATTR_FLAGS, FORWARD_CLEANUP_FLAGS), diff --git a/postfix/src/pickup/pickup.c b/postfix/src/pickup/pickup.c index 384a83eb0..390329d1e 100644 --- a/postfix/src/pickup/pickup.c +++ b/postfix/src/pickup/pickup.c @@ -486,6 +486,7 @@ static int pickup_file(PICKUP_INFO *info) cleanup_flags &= ~CLEANUP_FLAG_MILTER; else cleanup_flags |= smtputf8_autodetect(MAIL_SRC_MASK_SENDMAIL); + /* TODO(wietse) REQUIRETLS? */ cleanup = mail_connect_wait(MAIL_CLASS_PUBLIC, var_cleanup_service); if (attr_scan(cleanup, ATTR_FLAG_STRICT, diff --git a/postfix/src/qmqpd/qmqpd.c b/postfix/src/qmqpd/qmqpd.c index ad2169d63..2214d9855 100644 --- a/postfix/src/qmqpd/qmqpd.c +++ b/postfix/src/qmqpd/qmqpd.c @@ -271,6 +271,7 @@ static void qmqpd_open_file(QMQPD_STATE *state) cleanup_flags = input_transp_cleanup(CLEANUP_FLAG_MASK_EXTERNAL, qmqpd_input_transp_mask); cleanup_flags |= smtputf8_autodetect(MAIL_SRC_MASK_QMQPD); + /* TODO(wietse) REQUIRETLS? */ state->dest = mail_stream_service(MAIL_CLASS_PUBLIC, var_cleanup_service); if (state->dest == 0 || attr_print(state->dest->stream, ATTR_FLAG_NONE, diff --git a/postfix/src/smtp/smtp_proto.c b/postfix/src/smtp/smtp_proto.c index 847d0cc35..df2f74391 100644 --- a/postfix/src/smtp/smtp_proto.c +++ b/postfix/src/smtp/smtp_proto.c @@ -1778,6 +1778,7 @@ static int smtp_loop(SMTP_STATE *state, NOCLOBBER int send_state, if ((session->features & SMTP_FEATURE_SMTPUTF8) != 0 && (request->sendopts & SMTPUTF8_FLAG_REQUESTED) != 0) vstring_strcat(next_command, " SMTPUTF8"); + /* TODO(wietse) REQUIRETLS. */ /* * We authenticate the local MTA only, but not the sender. @@ -1842,7 +1843,8 @@ static int smtp_loop(SMTP_STATE *state, NOCLOBBER int send_state, vstring_sprintf(session->scratch2, "%s;%s", /* Fix 20140707: sender must request SMTPUTF8. */ ((request->sendopts & SMTPUTF8_FLAG_ALL) - && !allascii(vstring_str(session->scratch))) ? + && !allascii(vstring_str(session->scratch)) + && valid_utf8_stringz(vstring_str(session->scratch))) ? "utf-8" : "rfc822", vstring_str(session->scratch)); orcpt_type_addr = vstring_str(session->scratch2); diff --git a/postfix/src/smtpd/smtpd.c b/postfix/src/smtpd/smtpd.c index 968bac458..962b361c7 100644 --- a/postfix/src/smtpd/smtpd.c +++ b/postfix/src/smtpd/smtpd.c @@ -2213,6 +2213,7 @@ static int mail_open_stream(SMTPD_STATE *state) cleanup_flags |= CLEANUP_FLAG_SMTPUTF8; else cleanup_flags |= smtputf8_autodetect(MAIL_SRC_MASK_SMTPD); + /* TODO(wietse) REQUIRETLS. */ state->dest = mail_stream_service(MAIL_CLASS_PUBLIC, var_cleanup_service); if (state->dest == 0