postfix-3.9-20230419

postfix/HISTORY

@@ -27055,3 +27055,26 @@ Apologies for any names omitted.
 	Cleanup: in source-code comments, replaced redundant (and
 	sometimes incomplete) lookup table configuration info with
 	a reference to the corresponding *_table(5) manpage.
+
+20230417
+
+	Cleanup: in the MySQL client configuration file, the default
+	characterset is now configurable with the "charset" attribute.
+	Previously, the default was determined by the MySQL
+	implementation (utf8mb4 as of MySQL 8.0, latin1 with older
+	versions). This setting implicitly controls the collation
+	order. Files: proto/mysql_table, global/dict_mysql.c.
+
+20230418
+
+	Bugfix (introduced: Postfix 3.2): the MySQL client could
+	return "not found" instead of "error" (for example, resulting
+	in a 5XX SMTP status instead of 4XX) during the time that
+	all MySQL server connections were turned down after error.
+	Found during code maintenance. File: global/dict_mysql.c.
+
+20230419
+
+	Cleanup: in the PostgreSQL client, cosmetic changes to make
+	the code easier to maintain (in preparation for adding new
+	functionality). File: global/dict_pgsql.c.

postfix/RELEASE_NOTES

@@ -25,3 +25,11 @@ now also distributed with the more recent Eclipse Public License
 (EPL) 2.0. Recipients can choose to take the software under the
 license of their choice. Those who are more comfortable with the
 IPL can continue with that license.
+
+Incompatible changes with snapshot 20230419
+===========================================
+
+The MySQL client default characterset is now configurable with the
+"charset" configuration file attribute. The default is "utf8mb4",
+consistent with the MySQL 8.0 built-in default, but different from
+earlier MySQL versions where the built-in default was "latin1".

postfix/WISHLIST

@@ -18,6 +18,9 @@ Wish list:
 
 	Follow https://github.com/vdukhovni/postfix/commits/rpk
 
+	Figure out which mysql_*escape_string*() variant to use and
+	handle error results accordingly.
+
 	Multi-recipient support in sender/recipient_bcc_maps and
 	always_bcc.
 

postfix/html/mysql_table.5.html

@@ -72,6 +72,14 @@ MYSQL_TABLE(5)                                                  MYSQL_TABLE(5)
        <b>dbname</b> The database name on the servers. Example:
                   dbname = customer_database
 
+       <b>charset (default: utf8mb4)</b>
+              The default MySQL client character set; this  also  implies  the
+              collation order.
+
+              This  parameter  is  available with Postfix 3.9 and later.  With
+              earlier Postfix versions, the default was chosen  by  the  MySQL
+              implementation (<b>utf8mb4</b> as of MySQL 8.0, <b>latin1</b> historically).
+
        <b>query</b>  The  SQL query template used to search the database, where <b>%s</b> is
               a substitute for the address Postfix is trying to resolve,  e.g.
                   query = SELECT replacement FROM aliases WHERE mailbox = '%s'

postfix/man/man5/mysql_table.5

@@ -89,6 +89,14 @@ The database name on the servers. Example:
 .nf
     dbname = customer_database
 .fi
+.IP "\fBcharset (default: utf8mb4)\fR"
+The default MySQL client character set; this also implies
+the collation order.
+
+This parameter is available with Postfix 3.9 and later.
+With earlier Postfix versions, the default was chosen by
+the MySQL implementation (\fButf8mb4\fR as of MySQL 8.0,
+\fBlatin1\fR historically).
 .IP "\fBquery\fR"
 The SQL query template used to search the database, where \fB%s\fR
 is a substitute for the address Postfix is trying to resolve,

postfix/proto/mysql_table

@@ -79,6 +79,14 @@
 # .nf
 #	    dbname = customer_database
 # .fi
+# .IP "\fBcharset (default: utf8mb4)\fR"
+#	The default MySQL client character set; this also implies
+#	the collation order.
+#
+#	This parameter is available with Postfix 3.9 and later.
+#	With earlier Postfix versions, the default was chosen by
+#	the MySQL implementation (\fButf8mb4\fR as of MySQL 8.0,
+#	\fBlatin1\fR historically).
 # .IP "\fBquery\fR"
 #	The SQL query template used to search the database, where \fB%s\fR
 #	is a substitute for the address Postfix is trying to resolve,

postfix/proto/stop

@@ -1577,3 +1577,6 @@ Korbar
 ffdhe
 srv
 stderr
+charset
+latin
+utf

postfix/proto/stop.spell-cc

@@ -1443,7 +1443,7 @@ tv
 txn
 TXT
 Typechecking
-TYPECONNSTRING
+TYPECONNSTR
 typedef
 typedefs
 TYPEINET

postfix/src/global/dict_mysql.c

@@ -147,6 +147,7 @@ typedef struct {
     char   *username;
     char   *password;
     char   *dbname;
+    char   *charset;
     ARGV   *hosts;
     PLMYSQL *pldb;
 #if defined(MYSQL_VERSION_ID) && MYSQL_VERSION_ID >= 40000
@@ -439,7 +440,7 @@ static int plmysql_query(DICT_MYSQL *dict_mysql,
 {
     HOST   *host;
     MYSQL_RES *first_result = 0;
-    int     query_error;
+    int     query_error = 1;
 
     /*
      * Helper to avoid spamming the log with warnings.
@@ -602,6 +603,12 @@ static void plmysql_connect_single(DICT_MYSQL *dict_mysql, HOST *host)
 			   host->port,
 			   (host->type == TYPEUNIX ? host->name : 0),
 			   CLIENT_MULTI_RESULTS)) {
+	if (mysql_set_character_set(host->db, dict_mysql->charset) != 0) {
+	    msg_warn("dict_mysql: mysql_set_character_set '%s' failed: %s",
+		     dict_mysql->charset, mysql_error(host->db));
+	    plmysql_down_host(host);
+	    return;
+	}
 	if (msg_verbose)
 	    msg_info("dict_mysql: successful connection to host %s",
 		     host->hostname);
@@ -646,6 +653,7 @@ static void mysql_parse_config(DICT_MYSQL *dict_mysql, const char *mysqlcf)
     dict_mysql->username = cfg_get_str(p, "user", "", 0, 0);
     dict_mysql->password = cfg_get_str(p, "password", "", 0, 0);
     dict_mysql->dbname = cfg_get_str(p, "dbname", "", 1, 0);
+    dict_mysql->charset = cfg_get_str(p, "charset", "utf8mb4", 1, 0);
     dict_mysql->result_format = cfg_get_str(p, "result_format", "%s", 1, 0);
     dict_mysql->option_file = cfg_get_str(p, "option_file", NULL, 0, 0);
     dict_mysql->option_group = cfg_get_str(p, "option_group", "client", 0, 0);
@@ -826,6 +834,7 @@ static void dict_mysql_close(DICT *dict)
     myfree(dict_mysql->username);
     myfree(dict_mysql->password);
     myfree(dict_mysql->dbname);
+    myfree(dict_mysql->charset);
     myfree(dict_mysql->query);
     myfree(dict_mysql->result_format);
     if (dict_mysql->option_file)

postfix/src/global/dict_pgsql.c

@@ -108,7 +108,7 @@
 
 #define TYPEUNIX			(1<<0)
 #define TYPEINET			(1<<1)
-#define TYPECONNSTRING			(1<<2)
+#define TYPECONNSTR			(1<<2)
 
 #define RETRY_CONN_MAX			100
 #define RETRY_CONN_INTV			60	/* 1 minute */
@@ -119,7 +119,7 @@ typedef struct {
     char   *hostname;
     char   *name;
     char   *port;
-    unsigned type;			/* TYPEUNIX | TYPEINET | TYPECONNSTRING */
+    unsigned type;			/* TYPEUNIX | TYPEINET | TYPECONNSTR */
     unsigned stat;			/* STATUNTRIED | STATFAIL | STATCUR */
     time_t  ts;				/* used for attempting reconnection */
 } HOST;
@@ -152,12 +152,11 @@ typedef struct {
 
 /* internal function declarations */
 static PLPGSQL *plpgsql_init(ARGV *);
-static PGSQL_RES *plpgsql_query(DICT_PGSQL *, const char *, VSTRING *, char *,
+static PGSQL_RES *plpgsql_query(DICT_PGSQL *, const char *, VSTRING *);
-				        char *, char *, char *);
 static void plpgsql_dealloc(PLPGSQL *);
 static void plpgsql_close_host(HOST *);
 static void plpgsql_down_host(HOST *);
-static void plpgsql_connect_single(HOST *, char *, char *, char *, char *);
+static void plpgsql_connect_single(DICT_PGSQL *, HOST *);
 static const char *dict_pgsql_lookup(DICT *, const char *);
 DICT   *dict_pgsql_open(const char *, int, int);
 static void dict_pgsql_close(DICT *);
@@ -324,11 +323,7 @@ static const char *dict_pgsql_lookup(DICT *dict, const char *name)
 	return (0);
 
     /* do the query - set dict->error & cleanup if there's an error */
-    if ((query_res = plpgsql_query(dict_pgsql, name, query,
+    if ((query_res = plpgsql_query(dict_pgsql, name, query)) == 0) {
-				   dict_pgsql->dbname,
-				   dict_pgsql->encoding,
-				   dict_pgsql->username,
-				   dict_pgsql->password)) == 0) {
 	dict->error = DICT_ERR_RETRY;
 	return 0;
     }
@@ -404,8 +399,7 @@ static HOST *dict_pgsql_find_host(PLPGSQL *PLDB, unsigned stat, unsigned type)
 
 /* dict_pgsql_get_active - get an active connection */
 
-static HOST *dict_pgsql_get_active(PLPGSQL *PLDB, char *dbname, char *encoding,
+static HOST *dict_pgsql_get_active(DICT_PGSQL *dict_pgsql, PLPGSQL *PLDB)
-				           char *username, char *password)
 {
     const char *myname = "dict_pgsql_get_active";
     HOST   *host;
@@ -414,7 +408,7 @@ static HOST *dict_pgsql_get_active(PLPGSQL *PLDB, char *dbname, char *encoding,
     /* try the active connections first; prefer the ones to UNIX sockets */
     if ((host = dict_pgsql_find_host(PLDB, STATACTIVE, TYPEUNIX)) != NULL ||
 	(host = dict_pgsql_find_host(PLDB, STATACTIVE, TYPEINET)) != NULL ||
-	(host = dict_pgsql_find_host(PLDB, STATACTIVE, TYPECONNSTRING)) != NULL) {
+     (host = dict_pgsql_find_host(PLDB, STATACTIVE, TYPECONNSTR)) != NULL) {
 	if (msg_verbose)
 	    msg_info("%s: found active connection to host %s", myname,
 		     host->hostname);
@@ -432,11 +426,11 @@ static HOST *dict_pgsql_get_active(PLPGSQL *PLDB, char *dbname, char *encoding,
 	    (host = dict_pgsql_find_host(PLDB, STATUNTRIED | STATFAIL,
 					 TYPEINET)) != NULL ||
 	    (host = dict_pgsql_find_host(PLDB, STATUNTRIED | STATFAIL,
-					 TYPECONNSTRING)) != NULL)) {
+					 TYPECONNSTR)) != NULL)) {
 	if (msg_verbose)
 	    msg_info("%s: attempting to connect to host %s", myname,
 		     host->hostname);
-	plpgsql_connect_single(host, dbname, encoding, username, password);
+	plpgsql_connect_single(dict_pgsql, host);
 	if (host->stat == STATACTIVE)
 	    return host;
     }
@@ -464,18 +458,14 @@ static void dict_pgsql_event(int unused_event, void *context)
 
 static PGSQL_RES *plpgsql_query(DICT_PGSQL *dict_pgsql,
 				        const char *name,
-				        VSTRING *query,
+				        VSTRING *query)
-				        char *dbname,
-				        char *encoding,
-				        char *username,
-				        char *password)
 {
     PLPGSQL *PLDB = dict_pgsql->pldb;
     HOST   *host;
     PGSQL_RES *res = 0;
     ExecStatusType status;
 
-    while ((host = dict_pgsql_get_active(PLDB, dbname, encoding, username, password)) != NULL) {
+    while ((host = dict_pgsql_get_active(dict_pgsql, PLDB)) != NULL) {
 
 	/*
 	 * The active host is used to escape strings in the context of the
@@ -570,13 +560,14 @@ static PGSQL_RES *plpgsql_query(DICT_PGSQL *dict_pgsql,
  * used to reconnect to a single database when one is down or none is
  * connected yet. Log all errors and set the stat field of host accordingly
  */
-static void plpgsql_connect_single(HOST *host, char *dbname, char *encoding, char *username, char *password)
+static void plpgsql_connect_single(DICT_PGSQL *dict_pgsql, HOST *host)
 {
-    if (host->type == TYPECONNSTRING) {
+    if (host->type == TYPECONNSTR) {
 	host->db = PQconnectdb(host->name);
     } else {
 	host->db = PQsetdbLogin(host->name, host->port, NULL, NULL,
-				dbname, username, password);
+				dict_pgsql->dbname, dict_pgsql->username,
+				dict_pgsql->password);
     }
     if (host->db == NULL || PQstatus(host->db) != CONNECTION_OK) {
 	msg_warn("connect to pgsql server %s: %s",
@@ -584,9 +575,9 @@ static void plpgsql_connect_single(HOST *host, char *dbname, char *encoding, cha
 	plpgsql_down_host(host);
 	return;
     }
-    if (PQsetClientEncoding(host->db, encoding) != 0) {
+    if (PQsetClientEncoding(host->db, dict_pgsql->encoding) != 0) {
 	msg_warn("dict_pgsql: cannot set the encoding to %s, skipping %s",
-		 encoding, host->hostname);
+		 dict_pgsql->encoding, host->hostname);
 	plpgsql_down_host(host);
 	return;
     }
@@ -764,7 +755,7 @@ static HOST *host_init(const char *hostname)
      * Modern syntax: "postgresql://connection-info".
      */
     if (strncmp(d, "postgresql:", 11) == 0) {
-	host->type = TYPECONNSTRING;
+	host->type = TYPECONNSTR;
 	host->name = mystrdup(d);
 	host->port = 0;
     }

postfix/src/global/mail_version.h

@@ -20,7 +20,7 @@
   * Patches change both the patchlevel and the release date. Snapshots have no
   * patchlevel; they change the release date only.
   */
-#define MAIL_RELEASE_DATE	"20230416"
+#define MAIL_RELEASE_DATE	"20230419"
 #define MAIL_VERSION_NUMBER	"3.9"
 
 #ifdef SNAPSHOT