postfix-3.4-20181130

postfix/HISTORY

@@ -23854,3 +23854,21 @@ Apologies for any names omitted.
 	Cleanup: dict_file_to_xxx() takes a list of file names
 	separated by CHARS_COMMA_SP. Shoe-horned into the existing
 	API, make it nicer when there is time. File: util/dict_file.c.
+
+20181127
+
+	Cleanup: encapsulated clumsy 'read into VSTRING' code with
+	easier-to-use vstream_fread_buf() and vstream_fread_app()
+	primitives. Files: global/memcache_proto.c, global/record.c,
+	global/smtp_stream.c, global/smtp_stream.h, global/uxtext.c,
+	global/xtext.c, milter/milter8.c, util/dict_file.c,
+	util/hex_quote.c, util/netstring.c, util/vstream.c,
+	util/vstream.h. Verified with "make tests".
+
+	Cleanup: simplified the smtp_fread() API (introduced for
+	BDAT support), and changed the name to smtp_fread_buf().
+	Files: global/smtp_stream.c, smtpd/smtpd.c. Verified with
+	~megabyte BDAT commands.
+
+	Cleanup: simplified a tlsproxy-internal API. File:
+	tlsproxy/tlsproxy.c.

postfix/src/global/mail_version.h

@@ -20,7 +20,7 @@
   * Patches change both the patchlevel and the release date. Snapshots have no
   * patchlevel; they change the release date only.
   */
-#define MAIL_RELEASE_DATE	"20181125"
+#define MAIL_RELEASE_DATE	"20181130"
 #define MAIL_VERSION_NUMBER	"3.4"
 
 #ifdef SNAPSHOT

postfix/src/global/memcache_proto.c

@@ -39,6 +39,11 @@
 /*	IBM T.J. Watson Research
 /*	P.O. Box 704
 /*	Yorktown Heights, NY 10598, USA
+/*
+/*	Wietse Venema
+/*	Google, Inc.
+/*	111 8th Avenue
+/*	New York, NY 10011, USA
 /*--*/
 
 #include <sys_defs.h>
@@ -143,16 +148,13 @@ int     memcache_fread(VSTREAM *stream, VSTRING *buf, ssize_t todo)
     /*
      * Do the I/O.
      */
-    VSTRING_SPACE(buf, todo);
+    if (vstream_fread_buf(stream, buf, todo) != todo
-    VSTRING_AT_OFFSET(buf, todo);
-    if (vstream_fread(stream, STR(buf), todo) != todo
 	|| VSTREAM_GETC(stream) != '\r'
 	|| VSTREAM_GETC(stream) != '\n') {
 	if (msg_verbose)
 	    msg_info("%s read: error", VSTREAM_PATH(stream));
 	return (-1);
     } else {
-	vstring_truncate(buf, todo);
 	VSTRING_TERMINATE(buf);
 	if (msg_verbose)
 	    msg_info("%s read: %s", VSTREAM_PATH(stream), STR(buf));

postfix/src/global/record.c

@@ -287,14 +287,11 @@ int     rec_get_raw(VSTREAM *stream, VSTRING *buf, ssize_t maxsize, int flags)
 	 * Reserve buffer space for the result, and read the record data into
 	 * the buffer.
 	 */
-	VSTRING_RESET(buf);
+	if (vstream_fread_buf(stream, buf, len) != len) {
-	VSTRING_SPACE(buf, len);
-	if (vstream_fread(stream, vstring_str(buf), len) != len) {
 	    msg_warn("%s: unexpected EOF in data, record type %d length %ld",
 		     VSTREAM_PATH(stream), type, (long) len);
 	    return (REC_TYPE_ERROR);
 	}
-	VSTRING_AT_OFFSET(buf, len);
 	VSTRING_TERMINATE(buf);
 	if (msg_verbose > 2)
 	    msg_info("%s: type %c len %ld data %.10s", myname,

postfix/src/global/smtp_stream.c

@@ -37,7 +37,7 @@
 /*	ssize_t	len;
 /*	VSTREAM *stream;
 /*
-/*	void	smtp_fread(vp, len, stream)
+/*	void	smtp_fread_buf(vp, len, stream)
 /*	VSTRING	*vp;
 /*	ssize_t	len;
 /*	VSTREAM *stream;
@@ -111,8 +111,12 @@
 /*	Long strings are not broken. No CR LF is appended. The stream
 /*	is not flushed.
 /*
-/*	smtp_fread() appends the specified number of bytes from the
+/*	smtp_fread_buf() invokes vstream_fread_buf() to read the
-/*	stream to the buffer. The result is not null-terminated.
+/*	specified number of unformatted bytes from the stream. The
+/*	result is not null-terminated. NOTE: do not skip calling
+/*	smtp_fread_buf() when len == 0. This function has side
+/*	effects including resetting the buffer write position, and
+/*	skipping the call would invalidate the buffer state.
 /*
 /*	smtp_fputc() writes one character to the named stream.
 /*	The stream is not flushed.
@@ -474,23 +478,25 @@ void    smtp_fwrite(const char *cp, ssize_t todo, VSTREAM *stream)
 	smtp_longjmp(stream, SMTP_ERR_EOF, "smtp_fwrite");
 }
 
-/* smtp_fread - read one buffer from SMTP peer */
+/* smtp_fread_buf - read one buffer from SMTP peer */
 
-void    smtp_fread(VSTRING *vp, ssize_t todo, VSTREAM *stream)
+void    smtp_fread_buf(VSTRING *vp, ssize_t todo, VSTREAM *stream)
 {
     int     err;
 
-    if (todo <= 0)
+    /*
-	msg_panic("smtp_fread: zero or negative todo %ld", (long) todo);
+     * Do not return early if todo == 0. We still need the side effects from
+     * calling vstream_fread_buf() including resetting the buffer write
+     * position. Skipping the call would invalidate the buffer state.
+     */
+    if (todo < 0)
+	msg_panic("smtp_fread_buf: negative todo %ld", (long) todo);
 
     /*
      * Do the I/O, protected against timeout.
      */
     smtp_timeout_reset(stream);
-    VSTRING_SPACE(vp, todo);
+    err = (vstream_fread_buf(stream, vp, todo) != todo);
-    err = (vstream_fread(stream, vstring_end(vp), todo) != todo);
-    if (err == 0)
-	VSTRING_AT_OFFSET(vp, VSTRING_LEN(vp) + todo);
 
     /*
      * See if there was a problem.

postfix/src/global/smtp_stream.h

@@ -41,7 +41,7 @@ extern int smtp_get(VSTRING *, VSTREAM *, ssize_t, int);
 extern int smtp_get_noexcept(VSTRING *, VSTREAM *, ssize_t, int);
 extern void smtp_fputs(const char *, ssize_t len, VSTREAM *);
 extern void smtp_fwrite(const char *, ssize_t len, VSTREAM *);
-extern void smtp_fread(VSTRING *, ssize_t len, VSTREAM *);
+extern void smtp_fread_buf(VSTRING *, ssize_t len, VSTREAM *);
 extern void smtp_fputc(int, VSTREAM *);
 
 extern void smtp_vprintf(VSTREAM *, const char *, va_list);

postfix/src/global/uxtext.c

@@ -235,9 +235,7 @@ static ssize_t read_buf(VSTREAM *fp, VSTRING *buf)
 {
     ssize_t len;
 
-    VSTRING_RESET(buf);
+    len = vstream_fread_buf(fp, buf, BUFLEN);
-    len = vstream_fread(fp, STR(buf), vstring_avail(buf));
-    VSTRING_AT_OFFSET(buf, len);		/* XXX */
     VSTRING_TERMINATE(buf);
     return (len);
 }

postfix/src/global/xtext.c

@@ -155,9 +155,7 @@ static ssize_t read_buf(VSTREAM *fp, VSTRING *buf)
 {
     ssize_t len;
 
-    VSTRING_RESET(buf);
+    len = vstream_fread_buf(fp, buf, BUFLEN);
-    len = vstream_fread(fp, STR(buf), vstring_avail(buf));
-    VSTRING_AT_OFFSET(buf, len);		/* XXX */
     VSTRING_TERMINATE(buf);
     return (len);
 }

postfix/src/milter/milter8.c

@@ -51,6 +51,11 @@
 /*	IBM T.J. Watson Research
 /*	P.O. Box 704
 /*	Yorktown Heights, NY 10598, USA
+/*
+/*	Wietse Venema
+/*	Google, Inc.
+/*	111 8th Avenue
+/*	New York, NY 10011, USA
 /*--*/
 
 /* System library. */
@@ -663,14 +668,11 @@ static int vmilter8_read_data(MILTER8 *milter, ssize_t *data_len, va_list ap)
 		return (milter8_comm_error(milter));
 	    }
 	    buf = va_arg(ap, VSTRING *);
-	    VSTRING_RESET(buf);
+	    if (vstream_fread_buf(milter->fp, buf, *data_len)
-	    VSTRING_SPACE(buf, *data_len);
-	    if (vstream_fread(milter->fp, (void *) STR(buf), *data_len)
 		!= *data_len) {
 		msg_warn("milter %s: EOF while reading data: %m", milter->m.name);
 		return (milter8_comm_error(milter));
 	    }
-	    VSTRING_AT_OFFSET(buf, *data_len);
 	    *data_len = 0;
 	    break;
 

postfix/src/smtp/smtp_session.c

@@ -280,8 +280,10 @@ int     smtp_session_passivate(SMTP_SESSION *session, VSTRING *dest_prop,
      */
     if ((mp = vstream_memopen(endp_prop, O_WRONLY)) == 0
 	|| attr_print_plain(mp, ATTR_FLAG_NONE,
+#ifdef USE_TLS
 			    SEND_ATTR_INT(SESS_ATTR_TLS_LEVEL,
 					  session->state->tls->level),
+#endif
 			    SEND_ATTR_INT(SESS_ATTR_REUSE_COUNT,
 					  session->reuse_count),
 			    SEND_ATTR_INT(SESS_ATTR_ENDP_FEATURES,
@@ -329,9 +331,9 @@ SMTP_SESSION *smtp_session_activate(int fd, SMTP_ITERATOR *iter,
     int     dest_features;		/* server features */
     long    expire_time;		/* session re-use expiration time */
     int     reuse_count;		/* # times reused */
-    TLS_SESS_STATE *tls_context = 0;
 
 #ifdef USE_TLS
+    TLS_SESS_STATE *tls_context = 0;
     SMTP_TLS_POLICY *tls = iter->parent->tls;
 
 #endif
@@ -348,8 +350,10 @@ SMTP_SESSION *smtp_session_activate(int fd, SMTP_ITERATOR *iter,
      */
     if ((mp = vstream_memopen(endp_prop, O_RDONLY)) == 0
 	|| attr_scan_plain(mp, ATTR_FLAG_NONE,
+#ifdef USE_TLS
 			   RECV_ATTR_INT(SESS_ATTR_TLS_LEVEL,
 					 &tls->level),
+#endif
 			   RECV_ATTR_INT(SESS_ATTR_REUSE_COUNT,
 					 &reuse_count),
 			   RECV_ATTR_INT(SESS_ATTR_ENDP_FEATURES,

postfix/src/smtpd/smtpd.c

@@ -3727,10 +3727,9 @@ static int skip_bdat(SMTPD_STATE *state, off_t chunk_size,
      * connection in case of overload.
      */
     for (done = 0; done < chunk_size; done += len) {
-	VSTRING_RESET(state->buffer);
 	if ((len = chunk_size - done) > VSTREAM_BUFSIZE)
 	    len = VSTREAM_BUFSIZE;
-	smtp_fread(state->buffer, len, state->client);
+	smtp_fread_buf(state->buffer, len, state->client);
     }
 
     /*
@@ -3918,12 +3917,17 @@ static int bdat_cmd(SMTPD_STATE *state, int argc, SMTPD_TOKEN *argv)
      */
     done = 0;
     do {
+
+	/*
+	 * Do not skip the smtp_fread_buf() call if read_len == 0. We still
+	 * need the side effects which include resetting the buffer write
+	 * position. Skipping the call would invalidate the buffer state.
+	 * 
+	 * Caution: smtp_fread_buf() will long jump after EOF or timeout.
+	 */
 	if ((read_len = chunk_size - done) > VSTREAM_BUFSIZE)
 	    read_len = VSTREAM_BUFSIZE;
-	/* Caution: smtp_fread() makes a long jump in case of EOF or timeout. */
+	smtp_fread_buf(state->buffer, read_len, state->client);
-	VSTRING_RESET(state->buffer);
-	if (read_len > 0)
-	    smtp_fread(state->buffer, read_len, state->client);
 	state->bdat_get_stream = vstream_memreopen(
 			   state->bdat_get_stream, state->buffer, O_RDONLY);
 

postfix/src/tlsproxy/tlsproxy.c

@@ -1009,15 +1009,14 @@ static void tlsp_get_fd_event(int event, void *context)
  /*
   * Macro for readability.
   */
-#define TLSP_CLIENT_INIT(ctx, props, a1, a2, a3, a4, a5, a6, a7, a8, a9, \
+#define TLSP_CLIENT_INIT(props, a1, a2, a3, a4, a5, a6, a7, a8, a9, \
     a10, a11, a12, a13) \
-    tlsp_client_init((ctx), TLS_CLIENT_INIT_ARGS((props), a1, a2, a3, a4, \
+    tlsp_client_init(TLS_CLIENT_INIT_ARGS((props), a1, a2, a3, a4, \
     a5, a6, a7, a8, a9, a10, a11, a12, a13))
 
 /* tlsp_client_init - initialize a TLS client engine */
 
-static int tlsp_client_init(TLS_APPL_STATE **client_appl_state,
+static TLS_APPL_STATE *tlsp_client_init(TLS_CLIENT_INIT_PROPS *init_props)
-			            TLS_CLIENT_INIT_PROPS *init_props)
 {
     TLS_APPL_STATE *appl_state;
     VSTRING *buf;
@@ -1070,8 +1069,8 @@ static int tlsp_client_init(TLS_APPL_STATE **client_appl_state,
 		 "making this tls_client_init request, 2) configure a "
 		 "custom tlsproxy service with tlsproxy_client_* settings "
 		 "that match that SMTP client, and 3) configure that SMTP "
-		 "client with a tlsproxy_service setting that resolves to "
+		 "client with a tlsproxy_service_name setting that resolves "
-		 "that custom tlsproxy service");
+		 "to that custom tlsproxy service");
     }
 
     /*
@@ -1099,9 +1098,8 @@ static int tlsp_client_init(TLS_APPL_STATE **client_appl_state,
 			     SSL_MODE_ENABLE_PARTIAL_WRITE
 			     | SSL_MODE_ACCEPT_MOVING_WRITE_BUFFER);
     }
-    *client_appl_state = appl_state;
     vstring_free(buf);
-    return (appl_state != 0);
+    return (appl_state);
 }
 
 /* tlsp_close_event - pre-handshake plaintext-client close event */
@@ -1197,7 +1195,8 @@ static void tlsp_get_request_event(int event, void *context)
 	    tlsp_state_free(state);
 	    return;
 	}
-	ready = tlsp_client_init(&state->appl_state, state->client_init_props);
+	state->appl_state = tlsp_client_init(state->client_init_props);
+	ready = state->appl_state != 0;
 	break;
     case TLS_PROXY_FLAG_ROLE_SERVER:
 	state->is_server_role = 1;
@@ -1468,7 +1467,8 @@ static void pre_jail_init(char *unused_name, char **unused_argv)
 	 * Large parameter lists are error-prone, so we emulate a language
 	 * feature that C does not have natively: named parameter lists.
 	 */
-	if (TLSP_CLIENT_INIT(&tlsp_client_ctx, &props,
+	tlsp_client_ctx =
+	    TLSP_CLIENT_INIT(&props,
 			     log_param = var_tlsp_clnt_logparam,
 			     log_level = var_tlsp_clnt_loglevel,
 			     verifydepth = var_tlsp_clnt_scert_vd,
@@ -1481,7 +1481,8 @@ static void pre_jail_init(char *unused_name, char **unused_argv)
 			     eckey_file = var_tlsp_clnt_eckey_file,
 			     CAfile = var_tlsp_clnt_CAfile,
 			     CApath = var_tlsp_clnt_CApath,
-			     mdalg = var_tlsp_clnt_fpt_dgst) == 0)
+			     mdalg = var_tlsp_clnt_fpt_dgst);
+	if (tlsp_client_ctx == 0)
 	    msg_warn("TLS client initialization failed");
     }
 

postfix/src/util/dict_file.c

@@ -24,18 +24,15 @@
 /*	void	dict_file_purge_buffers(
 /*	DICT	*dict)
 /* DESCRIPTION
-/*	dict_file_to_buf() reads the content of the specified
+/*	dict_file_to_buf() reads the content of the specified files,
-/*	files, with names separated by CHARS_COMMA_SP, while inserting
+/*	with names separated by CHARS_COMMA_SP, while inserting a
-/*	a gratuitous newline character between files.
+/*	gratuitous newline character between files. It returns a
-/*	It returns a pointer to a buffer which is owned by the DICT,
+/*	pointer to a buffer which is owned by the DICT, or a null
-/*	or a null pointer in case of error.
+/*	pointer in case of error.
 /*
-/*	dict_file_to_b64() reads the content of the specified
+/*	dict_file_to_b64() invokes dict_file_to_buf() and converts
-/*	files, with names separated by CHARS_COMMA_SP, while inserting
+/*	the result to base64. It returns a pointer to a buffer which
-/*	a gratuitous newline character between files,
+/*	is owned by the DICT, or a null pointer in case of error.
-/*	and converts the result to base64.
-/*	It returns a pointer to a buffer which is owned by the DICT,
-/*	or a null pointer in case of error.
 /*
 /*	dict_file_from_b64() converts a value from base64. It returns
 /*	a pointer to a buffer which is owned by the DICT, or a null
@@ -121,14 +118,11 @@ VSTRING *dict_file_to_buf(DICT *dict, const char *pathnames)
 	    vstring_sprintf(dict->file_buf, "file too large: %s", pathnames);
 	    DICT_FILE_ERR_RETURN;
 	}
-	VSTRING_SPACE(dict->file_buf, st.st_size);
+	if (vstream_fread_app(fp, dict->file_buf, st.st_size) != st.st_size) {
-	if (vstream_fread(fp, STR(dict->file_buf) + LEN(dict->file_buf),
-			  st.st_size) != st.st_size) {
 	    vstring_sprintf(dict->file_buf, "read %s: %m", *cpp);
 	    DICT_FILE_ERR_RETURN;
 	}
 	(void) vstream_fclose(fp);
-	VSTRING_AT_OFFSET(dict->file_buf, LEN(dict->file_buf) + st.st_size);
 	if (cpp[1] != 0)
 	    VSTRING_ADDCH(dict->file_buf, '\n');
     }

postfix/src/util/hex_quote.c

@@ -35,6 +35,11 @@
 /*	IBM T.J. Watson Research
 /*	P.O. Box 704
 /*	Yorktown Heights, NY 10598, USA
+/*
+/*	Wietse Venema
+/*	Google, Inc.
+/*	111 8th Avenue
+/*	New York, NY 10011, USA
 /*--*/
 
 /* System library. */
@@ -119,9 +124,7 @@ static ssize_t read_buf(VSTREAM *fp, VSTRING *buf)
 {
     ssize_t len;
 
-    VSTRING_RESET(buf);
+    len = vstream_fread_buf(fp, buf, BUFLEN);
-    len = vstream_fread(fp, STR(buf), vstring_avail(buf));
-    VSTRING_AT_OFFSET(buf, len);		/* XXX */
     VSTRING_TERMINATE(buf);
     return (len);
 }

postfix/src/util/netstring.c

@@ -231,16 +231,10 @@ VSTRING *netstring_get_data(VSTREAM *stream, VSTRING *buf, ssize_t len)
 {
     const char *myname = "netstring_get_data";
 
-    /*
-     * Allocate buffer space.
-     */
-    VSTRING_RESET(buf);
-    VSTRING_SPACE(buf, len);
-
     /*
      * Read the payload and absorb the terminator.
      */
-    if (vstream_fread(stream, STR(buf), len) != len)
+    if (vstream_fread_buf(stream, buf, len) != len)
 	netstring_except(stream, vstream_ftimeout(stream) ?
 			 NETSTRING_ERR_TIME : NETSTRING_ERR_EOF);
     if (msg_verbose > 1)
@@ -249,9 +243,8 @@ VSTRING *netstring_get_data(VSTREAM *stream, VSTRING *buf, ssize_t len)
     netstring_get_terminator(stream);
 
     /*
-     * Position the buffer.
+     * Return the buffer.
      */
-    VSTRING_AT_OFFSET(buf, len);
     return (buf);
 }
 

postfix/src/util/vstream.c

@@ -78,7 +78,17 @@
 /*
 /*	ssize_t	vstream_fwrite(stream, buf, len)
 /*	VSTREAM	*stream;
-/*	const void *buf;
+/*	void *buf;
+/*	ssize_t	len;
+/*
+/*	ssize_t	vstream_fread_app(stream, buf, len)
+/*	VSTREAM	*stream;
+/*	VSTRING	*buf;
+/*	ssize_t	len;
+/*
+/*	ssize_t	vstream_fread_buf(stream, buf, len)
+/*	VSTREAM	*stream;
+/*	VSTRING	*buf;
 /*	ssize_t	len;
 /*
 /*	void	vstream_control(stream, name, ...)
@@ -287,6 +297,19 @@
 /*	transferred. A short count is returned in case of end-of-file
 /*	or error conditions.
 /*
+/*	vstream_fread_buf() resets the buffer write position,
+/*	allocates space for the specified number of bytes in the
+/*	buffer, reads the bytes from the specified VSTREAM, and
+/*	adjusts the buffer write position. The buffer is NOT
+/*	null-terminated. The result value is as with vstream_fread().
+/*      NOTE: do not skip calling vstream_fread_buf() when len == 0.
+/*      This function has side effects including resetting the buffer
+/*      write position, and skipping the call would invalidate the
+/*      buffer state.
+/*
+/*	vstream_fread_app() is like vstream_fread_buf() but appends
+/*	to existing buffer content, instead of writing over it.
+/*
 /*	vstream_control() allows the user to fine tune the behavior of
 /*	the specified stream.  The arguments are a list of macros with
 /*	zero or more arguments, terminated with CA_VSTREAM_CTL_END
@@ -1456,6 +1479,33 @@ int     vstream_fputs(const char *str, VSTREAM *stream)
     return (0);
 }
 
+/* vstream_fread_buf - unformatted read to VSTRING */
+
+ssize_t vstream_fread_buf(VSTREAM *fp, VSTRING *vp, ssize_t len)
+{
+    ssize_t ret;
+
+    VSTRING_RESET(vp);
+    VSTRING_SPACE(vp, len);
+    ret = vstream_fread(fp, vstring_str(vp), len);
+    if (ret > 0)
+	VSTRING_AT_OFFSET(vp, ret);
+    return (ret);
+}
+
+/* vstream_fread_app - unformatted read to VSTRING */
+
+ssize_t vstream_fread_app(VSTREAM *fp, VSTRING *vp, ssize_t len)
+{
+    ssize_t ret;
+
+    VSTRING_SPACE(vp, len);
+    ret = vstream_fread(fp, vstring_end(vp), len);
+    if (ret > 0)
+	VSTRING_AT_OFFSET(vp, VSTRING_LEN(vp) + ret);
+    return (ret);
+}
+
 /* vstream_control - fine control */
 
 void    vstream_control(VSTREAM *stream, int name,...)

postfix/src/util/vstream.h

@@ -132,6 +132,8 @@ extern int vstream_fdclose(VSTREAM *);
 
 #define vstream_fstat(vp, fl)	((vp)->buf.flags & (fl))
 
+extern ssize_t vstream_fread_buf(VSTREAM *, struct VSTRING *, ssize_t);
+extern ssize_t vstream_fread_app(VSTREAM *, struct VSTRING *, ssize_t);
 extern void vstream_control(VSTREAM *, int,...);
 
 /* Legacy API: type-unchecked arguments, internal use. */