postfix-3.4-20180708

postfix/HISTORY

@@ -23599,9 +23599,9 @@ Apologies for any names omitted.
 
 	Bugfix (introduced: Postfix 3.0): with smtputf8_enable=yes,
 	table lookups could casefold the search string when searching
-	a regexp, pcre, tcp, or other lookup table that does not
+	a lookup table that does not use fixed-string keys (regexp,
-	use fixed-string keys, which is inconsistent with historical
+	pcre, tcp, etc.). Historically, Postfix would not case-fold
-	behavior. File: util/dict_utf8.c.
+	the search string with such tables. File: util/dict_utf8.c.
 
 	Cleanup: removed unimplemented VSTRING support to enforce
 	a buffer size limit (by returning an error of sorts). In
@@ -23621,3 +23621,12 @@ Apologies for any names omitted.
 	for publication on the web. Also cleaned up some makedefs(1)
 	content. Files: man/Makefile.in, man/man1/makedefs.1,
 	html/Makefile.in, html/makedefs.1.html.
+
+20180708
+
+	Cleanup: VSTREAM support to "open" a VSTRING: added
+	vstream_ftell() support; documented what changes are needed
+	before this can support vstream_fseek(), without breaking a
+	VSTRING during vstream_fflush(); added a simple 'allow'
+	filter for vstream_control() requests; added a unit test.
+	File: util/vstream.c.

postfix/src/global/mail_version.h

@@ -20,7 +20,7 @@
   * Patches change both the patchlevel and the release date. Snapshots have no
   * patchlevel; they change the release date only.
   */
-#define MAIL_RELEASE_DATE	"20180707"
+#define MAIL_RELEASE_DATE	"20180708"
 #define MAIL_VERSION_NUMBER	"3.4"
 
 #ifdef SNAPSHOT

postfix/src/global/sent.c

@@ -64,6 +64,11 @@
 /*	IBM T.J. Watson Research
 /*	P.O. Box 704
 /*	Yorktown Heights, NY 10598, USA
+/*
+/*	Wietse Venema
+/*	Google, Inc.
+/*	111 8th Avenue
+/*	New York, NY 10011, USA
 /*--*/
 
 /* System library. */

postfix/src/postconf/Makefile.in

@@ -946,7 +946,7 @@ test69:	$(PROG) test69.ref
 	echo junk = junk >> test69.cf
 	touch -t 197101010000 main.cf
 	$(SHLIB_ENV) ./$(PROG) -nc . >test69.tmp 2>&1
-	diff test69.ref test69.tmp
+	sed "s;PWD;`pwd`;" test69.ref | diff - test69.tmp
 	rm -f main.cf master.cf test69.tmp test69.cf
 
 printfck: $(OBJS) $(PROG)

postfix/src/postconf/test69.ref

@@ -1,2 +1,2 @@
-./postconf: warning: ldap:/home/wietse/postfix-3.4-20180217/src/postconf/test69.cf: unused parameter: junk=junk
+./postconf: warning: ldap:PWD/test69.cf: unused parameter: junk=junk
 config_directory = .

postfix/src/util/Makefile.in

@@ -132,7 +132,7 @@ TESTPROG= dict_open dup2_pass_on_exec events exec_command fifo_open \
 	myaddrinfo myaddrinfo4 inet_proto sane_basename format_tv \
 	valid_utf8_string ip_match base32_code msg_rate_delay netstring \
 	vstream timecmp dict_cache midna_domain casefold strcasecmp_utf8 \
-	vbuf_print split_qnameval
+	vbuf_print split_qnameval vstream
 PLUGIN_MAP_SO = $(LIB_PREFIX)pcre$(LIB_SUFFIX)
 
 LIB_DIR	= ../../lib
@@ -543,7 +543,7 @@ tests: all valid_hostname_test mac_expand_test dict_test unescape_test \
 	dict_utf8_test strcasecmp_utf8_test vbuf_print_test dict_regexp_test \
 	dict_union_test dict_pipe_test miss_endif_cidr_test \
 	miss_endif_pcre_test miss_endif_regexp_test split_qnameval_test \
-	vstring_test
+	vstring_test vstream_test
 
 root_tests:
 
@@ -841,6 +841,11 @@ vstring_test: dict_open vstring vstring_test.ref
 	diff vstring_test.ref vstring_test.tmp
 	rm -f vstring_test.tmp
 
+vstream_test: dict_open vstream vstream_test.in vstream_test.ref
+	$(SHLIB_ENV) ./vstream <vstream_test.in >vstream_test.tmp 2>&1
+	diff vstream_test.ref vstream_test.tmp
+	rm -f vstream_test.tmp
+
 depend: $(MAKES)
 	(sed '1,/^# do not edit/!d' Makefile.in; \
 	set -e; for i in [a-z][a-z0-9]*.c; do \

postfix/src/util/dict_utf8.c

@@ -33,6 +33,11 @@
 /*	IBM T.J. Watson Research
 /*	P.O. Box 704
 /*	Yorktown Heights, NY 10598, USA
+/*
+/*	Wietse Venema
+/*	Google, Inc.
+/*	111 8th Avenue
+/*	New York, NY 10011, USA
 /*--*/
 
  /*

postfix/src/util/vstream.c

@@ -199,8 +199,11 @@
 /*	vstream_memopen() either succeeds or never returns. Streams
 /*	opened with vstream_memopen() have limitations: they can't
 /*	be opened in read/write mode, they can't seek beyond the
-/*	end of the VSTRING, and they support none of the methods
+/*	end of the VSTRING, and they don't support vstream_control()
-/*	that require a file descriptor.
+/*	methods that manipulate buffers, file descriptors, or I/O
+/*	functions. After a VSTRING is opened for writing, its content
+/*	will be in an indeterminate state while the stream is open,
+/*	and will be null-terminated when the stream is closed.
 /*
 /*	vstream_fclose() closes the named buffered stream. The result
 /*	is 0 in case of success, VSTREAM_EOF in case of problems.
@@ -1108,11 +1111,26 @@ off_t   vstream_fseek(VSTREAM *stream, off_t offset, int whence)
     VBUF   *bp = &stream->buf;
 
     /*
-     * TODO: fseek/ftell for memory buffer.
+     * TODO: support data length (data length != buffer length). Without data
+     * length information, vstream_fseek() would break vstream_fflush() for
+     * memory streams.
      */
     if (stream->buf.flags & VSTREAM_FLAG_MEMORY) {
-	stream->buf.flags |= VSTREAM_FLAG_ERR;
+#ifdef PENDING_VSTREAM_FSEEK_FOR_MEMORY
+	if (whence == SEEK_CUR)
+	    offset += (bp->ptr - bp->data);
+	else if (whence == SEEK_END)
+	    offset += bp->len;
+	if (offset < 0 || offset > bp->data_len) {
+	    errno = EINVAL;
+	    return (-1);
+	}
+	VSTREAM_BUF_AT_OFFSET(bp, offset);
+	return (offset);
+#else
+	errno = EOPNOTSUPP;
 	return (-1);
+#endif
     }
 
     /*
@@ -1179,12 +1197,10 @@ off_t   vstream_ftell(VSTREAM *stream)
     VBUF   *bp = &stream->buf;
 
     /*
-     * TODO: fseek/ftell for memory buffer.
+     * Special case for memory buffer.
      */
-    if (stream->buf.flags & VSTREAM_FLAG_MEMORY) {
+    if (stream->buf.flags & VSTREAM_FLAG_MEMORY)
-	stream->buf.flags |= VSTREAM_FLAG_ERR;
+	return (bp->ptr - bp->data);
-	return (-1);
-    }
 
     /*
      * Shave an unnecessary syscall.
@@ -1296,9 +1312,23 @@ VSTREAM *vstream_fopen(const char *path, int flags, mode_t mode)
 
 int     vstream_fflush(VSTREAM *stream)
 {
+
+    /*
+     * With VSTRING, the write pointer must be positioned behind the end of
+     * data. Without knowing the actual data length, VSTREAM can't support
+     * vstream_fseek() for memory streams, because vstream_fflush() would
+     * leave the VSTRING in a broken state.
+     */
     if (stream->buf.flags & VSTREAM_FLAG_MEMORY) {
-	if (stream->buf.flags & VSTREAM_FLAG_WRITE)
+	if (stream->buf.flags & VSTREAM_FLAG_WRITE) {
-	    memcpy(&stream->vstring->vbuf, &stream->buf, sizeof(stream->buf));
+	    VSTRING *string = stream->vstring;
+
+#ifdef PENDING_VSTREAM_FSEEK_FOR_MEMORY
+	    VSTREAM_BUF_AT_OFFSET(&stream->buf, stream->buf.data_len);
+#endif
+	    memcpy(&string->vbuf, &stream->buf, sizeof(stream->buf));
+	    VSTRING_TERMINATE(string);
+	}
 	return (0);
     }
     if ((stream->buf.flags & VSTREAM_FLAG_READ_DOUBLE)
@@ -1322,7 +1352,9 @@ int     vstream_fclose(VSTREAM *stream)
      */
     if (stream->pid != 0)
 	msg_panic("vstream_fclose: stream has process");
-    if ((stream->buf.flags & VSTREAM_FLAG_WRITE_DOUBLE) != 0 && stream->fd >= 0)
+    if ((stream->buf.flags & VSTREAM_FLAG_MEMORY)
+	|| ((stream->buf.flags & VSTREAM_FLAG_WRITE_DOUBLE) != 0
+	    && stream->fd >= 0))
 	vstream_fflush(stream);
     /* Do not remove: vstream_fdclose() depends on this error test. */
     err = vstream_ferror(stream);
@@ -1426,7 +1458,18 @@ void    vstream_control(VSTREAM *stream, int name,...)
 
 #define SWAP(type,a,b) do { type temp = (a); (a) = (b); (b) = (temp); } while (0)
 
+    /*
+     * A crude 'allow' filter for memory streams.
+     */
+    int     memory_ops =
+    ((1 << VSTREAM_CTL_END) | (1 << VSTREAM_CTL_CONTEXT)
+     | (1 << VSTREAM_CTL_PATH) | (1 << VSTREAM_CTL_EXCEPT));
+
     for (va_start(ap, name); name != VSTREAM_CTL_END; name = va_arg(ap, int)) {
+	if ((stream->buf.flags & VSTREAM_FLAG_MEMORY)
+	    && (memory_ops & (1 << name)) == 0)
+	    msg_panic("%s: memory stream does not support VSTREAM_CTL_%d",
+		      VSTREAM_PATH(stream), name);
 	switch (name) {
 	case VSTREAM_CTL_READ_FN:
 	    stream->read_fn = va_arg(ap, VSTREAM_RW_FN);
@@ -1443,9 +1486,6 @@ void    vstream_control(VSTREAM *stream, int name,...)
 	    stream->path = mystrdup(va_arg(ap, char *));
 	    break;
 	case VSTREAM_CTL_DOUBLE:
-	    if (stream->buf.flags & VSTREAM_FLAG_MEMORY)
-		msg_panic("%s: memory stream does not support double buffering",
-			  VSTREAM_PATH(stream));
 	    if ((stream->buf.flags & VSTREAM_FLAG_DOUBLE) == 0) {
 		stream->buf.flags |= VSTREAM_FLAG_DOUBLE;
 		if (stream->buf.flags & VSTREAM_FLAG_READ) {
@@ -1702,7 +1742,7 @@ static void copy_line(ssize_t bufsize)
 
 static void printf_number(void)
 {
-    vstream_printf("%d\n", __MAXINT__(int));
+    vstream_printf("%d\n", 1234567890);
     vstream_fflush(VSTREAM_OUT);
 }
 
@@ -1710,22 +1750,47 @@ static void do_memory_stream(void)
 {
     VSTRING *buf = vstring_alloc(1);
     VSTREAM *fp = vstream_memopen(buf, O_WRONLY);
+
+#ifdef PENDING_VSTREAM_FSEEK_FOR_MEMORY
+    off_t   offset;
+
+#endif
     int     ch;
 
-    vstream_fprintf(fp, "hello world\n");
+    vstream_fprintf(fp, "hallo world\n");
     if (vstream_fflush(fp))
 	msg_fatal("vstream_fflush: %m");
+    vstream_printf("final memory stream write offset: %ld\n",
+		   (long) vstream_ftell(fp));
+#ifdef PENDING_VSTREAM_FSEEK_FOR_MEMORY
+    vstream_fflush(fp);
+    vstream_printf("buffer size: %ld, content: %s",
+		   (long) VSTRING_LEN(buf), vstring_str(buf));
+    if ((offset = vstream_fseek(fp, 1, SEEK_SET)) != 1)
+	msg_panic("unexpected vstream_fseek return: %ld, expected: %ld",
+		  (long) offset, (long) 1);
+    VSTREAM_PUTC('e', fp);
+#endif
     vstream_fclose(fp);
-    VSTRING_TERMINATE(buf);
 
-    vstream_printf("content of buffer[%ld]: %s",
+    vstream_printf("buffer size: %ld, content: %s",
 		   (long) VSTRING_LEN(buf), vstring_str(buf));
     vstream_fflush(VSTREAM_OUT);
 
-    vstream_printf("read from buffer[%ld]: ", (long) VSTRING_LEN(buf));
     fp = vstream_memopen(buf, O_RDONLY);
+    vstream_printf("reading memory stream: ");
     while ((ch = VSTREAM_GETC(fp)) != VSTREAM_EOF)
 	VSTREAM_PUTCHAR(ch);
+#ifdef PENDING_VSTREAM_FSEEK_FOR_MEMORY
+    vstream_printf("reading memory stream from offset 6: ");
+    if ((offset = vstream_fseek(fp, 6, SEEK_SET)) != 6)
+	msg_panic("unexpected vstream_fseek return: %ld, expected: %ld",
+		  (long) offset, (long) 6);
+    while ((ch = VSTREAM_GETC(fp)) != VSTREAM_EOF)
+	VSTREAM_PUTCHAR(ch);
+#endif
+    vstream_printf("final memory stream read offset: %ld\n",
+		   (long) vstream_ftell(fp));
     vstream_fflush(VSTREAM_OUT);
     vstream_fclose(fp);
     vstring_free(buf);

postfix/src/util/vstream_test.in

@@ -0,0 +1,3 @@
+abcdef
+ghijkl
+mnopqr

postfix/src/util/vstream_test.ref

@@ -0,0 +1,8 @@
+abcdef
+ghijkl
+mnopqr
+1234567890
+final memory stream write offset: 12
+buffer size: 12, content: hallo world
+reading memory stream: hallo world
+final memory stream read offset: 12