diff --git a/postfix/HISTORY b/postfix/HISTORY index c8473aa89..20a72296c 100644 --- a/postfix/HISTORY +++ b/postfix/HISTORY @@ -17283,3 +17283,22 @@ Apologies for any names omitted. src/postlog/postlog.c, src/postmap/postmap.c, src/postmulti/postmulti.c, src/postqueue/postqueue.c, src/postsuper/postsuper.c, src/sendmail/sendmail.c. + +20111211 + + Feature: first/next (sequence) support in the proxymap + protocol. This is needed for cache cleanup of a proxied + postscreen or verify persistent cache. Files: + global/dict_proxy.[hc], proxymap/proxymap.c. + + Feature: memcache client support without libmemcache + dependencies. Files: global/memcache_proto.[hc], + global/dict_memcache.c. + + Feature: support for persistent backup database in the + memcache client. The database can be shared with the proxymap + service, but it needs to be listed as "proxy:maptype:mapname" + in the proxy_read_maps or proxy_write_maps parameter value + (depending on whether the access is read-only or read-write). + Support for proxymap-over-tcp (proxy:maptype:mapname@host:port) + is under development. File: global/dict_memcache.c. diff --git a/postfix/README_FILES/MEMCACHE_README b/postfix/README_FILES/MEMCACHE_README index 9c17f5551..4685f4c51 100644 --- a/postfix/README_FILES/MEMCACHE_README +++ b/postfix/README_FILES/MEMCACHE_README @@ -4,63 +4,36 @@ PPoossttffiixx mmeemmccaacchhee cclliieenntt HHoowwtto IInnttrroodduuccttiioonn -The Postfix memcache client type allows you to hook up Postfix to a memcache -server. This implementation supports multiple memcache servers for redundancy, -and multiple memcache clients that you can use for different table lookups. The -Postfix memcache client supports both lookup and update operations. +The Postfix memcache client allows you to hook up Postfix to a memcache server. +The current implementation supports one memcache server per Postfix table, with +one optional Postfix database that provides persistent backup. The Postfix +memcache client supports the lookup, update, delete and sequence operations. +The sequence (i.e. first/next) operation requires a backup database that +supports this operation. -Typically, a memcache map is used to reduce query load on a database server, or -to share a low-latency database among different Postfix instances. +Typically, the Postfix memcache client is used to reduce query load on a +persistent database, but it may also be used to query a memory-only database +for low-value, easy-to-create, information such as a reputation cache for +postscreen(8), verify(8) or greylisting. LLiimmiittaattiioonnss - * The Postfix memcache client is based on libmemcache, which will terminate - its process after a memcache server goes down. To avoid this, set up - redundant memcache servers that have no common source of failure. - * The Postfix memcache client cannot be used for security-sensitive tables such as alias_maps (these may contain "|command" and "/file/name" - destinations), or virtual_uid_maps and virtual_gid_maps (these specify UNIX - process privileges). Typically, a memcache database is shared via a TCP - socket, and is writable not only by Postfix, but by any process that can - talk to the memcache server. + destinations), or virtual_uid_maps, virtual_gid_maps and + virtual_mailbox_maps (these specify UNIX process privileges or "/file/name" + destinations). Typically, a memcache database is writable by any process + that can talk to the memcache server; in contrast, security-sensitive + tables must not be writable by the unprivileged Postfix user. - * The Postfix memcache client requires additional configuration when used - with the postscreen(8) and verify(8) daemons. For details see the ttl - parameter discussion in the memcache_table(5) manual page. - - * The Postfix memcache client is supported only with libmemcache version - 1.4.0. Some libmemcache features are documented by reading libmemcache - source code, instead of a proper API. + * The Postfix memcache client requires additional configuration when used as + postscreen(8) or verify(8) cache. For details see the backup and ttl + parameter discussions in the memcache_table(5) manual page. BBuuiillddiinngg PPoossttffiixx wwiitthh mmeemmccaacchhee ssuuppppoorrtt -To build Postfix with memcache client support, specify -DHAS_MEMCACHE, the -location of the libmemcache include files, and the location of the libmemcache -object library. - -For example: - - % make -f Makefile.init makefiles \ - 'CCARGS=-DHAS_MEMCACHE -I/usr/local/include' \ - 'AUXLIBS=-L/usr/local/lib -lmemcache' - -Then run 'make'. - -If the build fails with "undefined reference to `mcm_buf_len'" (and with a -similar error message for mcm_buf_remain_off), then you need to edit -libmemcache source code. - -The following instructions apply to libmemcache 1.4.0.rc2. - - * Open the libmemcache source file include/memcache/buffer.h. - - * Delete the "inline" words before the functions that were reported in the - "undefined reference" error messages. - - * Recompile and reinstall libmemcache. - -Then, continue building Postfix by running 'make'. +The Postfix memcache client has no external dependencies, and is therefore +built into Postfix by default. CCoonnffiigguurriinngg mmeemmccaacchhee llooookkuupp ttaabblleess @@ -68,8 +41,10 @@ Configuration is described in the memcache_table(5) manpage. CCrreeddiittss -The first memcache client for Postfix was written by Omar Kilani. +The first memcache client for Postfix was written by Omar Kilani, and was based +on the libmemcache library. -Wietse wrote a new memcache client from the ground up. Besides also using -libmemcache, the current implementation bears no resemblance to Omar's work. +Wietse wrote the current memcache client from the ground up. This +implementation does not use libmemcache, and bears no resemblance to earlier +work. diff --git a/postfix/RELEASE_NOTES b/postfix/RELEASE_NOTES index 48cd6b1a8..c375bb3ef 100644 --- a/postfix/RELEASE_NOTES +++ b/postfix/RELEASE_NOTES @@ -14,6 +14,33 @@ specifies the release date of a stable release or snapshot release. If you upgrade from Postfix 2.7 or earlier, read RELEASE_NOTES-2.8 before proceeding. +Major changes with snapshot 20111213 +==================================== + +Support for a persistent backup database in the memcache client. +The memcache client updates the memcache whenever it looks up or +modifies information in the persistent database. + +The persistent database can be shared with the proxymap service, +but it needs to be listed as "proxy:maptype:mapname" in the +proxy_read_maps or proxy_write_maps parameter value (depending on +whether the access is read-only or read-write). + +Support for proxymap-over-tcp (proxy:maptype:mapname@host:port) is +under development. + +Elimination of dependencies on the libmemcache library. Postfix +memcache support is now compiled in by default. + +Major changes with snapshot 20111209 +==================================== + +memcache lookup and update support. This provides a way to share +postscreen(8) or verify(8) caches between Postfix instances. The +Postfix memcache client can't be used for security-sensitive +information, and it supports only libmemcache version 1.4.0. See +MEMCACHE_README and memcache_table(5) for details and limitations. + Incompatible changes with snapshot 20111205 =========================================== diff --git a/postfix/WISHLIST b/postfix/WISHLIST index d2a4e8cb9..dd11556ff 100644 --- a/postfix/WISHLIST +++ b/postfix/WISHLIST @@ -6,6 +6,9 @@ Wish list: or require that they reset dict_errno on entry, either exit with a fatal error or set dict_errno on error. + dict_memcache: treat "bad" key as cache miss, i.e. read/write + the database as if the cache did not exist. + Is it possible to replace msg_fatal calls in match_ops.c by msg_warn and longjmp? The callers will have to specify if they want the code to return instead of terminate. diff --git a/postfix/html/MEMCACHE_README.html b/postfix/html/MEMCACHE_README.html index ff96b8bc9..5f3226cc6 100644 --- a/postfix/html/MEMCACHE_README.html +++ b/postfix/html/MEMCACHE_README.html @@ -19,82 +19,44 @@

Introduction

-

The Postfix memcache client type allows you to hook up Postfix to -a memcache server. This implementation supports multiple memcache -servers for redundancy, and multiple memcache clients that you can -use for different table lookups. The Postfix memcache client -supports both lookup and update operations.

+

The Postfix memcache client allows you to hook up Postfix to a +memcache server. The current implementation supports one memcache +server per Postfix table, with one optional Postfix database that +provides persistent backup. The Postfix memcache client supports +the lookup, update, delete and sequence operations. The sequence +(i.e. first/next) operation requires a backup database that supports +this operation.

-

Typically, a memcache map is used to reduce query load on a -database server, or to share a low-latency database among different -Postfix instances.

+

Typically, the Postfix memcache client is used to reduce query +load on a persistent database, but it may also be used to query a +memory-only database for low-value, easy-to-create, information +such as a reputation cache for postscreen(8), verify(8) or greylisting. +

Limitations

Building Postfix with memcache support

-

To build Postfix with memcache client support, specify --DHAS_MEMCACHE, the location of the libmemcache include -files, and the location of the libmemcache object library.

- -

For example:

- -
-
-% make -f Makefile.init makefiles \
-        'CCARGS=-DHAS_MEMCACHE -I/usr/local/include' \
-	'AUXLIBS=-L/usr/local/lib -lmemcache'
-
-
- -

Then run 'make'.

- -

If the build fails with "undefined reference to `mcm_buf_len'" -(and with a similar error message for mcm_buf_remain_off), -then you need to edit libmemcache source code.

- -

The following instructions apply to libmemcache 1.4.0.rc2.

- - - -

Then, continue building Postfix by running 'make'.

+

The Postfix memcache client has no external dependencies, +and is therefore built into Postfix by default.

Configuring memcache lookup tables

@@ -102,11 +64,12 @@ messages.

Credits

-

The first memcache client for Postfix was written by Omar Kilani.

+

The first memcache client for Postfix was written by Omar Kilani, +and was based on the libmemcache library.

-

Wietse wrote a new memcache client from the ground up. Besides -also using libmemcache, the current implementation bears no resemblance -to Omar's work.

+

Wietse wrote the current memcache client from the ground up. +This implementation does not use libmemcache, and bears no resemblance +to earlier work.

diff --git a/postfix/html/memcache_table.5.html b/postfix/html/memcache_table.5.html index ddf5db8c0..85610f977 100644 --- a/postfix/html/memcache_table.5.html +++ b/postfix/html/memcache_table.5.html @@ -20,27 +20,60 @@ MEMCACHE_TABLE(5) MEMCACHE_TABLE(5) or db format. Alternatively, lookup tables can be specified as memcache - instances. In order to use memcache lookups, define a - memcache source as a lookup table in main.cf, for example: + instances. To use memcache lookups, define a memcache + source as a lookup table in main.cf, for example: virtual_alias_maps = memcache:/etc/postfix/memcache-aliases.cf - The file /etc/postfix/memcache-aliases.cf has the same - format as the Postfix main.cf file, and specifies the + The file /etc/postfix/memcache-aliases.cf has the same + format as the Postfix main.cf file, and specifies the parameters described below. - The Postfix memcache client supports the lookup and update - operations. + The Postfix memcache client supports the lookup, update, + delete and sequence (first/next) operations. The sequence + operation requires a backup database that supports the + operation. MEMCACHE PARAMETERS - hosts (default: localhost:11211) - The memcache servers that Postfix will try to con- - nect to. Specify a hostname or address, optionally - followed by ":" and a port name or number. The - default port is 11211. Examples: + backup An optional Postfix database that provides persis- + tent backup for the memcache database. The Postfix + memcache client will update the memcache database + whenever it looks up or changes information in the + persistent database. Specify a Postfix "type:table" + database. Example: - hosts = memcache01.example.com - memcache02.example.com + backup = btree:/var/lib/postfix/postscreen_cache_map + + Access to remote proxymap servers is under develop- + ment. + + NOTE 1: When using memcache with persistent backup + as postscreen(8) or verify(8) cache, disable auto- + matic cache cleanup (*_cache_cleanup_interval = 0) + in all Postfix instances except for one instance + that will be responsible for cache cleanup. + + NOTE 2: In the case of a proxied database, the full + database name (including the "proxy:" prefix) must + be specified in the proxymap server's + proxy_read_maps or proxy_write_maps setting + (depending on whether the access is read-only or + read-write). + + memcache (default: inet:localhost:11211) + The memcache server (note: singular) that Postfix + will try to connect to. For a TCP server specify + "inet:" followed by a hostname or address, ":", and + a port name or number. For a UNIX-domain server + specify "unix:" followed by the socket pathname. + Examples: + + memcache = inet:memcache.example.com + memcache = unix:/path/to/socket + + NOTE: In the case of a UNIX-domain socket, it must + be accessible by the unprivileged postfix user and + by the memcached process. key_format (default: %s) Format of the lookup and update keys in memcache @@ -111,45 +144,37 @@ MEMCACHE_TABLE(5) MEMCACHE_TABLE(5) Optional flags that should be stored along with a memcache update. - ttl (default: 604800) + ttl (default: 3600) The expiration time in seconds of memcache updates. - The default is one week. - When using memcache tables with postscreen(8) or - verify(8), specify a zero *_cache_cleanup_interval - value, and specify the largest postscreen(8) *_ttl + NOTE 1: When using a memcache table as + postscreen(8) or verify(8) cache without persistent + backup, specify a zero *_cache_cleanup_interval + value with all Postfix instances that use the mem- + cache, and specify the largest postscreen(8) *_ttl value or verify(8) *_expire_time value as the mem- - cache map's ttl value. + cache table's ttl value. - Note: according to memcache protocol documentation, - a value greater than 30 days (2592000 seconds) - specifies absolute UNIX time. Smaller values are - relative to the time of the update. + NOTE 2: According to memcache protocol documenta- + tion, a value greater than 30 days (2592000 sec- + onds) specifies absolute UNIX time. Smaller values + are relative to the time of the update. BUGS - The Postfix memcache client is based on libmemcache, which - will terminate its process after a memcache server goes - down. To avoid this, set up redundant memcache servers - that have no common source of failure. - - The Postfix memcache client cannot be used for security- - sensitive tables such as alias_maps (these may contain - "|command and "/file/name" destinations), or vir- - tual_uid_maps and virtual_gid_maps (these specify UNIX - process privileges). In a typical deployment a memcache - database is shared via a TCP socket, and is therefore - writable not only by Postfix, but by any process that can - talk to the memcache server. + The Postfix memcache client cannot be used for security- + sensitive tables such as alias_maps (these may contain + "|command and "/file/name" destinations), or vir- + tual_uid_maps, virtual_gid_maps and virtual_mailbox_maps + (these specify UNIX process privileges or "/file/name" + destinations). In a typical deployment a memcache data- + base is writable by any process that can talk to the mem- + cache server; in contrast, security-sensitive tables must + not be writable by the unprivileged Postfix user. The Postfix memcache client requires additional configura- - tion when used with the postscreen(8) and verify(8) dae- - mons. For details see the ttl parameter discussion at the - end of the MEMCACHE PARAMETERS section in this document. - - The Postfix memcache client is supported only with libmem- - cache version 1.4.0. Some libmemcache features are docu- - mented by reading libmemcache source code, instead a - proper API. + tion when used as postscreen(8) or verify(8) cache. For + details see the backup and ttl parameter discussions in + the MEMCACHE PARAMETERS section above. SEE ALSO postmap(1), Postfix lookup table manager @@ -160,13 +185,14 @@ MEMCACHE_TABLE(5) MEMCACHE_TABLE(5) MEMCACHE_README, Postfix memcache client guide LICENSE - The Secure Mailer license must be distributed with this + The Secure Mailer license must be distributed with this software. HISTORY - The first memcache client for Postfix was written by Omar - Kilani. Besides being implemented on libmemcache, this - implementation bears no resemblance to his work. + The first memcache client for Postfix was written by Omar + Kilani, and was based on libmemcache. The Postfix imple- + mentation does not use libmemcache, and bears no resem- + blance to earlier work. AUTHOR(S) Wietse Venema diff --git a/postfix/html/proxymap.8.html b/postfix/html/proxymap.8.html index abdcf4ea8..8b674a85d 100644 --- a/postfix/html/proxymap.8.html +++ b/postfix/html/proxymap.8.html @@ -79,6 +79,14 @@ PROXYMAP(8) PROXYMAP(8) This request is supported in Postfix 2.5 and later. + sequence maptype:mapname flags function + Iterate over the specified database. The function + is one of DICT_SEQ_FUN_FIRST or DICT_SEQ_FUN_NEXT. + The reply is the request completion status code and + a lookup key and result value, if found. + + This request is supported in Postfix 2.9 and later. + The request completion status is one of OK, RETRY, NOKEY (lookup failed because the key was not found), BAD (mal- formed request) or DENY (the table is not approved for diff --git a/postfix/man/man5/memcache_table.5 b/postfix/man/man5/memcache_table.5 index 3e617b82d..4d8a48dba 100644 --- a/postfix/man/man5/memcache_table.5 +++ b/postfix/man/man5/memcache_table.5 @@ -19,8 +19,8 @@ rewriting or mail routing. These tables are usually in \fBdbm\fR or \fBdb\fR format. Alternatively, lookup tables can be specified as memcache -instances. In order to use memcache lookups, define a -memcache source as a lookup table in main.cf, for example: +instances. To use memcache lookups, define a memcache +source as a lookup table in main.cf, for example: .nf virtual_alias_maps = memcache:/etc/postfix/memcache-aliases.cf @@ -30,23 +30,53 @@ The file /etc/postfix/memcache-aliases.cf has the same format as the Postfix main.cf file, and specifies the parameters described below. -The Postfix memcache client supports the lookup and update -operations. +The Postfix memcache client supports the lookup, update, +delete and sequence (first/next) operations. The sequence +operation requires a backup database that supports the +operation. .SH "MEMCACHE PARAMETERS" .na .nf .ad .fi -.IP "\fBhosts (default: localhost:11211)\fR" -The memcache servers that Postfix will try to connect to. -Specify a hostname or address, optionally followed by ":" -and a port name or number. The default port is 11211. -Examples: +.IP \fBbackup\fR +An optional Postfix database that provides persistent backup +for the memcache database. The Postfix memcache client will +update the memcache database whenever it looks up or changes +information in the persistent database. Specify a Postfix +"type:table" database. Example: .nf - hosts = memcache01.example.com - memcache02.example.com + backup = btree:/var/lib/postfix/postscreen_cache_map .fi + +Access to remote proxymap servers is under development. + +NOTE 1: When using memcache with persistent backup as +\fBpostscreen\fR(8) or \fBverify\fR(8) cache, disable +automatic cache cleanup (*_cache_cleanup_interval = 0) in +all Postfix instances except for one instance that will be +responsible for cache cleanup. + +NOTE 2: In the case of a proxied database, the full database +name (including the "proxy:" prefix) must be specified in +the proxymap server's proxy_read_maps or proxy_write_maps +setting (depending on whether the access is read-only or +read-write). +.IP "\fBmemcache (default: inet:localhost:11211)\fR" +The memcache server (note: singular) that Postfix will try +to connect to. For a TCP server specify "inet:" followed by +a hostname or address, ":", and a port name or number. +For a UNIX-domain server specify "unix:" followed by the +socket pathname. Examples: + +.nf + memcache = inet:memcache.example.com + memcache = unix:/path/to/socket +.fi + +NOTE: In the case of a UNIX-domain socket, it must be accessible +by the unprivileged postfix user and by the memcached process. .IP "\fBkey_format (default: %s)\fB" Format of the lookup and update keys in memcache queries. By default, these are the same as the lookup and update @@ -110,45 +140,38 @@ are skipped with a warning). Example: .IP "\fBflags (default: 0)\fR" Optional flags that should be stored along with a memcache update. -.IP "\fBttl (default: 604800)\fR" +.IP "\fBttl (default: 3600)\fR" The expiration time in seconds of memcache updates. -The default is one week. -When using memcache tables with \fBpostscreen\fR(8) or -\fBverify\fR(8), specify a zero *_cache_cleanup_interval -value, and specify the largest \fBpostscreen\fR(8) *_ttl -value or \fBverify\fR(8) *_expire_time value as the memcache -map's \fBttl\fR value. +NOTE 1: When using a memcache table as \fBpostscreen\fR(8) +or \fBverify\fR(8) cache without persistent backup, specify +a zero *_cache_cleanup_interval value with all Postfix +instances that use the memcache, and specify the largest +\fBpostscreen\fR(8) *_ttl value or \fBverify\fR(8) *_expire_time +value as the memcache table's \fBttl\fR value. -Note: according to memcache protocol documentation, a value -greater than 30 days (2592000 seconds) specifies absolute UNIX +NOTE 2: According to memcache protocol documentation, a +value greater than 30 days (2592000 seconds) specifies +absolute UNIX time. Smaller values are relative to the time of the update. .SH BUGS .ad .fi -The Postfix memcache client is based on libmemcache, which -will terminate its process after a memcache server goes -down. To avoid this, set up redundant memcache servers that -have no common source of failure. - The Postfix memcache client cannot be used for security-sensitive tables such as \fBalias_maps\fR (these may contain "\fI|command\fR and "\fI/file/name\fR" destinations), or -\fBvirtual_uid_maps\fR and \fBvirtual_gid_maps\fR (these -specify UNIX process privileges). In a typical deployment -a memcache database is shared via a TCP socket, and is -therefore writable not only by Postfix, but by any process -that can talk to the memcache server. +\fBvirtual_uid_maps\fR, \fBvirtual_gid_maps\fR and +\fBvirtual_mailbox_maps\fR (these specify UNIX process +privileges or "\fI/file/name\fR" destinations). In a typical +deployment a memcache database is writable by any process +that can talk to the memcache server; in contrast, +security-sensitive tables must not be writable by the +unprivileged Postfix user. The Postfix memcache client requires additional configuration -when used with the \fBpostscreen\fR(8) and \fBverify\fR(8) -daemons. For details see the \fBttl\fR parameter discussion -at the end of the MEMCACHE PARAMETERS section in this -document. - -The Postfix memcache client is supported only with libmemcache -version 1.4.0. Some libmemcache features are documented -by reading libmemcache source code, instead a proper API. +when used as \fBpostscreen\fR(8) or \fBverify\fR(8) cache. +For details see the \fBbackup\fR and \fBttl\fR parameter +discussions in the MEMCACHE PARAMETERS section above. .SH "SEE ALSO" .na .nf @@ -177,8 +200,9 @@ The Secure Mailer license must be distributed with this software. .ad .fi The first memcache client for Postfix was written by Omar -Kilani. Besides being implemented on libmemcache, this -implementation bears no resemblance to his work. +Kilani, and was based on libmemcache. +The Postfix implementation does not use libmemcache, and +bears no resemblance to earlier work. .SH "AUTHOR(S)" .na .nf diff --git a/postfix/man/man8/proxymap.8 b/postfix/man/man8/proxymap.8 index 225dc3db4..cbec8c393 100644 --- a/postfix/man/man8/proxymap.8 +++ b/postfix/man/man8/proxymap.8 @@ -75,6 +75,13 @@ The \fImaptype:mapname\fR and \fIflags\fR are the same as with the \fBopen\fR request. .sp This request is supported in Postfix 2.5 and later. +.IP "\fBsequence\fR \fImaptype:mapname flags function\fR" +Iterate over the specified database. The \fIfunction\fR +is one of DICT_SEQ_FUN_FIRST or DICT_SEQ_FUN_NEXT. +The reply is the request completion status code and +a lookup key and result value, if found. +.sp +This request is supported in Postfix 2.9 and later. .PP The request completion status is one of OK, RETRY, NOKEY (lookup failed because the key was not found), BAD (malformed diff --git a/postfix/proto/MEMCACHE_README.html b/postfix/proto/MEMCACHE_README.html index fbf9006c7..62825d6a3 100644 --- a/postfix/proto/MEMCACHE_README.html +++ b/postfix/proto/MEMCACHE_README.html @@ -19,82 +19,44 @@

Introduction

-

The Postfix memcache client type allows you to hook up Postfix to -a memcache server. This implementation supports multiple memcache -servers for redundancy, and multiple memcache clients that you can -use for different table lookups. The Postfix memcache client -supports both lookup and update operations.

+

The Postfix memcache client allows you to hook up Postfix to a +memcache server. The current implementation supports one memcache +server per Postfix table, with one optional Postfix database that +provides persistent backup. The Postfix memcache client supports +the lookup, update, delete and sequence operations. The sequence +(i.e. first/next) operation requires a backup database that supports +this operation.

-

Typically, a memcache map is used to reduce query load on a -database server, or to share a low-latency database among different -Postfix instances.

+

Typically, the Postfix memcache client is used to reduce query +load on a persistent database, but it may also be used to query a +memory-only database for low-value, easy-to-create, information +such as a reputation cache for postscreen(8), verify(8) or greylisting. +

Limitations

Building Postfix with memcache support

-

To build Postfix with memcache client support, specify --DHAS_MEMCACHE, the location of the libmemcache include -files, and the location of the libmemcache object library.

- -

For example:

- -
-
-% make -f Makefile.init makefiles \
-        'CCARGS=-DHAS_MEMCACHE -I/usr/local/include' \
-	'AUXLIBS=-L/usr/local/lib -lmemcache'
-
-
- -

Then run 'make'.

- -

If the build fails with "undefined reference to `mcm_buf_len'" -(and with a similar error message for mcm_buf_remain_off), -then you need to edit libmemcache source code.

- -

The following instructions apply to libmemcache 1.4.0.rc2.

- - - -

Then, continue building Postfix by running 'make'.

+

The Postfix memcache client has no external dependencies, +and is therefore built into Postfix by default.

Configuring memcache lookup tables

@@ -102,11 +64,12 @@ messages.

Credits

-

The first memcache client for Postfix was written by Omar Kilani.

+

The first memcache client for Postfix was written by Omar Kilani, +and was based on the libmemcache library.

-

Wietse wrote a new memcache client from the ground up. Besides -also using libmemcache, the current implementation bears no resemblance -to Omar's work.

+

Wietse wrote the current memcache client from the ground up. +This implementation does not use libmemcache, and bears no resemblance +to earlier work.

diff --git a/postfix/proto/memcache_table b/postfix/proto/memcache_table index 83d687a39..f2beacd7c 100644 --- a/postfix/proto/memcache_table +++ b/postfix/proto/memcache_table @@ -13,8 +13,8 @@ # \fBdbm\fR or \fBdb\fR format. # # Alternatively, lookup tables can be specified as memcache -# instances. In order to use memcache lookups, define a -# memcache source as a lookup table in main.cf, for example: +# instances. To use memcache lookups, define a memcache +# source as a lookup table in main.cf, for example: # # .nf # virtual_alias_maps = memcache:/etc/postfix/memcache-aliases.cf @@ -24,21 +24,51 @@ # format as the Postfix main.cf file, and specifies the # parameters described below. # -# The Postfix memcache client supports the lookup and update -# operations. +# The Postfix memcache client supports the lookup, update, +# delete and sequence (first/next) operations. The sequence +# operation requires a backup database that supports the +# operation. # MEMCACHE PARAMETERS # .ad # .fi -# .IP "\fBhosts (default: localhost:11211)\fR" -# The memcache servers that Postfix will try to connect to. -# Specify a hostname or address, optionally followed by ":" -# and a port name or number. The default port is 11211. -# Examples: +# .IP \fBbackup\fR +# An optional Postfix database that provides persistent backup +# for the memcache database. The Postfix memcache client will +# update the memcache database whenever it looks up or changes +# information in the persistent database. Specify a Postfix +# "type:table" database. Example: # # .nf -# hosts = memcache01.example.com -# memcache02.example.com +# backup = btree:/var/lib/postfix/postscreen_cache_map # .fi +# +# Access to remote proxymap servers is under development. +# +# NOTE 1: When using memcache with persistent backup as +# \fBpostscreen\fR(8) or \fBverify\fR(8) cache, disable +# automatic cache cleanup (*_cache_cleanup_interval = 0) in +# all Postfix instances except for one instance that will be +# responsible for cache cleanup. +# +# NOTE 2: In the case of a proxied database, the full database +# name (including the "proxy:" prefix) must be specified in +# the proxymap server's proxy_read_maps or proxy_write_maps +# setting (depending on whether the access is read-only or +# read-write). +# .IP "\fBmemcache (default: inet:localhost:11211)\fR" +# The memcache server (note: singular) that Postfix will try +# to connect to. For a TCP server specify "inet:" followed by +# a hostname or address, ":", and a port name or number. +# For a UNIX-domain server specify "unix:" followed by the +# socket pathname. Examples: +# +# .nf +# memcache = inet:memcache.example.com +# memcache = unix:/path/to/socket +# .fi +# +# NOTE: In the case of a UNIX-domain socket, it must be accessible +# by the unprivileged postfix user and by the memcached process. # .IP "\fBkey_format (default: %s)\fB" # Format of the lookup and update keys in memcache queries. # By default, these are the same as the lookup and update @@ -102,43 +132,36 @@ # .IP "\fBflags (default: 0)\fR" # Optional flags that should be stored along with a memcache # update. -# .IP "\fBttl (default: 604800)\fR" +# .IP "\fBttl (default: 3600)\fR" # The expiration time in seconds of memcache updates. -# The default is one week. # -# When using memcache tables with \fBpostscreen\fR(8) or -# \fBverify\fR(8), specify a zero *_cache_cleanup_interval -# value, and specify the largest \fBpostscreen\fR(8) *_ttl -# value or \fBverify\fR(8) *_expire_time value as the memcache -# map's \fBttl\fR value. +# NOTE 1: When using a memcache table as \fBpostscreen\fR(8) +# or \fBverify\fR(8) cache without persistent backup, specify +# a zero *_cache_cleanup_interval value with all Postfix +# instances that use the memcache, and specify the largest +# \fBpostscreen\fR(8) *_ttl value or \fBverify\fR(8) *_expire_time +# value as the memcache table's \fBttl\fR value. # -# Note: according to memcache protocol documentation, a value -# greater than 30 days (2592000 seconds) specifies absolute UNIX +# NOTE 2: According to memcache protocol documentation, a +# value greater than 30 days (2592000 seconds) specifies +# absolute UNIX # time. Smaller values are relative to the time of the update. # BUGS -# The Postfix memcache client is based on libmemcache, which -# will terminate its process after a memcache server goes -# down. To avoid this, set up redundant memcache servers that -# have no common source of failure. -# # The Postfix memcache client cannot be used for security-sensitive # tables such as \fBalias_maps\fR (these may contain # "\fI|command\fR and "\fI/file/name\fR" destinations), or -# \fBvirtual_uid_maps\fR and \fBvirtual_gid_maps\fR (these -# specify UNIX process privileges). In a typical deployment -# a memcache database is shared via a TCP socket, and is -# therefore writable not only by Postfix, but by any process -# that can talk to the memcache server. +# \fBvirtual_uid_maps\fR, \fBvirtual_gid_maps\fR and +# \fBvirtual_mailbox_maps\fR (these specify UNIX process +# privileges or "\fI/file/name\fR" destinations). In a typical +# deployment a memcache database is writable by any process +# that can talk to the memcache server; in contrast, +# security-sensitive tables must not be writable by the +# unprivileged Postfix user. # # The Postfix memcache client requires additional configuration -# when used with the \fBpostscreen\fR(8) and \fBverify\fR(8) -# daemons. For details see the \fBttl\fR parameter discussion -# at the end of the MEMCACHE PARAMETERS section in this -# document. -# -# The Postfix memcache client is supported only with libmemcache -# version 1.4.0. Some libmemcache features are documented -# by reading libmemcache source code, instead a proper API. +# when used as \fBpostscreen\fR(8) or \fBverify\fR(8) cache. +# For details see the \fBbackup\fR and \fBttl\fR parameter +# discussions in the MEMCACHE PARAMETERS section above. # SEE ALSO # postmap(1), Postfix lookup table manager # postconf(5), configuration parameters @@ -159,8 +182,9 @@ # .ad # .fi # The first memcache client for Postfix was written by Omar -# Kilani. Besides being implemented on libmemcache, this -# implementation bears no resemblance to his work. +# Kilani, and was based on libmemcache. +# The Postfix implementation does not use libmemcache, and +# bears no resemblance to earlier work. # AUTHOR(S) # Wietse Venema # IBM T.J. Watson Research diff --git a/postfix/src/global/Makefile.in b/postfix/src/global/Makefile.in index 489027763..e771052cd 100644 --- a/postfix/src/global/Makefile.in +++ b/postfix/src/global/Makefile.in @@ -31,7 +31,7 @@ SRCS = abounce.c anvil_clnt.c been_here.c bounce.c bounce_log.c \ fold_addr.c header_body_checks.c mkmap_proxy.c data_redirect.c \ match_service.c mail_conf_nint.c addr_match_list.c mail_conf_nbool.c \ smtp_reply_footer.c safe_ultostr.c verify_sender_addr.c \ - dict_memcache.c mail_version.c + dict_memcache.c mail_version.c memcache_proto.c OBJS = abounce.o anvil_clnt.o been_here.o bounce.o bounce_log.o \ canon_addr.o cfg_parser.o cleanup_strerror.o cleanup_strflags.o \ clnt_stream.o conv_time.o db_common.o debug_peer.o debug_process.o \ @@ -64,7 +64,7 @@ OBJS = abounce.o anvil_clnt.o been_here.o bounce.o bounce_log.o \ fold_addr.o header_body_checks.o mkmap_proxy.o data_redirect.o \ match_service.o mail_conf_nint.o addr_match_list.o mail_conf_nbool.o \ smtp_reply_footer.o safe_ultostr.o verify_sender_addr.o \ - dict_memcache.o mail_version.o + dict_memcache.o mail_version.o memcache_proto.o HDRS = abounce.h anvil_clnt.h been_here.h bounce.h bounce_log.h \ canon_addr.h cfg_parser.h cleanup_user.h clnt_stream.h config.h \ conv_time.h db_common.h debug_peer.h debug_process.h defer.h \ @@ -90,7 +90,7 @@ HDRS = abounce.h anvil_clnt.h been_here.h bounce.h bounce_log.h \ verp_sender.h wildcard_inet_addr.h xtext.h delivered_hdr.h \ fold_addr.h header_body_checks.h data_redirect.h match_service.h \ addr_match_list.h smtp_reply_footer.h safe_ultostr.h \ - verify_sender_addr.h dict_memcache.h + verify_sender_addr.h dict_memcache.h memcache_proto.h TESTSRC = rec2stream.c stream2rec.c recdump.c DEFS = -I. -I$(INC_DIR) -D$(SYSTYPE) CFLAGS = $(DEBUG) $(OPT) $(DEFS) @@ -866,7 +866,7 @@ dict_ldap.o: dict_ldap.h dict_ldap.o: mail_conf.h dict_ldap.o: string_list.h dict_memcache.o: ../../include/argv.h -dict_memcache.o: ../../include/binhash.h +dict_memcache.o: ../../include/auto_clnt.h dict_memcache.o: ../../include/dict.h dict_memcache.o: ../../include/match_list.h dict_memcache.o: ../../include/match_ops.h @@ -881,6 +881,7 @@ dict_memcache.o: cfg_parser.h dict_memcache.o: db_common.h dict_memcache.o: dict_memcache.c dict_memcache.o: dict_memcache.h +dict_memcache.o: memcache_proto.h dict_memcache.o: string_list.h dict_mysql.o: ../../include/argv.h dict_mysql.o: ../../include/dict.h @@ -1470,6 +1471,7 @@ mail_trigger.o: ../../include/vstream.h mail_trigger.o: mail_params.h mail_trigger.o: mail_proto.h mail_trigger.o: mail_trigger.c +mail_version.o: ../../include/msg.h mail_version.o: ../../include/mymalloc.h mail_version.o: ../../include/split_at.h mail_version.o: ../../include/stringops.h @@ -1547,6 +1549,14 @@ mbox_open.o: dsn_buf.h mbox_open.o: mbox_conf.h mbox_open.o: mbox_open.c mbox_open.o: mbox_open.h +memcache_proto.o: ../../include/msg.h +memcache_proto.o: ../../include/sys_defs.h +memcache_proto.o: ../../include/vbuf.h +memcache_proto.o: ../../include/vstream.h +memcache_proto.o: ../../include/vstring.h +memcache_proto.o: ../../include/vstring_vstream.h +memcache_proto.o: memcache_proto.c +memcache_proto.o: memcache_proto.h mime_state.o: ../../include/msg.h mime_state.o: ../../include/mymalloc.h mime_state.o: ../../include/sys_defs.h @@ -1560,15 +1570,7 @@ mime_state.o: mail_params.h mime_state.o: mime_state.c mime_state.o: mime_state.h mime_state.o: rec_type.h -mkmap_cdb.o: ../../include/argv.h -mkmap_cdb.o: ../../include/dict.h -mkmap_cdb.o: ../../include/dict_cdb.h -mkmap_cdb.o: ../../include/mymalloc.h mkmap_cdb.o: ../../include/sys_defs.h -mkmap_cdb.o: ../../include/vbuf.h -mkmap_cdb.o: ../../include/vstream.h -mkmap_cdb.o: ../../include/vstring.h -mkmap_cdb.o: mkmap.h mkmap_cdb.o: mkmap_cdb.c mkmap_db.o: ../../include/argv.h mkmap_db.o: ../../include/dict.h diff --git a/postfix/src/global/dict_memcache.c b/postfix/src/global/dict_memcache.c index ee3c33c6a..8f635f717 100644 --- a/postfix/src/global/dict_memcache.c +++ b/postfix/src/global/dict_memcache.c @@ -2,7 +2,7 @@ /* NAME /* dict_memcache 3 /* SUMMARY -/* dictionary interface to memcache databases +/* dictionary interface to memcaches /* SYNOPSIS /* #include /* @@ -11,7 +11,7 @@ /* int open_flags; /* int dict_flags; /* DESCRIPTION -/* dict_memcache_open() opens a memcache database, providing +/* dict_memcache_open() opens a memcache, providing /* a dictionary interface for Postfix key->value mappings. /* The result is a pointer to the installed dictionary. /* @@ -19,9 +19,7 @@ /* /* Arguments: /* .IP name -/* Either the path to the Postfix memcache configuration file -/* (if it starts with '/' or '.'), or the parameter name prefix -/* which will be used to obtain main.cf configuration parameters. +/* The path to the Postfix memcache configuration file. /* .IP open_flags /* O_RDONLY or O_RDWR. This function ignores flags that don't /* specify a read, write or append mode. @@ -29,14 +27,13 @@ /* See dict_open(3). /* SEE ALSO /* dict(3) generic dictionary manager -/* BUGS -/* This code requires libmemcache 1.4.0, because some parts -/* of their API are documented by looking at the implementation. /* HISTORY -/* The first memcache client for Postfix was written by: -/* Omar Kilani -/* omar@tinysofa.com -/* This implementation bears no resemblance to his work. +/* .ad +/* .fi +/* The first memcache client for Postfix was written by Omar +/* Kilani, and was based on libmemcache. The current +/* implementation implements the memcache protocol directly, +/* and bears no resemblance to earlier work. /* AUTHOR(S) /* Wietse Venema /* IBM T.J. Watson Research @@ -46,15 +43,10 @@ /* System library. */ -#include "sys_defs.h" - -#ifdef HAS_MEMCACHE +#include #include -#include - -#if !defined(MEMCACHE_VERNUM) || MEMCACHE_VERNUM != 10400 -#error "Postfix memcache supports only libmemcache version 1.4.0" -#endif +#include +#include /* XXX sscanf() */ /* Utility library. */ @@ -63,50 +55,39 @@ #include #include #include -#include +#include +#include /* Global library. */ #include #include +#include /* Application-specific. */ #include - /* - * Robustness tests (with a single memcache server) proved disappointing. - * - * After failure to connect to the memcache server, libmemcache reports the - * error once. From then on it silently discards all updates and always - * reports "not found" for all lookups, without ever reporting an error. To - * avoid this, we destroy the memcache client and create a new one after - * libmemcache reports an error. - * - * Even more problematic is that libmemcache will terminate the process when - * the memcache server connection is lost (the libmemcache error message is: - * "read(2) failed: Socket is already connected"). Unfortunately, telling - * libmemcache not to terminate the process will result in an assertion - * failure followed by core dump. - * - * Conclusion: if we want robust code, then we should use our own memcache - * protocol implementation instead of libmemcache. - */ - /* * Structure of one memcache dictionary handle. */ typedef struct { DICT dict; /* parent class */ - struct memcache_ctxt *mc_ctxt; /* libmemcache context */ - struct memcache *mc; /* libmemcache object */ CFG_PARSER *parser; /* common parameter parser */ void *dbc_ctxt; /* db_common context */ char *key_format; /* query key translation */ + int timeout; /* client timeout */ int mc_ttl; /* memcache expiration */ int mc_flags; /* memcache flags */ + int mc_pause; /* sleep between errors */ + int mc_maxtry; /* number of tries */ + char *memcache; /* memcache server spec */ + AUTO_CLNT *clnt; /* memcache client stream */ + VSTRING *clnt_buf; /* memcache client buffer */ VSTRING *key_buf; /* lookup key */ VSTRING *res_buf; /* lookup result */ + int mc_errno; /* memcache dict_errno */ + DICT *backup; /* persistent backup */ } DICT_MC; /* @@ -114,23 +95,13 @@ typedef struct { */ #define DICT_MC_DEF_HOST "localhost" #define DICT_MC_DEF_PORT "11211" -#define DICT_MC_DEF_HOST_PORT DICT_MC_DEF_HOST ":" DICT_MC_DEF_PORT +#define DICT_MC_DEF_MEMCACHE "inet:" DICT_MC_DEF_HOST ":" DICT_MC_DEF_PORT #define DICT_MC_DEF_KEY_FMT "%s" -#define DICT_MC_DEF_TTL (7 * 86400) -#define DICT_MC_DEF_FLAGS 0 - - /* - * libmemcache can report errors through an application call-back function, - * but there is no support for passing application context to the call-back. - * The call-back API has two documented arguments: pointer to memcache_ctxt, - * and pointer to memcache_ectxt. The memcache_ctxt data structure has no - * space for application context, and the mcm_err() function zero-fills the - * memcache_ectxt data structure, making it useless for application context. - * - * We use our own hash table to find our dictionary handle, so that we can - * report errors in the proper context. - */ -static BINHASH *dict_mc_hash; +#define DICT_MC_DEF_MC_TTL 3600 +#define DICT_MC_DEF_MC_TIMEOUT 2 +#define DICT_MC_DEF_MC_FLAGS 0 +#define DICT_MC_DEF_MC_MAXTRY 2 +#define DICT_MC_DEF_MC_PAUSE 1 /* * SLMs. @@ -138,77 +109,94 @@ static BINHASH *dict_mc_hash; #define STR(x) vstring_str(x) #define LEN(x) VSTRING_LEN(x) -/*#define msg_verbose 1*/ +#define msg_verbose 1 -/* dict_memcache_error_cb - error call-back */ +/* dict_memcache_set - set memcache key/value */ -static int dict_memcache_error_cb(MCM_ERR_FUNC_ARGS) +static void dict_memcache_set(DICT_MC *dict_mc, const char *value, int ttl) { - const char *myname = "dict_memcache_error_cb"; - const struct memcache_ctxt *ctxt; - struct memcache_err_ctxt *ectxt; - DICT_MC *dict_mc; - void (*log_fn) (const char *,...); + VSTREAM *fp; + int count; - /* - * Play by the rules of the libmemcache API. - */ - MCM_ERR_INIT_CTXT(ctxt, ectxt); +#define MC_LINE_LIMIT 1024 - /* - * Locate our own dictionary handle for error reporting context. - * Unfortunately, the ctxt structure does not store application context, - * and mcm_err() zero-fills the ectxt structure, making it useless for - * storing application context. We use our own hash table instead. - */ - if ((dict_mc = (DICT_MC *) binhash_find(dict_mc_hash, (char *) &ctxt, - sizeof(ctxt))) == 0) - msg_panic("%s: can't locate DICT_MC database handle", myname); - - /* - * Report the error in our context, and set dict_errno for possible - * errors. We override dict_errno when an error was recoverable. - */ - switch (ectxt->severity) { - default: -#ifdef DICT_MC_RECOVER_FROM_DISCONNECT - /* Code below causes an assert failure and core dump. */ - if (ectxt->errcode == MCM_ERR_SYS_READ) - /* Also: MCM_ERR_SYS_WRITEV, MCM_ERR_SYS_SETSOCKOPT */ - ectxt->cont = 'y'; -#endif - /* FALLTHROUGH */ - case MCM_ERR_LVL_NOTICE: - log_fn = msg_warn; - dict_errno = 1; - break; - case MCM_ERR_LVL_INFO: - log_fn = msg_info; - break; + dict_mc->mc_errno = DICT_ERR_RETRY; + for (count = 0; count < dict_mc->mc_maxtry; count++) { + if (count > 0) + sleep(1); + if ((fp = auto_clnt_access(dict_mc->clnt)) != 0) { + if (memcache_printf(fp, "set %s %d %d %ld", + STR(dict_mc->key_buf), dict_mc->mc_flags, + ttl, strlen(value)) < 0 + || memcache_fwrite(fp, value, strlen(value)) < 0 + || memcache_get(fp, dict_mc->clnt_buf, MC_LINE_LIMIT) < 0) { + if (count > 0) + msg_warn("database %s:%s: I/O error: %m", + DICT_TYPE_MEMCACHE, dict_mc->dict.name); + auto_clnt_recover(dict_mc->clnt); + } else if (strcmp(STR(dict_mc->clnt_buf), "STORED") != 0) { + if (count > 0) + msg_warn("database %s:%s: update failed: %.30s", + DICT_TYPE_MEMCACHE, dict_mc->dict.name, + STR(dict_mc->clnt_buf)); + auto_clnt_recover(dict_mc->clnt); + } else { + /* Victory! */ + dict_mc->mc_errno = 0; + break; + } + } } - log_fn((ectxt)->errnum ? "database %s:%s: libmemcache error: %s: %m" : - "database %s:%s: libmemcache error: %s", - DICT_TYPE_MEMCACHE, dict_mc->dict.name, (ectxt)->errstr); - return (0); } -static void dict_memcache_mc_free(DICT_MC *); -static void dict_memcache_mc_init(DICT_MC *); +/* dict_memcache_get - get memcache key/value */ -/* dict_memcache_recover - recover after libmemcache error */ - -static void dict_memcache_recover(DICT_MC *dict_mc) +static const char *dict_memcache_get(DICT_MC *dict_mc) { - int saved_dict_errno; + VSTREAM *fp; + long todo; + const char *retval; + int count; - /* - * XXX If we don't try to recover from the first error, libmemcache will - * silently skip all subsequent database operations. - */ - saved_dict_errno = dict_errno; - dict_memcache_mc_free(dict_mc); - dict_memcache_mc_init(dict_mc); - dict_errno = saved_dict_errno; + dict_mc->mc_errno = DICT_ERR_RETRY; + retval = 0; + for (count = 0; count < dict_mc->mc_maxtry; count++) { + if (count > 0) + sleep(1); + if ((fp = auto_clnt_access(dict_mc->clnt)) != 0) { + if (memcache_printf(fp, "get %s", STR(dict_mc->key_buf)) < 0 + || memcache_get(fp, dict_mc->clnt_buf, MC_LINE_LIMIT) < 0) { + if (count > 0) + msg_warn("database %s:%s: I/O error: %m", + DICT_TYPE_MEMCACHE, dict_mc->dict.name); + auto_clnt_recover(dict_mc->clnt); + } else if (strcmp(STR(dict_mc->clnt_buf), "END") == 0) { + /* Not found. */ + dict_mc->mc_errno = 0; + break; + } else if (sscanf(STR(dict_mc->clnt_buf), + "VALUE %*s %*s %ld", &todo) != 1 || todo < 0) { + if (count > 0) + msg_warn("%s: unexpected memcache server reply: %.30s", + dict_mc->dict.name, STR(dict_mc->clnt_buf)); + auto_clnt_recover(dict_mc->clnt); + } else if (memcache_fread(fp, dict_mc->res_buf, todo) < 0) { + if (count > 0) + msg_warn("%s: EOF receiving memcache server reply", + dict_mc->dict.name); + auto_clnt_recover(dict_mc->clnt); + } else { + /* Victory! */ + retval = STR(dict_mc->res_buf); + dict_mc->mc_errno = 0; + if (memcache_get(fp, dict_mc->clnt_buf, MC_LINE_LIMIT) < 0 + || strcmp(STR(dict_mc->clnt_buf), "END") != 0) + auto_clnt_recover(dict_mc->clnt); + break; + } + } + } + return (retval); } /* dict_memcache_prepare_key - prepare lookup key */ @@ -249,189 +237,204 @@ static int dict_memcache_prepare_key(DICT_MC *dict_mc, const char *name) return (LEN(dict_mc->key_buf)); } -/* dict_memcache_update - update memcache database */ +/* dict_memcache_valid_key - validate key */ + +static int dict_memcache_valid_key(DICT_MC *dict_mc, + const char *name, + const char *operation, + void (*log_func) (const char *,...)) +{ + unsigned char *cp; + +#define DICT_MC_SKIP(why) do { \ + if (msg_verbose || log_func != msg_info) \ + log_func("%s: skipping %s for name \"%s\": %s", \ + dict_mc->dict.name, operation, name, (why)); \ + return(0); \ + } while (0) + + if (*name == 0) + DICT_MC_SKIP("empty lookup key"); + if (db_common_check_domain(dict_mc->dbc_ctxt, name) == 0) + DICT_MC_SKIP("domain mismatch"); + if (dict_memcache_prepare_key(dict_mc, name) == 0) + DICT_MC_SKIP("empty lookup key expansion"); + for (cp = (unsigned char *) STR(dict_mc->key_buf); *cp; cp++) + if (isascii(*cp) && isspace(*cp)) + DICT_MC_SKIP("name contains space"); + + return (1); +} + +/* dict_memcache_update - update memcache */ static void dict_memcache_update(DICT *dict, const char *name, const char *value) { const char *myname = "dict_memcache_update"; DICT_MC *dict_mc = (DICT_MC *) dict; + int backup_errno = 0; /* - * Skip updates with a null key, noisily. This would result in loss of - * information. + * Skip updates with an inapplicable key, noisily. This results in loss + * of information. */ - if (dict_memcache_prepare_key(dict_mc, name) == 0) { - dict_errno = 1; - msg_warn("database %s:%s: name \"%s\" expands to empty lookup key " - "-- skipping update", DICT_TYPE_MEMCACHE, - dict_mc->dict.name, name); + dict_errno = DICT_ERR_RETRY; + if (dict_memcache_valid_key(dict_mc, name, "update", msg_warn) == 0) return; + + /* + * Update the backup database first. + */ + if (dict_mc->backup) { + dict_errno = 0; + dict_mc->backup->update(dict_mc->backup, name, value); + backup_errno = dict_errno; } /* - * Our error call-back routine will report errors and set dict_errno. + * Update the memcache last. */ - dict_errno = (mcm_set(dict_mc->mc_ctxt, dict_mc->mc, STR(dict_mc->key_buf), - LEN(dict_mc->key_buf), value, strlen(value), - dict_mc->mc_ttl, dict_mc->mc_flags) != 0); + dict_memcache_set(dict_mc, value, dict_mc->mc_ttl); + if (msg_verbose) msg_info("%s: %s: update key \"%s\" => \"%s\" %s", myname, dict_mc->dict.name, STR(dict_mc->key_buf), value, - dict_errno ? "(error)" : "(no error)"); + dict_mc->mc_errno ? "(memcache error)" : + backup_errno ? "(backup error)" : "(no error)"); - /* - * Recover after server failure. - */ - if (dict_errno) - dict_memcache_recover(dict_mc); + dict_errno = (backup_errno ? backup_errno : dict_mc->mc_errno); } -/* dict_memcache_lookup - lookup memcache database */ +/* dict_memcache_lookup - lookup memcache */ static const char *dict_memcache_lookup(DICT *dict, const char *name) { const char *myname = "dict_memcache_lookup"; DICT_MC *dict_mc = (DICT_MC *) dict; - struct memcache_req *req; - struct memcache_res *res; const char *retval; + int backup_errno = 0; /* - * Skip lookups with a null key, silently. This is just asking for - * information that cannot exist. + * Skip lookups with an inapplicable key, silently. This is just asking + * for information that cannot exist. */ -#define DICT_MC_SKIP(why, map_name, key) do { \ - if (msg_verbose) \ - msg_info("%s: %s: skipping lookup of key \"%s\": %s", \ - myname, (map_name), (key), (why)); \ - return (0); \ - } while (0) - - if (*name == 0) - DICT_MC_SKIP("empty lookup key", dict_mc->dict.name, name); - if (db_common_check_domain(dict_mc->dbc_ctxt, name) == 0) - DICT_MC_SKIP("domain mismatch", dict_mc->dict.name, name); - if (dict_memcache_prepare_key(dict_mc, name) == 0) - DICT_MC_SKIP("empty lookup key expansion", dict_mc->dict.name, name); - - /* - * Our error call-back routine will report errors and set dict_errno. We - * reset dict_errno after an error turns out to be recoverable. - */ - if ((req = mcm_req_new(dict_mc->mc_ctxt)) == 0) - msg_fatal("%s: can't create new request: %m", myname); /* XXX */ - /* Not: mcm_req_add(), because that makes unnecessary copy of the key. */ - if ((res = mcm_req_add_ref(dict_mc->mc_ctxt, req, STR(dict_mc->key_buf), - LEN(dict_mc->key_buf))) == 0) - msg_fatal("%s: can't create new result: %m", myname); /* XXX */ - dict_errno = 0; - mcm_get(dict_mc->mc_ctxt, dict_mc->mc, req); - if (mcm_res_found(dict_mc->mc_ctxt, res) && res->bytes) { - vstring_strncpy(dict_mc->res_buf, res->val, res->bytes); - retval = STR(dict_mc->res_buf); - dict_errno = 0; - } else { - retval = 0; - } - mcm_res_free(dict_mc->mc_ctxt, req, res); - mcm_req_free(dict_mc->mc_ctxt, req); + if (dict_memcache_valid_key(dict_mc, name, "lookup", msg_info) == 0) + return (0); + /* + * Search the memcache first. + */ + retval = dict_memcache_get(dict_mc); + + /* + * Search the backup database last. Update the memcache if the data is + * found. + */ + if (retval == 0 && dict_mc->backup) { + retval = dict_mc->backup->lookup(dict_mc->backup, name); + backup_errno = dict_errno; + /* Update the cache. */ + if (retval != 0) + dict_memcache_set(dict_mc, retval, dict_mc->mc_ttl); + } if (msg_verbose) msg_info("%s: %s: key %s => %s", myname, dict_mc->dict.name, STR(dict_mc->key_buf), - retval ? STR(dict_mc->res_buf) : - dict_errno ? "(error)" : "(not found)"); - - /* - * Recover after server failure. - */ - if (dict_errno) - dict_memcache_recover(dict_mc); + retval ? retval : + dict_mc->mc_errno ? "(memcache error)" : + backup_errno ? "(backup error)" : "(not found)"); return (retval); } -/* dict_memcache_mc_free - destroy libmemcache objects */ +/* dict_memcache_delete - delete memcache entry */ -static void dict_memcache_mc_free(DICT_MC *dict_mc) +static int dict_memcache_delete(DICT *dict, const char *name) { - binhash_delete(dict_mc_hash, (char *) &dict_mc->mc_ctxt, - sizeof(dict_mc->mc_ctxt), (void (*) (char *)) 0); - mcm_free(dict_mc->mc_ctxt, dict_mc->mc); - mcMemFreeCtxt(dict_mc->mc_ctxt); -} - -/* dict_memcache_mc_init - create libmemcache objects */ - -static void dict_memcache_mc_init(DICT_MC *dict_mc) -{ - const char *myname = "dict_memcache_mc_init"; - char *servers; - char *server; - char *cp; + const char *myname = "dict_memcache_delete"; + DICT_MC *dict_mc = (DICT_MC *) dict; + const char *retval; + int backup_errno = 0; + int del_res = 0; /* - * Create the libmemcache objects. + * Skip lookups with an inapplicable key, silently. This is just deleting + * information that cannot exist. */ - dict_mc->mc_ctxt = - mcMemNewCtxt((mcFreeFunc) myfree, (mcMallocFunc) mymalloc, - (mcMallocFunc) mymalloc, (mcReallocFunc) myrealloc); - if (dict_mc->mc_ctxt == 0) - msg_fatal("error creating memcache context: %m"); /* XXX */ - dict_mc->mc = mcm_new(dict_mc->mc_ctxt); - if (dict_mc->mc == 0) - msg_fatal("error creating memcache object: %m"); /* XXX */ + dict_errno = 0; + if (dict_memcache_valid_key(dict_mc, name, "delete", msg_info) == 0) + return (1); /* - * Set up call-back info for error reporting. + * Update the persistent database first. */ - if (dict_mc_hash == 0) - dict_mc_hash = binhash_create(1); - binhash_enter(dict_mc_hash, (char *) &dict_mc->mc_ctxt, - sizeof(dict_mc->mc_ctxt), (char *) dict_mc); - mcErrSetupCtxt(dict_mc->mc_ctxt, dict_memcache_error_cb); - - /* - * Add the server list. - */ - cp = servers = cfg_get_str(dict_mc->parser, "hosts", - DICT_MC_DEF_HOST_PORT, 0, 0); - while ((server = mystrtok(&cp, " ,\t\r\n")) != 0) { - if (msg_verbose) - msg_info("%s: database %s:%s: adding server %s", - myname, DICT_TYPE_MEMCACHE, dict_mc->dict.name, server); - if (mcm_server_add4(dict_mc->mc_ctxt, dict_mc->mc, server) < 0) - msg_warn("database %s:%s: error adding server %s", - DICT_TYPE_MEMCACHE, dict_mc->dict.name, server); + if (dict_mc->backup) { + dict_errno = 0; + del_res = dict_mc->backup->delete(dict_mc->backup, name); + backup_errno = dict_errno; } - myfree(servers); + + /* + * Update the memcache last. There is no memcache delete operation. + * Instead, we set a short expiration time if the data exists. + */ + if ((retval = dict_memcache_get(dict_mc)) != 0) + dict_memcache_set(dict_mc, retval, 1); + + if (msg_verbose) + msg_info("%s: %s: delete key %s => %s", + myname, dict_mc->dict.name, STR(dict_mc->key_buf), + dict_mc->mc_errno ? "(memcache error)" : + backup_errno ? "(backup error)" : "(no error)"); + + dict_errno = (backup_errno ? backup_errno : dict_mc->mc_errno); + + return (del_res); } -/* dict_memcache_close - close memcache database */ +/* dict_memcache_sequence - first/next lookup */ + +static int dict_memcache_sequence(DICT *dict, int function, const char **key, + const char **value) +{ + DICT_MC *dict_mc = (DICT_MC *) dict; + + if (dict_mc->backup == 0) + msg_fatal("database %s:%s: first/next support requires backup database", + DICT_TYPE_MEMCACHE, dict_mc->dict.name); + return (dict_mc->backup->sequence(dict_mc->backup, function, key, value)); +} + +/* dict_memcache_close - close memcache */ static void dict_memcache_close(DICT *dict) { DICT_MC *dict_mc = (DICT_MC *) dict; - dict_memcache_mc_free(dict_mc); cfg_parser_free(dict_mc->parser); db_common_free_ctx(dict_mc->dbc_ctxt); - vstring_free(dict_mc->key_buf); - vstring_free(dict_mc->res_buf); if (dict_mc->key_format) myfree(dict_mc->key_format); + myfree(dict_mc->memcache); + auto_clnt_free(dict_mc->clnt); + vstring_free(dict_mc->clnt_buf); + vstring_free(dict_mc->key_buf); + vstring_free(dict_mc->res_buf); if (dict->fold_buf) vstring_free(dict->fold_buf); + if (dict_mc->backup) + dict_close(dict_mc->backup); dict_free(dict); } -/* dict_memcache_open - open memcache database */ +/* dict_memcache_open - open memcache */ DICT *dict_memcache_open(const char *name, int open_flags, int dict_flags) { DICT_MC *dict_mc; + char *backup; /* * Sanity checks. @@ -450,8 +453,11 @@ DICT *dict_memcache_open(const char *name, int open_flags, int dict_flags) dict_mc = (DICT_MC *) dict_alloc(DICT_TYPE_MEMCACHE, name, sizeof(*dict_mc)); dict_mc->dict.lookup = dict_memcache_lookup; - if (open_flags == O_RDWR) + if (open_flags == O_RDWR) { dict_mc->dict.update = dict_memcache_update; + dict_mc->dict.delete = dict_memcache_delete; + } + dict_mc->dict.sequence = dict_memcache_sequence; dict_mc->dict.close = dict_memcache_close; dict_mc->dict.flags = dict_flags; dict_mc->key_buf = vstring_alloc(10); @@ -463,15 +469,34 @@ DICT *dict_memcache_open(const char *name, int open_flags, int dict_flags) dict_mc->parser = cfg_parser_alloc(name); dict_mc->key_format = cfg_get_str(dict_mc->parser, "key_format", DICT_MC_DEF_KEY_FMT, 0, 0); + dict_mc->timeout = cfg_get_int(dict_mc->parser, "timeout", + DICT_MC_DEF_MC_TIMEOUT, 0, 0); dict_mc->mc_ttl = cfg_get_int(dict_mc->parser, "ttl", - DICT_MC_DEF_TTL, 0, 0); + DICT_MC_DEF_MC_TTL, 0, 0); dict_mc->mc_flags = cfg_get_int(dict_mc->parser, "flags", - DICT_MC_DEF_FLAGS, 0, 0); + DICT_MC_DEF_MC_FLAGS, 0, 0); + dict_mc->mc_pause = cfg_get_int(dict_mc->parser, "error_pause", + DICT_MC_DEF_MC_PAUSE, 1, 0); + dict_mc->mc_maxtry = cfg_get_int(dict_mc->parser, "maxtry", + DICT_MC_DEF_MC_MAXTRY, 1, 0); + dict_mc->memcache = cfg_get_str(dict_mc->parser, "memcache", + DICT_MC_DEF_MEMCACHE, 0, 0); /* - * Initialize the memcache objects. + * Initialize the memcache client. */ - dict_memcache_mc_init(dict_mc); + dict_mc->clnt = auto_clnt_create(dict_mc->memcache, dict_mc->timeout, 0, 0); + dict_mc->clnt_buf = vstring_alloc(100); + + /* + * Open the optional backup database. + */ + backup = cfg_get_str(dict_mc->parser, "backup", (char *) 0, 0, 0); + if (backup) { + dict_mc->backup = dict_open(backup, open_flags, dict_flags); + myfree(backup); + } else + dict_mc->backup = 0; /* * Parse templates and common database parameters. Maps that use @@ -489,5 +514,3 @@ DICT *dict_memcache_open(const char *name, int open_flags, int dict_flags) return (&dict_mc->dict); } - -#endif diff --git a/postfix/src/global/dict_proxy.c b/postfix/src/global/dict_proxy.c index 78a5dd73e..e09399cab 100644 --- a/postfix/src/global/dict_proxy.c +++ b/postfix/src/global/dict_proxy.c @@ -11,11 +11,11 @@ /* int open_flags; /* int dict_flags; /* DESCRIPTION -/* dict_proxy_open() relays read-only operations through -/* the Postfix proxymap server. +/* dict_proxy_open() relays read-only or read-write operations +/* through the Postfix proxymap server. /* /* The \fIopen_flags\fR argument must specify O_RDONLY -/* or O_RDWR|O_CREAT. Depending on this, the client +/* or O_RDWR. Depending on this, the client /* connects to the proxymap multiserver or to the /* proxywrite single updater. /* @@ -73,6 +73,7 @@ typedef struct { CLNT_STREAM *clnt; /* client handle (shared) */ const char *service; /* service name */ int in_flags; /* caller-specified flags */ + VSTRING *reskey; /* result key storage */ VSTRING *result; /* storage */ } DICT_PROXY; @@ -88,6 +89,86 @@ typedef struct { static CLNT_STREAM *proxymap_stream; /* read-only maps */ static CLNT_STREAM *proxywrite_stream; /* read-write maps */ +/* dict_proxy_sequence - find first/next entry */ + +static int dict_proxy_sequence(DICT *dict, int function, + const char **key, const char **value) +{ + const char *myname = "dict_proxy_sequence"; + DICT_PROXY *dict_proxy = (DICT_PROXY *) dict; + VSTREAM *stream; + int status; + int count = 0; + int request_flags; + + /* + * The client and server live in separate processes that may start and + * terminate independently. We cannot rely on a persistent connection, + * let alone on persistent state (such as a specific open table) that is + * associated with a specific connection. Each lookup needs to specify + * the table and the flags that were specified to dict_proxy_open(). + */ + VSTRING_RESET(dict_proxy->reskey); + VSTRING_TERMINATE(dict_proxy->reskey); + VSTRING_RESET(dict_proxy->result); + VSTRING_TERMINATE(dict_proxy->result); + request_flags = (dict_proxy->in_flags & DICT_FLAG_RQST_MASK) + | (dict->flags & DICT_FLAG_RQST_MASK); + for (;;) { + stream = clnt_stream_access(dict_proxy->clnt); + errno = 0; + count += 1; + if (attr_print(stream, ATTR_FLAG_NONE, + ATTR_TYPE_STR, MAIL_ATTR_REQ, PROXY_REQ_SEQUENCE, + ATTR_TYPE_STR, MAIL_ATTR_TABLE, dict->name, + ATTR_TYPE_INT, MAIL_ATTR_FLAGS, request_flags, + ATTR_TYPE_INT, MAIL_ATTR_FUNC, function, + ATTR_TYPE_END) != 0 + || vstream_fflush(stream) + || attr_scan(stream, ATTR_FLAG_STRICT, + ATTR_TYPE_INT, MAIL_ATTR_STATUS, &status, + ATTR_TYPE_STR, MAIL_ATTR_KEY, dict_proxy->reskey, + ATTR_TYPE_STR, MAIL_ATTR_VALUE, dict_proxy->result, + ATTR_TYPE_END) != 3) { + if (msg_verbose || count > 1 || (errno && errno != EPIPE && errno != ENOENT)) + msg_warn("%s: service %s: %m", myname, VSTREAM_PATH(stream)); + } else { + if (msg_verbose) + msg_info("%s: table=%s flags=%s func=%d -> status=%d key=%s val=%s", + myname, dict->name, dict_flags_str(request_flags), + function, status, STR(dict_proxy->reskey), + STR(dict_proxy->result)); + switch (status) { + case PROXY_STAT_BAD: + msg_fatal("%s sequence failed for table \"%s\" function %d: " + "invalid request", + dict_proxy->service, dict->name, function); + case PROXY_STAT_DENY: + msg_fatal("%s service is not configured for table \"%s\"", + dict_proxy->service, dict->name); + case PROXY_STAT_OK: + *key = STR(dict_proxy->reskey); + *value = STR(dict_proxy->result); + return (0); + case PROXY_STAT_NOKEY: + dict_errno = 0; + *key = *value = 0; + return (1); + case PROXY_STAT_RETRY: + dict_errno = DICT_ERR_RETRY; + *key = *value = 0; + return (1); + default: + msg_warn("%s sequence failed for table \"%s\" function %d: " + "unexpected reply status %d", + dict_proxy->service, dict->name, function, status); + } + } + clnt_stream_recover(dict_proxy->clnt); + sleep(1); /* XXX make configurable */ + } +} + /* dict_proxy_lookup - find table entry */ static const char *dict_proxy_lookup(DICT *dict, const char *key) @@ -212,6 +293,9 @@ static void dict_proxy_update(DICT *dict, const char *key, const char *value) dict_proxy->service, dict->name); case PROXY_STAT_OK: return; + case PROXY_STAT_RETRY: + dict_errno = DICT_ERR_RETRY; + return; default: msg_warn("%s update failed for table \"%s\" key \"%s\": " "unexpected reply status %d", @@ -275,8 +359,12 @@ static int dict_proxy_delete(DICT *dict, const char *key) dict_proxy->service, dict->name); case PROXY_STAT_OK: return 0; + case PROXY_STAT_RETRY: + dict_errno = DICT_ERR_RETRY; + return (-1); case PROXY_STAT_NOKEY: - return 1; + dict_errno = 0; + return (1); default: msg_warn("%s delete failed for table \"%s\" key \"%s\": " "unexpected reply status %d", @@ -294,6 +382,7 @@ static void dict_proxy_close(DICT *dict) { DICT_PROXY *dict_proxy = (DICT_PROXY *) dict; + vstring_free(dict_proxy->reskey); vstring_free(dict_proxy->result); dict_free(dict); } @@ -334,11 +423,11 @@ DICT *dict_proxy_open(const char *map, int open_flags, int dict_flags) if (open_flags == O_RDONLY) { pstream = &proxymap_stream; service = var_proxymap_service; - } else if (open_flags == (O_RDWR | O_CREAT)) { + } else if ((open_flags & O_RDWR) == O_RDWR) { pstream = &proxywrite_stream; service = var_proxywrite_service; } else - msg_fatal("%s: %s map open requires O_RDONLY or O_RDWR|O_CREAT mode", + msg_fatal("%s: %s map open requires O_RDONLY or O_RDWR mode", map, DICT_TYPE_PROXY); if (*pstream == 0) { @@ -364,8 +453,10 @@ DICT *dict_proxy_open(const char *map, int open_flags, int dict_flags) dict_proxy->dict.lookup = dict_proxy_lookup; dict_proxy->dict.update = dict_proxy_update; dict_proxy->dict.delete = dict_proxy_delete; + dict_proxy->dict.sequence = dict_proxy_sequence; dict_proxy->dict.close = dict_proxy_close; dict_proxy->in_flags = dict_flags; + dict_proxy->reskey = vstring_alloc(10); dict_proxy->result = vstring_alloc(10); dict_proxy->clnt = *pstream; dict_proxy->service = service; diff --git a/postfix/src/global/dict_proxy.h b/postfix/src/global/dict_proxy.h index 6d55842d4..80dd6e352 100644 --- a/postfix/src/global/dict_proxy.h +++ b/postfix/src/global/dict_proxy.h @@ -30,6 +30,7 @@ extern DICT *dict_proxy_open(const char *, int, int); #define PROXY_REQ_LOOKUP "lookup" #define PROXY_REQ_UPDATE "update" #define PROXY_REQ_DELETE "delete" +#define PROXY_REQ_SEQUENCE "sequence" #define PROXY_STAT_OK 0 /* operation succeeded */ #define PROXY_STAT_NOKEY 1 /* requested key not found */ diff --git a/postfix/src/global/mail_dict.c b/postfix/src/global/mail_dict.c index 3d456ea7d..6dbf6eecc 100644 --- a/postfix/src/global/mail_dict.c +++ b/postfix/src/global/mail_dict.c @@ -59,9 +59,7 @@ static const DICT_OPEN_INFO dict_open_info[] = { #ifdef HAS_SQLITE DICT_TYPE_SQLITE, dict_sqlite_open, #endif -#ifdef HAS_MEMCACHE DICT_TYPE_MEMCACHE, dict_memcache_open, -#endif 0, }; @@ -76,12 +74,19 @@ void mail_dict_init(void) } #ifdef TEST - /* * Proof-of-concept test program. */ + +#include +#include + int main(int argc, char **argv) { + var_queue_dir = DEF_QUEUE_DIR; + var_proxymap_service = DEF_PROXYMAP_SERVICE; + var_proxywrite_service = DEF_PROXYWRITE_SERVICE; + var_ipc_timeout = 3600; mail_dict_init(); dict_test(argc, argv); return (0); diff --git a/postfix/src/global/mail_proto.h b/postfix/src/global/mail_proto.h index da5dc31af..66f6ff521 100644 --- a/postfix/src/global/mail_proto.h +++ b/postfix/src/global/mail_proto.h @@ -141,6 +141,7 @@ extern char *mail_pathname(const char *, const char *); #define MAIL_ATTR_TTL "ttl" #define MAIL_ATTR_LABEL "label" #define MAIL_ATTR_PROP "property" +#define MAIL_ATTR_FUNC "function" #define MAIL_ATTR_CCERT_SUBJECT "ccert_subject" #define MAIL_ATTR_CCERT_ISSUER "ccert_issuer" #define MAIL_ATTR_CCERT_FINGERPRINT "ccert_fingerprint" diff --git a/postfix/src/global/mail_version.h b/postfix/src/global/mail_version.h index 2f815b661..f35591f45 100644 --- a/postfix/src/global/mail_version.h +++ b/postfix/src/global/mail_version.h @@ -20,7 +20,7 @@ * Patches change both the patchlevel and the release date. Snapshots have no * patchlevel; they change the release date only. */ -#define MAIL_RELEASE_DATE "20111209" +#define MAIL_RELEASE_DATE "20111213" #define MAIL_VERSION_NUMBER "2.9" #ifdef SNAPSHOT diff --git a/postfix/src/global/memcache_proto.c b/postfix/src/global/memcache_proto.c new file mode 100644 index 000000000..9d7e986b7 --- /dev/null +++ b/postfix/src/global/memcache_proto.c @@ -0,0 +1,202 @@ +/*++ +/* NAME +/* memcache_proto 3 +/* SUMMARY +/* memcache low-level protocol +/* SYNOPSIS +/* #include +/* +/* int memcache_get(fp, buf, len) +/* VSTREAM *fp; +/* VSTRING *buf; +/* ssize_t len; +/* +/* int memcache_printf(fp, format, ...) +/* VSTREAM *fp; +/* const char *format; +/* +/* int memcache_vprintf(fp, format, ap) +/* VSTREAM *fp; +/* const char *format; +/* va_list ap; +/* +/* int memcache_fread(fp, buf, len) +/* VSTREAM *fp; +/* VSTRING *buf; +/* ssize_t len; +/* +/* int memcache_fwrite(fp, buf, len) +/* VSTREAM *fp; +/* const char *buf; +/* ssize_t len; +/* DESCRIPTION +/* This module implements the low-level memcache protocol. +/* All functions return -1 on error and 0 on succcess. +/* SEE ALSO +/* smtp_proto(3) SMTP low-level protocol. +/* AUTHOR(S) +/* Wietse Venema +/* IBM T.J. Watson Research +/* P.O. Box 704 +/* Yorktown Heights, NY 10598, USA +/*--*/ + +#include + +/* Utility library. */ + +#include +#include +#include +#include + +/* Application-specific. */ + +#include + +#define STR(x) vstring_str(x) +#define LEN(x) VSTRING_LEN(x) + +/* memcache_get - read one line from peer */ + +int memcache_get(VSTREAM *stream, VSTRING *vp, ssize_t bound) +{ + int last_char; + int next_char; + + last_char = (bound == 0 ? vstring_get(vp, stream) : + vstring_get_bound(vp, stream, bound)); + + switch (last_char) { + + /* + * Do some repair in the rare case that we stopped reading in the + * middle of the CRLF record terminator. + */ + case '\r': + if ((next_char = VSTREAM_GETC(stream)) == '\n') { + VSTRING_ADDCH(vp, '\n'); + /* FALLTRHOUGH */ + } else { + if (next_char != VSTREAM_EOF) + vstream_ungetc(stream, next_char); + + /* + * Input too long, or EOF + */ + default: + if (msg_verbose) + msg_info("%s got %s", VSTREAM_PATH(stream), + LEN(vp) < bound ? "EOF" : "input too long"); + return (-1); + } + + /* + * Strip off the record terminator: either CRLF or just bare LF. + */ + case '\n': + vstring_truncate(vp, VSTRING_LEN(vp) - 1); + if (VSTRING_LEN(vp) > 0 && vstring_end(vp)[-1] == '\r') + vstring_truncate(vp, VSTRING_LEN(vp) - 1); + VSTRING_TERMINATE(vp); + if (msg_verbose) + msg_info("%s got: %s", VSTREAM_PATH(stream), STR(vp)); + return (0); + } +} + +/* memcache_fwrite - write one blob to peer */ + +int memcache_fwrite(VSTREAM *stream, const char *cp, ssize_t todo) +{ + + /* + * Sanity check. + */ + if (todo < 0) + msg_panic("memcache_fwrite: negative todo %ld", (long) todo); + + /* + * Do the I/O. + */ + if (msg_verbose) + msg_info("%s write: %.*s", VSTREAM_PATH(stream), (int) todo, cp); + if (vstream_fwrite(stream, cp, todo) != todo + || vstream_fputs("\r\n", stream) == VSTREAM_EOF) + return (-1); + else + return (0); +} + +/* memcache_fread - read one blob from peer */ + +int memcache_fread(VSTREAM *stream, VSTRING *buf, ssize_t todo) +{ + + /* + * Sanity check. + */ + if (todo < 0) + msg_panic("memcache_fread: negative todo %ld", (long) todo); + + /* + * Do the I/O. + */ + VSTRING_SPACE(buf, todo); + VSTRING_AT_OFFSET(buf, todo); + if (vstream_fread(stream, STR(buf), todo) != todo + || VSTREAM_GETC(stream) != '\r' + || VSTREAM_GETC(stream) != '\n') { + if (msg_verbose) + msg_info("%s read: error", VSTREAM_PATH(stream)); + return (-1); + } else { + vstring_truncate(buf, todo); + VSTRING_TERMINATE(buf); + if (msg_verbose) + msg_info("%s read: %s", VSTREAM_PATH(stream), STR(buf)); + return (0); + } +} + +/* memcache_vprintf - write one line to peer */ + +int memcache_vprintf(VSTREAM *stream, const char *fmt, va_list ap) +{ + + /* + * Do the I/O. + */ + vstream_vfprintf(stream, fmt, ap); + vstream_fputs("\r\n", stream); + if (vstream_ferror(stream)) + return (-1); + else + return (0); +} + +/* memcache_printf - write one line to peer */ + +int memcache_printf(VSTREAM *stream, const char *fmt,...) +{ + va_list ap; + int ret; + + if (msg_verbose) { + VSTRING *buf = vstring_alloc(100); + + va_start(ap, fmt); + vstring_vsprintf(buf, fmt, ap); + va_end(ap); + msg_info("%s write: %s", VSTREAM_PATH(stream), STR(buf)); + vstring_free(buf); + } + + /* + * Do the I/O. + */ + va_start(ap, fmt); + ret = memcache_vprintf(stream, fmt, ap); + va_end(ap); + return (ret); +} diff --git a/postfix/src/global/memcache_proto.h b/postfix/src/global/memcache_proto.h new file mode 100644 index 000000000..88a1d191f --- /dev/null +++ b/postfix/src/global/memcache_proto.h @@ -0,0 +1,34 @@ +#ifndef _MEMCACHE_PROTO_H_INCLUDED_ +#define _MEMCACHE_PROTO_H_INCLUDED_ + +/*++ +/* NAME +/* memcache_proto 3h +/* SUMMARY +/* memcache low-level protocol +/* SYNOPSIS +/* #include +/* DESCRIPTION +/* .nf + + /* + * External interface. + */ +extern int memcache_get(VSTREAM *, VSTRING *, ssize_t); +extern int memcache_vprintf(VSTREAM *, const char *, va_list); +extern int memcache_printf(VSTREAM *, const char *fmt,...); +extern int memcache_fread(VSTREAM *, VSTRING *, ssize_t); +extern int memcache_fwrite(VSTREAM *, const char *, ssize_t); + +/* LICENSE +/* .ad +/* .fi +/* The Secure Mailer license must be distributed with this software. +/* AUTHOR(S) +/* Wietse Venema +/* IBM T.J. Watson Research +/* P.O. Box 704 +/* Yorktown Heights, NY 10598, USA +/*--*/ + +#endif diff --git a/postfix/src/proxymap/proxymap.c b/postfix/src/proxymap/proxymap.c index 24016ab72..9caea9350 100644 --- a/postfix/src/proxymap/proxymap.c +++ b/postfix/src/proxymap/proxymap.c @@ -69,6 +69,13 @@ /* as with the \fBopen\fR request. /* .sp /* This request is supported in Postfix 2.5 and later. +/* .IP "\fBsequence\fR \fImaptype:mapname flags function\fR" +/* Iterate over the specified database. The \fIfunction\fR +/* is one of DICT_SEQ_FUN_FIRST or DICT_SEQ_FUN_NEXT. +/* The reply is the request completion status code and +/* a lookup key and result value, if found. +/* .sp +/* This request is supported in Postfix 2.9 and later. /* .PP /* The request completion status is one of OK, RETRY, NOKEY /* (lookup failed because the key was not found), BAD (malformed @@ -235,6 +242,7 @@ * aren't too broken. The fix is to gather all parameter default settings in * one place. */ +char *var_alias_maps; char *var_local_rcpt_maps; char *var_virt_alias_maps; char *var_virt_alias_doms; @@ -329,6 +337,54 @@ static DICT *proxy_map_find(const char *map_type_name, int request_flags, return (dict); } +/* proxymap_sequence_service - remote sequence service */ + +static void proxymap_sequence_service(VSTREAM *client_stream) +{ + int request_flags; + DICT *dict; + int request_func; + const char *reply_key; + const char *reply_value; + int reply_status; + + /* + * Process the request. + */ + if (attr_scan(client_stream, ATTR_FLAG_STRICT, + ATTR_TYPE_STR, MAIL_ATTR_TABLE, request_map, + ATTR_TYPE_INT, MAIL_ATTR_FLAGS, &request_flags, + ATTR_TYPE_INT, MAIL_ATTR_FUNC, &request_func, + ATTR_TYPE_END) != 3 + || (request_func != DICT_SEQ_FUN_FIRST + && request_func != DICT_SEQ_FUN_NEXT)) { + reply_status = PROXY_STAT_BAD; + reply_key = reply_value = ""; + } else if ((dict = proxy_map_find(STR(request_map), request_flags, + &reply_status)) == 0) { + reply_key = reply_value = ""; + } else if (dict->flags = ((dict->flags & ~DICT_FLAG_RQST_MASK) + | (request_flags & DICT_FLAG_RQST_MASK)), + dict_seq(dict, request_func, &reply_key, &reply_value) == 0) { + reply_status = PROXY_STAT_OK; + } else if (dict_errno == 0) { + reply_status = PROXY_STAT_NOKEY; + reply_key = reply_value = ""; + } else { + reply_status = PROXY_STAT_RETRY; + reply_key = reply_value = ""; + } + + /* + * Respond to the client. + */ + attr_print(client_stream, ATTR_FLAG_NONE, + ATTR_TYPE_INT, MAIL_ATTR_STATUS, reply_status, + ATTR_TYPE_STR, MAIL_ATTR_KEY, reply_key, + ATTR_TYPE_STR, MAIL_ATTR_VALUE, reply_value, + ATTR_TYPE_END); +} + /* proxymap_lookup_service - remote lookup service */ static void proxymap_lookup_service(VSTREAM *client_stream) @@ -407,8 +463,9 @@ static void proxymap_update_service(VSTREAM *client_stream) dict->flags = ((dict->flags & ~DICT_FLAG_RQST_MASK) | (request_flags & DICT_FLAG_RQST_MASK) | DICT_FLAG_SYNC_UPDATE | DICT_FLAG_DUP_REPLACE); + dict_errno = 0; dict_put(dict, STR(request_key), STR(request_value)); - reply_status = PROXY_STAT_OK; + reply_status = (dict_errno ? PROXY_STAT_RETRY : PROXY_STAT_OK); } /* @@ -425,6 +482,7 @@ static void proxymap_delete_service(VSTREAM *client_stream) { int request_flags; DICT *dict; + int dict_status; int reply_status; /* @@ -450,8 +508,11 @@ static void proxymap_delete_service(VSTREAM *client_stream) dict->flags = ((dict->flags & ~DICT_FLAG_RQST_MASK) | (request_flags & DICT_FLAG_RQST_MASK) | DICT_FLAG_SYNC_UPDATE); - reply_status = - dict_del(dict, STR(request_key)) ? PROXY_STAT_OK : PROXY_STAT_NOKEY; + dict_errno = 0; + dict_status = dict_del(dict, STR(request_key)); + reply_status = (dict_status == 0 ? PROXY_STAT_OK : + dict_status > 0 ? PROXY_STAT_NOKEY : + PROXY_STAT_RETRY); } /* @@ -524,6 +585,8 @@ static void proxymap_service(VSTREAM *client_stream, char *unused_service, proxymap_update_service(client_stream); } else if (VSTREQ(request, PROXY_REQ_DELETE)) { proxymap_delete_service(client_stream); + } else if (VSTREQ(request, PROXY_REQ_SEQUENCE)) { + proxymap_sequence_service(client_stream); } else if (VSTREQ(request, PROXY_REQ_OPEN)) { proxymap_open_service(client_stream); } else { @@ -620,6 +683,7 @@ MAIL_VERSION_STAMP_DECLARE; int main(int argc, char **argv) { static const CONFIG_STR_TABLE str_table[] = { + VAR_ALIAS_MAPS, DEF_ALIAS_MAPS, &var_alias_maps, 0, 0, VAR_LOCAL_RCPT_MAPS, DEF_LOCAL_RCPT_MAPS, &var_local_rcpt_maps, 0, 0, VAR_VIRT_ALIAS_MAPS, DEF_VIRT_ALIAS_MAPS, &var_virt_alias_maps, 0, 0, VAR_VIRT_ALIAS_DOMS, DEF_VIRT_ALIAS_DOMS, &var_virt_alias_doms, 0, 0, diff --git a/postfix/src/util/dict.c b/postfix/src/util/dict.c index 0731a2b87..8e170e62f 100644 --- a/postfix/src/util/dict.c +++ b/postfix/src/util/dict.c @@ -107,7 +107,9 @@ /* modified, or if the result is to survive multiple dict_lookup() calls. /* /* dict_delete() removes the named member from the named dictionary. -/* The result value is zero when the member was found. +/* The result value is zero when the member was found, > 0 if +/* it was not found, and < 0 in case of error (a database may +/* not return after error). /* /* dict_sequence() steps through the named dictionary and returns /* keys and values in some implementation-defined order. The func diff --git a/postfix/src/util/dict_test.c b/postfix/src/util/dict_test.c index e4f780b6a..afd41b790 100644 --- a/postfix/src/util/dict_test.c +++ b/postfix/src/util/dict_test.c @@ -87,7 +87,7 @@ void dict_test(int argc, char **argv) if (*bufp == '#') continue; if ((cmd = mystrtok(&bufp, " ")) == 0) { - vstream_printf("usage: del key|get key|put key=value|first|next\n"); + vstream_printf("usage: verbose|del key|get key|put key=value|first|next\n"); vstream_fflush(VSTREAM_OUT); continue; } @@ -95,7 +95,9 @@ void dict_test(int argc, char **argv) msg_warn("dictionary has changed"); key = *bufp ? vstring_str(unescape(keybuf, mystrtok(&bufp, " ="))) : 0; value = mystrtok(&bufp, " ="); - if (strcmp(cmd, "del") == 0 && key && !value) { + if (strcmp(cmd, "verbose") == 0 && !key) { + msg_verbose++; + } else if (strcmp(cmd, "del") == 0 && key && !value) { if (dict_del(dict, key)) vstream_printf("%s: not found\n", key); else