diff --git a/postfix/.indent.pro b/postfix/.indent.pro
index 61201ac29..60bdd8d01 100644
--- a/postfix/.indent.pro
+++ b/postfix/.indent.pro
@@ -1,4 +1,3 @@
--TMDB_val
-TABOUNCE
-TADDR_MATCH_LIST
-TADDR_PATTERN
@@ -178,6 +177,7 @@
-TMBLOCK
-TMBOX
-TMDB_txn
+-TMDB_val
-TMILTER
-TMILTER8
-TMILTERS
@@ -270,6 +270,7 @@
-TSINGLE_SERVER
-TSINK_COMMAND
-TSINK_STATE
+-TSLMDB
-TSMFICTX
-TSMTPD_CMD
-TSMTPD_DEFER
diff --git a/postfix/HISTORY b/postfix/HISTORY
index a8c8893a4..f5a5ff460 100644
--- a/postfix/HISTORY
+++ b/postfix/HISTORY
@@ -10876,7 +10876,6 @@ Apologies for any names omitted.
Postfix 2.3 code review. Files: util/netstring.c,
util/myaddrinfo.c, util/attr_clnt.c, util/vstream.c.
-
Bugfix: the SMTP server now separates the message size check
from the queue space check, so that the size check can be
done before an SMTPD proxy filter. Files: smtpd/smtpd.c,
@@ -13536,7 +13535,6 @@ Apologies for any names omitted.
Bugfix: Content-Transfer-Encoding: attribute values are
case insensitive. File: src/cleanup/cleanup_message.c.
-
20070514
Bugfix: the makedefs EPOLL workaround broke any attempt to
@@ -17839,7 +17837,6 @@ Apologies for any names omitted.
util/Makefile.in, util/listen.h, util/recv_pass_attr.c,
util/stream_listen.c, util/sys_defs.h, util/unix_pass_listen.c.
-
20120618
Cleanup: made the postscreen-to-smtpd haproxy attribute
@@ -18977,3 +18974,52 @@ Apologies for any names omitted.
in multi-programmed systems, and prohibit database sharing
between privileged writer processes and unprivileged reader
processes.
+
+20131009
+
+ Documentation: inet_protols description was not updated
+ when smtp_address_preference was added. File: proto/postconf.proto
+
+20131013
+
+ Documentation: why postscreen(8) uses hash-table lookups
+ instead of direct pointers to find the DNSBL lookup result
+ for a specific session. File: postscreen/postscreen_early.c.
+
+20131022
+
+ Cleanup: add more &code; to postconf2man. Someone has been
+ writing documentation without checking the result, File:
+ mantools/postconf2man.
+
+ Documentation: in the discard(8) manpage, the reason is not
+ a host or domain name. File: discard/discard.c.
+
+20131025
+
+ Documentation: specify the expected result format with
+ "list" tables. File: proto/DATABASE_README.html.
+
+20131026
+
+ Future proofing: API changes in the PCRE library. File:
+ util/dict_pcre.c.
+
+20131028
+
+ Feature: check_sasl_access to block hijacked logins. Files:
+ mantools/postlink, proto/postconf.proto, global/mail_params.h,
+ smtpd/smtpd_check.c, smtpd/smtpd_dsn_fix.h.
+
+20131029-31
+
+ Cleanup: slmdb(3) simplified LMDB API that hides recoverable
+ LMDB errors from applications so that they can focus on
+ their own job. Files: util/slmdb.[hc].
+
+ Cleanup: LMDB functionality restored, after elimination of
+ 1) world-writable lockfiles, 2) hard limits on the number
+ of concurrent readers, and 3) hard-coded database file inode
+ numbers in lockfiles that can prevent automatic crash
+ recovery. Files: proto/LMDB_README.html, proto/postconf.proto,
+ mantools/postlink, util/dict_lmdb.c.
diff --git a/postfix/README_FILES/DATABASE_README b/postfix/README_FILES/DATABASE_README
index 6ba7772fd..53af00125 100644
--- a/postfix/README_FILES/DATABASE_README
+++ b/postfix/README_FILES/DATABASE_README
@@ -55,12 +55,13 @@ new address) or access control (the lookup string is the client, sender or
recipient, and the result is an action such as "reject").
With some tables, however, Postfix needs to know only if the lookup key exists.
-The lookup result itself is not used. Examples are the local_recipient_maps
-that determine what local recipients Postfix accepts in mail from the network,
-the mydestination parameter that specifies what domains Postfix delivers
-locally, or the mynetworks parameter that specifies the IP addresses of trusted
-clients or client networks. Technically, these are lists, not tables. Despite
-the difference, Postfix lists are described here because they use the same
+Any non-empty lookup result value may be used here: the lookup result is not
+used. Examples are the local_recipient_maps that determine what local
+recipients Postfix accepts in mail from the network, the mydestination
+parameter that specifies what domains Postfix delivers locally, or the
+mynetworks parameter that specifies the IP addresses of trusted clients or
+client networks. Technically, these are lists, not tables. Despite the
+difference, Postfix lists are described here because they use the same
underlying infrastructure as Postfix lookup tables.
PPrreeppaarriinngg PPoossttffiixx ffoorr LLDDAAPP oorr SSQQLL llooookkuuppss
@@ -119,18 +120,21 @@ performance.
UUppddaattiinngg BBeerrkkeelleeyy DDBB ffiilleess ssaaffeellyy
-Although Postfix uses file locking to avoid access conflicts while updating
-Berkeley DB or other local database files, you still have a problem when the
-update fails because the disk is full or because something else happens. This
-is because commands such as postmap(1) or postalias(1) overwrite existing
-files. If the update fails in the middle then you have no usable database, and
-Postfix will stop working. This is not an issue with the CDB database type
+Postfix uses file locking to avoid access conflicts while updating Berkeley DB
+or other local database files. This used to be safe, but as Berkeley DB has
+evolved to use more aggressive caching, file locking may no longer be
+sufficient.
+
+Furthermore, file locking would not prevent problems when the update fails
+because the disk is full or something else causes a database update to fail. In
+particular, commands such as postmap(1) or postalias(1) overwrite existing
+files. If the overwrite fails in the middle then you have no usable database,
+and Postfix will stop working. This is not an issue with the CDB database type
available with Postfix 2.2 and later: CDB creates a new file, and renames the
file upon successful completion.
-With multi-file databases such as DBM, there is no simple solution. With
-Berkeley DB and other "one file" databases, it is possible to add some extra
-robustness by using "mv" to REPLACE an existing database file instead of
+With Berkeley DB and other "one file" databases, it is possible to add some
+extra robustness by using "mv" to REPLACE an existing database file instead of
overwriting it:
# ppoossttmmaapp aacccceessss..iinn &&&& mmvv aacccceessss..iinn..ddbb aacccceessss..ddbb
diff --git a/postfix/README_FILES/LMDB_README b/postfix/README_FILES/LMDB_README
index 63a7051fd..2ce6b726d 100644
--- a/postfix/README_FILES/LMDB_README
+++ b/postfix/README_FILES/LMDB_README
@@ -1,9 +1,103 @@
PPoossttffiixx OOppeennLLDDAAPP LLMMDDBB HHoowwttoo
-------------------------------------------------------------------------------
--------------------------------------------------------------------------------
-Postfix LMDB support is forbidden due to problems with LMDB lock management.
-These problems hinder error recovery in multi-programmed systems, and prohibit
-database sharing between privileged writer processes and unprivileged reader
-processes.
+
+IInnttrroodduuccttiioonn
+
+Postfix uses databases of various kinds to store and look up information.
+Postfix databases are specified as "type:name". OpenLDAP LMDB implements the
+Postfix database type "lmdb". The name of a Postfix OpenLDAP LMDB database is
+the name of the database file without the ".lmdb" suffix.
+
+This document describes:
+
+ 1. How to build Postfix with OpenLDAP LMDB support.
+
+ 2. How to configure LMDB settings.
+
+ 3. Missing pthread library trouble.
+
+ 4. Unexpected failure modes that don't exist with other Postfix databases.
+
+BBuuiillddiinngg PPoossttffiixx wwiitthh OOppeennLLDDAAPP LLMMDDBB ssuuppppoorrtt
+
+Postfix normally does not enable OpenLDAP LMDB support. To build Postfix with
+OpenLDAP LMDB support, use something like:
+
+ % make makefiles CCARGS="-DHAS_LMDB -I/usr/local/include" \
+ AUXLIBS="-L/usr/local/lib -llmdb"
+ % make
+
+Solaris may need this:
+
+ % make makefiles CCARGS="-DHAS_LMDB -I/usr/local/include" \
+ AUXLIBS="-R/usr/local/lib -L/usr/local/lib -llmdb"
+ % make
+
+The exact pathnames depend on how OpenLDAP LMDB was installed.
+
+CCoonnffiigguurree LLMMDDBB sseettttiinnggss
+
+Postfix provides one configuration parameter that controls OpenLDAP LMDB
+database behavior.
+
+ * lmdb_map_size (default: 16777216). This setting specifies the initial
+ OpenLDAP LMDB database size limit in bytes. Each time a database becomes
+ full, its size limit is doubled. The maximum size is the largest signed
+ integer value of "long".
+
+MMiissssiinngg pptthhrreeaadd lliibbrraarryy ttrroouubbllee
+
+When building Postfix fails with:
+
+ undefined reference to `pthread_mutexattr_destroy'
+ undefined reference to `pthread_mutexattr_init'
+ undefined reference to `pthread_mutex_lock'
+
+Add the "-lpthread" library to the "make makefiles" command.
+
+ % make makefiles .... AUXLIBS="... -lpthread"
+
+Source code for OpenLDAP LMDB is available at http://www.openldap.org. More
+information is available at http://highlandsun.com/hyc/mdb/.
+
+UUnneexxppeecctteedd ffaaiilluurree mmooddeess ooff PPoossttffiixx LLMMDDBB ddaattaabbaasseess..
+
+As documented below, conversion to LMDB introduces a number of failure modes
+that don't exist with other Postfix databases. Some failure modes have been
+eliminated in the course of time. The writeup below reflects the status as of
+LMDB 0.9.9.
+
+NNoonn--oobbvviioouuss rreeccoovveerryy wwiitthh ppoossttmmaapp((11)),, ppoossttaalliiaass((11)),, oorr ttllssmmggrr((88)) ffrroomm aa
+ccoorrrruupptteedd ddaattaabbaassee..
+
+Problem:
+ A corrupted LMDB database cann't be rebuilt simply by re-running postmap(1)
+ or postalias(1), or by waiting until a tlsmgr(8) daemon restarts. This
+ problem does not exist with other Postfix databases.
+
+Background:
+ The Postfix LMDB database client does not truncate the database file.
+ Instead it attempts to create a transaction for a "drop" request plus
+ subsequent "store" requests. That is obviously not possible with a
+ corrupted database file.
+
+Impact:
+ Postfix does not process mail until someone fixes the problem.
+
+Recovery:
+ First delete the ".lmdb" file by hand. Then rebuild the file with the
+ postmap(1) or postalias(1) command if the file was created with those
+ commands, or restart postfix daemons if the file is maintained by tlsmgr
+ (8).
+
+Prevention:
+ Arrange your file systems such that they never run out of free space.
+
+ Use ECC memory to detect and correct silent corruption of in-memory file
+ system data and metadata.
+
+ Use a file system such as ZFS to detect and correct silent corruption of
+ on-disk file system data and metadata. DO NOT use ZFS on systems without
+ ECC memory error correction.
diff --git a/postfix/RELEASE_NOTES b/postfix/RELEASE_NOTES
index 956aafeaa..0a3ded0ec 100644
--- a/postfix/RELEASE_NOTES
+++ b/postfix/RELEASE_NOTES
@@ -14,13 +14,14 @@ specifies the release date of a stable release or snapshot release.
If you upgrade from Postfix 2.9 or earlier, read RELEASE_NOTES-2.10
before proceeding.
-Major changes with snapshot 20131001
+Major changes with snapshot 20131031
====================================
-LMDB support is forbidden due to problems with LMDB lock management.
-These problems hinder error recovery in multi-programmed systems,
-and prohibit database sharing between privileged writer processes
-and unprivileged reader processes.
+LMDB support is enabled after changes to LMDB lock management. This
+includes creating databases with postmap(1) and postalias(1);
+read/write access by postscreen(8), proxymap(8), verify(8), and
+tlsmgr(8); and database sharing between privileged writer processes
+and unprivileged reader processes without world-writable files.
Major changes with snapshot 20130929
====================================
diff --git a/postfix/WISHLIST b/postfix/WISHLIST
index a40d6cf50..be55ea52c 100644
--- a/postfix/WISHLIST
+++ b/postfix/WISHLIST
@@ -1,5 +1,9 @@
Wish list:
+ Per SASL account rate limits.
+
+ Add watchdog timer to postmap/postalias.
+
Things to do before the stable release:
Spell-check, double-word check, and HTML validator check.
diff --git a/postfix/html/DATABASE_README.html b/postfix/html/DATABASE_README.html
index 68606057a..299b8e6bc 100644
--- a/postfix/html/DATABASE_README.html
+++ b/postfix/html/DATABASE_README.html
@@ -94,7 +94,8 @@ string is the client, sender or recipient, and the result is an
action such as "reject").
With some tables, however, Postfix needs to know only if the
-lookup key exists. The lookup result itself is not used. Examples
+lookup key exists. Any non-empty lookup result value may be used
+here: the lookup result is not used. Examples
are the local_recipient_maps that determine what local recipients
Postfix accepts in mail from the network, the mydestination parameter
that specifies what domains Postfix delivers locally, or the
@@ -185,19 +186,22 @@ process can initialize with the new database.
- Although Postfix uses file locking to avoid access conflicts
-while updating Berkeley DB or other local database files, you still
-have a problem when the update fails because the disk is full or
-because something else happens. This is because commands such as
-postmap(1) or postalias(1) overwrite existing files. If the update
+
Postfix uses file locking to avoid access conflicts while
+updating Berkeley DB or other local database files. This used to
+be safe, but as Berkeley DB has evolved to use more aggressive
+caching, file locking may no longer be sufficient.
+
+ Furthermore, file locking would not prevent problems when the
+update fails because the disk is full or something else causes a
+database update to fail. In particular, commands such as postmap(1)
+or postalias(1) overwrite existing files. If the overwrite
fails in the middle then you have no usable database, and Postfix
will stop working. This is not an issue with the CDB database type
available with Postfix 2.2 and later: CDB
creates a new file, and renames the file upon successful completion.
- With multi-file databases such as DBM, there is no simple
-solution. With Berkeley DB and other "one file" databases, it is
+
With Berkeley DB and other "one file" databases, it is
possible to add some extra robustness by using "mv" to REPLACE an
existing database file instead of overwriting it:
diff --git a/postfix/html/LMDB_README.html b/postfix/html/LMDB_README.html
index e76cc9005..a607b5235 100644
--- a/postfix/html/LMDB_README.html
+++ b/postfix/html/LMDB_README.html
@@ -17,35 +17,8 @@
-
-
-
- Postfix LMDB support is forbidden due to problems with LMDB lock
-management. These problems hinder error recovery in multi-programmed
-systems, and prohibit database sharing between privileged writer
-processes and unprivileged reader processes.
-
-
+
+
+
+
+
+
+
+
- Non-obvious recovery with postmap(1), postalias(1), - ->
-postscreen(8), tlsmgr(8), or verify(8) from a corrupted database.
-
+ Non-obvious recovery with postmap(1), postalias(1), or
+tlsmgr(8) from a corrupted database.
-- Problem:
-
You cannot rebuild a corrupted LMDB
-database simply by postmap(1) or postalias(1), or
-by - -> waiting until a daemon restarts. This problem does not exist
-with other Postfix databases.
+- Problem:
-
A corrupted LMDB database cann't be
+rebuilt simply by re-running postmap(1) or postalias(1), or by
+waiting until a tlsmgr(8) daemon restarts. This problem does not
+exist with other Postfix databases.
- Background:
-
The Postfix LMDB database client
does not truncate the database file. Instead it attempts to create
@@ -323,10 +309,10 @@ That is obviously not possible with a corrupted database file.
someone fixes the problem.
- Recovery:
-
First delete the ".lmdb" file by hand.
-Then, postmap(1) or postalias(1)
-command if the file was created with those commands, or - -> restart
-postfix.
+Then rebuild the file with the postmap(1) or postalias(1)
+command if the file was created with those commands, or restart
+postfix daemons if the file is maintained by tlsmgr(8).
+
- Prevention:
-
@@ -337,10 +323,7 @@ space.
in-memory file system data and metadata.
Use a file system such as ZFS to detect and correct silent
-corruption of on-disk file system data and metadata.
+corruption of on-disk file system data and metadata. DO NOT
+use ZFS on systems without ECC memory error correction.
-
--->
-
-
diff --git a/postfix/html/discard.8.html b/postfix/html/discard.8.html
index e4b7a15fe..ff5574a86 100644
--- a/postfix/html/discard.8.html
+++ b/postfix/html/discard.8.html
@@ -15,7 +15,7 @@ DISCARD(8) DISCARD(8)
DESCRIPTION
The Postfix discard(8) delivery agent processes delivery
requests from the queue manager. Each request specifies a
- queue file, a sender address, a domain or host name that
+ queue file, a sender address, a next-hop destination that
is treated as the reason for discarding the mail, and
recipient information. The reason may be prefixed with an
RFC 3463-compatible detail code. This program expects to
@@ -23,10 +23,10 @@ DISCARD(8) DISCARD(8)
The discard(8) delivery agent pretends to deliver all
recipients in the delivery request, logs the "next-hop"
- domain or host information as the reason for discarding
- the mail, updates the queue file and marks recipients as
- finished or informs the queue manager that delivery should
- be tried again at a later time.
+ destination as the reason for discarding the mail, updates
+ the queue file, and either marks recipients as finished or
+ informs the queue manager that delivery should be tried
+ again at a later time.
Delivery status reports are sent to the trace(8) daemon as
appropriate.
diff --git a/postfix/html/postconf.5.html b/postfix/html/postconf.5.html
index 79ca1b2d7..2c013792f 100644
--- a/postfix/html/postconf.5.html
+++ b/postfix/html/postconf.5.html
@@ -3630,8 +3630,9 @@ IPV6_V6ONLY support (RFC 3493).
Postfix will do DNS type AAAA record lookups.
When both IPv4 and IPv6 support are enabled, the Postfix SMTP
-client will attempt to connect via IPv6 before attempting to use
-IPv4.
+client will choose the protocol as specified with the
+smtp_address_preference parameter. Postfix versions before 2.8
+attempt to connect via IPv6 before attempting to use IPv4.
Examples:
@@ -3794,18 +3795,6 @@ This feature is available in Postfix 2.11 and later.
-
-
-lmdb_max_readers
-(default: $default_process_limit)
-
- The hard limit on the number of read transactions that may be
-open at the same time for the same OpenLDAP LMDB database. When
-this number is too small, the Postfix LMDB client will log
-MDB_READERS_FULL errors, and will run with reduced performance.
-
-
-
lmtp_address_preference
@@ -12746,6 +12735,16 @@ action. Note: a result of "OK" is not allowed for safety reasons.
Instead, use DUNNO in order to exclude specific hosts from blacklists.
This feature is available in Postfix 2.7 and later.
+check_sasl_access type:table
+
+ Use the remote SMTP client SASL user name as lookup key for
+the specified access(5) database. The lookup key has the form
+"username@domainname" when the smtpd_sasl_local_domain parameter
+value is non-empty. Unlike the check_client_access feature,
+check_sasl_access does not perform matches of parent domains or IP
+subnet ranges. This feature is available with Postfix version 2.11
+and later.
+
permit_inet_interfaces
Permit the request when the client IP address matches
diff --git a/postfix/man/man5/postconf.5 b/postfix/man/man5/postconf.5
index f566d57c2..58b9886bc 100644
--- a/postfix/man/man5/postconf.5
+++ b/postfix/man/man5/postconf.5
@@ -2158,8 +2158,9 @@ When IPv6 support is enabled via the inet_protocols parameter,
Postfix will do DNS type AAAA record lookups.
.PP
When both IPv4 and IPv6 support are enabled, the Postfix SMTP
-client will attempt to connect via IPv6 before attempting to use
-IPv4.
+client will choose the protocol as specified with the
+smtp_address_preference parameter. Postfix versions before 2.8
+attempt to connect via IPv6 before attempting to use IPv4.
.PP
Examples:
.PP
@@ -2247,11 +2248,6 @@ The initial OpenLDAP LMDB database size limit in bytes. Each time
a database becomes full, its size limit is doubled.
.PP
This feature is available in Postfix 2.11 and later.
-.SH lmdb_max_readers (default: $default_process_limit)
-The hard limit on the number of read transactions that may be
-open at the same time for the same OpenLDAP LMDB database. When
-this number is too small, the Postfix LMDB client will log
-MDB_READERS_FULL errors, and will run with reduced performance.
.SH lmtp_address_preference (default: ipv6)
The LMTP-specific version of the smtp_address_preference
configuration parameter. See there for details.
@@ -5234,7 +5230,7 @@ For more fine-grained control, use check_ccert_access to select
an appropriate \fBaccess\fR(5) policy for each client.
See RESTRICTION_CLASS_README.
.PP
-\fBNote:\fR Postfix 2.9.0–2.9.5 computed the public key
+\fBNote:\fR Postfix 2.9.0-2.9.5 computed the public key
fingerprint incorrectly. To use public-key fingerprints, upgrade
to Postfix 2.9.6 or later.
.PP
@@ -7074,7 +7070,7 @@ The Postfix SMTP server and client log the peer (leaf) certificate
fingerprint and public key fingerprint when the TLS loglevel is 2 or
higher.
.PP
-\fBNote:\fR Postfix 2.9.0–2.9.5 computed the public key
+\fBNote:\fR Postfix 2.9.0-2.9.5 computed the public key
fingerprint incorrectly. To use public-key fingerprints, upgrade
to Postfix 2.9.6 or later.
.PP
@@ -7118,7 +7114,7 @@ Each logging level also includes the information that is logged at
a lower logging level.
.IP ""
0 Log only a summary message on TLS handshake completion
-— no logging of remote SMTP server certificate trust-chain
+- no logging of remote SMTP server certificate trust-chain
verification errors if server certificate verification is not required.
With Postfix 2.8 and earlier, disable logging of TLS activity.
.br
@@ -8284,6 +8280,15 @@ action. Note: a result of "OK" is not allowed for safety reasons.
Instead, use DUNNO in order to exclude specific hosts from blacklists.
This feature is available in Postfix 2.7 and later.
.br
+.IP "\fBcheck_sasl_access \fItype:table\fR\fR"
+Use the remote SMTP client SASL user name as lookup key for
+the specified \fBaccess\fR(5) database. The lookup key has the form
+"username@domainname" when the smtpd_sasl_local_domain parameter
+value is non-empty. Unlike the check_client_access feature,
+check_sasl_access does not perform matches of parent domains or IP
+subnet ranges. This feature is available with Postfix version 2.11
+and later.
+.br
.IP "\fBpermit_inet_interfaces\fR"
Permit the request when the client IP address matches
$inet_interfaces.
@@ -10442,7 +10447,7 @@ The Postfix SMTP server and client log the peer (leaf) certificate
fingerprint and public key fingerprint when the TLS loglevel is 2 or
higher.
.PP
-\fBNote:\fR Postfix 2.9.0–2.9.5 computed the public key
+\fBNote:\fR Postfix 2.9.0-2.9.5 computed the public key
fingerprint incorrectly. To use public-key fingerprints, upgrade
to Postfix 2.9.6 or later.
.PP
@@ -10490,7 +10495,7 @@ Each logging level also includes the information that is logged at
a lower logging level.
.IP ""
0 Log only a summary message on TLS handshake completion
-— no logging of remote SMTP client certificate trust-chain verification
+- no logging of remote SMTP client certificate trust-chain verification
errors
if client certificate verification is not required. With Postfix 2.8
and earlier, disable logging of TLS activity.
diff --git a/postfix/man/man8/discard.8 b/postfix/man/man8/discard.8
index a96cd7d6d..05c283970 100644
--- a/postfix/man/man8/discard.8
+++ b/postfix/man/man8/discard.8
@@ -15,16 +15,16 @@ Postfix discard mail delivery agent
The Postfix \fBdiscard\fR(8) delivery agent processes
delivery requests from
the queue manager. Each request specifies a queue file, a sender
-address, a domain or host name that is treated as the reason for
+address, a next-hop destination that is treated as the reason for
discarding the mail, and recipient information.
The reason may be prefixed with an RFC 3463-compatible detail code.
This program expects to be run from the \fBmaster\fR(8) process
manager.
The \fBdiscard\fR(8) delivery agent pretends to deliver all recipients
-in the delivery request, logs the "next-hop" domain or host
-information as the reason for discarding the mail, updates the
-queue file and marks recipients as finished or informs the
+in the delivery request, logs the "next-hop" destination
+as the reason for discarding the mail, updates the
+queue file, and either marks recipients as finished or informs the
queue manager that delivery should be tried again at a later time.
Delivery status reports are sent to the \fBtrace\fR(8)
diff --git a/postfix/mantools/postconf2man b/postfix/mantools/postconf2man
index 36bbea61a..ba6205335 100755
--- a/postfix/mantools/postconf2man
+++ b/postfix/mantools/postconf2man
@@ -78,6 +78,8 @@ while(<>) {
$block =~ s/≥/>=/g;
$block =~ s/>/>/g;
$block =~ s/&/\&/g;
+ $block =~ s/–/-/g;
+ $block =~ s/—/-/g;
$block =~ s/\s+\n/\n/g;
$block =~ s/^\n//g;
$block =~ s/([a-z][_a-zA-Z0-9-]*)(\([0-9]\))/\\fB\1\\fR\2/g;
diff --git a/postfix/mantools/postlink b/postfix/mantools/postlink
index 4ed38aff5..75083a73c 100755
--- a/postfix/mantools/postlink
+++ b/postfix/mantools/postlink
@@ -209,7 +209,6 @@ while (<>) {
s;\bipc_ttl\b;$&;g;
s;\bline_length_limit\b;$&;g;
s;\blmdb_map_size\b;$&;g;
- s;\blmdb_max_readers\b;$&;g;
s;\blmtp_address_preference\b;$&;g;
s;\blmtp_body_checks\b;$&;g;
s;\blmtp_cname_overrides_servername\b;$&;g;
@@ -864,6 +863,7 @@ while (<>) {
s;\bcheck_reverse_client_hostname_access\b;$&;g;
s;\bcheck_reverse_client_hostname_mx_access\b;$&;g;
s;\bcheck_reverse_client_hostname_ns_access\b;$&;g;
+ s;\bcheck_sasl_access\b;$&;g;
s;\bpermit_inet_interfaces\b;$&;g;
s;\bpermit_mynetworks\b;$&;g;
s;\bper[-]*\n* *[]*mit_sasl_authenticated\b;$&;g;
diff --git a/postfix/proto/DATABASE_README.html b/postfix/proto/DATABASE_README.html
index af0c2de95..2f05dc0e6 100644
--- a/postfix/proto/DATABASE_README.html
+++ b/postfix/proto/DATABASE_README.html
@@ -94,7 +94,8 @@ string is the client, sender or recipient, and the result is an
action such as "reject").
With some tables, however, Postfix needs to know only if the
-lookup key exists. The lookup result itself is not used. Examples
+lookup key exists. Any non-empty lookup result value may be used
+here: the lookup result is not used. Examples
are the local_recipient_maps that determine what local recipients
Postfix accepts in mail from the network, the mydestination parameter
that specifies what domains Postfix delivers locally, or the
@@ -185,19 +186,22 @@ process can initialize with the new database.
- Although Postfix uses file locking to avoid access conflicts
-while updating Berkeley DB or other local database files, you still
-have a problem when the update fails because the disk is full or
-because something else happens. This is because commands such as
-postmap(1) or postalias(1) overwrite existing files. If the update
+
Postfix uses file locking to avoid access conflicts while
+updating Berkeley DB or other local database files. This used to
+be safe, but as Berkeley DB has evolved to use more aggressive
+caching, file locking may no longer be sufficient.
+
+ Furthermore, file locking would not prevent problems when the
+update fails because the disk is full or something else causes a
+database update to fail. In particular, commands such as postmap(1)
+or postalias(1) overwrite existing files. If the overwrite
fails in the middle then you have no usable database, and Postfix
will stop working. This is not an issue with the CDB database type
available with Postfix 2.2 and later: CDB
creates a new file, and renames the file upon successful completion.
- With multi-file databases such as DBM, there is no simple
-solution. With Berkeley DB and other "one file" databases, it is
+
With Berkeley DB and other "one file" databases, it is
possible to add some extra robustness by using "mv" to REPLACE an
existing database file instead of overwriting it:
diff --git a/postfix/proto/LMDB_README.html b/postfix/proto/LMDB_README.html
index b69f81085..91e3a7fc4 100644
--- a/postfix/proto/LMDB_README.html
+++ b/postfix/proto/LMDB_README.html
@@ -17,35 +17,8 @@
-
-
-
- Postfix LMDB support is forbidden due to problems with LMDB lock
-management. These problems hinder error recovery in multi-programmed
-systems, and prohibit database sharing between privileged writer
-processes and unprivileged reader processes.
-
-
+
+
+
+
+
+
+
+
- Non-obvious recovery with
-postscreen(8), tlsmgr(8), or verify(8) from a corrupted database.
-
+ Non-obvious recovery with postmap(1), postalias(1), or
+tlsmgr(8) from a corrupted database.
-- Problem:
-
You cannot rebuild a corrupted LMDB
-database simply by waiting until a daemon restarts. This problem does not exist
-with other Postfix databases.
+- Problem:
-
A corrupted LMDB database cann't be
+rebuilt simply by re-running postmap(1) or postalias(1), or by
+waiting until a tlsmgr(8) daemon restarts. This problem does not
+exist with other Postfix databases.
- Background:
-
The Postfix LMDB database client
does not truncate the database file. Instead it attempts to create
@@ -323,10 +309,10 @@ That is obviously not possible with a corrupted database file.
someone fixes the problem.
- Recovery:
-
First delete the ".lmdb" file by hand.
-Then, restart
-postfix.
+Then rebuild the file with the postmap(1) or postalias(1)
+command if the file was created with those commands, or restart
+postfix daemons if the file is maintained by tlsmgr(8).
+
Prevention:
@@ -337,8 +323,7 @@ space.
in-memory file system data and metadata.
Use a file system such as ZFS to detect and correct silent
-corruption of on-disk file system data and metadata.
+corruption of on-disk file system data and metadata. DO NOT
+use ZFS on systems without ECC memory error correction.
-
--->
diff --git a/postfix/proto/postconf.proto b/postfix/proto/postconf.proto
index 993cf8fb6..1a72ee4e5 100644
--- a/postfix/proto/postconf.proto
+++ b/postfix/proto/postconf.proto
@@ -1975,8 +1975,9 @@ IPV6_V6ONLY support (RFC 3493).
Postfix will do DNS type AAAA record lookups.
When both IPv4 and IPv6 support are enabled, the Postfix SMTP
-client will attempt to connect via IPv6 before attempting to use
-IPv4.
+client will choose the protocol as specified with the
+smtp_address_preference parameter. Postfix versions before 2.8
+attempt to connect via IPv6 before attempting to use IPv4.
Examples:
@@ -2848,15 +2849,6 @@ a database becomes full, its size limit is doubled.
This feature is available in Postfix 2.11 and later.
-%PARAM lmdb_max_readers $default_process_limit
-
- The hard limit on the number of read transactions that may be
-open at the same time for the same OpenLDAP LMDB database. When
-this number is too small, the Postfix LMDB client will log
-MDB_READERS_FULL errors, and will run with reduced performance.
-
-
-
%PARAM message_size_limit 10240000
@@ -4982,6 +4974,16 @@ action. Note: a result of "OK" is not allowed for safety reasons.
Instead, use DUNNO in order to exclude specific hosts from blacklists.
This feature is available in Postfix 2.7 and later.
+
check_sasl_access type:table
+
+ Use the remote SMTP client SASL user name as lookup key for
+the specified access(5) database. The lookup key has the form
+"username@domainname" when the smtpd_sasl_local_domain parameter
+value is non-empty. Unlike the check_client_access feature,
+check_sasl_access does not perform matches of parent domains or IP
+subnet ranges. This feature is available with Postfix version 2.11
+and later.
+
permit_inet_interfaces
Permit the request when the client IP address matches
diff --git a/postfix/src/discard/discard.c b/postfix/src/discard/discard.c
index 4f90926b6..fec6a9f85 100644
--- a/postfix/src/discard/discard.c
+++ b/postfix/src/discard/discard.c
@@ -9,16 +9,16 @@
/* The Postfix \fBdiscard\fR(8) delivery agent processes
/* delivery requests from
/* the queue manager. Each request specifies a queue file, a sender
-/* address, a domain or host name that is treated as the reason for
+/* address, a next-hop destination that is treated as the reason for
/* discarding the mail, and recipient information.
/* The reason may be prefixed with an RFC 3463-compatible detail code.
/* This program expects to be run from the \fBmaster\fR(8) process
/* manager.
/*
/* The \fBdiscard\fR(8) delivery agent pretends to deliver all recipients
-/* in the delivery request, logs the "next-hop" domain or host
-/* information as the reason for discarding the mail, updates the
-/* queue file and marks recipients as finished or informs the
+/* in the delivery request, logs the "next-hop" destination
+/* as the reason for discarding the mail, updates the
+/* queue file, and either marks recipients as finished or informs the
/* queue manager that delivery should be tried again at a later time.
/*
/* Delivery status reports are sent to the \fBtrace\fR(8)
diff --git a/postfix/src/global/mail_params.c b/postfix/src/global/mail_params.c
index 3dabacbcd..b2ec918d8 100644
--- a/postfix/src/global/mail_params.c
+++ b/postfix/src/global/mail_params.c
@@ -98,7 +98,6 @@
/* int var_db_read_buf;
/* long var_lmdb_map_size;
/* int var_proc_limit;
-/* int var_lmdb_max_readers;
/* int var_mime_maxdepth;
/* int var_mime_bound_len;
/* int var_header_limit;
@@ -291,7 +290,6 @@ char *var_proxywrite_service;
int var_db_create_buf;
int var_db_read_buf;
long var_lmdb_map_size;
-int var_lmdb_max_readers;
int var_proc_limit;
int var_mime_maxdepth;
int var_mime_bound_len;
@@ -614,13 +612,9 @@ void mail_params_init()
VAR_INET_WINDOW, DEF_INET_WINDOW, &var_inet_windowsize, 0, 0,
0,
};
- static const CONFIG_NINT_TABLE nint_defaults[] = {
- VAR_LMDB_MAX_READERS, DEF_LMDB_MAX_READERS, &var_lmdb_max_readers, 1, 0,
- 0,
- };
static const CONFIG_LONG_TABLE long_defaults[] = {
VAR_MESSAGE_LIMIT, DEF_MESSAGE_LIMIT, &var_message_limit, 0, 0,
- VAR_LMDB_MAP_SIZE, DEF_LMDB_MAP_SIZE, &var_lmdb_map_size, 8192, 0,
+ VAR_LMDB_MAP_SIZE, DEF_LMDB_MAP_SIZE, &var_lmdb_map_size, 1, 0,
0,
};
static const CONFIG_TIME_TABLE time_defaults[] = {
@@ -718,7 +712,6 @@ void mail_params_init()
}
#endif
get_mail_conf_int_table(other_int_defaults);
- get_mail_conf_nint_table(nint_defaults);
get_mail_conf_long_table(long_defaults);
get_mail_conf_bool_table(bool_defaults);
get_mail_conf_time_table(time_defaults);
@@ -731,7 +724,6 @@ void mail_params_init()
#endif
#ifdef HAS_LMDB
dict_lmdb_map_size = var_lmdb_map_size;
- dict_lmdb_max_readers = var_lmdb_max_readers;
#endif
inet_windowsize = var_inet_windowsize;
diff --git a/postfix/src/global/mail_params.h b/postfix/src/global/mail_params.h
index 8a8e5b5e5..063977b4f 100644
--- a/postfix/src/global/mail_params.h
+++ b/postfix/src/global/mail_params.h
@@ -2184,6 +2184,7 @@ extern int var_map_defer_code;
#define CHECK_CLIENT_ACL "check_client_access"
#define CHECK_REVERSE_CLIENT_ACL "check_reverse_client_hostname_access"
#define CHECK_CCERT_ACL "check_ccert_access"
+#define CHECK_SASL_ACL "check_sasl_access"
#define CHECK_HELO_ACL "check_helo_access"
#define CHECK_SENDER_ACL "check_sender_access"
#define CHECK_RECIP_ACL "check_recipient_access"
@@ -2775,10 +2776,6 @@ extern int var_db_read_buf;
#define DEF_LMDB_MAP_SIZE (16 * 1024 *1024)
extern long var_lmdb_map_size;
-#define VAR_LMDB_MAX_READERS "lmdb_max_readers"
-#define DEF_LMDB_MAX_READERS "$" VAR_PROC_LIMIT
-extern int var_lmdb_max_readers;
-
/*
* Named queue file attributes.
*/
diff --git a/postfix/src/global/mail_version.h b/postfix/src/global/mail_version.h
index c306fbbaf..47daf13c3 100644
--- a/postfix/src/global/mail_version.h
+++ b/postfix/src/global/mail_version.h
@@ -20,7 +20,7 @@
* Patches change both the patchlevel and the release date. Snapshots have no
* patchlevel; they change the release date only.
*/
-#define MAIL_RELEASE_DATE "20131001"
+#define MAIL_RELEASE_DATE "20131031"
#define MAIL_VERSION_NUMBER "2.11"
#ifdef SNAPSHOT
diff --git a/postfix/src/global/mkmap_open.c b/postfix/src/global/mkmap_open.c
index f84ef692d..80df1bd21 100644
--- a/postfix/src/global/mkmap_open.c
+++ b/postfix/src/global/mkmap_open.c
@@ -103,7 +103,6 @@ static const MKMAP_OPEN_INFO mkmap_types[] = {
DICT_TYPE_BTREE, mkmap_btree_open,
#endif
#ifdef HAS_LMDB
-#error "LMDB support is forbidden"
DICT_TYPE_LMDB, mkmap_lmdb_open,
#endif
DICT_TYPE_FAIL, mkmap_fail_open,
@@ -189,7 +188,11 @@ MKMAP *mkmap_open(const char *type, const char *path,
/*
* Truncate the database upon open, and update it. Read-write mode is
- * needed because the underlying routines read as well as write.
+ * needed because the underlying routines read as well as write. We
+ * explicitly clobber lock_fd to trigger a fatal error when a map wants
+ * to unlock the database after individual transactions: that would
+ * result in race condition problems. We clobbber stat_fd as well,
+ * because that, too, is used only for individual-transaction clients.
*/
mkmap->dict = mkmap->open(path, open_flags, dict_flags);
mkmap->dict->lock_fd = -1; /* XXX just in case */
diff --git a/postfix/src/postscreen/postscreen_early.c b/postfix/src/postscreen/postscreen_early.c
index a540703d9..459584715 100644
--- a/postfix/src/postscreen/postscreen_early.c
+++ b/postfix/src/postscreen/postscreen_early.c
@@ -120,6 +120,9 @@ static void psc_early_event(int event, char *context)
* XXX We can avoid "forgetting" to do this by keeping a pointer to the
* DNSBL lookup buffer in the PSC_STATE structure. This also allows us to
* shave off a hash table lookup when retrieving the DNSBL result.
+ *
+ * A direct pointer increases the odds of dangling pointers. Hash-table
+ * lookup is safer, and that is why it's done that way.
*/
switch (event) {
diff --git a/postfix/src/smtpd/smtpd_check.c b/postfix/src/smtpd/smtpd_check.c
index 3b9508328..4532b67d4 100644
--- a/postfix/src/smtpd/smtpd_check.c
+++ b/postfix/src/smtpd/smtpd_check.c
@@ -2823,6 +2823,26 @@ static int check_ccert_access(SMTPD_STATE *state, const char *table,
return (result);
}
+/* check_sasl_access - access by SASL user name */
+
+#ifdef USE_SASL_AUTH
+
+static int check_sasl_access(SMTPD_STATE *state, const char *table,
+ const char *def_acl)
+{
+ int result;
+ int unused_found;
+ char *sane_username = printable(mystrdup(state->sasl_username), '_');
+
+ result = check_access(state, table, state->sasl_username,
+ DICT_FLAG_NONE, &unused_found, sane_username,
+ SMTPD_NAME_SASL_USER, def_acl);
+ myfree(sane_username);
+ return (result);
+}
+
+#endif
+
/* check_mail_access - OK/FAIL based on mail address lookup */
static int check_mail_access(SMTPD_STATE *state, const char *table,
@@ -3882,6 +3902,14 @@ static int generic_checks(SMTPD_STATE *state, ARGV *restrictions,
}
} else if (is_map_command(state, name, CHECK_CCERT_ACL, &cpp)) {
status = check_ccert_access(state, *cpp, def_acl);
+#ifdef USE_SASL_AUTH
+ } else if (is_map_command(state, name, CHECK_SASL_ACL, &cpp)) {
+ if (var_smtpd_sasl_enable) {
+ if (state->sasl_username && state->sasl_username[0])
+ status = check_sasl_access(state, *cpp, def_acl);
+ } else
+#endif
+ msg_warn("restriction `%s' ignored: no SASL support", name);
} else if (is_map_command(state, name, CHECK_CLIENT_NS_ACL, &cpp)) {
if (strcasecmp(state->name, "unknown") != 0) {
status = check_server_access(state, *cpp, state->name,
diff --git a/postfix/src/smtpd/smtpd_dsn_fix.h b/postfix/src/smtpd/smtpd_dsn_fix.h
index 30e0c93d3..c608e34cf 100644
--- a/postfix/src/smtpd/smtpd_dsn_fix.h
+++ b/postfix/src/smtpd/smtpd_dsn_fix.h
@@ -15,6 +15,7 @@
#define SMTPD_NAME_CLIENT "Client host"
#define SMTPD_NAME_REV_CLIENT "Unverified Client host"
#define SMTPD_NAME_CCERT "Client certificate"
+#define SMTPD_NAME_SASL_USER "SASL login name"
#define SMTPD_NAME_HELO "Helo command"
#define SMTPD_NAME_SENDER "Sender address"
#define SMTPD_NAME_RECIPIENT "Recipient address"
diff --git a/postfix/src/util/Makefile.in b/postfix/src/util/Makefile.in
index 0dacf13b8..609bd0b7b 100644
--- a/postfix/src/util/Makefile.in
+++ b/postfix/src/util/Makefile.in
@@ -36,7 +36,7 @@ SRCS = alldig.c allprint.c argv.c argv_split.c attr_clnt.c attr_print0.c \
ip_match.c nbbio.c base32_code.c dict_test.c \
dict_fail.c msg_rate_delay.c dict_surrogate.c warn_stat.c \
dict_sockmap.c line_number.c recv_pass_attr.c pass_accept.c \
- poll_fd.c timecmp.c
+ poll_fd.c timecmp.c slmdb.c
OBJS = alldig.o allprint.o argv.o argv_split.o attr_clnt.o attr_print0.o \
attr_print64.o attr_print_plain.o attr_scan0.o attr_scan64.o \
attr_scan_plain.o auto_clnt.o base64_code.o basename.o binhash.o \
@@ -74,7 +74,7 @@ OBJS = alldig.o allprint.o argv.o argv_split.o attr_clnt.o attr_print0.o \
ip_match.o nbbio.o base32_code.o dict_test.o \
dict_fail.o msg_rate_delay.o dict_surrogate.o warn_stat.o \
dict_sockmap.o line_number.o recv_pass_attr.o pass_accept.o \
- poll_fd.o timecmp.o
+ poll_fd.o timecmp.o slmdb.o
HDRS = argv.h attr.h attr_clnt.h auto_clnt.h base64_code.h binhash.h \
chroot_uid.h cidr_match.h clean_env.h connect.h ctable.h dict.h \
dict_cdb.h dict_cidr.h dict_db.h dict_dbm.h dict_env.h dict_ht.h \
@@ -95,7 +95,8 @@ HDRS = argv.h attr.h attr_clnt.h auto_clnt.h base64_code.h binhash.h \
username.h valid_hostname.h vbuf.h vbuf_print.h vstream.h vstring.h \
vstring_vstream.h watchdog.h format_tv.h load_file.h killme_after.h \
edit_file.h dict_cache.h dict_thash.h ip_match.h nbbio.h base32_code.h \
- dict_fail.h warn_stat.h dict_sockmap.h line_number.h timecmp.h
+ dict_fail.h warn_stat.h dict_sockmap.h line_number.h timecmp.h \
+ slmdb.h
TESTSRC = fifo_open.c fifo_rdwr_bug.c fifo_rdonly_bug.c select_bug.c \
stream_test.c dup2_pass_on_exec.c
DEFS = -I. -D$(SYSTYPE)
@@ -1002,6 +1003,7 @@ dict_lmdb.o: iostuff.h
dict_lmdb.o: msg.h
dict_lmdb.o: myflock.h
dict_lmdb.o: mymalloc.h
+dict_lmdb.o: slmdb.h
dict_lmdb.o: stringops.h
dict_lmdb.o: sys_defs.h
dict_lmdb.o: vbuf.h
@@ -1764,6 +1766,8 @@ skipblanks.o: stringops.h
skipblanks.o: sys_defs.h
skipblanks.o: vbuf.h
skipblanks.o: vstring.h
+slmdb.o: slmdb.c
+slmdb.o: slmdb.h
sock_addr.o: msg.h
sock_addr.o: sock_addr.c
sock_addr.o: sock_addr.h
diff --git a/postfix/src/util/dict_lmdb.c b/postfix/src/util/dict_lmdb.c
index 27e8542da..f02f93267 100644
--- a/postfix/src/util/dict_lmdb.c
+++ b/postfix/src/util/dict_lmdb.c
@@ -7,7 +7,6 @@
/* #include
/*
/* size_t dict_lmdb_map_size;
-/* unsigned int dict_lmdb_max_readers;
/*
/* DICT *dict_lmdb_open(path, open_flags, dict_flags)
/* const char *name;
@@ -22,18 +21,13 @@
/* The dict_lmdb_map_size variable specifies the initial
/* database memory map size. When a map becomes full its size
/* is doubled, and other programs pick up the size change.
-/*
-/* The dict_lmdb_max_readers variable specifies the hard (ugh)
-/* limit on the number of read transactions that may be open
-/* at the same time. This should be propertional to the number
-/* of processes that read the table.
/* DIAGNOSTICS
/* Fatal errors: cannot open file, file write error, out of
/* memory.
/* BUGS
/* The on-the-fly map resize operations require no concurrent
/* activity in the same database by other threads in the same
-/* process.
+/* memory address space.
/* SEE ALSO
/* dict(3) generic dictionary manager
/* LICENSE
@@ -61,21 +55,6 @@
#include
#include
-#ifdef PATH_LMDB_H
-#include PATH_LMDB_H
-#else
-#include
-#endif
-
- /*
- * As of LMDB 0.9.8 the database size limit can be updated on-the-fly. The
- * only limit that remains is imposed by the hardware address space. Earlier
- * LMDB versions are not suitable for use with Postfix.
- */
-#if MDB_VERSION_FULL < MDB_VERINT(0, 9, 8)
-#error "Build with LMDB version 0.9.8 or later"
-#endif
-
/* Utility library. */
#include
@@ -85,6 +64,7 @@
#include
#include
#include
+#include
#include
#include
#include
@@ -93,19 +73,9 @@
typedef struct {
DICT dict; /* generic members */
- MDB_env *env; /* LMDB environment */
- MDB_dbi dbi; /* database handle */
- MDB_txn *txn; /* bulk update transaction */
- MDB_cursor *cursor; /* for sequence ops */
- size_t map_size; /* per-database size limit */
+ SLMDB slmdb; /* sane LMDB API */
VSTRING *key_buf; /* key buffer */
VSTRING *val_buf; /* value buffer */
- /* The following facilitate LMDB quirk workarounds. */
- int dict_api_retries; /* workarounds per dict(3) call */
- int bulk_mode_retries; /* workarounds per bulk transaction */
- int dict_open_flags; /* dict(3) open flags */
- int mdb_open_flags; /* LMDB open flags */
- int readers_full; /* MDB_READERS_FULL errors */
} DICT_LMDB;
/*
@@ -137,411 +107,15 @@ typedef struct {
#define DICT_LMDB_SIZE_INCR 2 /* Increase size by 1 bit on retry */
#define DICT_LMDB_SIZE_MAX SSIZE_T_MAX
-#define DICT_LMDB_API_RETRY_LIMIT 100 /* Retries per dict(3) API call */
+#define DICT_LMDB_API_RETRY_LIMIT 2 /* Retries per dict(3) API call */
#define DICT_LMDB_BULK_RETRY_LIMIT \
- (2 * sizeof(size_t) * CHAR_BIT) /* Retries per bulk-mode transaction */
+ ((int) (2 * sizeof(size_t) * CHAR_BIT)) /* Retries per bulk-mode
+ * transaction */
- /*
- * XXX Should dict_lmdb_max_readers be configurable? Is this a per-database
- * property? Per-process? Does it need to be the same for all processes?
- */
size_t dict_lmdb_map_size = 8192; /* Minimum size without SIGSEGV */
-unsigned int dict_lmdb_max_readers = 216; /* 200 postfix processes,
- * plus some extra */
/* #define msg_verbose 1 */
- /*
- * The purpose of the error-recovering functions below is to hide LMDB
- * quirks (MAP_FULL, MAP_CHANGED, READERS_FULL), so that the dict(3) API
- * routines can pretend that those quirks don't exist, and focus on their
- * own job.
- *
- * - To recover from a single-transaction LMDB error, each wrapper function
- * uses tail recursion instead of goto. Since LMDB errors are rare, code
- * clarity is more important than speed.
- *
- * - To recover from a bulk-mode transaction LMDB error, the error-recovery
- * code jumps back into the caller to some pre-arranged point (the closest
- * thing that C has to exception handling). With postmap, this means that
- * bulk-mode LMDB error recovery is limited to input that is seekable.
- */
-
-/* dict_lmdb_prepare - LMDB-specific (re)initialization before actual access */
-
-static void dict_lmdb_prepare(DICT_LMDB *dict_lmdb)
-{
- int status;
-
- /*
- * This is called before accessing the database, or after recovery from
- * an LMDB error. dict_lmdb->txn is either the database open()
- * transaction or a freshly-created bulk-mode transaction.
- *
- * - With O_TRUNC we make a "drop" request before populating the database.
- *
- * - With DICT_FLAG_BULK_UPDATE we commit a bulk-mode transaction when the
- * database is closed.
- */
- if (dict_lmdb->dict_open_flags & O_TRUNC) {
- if ((status = mdb_drop(dict_lmdb->txn, dict_lmdb->dbi, 0)) != 0)
- msg_fatal("truncate %s:%s: %s",
- dict_lmdb->dict.type, dict_lmdb->dict.name,
- mdb_strerror(status));
- if ((dict_lmdb->dict.flags & DICT_FLAG_BULK_UPDATE) == 0) {
- if ((status = mdb_txn_commit(dict_lmdb->txn)))
- msg_fatal("truncate %s:%s: %s",
- dict_lmdb->dict.type, dict_lmdb->dict.name,
- mdb_strerror(status));
- dict_lmdb->txn = NULL;
- }
- } else if ((dict_lmdb->mdb_open_flags & MDB_RDONLY) != 0
- || (dict_lmdb->dict.flags & DICT_FLAG_BULK_UPDATE) == 0) {
- mdb_txn_abort(dict_lmdb->txn);
- dict_lmdb->txn = NULL;
- }
-}
-
-/* dict_lmdb_recover - recover from LMDB errors */
-
-static int dict_lmdb_recover(DICT_LMDB *dict_lmdb, int status)
-{
- const char *myname = "dict_lmdb_recover";
- MDB_envinfo info;
-
- /*
- * Limit the number of recovery attempts per dict(3) API request.
- */
- if ((dict_lmdb->dict_api_retries += 1) > DICT_LMDB_API_RETRY_LIMIT) {
- if (msg_verbose)
- msg_info("%s: %s:%s too many recovery attempts %d",
- myname, dict_lmdb->dict.type, dict_lmdb->dict.name,
- dict_lmdb->dict_api_retries);
- return (status);
- }
-
- /*
- * If we can recover from the error, we clear the error condition and the
- * caller should retry the failed operation immediately. Otherwise, the
- * caller should terminate with a fatal run-time error and the program
- * should be re-run later.
- *
- * dict_lmdb->txn is either null (non-bulk transaction error), or an aborted
- * bulk-mode transaction. If we want to make this wrapper layer suitable
- * for general use, then the bulk/non-bulk distinction should be made
- * less specific to Postfix.
- */
- switch (status) {
-
- /*
- * As of LMDB 0.9.8 when a non-bulk update runs into a "map full"
- * error, we can resize the environment's memory map and clear the
- * error condition. The caller should retry immediately.
- */
- case MDB_MAP_FULL:
- /* Can we increase the memory map? Give up if we can't. */
- if (dict_lmdb->map_size < DICT_LMDB_SIZE_MAX / DICT_LMDB_SIZE_INCR) {
- dict_lmdb->map_size = dict_lmdb->map_size * DICT_LMDB_SIZE_INCR;
- } else if (dict_lmdb->map_size < DICT_LMDB_SIZE_MAX) {
- dict_lmdb->map_size = DICT_LMDB_SIZE_MAX;
- } else {
- /* Sorry, but we are already maxed out. */
- break;
- }
- /* Resize the memory map. */
- if (msg_verbose)
- msg_info("updating database %s:%s size limit to %lu",
- dict_lmdb->dict.type, dict_lmdb->dict.name,
- (unsigned long) dict_lmdb->map_size);
- if ((status = mdb_env_set_mapsize(dict_lmdb->env,
- dict_lmdb->map_size)) != 0)
- msg_fatal("env_set_mapsize %s:%s to %lu: %s",
- dict_lmdb->dict.type, dict_lmdb->dict.name,
- (unsigned long) dict_lmdb->map_size,
- mdb_strerror(status));
- break;
-
- /*
- * When a writer resizes the database, read-only applications must
- * increase their LMDB memory map size limit, too. Otherwise, they
- * won't be able to read a table after it grows.
- *
- * As of LMDB 0.9.8 we can import the new memory map size limit into the
- * database environment by calling mdb_env_set_mapsize() with a zero
- * size argument. Then we extract the map size limit for later use.
- * The caller should retry immediately.
- */
- case MDB_MAP_RESIZED:
- if ((status = mdb_env_set_mapsize(dict_lmdb->env, 0)) != 0)
- msg_fatal("env_set_mapsize %s:%s to 0: %s",
- dict_lmdb->dict.type, dict_lmdb->dict.name,
- mdb_strerror(status));
- /* Do not panic. Maps may shrink after bulk update. */
- mdb_env_info(dict_lmdb->env, &info);
- dict_lmdb->map_size = info.me_mapsize;
- if (msg_verbose)
- msg_info("importing database %s:%s new size limit %lu",
- dict_lmdb->dict.type, dict_lmdb->dict.name,
- (unsigned long) dict_lmdb->map_size);
- break;
-
- /*
- * What is it with these built-in hard limits that cause systems to
- * fail when resources are needed most? When the system is under
- * stress it should slow down, not stop working.
- */
- case MDB_READERS_FULL:
- if (dict_lmdb->readers_full++ == 0)
- msg_warn("database %s:%s: %s - increase lmdb_max_readers",
- dict_lmdb->dict.type, dict_lmdb->dict.name,
- mdb_strerror(status));
- rand_sleep(1000000, 1000000);
- status = 0;
- break;
-
- /*
- * We can't solve this problem. The application should terminate with
- * a fatal run-time error and the program should be re-run later.
- */
- default:
- break;
- }
-
- /*
- * If a bulk-mode transaction error is recoverable, build a new bulk-mode
- * transaction from scratch, by making a long jump back into the caller
- * at some pre-arranged point.
- */
- if (dict_lmdb->txn != 0 && status == 0
- && (dict_lmdb->bulk_mode_retries += 1) <= DICT_LMDB_BULK_RETRY_LIMIT) {
- status = mdb_txn_begin(dict_lmdb->env, NULL,
- dict_lmdb->mdb_open_flags & MDB_RDONLY,
- &dict_lmdb->txn);
- if (status != 0)
- msg_fatal("txn_begin %s:%s: %s",
- dict_lmdb->dict.type, dict_lmdb->dict.name,
- mdb_strerror(status));
- dict_lmdb_prepare(dict_lmdb);
- dict_longjmp(&dict_lmdb->dict, 1);
- }
- return (status);
-}
-
-/* dict_lmdb_txn_begin - mdb_txn_begin() wrapper with LMDB error recovery */
-
-static void dict_lmdb_txn_begin(DICT_LMDB *dict_lmdb, int rdonly, MDB_txn **txn)
-{
- int status;
-
- if ((status = mdb_txn_begin(dict_lmdb->env, NULL, rdonly, txn)) != 0) {
- if ((status = dict_lmdb_recover(dict_lmdb, status)) == 0) {
- dict_lmdb_txn_begin(dict_lmdb, rdonly, txn);
- return;
- }
- msg_fatal("%s:%s: error starting %s transaction: %s",
- dict_lmdb->dict.type, dict_lmdb->dict.name,
- rdonly ? "read" : "write", mdb_strerror(status));
- }
-}
-
-/* dict_lmdb_get - mdb_get() wrapper with LMDB error recovery */
-
-static int dict_lmdb_get(DICT_LMDB *dict_lmdb, MDB_val *mdb_key,
- MDB_val *mdb_value)
-{
- MDB_txn *txn;
- int status;
-
- /*
- * Start a read transaction if there's no bulk-mode txn.
- */
- if (dict_lmdb->txn)
- txn = dict_lmdb->txn;
- else
- dict_lmdb_txn_begin(dict_lmdb, MDB_RDONLY, &txn);
-
- /*
- * Do the lookup.
- */
- if ((status = mdb_get(txn, dict_lmdb->dbi, mdb_key, mdb_value)) != 0
- && status != MDB_NOTFOUND) {
- mdb_txn_abort(txn);
- if (dict_lmdb->txn == 0)
- txn = 0;
- if ((status = dict_lmdb_recover(dict_lmdb, status)) == 0)
- return (dict_lmdb_get(dict_lmdb, mdb_key, mdb_value));
- }
-
- /*
- * Close the read txn if it's not the bulk-mode txn.
- */
- if (txn && dict_lmdb->txn == 0)
- mdb_txn_abort(txn);
-
- return (status);
-}
-
-/* dict_lmdb_put - mdb_put() wrapper with LMDB error recovery */
-
-static int dict_lmdb_put(DICT_LMDB *dict_lmdb, MDB_val *mdb_key,
- MDB_val *mdb_value, int flags)
-{
- MDB_txn *txn;
- int status;
-
- /*
- * Start a write transaction if there's no bulk-mode txn.
- */
- if (dict_lmdb->txn)
- txn = dict_lmdb->txn;
- else
- dict_lmdb_txn_begin(dict_lmdb, 0, &txn);
-
- /*
- * Do the update.
- */
- if ((status = mdb_put(txn, dict_lmdb->dbi, mdb_key, mdb_value, flags)) != 0
- && status != MDB_KEYEXIST) {
- mdb_txn_abort(txn);
- if (dict_lmdb->txn == 0)
- txn = 0;
- if ((status = dict_lmdb_recover(dict_lmdb, status)) == 0)
- return (dict_lmdb_put(dict_lmdb, mdb_key, mdb_value, flags));
- }
-
- /*
- * Commit the transaction if it's not the bulk-mode txn.
- */
- if (txn && dict_lmdb->txn == 0 && (status = mdb_txn_commit(txn)) != 0) {
- if ((status = dict_lmdb_recover(dict_lmdb, status)) == 0)
- return (dict_lmdb_put(dict_lmdb, mdb_key, mdb_value, flags));
- msg_fatal("error committing database %s:%s: %s",
- dict_lmdb->dict.type, dict_lmdb->dict.name,
- mdb_strerror(status));
- }
- return (status);
-}
-
-/* dict_lmdb_del - mdb_del() wrapper with LMDB error recovery */
-
-static int dict_lmdb_del(DICT_LMDB *dict_lmdb, MDB_val *mdb_key)
-{
- MDB_txn *txn;
- int status;
-
- /*
- * Start a write transaction if there's no bulk-mode txn.
- */
- if (dict_lmdb->txn)
- txn = dict_lmdb->txn;
- else
- dict_lmdb_txn_begin(dict_lmdb, 0, &txn);
-
- /*
- * Do the update.
- */
- if ((status = mdb_del(txn, dict_lmdb->dbi, mdb_key, NULL)) != 0
- && status != MDB_NOTFOUND) {
- mdb_txn_abort(txn);
- if (dict_lmdb->txn == 0)
- txn = 0;
- if ((status = dict_lmdb_recover(dict_lmdb, status)) == 0)
- return (dict_lmdb_del(dict_lmdb, mdb_key));
- }
-
- /*
- * Commit the transaction if it's not the bulk-mode txn.
- */
- if (txn && dict_lmdb->txn == 0 && (status = mdb_txn_commit(txn)) != 0) {
- if ((status = dict_lmdb_recover(dict_lmdb, status)) == 0)
- return (dict_lmdb_del(dict_lmdb, mdb_key));
- msg_fatal("error committing database %s:%s: %s",
- dict_lmdb->dict.type, dict_lmdb->dict.name,
- mdb_strerror(status));
- }
- return (status);
-}
-
-/* dict_lmdb_cursor_get - mdb_cursor_get() wrapper with LMDB error recovery */
-
-static int dict_lmdb_cursor_get(DICT_LMDB *dict_lmdb, MDB_val *mdb_key,
- MDB_val *mdb_value, MDB_cursor_op op)
-{
- MDB_txn *txn;
- int status;
-
- /*
- * Open a read transaction and cursor if needed.
- */
- if (dict_lmdb->cursor == 0) {
- dict_lmdb_txn_begin(dict_lmdb, MDB_RDONLY, &txn);
- if ((status = mdb_cursor_open(txn, dict_lmdb->dbi, &dict_lmdb->cursor))) {
- if ((status = dict_lmdb_recover(dict_lmdb, status)) == 0)
- return (dict_lmdb_cursor_get(dict_lmdb, mdb_key, mdb_value, op));
- msg_fatal("%s:%s: cursor_open database: %s",
- dict_lmdb->dict.type, dict_lmdb->dict.name,
- mdb_strerror(status));
- }
- }
-
- /*
- * Database lookup.
- */
- status = mdb_cursor_get(dict_lmdb->cursor, mdb_key, mdb_value, op);
-
- /*
- * Handle end-of-database or other error.
- */
- if (status != 0) {
- if (status == MDB_NOTFOUND) {
- txn = mdb_cursor_txn(dict_lmdb->cursor);
- mdb_cursor_close(dict_lmdb->cursor);
- mdb_txn_abort(txn);
- dict_lmdb->cursor = 0;
- } else {
- if ((status = dict_lmdb_recover(dict_lmdb, status)) == 0)
- return (dict_lmdb_cursor_get(dict_lmdb, mdb_key, mdb_value, op));
- }
- }
- return (status);
-}
-
-/* dict_lmdb_finish - wrapper with LMDB error recovery */
-
-static void dict_lmdb_finish(DICT_LMDB *dict_lmdb)
-{
- int status;
-
- /*
- * Finish the bulk-mode transaction. If dict_lmdb_recover() returns after
- * a bulk-mode transaction error, then it was unable to recover.
- */
- if (dict_lmdb->txn) {
- if ((status = mdb_txn_commit(dict_lmdb->txn)) != 0) {
- (void) dict_lmdb_recover(dict_lmdb, status);
- msg_fatal("%s:%s: closing dictionary: %s",
- dict_lmdb->dict.type, dict_lmdb->dict.name,
- mdb_strerror(status));
- }
- }
-
- /*
- * Clean up after an unfinished sequence() operation.
- */
- if (dict_lmdb->cursor) {
- MDB_txn *txn = mdb_cursor_txn(dict_lmdb->cursor);
-
- mdb_cursor_close(dict_lmdb->cursor);
- mdb_txn_abort(txn);
- }
-}
-
- /*
- * With all recovery from LMDB quirks encapsulated in the routines above,
- * the dict(3) API routines below can pretend that LMDB quirks don't exist
- * and focus on their own job: accessing or updating the database.
- */
-
/* dict_lmdb_lookup - find database entry */
static const char *dict_lmdb_lookup(DICT *dict, const char *name)
@@ -553,7 +127,6 @@ static const char *dict_lmdb_lookup(DICT *dict, const char *name)
int status, klen;
dict->error = 0;
- dict_lmdb->dict_api_retries = 0;
klen = strlen(name);
/*
@@ -572,6 +145,13 @@ static const char *dict_lmdb_lookup(DICT *dict, const char *name)
name = lowercase(vstring_str(dict->fold_buf));
}
+ /*
+ * Acquire a shared lock.
+ */
+ if ((dict->flags & DICT_FLAG_LOCK)
+ && myflock(dict->lock_fd, MYFLOCK_STYLE_FCNTL, MYFLOCK_OP_SHARED) < 0)
+ msg_fatal("%s: lock dictionary: %m", dict->name);
+
/*
* See if this LMDB file was written with one null byte appended to key
* and value.
@@ -579,7 +159,7 @@ static const char *dict_lmdb_lookup(DICT *dict, const char *name)
if (dict->flags & DICT_FLAG_TRY1NULL) {
mdb_key.mv_data = (void *) name;
mdb_key.mv_size = klen + 1;
- status = dict_lmdb_get(dict_lmdb, &mdb_key, &mdb_value);
+ status = slmdb_get(&dict_lmdb->slmdb, &mdb_key, &mdb_value);
if (status == 0) {
dict->flags &= ~DICT_FLAG_TRY0NULL;
result = SCOPY(dict_lmdb->val_buf, mdb_value.mv_data,
@@ -598,7 +178,7 @@ static const char *dict_lmdb_lookup(DICT *dict, const char *name)
if (result == 0 && (dict->flags & DICT_FLAG_TRY0NULL)) {
mdb_key.mv_data = (void *) name;
mdb_key.mv_size = klen;
- status = dict_lmdb_get(dict_lmdb, &mdb_key, &mdb_value);
+ status = slmdb_get(&dict_lmdb->slmdb, &mdb_key, &mdb_value);
if (status == 0) {
dict->flags &= ~DICT_FLAG_TRY1NULL;
result = SCOPY(dict_lmdb->val_buf, mdb_value.mv_data,
@@ -609,6 +189,14 @@ static const char *dict_lmdb_lookup(DICT *dict, const char *name)
mdb_strerror(status));
}
}
+
+ /*
+ * Release the shared lock.
+ */
+ if ((dict->flags & DICT_FLAG_LOCK)
+ && myflock(dict->lock_fd, MYFLOCK_STYLE_FCNTL, MYFLOCK_OP_NONE) < 0)
+ msg_fatal("%s: unlock dictionary: %m", dict->name);
+
return (result);
}
@@ -622,7 +210,6 @@ static int dict_lmdb_update(DICT *dict, const char *name, const char *value)
int status;
dict->error = 0;
- dict_lmdb->dict_api_retries = 0;
/*
* Sanity check.
@@ -666,10 +253,17 @@ static int dict_lmdb_update(DICT *dict, const char *name, const char *value)
mdb_value.mv_size++;
}
+ /*
+ * Acquire an exclusive lock.
+ */
+ if ((dict->flags & DICT_FLAG_LOCK)
+ && myflock(dict->lock_fd, MYFLOCK_STYLE_FCNTL, MYFLOCK_OP_EXCLUSIVE) < 0)
+ msg_fatal("%s: lock dictionary: %m", dict->name);
+
/*
* Do the update.
*/
- status = dict_lmdb_put(dict_lmdb, &mdb_key, &mdb_value,
+ status = slmdb_put(&dict_lmdb->slmdb, &mdb_key, &mdb_value,
(dict->flags & DICT_FLAG_DUP_REPLACE) ? 0 : MDB_NOOVERWRITE);
if (status != 0) {
if (status == MDB_KEYEXIST) {
@@ -687,6 +281,14 @@ static int dict_lmdb_update(DICT *dict, const char *name, const char *value)
mdb_strerror(status));
}
}
+
+ /*
+ * Release the exclusive lock.
+ */
+ if ((dict->flags & DICT_FLAG_LOCK)
+ && myflock(dict->lock_fd, MYFLOCK_STYLE_FCNTL, MYFLOCK_OP_NONE) < 0)
+ msg_fatal("%s: unlock dictionary: %m", dict->name);
+
return (status);
}
@@ -699,7 +301,6 @@ static int dict_lmdb_delete(DICT *dict, const char *name)
int status = 1, klen;
dict->error = 0;
- dict_lmdb->dict_api_retries = 0;
klen = strlen(name);
/*
@@ -718,6 +319,13 @@ static int dict_lmdb_delete(DICT *dict, const char *name)
name = lowercase(vstring_str(dict->fold_buf));
}
+ /*
+ * Acquire an exclusive lock.
+ */
+ if ((dict->flags & DICT_FLAG_LOCK)
+ && myflock(dict->lock_fd, MYFLOCK_STYLE_FCNTL, MYFLOCK_OP_EXCLUSIVE) < 0)
+ msg_fatal("%s: lock dictionary: %m", dict->name);
+
/*
* See if this LMDB file was written with one null byte appended to key
* and value.
@@ -725,7 +333,7 @@ static int dict_lmdb_delete(DICT *dict, const char *name)
if (dict->flags & DICT_FLAG_TRY1NULL) {
mdb_key.mv_data = (void *) name;
mdb_key.mv_size = klen + 1;
- status = dict_lmdb_del(dict_lmdb, &mdb_key);
+ status = slmdb_del(&dict_lmdb->slmdb, &mdb_key);
if (status != 0) {
if (status == MDB_NOTFOUND)
status = 1;
@@ -745,7 +353,7 @@ static int dict_lmdb_delete(DICT *dict, const char *name)
if (status > 0 && (dict->flags & DICT_FLAG_TRY0NULL)) {
mdb_key.mv_data = (void *) name;
mdb_key.mv_size = klen;
- status = dict_lmdb_del(dict_lmdb, &mdb_key);
+ status = slmdb_del(&dict_lmdb->slmdb, &mdb_key);
if (status != 0) {
if (status == MDB_NOTFOUND)
status = 1;
@@ -757,6 +365,14 @@ static int dict_lmdb_delete(DICT *dict, const char *name)
dict->flags &= ~DICT_FLAG_TRY1NULL; /* found */
}
}
+
+ /*
+ * Release the exclusive lock.
+ */
+ if ((dict->flags & DICT_FLAG_LOCK)
+ && myflock(dict->lock_fd, MYFLOCK_STYLE_FCNTL, MYFLOCK_OP_NONE) < 0)
+ msg_fatal("%s: unlock dictionary: %m", dict->name);
+
return (status);
}
@@ -773,7 +389,6 @@ static int dict_lmdb_sequence(DICT *dict, int function,
int status;
dict->error = 0;
- dict_lmdb->dict_api_retries = 0;
/*
* Determine the seek function.
@@ -789,10 +404,17 @@ static int dict_lmdb_sequence(DICT *dict, int function,
msg_panic("%s: invalid function: %d", myname, function);
}
+ /*
+ * Acquire a shared lock.
+ */
+ if ((dict->flags & DICT_FLAG_LOCK)
+ && myflock(dict->lock_fd, MYFLOCK_STYLE_FCNTL, MYFLOCK_OP_SHARED) < 0)
+ msg_fatal("%s: lock dictionary: %m", dict->name);
+
/*
* Database lookup.
*/
- status = dict_lmdb_cursor_get(dict_lmdb, &mdb_key, &mdb_value, op);
+ status = slmdb_cursor_get(&dict_lmdb->slmdb, &mdb_key, &mdb_value, op);
switch (status) {
@@ -811,6 +433,7 @@ static int dict_lmdb_sequence(DICT *dict, int function,
*/
case MDB_NOTFOUND:
status = 1;
+ /* Not: mdb_cursor_close(). Wrong abstraction level. */
break;
/*
@@ -821,28 +444,24 @@ static int dict_lmdb_sequence(DICT *dict, int function,
dict_lmdb->dict.type, dict_lmdb->dict.name,
mdb_strerror(status));
}
+
+ /*
+ * Release the shared lock.
+ */
+ if ((dict->flags & DICT_FLAG_LOCK)
+ && myflock(dict->lock_fd, MYFLOCK_STYLE_FCNTL, MYFLOCK_OP_NONE) < 0)
+ msg_fatal("%s: unlock dictionary: %m", dict->name);
+
return (status);
}
-/* dict_lmdb_lock - noop lock handler */
-
-static int dict_lmdb_lock(DICT *dict, int unused_op)
-{
- /* LMDB does its own concurrency control */
- return 0;
-}
-
/* dict_lmdb_close - disassociate from data base */
static void dict_lmdb_close(DICT *dict)
{
DICT_LMDB *dict_lmdb = (DICT_LMDB *) dict;
- dict_lmdb->dict_api_retries = 0;
- dict_lmdb_finish(dict_lmdb);
- if (dict_lmdb->dict.stat_fd >= 0)
- close(dict_lmdb->dict.stat_fd);
- mdb_env_close(dict_lmdb->env);
+ slmdb_close(&dict_lmdb->slmdb);
if (dict_lmdb->key_buf)
vstring_free(dict_lmdb->key_buf);
if (dict_lmdb->val_buf)
@@ -852,62 +471,96 @@ static void dict_lmdb_close(DICT *dict)
dict_free(dict);
}
+/* dict_lmdb_longjmp - debug logging */
+
+static void dict_lmdb_longjmp(void *context, int val)
+{
+ DICT_LMDB *dict_lmdb = (DICT_LMDB *) context;
+
+ dict_longjmp(&dict_lmdb->dict, val);
+}
+
+/* dict_lmdb_notify - debug logging */
+
+static void dict_lmdb_notify(void *context, int error_code,...)
+{
+ DICT_LMDB *dict_lmdb = (DICT_LMDB *) context;
+ va_list ap;
+
+ va_start(ap, error_code);
+ switch (error_code) {
+ case MDB_SUCCESS:
+ msg_info("database %s:%s: using size limit %lu during open",
+ dict_lmdb->dict.type, dict_lmdb->dict.name,
+ (unsigned long) va_arg(ap, size_t));
+ break;
+ case MDB_MAP_FULL:
+ msg_info("database %s:%s: using size limit %lu after MDB_MAP_FULL",
+ dict_lmdb->dict.type, dict_lmdb->dict.name,
+ (unsigned long) va_arg(ap, size_t));
+ break;
+ case MDB_MAP_RESIZED:
+ msg_info("database %s:%s: using size limit %lu after MDB_MAP_RESIZED",
+ dict_lmdb->dict.type, dict_lmdb->dict.name,
+ (unsigned long) va_arg(ap, size_t));
+ break;
+ case MDB_READERS_FULL:
+ msg_info("database %s:%s: pausing after MDB_READERS_FULL",
+ dict_lmdb->dict.type, dict_lmdb->dict.name);
+ break;
+ default:
+ msg_warn("unknown MDB error code: %d", error_code);
+ break;
+ }
+ va_end(ap);
+}
+
/* dict_lmdb_open - open LMDB data base */
-DICT *dict_lmdb_open(const char *path, int open_flags, int dict_flags)
+DICT *dict_lmdb_open(const char *path, int dict_open_flags, int dict_flags)
{
DICT_LMDB *dict_lmdb;
+ DICT *dict;
struct stat st;
- MDB_env *env;
- MDB_txn *txn;
- MDB_dbi dbi;
+ SLMDB slmdb;
char *mdb_path;
- int env_flags, status;
- size_t map_size = dict_lmdb_map_size;
+ int mdb_open_flags, status;
+ int db_fd;
mdb_path = concatenate(path, "." DICT_TYPE_LMDB, (char *) 0);
- env_flags = MDB_NOSUBDIR;
- if (open_flags == O_RDONLY)
- env_flags |= MDB_RDONLY;
-
- if ((status = mdb_env_create(&env)))
- msg_fatal("env_create %s: %s", mdb_path, mdb_strerror(status));
-
- if (stat(mdb_path, &st) == 0 && st.st_size > map_size) {
- if (st.st_size / map_size < DICT_LMDB_SIZE_MAX / map_size) {
- map_size = (st.st_size / map_size + 1) * map_size;
- } else {
- map_size = st.st_size;
- }
- if (msg_verbose)
- msg_info("using %s:%s map size %lu",
- DICT_TYPE_LMDB, path, (unsigned long) map_size);
- }
- if ((status = mdb_env_set_mapsize(env, map_size)))
- msg_fatal("env_set_mapsize %s: %s", mdb_path, mdb_strerror(status));
-
- if ((status = mdb_env_set_maxreaders(env, dict_lmdb_max_readers)))
- msg_fatal("env_set_maxreaders %s: %s", mdb_path, mdb_strerror(status));
+ mdb_open_flags = MDB_NOSUBDIR | MDB_NOLOCK;
+ if (dict_open_flags == O_RDONLY)
+ mdb_open_flags |= MDB_RDONLY;
/*
- * Gracefully handle the most common mistake.
+ * Gracefully handle most database open errors.
*/
- if ((status = mdb_env_open(env, mdb_path, env_flags, 0644))) {
- mdb_env_close(env);
- return (dict_surrogate(DICT_TYPE_LMDB, path, open_flags, dict_flags,
- "open database %s: %m", mdb_path));
+ if ((status = slmdb_open(&slmdb, mdb_path, dict_open_flags, mdb_open_flags,
+ dict_flags & DICT_FLAG_BULK_UPDATE, dict_lmdb_map_size,
+ DICT_LMDB_SIZE_INCR, DICT_LMDB_SIZE_MAX)) != 0) {
+ dict = dict_surrogate(DICT_TYPE_LMDB, path, dict_open_flags,
+ dict_flags, "open database %s: %m", mdb_path);
+ myfree(mdb_path);
+ return (dict);
}
- if ((status = mdb_txn_begin(env, NULL, env_flags & MDB_RDONLY, &txn)))
- msg_fatal("txn_begin %s: %s", mdb_path, mdb_strerror(status));
/*
- * mdb_open() requires a txn, but since the default DB always exists in
- * an LMDB environment, we usually don't need to do anything else with
- * the txn. It is currently used for bulk transactions.
+ * XXX Persistent locking belongs in mkmap_lmdb.
+ *
+ * We just need to acquire exclusive access momentarily. This establishes
+ * that no readers are accessing old (obsoleted by copy-on-write) txn
+ * snapshots, so we are free to reuse all eligible old pages. Downgrade
+ * the lock right after acquiring it. This is sufficient to keep out
+ * other writers until we are done.
*/
- if ((status = mdb_open(txn, NULL, 0, &dbi)))
- msg_fatal("mdb_open %s: %s", mdb_path, mdb_strerror(status));
+ db_fd = slmdb_fd(&slmdb);
+ if (dict_flags & DICT_FLAG_BULK_UPDATE) {
+ if (myflock(db_fd, MYFLOCK_STYLE_FCNTL, MYFLOCK_OP_EXCLUSIVE) < 0)
+ msg_fatal("%s: lock dictionary: %m", mdb_path);
+ if (myflock(db_fd, MYFLOCK_STYLE_FCNTL, MYFLOCK_OP_SHARED) < 0)
+ msg_fatal("%s: unlock dictionary: %m", mdb_path);
+ }
/*
* Bundle up.
@@ -918,15 +571,17 @@ DICT *dict_lmdb_open(const char *path, int open_flags, int dict_flags)
dict_lmdb->dict.delete = dict_lmdb_delete;
dict_lmdb->dict.sequence = dict_lmdb_sequence;
dict_lmdb->dict.close = dict_lmdb_close;
- dict_lmdb->dict.lock = dict_lmdb_lock;
- if ((dict_lmdb->dict.stat_fd = open(mdb_path, O_RDONLY)) < 0)
- msg_fatal("dict_lmdb_open: %s: %m", mdb_path);
- if (fstat(dict_lmdb->dict.stat_fd, &st) < 0)
+
+ if (fstat(db_fd, &st) < 0)
msg_fatal("dict_lmdb_open: fstat: %m");
+ dict_lmdb->dict.lock_fd = dict_lmdb->dict.stat_fd = db_fd;
dict_lmdb->dict.mtime = st.st_mtime;
dict_lmdb->dict.owner.uid = st.st_uid;
dict_lmdb->dict.owner.status = (st.st_uid != 0);
+ dict_lmdb->key_buf = 0;
+ dict_lmdb->val_buf = 0;
+
/*
* Warn if the source file is newer than the indexed file, except when
* the source file changed only seconds ago.
@@ -937,30 +592,37 @@ DICT *dict_lmdb_open(const char *path, int open_flags, int dict_flags)
&& st.st_mtime < time((time_t *) 0) - 100)
msg_warn("database %s is older than source file %s", mdb_path, path);
- close_on_exec(dict_lmdb->dict.stat_fd, CLOSE_ON_EXEC);
dict_lmdb->dict.flags = dict_flags | DICT_FLAG_FIXED;
if ((dict_flags & (DICT_FLAG_TRY0NULL | DICT_FLAG_TRY1NULL)) == 0)
dict_lmdb->dict.flags |= (DICT_FLAG_TRY0NULL | DICT_FLAG_TRY1NULL);
if (dict_flags & DICT_FLAG_FOLD_FIX)
dict_lmdb->dict.fold_buf = vstring_alloc(10);
- dict_lmdb->env = env;
- dict_lmdb->dbi = dbi;
- dict_lmdb->map_size = map_size;
- dict_lmdb->cursor = 0;
- dict_lmdb->key_buf = 0;
- dict_lmdb->val_buf = 0;
-
- /* The following facilitate transparent error recovery. */
- dict_lmdb->dict_api_retries = 0;
- dict_lmdb->bulk_mode_retries = 0;
- dict_lmdb->dict_open_flags = open_flags;
- dict_lmdb->mdb_open_flags = env_flags;
- dict_lmdb->txn = txn;
- dict_lmdb->readers_full = 0;
- dict_lmdb_prepare(dict_lmdb);
if (dict_flags & DICT_FLAG_BULK_UPDATE)
- dict_jmp_alloc(&dict_lmdb->dict); /* build into dict_alloc() */
+ dict_jmp_alloc(&dict_lmdb->dict);
+
+ /*
+ * The following requests return an error result only if we have serious
+ * memory corruption problem.
+ */
+ slmdb_control(&slmdb,
+ SLMDB_CTL_API_RETRY_LIMIT, DICT_LMDB_API_RETRY_LIMIT,
+ SLMDB_CTL_BULK_RETRY_LIMIT, DICT_LMDB_BULK_RETRY_LIMIT,
+ SLMDB_CTL_LONGJMP_FN, dict_lmdb_longjmp,
+ SLMDB_CTL_CONTEXT, (void *) dict_lmdb,
+ SLMDB_CTL_END);
+ if (msg_verbose) {
+ slmdb_control(&slmdb,
+ SLMDB_CTL_NOTIFY_FN, dict_lmdb_notify,
+ SLMDB_CTL_END);
+ dict_lmdb_notify((void *) dict_lmdb, MDB_SUCCESS,
+ slmdb_curr_limit(&slmdb));
+ }
+
+ /*
+ * From here on no direct assignments to slmdb.
+ */
+ dict_lmdb->slmdb = slmdb;
myfree(mdb_path);
diff --git a/postfix/src/util/dict_open.c b/postfix/src/util/dict_open.c
index d3b33aa0f..aee1f8ddd 100644
--- a/postfix/src/util/dict_open.c
+++ b/postfix/src/util/dict_open.c
@@ -299,7 +299,6 @@ static const DICT_OPEN_INFO dict_open_info[] = {
DICT_TYPE_BTREE, dict_btree_open,
#endif
#ifdef HAS_LMDB
-#error "LMDB support is forbidden"
DICT_TYPE_LMDB, dict_lmdb_open,
#endif
#ifdef HAS_NIS
diff --git a/postfix/src/util/dict_pcre.c b/postfix/src/util/dict_pcre.c
index 060019ec5..3ae2104b9 100644
--- a/postfix/src/util/dict_pcre.c
+++ b/postfix/src/util/dict_pcre.c
@@ -61,6 +61,15 @@
#include "pcre.h"
#include "warn_stat.h"
+ /*
+ * Backwards compatibility.
+ */
+#ifdef PCRE_STUDY_JIT_COMPILE
+#define DICT_PCRE_FREE_STUDY(x) pcre_free_study(x)
+#else
+#define DICT_PCRE_FREE_STUDY(x) pcre_free((char *) (x))
+#endif
+
/*
* Support for IF/ENDIF based on an idea by Bert Driehuis.
*/
@@ -389,7 +398,7 @@ static void dict_pcre_close(DICT *dict)
if (match_rule->pattern)
myfree((char *) match_rule->pattern);
if (match_rule->hints)
- myfree((char *) match_rule->hints);
+ DICT_PCRE_FREE_STUDY(match_rule->hints);
if (match_rule->replacement)
myfree((char *) match_rule->replacement);
break;
@@ -398,7 +407,7 @@ static void dict_pcre_close(DICT *dict)
if (if_rule->pattern)
myfree((char *) if_rule->pattern);
if (if_rule->hints)
- myfree((char *) if_rule->hints);
+ DICT_PCRE_FREE_STUDY(if_rule->hints);
break;
case DICT_PCRE_OP_ENDIF:
break;
@@ -679,7 +688,7 @@ static DICT_PCRE_RULE *dict_pcre_parse_rule(const char *mapname, int lineno,
if (engine.pattern)
myfree((char *) engine.pattern);
if (engine.hints)
- myfree((char *) engine.hints);
+ DICT_PCRE_FREE_STUDY(engine.hints);
CREATE_MATCHOP_ERROR_RETURN(0);
}
#endif
diff --git a/postfix/src/util/slmdb.c b/postfix/src/util/slmdb.c
new file mode 100644
index 000000000..25f147f8c
--- /dev/null
+++ b/postfix/src/util/slmdb.c
@@ -0,0 +1,664 @@
+/*++
+/* NAME
+/* slmdb 3
+/* SUMMARY
+/* Simplified LMDB API
+/* SYNOPSIS
+/* #include
+/*
+/* size_t slmdb_map_size;
+/*
+/* int slmdb_open(slmdb, path, open_flags, lmdb_flags, bulk_mode,
+/* curr_limit, size_incr, hard_limit)
+/* SLMDB *slmdb;
+/* const char *path;
+/* int open_flags;
+/* int lmdb_flags;
+/* int bulk_mode;
+/* size_t curr_limit;
+/* int size_incr;
+/* size_t hard_limit;
+/*
+/* int slmdb_close(slmdb)
+/* SLMDB *slmdb;
+/*
+/* int slmdb_get(slmdb, mdb_key, mdb_value)
+/* SLMDB *slmdb;
+/* MDB_val *mdb_key;
+/* MDB_val *mdb_value;
+/*
+/* int slmdb_put(slmdb, mdb_key, mdb_value, flags)
+/* SLMDB *slmdb;
+/* MDB_val *mdb_key;
+/* MDB_val *mdb_value;
+/* int flags;
+/*
+/* int slmdb_del(slmdb, mdb_key)
+/* SLMDB *slmdb;
+/* MDB_val *mdb_key;
+/*
+/* int slmdb_cursor_get(slmdb, mdb_key, mdb_value, op)
+/* SLMDB *slmdb;
+/* MDB_val *mdb_key;
+/* MDB_val *mdb_value;
+/* MDB_cursor_op op;
+/* AUXILIARY FUNCTIONS
+/* int slmdb_fd(slmdb)
+/* SLMDB *slmdb;
+/*
+/* size_t slmdb_curr_limit(slmdb)
+/* SLMDB *slmdb;
+/*
+/* int slmdb_control(slmdb, id, ...)
+/* SLMDB *slmdb;
+/* int id;
+/* DESCRIPTION
+/* This module simplifies the LMDB API by hiding recoverable
+/* errors from the application. Details are given in the
+/* section "ERROR RECOVERY".
+/*
+/* slmdb_open() opens an LMDB database. The result value is
+/* an LMDB status code (zero in case of success).
+/*
+/* slmdb_close() finalizes an optional bulk-mode transaction
+/* and closes a successfully-opened LMDB database. The result
+/* value is an LMDB status code (zero in case of success).
+/*
+/* slmdb_get() is an mdb_get() wrapper with automatic error
+/* recovery. The result value is an LMDB status code (zero
+/* in case of success).
+/*
+/* slmdb_put() is an mdb_put() wrapper with automatic error
+/* recovery. The result value is an LMDB status code (zero
+/* in case of success).
+/*
+/* slmdb_del() is an mdb_del() wrapper with automatic error
+/* recovery. The result value is an LMDB status code (zero
+/* in case of success).
+/*
+/* slmdb_cursor_get() iterates over an LMDB database. The
+/* result value is an LMDB status code (zero in case of success).
+/*
+/* slmdb_fd() returns the file descriptor for an open LMDB
+/* database. This may be used for file status queries or
+/* application-controlled locking.
+/*
+/* slmdb_curr_limit() returns the current database size limit
+/* for the specified database.
+/*
+/* slmdb_control() specifies optional features. The arguments
+/* are a list of (name, value) pairs, terminated with
+/* SLMDB_CTL_END. The result is 0 in case of success, or -1
+/* with errno indicating the nature of the problem. The following
+/* text enumerates the symbolic request names and the types
+/* of the corresponding additional arguments.
+/* .IP "SLMDB_CTL_LONGJMP_FN (void (*)(void *, int))
+/* Application long-jump call-back function pointer. The
+/* function must not return and is called to repeat a failed
+/* bulk-mode transaction from the start. The arguments are
+/* the application context and the setjmp() or sigsetjmp()
+/* result value.
+/* .IP "SLMDB_CTL_NOTIFY_FN (void (*)(void *, int, ...))"
+/* Application notification call-back function pointer. The
+/* function is called after succesful error recovery with as
+/* arguments the application context, the MDB error code, and
+/* additional arguments that depend on the error code.
+/* Details are given in the section "ERROR RECOVERY".
+/* .IP "SLMDB_CTL_CONTEXT (void *)"
+/* Application context that is passed in application notification
+/* and long-jump call-back function calls.
+/* .IP "SLMDB_CTL_API_RETRY_LIMIT (int)"
+/* How many times to recover from LMDB errors within the
+/* execution of a single slmdb(3) API call before giving up.
+/* .IP "SLMDB_CTL_BULK_RETRY_LIMIT (int)"
+/* How many times to recover from a bulk-mode transaction
+/* before giving up.
+/* ERROR RECOVERY
+/* .ad
+/* .fi
+/* This module automatically repeats failed requests after
+/* recoverable errors, up to limits specified with slmdb_control().
+/*
+/* Recoverable errors are reported through an optional
+/* notification function specified with slmdb_control(). With
+/* recoverable MDB_MAP_FULL and MDB_MAP_RESIZED errors, the
+/* additional argument is a size_t value with the updated
+/* current database size limit; with recoverable MDB_READERS_FULL
+/* errors there is no additional argument.
+/* BUGS
+/* Recovery from MDB_MAP_FULL involves resizing the database
+/* memory mapping. According to LMDB documentation this
+/* requires that there is no concurrent activity in the same
+/* database by other threads in the same memory address space.
+/* SEE ALSO
+/* lmdb(3) API manpage (currently, non-existent).
+/* LICENSE
+/* .ad
+/* .fi
+/* The Secure Mailer license must be distributed with this software.
+/* AUTHOR(S)
+/* Howard Chu
+/* Symas Corporation
+/*
+/* Wietse Venema
+/* IBM T.J. Watson Research
+/* P.O. Box 704
+/* Yorktown Heights, NY 10598, USA
+/*--*/
+
+/* System library. */
+
+#include
+#include
+#include
+#include
+#include
+#include
+
+/* Application-specific. */
+
+#include
+
+ /*
+ * LMDB 0.9.8 allows the application to update the database size limit
+ * on-the-fly (typically after an MDB_MAP_FULL error). The only limit that
+ * remains is imposed by the hardware address space. The implementation is
+ * supposed to handle databases larger than physical memory. However, at
+ * some point in time there was no such guarantee for (bulk) transactions
+ * larger than physical memory.
+ *
+ * LMDB 0.9.9 allows the application to manage locks. This elimimates multiple
+ * problems:
+ *
+ * - The need for a (world-)writable lockfile, which is a show-stopper for
+ * multiprogrammed applications that have privileged writers and
+ * unprivileged readers.
+ *
+ * - Hard-coded inode numbers (in ftok() output) in lockfile content that can
+ * prevent automatic crash recovery, and related to that, sub-optimal
+ * semaphore performance on BSD systems.
+ */
+#if MDB_VERSION_FULL < MDB_VERINT(0, 9, 9)
+#error "Build with LMDB version 0.9.9 or later"
+#endif
+
+#define SLMDB_DEF_API_RETRY_LIMIT 2 /* Retries per dict(3) API call */
+#define SLMDB_DEF_BULK_RETRY_LIMIT \
+ (2 * sizeof(size_t) * CHAR_BIT) /* Retries per bulk-mode transaction */
+
+ /*
+ * The purpose of the error-recovering functions below is to hide LMDB
+ * quirks (MAP_FULL, MAP_RESIZED, MDB_READERS_FULL), so that the caller can
+ * pretend that those quirks don't exist, and focus on its own job.
+ *
+ * - To recover from a single-transaction LMDB error, each wrapper function
+ * uses tail recursion instead of goto. Since LMDB errors are rare, code
+ * clarity is more important than speed.
+ *
+ * - To recover from a bulk-transaction LMDB error, the error-recovery code
+ * jumps back into the caller to some pre-arranged point (the closest thing
+ * that C has to exception handling). The application is then expected to
+ * repeat the bulk transaction from scratch.
+ */
+
+ /*
+ * We increment the recursion counter each time we try to recover from
+ * error, and reset the recursion counter when returning to the application
+ * from the slmdb API.
+ */
+#define SLMDB_API_RETURN(slmdb, status) do { \
+ (slmdb)->api_retry_count = 0; \
+ return (status); \
+ } while (0)
+
+/* slmdb_prepare - LMDB-specific (re)initialization before actual access */
+
+static int slmdb_prepare(SLMDB *slmdb)
+{
+ int status;
+
+ /*
+ * This is called before accessing the database, or after recovery from
+ * an LMDB error. Note: this code cannot recover from errors itself.
+ * slmdb->txn is either the database open() transaction or a
+ * freshly-created bulk-mode transaction.
+ *
+ * - With O_TRUNC we make a "drop" request before updating the database.
+ *
+ * - With a bulk-mode transaction we commit when the database is closed.
+ *
+ * XXX If we want to make the slmdb API suitable for general use, then the
+ * bulk/non-bulk handling must be generalized.
+ */
+ if (slmdb->open_flags & O_TRUNC) {
+ if ((status = mdb_drop(slmdb->txn, slmdb->dbi, 0)) != 0)
+ return (status);
+ if ((slmdb->bulk_mode) == 0) {
+ if ((status = mdb_txn_commit(slmdb->txn)))
+ return (status);
+ slmdb->txn = 0;
+ }
+ } else if ((slmdb->lmdb_flags & MDB_RDONLY) != 0
+ || (slmdb->bulk_mode) == 0) {
+ mdb_txn_abort(slmdb->txn);
+ slmdb->txn = 0;
+ }
+ slmdb->api_retry_count = 0;
+ return (status);
+}
+
+/* slmdb_recover - recover from LMDB errors */
+
+static int slmdb_recover(SLMDB *slmdb, int status)
+{
+ MDB_envinfo info;
+
+ /*
+ * Limit the number of recovery attempts per slmdb(3) API request.
+ */
+ if ((slmdb->api_retry_count += 1) >= slmdb->api_retry_limit)
+ return (status);
+
+ /*
+ * If we can recover from the error, we clear the error condition and the
+ * caller should retry the failed operation immediately. Otherwise, the
+ * caller should terminate with a fatal run-time error and the program
+ * should be re-run later.
+ *
+ * slmdb->txn must be either null (non-bulk transaction error), or an
+ * aborted bulk-mode transaction.
+ *
+ * XXX If we want to make the slmdb API suitable for general use, then the
+ * bulk/non-bulk handling must be generalized.
+ */
+ switch (status) {
+
+ /*
+ * As of LMDB 0.9.8 when a non-bulk update runs into a "map full"
+ * error, we can resize the environment's memory map and clear the
+ * error condition. The caller should retry immediately.
+ */
+ case MDB_MAP_FULL:
+ /* Can we increase the memory map? Give up if we can't. */
+ if (slmdb->curr_limit < slmdb->hard_limit / slmdb->size_incr) {
+ slmdb->curr_limit = slmdb->curr_limit * slmdb->size_incr;
+ } else if (slmdb->curr_limit < slmdb->hard_limit) {
+ slmdb->curr_limit = slmdb->hard_limit;
+ } else {
+ /* Sorry, we are already maxed out. */
+ break;
+ }
+ if (slmdb->notify_fn)
+ slmdb->notify_fn(slmdb->cb_context, MDB_MAP_FULL,
+ slmdb->curr_limit);
+ status = mdb_env_set_mapsize(slmdb->env, slmdb->curr_limit);
+ break;
+
+ /*
+ * When a writer resizes the database, read-only applications must
+ * increase their LMDB memory map size limit, too. Otherwise, they
+ * won't be able to read a table after it grows.
+ *
+ * As of LMDB 0.9.8 we can import the new memory map size limit into the
+ * database environment by calling mdb_env_set_mapsize() with a zero
+ * size argument. Then we extract the map size limit for later use.
+ * The caller should retry immediately.
+ */
+ case MDB_MAP_RESIZED:
+ if ((status = mdb_env_set_mapsize(slmdb->env, 0)) == 0) {
+ /* Do not panic. Maps may shrink after bulk update. */
+ mdb_env_info(slmdb->env, &info);
+ slmdb->curr_limit = info.me_mapsize;
+ if (slmdb->notify_fn)
+ slmdb->notify_fn(slmdb->cb_context, MDB_MAP_RESIZED,
+ slmdb->curr_limit);
+ }
+ break;
+
+ /*
+ * What is it with these built-in hard limits that cause systems to
+ * stop when demand is at its highest? When the system is under
+ * stress it should slow down and keep making progress.
+ */
+ case MDB_READERS_FULL:
+ if (slmdb->notify_fn)
+ slmdb->notify_fn(slmdb->cb_context, MDB_READERS_FULL);
+ sleep(1);
+ status = 0;
+ break;
+
+ /*
+ * We can't solve this problem. The application should terminate with
+ * a fatal run-time error and the program should be re-run later.
+ */
+ default:
+ break;
+ }
+
+ /*
+ * If a bulk-transaction error is recoverable, build a new bulk
+ * transaction from scratch, by making a long jump back into the caller
+ * at some pre-arranged point.
+ */
+ if (slmdb->txn != 0 && status == 0 && slmdb->longjmp_fn != 0
+ && (slmdb->bulk_retry_count += 1) <= slmdb->bulk_retry_limit) {
+ if ((status = mdb_txn_begin(slmdb->env, (MDB_txn *) 0,
+ slmdb->lmdb_flags & MDB_RDONLY,
+ &slmdb->txn)) == 0
+ && (status = slmdb_prepare(slmdb)) == 0)
+ slmdb->longjmp_fn(slmdb->cb_context, 1);
+ }
+ return (status);
+}
+
+/* slmdb_txn_begin - mdb_txn_begin() wrapper with LMDB error recovery */
+
+static int slmdb_txn_begin(SLMDB *slmdb, int rdonly, MDB_txn **txn)
+{
+ int status;
+
+ if ((status = mdb_txn_begin(slmdb->env, (MDB_txn *) 0, rdonly, txn)) != 0
+ && (status = slmdb_recover(slmdb, status)) == 0)
+ status = slmdb_txn_begin(slmdb, rdonly, txn);
+
+ return (status);
+}
+
+/* slmdb_get - mdb_get() wrapper with LMDB error recovery */
+
+int slmdb_get(SLMDB *slmdb, MDB_val *mdb_key, MDB_val *mdb_value)
+{
+ MDB_txn *txn;
+ int status;
+
+ /*
+ * Start a read transaction if there's no bulk-mode txn.
+ */
+ if (slmdb->txn)
+ txn = slmdb->txn;
+ else if ((status = slmdb_txn_begin(slmdb, MDB_RDONLY, &txn)) != 0)
+ SLMDB_API_RETURN(slmdb, status);
+
+ /*
+ * Do the lookup.
+ */
+ if ((status = mdb_get(txn, slmdb->dbi, mdb_key, mdb_value)) != 0
+ && status != MDB_NOTFOUND) {
+ mdb_txn_abort(txn);
+ if ((status = slmdb_recover(slmdb, status)) == 0)
+ status = slmdb_get(slmdb, mdb_key, mdb_value);
+ SLMDB_API_RETURN(slmdb, status);
+ }
+
+ /*
+ * Close the read txn if it's not the bulk-mode txn.
+ */
+ if (slmdb->txn == 0)
+ mdb_txn_abort(txn);
+
+ SLMDB_API_RETURN(slmdb, status);
+}
+
+/* slmdb_put - mdb_put() wrapper with LMDB error recovery */
+
+int slmdb_put(SLMDB *slmdb, MDB_val *mdb_key,
+ MDB_val *mdb_value, int flags)
+{
+ MDB_txn *txn;
+ int status;
+
+ /*
+ * Start a write transaction if there's no bulk-mode txn.
+ */
+ if (slmdb->txn)
+ txn = slmdb->txn;
+ else if ((status = slmdb_txn_begin(slmdb, 0, &txn)) != 0)
+ SLMDB_API_RETURN(slmdb, status);
+
+ /*
+ * Do the update.
+ */
+ if ((status = mdb_put(txn, slmdb->dbi, mdb_key, mdb_value, flags)) != 0) {
+ mdb_txn_abort(txn);
+ if (status != MDB_KEYEXIST) {
+ if ((status = slmdb_recover(slmdb, status)) == 0)
+ status = slmdb_put(slmdb, mdb_key, mdb_value, flags);
+ SLMDB_API_RETURN(slmdb, status);
+ }
+ }
+
+ /*
+ * Commit the transaction if it's not the bulk-mode txn.
+ */
+ if (status == 0 && slmdb->txn == 0 && (status = mdb_txn_commit(txn)) != 0
+ && (status = slmdb_recover(slmdb, status)) == 0)
+ status = slmdb_put(slmdb, mdb_key, mdb_value, flags);
+
+ SLMDB_API_RETURN(slmdb, status);
+}
+
+/* slmdb_del - mdb_del() wrapper with LMDB error recovery */
+
+int slmdb_del(SLMDB *slmdb, MDB_val *mdb_key)
+{
+ MDB_txn *txn;
+ int status;
+
+ /*
+ * Start a write transaction if there's no bulk-mode txn.
+ */
+ if (slmdb->txn)
+ txn = slmdb->txn;
+ else if ((status = slmdb_txn_begin(slmdb, 0, &txn)) != 0)
+ SLMDB_API_RETURN(slmdb, status);
+
+ /*
+ * Do the update.
+ */
+ if ((status = mdb_del(txn, slmdb->dbi, mdb_key, (MDB_val *) 0)) != 0) {
+ mdb_txn_abort(txn);
+ if (status != MDB_NOTFOUND) {
+ if ((status = slmdb_recover(slmdb, status)) == 0)
+ status = slmdb_del(slmdb, mdb_key);
+ SLMDB_API_RETURN(slmdb, status);
+ }
+ }
+
+ /*
+ * Commit the transaction if it's not the bulk-mode txn.
+ */
+ if (status == 0 && slmdb->txn == 0 && (status = mdb_txn_commit(txn)) != 0
+ && (status = slmdb_recover(slmdb, status)) == 0)
+ status = slmdb_del(slmdb, mdb_key);
+
+ SLMDB_API_RETURN(slmdb, status);
+}
+
+/* slmdb_cursor_get - mdb_cursor_get() wrapper with LMDB error recovery */
+
+int slmdb_cursor_get(SLMDB *slmdb, MDB_val *mdb_key,
+ MDB_val *mdb_value, MDB_cursor_op op)
+{
+ MDB_txn *txn;
+ int status;
+
+ /*
+ * Open a read transaction and cursor if needed.
+ */
+ if (slmdb->cursor == 0) {
+ slmdb_txn_begin(slmdb, MDB_RDONLY, &txn);
+ if ((status = mdb_cursor_open(txn, slmdb->dbi, &slmdb->cursor)) != 0) {
+ mdb_txn_abort(txn);
+ if ((status = slmdb_recover(slmdb, status)) == 0)
+ status = slmdb_cursor_get(slmdb, mdb_key, mdb_value, op);
+ SLMDB_API_RETURN(slmdb, status);
+ }
+ }
+
+ /*
+ * Database lookup.
+ */
+ status = mdb_cursor_get(slmdb->cursor, mdb_key, mdb_value, op);
+
+ /*
+ * Handle end-of-database or other error.
+ */
+ if (status != 0) {
+ if (status == MDB_NOTFOUND) {
+ txn = mdb_cursor_txn(slmdb->cursor);
+ mdb_cursor_close(slmdb->cursor);
+ mdb_txn_abort(txn);
+ slmdb->cursor = 0;
+ } else {
+ if ((status = slmdb_recover(slmdb, status)) == 0)
+ status = slmdb_cursor_get(slmdb, mdb_key, mdb_value, op);
+ SLMDB_API_RETURN(slmdb, status);
+ /* Do not hand-optimize out the above return statement. */
+ }
+ }
+ SLMDB_API_RETURN(slmdb, status);
+}
+
+/* slmdb_control - control optional settings */
+
+int slmdb_control(SLMDB *slmdb, int first,...)
+{
+ va_list ap;
+ int status = 0;
+ int reqno;
+
+ va_start(ap, first);
+ for (reqno = first; reqno != SLMDB_CTL_END; reqno = va_arg(ap, int)) {
+ switch (reqno) {
+ case SLMDB_CTL_LONGJMP_FN:
+ slmdb->longjmp_fn = va_arg(ap, SLMDB_LONGJMP_FN);
+ break;
+ case SLMDB_CTL_NOTIFY_FN:
+ slmdb->notify_fn = va_arg(ap, SLMDB_NOTIFY_FN);
+ break;
+ case SLMDB_CTL_CONTEXT:
+ slmdb->cb_context = va_arg(ap, void *);
+ break;
+ case SLMDB_CTL_API_RETRY_LIMIT:
+ slmdb->api_retry_limit = va_arg(ap, int);
+ break;
+ case SLMDB_CTL_BULK_RETRY_LIMIT:
+ slmdb->bulk_retry_limit = va_arg(ap, int);
+ break;
+ default:
+ errno = EINVAL;
+ status = -1;
+ break;
+ }
+ }
+ va_end(ap);
+ return (status);
+}
+
+/* slmdb_close - wrapper with LMDB error recovery */
+
+int slmdb_close(SLMDB *slmdb)
+{
+ int status = 0;
+
+ /*
+ * Finish an open bulk transaction. If slmdb_recover() returns after a
+ * bulk-transaction error, then it was unable to recover.
+ */
+ if (slmdb->txn != 0
+ && (status = mdb_txn_commit(slmdb->txn)) != 0)
+ status = slmdb_recover(slmdb, status);
+
+ /*
+ * Clean up after an unfinished sequence() operation.
+ */
+ if (slmdb->cursor) {
+ MDB_txn *txn = mdb_cursor_txn(slmdb->cursor);
+
+ mdb_cursor_close(slmdb->cursor);
+ mdb_txn_abort(txn);
+ }
+ mdb_env_close(slmdb->env);
+
+ SLMDB_API_RETURN(slmdb, status);
+}
+
+/* slmdb_open - open wrapped LMDB database */
+
+int slmdb_open(SLMDB *slmdb, const char *path, int open_flags,
+ int lmdb_flags, int bulk_mode, size_t curr_limit,
+ int size_incr, size_t hard_limit)
+{
+ struct stat st;
+ MDB_env *env;
+ MDB_txn *txn;
+ MDB_dbi dbi;
+ int db_fd;
+ int status;
+
+ /*
+ * Create LMDB environment.
+ */
+ if ((status = mdb_env_create(&env)) != 0)
+ return (status);
+
+ /*
+ * Make sure that the memory map has room to store and commit an initial
+ * "drop" transaction. We have no way to recover from errors before the
+ * first application-level request.
+ */
+#define SLMDB_FUDGE 8192
+
+ if (curr_limit < SLMDB_FUDGE)
+ curr_limit = SLMDB_FUDGE;
+ if (stat(path, &st) == 0 && st.st_size > curr_limit - SLMDB_FUDGE) {
+ if (st.st_size > hard_limit)
+ hard_limit = st.st_size;
+ if (st.st_size < hard_limit - SLMDB_FUDGE)
+ curr_limit = st.st_size + SLMDB_FUDGE;
+ else
+ curr_limit = hard_limit;
+ }
+
+ /*
+ * mdb_open() requires a txn, but since the default DB always exists in
+ * an LMDB environment, we usually don't need to do anything else with
+ * the txn. It is currently used for truncate and for bulk transactions.
+ */
+ if ((status = mdb_env_set_mapsize(env, curr_limit)) != 0
+ || (status = mdb_env_open(env, path, lmdb_flags, 0644)) != 0
+ || (status = mdb_txn_begin(env, (MDB_txn *) 0,
+ lmdb_flags & MDB_RDONLY, &txn)) != 0
+ || (status = mdb_open(txn, (const char *) 0, 0, &dbi)) != 0
+ || (status = mdb_env_get_fd(env, &db_fd)) != 0) {
+ mdb_env_close(env);
+ return (status);
+ }
+
+ /*
+ * Bundle up.
+ */
+ slmdb->open_flags = open_flags;
+ slmdb->lmdb_flags = lmdb_flags;
+ slmdb->bulk_mode = bulk_mode;
+ slmdb->curr_limit = curr_limit;
+ slmdb->size_incr = size_incr;
+ slmdb->hard_limit = hard_limit;
+ slmdb->env = env;
+ slmdb->dbi = dbi;
+ slmdb->db_fd = db_fd;
+ slmdb->cursor = 0;
+ slmdb->api_retry_count = 0;
+ slmdb->bulk_retry_count = 0;
+ slmdb->api_retry_limit = SLMDB_DEF_API_RETRY_LIMIT;
+ slmdb->bulk_retry_limit = SLMDB_DEF_BULK_RETRY_LIMIT;
+ slmdb->longjmp_fn = 0;
+ slmdb->notify_fn = 0;
+ slmdb->cb_context = 0;
+ slmdb->txn = txn;
+
+ if ((status = slmdb_prepare(slmdb)) != 0)
+ mdb_env_close(env);
+
+ return (status);
+}
diff --git a/postfix/src/util/slmdb.h b/postfix/src/util/slmdb.h
new file mode 100644
index 000000000..1902c1163
--- /dev/null
+++ b/postfix/src/util/slmdb.h
@@ -0,0 +1,87 @@
+#ifndef _SLMDB_H_INCLUDED_
+#define _SLMDB_H_INCLUDED_
+
+/*++
+/* NAME
+/* slmdb 3h
+/* SUMMARY
+/* LMDB API wrapper
+/* SYNOPSIS
+/* #include
+/* DESCRIPTION
+/* .nf
+
+ /*
+ * System library.
+ */
+#include
+#include
+
+#ifdef PATH_LMDB_H
+#include PATH_LMDB_H
+#else
+#include
+#endif
+
+ /*
+ * External interface.
+ */
+#ifdef NO_SIGSETJMP
+#define SLMDB_JMP_BUF jmp_buf
+#else
+#define SLMDB_JMP_BUF sigjmp_buf
+#endif
+
+typedef struct {
+ int open_flags; /* open() flags */
+ int lmdb_flags; /* LMDB-specific flags */
+ int bulk_mode; /* bulk-mode flag */
+ size_t curr_limit; /* database soft size limit */
+ int size_incr; /* database growth factor */
+ size_t hard_limit; /* database hard size limit */
+ MDB_env *env; /* database environment */
+ MDB_dbi dbi; /* database instance */
+ MDB_txn *txn; /* bulk transaction */
+ int db_fd; /* database file handle */
+ MDB_cursor *cursor; /* iterator */
+ void (*longjmp_fn) (void *, int); /* exception handling */
+ void (*notify_fn) (void *, int,...); /* workaround notification */
+ void *cb_context; /* call-back context */
+ int api_retry_count; /* slmdb(3) API call retry count */
+ int bulk_retry_count; /* bulk_mode retry count */
+ int api_retry_limit; /* slmdb(3) API call retry limit */
+ int bulk_retry_limit; /* bulk_mode retry limit */
+} SLMDB;
+
+extern int slmdb_open(SLMDB *, const char *, int, int, int, size_t, int, size_t);
+extern int slmdb_get(SLMDB *, MDB_val *, MDB_val *);
+extern int slmdb_put(SLMDB *, MDB_val *, MDB_val *, int);
+extern int slmdb_del(SLMDB *, MDB_val *);
+extern int slmdb_cursor_get(SLMDB *, MDB_val *, MDB_val *, MDB_cursor_op);
+extern int slmdb_control(SLMDB *, int, ...);
+extern int slmdb_close(SLMDB *);
+
+#define slmdb_fd(slmdb) ((slmdb)->db_fd)
+#define slmdb_curr_limit(slmdb) ((slmdb)->curr_limit)
+
+#define SLMDB_CTL_END 0
+#define SLMDB_CTL_LONGJMP_FN 1 /* exception handling */
+#define SLMDB_CTL_NOTIFY_FN 2 /* debug logging function */
+#define SLMDB_CTL_CONTEXT 3 /* exception/debug logging context */
+#define SLMDB_CTL_HARD_LIMIT 4 /* hard database size limit */
+#define SLMDB_CTL_API_RETRY_LIMIT 5 /* per slmdb(3) API call */
+#define SLMDB_CTL_BULK_RETRY_LIMIT 6 /* per bulk update */
+
+typedef void (*SLMDB_NOTIFY_FN)(void *, int, ...);
+typedef void (*SLMDB_LONGJMP_FN)(void *, int);
+
+/* LICENSE
+/* .ad
+/* .fi
+/* The Secure Mailer license must be distributed with this software.
+/* AUTHOR(S)
+/* Howard Chu
+/* Symas Corporation
+/*--*/
+
+#endif