From c5d32fd76b0b47d55e897377df4a8a2c0f6ce1e9 Mon Sep 17 00:00:00 2001 From: Wietse Venema Date: Sun, 17 Mar 2013 00:00:00 -0500 Subject: [PATCH] postfix-2.11-20130317 --- postfix/HISTORY | 37 ++++++-- postfix/README_FILES/LMDB_README | 137 ++++++++++++++------------- postfix/RELEASE_NOTES | 8 +- postfix/html/LMDB_README.html | 152 +++++++++++++++--------------- postfix/proto/LMDB_README.html | 152 +++++++++++++++--------------- postfix/src/global/mail_version.h | 2 +- postfix/src/global/mkmap_lmdb.c | 2 +- postfix/src/global/mkmap_open.c | 2 +- postfix/src/postalias/postalias.c | 12 ++- postfix/src/postmap/postmap.c | 12 ++- postfix/src/tls/tls_scache.c | 4 + postfix/src/util/dict.c | 16 +++- postfix/src/util/dict.h | 2 + postfix/src/util/dict_cache.c | 5 + postfix/src/util/dict_lmdb.c | 61 +++++++++--- postfix/src/util/dict_open.c | 2 +- postfix/src/util/dict_test.c | 21 ++--- 17 files changed, 353 insertions(+), 274 deletions(-) diff --git a/postfix/HISTORY b/postfix/HISTORY index 1f3a451f6..4041b6f83 100644 --- a/postfix/HISTORY +++ b/postfix/HISTORY @@ -18256,12 +18256,31 @@ Apologies for any names omitted. 20130315 - Feature: preliminary LMDB (memory-mapped persistent file) - support by Howard Chu. This feature has unexpected limitations - that don't exist with other Postfix databases, and is - therefore "snapshot only", i.e. it will not be part of a - stable release without further changes to the Postfix LMDB - client or the Postfix dictionary API. Files: proto/postconf.proto, - proto/LMDB_README.html, proto/DATABASE_README.html, - proto/INSTALL.html util/dict_lmdb.[hc], util/dict_open.c, - global/mkmap_lmdb.[hc], global/mkmap_open.c, postconf/postconf.c. + Feature: LMDB (memory-mapped persistent file) support by + Howard Chu. This implementation has unexpected failure modes + that don't exist with other Postfix databases, so don't + just yet abandon CDB. See LMDB_README for details. Files: + proto/postconf.proto, proto/LMDB_README.html, + proto/DATABASE_README.html, proto/INSTALL.html util/dict_lmdb.[hc], + util/dict_open.c, global/mkmap_lmdb.[hc], global/mkmap_open.c, + postconf/postconf.c. + +20130316 + + Cleanup: new Postfix dictionary API flag to control the use + of (LMDB) bulk database transactions. With this, LMDB + databases no longer fail to commit any transactions with + tlsmgr(8), and LMDB databases no longer perform glacially + slow with postmap -i/postalias -i. Files: util/dict.h, + util/dict_lmdb.c, postmap/postmap.c, postalias/postalias.c. + +20130317 + + Debugging: generalized setting of dictionary API flags. + File: util/dict.[hc], util/dict_test.c. + + Robustness: Postfix programs can now recover from LMDB + "database full" errors without requiring human intervention. + When a program opens an LMDB file larger than lmdb_map_size/3, + it logs a warning and uses a larger size limit instead. + Files: util/dict_lmdb.c, proto/LMDB_README.html. diff --git a/postfix/README_FILES/LMDB_README b/postfix/README_FILES/LMDB_README index dfd131373..e39d2777d 100644 --- a/postfix/README_FILES/LMDB_README +++ b/postfix/README_FILES/LMDB_README @@ -18,11 +18,9 @@ This document describes: 3. Missing pthread library trouble. -Caution: - The current Postfix LMDB client has unexpected limitations that don't exist - with other Postfix databases. For this reason, LMDB support will not be - part of the stable Postfix release without further changes to the Postfix - LMDB client or the Postfix dictionary API. +Note: + The Postfix LMDB client implementation introduces unexpected failure modes + that don't exist with other Postfix databases. Don't just yet abandon CDB. BBuuiillddiinngg PPoossttffiixx wwiitthh OOppeennLLDDAAPP LLMMDDBB ssuuppppoorrtt @@ -65,92 +63,95 @@ Add the "-lpthread" library to the "make makefiles" command. Source code for OpenLDAP LMDB is available at http://www.openldap.org. More information is available at http://highlandsun.com/hyc/mdb/. -LLiimmiittaattiioonnss ooff PPoossttffiixx LLMMDDBB ddaattaabbaasseess.. +UUnneexxppeecctteedd ffaaiilluurree mmooddeess ooff PPoossttffiixx LLMMDDBB ddaattaabbaasseess.. + +As documented below, conversion to LMDB introduces a number of failure modes +that don't exist with other Postfix databases. UUnneexxppeecctteedd ppoossttmmaapp((11))//ppoossttaalliiaass((11)) ""ddaattaabbaassee ffuullll"" eerrrroorrss.. Problem: - Even if the "postmap lmdb:filename" command succeeds, the exact same - command (with the exact same input data) may fail subsequently with an - MDB_MAP_FULL error. This problem does not exist with other Postfix - databases. + The "postmap lmdb:filename" command fails with an MDB_MAP_FULL error. This + problem does not exist with other Postfix databases. Background: LMDB databases have a hard size limit (configured with the lmdb_map_size configuration parameter). When executing "postmap lmdb:filename", the Postfix LMDB database client - does not truncate the database file. Instead it saves the "drop" request - and subsequent "store" requests to a transaction (which takes up space in - addition to the existing data), and commits the transaction when it closes - the database. Only then can the space for old data be reused. + stores the new data in a transaction which takes up space in addition to + the existing data, and commits the transaction when it closes the database. + Only then can the space for old data be reused. Impact: This failure does not affect Postfix availability, because the old data still exists in the database. -Recovery: - Increase the lmdb_map_size limit in main.cf, and retry the postmap(1) or - postalias(1) command. +Mitigation: + When the postmap(1) or postalias(1) command opens an LMDB file larger than + lmdb_map_size/3, it logs a warning and uses a larger size limit instead: -PPoossttffiixx ddaaeemmoonn ""ddaattaabbaassee ffuullll"" eerrrroorrss.. + warning: filename.lmdb: file size 15024128 >= (lmdb map size limit + 16777216)/3 -- using a larger map size limit + + This can be used to automate recovery and avoid the need for human + intervention. Just keep running "postmap lmdb:filename". After each failure + it will use a 3x larger size limit, and eventually the "database full" + error will disappear. + +Prevention: + Monitor your LMDB files and make sure that lmdb_map_size > 3x the largest + LMDB file size. + +UUnneexxppeecctteedd PPoossttffiixx ddaaeemmoonn ""ddaattaabbaassee ffuullll"" eerrrroorrss.. Problem: - "database full" errors with daemon programs such as postscreen(8), tlsmgr - (8) or verify(8). This problem does not exist with other Postfix databases. + Postfix daemon programs fail with "database full" errors, such as + postscreen(8), tlsmgr(8) or verify(8). This problem does not exist with + other Postfix databases. + +Impact: + This failure temporarily affects Postfix availability. The daemon restarts + automatically and tries to open the database again as described next. + +Mitigation: + When a Postfix daemon opens an LMDB file larger than lmdb_map_size/3, it + logs a warning and uses a larger size limit instead: + + warning: filename.lmdb: file size 15024128 >= (lmdb map size limit + 16777216)/3 -- using a larger map size limit + + This can be used to automate recovery and avoid the need for human + intervention. Each time the daemon runs into a "database full" error, it + restarts and uses a 3x larger size limit. The "database full" error will + disappear, at least for a while. + +Prevention: + Monitor your LMDB files and make sure that lmdb_map_size > 3x the largest + LMDB file size. + +NNoonn--oobbvviioouuss rreeccoovveerryy wwiitthh ppoossttmmaapp((11))//ppoossttaalliiaass((11))//ttllssmmggrr((88)) ffrroomm aa ccoorrrruupptteedd +ddaattaabbaassee.. + +Problem: + You cannot rebuild a corrupted LMDB database simply by running postmap(1) + or postalias(1), or by waiting until the tlsmgr(8) daemon restarts + automatically. This problem does not exist with other Postfix databases. + +Background: + The Postfix LMDB database client does not truncate the database file. + Instead it attempts to create a transaction for a "drop" request and + subsequent "store" requests. That is obviously not possible with a + corrupted database file. Impact: Postfix does not process mail until someone fixes the problem. -Recovery: - Increase the lmdb_map_size limit in main.cf, and "reload" Postfix. - -NNoonn--oobbvviioouuss rreeccoovveerryy wwiitthh ppoossttmmaapp((11))//ppoossttaalliiaass((11)) ffrroomm aa ccoorrrruupptteedd ddaattaabbaassee.. - -Problem: - You cannot rebuild a corrupted LMDB database simply by running postmap(1) - or postalias(1). This problem does not exist with other Postfix databases. - -Background: - The reason for this limitation is that the Postfix LMDB database client - does not truncate the database file. Instead it attempts to save the "drop" - request and subsequent "store" requests to a transaction for later - processing. That is obviously not possible with a corrupted database file. - Recovery: First delete the ".lmdb" file by hand, then rebuild the file with the - postmap(1) or postalias(1) command. + postmap(1) or postalias(1) command, or wait until the tlsmgr(8) daemon + restarts automatically. -IInnccoommppaattiibbiilliittyy wwiitthh ttllssmmggrr((88)).. - -Problem: - The Postfix LMDB database client never commits any tlsmgr(8) transaction. - This problem does not exist with other Postfix databases. - -Background: - Instead, it creates a single transaction that accumulates a "drop" request - and all tlsmgr(8) "store" requests that are made during the lifetime of the - process. - -Solution: - This requires changes to the Postfix dictionary API, or to the Postfix LMDB - database client. - -Problem: - The Postfix LMDB database client breaks how tlsmgr(8) automatically - recovers from a corrupted database file. This problem does not exist with - other Postfix databases. - -Background: - The Postfix LMDB database client does not truncate the database file. - Instead it attempts to create a transaction which obviously is not possible - when the database file is corrupted. - -Impact: - The tlsmgr(8) process will keep crashing until someone removes the ".lmdb" - file. - -Recovery: - Remove the the ".lmdb" file by hand, and wait until the tlsmgr(8) process - restarts. +Prevention: + Arrange your file systems such that they never run out of free space. diff --git a/postfix/RELEASE_NOTES b/postfix/RELEASE_NOTES index 64fee86b6..ef167571b 100644 --- a/postfix/RELEASE_NOTES +++ b/postfix/RELEASE_NOTES @@ -17,8 +17,6 @@ before proceeding. Major changes with snapshot 20130315 ==================================== -Preliminary LMDB support by Howard Chu. This implementation has -unexpected limitations that don't exist with other Postfix databases, -and therefore the code is "snapshot only", i.e. it will not be part -of the stable release without further changes to the Postfix LMDB -client or the Postfix dictionary API. See LMDB_README for details. +LMDB support by Howard Chu. This implementation has unexpected +failure modes that don't exist with other Postfix databases, so +don't just yet abandon CDB. See LMDB_README for details. diff --git a/postfix/html/LMDB_README.html b/postfix/html/LMDB_README.html index e6f1bb984..4b3368145 100644 --- a/postfix/html/LMDB_README.html +++ b/postfix/html/LMDB_README.html @@ -39,12 +39,10 @@ LMDB support.

-
Caution:

The current Postfix LMDB client -has unexpected limitations that don't -exist with other Postfix databases. For this reason, LMDB support -will not be part of the stable Postfix release without further -changes to the Postfix LMDB client or the Postfix dictionary API. -

+
Note:

The Postfix LMDB client implementation +introduces unexpected failure modes that +don't exist with other Postfix databases. Don't just yet abandon +CDB.

Building Postfix with OpenLDAP LMDB support

@@ -111,32 +109,33 @@ undefined reference to `pthread_mutex_lock' More information is available at http://highlandsun.com/hyc/mdb/.

-

Limitations of Postfix LMDB databases. -

+

Unexpected failure modes of Postfix LMDB +databases.

-

Unexpected postmap(1)/postalias(1) "database full" errors. -

+

As documented below, conversion to LMDB introduces a number of +failure modes that don't exist with other Postfix databases.

+ +

Unexpected postmap(1)/postalias(1) "database full" +errors.

-
Problem:

Even if the "postmap lmdb:filename" -command succeeds, the exact same command (with the exact same input -data) may fail subsequently with an MDB_MAP_FULL error. This problem -does not exist with other Postfix databases.

+
Problem:

The "postmap lmdb:filename" command +fails with an MDB_MAP_FULL error. This problem does not exist with +other Postfix databases.

-
Background:
+
Background:
-
+

LMDB databases have a hard size limit (configured with the lmdb_map_size configuration parameter).

When executing "postmap lmdb:filename", the Postfix LMDB database -client does not truncate the database file. Instead it saves the -"drop" request and subsequent "store" requests to a transaction -(which takes up space in addition to the existing data), and commits -the transaction when it closes the database. Only then can the -space for old data be reused.

+client stores the new data in a transaction which takes up space +in addition to the existing data, and commits the transaction when +it closes the database. Only then can the space for old data be +reused.

@@ -144,82 +143,81 @@ space for old data be reused.

availability, because the old data still exists in the database.

-
Recovery:

Increase the lmdb_map_size limit in -main.cf, and retry the postmap(1) or postalias(1) command.

-
+
Mitigation:

When the postmap(1) or postalias(1) +command opens an LMDB file larger than lmdb_map_size/3, it logs a +warning and uses a larger size limit instead:

-
+

warning: filename.lmdb: file size 15024128 ≥ +(lmdb map size limit 16777216)/3 -- using a larger map size limit +

-

Postfix daemon "database full" errors.

+

This can be used to automate recovery and avoid the need for +human intervention. Just keep running "postmap lmdb:filename". +After each failure it will use a 3x larger size limit, and eventually +the "database full" error will disappear.

+ +
Prevention:

Monitor your LMDB files and make +sure that lmdb_map_size > 3x the largest LMDB file size.

+
+ +

Unexpected Postfix daemon "database full" errors. +

-
Problem:

"database full" errors with daemon -programs such as postscreen(8), tlsmgr(8) or verify(8). This problem -does not exist with other Postfix databases.

+
Problem:

Postfix daemon programs fail with +"database full" errors, such as postscreen(8), tlsmgr(8) or verify(8). +This problem does not exist with other Postfix databases.

+
-
Impact:

Postfix does not process mail until -someone fixes the problem.

+
Impact:

This failure temporarily affects Postfix +availability. The daemon restarts automatically and tries to open +the database again as described next.

-
Recovery:

Increase the lmdb_map_size limit in -main.cf, and "reload" Postfix.

+
Mitigation:

When a Postfix daemon opens an LMDB +file larger than lmdb_map_size/3, it logs a warning and uses a +larger size limit instead:

+ +

warning: filename.lmdb: file size 15024128 ≥ +(lmdb map size limit 16777216)/3 -- using a larger map size limit +

+ +

This can be used to automate recovery and avoid the need for +human intervention. Each time the daemon runs into a "database full" +error, it restarts and uses a 3x larger size limit. The "database +full" error will disappear, at least for a while.

+ +
Prevention:

Monitor your LMDB files and make +sure that lmdb_map_size > 3x the largest LMDB file size.

+
-

Non-obvious recovery with postmap(1)/postalias(1) +

Non-obvious recovery with postmap(1)/postalias(1)/tlsmgr(8) from a corrupted database.

Problem:

You cannot rebuild a corrupted LMDB -database simply by running postmap(1) or postalias(1). This problem +database simply by running postmap(1) or postalias(1), or by waiting +until the tlsmgr(8) daemon restarts automatically. This problem does not exist with other Postfix databases.

-
Background:

The reason for this limitation is -that the Postfix LMDB database client does not truncate the database -file. Instead it attempts to save the "drop" request and subsequent -"store" requests to a transaction for later processing. That is -obviously not possible with a corrupted database file.

- -
Recovery:

First delete the ".lmdb" file by hand, -then rebuild the file with the postmap(1) or postalias(1) command. -

- -
- -

Incompatibility with tlsmgr(8).

- -
- -
Problem:

The Postfix LMDB database client never -commits any tlsmgr(8) transaction. This problem does not exist with -other Postfix databases.

- -
Background:

Instead, it creates a single -transaction that accumulates a "drop" request and all tlsmgr(8) -"store" requests that are made during the lifetime of the process. -

- -
Solution:

This requires changes to the Postfix -dictionary API, or to the Postfix LMDB database client.

- -
Problem:

The Postfix LMDB database client breaks -how tlsmgr(8) automatically recovers from a corrupted database file. -This problem does not exist with other Postfix databases.

-
Background:

The Postfix LMDB database client does not truncate the database file. Instead it attempts to create -a transaction which obviously is not possible when the database -file is corrupted.

+a transaction for a "drop" request and subsequent "store" requests. +That is obviously not possible with a corrupted database file.

+ -
Impact:

The tlsmgr(8) process will keep crashing -until someone removes the ".lmdb" file.

+
Impact:

Postfix does not process mail until +someone fixes the problem.

-
Recovery:

Remove the the ".lmdb" file by hand, -and wait until the tlsmgr(8) process restarts.

+
Recovery:

First delete the ".lmdb" file by hand, +then rebuild the file with the postmap(1) or postalias(1) command, +or wait until the tlsmgr(8) daemon restarts automatically.

+
+ +
Prevention:

Arrange your file systems such that +they never run out of free space.

- - - - diff --git a/postfix/proto/LMDB_README.html b/postfix/proto/LMDB_README.html index adf44346f..4220785dd 100644 --- a/postfix/proto/LMDB_README.html +++ b/postfix/proto/LMDB_README.html @@ -39,12 +39,10 @@ LMDB support.

-
Caution:

The current Postfix LMDB client -has unexpected limitations that don't -exist with other Postfix databases. For this reason, LMDB support -will not be part of the stable Postfix release without further -changes to the Postfix LMDB client or the Postfix dictionary API. -

+
Note:

The Postfix LMDB client implementation +introduces unexpected failure modes that +don't exist with other Postfix databases. Don't just yet abandon +CDB.

Building Postfix with OpenLDAP LMDB support

@@ -111,32 +109,33 @@ http://www.openldap.org. More information is available at http://highlandsun.com/hyc/mdb/.

-

Limitations of Postfix LMDB databases. -

+

Unexpected failure modes of Postfix LMDB +databases.

-

Unexpected postmap(1)/postalias(1) "database full" errors. -

+

As documented below, conversion to LMDB introduces a number of +failure modes that don't exist with other Postfix databases.

+ +

Unexpected postmap(1)/postalias(1) "database full" +errors.

-
Problem:

Even if the "postmap lmdb:filename" -command succeeds, the exact same command (with the exact same input -data) may fail subsequently with an MDB_MAP_FULL error. This problem -does not exist with other Postfix databases.

+
Problem:

The "postmap lmdb:filename" command +fails with an MDB_MAP_FULL error. This problem does not exist with +other Postfix databases.

-
Background:
+
Background:
-
+

LMDB databases have a hard size limit (configured with the lmdb_map_size configuration parameter).

When executing "postmap lmdb:filename", the Postfix LMDB database -client does not truncate the database file. Instead it saves the -"drop" request and subsequent "store" requests to a transaction -(which takes up space in addition to the existing data), and commits -the transaction when it closes the database. Only then can the -space for old data be reused.

+client stores the new data in a transaction which takes up space +in addition to the existing data, and commits the transaction when +it closes the database. Only then can the space for old data be +reused.

@@ -144,82 +143,81 @@ space for old data be reused.

availability, because the old data still exists in the database.

-
Recovery:

Increase the lmdb_map_size limit in -main.cf, and retry the postmap(1) or postalias(1) command.

-
+
Mitigation:

When the postmap(1) or postalias(1) +command opens an LMDB file larger than lmdb_map_size/3, it logs a +warning and uses a larger size limit instead:

-
+

warning: filename.lmdb: file size 15024128 ≥ +(lmdb map size limit 16777216)/3 -- using a larger map size limit +

-

Postfix daemon "database full" errors.

+

This can be used to automate recovery and avoid the need for +human intervention. Just keep running "postmap lmdb:filename". +After each failure it will use a 3x larger size limit, and eventually +the "database full" error will disappear.

+ +
Prevention:

Monitor your LMDB files and make +sure that lmdb_map_size > 3x the largest LMDB file size.

+
+ +

Unexpected Postfix daemon "database full" errors. +

-
Problem:

"database full" errors with daemon -programs such as postscreen(8), tlsmgr(8) or verify(8). This problem -does not exist with other Postfix databases.

+
Problem:

Postfix daemon programs fail with +"database full" errors, such as postscreen(8), tlsmgr(8) or verify(8). +This problem does not exist with other Postfix databases.

+
-
Impact:

Postfix does not process mail until -someone fixes the problem.

+
Impact:

This failure temporarily affects Postfix +availability. The daemon restarts automatically and tries to open +the database again as described next.

-
Recovery:

Increase the lmdb_map_size limit in -main.cf, and "reload" Postfix.

+
Mitigation:

When a Postfix daemon opens an LMDB +file larger than lmdb_map_size/3, it logs a warning and uses a +larger size limit instead:

+ +

warning: filename.lmdb: file size 15024128 ≥ +(lmdb map size limit 16777216)/3 -- using a larger map size limit +

+ +

This can be used to automate recovery and avoid the need for +human intervention. Each time the daemon runs into a "database full" +error, it restarts and uses a 3x larger size limit. The "database +full" error will disappear, at least for a while.

+ +
Prevention:

Monitor your LMDB files and make +sure that lmdb_map_size > 3x the largest LMDB file size.

+
-

Non-obvious recovery with postmap(1)/postalias(1) +

Non-obvious recovery with postmap(1)/postalias(1)/tlsmgr(8) from a corrupted database.

Problem:

You cannot rebuild a corrupted LMDB -database simply by running postmap(1) or postalias(1). This problem +database simply by running postmap(1) or postalias(1), or by waiting +until the tlsmgr(8) daemon restarts automatically. This problem does not exist with other Postfix databases.

-
Background:

The reason for this limitation is -that the Postfix LMDB database client does not truncate the database -file. Instead it attempts to save the "drop" request and subsequent -"store" requests to a transaction for later processing. That is -obviously not possible with a corrupted database file.

- -
Recovery:

First delete the ".lmdb" file by hand, -then rebuild the file with the postmap(1) or postalias(1) command. -

- -
- -

Incompatibility with tlsmgr(8).

- -
- -
Problem:

The Postfix LMDB database client never -commits any tlsmgr(8) transaction. This problem does not exist with -other Postfix databases.

- -
Background:

Instead, it creates a single -transaction that accumulates a "drop" request and all tlsmgr(8) -"store" requests that are made during the lifetime of the process. -

- -
Solution:

This requires changes to the Postfix -dictionary API, or to the Postfix LMDB database client.

- -
Problem:

The Postfix LMDB database client breaks -how tlsmgr(8) automatically recovers from a corrupted database file. -This problem does not exist with other Postfix databases.

-
Background:

The Postfix LMDB database client does not truncate the database file. Instead it attempts to create -a transaction which obviously is not possible when the database -file is corrupted.

+a transaction for a "drop" request and subsequent "store" requests. +That is obviously not possible with a corrupted database file.

+ -
Impact:

The tlsmgr(8) process will keep crashing -until someone removes the ".lmdb" file.

+
Impact:

Postfix does not process mail until +someone fixes the problem.

-
Recovery:

Remove the the ".lmdb" file by hand, -and wait until the tlsmgr(8) process restarts.

+
Recovery:

First delete the ".lmdb" file by hand, +then rebuild the file with the postmap(1) or postalias(1) command, +or wait until the tlsmgr(8) daemon restarts automatically.

+
+ +
Prevention:

Arrange your file systems such that +they never run out of free space.

- - - - diff --git a/postfix/src/global/mail_version.h b/postfix/src/global/mail_version.h index 848be5bb0..dfe4d1ec0 100644 --- a/postfix/src/global/mail_version.h +++ b/postfix/src/global/mail_version.h @@ -20,7 +20,7 @@ * Patches change both the patchlevel and the release date. Snapshots have no * patchlevel; they change the release date only. */ -#define MAIL_RELEASE_DATE "20130316" +#define MAIL_RELEASE_DATE "20130317" #define MAIL_VERSION_NUMBER "2.11" #ifdef SNAPSHOT diff --git a/postfix/src/global/mkmap_lmdb.c b/postfix/src/global/mkmap_lmdb.c index 7ab120954..674ddf4c8 100644 --- a/postfix/src/global/mkmap_lmdb.c +++ b/postfix/src/global/mkmap_lmdb.c @@ -45,7 +45,7 @@ #include #include -#if defined(HAS_LMDB) && defined(SNAPSHOT) +#ifdef HAS_LMDB #ifdef PATH_LMDB_H #include PATH_LMDB_H #else diff --git a/postfix/src/global/mkmap_open.c b/postfix/src/global/mkmap_open.c index e3e2d9595..e53b093b3 100644 --- a/postfix/src/global/mkmap_open.c +++ b/postfix/src/global/mkmap_open.c @@ -102,7 +102,7 @@ static const MKMAP_OPEN_INFO mkmap_types[] = { DICT_TYPE_HASH, mkmap_hash_open, DICT_TYPE_BTREE, mkmap_btree_open, #endif -#if defined(HAS_LMDB) && defined(SNAPSHOT) +#ifdef HAS_LMDB DICT_TYPE_LMDB, mkmap_lmdb_open, #endif DICT_TYPE_FAIL, mkmap_fail_open, diff --git a/postfix/src/postalias/postalias.c b/postfix/src/postalias/postalias.c index b11d8fa4b..750b263c8 100644 --- a/postfix/src/postalias/postalias.c +++ b/postfix/src/postalias/postalias.c @@ -279,13 +279,17 @@ static void postalias(char *map_type, char *path_name, int postalias_flags, key_buffer = vstring_alloc(100); value_buffer = vstring_alloc(100); if ((open_flags & O_TRUNC) == 0) { + /* Incremental mode. */ source_fp = VSTREAM_IN; vstream_control(source_fp, VSTREAM_CTL_PATH, "stdin", VSTREAM_CTL_END); - } else if (strcmp(map_type, DICT_TYPE_PROXY) == 0) { - msg_fatal("can't create maps via the proxy service"); - } else if ((source_fp = vstream_fopen(path_name, O_RDONLY, 0)) == 0) { - msg_fatal("open %s: %m", path_name); + } else { + /* Create database. */ + if (strcmp(map_type, DICT_TYPE_PROXY) == 0) + msg_fatal("can't create maps via the proxy service"); + if ((source_fp = vstream_fopen(path_name, O_RDONLY, 0)) == 0) + msg_fatal("open %s: %m", path_name); } + dict_flags |= DICT_FLAG_BULK_UPDATE; if (fstat(vstream_fileno(source_fp), &st) < 0) msg_fatal("fstat %s: %m", path_name); diff --git a/postfix/src/postmap/postmap.c b/postfix/src/postmap/postmap.c index b3a545b87..5d891aa6d 100644 --- a/postfix/src/postmap/postmap.c +++ b/postfix/src/postmap/postmap.c @@ -342,13 +342,17 @@ static void postmap(char *map_type, char *path_name, int postmap_flags, */ line_buffer = vstring_alloc(100); if ((open_flags & O_TRUNC) == 0) { + /* Incremental mode. */ source_fp = VSTREAM_IN; vstream_control(source_fp, VSTREAM_CTL_PATH, "stdin", VSTREAM_CTL_END); - } else if (strcmp(map_type, DICT_TYPE_PROXY) == 0) { - msg_fatal("can't create maps via the proxy service"); - } else if ((source_fp = vstream_fopen(path_name, O_RDONLY, 0)) == 0) { - msg_fatal("open %s: %m", path_name); + } else { + /* Create database. */ + if (strcmp(map_type, DICT_TYPE_PROXY) == 0) + msg_fatal("can't create maps via the proxy service"); + if ((source_fp = vstream_fopen(path_name, O_RDONLY, 0)) == 0) + msg_fatal("open %s: %m", path_name); } + dict_flags |= DICT_FLAG_BULK_UPDATE; if (fstat(vstream_fileno(source_fp), &st) < 0) msg_fatal("fstat %s: %m", path_name); diff --git a/postfix/src/tls/tls_scache.c b/postfix/src/tls/tls_scache.c index 5ff68ec0f..3fd8b5313 100644 --- a/postfix/src/tls/tls_scache.c +++ b/postfix/src/tls/tls_scache.c @@ -376,6 +376,10 @@ int tls_scache_sequence(TLS_SCACHE *cp, int first_next, * Delete behind. This is a no-op if an expired cache entry was updated * in the mean time. Use the saved lookup criteria so that the "delete * behind" operation works as promised. + * + * The delete-behind strategy assumes that all updates are made by a single + * process. Otherwise, delete-behind may remove an entry that was updated + * after it was scheduled for deletion. */ if (cp->flags & TLS_SCACHE_FLAG_DEL_SAVED_CURSOR) { cp->flags &= ~TLS_SCACHE_FLAG_DEL_SAVED_CURSOR; diff --git a/postfix/src/util/dict.c b/postfix/src/util/dict.c index 173544c37..8d77883f5 100644 --- a/postfix/src/util/dict.c +++ b/postfix/src/util/dict.c @@ -59,6 +59,9 @@ /* /* const char *dict_flags_str(dict_flags) /* int dict_flags; +/* +/* int dict_flags_mask(names) +/* const char *names; /* DESCRIPTION /* This module maintains a collection of name-value dictionaries. /* Each dictionary has its own name and has its own methods to read @@ -153,6 +156,9 @@ /* dict_flags_str() returns a printable representation of the /* specified dictionary flags. The result is overwritten upon /* each call. +/* +/* dict_flags_mask() returns the bitmask for the specified +/* comma/space-separated dictionary flag names. /* SEE ALSO /* htable(3) /* BUGS @@ -578,10 +584,11 @@ static const NAME_MASK dict_mask[] = { "fold_fix", DICT_FLAG_FOLD_FIX, /* case-fold with fixed-case key map */ "fold_mul", DICT_FLAG_FOLD_MUL, /* case-fold with multi-case key map */ "open_lock", DICT_FLAG_OPEN_LOCK, /* permanent lock upon open */ + "bulk_update", DICT_FLAG_BULK_UPDATE, /* bulk update if supported */ 0, }; -/* dict_flags_str - convert mask to string for debugging purposes */ +/* dict_flags_str - convert bitmask to symbolic flag names */ const char *dict_flags_str(int dict_flags) { @@ -593,3 +600,10 @@ const char *dict_flags_str(int dict_flags) return (str_name_mask_opt(buf, "dictionary flags", dict_mask, dict_flags, NAME_MASK_NUMBER | NAME_MASK_PIPE)); } + +/* dict_flags_mask - convert symbolic flag names to bitmask */ + +int dict_flags_mask(const char *names) +{ + return (name_mask("dictionary flags", dict_mask, names)); +} diff --git a/postfix/src/util/dict.h b/postfix/src/util/dict.h index db3be4a4c..c47c20657 100644 --- a/postfix/src/util/dict.h +++ b/postfix/src/util/dict.h @@ -87,6 +87,7 @@ extern DICT *dict_debug(DICT *); #define DICT_FLAG_FOLD_MUL (1<<15) /* case-fold key with multi-case map */ #define DICT_FLAG_FOLD_ANY (DICT_FLAG_FOLD_FIX | DICT_FLAG_FOLD_MUL) #define DICT_FLAG_OPEN_LOCK (1<<16) /* perm lock if not multi-writer safe */ +#define DICT_FLAG_BULK_UPDATE (1<<17) /* optimize for bulk updates */ /* IMPORTANT: Update the dict_mask[] table when the above changes */ @@ -186,6 +187,7 @@ extern void dict_walk(DICT_WALK_ACTION, char *); extern int dict_changed(void); extern const char *dict_changed_name(void); extern const char *dict_flags_str(int); +extern int dict_flags_mask(const char *); /* * Driver for interactive or scripted tests. diff --git a/postfix/src/util/dict_cache.c b/postfix/src/util/dict_cache.c index 94907dd12..3007d6da6 100644 --- a/postfix/src/util/dict_cache.c +++ b/postfix/src/util/dict_cache.c @@ -159,6 +159,11 @@ /* until a cache cleanup run is completed. Some entries may /* never be removed when the process max_idle time is less /* than the time needed to make a full pass over the cache. +/* +/* The delete-behind strategy assumes that all updates are +/* made by a single process. Otherwise, delete-behind may +/* remove an entry that was updated after it was scheduled for +/* deletion. /* LICENSE /* .ad /* .fi diff --git a/postfix/src/util/dict_lmdb.c b/postfix/src/util/dict_lmdb.c index e60d44a79..8c9f995d8 100644 --- a/postfix/src/util/dict_lmdb.c +++ b/postfix/src/util/dict_lmdb.c @@ -6,6 +6,8 @@ /* SYNOPSIS /* #include /* +/* size_t dict_lmdb_map_size; +/* /* DICT *dict_lmdb_open(path, open_flags, dict_flags) /* const char *name; /* const char *path; @@ -15,10 +17,16 @@ /* dict_lmdb_open() opens the named LMDB database and makes it available /* via the generic interface described in dict_open(3). /* -/* The dict_lmdb_map_size variable specifies a non-default per-table -/* memory map size. The map size is 10MB. The map size is also the -/* maximum size the table can grow to, so it must be set large enough +/* The dict_lmdb_map_size variable specifies a non-default +/* per-table memory map size. The map size is also the maximum +/* size the table can grow to, so it must be set large enough /* to accomodate the largest tables in use. +/* +/* As a safety measure, when Postfix opens an LMDB database it +/* will set the memory size limit to at least 3x the +/* ".lmdb" file size, so that there is room for the file to +/* grow. This ensures continued availability of Postfix daemon +/* processes. /* DIAGNOSTICS /* Fatal errors: cannot open file, file write error, out of memory. /* SEE ALSO @@ -34,13 +42,14 @@ #include "sys_defs.h" -#if defined(HAS_LMDB) && defined(SNAPSHOT) +#ifdef HAS_LMDB /* System library. */ #include #include #include +#include #ifdef PATH_LMDB_H #include PATH_LMDB_H @@ -67,7 +76,7 @@ typedef struct { DICT dict; /* generic members */ MDB_env *env; /* LMDB environment */ MDB_dbi dbi; /* database handle */ - MDB_txn *txn; /* write transaction for O_TRUNC */ + MDB_txn *txn; /* bulk update transaction */ MDB_cursor *cursor; /* for sequence ops */ VSTRING *key_buf; /* key buffer */ VSTRING *val_buf; /* result buffer */ @@ -461,6 +470,24 @@ DICT *dict_lmdb_open(const char *path, int open_flags, int dict_flags) if ((status = mdb_env_create(&env))) msg_fatal("env_create %s: %s", mdb_path, mdb_strerror(status)); + /* + * For continued availability, try to ensure that the LMDB size limit is + * at least 3x the current LMDB file size. This should be sufficient for + * short-lived Postfix daemon processes. + */ +#ifdef SIZE_T_MAX +#define SIZE_T_MAX __MAXINT__(size_t) +#endif + + if (stat(mdb_path, &st) == 0 && st.st_size >= dict_lmdb_map_size / 2) { + msg_warn("%s: file size %lu >= (%s map size limit %ld)/3 -- " + "using a larger map size limit", + mdb_path, (unsigned long) st.st_size, + DICT_TYPE_LMDB, (long) dict_lmdb_map_size); + dict_lmdb_map_size = 3 * st.st_size; + if (dict_lmdb_map_size / 3 != st.st_size) + dict_lmdb_map_size = SIZE_T_MAX; + } if ((status = mdb_env_set_mapsize(env, dict_lmdb_map_size))) msg_fatal("env_set_mapsize %s: %s", mdb_path, mdb_strerror(status)); @@ -475,24 +502,33 @@ DICT *dict_lmdb_open(const char *path, int open_flags, int dict_flags) /* * mdb_open requires a txn, but since the default DB always exists in an - * LMDB environment, we don't need to do anything else with the txn. + * LMDB environment, we usually don't need to do anything else with the + * txn. */ if ((status = mdb_open(txn, NULL, 0, &dbi))) msg_fatal("mdb_open %s: %s", mdb_path, mdb_strerror(status)); /* - * However, if O_TRUNC was specified, we need to do it now. Also with - * O_TRUNC we keep this write txn for as long as the database is open, - * since we'll probably be doing a bulk import immediately after. + * Cases where we use the mdb_open transaction: + * + * - With O_TRUNC we make the "drop" request before populating the database. + * + * - With DICT_FLAG_BULK_UPDATE we commit the transaction when the database + * is closed. */ if (open_flags & O_TRUNC) { if ((status = mdb_drop(txn, dbi, 0))) msg_fatal("truncate %s: %s", mdb_path, mdb_strerror(status)); - } else { + if ((dict_flags & DICT_FLAG_BULK_UPDATE) == 0) { + if ((status = mdb_txn_commit(txn))) + msg_fatal("truncate %s: %s", mdb_path, mdb_strerror(status)); + txn = NULL; + } + } else if ((env_flags & MDB_RDONLY) != 0 + || (dict_flags & DICT_FLAG_BULK_UPDATE) == 0) { mdb_txn_abort(txn); txn = NULL; } - dict_lmdb = (DICT_LMDB *) dict_alloc(DICT_TYPE_LMDB, path, sizeof(*dict_lmdb)); dict_lmdb->dict.lookup = dict_lmdb_lookup; dict_lmdb->dict.update = dict_lmdb_update; @@ -500,7 +536,8 @@ DICT *dict_lmdb_open(const char *path, int open_flags, int dict_flags) dict_lmdb->dict.sequence = dict_lmdb_sequence; dict_lmdb->dict.close = dict_lmdb_close; dict_lmdb->dict.lock = dict_lmdb_lock; - dict_lmdb->dict.stat_fd = open(mdb_path, O_RDONLY); + if ((dict_lmdb->dict.stat_fd = open(mdb_path, O_RDONLY)) < 0) + msg_fatal("dict_lmdb_open: %s: %m", mdb_path); if (fstat(dict_lmdb->dict.stat_fd, &st) < 0) msg_fatal("dict_lmdb_open: fstat: %m"); dict_lmdb->dict.mtime = st.st_mtime; diff --git a/postfix/src/util/dict_open.c b/postfix/src/util/dict_open.c index f0dca1db7..580d2b3a9 100644 --- a/postfix/src/util/dict_open.c +++ b/postfix/src/util/dict_open.c @@ -272,7 +272,7 @@ static const DICT_OPEN_INFO dict_open_info[] = { DICT_TYPE_HASH, dict_hash_open, DICT_TYPE_BTREE, dict_btree_open, #endif -#if defined(HAS_LMDB) && defined(SNAPSHOT) +#ifdef HAS_LMDB DICT_TYPE_LMDB, dict_lmdb_open, #endif #ifdef HAS_NIS diff --git a/postfix/src/util/dict_test.c b/postfix/src/util/dict_test.c index 69591ea5a..b7a3df586 100644 --- a/postfix/src/util/dict_test.c +++ b/postfix/src/util/dict_test.c @@ -26,7 +26,7 @@ static NORETURN usage(char *myname) { - msg_fatal("usage: %s type:file read|write|create [fold] [sync]", myname); + msg_fatal("usage: %s type:file read|write|create [flags...]", myname); } void dict_test(int argc, char **argv) @@ -41,7 +41,7 @@ void dict_test(int argc, char **argv) const char *key; const char *value; int ch; - int dict_flags = DICT_FLAG_LOCK | DICT_FLAG_DUP_REPLACE; + int dict_flags = 0; int n; int rc; @@ -68,17 +68,12 @@ void dict_test(int argc, char **argv) open_flags = O_RDONLY; else msg_fatal("unknown access mode: %s", argv[2]); - for (n = 2; argv[optind + n]; n++) { - if (strcasecmp(argv[optind + 2], "fold") == 0) - dict_flags |= DICT_FLAG_FOLD_ANY; - else if (strcasecmp(argv[optind + 2], "sync") == 0) - dict_flags |= DICT_FLAG_SYNC_UPDATE; - else if (strcasecmp(argv[optind + 2], "open_lock") == 0) { - dict_flags |= DICT_FLAG_OPEN_LOCK; - dict_flags &= ~DICT_FLAG_LOCK; - } else - usage(argv[0]); - } + for (n = 2; argv[optind + n]; n++) + dict_flags |= dict_flags_mask(argv[optind + 2]); + if ((dict_flags & DICT_FLAG_OPEN_LOCK) == 0) + dict_flags |= DICT_FLAG_LOCK; + if ((dict_flags & (DICT_FLAG_DUP_WARN | DICT_FLAG_DUP_IGNORE)) == 0) + dict_flags |= DICT_FLAG_DUP_REPLACE; vstream_fflush(VSTREAM_OUT); dict_name = argv[optind]; dict_allow_surrogate = 1;