mir/postfix
2
0
Fork 0
mirror of https://github.com/vdukhovni/postfix synced 2025-08-30 05:38:06 +00:00
Code Issues Releases Wiki Activity

postfix-2.11-20131101

Browse Source
This commit is contained in:
Wietse Venema 2013-11-01 00:00:00 -05:00 committed by Viktor Dukhovni
parent 274fd80ff6
commit 11ee4ba923
5 changed files with 155 additions and 83 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
postfix/HISTORY
View File

@ -19023,3 +19023,9 @@ Apologies for any names omitted.
numbers in lockfiles that can prevent automatic crash numbers in lockfiles that can prevent automatic crash
recovery. Files: proto/LMDB_README.html, proto/postconf.proto, recovery. Files: proto/LMDB_README.html, proto/postconf.proto,
mantools/postlink, util/dict_lmdb.c. mantools/postlink, util/dict_lmdb.c.
20131101
Cleanup: restore ability to build without LMDB support;
further slmdb API streamlining. Files: util/slmdb.[hc],
util/dict_lmdb.c.

2
postfix/src/global/mail_version.h
View File

@ -20,7 +20,7 @@
* Patches change both the patchlevel and the release date. Snapshots have no * Patches change both the patchlevel and the release date. Snapshots have no
* patchlevel; they change the release date only. * patchlevel; they change the release date only.
*/ */
#define MAIL_RELEASE_DATE "20131031" #define MAIL_RELEASE_DATE "20131101"
#define MAIL_VERSION_NUMBER "2.11" #define MAIL_VERSION_NUMBER "2.11"
#ifdef SNAPSHOT #ifdef SNAPSHOT

33
postfix/src/util/dict_lmdb.c
View File

@ -99,6 +99,9 @@ typedef struct {
* Each dict(3) API call is retried no more than a few times. For bulk-mode * Each dict(3) API call is retried no more than a few times. For bulk-mode
* transactions the number of retries is proportional to the size of the * transactions the number of retries is proportional to the size of the
* address space. * address space.
*
* We do not expise these details to the Postfix user interface. The purpose of
* Postfix is to solve problems, not punt them to the user.
*/ */
#ifndef SSIZE_T_MAX /* The maximum map size */ #ifndef SSIZE_T_MAX /* The maximum map size */
#define SSIZE_T_MAX __MAXINT__(ssize_t) /* XXX Assumes two's complement */ #define SSIZE_T_MAX __MAXINT__(ssize_t) /* XXX Assumes two's complement */
@ -471,7 +474,7 @@ static void dict_lmdb_close(DICT *dict)
dict_free(dict); dict_free(dict);
} }
/* dict_lmdb_longjmp - debug logging */ /* dict_lmdb_longjmp - repeat bulk transaction */
static void dict_lmdb_longjmp(void *context, int val) static void dict_lmdb_longjmp(void *context, int val)
{ {
@ -517,30 +520,38 @@ static void dict_lmdb_notify(void *context, int error_code,...)
/* dict_lmdb_open - open LMDB data base */ /* dict_lmdb_open - open LMDB data base */
DICT *dict_lmdb_open(const char *path, int dict_open_flags, int dict_flags) DICT *dict_lmdb_open(const char *path, int open_flags, int dict_flags)
{ {
DICT_LMDB *dict_lmdb; DICT_LMDB *dict_lmdb;
DICT *dict; DICT *dict;
struct stat st; struct stat st;
SLMDB slmdb; SLMDB slmdb;
char *mdb_path; char *mdb_path;
int mdb_open_flags, status; int mdb_flags, slmdb_flags, status;
int db_fd; int db_fd;
mdb_path = concatenate(path, "." DICT_TYPE_LMDB, (char *) 0); mdb_path = concatenate(path, "." DICT_TYPE_LMDB, (char *) 0);
mdb_open_flags = MDB_NOSUBDIR | MDB_NOLOCK; /*
if (dict_open_flags == O_RDONLY) * Impedance adapters.
mdb_open_flags |= MDB_RDONLY; */
mdb_flags = MDB_NOSUBDIR | MDB_NOLOCK;
if (open_flags == O_RDONLY)
mdb_flags |= MDB_RDONLY;
slmdb_flags = 0;
if (dict_flags & DICT_FLAG_BULK_UPDATE)
slmdb_flags |= SLMDB_FLAG_BULK;
/* /*
* Gracefully handle most database open errors. * Gracefully handle most database open errors.
*/ */
if ((status = slmdb_open(&slmdb, mdb_path, dict_open_flags, mdb_open_flags, if ((status = slmdb_init(&slmdb, dict_lmdb_map_size, DICT_LMDB_SIZE_INCR,
dict_flags & DICT_FLAG_BULK_UPDATE, dict_lmdb_map_size, DICT_LMDB_SIZE_MAX)) != 0
DICT_LMDB_SIZE_INCR, DICT_LMDB_SIZE_MAX)) != 0) { || (status = slmdb_open(&slmdb, mdb_path, open_flags, mdb_flags,
dict = dict_surrogate(DICT_TYPE_LMDB, path, dict_open_flags, slmdb_flags)) != 0) {
dict_flags, "open database %s: %m", mdb_path); dict = dict_surrogate(DICT_TYPE_LMDB, path, open_flags, dict_flags,
"open database %s: %s", mdb_path, mdb_strerror(status));
myfree(mdb_path); myfree(mdb_path);
return (dict); return (dict);
} }

176
postfix/src/util/slmdb.c
View File

@ -6,57 +6,61 @@
/* SYNOPSIS /* SYNOPSIS
/* #include <slmdb.h> /* #include <slmdb.h>
/* /*
/* size_t slmdb_map_size; /* int slmdb_init(slmdb, curr_limit, size_incr, hard_limit)
/* /* SLMDB *slmdb;
/* int slmdb_open(slmdb, path, open_flags, lmdb_flags, bulk_mode,
/* curr_limit, size_incr, hard_limit)
/* SLMDB *slmdb;
/* const char *path;
/* int open_flags;
/* int lmdb_flags;
/* int bulk_mode;
/* size_t curr_limit; /* size_t curr_limit;
/* int size_incr; /* int size_incr;
/* size_t hard_limit; /* size_t hard_limit;
/* /*
/* int slmdb_open(slmdb, path, open_flags, lmdb_flags, slmdb_flags)
/* SLMDB *slmdb;
/* const char *path;
/* int open_flags;
/* int lmdb_flags;
/* int slmdb_flags;
/*
/* int slmdb_close(slmdb) /* int slmdb_close(slmdb)
/* SLMDB *slmdb; /* SLMDB *slmdb;
/* /*
/* int slmdb_get(slmdb, mdb_key, mdb_value) /* int slmdb_get(slmdb, mdb_key, mdb_value)
/* SLMDB *slmdb; /* SLMDB *slmdb;
/* MDB_val *mdb_key; /* MDB_val *mdb_key;
/* MDB_val *mdb_value; /* MDB_val *mdb_value;
/* /*
/* int slmdb_put(slmdb, mdb_key, mdb_value, flags) /* int slmdb_put(slmdb, mdb_key, mdb_value, flags)
/* SLMDB *slmdb; /* SLMDB *slmdb;
/* MDB_val *mdb_key; /* MDB_val *mdb_key;
/* MDB_val *mdb_value; /* MDB_val *mdb_value;
/* int flags; /* int flags;
/* /*
/* int slmdb_del(slmdb, mdb_key) /* int slmdb_del(slmdb, mdb_key)
/* SLMDB *slmdb; /* SLMDB *slmdb;
/* MDB_val *mdb_key; /* MDB_val *mdb_key;
/* /*
/* int slmdb_cursor_get(slmdb, mdb_key, mdb_value, op) /* int slmdb_cursor_get(slmdb, mdb_key, mdb_value, op)
/* SLMDB *slmdb; /* SLMDB *slmdb;
/* MDB_val *mdb_key; /* MDB_val *mdb_key;
/* MDB_val *mdb_value; /* MDB_val *mdb_value;
/* MDB_cursor_op op; /* MDB_cursor_op op;
/* AUXILIARY FUNCTIONS /* AUXILIARY FUNCTIONS
/* int slmdb_fd(slmdb) /* int slmdb_fd(slmdb)
/* SLMDB *slmdb; /* SLMDB *slmdb;
/* /*
/* size_t slmdb_curr_limit(slmdb) /* size_t slmdb_curr_limit(slmdb)
/* SLMDB *slmdb; /* SLMDB *slmdb;
/* /*
/* int slmdb_control(slmdb, id, ...) /* int slmdb_control(slmdb, request, ...)
/* SLMDB *slmdb; /* SLMDB *slmdb;
/* int id; /* int request;
/* DESCRIPTION /* DESCRIPTION
/* This module simplifies the LMDB API by hiding recoverable /* This module simplifies the LMDB API by hiding recoverable
/* errors from the application. Details are given in the /* errors from the application. Details are given in the
/* section "ERROR RECOVERY". /* section "ERROR RECOVERY".
/* /*
/* slmdb_init() performs mandatory initialization before opening
/* an LMDB database. The result value is an LMDB status code
/* (zero in case of success).
/*
/* slmdb_open() opens an LMDB database. The result value is /* slmdb_open() opens an LMDB database. The result value is
/* an LMDB status code (zero in case of success). /* an LMDB status code (zero in case of success).
/* /*
@ -76,34 +80,68 @@
/* recovery. The result value is an LMDB status code (zero /* recovery. The result value is an LMDB status code (zero
/* in case of success). /* in case of success).
/* /*
/* slmdb_cursor_get() iterates over an LMDB database. The /* slmdb_cursor_get() is an mdb_cursor_get() wrapper with
/* result value is an LMDB status code (zero in case of success). /* automatic error recovery. The result value is an LMDB
/* status code (zero in case of success).
/* /*
/* slmdb_fd() returns the file descriptor for an open LMDB /* slmdb_fd() returns the file descriptor for the specified
/* database. This may be used for file status queries or /* database. This may be used for file status queries or
/* application-controlled locking. /* application-controlled locking.
/* /*
/* slmdb_curr_limit() returns the current database size limit /* slmdb_curr_limit() returns the current database size limit
/* for the specified database. /* for the specified database.
/* /*
/* slmdb_control() specifies optional features. The arguments /* slmdb_control() specifies optional features. The result is
/* are a list of (name, value) pairs, terminated with /* 0 in case of success, or -1 with errno indicating the nature
/* SLMDB_CTL_END. The result is 0 in case of success, or -1 /* of the problem.
/* with errno indicating the nature of the problem. The following /*
/* text enumerates the symbolic request names and the types /* Arguments:
/* of the corresponding additional arguments. /* .IP slmdb
/* .IP "SLMDB_CTL_LONGJMP_FN (void (*)(void *, int)) /* Pointer to caller-provided storage.
/* .IP curr_limit
/* The initial memory mapping size limit. This limit is
/* automatically increased when the database becomes full.
/* .IP size_incr
/* An integer factor by which the memory mapping size limit
/* is increased when the database becomes full.
/* .IP hard_limit
/* The upper bound for the memory mapping size limit.
/* .IP path
/* LMDB database pathname.
/* .IP open_flags
/* Flags that control file open operations. Do not specify
/* locking flags here.
/* .IP lmdb_flags
/* Flags that control the LMDB environment.
/* .IP slmdb_flags
/* Bit-wise OR of zero or more of the following:
/* .RS
/* .IP SLMDB_FLAG_BULK
/* Open the database for a "bulk" transaction that is not
/* committed until the database is closed.
/* .RE
/* .IP mdb_key
/* Pointer to caller-provided lookup key storage.
/* .IP mdb_value
/* Pointer to caller-provided value storage.
/* .IP op
/* LMDB cursor operation.
/* .IP request
/* The start of a list of (name, value) pairs, terminated with
/* SLMDB_CTL_END. The following text enumerates the symbolic
/* request names and the corresponding value types.
/* .RS .IP "SLMDB_CTL_LONGJMP_FN (void (*)(void *, int))
/* Application long-jump call-back function pointer. The /* Application long-jump call-back function pointer. The
/* function must not return and is called to repeat a failed /* function must not return and is called to repeat a failed
/* bulk-mode transaction from the start. The arguments are /* bulk-mode transaction from the start. The arguments are the
/* the application context and the setjmp() or sigsetjmp() /* application context and the setjmp() or sigsetjmp() result
/* result value. /* value.
/* .IP "SLMDB_CTL_NOTIFY_FN (void (*)(void *, int, ...))" /* .IP "SLMDB_CTL_NOTIFY_FN (void (*)(void *, int, ...))"
/* Application notification call-back function pointer. The /* Application notification call-back function pointer. The
/* function is called after succesful error recovery with as /* function is called after succesful error recovery with
/* arguments the application context, the MDB error code, and /* arguments the application context, the MDB error code, and
/* additional arguments that depend on the error code. /* additional arguments that depend on the error code. Details
/* Details are given in the section "ERROR RECOVERY". /* are given in the section "ERROR RECOVERY".
/* .IP "SLMDB_CTL_CONTEXT (void *)" /* .IP "SLMDB_CTL_CONTEXT (void *)"
/* Application context that is passed in application notification /* Application context that is passed in application notification
/* and long-jump call-back function calls. /* and long-jump call-back function calls.
@ -113,11 +151,13 @@
/* .IP "SLMDB_CTL_BULK_RETRY_LIMIT (int)" /* .IP "SLMDB_CTL_BULK_RETRY_LIMIT (int)"
/* How many times to recover from a bulk-mode transaction /* How many times to recover from a bulk-mode transaction
/* before giving up. /* before giving up.
/* .RE
/* ERROR RECOVERY /* ERROR RECOVERY
/* .ad /* .ad
/* .fi /* .fi
/* This module automatically repeats failed requests after /* This module automatically repeats failed requests after
/* recoverable errors, up to limits specified with slmdb_control(). /* recoverable errors, up to the limits specified with
/* slmdb_control().
/* /*
/* Recoverable errors are reported through an optional /* Recoverable errors are reported through an optional
/* notification function specified with slmdb_control(). With /* notification function specified with slmdb_control(). With
@ -146,6 +186,8 @@
/* Yorktown Heights, NY 10598, USA /* Yorktown Heights, NY 10598, USA
/*--*/ /*--*/
#ifdef HAS_LMDB
/* System library. */ /* System library. */
#include <sys/stat.h> #include <sys/stat.h>
@ -226,20 +268,17 @@ static int slmdb_prepare(SLMDB *slmdb)
* - With O_TRUNC we make a "drop" request before updating the database. * - With O_TRUNC we make a "drop" request before updating the database.
* *
* - With a bulk-mode transaction we commit when the database is closed. * - With a bulk-mode transaction we commit when the database is closed.
*
* XXX If we want to make the slmdb API suitable for general use, then the
* bulk/non-bulk handling must be generalized.
*/ */
if (slmdb->open_flags & O_TRUNC) { if (slmdb->open_flags & O_TRUNC) {
if ((status = mdb_drop(slmdb->txn, slmdb->dbi, 0)) != 0) if ((status = mdb_drop(slmdb->txn, slmdb->dbi, 0)) != 0)
return (status); return (status);
if ((slmdb->bulk_mode) == 0) { if ((slmdb->slmdb_flags & SLMDB_FLAG_BULK) == 0) {
if ((status = mdb_txn_commit(slmdb->txn))) if ((status = mdb_txn_commit(slmdb->txn)))
return (status); return (status);
slmdb->txn = 0; slmdb->txn = 0;
} }
} else if ((slmdb->lmdb_flags & MDB_RDONLY) != 0 } else if ((slmdb->lmdb_flags & MDB_RDONLY) != 0
|| (slmdb->bulk_mode) == 0) { || (slmdb->slmdb_flags & SLMDB_FLAG_BULK) == 0) {
mdb_txn_abort(slmdb->txn); mdb_txn_abort(slmdb->txn);
slmdb->txn = 0; slmdb->txn = 0;
} }
@ -267,9 +306,6 @@ static int slmdb_recover(SLMDB *slmdb, int status)
* *
* slmdb->txn must be either null (non-bulk transaction error), or an * slmdb->txn must be either null (non-bulk transaction error), or an
* aborted bulk-mode transaction. * aborted bulk-mode transaction.
*
* XXX If we want to make the slmdb API suitable for general use, then the
* bulk/non-bulk handling must be generalized.
*/ */
switch (status) { switch (status) {
@ -583,11 +619,27 @@ int slmdb_close(SLMDB *slmdb)
SLMDB_API_RETURN(slmdb, status); SLMDB_API_RETURN(slmdb, status);
} }
/* slmdb_init - mandatory initialization */
int slmdb_init(SLMDB *slmdb, size_t curr_limit, int size_incr,
size_t hard_limit)
{
/*
* This is a separate operation to keep the slmdb_open() API simple.
* Don't allocate resources here. Just store control information,
*/
slmdb->curr_limit = curr_limit;
slmdb->size_incr = size_incr;
slmdb->hard_limit = hard_limit;
return (MDB_SUCCESS);
}
/* slmdb_open - open wrapped LMDB database */ /* slmdb_open - open wrapped LMDB database */
int slmdb_open(SLMDB *slmdb, const char *path, int open_flags, int slmdb_open(SLMDB *slmdb, const char *path, int open_flags,
int lmdb_flags, int bulk_mode, size_t curr_limit, int lmdb_flags, int slmdb_flags)
int size_incr, size_t hard_limit)
{ {
struct stat st; struct stat st;
MDB_env *env; MDB_env *env;
@ -604,20 +656,21 @@ int slmdb_open(SLMDB *slmdb, const char *path, int open_flags,
/* /*
* Make sure that the memory map has room to store and commit an initial * Make sure that the memory map has room to store and commit an initial
* "drop" transaction. We have no way to recover from errors before the * "drop" transaction as well as fixed database metadata. We have no way
* first application-level request. * to recover from errors before the first application-level I/O request.
*/ */
#define SLMDB_FUDGE 8192 #define SLMDB_FUDGE 10240
if (curr_limit < SLMDB_FUDGE) if (slmdb->curr_limit < SLMDB_FUDGE)
curr_limit = SLMDB_FUDGE; slmdb->curr_limit = SLMDB_FUDGE;
if (stat(path, &st) == 0 && st.st_size > curr_limit - SLMDB_FUDGE) { if (stat(path, &st) == 0
if (st.st_size > hard_limit) && st.st_size > slmdb->curr_limit - SLMDB_FUDGE) {
hard_limit = st.st_size; if (st.st_size > slmdb->hard_limit)
if (st.st_size < hard_limit - SLMDB_FUDGE) slmdb->hard_limit = st.st_size;
curr_limit = st.st_size + SLMDB_FUDGE; if (st.st_size < slmdb->hard_limit - SLMDB_FUDGE)
slmdb->curr_limit = st.st_size + SLMDB_FUDGE;
else else
curr_limit = hard_limit; slmdb->curr_limit = slmdb->hard_limit;
} }
/* /*
@ -625,7 +678,7 @@ int slmdb_open(SLMDB *slmdb, const char *path, int open_flags,
* an LMDB environment, we usually don't need to do anything else with * an LMDB environment, we usually don't need to do anything else with
* the txn. It is currently used for truncate and for bulk transactions. * the txn. It is currently used for truncate and for bulk transactions.
*/ */
if ((status = mdb_env_set_mapsize(env, curr_limit)) != 0 if ((status = mdb_env_set_mapsize(env, slmdb->curr_limit)) != 0
|| (status = mdb_env_open(env, path, lmdb_flags, 0644)) != 0 || (status = mdb_env_open(env, path, lmdb_flags, 0644)) != 0
|| (status = mdb_txn_begin(env, (MDB_txn *) 0, || (status = mdb_txn_begin(env, (MDB_txn *) 0,
lmdb_flags & MDB_RDONLY, &txn)) != 0 lmdb_flags & MDB_RDONLY, &txn)) != 0
@ -640,10 +693,7 @@ int slmdb_open(SLMDB *slmdb, const char *path, int open_flags,
*/ */
slmdb->open_flags = open_flags; slmdb->open_flags = open_flags;
slmdb->lmdb_flags = lmdb_flags; slmdb->lmdb_flags = lmdb_flags;
slmdb->bulk_mode = bulk_mode; slmdb->slmdb_flags = slmdb_flags;
slmdb->curr_limit = curr_limit;
slmdb->size_incr = size_incr;
slmdb->hard_limit = hard_limit;
slmdb->env = env; slmdb->env = env;
slmdb->dbi = dbi; slmdb->dbi = dbi;
slmdb->db_fd = db_fd; slmdb->db_fd = db_fd;
@ -662,3 +712,5 @@ int slmdb_open(SLMDB *slmdb, const char *path, int open_flags,
return (status); return (status);
} }
#endif

21
postfix/src/util/slmdb.h
View File

@ -33,18 +33,18 @@
#endif #endif
typedef struct { typedef struct {
size_t curr_limit; /* database soft size limit */
int size_incr; /* database expansion factor */
size_t hard_limit; /* database hard size limit */
int open_flags; /* open() flags */ int open_flags; /* open() flags */
int lmdb_flags; /* LMDB-specific flags */ int lmdb_flags; /* LMDB-specific flags */
int bulk_mode; /* bulk-mode flag */ int slmdb_flags; /* bulk-mode flag */
size_t curr_limit; /* database soft size limit */
int size_incr; /* database growth factor */
size_t hard_limit; /* database hard size limit */
MDB_env *env; /* database environment */ MDB_env *env; /* database environment */
MDB_dbi dbi; /* database instance */ MDB_dbi dbi; /* database instance */
MDB_txn *txn; /* bulk transaction */ MDB_txn *txn; /* bulk transaction */
int db_fd; /* database file handle */ int db_fd; /* database file handle */
MDB_cursor *cursor; /* iterator */ MDB_cursor *cursor; /* iterator */
void (*longjmp_fn) (void *, int); /* exception handling */ void (*longjmp_fn) (void *, int);/* exception handling */
void (*notify_fn) (void *, int,...); /* workaround notification */ void (*notify_fn) (void *, int,...); /* workaround notification */
void *cb_context; /* call-back context */ void *cb_context; /* call-back context */
int api_retry_count; /* slmdb(3) API call retry count */ int api_retry_count; /* slmdb(3) API call retry count */
@ -53,12 +53,15 @@ typedef struct {
int bulk_retry_limit; /* bulk_mode retry limit */ int bulk_retry_limit; /* bulk_mode retry limit */
} SLMDB; } SLMDB;
extern int slmdb_open(SLMDB *, const char *, int, int, int, size_t, int, size_t); #define SLMDB_FLAG_BULK (1 << 0)
extern int slmdb_init(SLMDB *, size_t, int, size_t);
extern int slmdb_open(SLMDB *, const char *, int, int, int);
extern int slmdb_get(SLMDB *, MDB_val *, MDB_val *); extern int slmdb_get(SLMDB *, MDB_val *, MDB_val *);
extern int slmdb_put(SLMDB *, MDB_val *, MDB_val *, int); extern int slmdb_put(SLMDB *, MDB_val *, MDB_val *, int);
extern int slmdb_del(SLMDB *, MDB_val *); extern int slmdb_del(SLMDB *, MDB_val *);
extern int slmdb_cursor_get(SLMDB *, MDB_val *, MDB_val *, MDB_cursor_op); extern int slmdb_cursor_get(SLMDB *, MDB_val *, MDB_val *, MDB_cursor_op);
extern int slmdb_control(SLMDB *, int, ...); extern int slmdb_control(SLMDB *, int,...);
extern int slmdb_close(SLMDB *); extern int slmdb_close(SLMDB *);
#define slmdb_fd(slmdb) ((slmdb)->db_fd) #define slmdb_fd(slmdb) ((slmdb)->db_fd)
@ -72,8 +75,8 @@ extern int slmdb_close(SLMDB *);
#define SLMDB_CTL_API_RETRY_LIMIT 5 /* per slmdb(3) API call */ #define SLMDB_CTL_API_RETRY_LIMIT 5 /* per slmdb(3) API call */
#define SLMDB_CTL_BULK_RETRY_LIMIT 6 /* per bulk update */ #define SLMDB_CTL_BULK_RETRY_LIMIT 6 /* per bulk update */
typedef void (*SLMDB_NOTIFY_FN)(void *, int, ...); typedef void (*SLMDB_NOTIFY_FN) (void *, int,...);
typedef void (*SLMDB_LONGJMP_FN)(void *, int); typedef void (*SLMDB_LONGJMP_FN) (void *, int);
/* LICENSE /* LICENSE
/* .ad /* .ad