From ee19966326b9d6ddcd8c97d6fd934e144bd7f3e3 Mon Sep 17 00:00:00 2001 From: Evan Hunt Date: Fri, 19 Feb 2021 15:04:50 -0800 Subject: [PATCH 1/5] allow dns_journal_rollforward() to read old journal files when the 'max-ixfr-ratio' option was added, journal transaction headers were revised to include a count of RR's in each transaction. this made it impossible to read old journal files after an upgrade. this branch restores the ability to read version 1 transaction headers. when rolling forward, printing journal contents, if the wrong transaction header format is found, we can switch. when dns_journal_rollforward() detects a version 1 transaction header, it returns DNS_R_RECOVERABLE. this triggers zone_postload() to force a rewrite of the journal file in the new format, and also to schedule a dump of the zone database with minimal delay. journal repair is done by dns_journal_compact(), which rewrites the entire journal, ignoring 'max-journal-size'. journal size is corrected later. newly created journal files now have "BIND LOG V9.2" in their headers instead of "BIND LOG V9". files with the new version string cannot be read using the old transaction header format. note that this means newly created journal files will be rejected by older versions of named. named-journalprint now takes a "-x" option, causing it to print transaction header information before each delta, including its format version. --- bin/tools/named-journalprint.c | 33 ++- bin/tools/named-journalprint.rst | 5 +- doc/man/named-journalprint.1in | 5 +- lib/dns/include/dns/events.h | 1 + lib/dns/include/dns/journal.h | 18 +- lib/dns/journal.c | 493 +++++++++++++++++++++++-------- lib/dns/zone.c | 43 ++- 7 files changed, 450 insertions(+), 148 deletions(-) diff --git a/bin/tools/named-journalprint.c b/bin/tools/named-journalprint.c index 20b8740021..0fcfa2d0de 100644 --- a/bin/tools/named-journalprint.c +++ b/bin/tools/named-journalprint.c @@ -13,6 +13,7 @@ #include +#include #include #include #include @@ -23,6 +24,14 @@ #include #include +const char *progname = NULL; + +static void +usage(void) { + fprintf(stderr, "Usage: %s [-x] journal\n", progname); + exit(1); +} + /* * Setup logging to use stderr. */ @@ -57,18 +66,32 @@ main(int argc, char **argv) { isc_mem_t *mctx = NULL; isc_result_t result; isc_log_t *lctx = NULL; + uint32_t flags = 0U; + char ch; - if (argc != 2) { - printf("usage: %s journal\n", argv[0]); - return (1); + progname = argv[0]; + while ((ch = isc_commandline_parse(argc, argv, "x")) != -1) { + switch (ch) { + case 'x': + flags |= DNS_JOURNAL_PRINTXHDR; + break; + default: + usage(); + } } - file = argv[1]; + argc -= isc_commandline_index; + argv += isc_commandline_index; + + if (argc != 1) { + usage(); + } + file = argv[0]; isc_mem_create(&mctx); RUNTIME_CHECK(setup_logging(mctx, stderr, &lctx) == ISC_R_SUCCESS); - result = dns_journal_print(mctx, file, stdout); + result = dns_journal_print(mctx, flags, file, stdout); if (result == DNS_R_NOJOURNAL) { fprintf(stderr, "%s\n", dns_result_totext(result)); } diff --git a/bin/tools/named-journalprint.rst b/bin/tools/named-journalprint.rst index ffa9ac9577..2ce6a05bb4 100644 --- a/bin/tools/named-journalprint.rst +++ b/bin/tools/named-journalprint.rst @@ -29,7 +29,7 @@ named-journalprint - print zone journal in human-readable form Synopsis ~~~~~~~~ -:program:`named-journalprint` {journal} +:program:`named-journalprint` [**-x**] {journal} Description ~~~~~~~~~~~ @@ -50,6 +50,9 @@ into a human-readable text format. Each line begins with ``add`` or ``del``, to indicate whether the record was added or deleted, and continues with the resource record in master-file format. +The ``-x`` option causes additional information about the transaction +header to be printed before each group of changes. + See Also ~~~~~~~~ diff --git a/doc/man/named-journalprint.1in b/doc/man/named-journalprint.1in index 2c86eec8df..84684cb8d6 100644 --- a/doc/man/named-journalprint.1in +++ b/doc/man/named-journalprint.1in @@ -32,7 +32,7 @@ level margin: \\n[rst2man-indent\\n[rst2man-indent-level]] .. .SH SYNOPSIS .sp -\fBnamed\-journalprint\fP {journal} +\fBnamed\-journalprint\fP [\fB\-x\fP] {journal} .SH DESCRIPTION .sp \fBnamed\-journalprint\fP prints the contents of a zone journal file in a @@ -50,6 +50,9 @@ file. into a human\-readable text format. Each line begins with \fBadd\fP or \fBdel\fP, to indicate whether the record was added or deleted, and continues with the resource record in master\-file format. +.sp +The \fB\-x\fP option causes additional information about the transaction +header to be printed before each group of changes. .SH SEE ALSO .sp \fBnamed(8)\fP, \fBnsupdate(1)\fP, BIND 9 Administrator Reference Manual. diff --git a/lib/dns/include/dns/events.h b/lib/dns/include/dns/events.h index 50a8d01ac6..894c4dd211 100644 --- a/lib/dns/include/dns/events.h +++ b/lib/dns/include/dns/events.h @@ -79,6 +79,7 @@ #define DNS_EVENT_RPZUPDATED (ISC_EVENTCLASS_DNS + 57) #define DNS_EVENT_STARTUPDATE (ISC_EVENTCLASS_DNS + 58) #define DNS_EVENT_TRYSTALE (ISC_EVENTCLASS_DNS + 59) +#define DNS_EVENT_ZONEFLUSH (ISC_EVENTCLASS_DNS + 60) #define DNS_EVENT_FIRSTEVENT (ISC_EVENTCLASS_DNS + 0) #define DNS_EVENT_LASTEVENT (ISC_EVENTCLASS_DNS + 65535) diff --git a/lib/dns/include/dns/journal.h b/lib/dns/include/dns/journal.h index 8acf9ab44e..4d3b73efb5 100644 --- a/lib/dns/include/dns/journal.h +++ b/lib/dns/include/dns/journal.h @@ -48,6 +48,12 @@ #define DNS_JOURNAL_SIZE_MAX INT32_MAX #define DNS_JOURNAL_SIZE_MIN 4096 +/*% Print transaction header data */ +#define DNS_JOURNAL_PRINTXHDR 0x0001 + +/*% Rewrite whole journal file instead of compacting */ +#define DNS_JOURNAL_COMPACTALL 0x0001 + /*** *** Types ***/ @@ -258,12 +264,18 @@ dns_journal_rollforward(isc_mem_t *mctx, dns_db_t *db, unsigned int options, *\li DNS_R_NOJOURNAL when journal does not exist. *\li ISC_R_NOTFOUND when current serial in not in journal. *\li ISC_R_RANGE when current serial in not in journals range. - *\li ISC_R_SUCCESS journal has been applied successfully to database. + *\li DNS_R_UPTODATE when the database was already up to date. + *\li ISC_R_SUCCESS journal has been applied successfully to the + * database without any issues. + *\li DNS_R_RECOVERABLE if successful or up to date, but the journal + * was found to contain at least one outdated transaction header. + * * others */ isc_result_t -dns_journal_print(isc_mem_t *mctx, const char *filename, FILE *file); +dns_journal_print(isc_mem_t *mctx, uint32_t flags, const char *filename, + FILE *file); /* For debugging not general use */ isc_result_t @@ -286,7 +298,7 @@ dns_db_diffx(dns_diff_t *diff, dns_db_t *dba, dns_dbversion_t *dbvera, isc_result_t dns_journal_compact(isc_mem_t *mctx, char *filename, uint32_t serial, - uint32_t target_size); + uint32_t flags, uint32_t target_size); /*%< * Attempt to compact the journal if it is greater that 'target_size'. * Changes from 'serial' onwards will be preserved. If the journal diff --git a/lib/dns/journal.c b/lib/dns/journal.c index bc1ad96314..5007f0c9ef 100644 --- a/lib/dns/journal.c +++ b/lib/dns/journal.c @@ -18,6 +18,7 @@ #include #include #include +#include #include #include #include @@ -193,6 +194,11 @@ typedef struct { */ #define JOURNAL_HEADER_SIZE 64 /* Bytes. */ +typedef enum { + XHDR_VERSION1 = 1, + XHDR_VERSION2 = 2, +} xhdr_version_t; + /*% * The on-disk representation of the journal header. * All numbers are stored in big-endian order. @@ -216,7 +222,7 @@ typedef union { } journal_rawheader_t; /*% - * The on-disk representation of the transaction header. + * The on-disk representation of the transaction header, version 2. * There is one of these at the beginning of each transaction. */ typedef struct { @@ -226,6 +232,16 @@ typedef struct { unsigned char serial1[4]; /*%< SOA serial after update. */ } journal_rawxhdr_t; +/*% + * Old-style raw transaction header, version 1, used for backward + * compatibility mode. + */ +typedef struct { + unsigned char size[4]; + unsigned char serial0[4]; + unsigned char serial1[4]; +} journal_rawxhdr_ver1_t; + /*% * The on-disk representation of the RR header. * There is one of these at the beginning of each RR. @@ -275,16 +291,19 @@ typedef struct { * Initial contents to store in the header of a newly created * journal file. * - * The header starts with the magic string ";BIND LOG V9\n" + * The header starts with the magic string ";BIND LOG V9.2\n" * to identify the file as a BIND 9 journal file. An ASCII * identification string is used rather than a binary magic * number to be consistent with BIND 8 (BIND 8 journal files * are ASCII text files). */ -static journal_header_t initial_journal_header = { +static journal_header_t journal_header_ver1 = { ";BIND LOG V9\n", { 0, 0 }, { 0, 0 }, 0, 0, 0 }; +static journal_header_t initial_journal_header = { + ";BIND LOG V9.2\n", { 0, 0 }, { 0, 0 }, 0, 0, 0 +}; #define JOURNAL_EMPTY(h) ((h)->begin.offset == (h)->end.offset) @@ -300,13 +319,19 @@ struct dns_journal { unsigned int magic; /*%< JOUR */ isc_mem_t *mctx; /*%< Memory context */ journal_state_t state; - char *filename; /*%< Journal file name */ - FILE *fp; /*%< File handle */ - isc_offset_t offset; /*%< Current file offset */ - journal_header_t header; /*%< In-core journal header */ - unsigned char *rawindex; /*%< In-core buffer for journal index - * in on-disk format */ - journal_pos_t *index; /*%< In-core journal index */ + xhdr_version_t xhdr_version; /*%< Expected transaction header version */ + bool header_ver1; /*%< Transaction header compatibility + * mode is allowed */ + bool recovered; /*%< A recoverable error was found + * while reading the journal */ + char *filename; /*%< Journal file name */ + FILE *fp; /*%< File handle */ + isc_offset_t offset; /*%< Current file offset */ + journal_xhdr_t curxhdr; /*%< Current transaction header */ + journal_header_t header; /*%< In-core journal header */ + unsigned char *rawindex; /*%< In-core buffer for journal index + * in on-disk format */ + journal_pos_t *index; /*%< In-core journal index */ /*% Current transaction state (when writing). */ struct { @@ -321,8 +346,7 @@ struct dns_journal { journal_pos_t bpos; /*%< Position before first, */ journal_pos_t epos; /*%< and after last transaction */ /* The rest is iterator state. */ - uint32_t current_serial; /*%< Current SOA serial - * */ + uint32_t current_serial; /*%< Current SOA serial */ isc_buffer_t source; /*%< Data from disk */ isc_buffer_t target; /*%< Data from _fromwire check */ dns_decompress_t dctx; /*%< Dummy decompression ctx */ @@ -353,6 +377,7 @@ journal_pos_encode(journal_rawpos_t *raw, journal_pos_t *cooked) { static void journal_header_decode(journal_rawheader_t *raw, journal_header_t *cooked) { INSIST(sizeof(cooked->format) == sizeof(raw->h.format)); + memmove(cooked->format, raw->h.format, sizeof(cooked->format)); journal_pos_decode(&raw->h.begin, &cooked->begin); journal_pos_decode(&raw->h.end, &cooked->end); @@ -366,6 +391,7 @@ journal_header_encode(journal_header_t *cooked, journal_rawheader_t *raw) { unsigned char flags = 0; INSIST(sizeof(cooked->format) == sizeof(raw->h.format)); + memset(raw->pad, 0, sizeof(raw->pad)); memmove(raw->h.format, cooked->format, sizeof(raw->h.format)); journal_pos_encode(&raw->h.begin, &cooked->begin); @@ -432,6 +458,7 @@ journal_write(dns_journal_t *j, void *mem, size_t nbytes) { static isc_result_t journal_fsync(dns_journal_t *j) { isc_result_t result; + result = isc_stdio_flush(j->fp); if (result != ISC_R_SUCCESS) { isc_log_write(JOURNAL_COMMON_LOGARGS, ISC_LOG_ERROR, @@ -452,26 +479,49 @@ journal_fsync(dns_journal_t *j) { /* * Read/write a transaction header at the current file position. */ - static isc_result_t journal_read_xhdr(dns_journal_t *j, journal_xhdr_t *xhdr) { - journal_rawxhdr_t raw; isc_result_t result; - result = journal_read(j, &raw, sizeof(raw)); - if (result != ISC_R_SUCCESS) { - return (result); + + switch (j->xhdr_version) { + case XHDR_VERSION1: { + journal_rawxhdr_ver1_t raw; + result = journal_read(j, &raw, sizeof(raw)); + if (result != ISC_R_SUCCESS) { + return (result); + } + xhdr->size = decode_uint32(raw.size); + xhdr->count = 0; + xhdr->serial0 = decode_uint32(raw.serial0); + xhdr->serial1 = decode_uint32(raw.serial1); + j->curxhdr = *xhdr; + return (ISC_R_SUCCESS); + } + + case XHDR_VERSION2: { + journal_rawxhdr_t raw; + result = journal_read(j, &raw, sizeof(raw)); + if (result != ISC_R_SUCCESS) { + return (result); + } + xhdr->size = decode_uint32(raw.size); + xhdr->count = decode_uint32(raw.count); + xhdr->serial0 = decode_uint32(raw.serial0); + xhdr->serial1 = decode_uint32(raw.serial1); + j->curxhdr = *xhdr; + return (ISC_R_SUCCESS); + } + + default: + return (ISC_R_NOTIMPLEMENTED); } - xhdr->size = decode_uint32(raw.size); - xhdr->count = decode_uint32(raw.count); - xhdr->serial0 = decode_uint32(raw.serial0); - xhdr->serial1 = decode_uint32(raw.serial1); - return (ISC_R_SUCCESS); } static isc_result_t journal_write_xhdr(dns_journal_t *j, uint32_t size, uint32_t count, uint32_t serial0, uint32_t serial1) { journal_rawxhdr_t raw; + encode_uint32(size, raw.size); encode_uint32(count, raw.count); encode_uint32(serial0, raw.serial0); @@ -487,6 +537,7 @@ static isc_result_t journal_read_rrhdr(dns_journal_t *j, journal_rrhdr_t *rrhdr) { journal_rawrrhdr_t raw; isc_result_t result; + result = journal_read(j, &raw, sizeof(raw)); if (result != ISC_R_SUCCESS) { return (result); @@ -503,7 +554,7 @@ journal_file_create(isc_mem_t *mctx, const char *filename) { journal_rawheader_t rawheader; int index_size = 56; /* XXX configurable */ int size; - void *mem; /* Memory for temporary index image. */ + void *mem = NULL; /* Memory for temporary index image. */ INSIST(sizeof(journal_rawheader_t) == JOURNAL_HEADER_SIZE); @@ -558,23 +609,15 @@ journal_open(isc_mem_t *mctx, const char *filename, bool writable, bool create, journal_rawheader_t rawheader; dns_journal_t *j; - INSIST(journalp != NULL && *journalp == NULL); + REQUIRE(journalp != NULL && *journalp == NULL); + j = isc_mem_get(mctx, sizeof(*j)); - - j->mctx = NULL; + *j = (dns_journal_t){ .state = JOURNAL_STATE_INVALID, + .filename = isc_mem_strdup(mctx, filename), + .xhdr_version = XHDR_VERSION2 }; isc_mem_attach(mctx, &j->mctx); - j->state = JOURNAL_STATE_INVALID; - j->fp = NULL; - j->filename = isc_mem_strdup(mctx, filename); - j->index = NULL; - j->rawindex = NULL; - - if (j->filename == NULL) { - FAIL(ISC_R_NOMEMORY); - } result = isc_stdio_open(j->filename, writable ? "rb+" : "rb", &fp); - if (result == ISC_R_FILENOTFOUND) { if (create) { isc_log_write(JOURNAL_COMMON_LOGARGS, ISC_LOG_DEBUG(1), @@ -607,9 +650,29 @@ journal_open(isc_mem_t *mctx, const char *filename, bool writable, bool create, CHECK(journal_seek(j, 0)); CHECK(journal_read(j, &rawheader, sizeof(rawheader))); - if (memcmp(rawheader.h.format, initial_journal_header.format, - sizeof(initial_journal_header.format)) != 0) + if (memcmp(rawheader.h.format, journal_header_ver1.format, + sizeof(journal_header_ver1.format)) == 0) { + /* + * The file header says it's the old format, but it + * still might have the new xhdr format because we + * forgot to change the format string when we introduced + * the new xhdr. When we first try to read it, we assume + * it uses the new xhdr format. If that fails, we'll be + * called a second time with compat set to true, in which + * case we can lower xhdr_version to 1 if we find a + * corrupt transaction. + */ + j->header_ver1 = true; + } else if (memcmp(rawheader.h.format, initial_journal_header.format, + sizeof(initial_journal_header.format)) == 0) + { + /* + * File header says this is format version 2; all + * transactions have to match. + */ + j->header_ver1 = false; + } else { isc_log_write(JOURNAL_COMMON_LOGARGS, ISC_LOG_ERROR, "%s: journal format not recognized", j->filename); FAIL(ISC_R_UNEXPECTED); @@ -795,9 +858,11 @@ ixfr_order(const void *av, const void *bv) { * Other results due to file errors are possible. */ static isc_result_t -journal_next(dns_journal_t *j, journal_pos_t *pos) { +journal_next(dns_journal_t *j, journal_pos_t *pos, bool retry) { isc_result_t result; journal_xhdr_t xhdr; + size_t hdrsize; + REQUIRE(DNS_JOURNAL_VALID(j)); result = journal_seek(j, pos->offset); @@ -808,6 +873,7 @@ journal_next(dns_journal_t *j, journal_pos_t *pos) { if (pos->serial == j->header.end.serial) { return (ISC_R_NOMORE); } + /* * Read the header of the current transaction. * This will return ISC_R_NOMORE if we are at EOF. @@ -820,26 +886,59 @@ journal_next(dns_journal_t *j, journal_pos_t *pos) { /* * Check serial number consistency. */ - if (xhdr.serial0 != pos->serial) { - isc_log_write(JOURNAL_COMMON_LOGARGS, ISC_LOG_ERROR, - "%s: journal file corrupt: " - "expected serial %u, got %u", - j->filename, pos->serial, xhdr.serial0); - return (ISC_R_UNEXPECTED); + if (xhdr.serial0 != pos->serial || + isc_serial_le(xhdr.serial1, xhdr.serial0)) { + if (j->header_ver1 && j->xhdr_version == XHDR_VERSION1 && + xhdr.serial1 == pos->serial && !retry) + { + /* XHDR_VERSION1 -> XHDR_VERSION2 */ + isc_log_write( + JOURNAL_COMMON_LOGARGS, ISC_LOG_DEBUG(3), + "%s: XHDR_VERSION1 -> XHDR_VERSION2 at %u\n", + j->filename, pos->serial); + j->xhdr_version = XHDR_VERSION2; + result = journal_next(j, pos, true); + if (result == ISC_R_SUCCESS) { + j->recovered = true; + } + return (result); + } else if (j->header_ver1 && j->xhdr_version == XHDR_VERSION2 && + xhdr.count == pos->serial && !retry) + { + /* XHDR_VERSION2 -> XHDR_VERSION1 */ + isc_log_write( + JOURNAL_COMMON_LOGARGS, ISC_LOG_DEBUG(3), + "%s: XHDR_VERSION2 -> XHDR_VERSION1 at %u\n", + j->filename, pos->serial); + j->xhdr_version = XHDR_VERSION1; + result = journal_next(j, pos, true); + if (result == ISC_R_SUCCESS) { + j->recovered = true; + } + return (result); + } else { + isc_log_write(JOURNAL_COMMON_LOGARGS, ISC_LOG_ERROR, + "%s: journal file corrupt: " + "expected serial %u, got %u", + j->filename, pos->serial, xhdr.serial0); + return (ISC_R_UNEXPECTED); + } } /* * Check for offset wraparound. */ - if ((isc_offset_t)(pos->offset + sizeof(journal_rawxhdr_t) + - xhdr.size) < pos->offset) - { + hdrsize = (j->xhdr_version == XHDR_VERSION2) + ? sizeof(journal_rawxhdr_t) + : sizeof(journal_rawxhdr_ver1_t); + + if ((isc_offset_t)(pos->offset + hdrsize + xhdr.size) < pos->offset) { isc_log_write(JOURNAL_COMMON_LOGARGS, ISC_LOG_ERROR, "%s: offset too large", j->filename); return (ISC_R_UNEXPECTED); } - pos->offset += sizeof(journal_rawxhdr_t) + xhdr.size; + pos->offset += hdrsize + xhdr.size; pos->serial = xhdr.serial1; return (ISC_R_SUCCESS); } @@ -879,9 +978,11 @@ index_find(dns_journal_t *j, uint32_t serial, journal_pos_t *best_guess) { static void index_add(dns_journal_t *j, journal_pos_t *pos) { unsigned int i; + if (j->index == NULL) { return; } + /* * Search for a vacant position. */ @@ -953,6 +1054,7 @@ static isc_result_t journal_find(dns_journal_t *j, uint32_t serial, journal_pos_t *pos) { isc_result_t result; journal_pos_t current_pos; + REQUIRE(DNS_JOURNAL_VALID(j)); if (DNS_SERIAL_GT(j->header.begin.serial, serial)) { @@ -973,7 +1075,7 @@ journal_find(dns_journal_t *j, uint32_t serial, journal_pos_t *pos) { if (DNS_SERIAL_GT(current_pos.serial, serial)) { return (ISC_R_NOTFOUND); } - result = journal_next(j, ¤t_pos); + result = journal_next(j, ¤t_pos, false); if (result != ISC_R_SUCCESS) { return (result); } @@ -1187,7 +1289,7 @@ dns_journal_commit(dns_journal_t *j) { if (!JOURNAL_EMPTY(&j->header)) { while (!DNS_SERIAL_GT(j->x.pos[1].serial, j->header.begin.serial)) { - CHECK(journal_next(j, &j->header.begin)); + CHECK(journal_next(j, &j->header.begin, false)); } index_invalidate(j, j->x.pos[1].serial); } @@ -1256,6 +1358,7 @@ failure: isc_result_t dns_journal_write_transaction(dns_journal_t *j, dns_diff_t *diff) { isc_result_t result; + CHECK(dns_diff_sort(diff, ixfr_order)); CHECK(dns_journal_begin_transaction(j)); CHECK(dns_journal_writediff(j, diff)); @@ -1267,9 +1370,13 @@ failure: void dns_journal_destroy(dns_journal_t **journalp) { - dns_journal_t *j = *journalp; + dns_journal_t *j = NULL; + + REQUIRE(journalp != NULL); + REQUIRE(DNS_JOURNAL_VALID(*journalp)); + + j = *journalp; *journalp = NULL; - REQUIRE(DNS_JOURNAL_VALID(j)); j->it.result = ISC_R_FAILURE; dns_name_invalidate(&j->it.name); @@ -1346,33 +1453,38 @@ roll_forward(dns_journal_t *j, dns_db_t *db, unsigned int options) { * Locate a journal entry for the current database serial. */ CHECK(journal_find(j, db_serial, &pos)); - /* - * XXX do more drastic things, like marking zone stale, - * if this fails? - */ - /* - * XXXRTH The zone code should probably mark the zone as bad and - * scream loudly into the log if this is a dynamic update - * log reply that failed. - */ end_serial = dns_journal_last_serial(j); + + /* + * If we're reading a version 1 file, scan all the transactions + * to see if the journal needs rewriting: if any outdated + * transaction headers are found, j->recovered will be set. + */ + if (j->header_ver1) { + uint32_t start_serial = dns_journal_first_serial(j); + + CHECK(dns_journal_iter_init(j, start_serial, db_serial, NULL)); + for (result = dns_journal_first_rr(j); result == ISC_R_SUCCESS; + result = dns_journal_next_rr(j)) + { + continue; + } + } + if (db_serial == end_serial) { CHECK(DNS_R_UPTODATE); } CHECK(dns_journal_iter_init(j, db_serial, end_serial, NULL)); - for (result = dns_journal_first_rr(j); result == ISC_R_SUCCESS; result = dns_journal_next_rr(j)) { - dns_name_t *name; - uint32_t ttl; - dns_rdata_t *rdata; + dns_name_t *name = NULL; + dns_rdata_t *rdata = NULL; dns_difftuple_t *tuple = NULL; + uint32_t ttl; - name = NULL; - rdata = NULL; dns_journal_current_rr(j, &name, &ttl, &rdata); if (rdata->type == dns_rdatatype_soa) { @@ -1450,13 +1562,12 @@ failure: isc_result_t dns_journal_rollforward(isc_mem_t *mctx, dns_db_t *db, unsigned int options, const char *filename) { - dns_journal_t *j; + dns_journal_t *j = NULL; isc_result_t result; REQUIRE(DNS_DB_VALID(db)); REQUIRE(filename != NULL); - j = NULL; result = dns_journal_open(mctx, filename, DNS_JOURNAL_READ, &j); if (result == ISC_R_NOTFOUND) { isc_log_write(JOURNAL_DEBUG_LOGARGS(3), "no journal file, but " @@ -1466,20 +1577,26 @@ dns_journal_rollforward(isc_mem_t *mctx, dns_db_t *db, unsigned int options, if (result != ISC_R_SUCCESS) { return (result); } + if (JOURNAL_EMPTY(&j->header)) { - result = DNS_R_UPTODATE; - } else { - result = roll_forward(j, db, options); + CHECK(DNS_R_UPTODATE); } - dns_journal_destroy(&j); + result = roll_forward(j, db, options); + if ((result == ISC_R_SUCCESS || result == DNS_R_UPTODATE) && + j->recovered) { + result = DNS_R_RECOVERABLE; + } +failure: + dns_journal_destroy(&j); return (result); } isc_result_t -dns_journal_print(isc_mem_t *mctx, const char *filename, FILE *file) { - dns_journal_t *j; +dns_journal_print(isc_mem_t *mctx, uint32_t flags, const char *filename, + FILE *file) { + dns_journal_t *j = NULL; isc_buffer_t source; /* Transaction data from disk */ isc_buffer_t target; /* Ditto after _fromwire check */ uint32_t start_serial; /* Database SOA serial */ @@ -1488,17 +1605,15 @@ dns_journal_print(isc_mem_t *mctx, const char *filename, FILE *file) { dns_diff_t diff; unsigned int n_soa = 0; unsigned int n_put = 0; + bool printxhdr = ((flags & DNS_JOURNAL_PRINTXHDR) != 0); REQUIRE(filename != NULL); - j = NULL; result = dns_journal_open(mctx, filename, DNS_JOURNAL_READ, &j); if (result == ISC_R_NOTFOUND) { isc_log_write(JOURNAL_DEBUG_LOGARGS(3), "no journal file"); return (DNS_R_NOJOURNAL); - } - - if (result != ISC_R_SUCCESS) { + } else if (result != ISC_R_SUCCESS) { isc_log_write(JOURNAL_COMMON_LOGARGS, ISC_LOG_ERROR, "journal open failure: %s: %s", isc_result_totext(result), filename); @@ -1526,13 +1641,11 @@ dns_journal_print(isc_mem_t *mctx, const char *filename, FILE *file) { for (result = dns_journal_first_rr(j); result == ISC_R_SUCCESS; result = dns_journal_next_rr(j)) { - dns_name_t *name; - uint32_t ttl; - dns_rdata_t *rdata; + dns_name_t *name = NULL; + dns_rdata_t *rdata = NULL; dns_difftuple_t *tuple = NULL; + uint32_t ttl; - name = NULL; - rdata = NULL; dns_journal_current_rr(j, &name, &ttl, &rdata); if (rdata->type == dns_rdatatype_soa) { @@ -1549,12 +1662,21 @@ dns_journal_print(isc_mem_t *mctx, const char *filename, FILE *file) { j->filename); FAIL(ISC_R_UNEXPECTED); } + + if (printxhdr && n_soa == 1) { + fprintf(file, + "Transaction: version %d size %u rrcount %u " + "startserial %u endserial %u\n", + j->xhdr_version, j->curxhdr.size, + j->curxhdr.count, j->curxhdr.serial0, + j->curxhdr.serial1); + } CHECK(dns_difftuple_create( diff.mctx, n_soa == 1 ? DNS_DIFFOP_DEL : DNS_DIFFOP_ADD, name, ttl, rdata, &tuple)); dns_diff_append(&diff, &tuple); - if (++n_put > 100) { + if (++n_put != 0 || printxhdr) { result = dns_diff_print(&diff, file); dns_diff_clear(&diff); n_put = 0; @@ -1568,7 +1690,7 @@ dns_journal_print(isc_mem_t *mctx, const char *filename, FILE *file) { } CHECK(result); - if (n_put != 0) { + if (!printxhdr && n_put != 0) { result = dns_diff_print(&diff, file); dns_diff_clear(&diff); } @@ -1648,7 +1770,7 @@ dns_journal_get_sourceserial(dns_journal_t *j, uint32_t *sourceserial) { */ static isc_result_t -read_one_rr(dns_journal_t *j); +read_one_rr(dns_journal_t *j, bool retry); /* * Make sure the buffer 'b' is has at least 'size' bytes @@ -1706,7 +1828,7 @@ dns_journal_iter_init(dns_journal_t *j, uint32_t begin_serial, size += xhdr.size; count += xhdr.count; - result = journal_next(j, &pos); + result = journal_next(j, &pos, false); if (result == ISC_R_NOMORE) { result = ISC_R_SUCCESS; } @@ -1742,22 +1864,22 @@ dns_journal_first_rr(dns_journal_t *j) { j->it.xsize = 0; /* We have no transaction data yet... */ j->it.xpos = 0; /* ...and haven't used any of it. */ - return (read_one_rr(j)); + return (read_one_rr(j, false)); failure: return (result); } static isc_result_t -read_one_rr(dns_journal_t *j) { +read_one_rr(dns_journal_t *j, bool retry) { isc_result_t result; - dns_rdatatype_t rdtype; dns_rdataclass_t rdclass; unsigned int rdlen; uint32_t ttl; journal_xhdr_t xhdr; journal_rrhdr_t rrhdr; + dns_journal_t save = *j; if (j->offset > j->it.epos.offset) { isc_log_write(JOURNAL_COMMON_LOGARGS, ISC_LOG_ERROR, @@ -1780,7 +1902,34 @@ read_one_rr(dns_journal_t *j) { j->filename); FAIL(ISC_R_UNEXPECTED); } - if (xhdr.serial0 != j->it.current_serial) { + if (xhdr.serial0 != j->it.current_serial || + isc_serial_le(xhdr.serial1, xhdr.serial0)) + { + if (!retry && j->header_ver1 && + j->xhdr_version == XHDR_VERSION2 && + xhdr.count == j->it.current_serial) + { + /* XHDR_VERSION2 -> XHDR_VERSION1 */ + j->xhdr_version = XHDR_VERSION1; + CHECK(journal_seek(j, save.offset)); + result = read_one_rr(j, true); + if (result == ISC_R_SUCCESS) { + j->recovered = true; + } + return (result); + } else if (!retry && j->header_ver1 && + j->xhdr_version == XHDR_VERSION1 && + xhdr.serial1 == j->it.current_serial) + { + /* XHDR_VERSION1 -> XHDR_VERSION2 */ + j->xhdr_version = XHDR_VERSION2; + CHECK(journal_seek(j, save.offset)); + result = read_one_rr(j, true); + if (result == ISC_R_SUCCESS) { + j->recovered = true; + } + return (result); + } isc_log_write(JOURNAL_COMMON_LOGARGS, ISC_LOG_ERROR, "%s: journal file corrupt: " "expected serial %u, got %u", @@ -1871,7 +2020,7 @@ failure: isc_result_t dns_journal_next_rr(dns_journal_t *j) { - j->it.result = read_one_rr(j); + j->it.result = read_one_rr(j, false); return (j->it.result); } @@ -2200,16 +2349,33 @@ failure: return (result); } +static uint32_t +rrcount(char *buf, unsigned int size) { + isc_buffer_t b; + uint32_t rrsize, count = 0; + + isc_buffer_init(&b, buf, size); + isc_buffer_add(&b, size); + while (isc_buffer_remaininglength(&b) > 0) { + rrsize = isc_buffer_getuint32(&b); + INSIST(isc_buffer_remaininglength(&b) >= rrsize); + isc_buffer_forward(&b, rrsize); + count++; + } + + return (count); +} + isc_result_t dns_journal_compact(isc_mem_t *mctx, char *filename, uint32_t serial, - uint32_t target_size) { + uint32_t flags, uint32_t target_size) { unsigned int i; journal_pos_t best_guess; journal_pos_t current_pos; dns_journal_t *j1 = NULL; dns_journal_t *j2 = NULL; journal_rawheader_t rawheader; - unsigned int copy_length; + unsigned int len; size_t namelen; char *buf = NULL; unsigned int size = 0; @@ -2218,6 +2384,7 @@ dns_journal_compact(isc_mem_t *mctx, char *filename, uint32_t serial, char newname[PATH_MAX]; char backup[PATH_MAX]; bool is_backup = false; + bool rewrite = false; REQUIRE(filename != NULL); @@ -2243,7 +2410,14 @@ dns_journal_compact(isc_mem_t *mctx, char *filename, uint32_t serial, return (result); } - if (JOURNAL_EMPTY(&j1->header)) { + /* + * Check whether we need to rewrite the whole journal + * file (for example, to upversion it). + */ + if ((flags & DNS_JOURNAL_COMPACTALL) != 0) { + rewrite = true; + serial = dns_journal_first_serial(j1); + } else if (JOURNAL_EMPTY(&j1->header)) { dns_journal_destroy(&j1); return (ISC_R_SUCCESS); } @@ -2270,12 +2444,13 @@ dns_journal_compact(isc_mem_t *mctx, char *filename, uint32_t serial, /* * See if there is any work to do. */ - if ((uint32_t)j1->header.end.offset < target_size) { + if (!rewrite && (uint32_t)j1->header.end.offset < target_size) { dns_journal_destroy(&j1); return (ISC_R_SUCCESS); } CHECK(journal_open(mctx, newname, true, true, &j2)); + CHECK(journal_seek(j2, indexend)); /* * Remove overhead so space test below can succeed. @@ -2301,7 +2476,7 @@ dns_journal_compact(isc_mem_t *mctx, char *filename, uint32_t serial, current_pos = best_guess; while (current_pos.serial != serial) { - CHECK(journal_next(j1, ¤t_pos)); + CHECK(journal_next(j1, ¤t_pos, false)); if (current_pos.serial == j1->header.end.serial) { break; } @@ -2319,7 +2494,7 @@ dns_journal_compact(isc_mem_t *mctx, char *filename, uint32_t serial, INSIST(best_guess.serial != j1->header.end.serial); if (best_guess.serial != serial) { - CHECK(journal_next(j1, &best_guess)); + CHECK(journal_next(j1, &best_guess, false)); } /* @@ -2327,40 +2502,98 @@ dns_journal_compact(isc_mem_t *mctx, char *filename, uint32_t serial, * we did not reach 'serial'. If not we will just copy * all uncommitted deltas regardless of the size. */ - copy_length = j1->header.end.offset - best_guess.offset; - - if (copy_length != 0) { - /* - * Copy best_guess to end into space just freed. - */ - size = 64 * 1024; - if (copy_length < size) { - size = copy_length; - } - buf = isc_mem_get(mctx, size); - + len = j1->header.end.offset - best_guess.offset; + if (len != 0) { CHECK(journal_seek(j1, best_guess.offset)); - CHECK(journal_seek(j2, indexend)); - for (i = 0; i < copy_length; i += size) { - unsigned int len = (copy_length - i) > size - ? size - : (copy_length - i); - CHECK(journal_read(j1, buf, len)); - CHECK(journal_write(j2, buf, len)); + + /* Prepare new header */ + j2->header.begin.serial = best_guess.serial; + j2->header.begin.offset = indexend; + j2->header.sourceserial = j1->header.sourceserial; + j2->header.serialset = j1->header.serialset; + j2->header.end.serial = j1->header.end.serial; + + /* + * Only use this method if we're rewriting the + * journal to fix outdated transaction headers; + * otherwise we'll copy the whole journal without + * parsing individual deltas below. + */ + while (rewrite && len > 0) { + journal_xhdr_t xhdr; + isc_offset_t offset = j1->offset; + uint32_t count; + + result = journal_read_xhdr(j1, &xhdr); + if (rewrite && result == ISC_R_NOMORE) { + break; + } + CHECK(result); + + /* + * If we're repairing an outdated journal, the + * xhdr format may be wrong. + */ + if (rewrite && + (xhdr.serial0 != serial || + isc_serial_le(xhdr.serial1, xhdr.serial0))) + { + if (j1->xhdr_version == XHDR_VERSION2 && + xhdr.count == serial) { + /* XHDR_VERSION2 -> XHDR_VERSION1 */ + j1->xhdr_version = XHDR_VERSION1; + CHECK(journal_seek(j1, offset)); + CHECK(journal_read_xhdr(j1, &xhdr)); + } else if (j1->xhdr_version == XHDR_VERSION1 && + xhdr.serial1 == serial) { + /* XHDR_VERSION1 -> XHDR_VERSION2 */ + j1->xhdr_version = XHDR_VERSION2; + CHECK(journal_seek(j1, offset)); + CHECK(journal_read_xhdr(j1, &xhdr)); + } + + /* Check again */ + if (xhdr.serial0 != serial || + isc_serial_le(xhdr.serial1, xhdr.serial0)) { + CHECK(ISC_R_UNEXPECTED); + } + } + + size = xhdr.size; + buf = isc_mem_get(mctx, size); + CHECK(journal_read(j1, buf, size)); + + count = rrcount(buf, size); + CHECK(journal_write_xhdr(j2, xhdr.size, count, + xhdr.serial0, xhdr.serial1)); + CHECK(journal_write(j2, buf, size)); + + j2->header.end.offset = j2->offset; + + serial = xhdr.serial1; + + len = j1->header.end.offset - j1->offset; + isc_mem_put(mctx, buf, size); + } + + /* + * If we're not rewriting transaction headers, we can use + * this faster method instead. + */ + if (!rewrite) { + size = ISC_MIN(64 * 1024, len); + buf = isc_mem_get(mctx, size); + for (i = 0; i < len; i += size) { + unsigned int blob = ISC_MIN(size, len - i); + CHECK(journal_read(j1, buf, blob)); + CHECK(journal_write(j2, buf, blob)); + } + + j2->header.end.offset = indexend + len; } CHECK(journal_fsync(j2)); - /* - * Compute new header. - */ - j2->header.begin.serial = best_guess.serial; - j2->header.begin.offset = indexend; - j2->header.end.serial = j1->header.end.serial; - j2->header.end.offset = indexend + copy_length; - j2->header.sourceserial = j1->header.sourceserial; - j2->header.serialset = j1->header.serialset; - /* * Update the journal header. */ @@ -2375,7 +2608,7 @@ dns_journal_compact(isc_mem_t *mctx, char *filename, uint32_t serial, current_pos = j2->header.begin; while (current_pos.serial != j2->header.end.serial) { index_add(j2, ¤t_pos); - CHECK(journal_next(j2, ¤t_pos)); + CHECK(journal_next(j2, ¤t_pos, false)); } /* diff --git a/lib/dns/zone.c b/lib/dns/zone.c index 5b8b1f1ddc..ae21050106 100644 --- a/lib/dns/zone.c +++ b/lib/dns/zone.c @@ -503,7 +503,10 @@ typedef enum { DNS_ZONEFLG_NEEDSTARTUPNOTIFY = 0x80000000U, /*%< need to send out * notify due to the zone * just being loaded for - * the first time. */ + * the first time. */ + DNS_ZONEFLG_FIXJOURNAL = 0x100000000U, /*%< journal file had + * recoverable error, + * needs rewriting */ DNS_ZONEFLG___MAX = UINT64_MAX, /* trick to make the ENUM 64-bit wide */ } dns_zoneflg_t; @@ -890,6 +893,8 @@ static isc_result_t zone_send_securedb(dns_zone_t *zone, dns_db_t *db); static void setrl(isc_ratelimiter_t *rl, unsigned int *rate, unsigned int value); +static void +zone_journal_compact(dns_zone_t *zone, dns_db_t *db, uint32_t serial); #define ENTER zone_debuglog(zone, me, 1, "enter") @@ -4743,6 +4748,7 @@ zone_postload(dns_zone_t *zone, dns_db_t *db, isc_time_t loadtime, uint32_t serial, oldserial, refresh, retry, expire, minimum; isc_time_t now; bool needdump = false; + bool fixjournal = false; bool hasinclude = DNS_ZONE_FLAG(zone, DNS_ZONEFLG_HASINCLUDE); bool nomaster = false; bool had_db = false; @@ -4844,9 +4850,9 @@ zone_postload(dns_zone_t *zone, dns_db_t *db, isc_time_t loadtime, } result = dns_journal_rollforward(zone->mctx, db, options, zone->journal); - if (result != ISC_R_SUCCESS && result != ISC_R_NOTFOUND && - result != DNS_R_UPTODATE && result != DNS_R_NOJOURNAL && - result != ISC_R_RANGE) + if (result != ISC_R_SUCCESS && result != DNS_R_RECOVERABLE && + result != ISC_R_NOTFOUND && result != DNS_R_UPTODATE && + result != DNS_R_NOJOURNAL && result != ISC_R_RANGE) { dns_zone_logc(zone, DNS_LOGCATEGORY_ZONELOAD, ISC_LOG_ERROR, @@ -4867,6 +4873,12 @@ zone_postload(dns_zone_t *zone, dns_db_t *db, isc_time_t loadtime, dns_result_totext(result)); if (result == ISC_R_SUCCESS) { needdump = true; + } else if (result == DNS_R_RECOVERABLE) { + dns_zone_logc(zone, DNS_LOGCATEGORY_ZONELOAD, + ISC_LOG_ERROR, + "retried using old journal format"); + needdump = true; + fixjournal = true; } } @@ -5192,7 +5204,13 @@ zone_postload(dns_zone_t *zone, dns_db_t *db, isc_time_t loadtime, if (zone->type == dns_zone_key) { zone_needdump(zone, 30); } else { - zone_needdump(zone, DNS_DUMP_DELAY); + if (fixjournal) { + DNS_ZONE_SETFLAG(zone, DNS_ZONEFLG_FIXJOURNAL); + zone_journal_compact(zone, zone->db, 0); + zone_needdump(zone, 0); + } else { + zone_needdump(zone, DNS_DUMP_DELAY); + } } } @@ -11435,6 +11453,7 @@ zone_journal_compact(dns_zone_t *zone, dns_db_t *db, uint32_t serial) { int32_t journalsize; dns_dbversion_t *ver = NULL; uint64_t dbsize; + uint32_t options = 0; INSIST(LOCKED_ZONE(zone)); if (inline_raw(zone)) { @@ -11456,9 +11475,16 @@ zone_journal_compact(dns_zone_t *zone, dns_db_t *db, uint32_t serial) { journalsize = (int32_t)dbsize * 2; } } - zone_debuglog(zone, "zone_journal_compact", 1, "target journal size %d", - journalsize); - result = dns_journal_compact(zone->mctx, zone->journal, serial, + if (DNS_ZONE_FLAG(zone, DNS_ZONEFLG_FIXJOURNAL)) { + options |= DNS_JOURNAL_COMPACTALL; + DNS_ZONE_CLRFLAG(zone, DNS_ZONEFLG_FIXJOURNAL); + zone_debuglog(zone, "zone_journal_compact", 1, + "repair full journal"); + } else { + zone_debuglog(zone, "zone_journal_compact", 1, + "target journal size %d", journalsize); + } + result = dns_journal_compact(zone->mctx, zone->journal, serial, options, journalsize); switch (result) { case ISC_R_SUCCESS: @@ -11486,6 +11512,7 @@ dns_zone_flush(dns_zone_t *zone) { DNS_ZONE_SETFLAG(zone, DNS_ZONEFLG_FLUSH); if (DNS_ZONE_FLAG(zone, DNS_ZONEFLG_NEEDDUMP) && zone->masterfile != NULL) { + DNS_ZONE_SETFLAG(zone, DNS_ZONEFLG_NEEDCOMPACT); result = ISC_R_ALREADYRUNNING; dumping = was_dumping(zone); } else { From fb2d0e2897e10e71627bf895aeb6b8664ee54ad2 Mon Sep 17 00:00:00 2001 From: Mark Andrews Date: Wed, 3 Mar 2021 17:07:15 +1100 Subject: [PATCH 2/5] extend named-journalprint to be able to force the journal version named-journalprint can now upgrade or downgrade a journal file in place; the '-u' option upgrades and the '-d' option downgrades. --- bin/tools/named-journalprint.c | 26 +++++++++++++--- bin/tools/named-journalprint.rst | 19 +++++++++--- doc/man/named-journalprint.1in | 19 +++++++++--- lib/dns/include/dns/journal.h | 1 + lib/dns/journal.c | 52 ++++++++++++++++++++++---------- 5 files changed, 86 insertions(+), 31 deletions(-) diff --git a/bin/tools/named-journalprint.c b/bin/tools/named-journalprint.c index 0fcfa2d0de..37cc0ff205 100644 --- a/bin/tools/named-journalprint.c +++ b/bin/tools/named-journalprint.c @@ -28,7 +28,7 @@ const char *progname = NULL; static void usage(void) { - fprintf(stderr, "Usage: %s [-x] journal\n", progname); + fprintf(stderr, "Usage: %s [-dux] journal\n", progname); exit(1); } @@ -68,10 +68,18 @@ main(int argc, char **argv) { isc_log_t *lctx = NULL; uint32_t flags = 0U; char ch; + bool downgrade = false; + bool upgrade = false; progname = argv[0]; - while ((ch = isc_commandline_parse(argc, argv, "x")) != -1) { + while ((ch = isc_commandline_parse(argc, argv, "dux")) != -1) { switch (ch) { + case 'd': + downgrade = true; + break; + case 'u': + upgrade = true; + break; case 'x': flags |= DNS_JOURNAL_PRINTXHDR; break; @@ -91,9 +99,17 @@ main(int argc, char **argv) { isc_mem_create(&mctx); RUNTIME_CHECK(setup_logging(mctx, stderr, &lctx) == ISC_R_SUCCESS); - result = dns_journal_print(mctx, flags, file, stdout); - if (result == DNS_R_NOJOURNAL) { - fprintf(stderr, "%s\n", dns_result_totext(result)); + if (upgrade) { + flags = DNS_JOURNAL_COMPACTALL; + result = dns_journal_compact(mctx, file, 0, flags, 0); + } else if (downgrade) { + flags = DNS_JOURNAL_COMPACTALL | DNS_JOURNAL_VERSION1; + result = dns_journal_compact(mctx, file, 0, flags, 0); + } else { + result = dns_journal_print(mctx, flags, file, stdout); + if (result == DNS_R_NOJOURNAL) { + fprintf(stderr, "%s\n", dns_result_totext(result)); + } } isc_log_destroy(&lctx); isc_mem_detach(&mctx); diff --git a/bin/tools/named-journalprint.rst b/bin/tools/named-journalprint.rst index 2ce6a05bb4..633bcb628b 100644 --- a/bin/tools/named-journalprint.rst +++ b/bin/tools/named-journalprint.rst @@ -29,13 +29,14 @@ named-journalprint - print zone journal in human-readable form Synopsis ~~~~~~~~ -:program:`named-journalprint` [**-x**] {journal} +:program:`named-journalprint` [**-dux**] {journal} Description ~~~~~~~~~~~ -``named-journalprint`` prints the contents of a zone journal file in a -human-readable form. +``named-journalprint`` scans the contents of a zone journal file, +printing it in a human-readable form, or, optionally, converting it +to a different journal file format. Journal files are automatically created by ``named`` when changes are made to dynamic zones (e.g., by ``nsupdate``). They record each addition @@ -50,8 +51,16 @@ into a human-readable text format. Each line begins with ``add`` or ``del``, to indicate whether the record was added or deleted, and continues with the resource record in master-file format. -The ``-x`` option causes additional information about the transaction -header to be printed before each group of changes. +The ``-x`` option causes additional data about the journal file to be +printed at the beginning of the output and before each group of changes. + +The ``-u`` (upgrade) and ``-d`` (downgrade) options recreate the journal +file with a modified format version. The existing journal file is +replaced. ``-d`` writes out the journal in the format used by +versions of BIND up to 9.16.11; ``-u`` writes it out in the format used +by versions since 9.16.13. (9.16.12 is omitted due to a journal-formatting +bug in that release.) Note that these options *must not* be used while +``named`` is running. See Also ~~~~~~~~ diff --git a/doc/man/named-journalprint.1in b/doc/man/named-journalprint.1in index 84684cb8d6..b18f896547 100644 --- a/doc/man/named-journalprint.1in +++ b/doc/man/named-journalprint.1in @@ -32,11 +32,12 @@ level margin: \\n[rst2man-indent\\n[rst2man-indent-level]] .. .SH SYNOPSIS .sp -\fBnamed\-journalprint\fP [\fB\-x\fP] {journal} +\fBnamed\-journalprint\fP [\fB\-dux\fP] {journal} .SH DESCRIPTION .sp -\fBnamed\-journalprint\fP prints the contents of a zone journal file in a -human\-readable form. +\fBnamed\-journalprint\fP scans the contents of a zone journal file, +printing it in a human\-readable form, or, optionally, converting it +to a different journal file format. .sp Journal files are automatically created by \fBnamed\fP when changes are made to dynamic zones (e.g., by \fBnsupdate\fP). They record each addition @@ -51,8 +52,16 @@ into a human\-readable text format. Each line begins with \fBadd\fP or \fBdel\fP to indicate whether the record was added or deleted, and continues with the resource record in master\-file format. .sp -The \fB\-x\fP option causes additional information about the transaction -header to be printed before each group of changes. +The \fB\-x\fP option causes additional data about the journal file to be +printed at the beginning of the output and before each group of changes. +.sp +The \fB\-u\fP (upgrade) and \fB\-d\fP (downgrade) options recreate the journal +file with a modified format version. The existing journal file is +replaced. \fB\-d\fP writes out the journal in the format used by +versions of BIND up to 9.16.11; \fB\-u\fP writes it out in the format used +by versions since 9.16.13. (9.16.12 is omitted due to a journal\-formatting +bug in that release.) Note that these options \fImust not\fP be used while +\fBnamed\fP is running. .SH SEE ALSO .sp \fBnamed(8)\fP, \fBnsupdate(1)\fP, BIND 9 Administrator Reference Manual. diff --git a/lib/dns/include/dns/journal.h b/lib/dns/include/dns/journal.h index 4d3b73efb5..e115bbc226 100644 --- a/lib/dns/include/dns/journal.h +++ b/lib/dns/include/dns/journal.h @@ -53,6 +53,7 @@ /*% Rewrite whole journal file instead of compacting */ #define DNS_JOURNAL_COMPACTALL 0x0001 +#define DNS_JOURNAL_VERSION1 0x0002 /*** *** Types diff --git a/lib/dns/journal.c b/lib/dns/journal.c index 5007f0c9ef..32756735f1 100644 --- a/lib/dns/journal.c +++ b/lib/dns/journal.c @@ -520,13 +520,20 @@ journal_read_xhdr(dns_journal_t *j, journal_xhdr_t *xhdr) { static isc_result_t journal_write_xhdr(dns_journal_t *j, uint32_t size, uint32_t count, uint32_t serial0, uint32_t serial1) { - journal_rawxhdr_t raw; - - encode_uint32(size, raw.size); - encode_uint32(count, raw.count); - encode_uint32(serial0, raw.serial0); - encode_uint32(serial1, raw.serial1); - return (journal_write(j, &raw, sizeof(raw))); + if (j->header_ver1) { + journal_rawxhdr_ver1_t raw; + encode_uint32(size, raw.size); + encode_uint32(serial0, raw.serial0); + encode_uint32(serial1, raw.serial1); + return (journal_write(j, &raw, sizeof(raw))); + } else { + journal_rawxhdr_t raw; + encode_uint32(size, raw.size); + encode_uint32(count, raw.count); + encode_uint32(serial0, raw.serial0); + encode_uint32(serial1, raw.serial1); + return (journal_write(j, &raw, sizeof(raw))); + } } /* @@ -547,7 +554,7 @@ journal_read_rrhdr(dns_journal_t *j, journal_rrhdr_t *rrhdr) { } static isc_result_t -journal_file_create(isc_mem_t *mctx, const char *filename) { +journal_file_create(isc_mem_t *mctx, bool downgrade, const char *filename) { FILE *fp = NULL; isc_result_t result; journal_header_t header; @@ -566,7 +573,11 @@ journal_file_create(isc_mem_t *mctx, const char *filename) { return (ISC_R_UNEXPECTED); } - header = initial_journal_header; + if (downgrade) { + header = journal_header_ver1; + } else { + header = initial_journal_header; + } header.index_size = index_size; journal_header_encode(&header, &rawheader); @@ -603,7 +614,7 @@ journal_file_create(isc_mem_t *mctx, const char *filename) { static isc_result_t journal_open(isc_mem_t *mctx, const char *filename, bool writable, bool create, - dns_journal_t **journalp) { + bool downgrade, dns_journal_t **journalp) { FILE *fp = NULL; isc_result_t result; journal_rawheader_t rawheader; @@ -624,7 +635,7 @@ journal_open(isc_mem_t *mctx, const char *filename, bool writable, bool create, "journal file %s does not exist, " "creating it", j->filename); - CHECK(journal_file_create(mctx, filename)); + CHECK(journal_file_create(mctx, downgrade, filename)); /* * Retry. */ @@ -758,7 +769,8 @@ dns_journal_open(isc_mem_t *mctx, const char *filename, unsigned int mode, create = ((mode & DNS_JOURNAL_CREATE) != 0); writable = ((mode & (DNS_JOURNAL_WRITE | DNS_JOURNAL_CREATE)) != 0); - result = journal_open(mctx, filename, writable, create, journalp); + result = journal_open(mctx, filename, writable, create, false, + journalp); if (result == ISC_R_NOTFOUND) { namelen = strlen(filename); if (namelen > 4U && strcmp(filename + namelen - 4, ".jnl") == 0) @@ -771,7 +783,7 @@ dns_journal_open(isc_mem_t *mctx, const char *filename, unsigned int mode, if (result >= sizeof(backup)) { return (ISC_R_NOSPACE); } - result = journal_open(mctx, backup, writable, writable, + result = journal_open(mctx, backup, writable, writable, false, journalp); } return (result); @@ -1620,6 +1632,10 @@ dns_journal_print(isc_mem_t *mctx, uint32_t flags, const char *filename, return (result); } + if (printxhdr) { + fprintf(file, "Journal format = %sHeader version = %d\n", + j->header.format + 1, j->header_ver1 ? 1 : 2); + } if (j->header.serialset) { fprintf(file, "Source serial = %u\n", j->header.sourceserial); } @@ -2385,6 +2401,7 @@ dns_journal_compact(isc_mem_t *mctx, char *filename, uint32_t serial, char backup[PATH_MAX]; bool is_backup = false; bool rewrite = false; + bool downgrade = false; REQUIRE(filename != NULL); @@ -2401,10 +2418,10 @@ dns_journal_compact(isc_mem_t *mctx, char *filename, uint32_t serial, filename); RUNTIME_CHECK(result < sizeof(backup)); - result = journal_open(mctx, filename, false, false, &j1); + result = journal_open(mctx, filename, false, false, false, &j1); if (result == ISC_R_NOTFOUND) { is_backup = true; - result = journal_open(mctx, backup, false, false, &j1); + result = journal_open(mctx, backup, false, false, false, &j1); } if (result != ISC_R_SUCCESS) { return (result); @@ -2415,6 +2432,9 @@ dns_journal_compact(isc_mem_t *mctx, char *filename, uint32_t serial, * file (for example, to upversion it). */ if ((flags & DNS_JOURNAL_COMPACTALL) != 0) { + if ((flags & DNS_JOURNAL_VERSION1) != 0) { + downgrade = true; + } rewrite = true; serial = dns_journal_first_serial(j1); } else if (JOURNAL_EMPTY(&j1->header)) { @@ -2449,7 +2469,7 @@ dns_journal_compact(isc_mem_t *mctx, char *filename, uint32_t serial, return (ISC_R_SUCCESS); } - CHECK(journal_open(mctx, newname, true, true, &j2)); + CHECK(journal_open(mctx, newname, true, true, downgrade, &j2)); CHECK(journal_seek(j2, indexend)); /* From a4972324a6846f4c338e5c7ea88ee88eea95697e Mon Sep 17 00:00:00 2001 From: Evan Hunt Date: Wed, 3 Mar 2021 14:59:30 -0800 Subject: [PATCH 3/5] print journal index data and test for consistency 'named-journalprint -x' now prints the journal's index table and the offset of each transaction in the journal, so that index consistency can be confirmed. --- lib/dns/journal.c | 44 ++++++++++++++++++++++++++++++++++---------- 1 file changed, 34 insertions(+), 10 deletions(-) diff --git a/lib/dns/journal.c b/lib/dns/journal.c index 32756735f1..fad38c0f57 100644 --- a/lib/dns/journal.c +++ b/lib/dns/journal.c @@ -344,6 +344,7 @@ struct dns_journal { struct { /* These define the part of the journal we iterate over. */ journal_pos_t bpos; /*%< Position before first, */ + journal_pos_t cpos; /*%< before current, */ journal_pos_t epos; /*%< and after last transaction */ /* The rest is iterator state. */ uint32_t current_serial; /*%< Current SOA serial */ @@ -483,6 +484,8 @@ static isc_result_t journal_read_xhdr(dns_journal_t *j, journal_xhdr_t *xhdr) { isc_result_t result; + j->it.cpos.offset = j->offset; + switch (j->xhdr_version) { case XHDR_VERSION1: { journal_rawxhdr_ver1_t raw; @@ -1635,6 +1638,15 @@ dns_journal_print(isc_mem_t *mctx, uint32_t flags, const char *filename, if (printxhdr) { fprintf(file, "Journal format = %sHeader version = %d\n", j->header.format + 1, j->header_ver1 ? 1 : 2); + fprintf(file, "Index (size = %u):\n", j->header.index_size); + for (uint32_t i = 0; i < j->header.index_size; i++) { + if (j->index[i].offset == 0) { + fputc('\n', file); + break; + } + fprintf(file, "%lld", (long long)j->index[i].offset); + fputc((i + 1) % 8 == 0 ? '\n' : ' ', file); + } } if (j->header.serialset) { fprintf(file, "Source serial = %u\n", j->header.sourceserial); @@ -1660,16 +1672,20 @@ dns_journal_print(isc_mem_t *mctx, uint32_t flags, const char *filename, dns_name_t *name = NULL; dns_rdata_t *rdata = NULL; dns_difftuple_t *tuple = NULL; + static uint32_t i = 0; + bool print = false; uint32_t ttl; dns_journal_current_rr(j, &name, &ttl, &rdata); if (rdata->type == dns_rdatatype_soa) { n_soa++; - } - - if (n_soa == 3) { - n_soa = 1; + if (n_soa == 3) { + n_soa = 1; + } + if (n_soa == 1) { + print = printxhdr; + } } if (n_soa == 0) { isc_log_write(JOURNAL_COMMON_LOGARGS, ISC_LOG_ERROR, @@ -1679,13 +1695,21 @@ dns_journal_print(isc_mem_t *mctx, uint32_t flags, const char *filename, FAIL(ISC_R_UNEXPECTED); } - if (printxhdr && n_soa == 1) { + if (print) { fprintf(file, - "Transaction: version %d size %u rrcount %u " - "startserial %u endserial %u\n", - j->xhdr_version, j->curxhdr.size, - j->curxhdr.count, j->curxhdr.serial0, - j->curxhdr.serial1); + "Transaction: version %d offset %lld size %u " + "rrcount %u start %u end %u\n", + j->xhdr_version, (long long)j->it.cpos.offset, + j->curxhdr.size, j->curxhdr.count, + j->curxhdr.serial0, j->curxhdr.serial1); + if (j->it.cpos.offset > j->index[i].offset) { + fprintf(file, + "ERROR: Offset mismatch, " + "expected %lld\n", + (long long)j->index[i].offset); + } else if (j->it.cpos.offset == j->index[i].offset) { + i++; + } } CHECK(dns_difftuple_create( diff.mctx, n_soa == 1 ? DNS_DIFFOP_DEL : DNS_DIFFOP_ADD, From a0aefa1de6459000ddfd1876be886e3b12cf7692 Mon Sep 17 00:00:00 2001 From: Evan Hunt Date: Mon, 22 Feb 2021 15:17:54 -0800 Subject: [PATCH 4/5] create 'journal' system test tests that version 1 journal files containing version 1 transaction headers are rolled forward correctly on server startup, then updated into version 2 journals. also checks journal file consistency and 'max-journal-size' behavior. --- bin/tests/system/Makefile.am | 1 + bin/tests/system/journal/clean.sh | 14 ++ .../system/journal/ns1/changed.ver1.jnl.saved | Bin 0 -> 707 bytes .../system/journal/ns1/changed.ver2.jnl.saved | Bin 0 -> 718 bytes bin/tests/system/journal/ns1/d1212.jnl.saved | Bin 0 -> 1478 bytes bin/tests/system/journal/ns1/d2121.jnl.saved | Bin 0 -> 1478 bytes bin/tests/system/journal/ns1/generic.db.in | 16 ++ bin/tests/system/journal/ns1/ixfr.db.in | 17 ++ .../system/journal/ns1/ixfr.ver1.jnl.saved | Bin 0 -> 686 bytes .../system/journal/ns1/maxjournal.jnl.saved | Bin 0 -> 13660 bytes .../system/journal/ns1/maxjournal2.jnl.saved | Bin 0 -> 14259 bytes bin/tests/system/journal/ns1/named.conf.in | 90 ++++++++ .../journal/ns1/unchanged.ver1.jnl.saved | Bin 0 -> 512 bytes .../journal/ns1/unchanged.ver2.jnl.saved | Bin 0 -> 512 bytes bin/tests/system/journal/setup.sh | 42 ++++ bin/tests/system/journal/tests.sh | 201 ++++++++++++++++++ util/copyrights | 12 ++ 17 files changed, 393 insertions(+) create mode 100644 bin/tests/system/journal/clean.sh create mode 100644 bin/tests/system/journal/ns1/changed.ver1.jnl.saved create mode 100644 bin/tests/system/journal/ns1/changed.ver2.jnl.saved create mode 100644 bin/tests/system/journal/ns1/d1212.jnl.saved create mode 100644 bin/tests/system/journal/ns1/d2121.jnl.saved create mode 100644 bin/tests/system/journal/ns1/generic.db.in create mode 100644 bin/tests/system/journal/ns1/ixfr.db.in create mode 100644 bin/tests/system/journal/ns1/ixfr.ver1.jnl.saved create mode 100644 bin/tests/system/journal/ns1/maxjournal.jnl.saved create mode 100644 bin/tests/system/journal/ns1/maxjournal2.jnl.saved create mode 100644 bin/tests/system/journal/ns1/named.conf.in create mode 100644 bin/tests/system/journal/ns1/unchanged.ver1.jnl.saved create mode 100644 bin/tests/system/journal/ns1/unchanged.ver2.jnl.saved create mode 100644 bin/tests/system/journal/setup.sh create mode 100644 bin/tests/system/journal/tests.sh diff --git a/bin/tests/system/Makefile.am b/bin/tests/system/Makefile.am index 92980fa441..feb7598241 100644 --- a/bin/tests/system/Makefile.am +++ b/bin/tests/system/Makefile.am @@ -111,6 +111,7 @@ TESTS += \ inline \ integrity \ hooks \ + journal \ keepalive \ legacy \ limits \ diff --git a/bin/tests/system/journal/clean.sh b/bin/tests/system/journal/clean.sh new file mode 100644 index 0000000000..1b8649595a --- /dev/null +++ b/bin/tests/system/journal/clean.sh @@ -0,0 +1,14 @@ +# Copyright (C) Internet Systems Consortium, Inc. ("ISC") +# +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, you can obtain one at https://mozilla.org/MPL/2.0/. +# +# See the COPYRIGHT file distributed with this work for additional +# information regarding copyright ownership. + +rm -f */named.memstats +rm -f */named.run +rm -f */named.conf +rm -f */*.db */*.jnl +rm -f dig.out* diff --git a/bin/tests/system/journal/ns1/changed.ver1.jnl.saved b/bin/tests/system/journal/ns1/changed.ver1.jnl.saved new file mode 100644 index 0000000000000000000000000000000000000000..b449a7dbccfae7280bd90bdcd0813866c46f1fae GIT binary patch literal 707 zcmcC^^7M02@bPz72(#p3U|=YJb8{*K0~3gz2BZ%Ig)D#=CxEHPsbExMu!q2QP#i+y zo57tuIU_MIJvD`afsKI?m;fReOqlYD;i6m_`Nbu@#9$ABeLx2? zgW?eq=L}vP$r*`x>8UA33=C`xjKEY7!C=OeSBwzm%E&J+$xSRSNi9MMLQ(>s00RTd m1_n;61~AP)foZ_>hhdr=Yhp@DkrBdHp!*;$WZ^PpU;+TC+d*mo literal 0 HcmV?d00001 diff --git a/bin/tests/system/journal/ns1/d1212.jnl.saved b/bin/tests/system/journal/ns1/d1212.jnl.saved new file mode 100644 index 0000000000000000000000000000000000000000..57c149722ebad409a260ccd53d8abd0319461e85 GIT binary patch literal 1478 zcmcC^^7M02@bPz72(#p3U|=YJb8{*K0~3gz$-uyR3@Bs)#5e(x`e{IY?}7a33=GVV zfbN=ojw!DgRf;Pk zzqlkfvA86)2vq`P8Uq8L05JA9FmPHmfN5%(40I!g$=ay)GYEj(3Q^44|KZr5Kg`jLl`#@iWkk6#3Z*0K()DN&o-= literal 0 HcmV?d00001 diff --git a/bin/tests/system/journal/ns1/d2121.jnl.saved b/bin/tests/system/journal/ns1/d2121.jnl.saved new file mode 100644 index 0000000000000000000000000000000000000000..ec21372f445e86cbf37dac465ca5c5464cae1099 GIT binary patch literal 1478 zcmcC^^7M02@bPz72(#p3U|=YJb8{*K0~3gz$-uyR3@Bs)#5e(x`e{IYpMm`83=GVV zfb6tIDT)2abXQ_Eza8!=4QMzxC(NtH_8#pWpL1SHUnlz7(==*1^M42t3DKn%>4RPr)5 whf&ANKsRD|SqDc9i?9W!CgSIQUCw| literal 0 HcmV?d00001 diff --git a/bin/tests/system/journal/ns1/generic.db.in b/bin/tests/system/journal/ns1/generic.db.in new file mode 100644 index 0000000000..bb75d3ce82 --- /dev/null +++ b/bin/tests/system/journal/ns1/generic.db.in @@ -0,0 +1,16 @@ +; Copyright (C) Internet Systems Consortium, Inc. ("ISC") +; +; This Source Code Form is subject to the terms of the Mozilla Public +; License, v. 2.0. If a copy of the MPL was not distributed with this +; file, You can obtain one at http://mozilla.org/MPL/2.0/. +; +; See the COPYRIGHT file distributed with this work for additional +; information regarding copyright ownership. + + +$TTL 600 +@ SOA ns hostmaster 2012010901 3600 1200 604800 1200 + NS ns +ns A 192.0.2.1 + +addr1 A 10.53.0.1 diff --git a/bin/tests/system/journal/ns1/ixfr.db.in b/bin/tests/system/journal/ns1/ixfr.db.in new file mode 100644 index 0000000000..cee52ca6c5 --- /dev/null +++ b/bin/tests/system/journal/ns1/ixfr.db.in @@ -0,0 +1,17 @@ +; Copyright (C) Internet Systems Consortium, Inc. ("ISC") +; +; This Source Code Form is subject to the terms of the Mozilla Public +; License, v. 2.0. If a copy of the MPL was not distributed with this +; file, You can obtain one at http://mozilla.org/MPL/2.0/. +; +; See the COPYRIGHT file distributed with this work for additional +; information regarding copyright ownership. + + +$TTL 600 +@ SOA ns hostmaster 2012010902 3600 1200 604800 1200 + NS ns +ns A 192.0.2.1 + +addr1 A 10.53.0.1 +addr2 A 10.53.0.2 diff --git a/bin/tests/system/journal/ns1/ixfr.ver1.jnl.saved b/bin/tests/system/journal/ns1/ixfr.ver1.jnl.saved new file mode 100644 index 0000000000000000000000000000000000000000..10b5116ef2bfcdc182ce801d144738e6dd157f9d GIT binary patch literal 686 zcmcC^^7M02@bPz72(#p3U|=YJb8{*K0~3gz2Bg;kg)D#=CxEHPsbExMu!q1RP#i+y zo57AHvm&jCfq{*I5g7Xs40=p?#SkH`jQrw~+{EIN)FOxgBU`sUfD2m=+>UkO&qc7B)2`BBp4Ch=qlPg>BWWC~dW< zd8SK$fCY(#g@r{I7XARg`>yIyES|pc=52Cw()+dNo^zhl-h03AxvSQ0*i>J;ar3&` zZ7YhBj`e+NmdZ-lD=pi>UCX)s@vGm@pX2vF@O_I^J_0;14P6ahkcJ%wFG|DPz)RAI zkKkpgSOH#M8J^H0?2XU#k5E_DIufzz5Qd zE#O0G=4tSeH0wF|Seo4jK9TBb!Kc!kt>80h?iuj8H19R|LYn^_d?_uM3BHmRZU|{8)r@7ME^%dCrS(RRbH%`N_K_1 zk{@n#(d}uV$M@!ht^qtX4D6{;Hl!RMHHDr3IKJy2$9Mnjc*v}U%=o)?XO^Hpw1vJ-b9^ zyw1qcpW73;*qMc%T_!VLlf~e7bE3;?PNHX5$c)#oIr?*ZA{RTe(6g&##%ui;{BBNk zS>8Q14nk%L`g6~WT|+bu#1qc8>nsp2)?{EcEOK znQ?9+2EUsVT~>1vJ-bO}oX5%0pW73;*qMc%wUHU;ykhXXIniY`C(*N8WXAcy9R0aH zk&B&K=-F*D<6LSCem5t&tmY(o)=p-ech1qD+Y`CinT4KpkQwLbWAM8<(PcF!(X&o6 z<9bGp{@k9(#m+4B><*c6?I#Amn-g7Da}qu4A~UX&<>=4tiCpZ=LeEOqB)O&-gWt`G zE~`0-p4}xguD|B!&+Um^?94*X?vWYSs$=lGIniY`C(*O}WX5&<9R0aHk&B&K=virZ zf*pt${BBNkS;ajvFOs7_w@LUPcXOi4 zYEGhOugHu&?;QQPJ&}u@S?F15XP%w-82oNdbXm fvlcP<-JIyMnv>|+J2LAXgv=83=bjn4{5rFr`Xv;4 literal 0 HcmV?d00001 diff --git a/bin/tests/system/journal/ns1/maxjournal2.jnl.saved b/bin/tests/system/journal/ns1/maxjournal2.jnl.saved new file mode 100644 index 0000000000000000000000000000000000000000..e200905955a533116a92cdafedbb607b61945b3f GIT binary patch literal 14259 zcmds;No3_lrBPytS=1aQYO0uRh(shpL?SG&l9+`O!bW0gDdwRy&(u7$hK;3I zSy+gb*bqxgp5Nj`Kdo-QlboC;H*eef_xs-W{odaHzvn-9!Qy2rmW*D$a_Q(b^QP8U zYu~qz$E1n^-$-*FN}9`~)33jdKhEEM=I0YqbCH1%ho|dY(a2p;0?*KD8g~@DFOB~I zK9D910r8}E9r#F^cm{kdP5J~rktUAB_68K!2`UQL;O|J!8r5T&SHmUv= z_)?nr9egFtng+g>X7B3MaQ`RXQ?eUwi^Dlyo#T3E58 zvE4J(n|3zt*|Aa6Ay zwecuhyb-cj(J7san{cq%$H7FUdQ1_A>=Z)=n{kl0w~^X-l!cvTpVaH%1sr5yAY-W> z(s0O5F=X%}4zi>Ysf|Zj*je^TWw2P}VKFLWsUFgB$WAe2@G=gvj2EemM_Jfe_DN;1 zSYTxVG-IhA(s0O5F=X&64zhF{sf|Zj*je^TW$+pfvgn?%R1ax5WTzN1cpV41T@k5` zM_Jfe_DN;%1`cwgCu6A|(s0O5F=X&24sweuQX7x5u(Rxw%3yI*kDH4bOZAY3Lw1TG zgST;z+p>|`c$9^mWuH_AiyP|P(9T$@hcq0rQw$lri-T+hh}6cTEbJ`%q%wF92iZiC zu~ZLfIAo_7GT4HHY(I(A#-l9kEc>J~cpnGZxRbF|4{11Lrx-H$00)b0U=ABxI%wlj z7Iv0>QW<=RgKQSdSgMCK9I{gk8GM9;Y_p5h#-l9kEc>J~SZttVgJQ-~J*44~onpw~ z6C7k~XQVbBWnpL8CzZjcILM~hjHP-=!y!Axkilm-$admLZ9K}t&azJ`gU@l0jo2AW z^^k@`c8Vc`FL018?2+1dl!cvTpHv20agfdb8B6t$hC_CWA%kr=$j2rkwectmJIg+) z3>F`l;R7cbOZAY3Lw1TGgRgLqkLpBf<53oNmVHtge2s%|d>l+vs>c*@sQnaw1D`8! ASpWb4 literal 0 HcmV?d00001 diff --git a/bin/tests/system/journal/ns1/named.conf.in b/bin/tests/system/journal/ns1/named.conf.in new file mode 100644 index 0000000000..3fdd96b0ed --- /dev/null +++ b/bin/tests/system/journal/ns1/named.conf.in @@ -0,0 +1,90 @@ +/* + * Copyright (C) Internet Systems Consortium, Inc. ("ISC") + * + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. + * + * See the COPYRIGHT file distributed with this work for additional + * information regarding copyright ownership. + */ + +options { + query-source address 10.53.0.1; + notify-source 10.53.0.1; + transfer-source 10.53.0.1; + port @PORT@; + session-keyfile "session.key"; + pid-file "named.pid"; + listen-on { 10.53.0.1; }; + listen-on-v6 { none; }; + dnssec-validation yes; + minimal-responses no; + recursion no; + notify yes; +}; + +key rndc_key { + secret "1234abcd8765"; + algorithm hmac-sha256; +}; + +controls { + inet 10.53.0.1 port @CONTROLPORT@ allow { any; } keys { rndc_key; }; +}; + +zone changed { + type primary; + update-policy local; + file "changed.db"; +}; + +zone unchanged { + type primary; + update-policy local; + file "unchanged.db"; +}; + +zone changed2 { + type primary; + update-policy local; + file "changed2.db"; +}; + +zone unchanged2 { + type primary; + update-policy local; + file "unchanged2.db"; +}; + +zone hdr1d1d2d1d2 { + type primary; + update-policy local; + file "d1212.db"; +}; + +zone hdr1d2d1d2d1 { + type primary; + update-policy local; + file "d2121.db"; +}; + +zone ixfr { + type primary; + ixfr-from-differences yes; + file "ixfr.db"; +}; + +zone maxjournal { + type primary; + max-journal-size 1k; + update-policy local; + file "maxjournal.db"; +}; + +zone maxjournal2 { + type primary; + max-journal-size 1k; + update-policy local; + file "maxjournal2.db"; +}; diff --git a/bin/tests/system/journal/ns1/unchanged.ver1.jnl.saved b/bin/tests/system/journal/ns1/unchanged.ver1.jnl.saved new file mode 100644 index 0000000000000000000000000000000000000000..f7885d93dd7abfd1b09078e3cba0d724c1472acd GIT binary patch literal 512 fcmcC^^7M02@bPz72(#p3zzQs|ijE2nix2<+f3yQk literal 0 HcmV?d00001 diff --git a/bin/tests/system/journal/ns1/unchanged.ver2.jnl.saved b/bin/tests/system/journal/ns1/unchanged.ver2.jnl.saved new file mode 100644 index 0000000000000000000000000000000000000000..d974be4df70fdbea4ba8ed85ced6fb814b1dbc80 GIT binary patch literal 512 hcmcC^^7M02@bPz72(#2P;$px8EU<`;3JsGG0027u1FQf5 literal 0 HcmV?d00001 diff --git a/bin/tests/system/journal/setup.sh b/bin/tests/system/journal/setup.sh new file mode 100644 index 0000000000..23c963920f --- /dev/null +++ b/bin/tests/system/journal/setup.sh @@ -0,0 +1,42 @@ +#!/bin/sh +# +# Copyright (C) Internet Systems Consortium, Inc. ("ISC") +# +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, you can obtain one at https://mozilla.org/MPL/2.0/. +# +# See the COPYRIGHT file distributed with this work for additional +# information regarding copyright ownership. + +. ../conf.sh + +$SHELL clean.sh + +copy_setports ns1/named.conf.in ns1/named.conf +cp ns1/generic.db.in ns1/changed.db +cp ns1/changed.ver1.jnl.saved ns1/changed.db.jnl + +cp ns1/generic.db.in ns1/unchanged.db +cp ns1/unchanged.ver1.jnl.saved ns1/unchanged.db.jnl + +cp ns1/generic.db.in ns1/changed2.db +cp ns1/changed.ver2.jnl.saved ns1/changed2.db.jnl + +cp ns1/generic.db.in ns1/unchanged2.db +cp ns1/unchanged.ver2.jnl.saved ns1/unchanged2.db.jnl + +cp ns1/ixfr.db.in ns1/ixfr.db +cp ns1/ixfr.ver1.jnl.saved ns1/ixfr.db.jnl + +cp ns1/generic.db.in ns1/d1212.db +cp ns1/d1212.jnl.saved ns1/d1212.db.jnl + +cp ns1/generic.db.in ns1/d2121.db +cp ns1/d2121.jnl.saved ns1/d2121.db.jnl + +cp ns1/generic.db.in ns1/maxjournal.db +cp ns1/maxjournal.jnl.saved ns1/maxjournal.db.jnl + +cp ns1/generic.db.in ns1/maxjournal2.db +cp ns1/maxjournal2.jnl.saved ns1/maxjournal2.db.jnl diff --git a/bin/tests/system/journal/tests.sh b/bin/tests/system/journal/tests.sh new file mode 100644 index 0000000000..7fd12dc14c --- /dev/null +++ b/bin/tests/system/journal/tests.sh @@ -0,0 +1,201 @@ +#!/bin/sh +# +# Copyright (C) Internet Systems Consortium, Inc. ("ISC") +# +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, you can obtain one at https://mozilla.org/MPL/2.0/. +# +# See the COPYRIGHT file distributed with this work for additional +# information regarding copyright ownership. + +. ../conf.sh + +dig_with_opts() { + "$DIG" @10.53.0.1 -p "$PORT" +tcp "$@" +} + +rndc_with_opts() { + "$RNDC" -c ../common/rndc.conf -p "$CONTROLPORT" -s "$@" +} + +status=0 +n=0 + +n=`expr $n + 1` +echo_i "check outdated journal rolled forward (dynamic) ($n)" +ret=0 +dig_with_opts changed soa > dig.out.test$n +grep 'status: NOERROR' dig.out.test$n > /dev/null || ret=1 +grep '2012010902' dig.out.test$n > /dev/null || ret=1 +grep 'zone changed/IN: retried using old journal format' ns1/named.run > /dev/null || ret=1 +[ $ret -eq 0 ] || echo_i "failed" +status=`expr $status + $ret` + +n=`expr $n + 1` +echo_i "check outdated empty journal did not cause an error (dynamic) ($n)" +ret=0 +dig_with_opts unchanged soa > dig.out.test$n +grep 'status: NOERROR' dig.out.test$n > /dev/null || ret=1 +grep '2012010901' dig.out.test$n > /dev/null || ret=1 +[ $ret -eq 0 ] || echo_i "failed" +status=`expr $status + $ret` + +n=`expr $n + 1` +echo_i "check outdated journals were updated or removed (dynamic) ($n)" +ret=0 +cat -v ns1/changed.db.jnl | grep "BIND LOG V9.2" > /dev/null || ret=1 +[ -f ns1/unchanged.db.jnl ] && ret=1 +[ $ret -eq 0 ] || echo_i "failed" +status=`expr $status + $ret` + +n=`expr $n + 1` +echo_i "check updated journal has correct RR count (dynamic) ($n)" +ret=0 +$JOURNALPRINT -x ns1/changed.db.jnl | grep "rrcount 3 " > /dev/null || ret=1 +[ $ret -eq 0 ] || echo_i "failed" +status=`expr $status + $ret` + +n=`expr $n + 1` +echo_i "check new-format journal rolled forward (dynamic) ($n)" +ret=0 +dig_with_opts changed2 soa > dig.out.test$n +grep 'status: NOERROR' dig.out.test$n > /dev/null || ret=1 +grep '2012010902' dig.out.test$n > /dev/null || ret=1 +grep 'zone changed2/IN: retried using old journal format' ns1/named.run > /dev/null && ret=1 +[ $ret -eq 0 ] || echo_i "failed" +status=`expr $status + $ret` + +n=`expr $n + 1` +echo_i "check new-format empty journal did not cause error (dynamic) ($n)" +ret=0 +dig_with_opts unchanged2 soa > dig.out.test$n +grep 'status: NOERROR' dig.out.test$n > /dev/null || ret=1 +grep '2012010901' dig.out.test$n > /dev/null || ret=1 +grep 'zone unchanged2/IN: retried using old journal format' ns1/named.run > /dev/null && ret=1 +[ $ret -eq 0 ] || echo_i "failed" +status=`expr $status + $ret` + +n=`expr $n + 1` +echo_i "check new-format journals were updated or removed (dynamic) ($n)" +ret=0 +cat -v ns1/changed2.db.jnl | grep "BIND LOG V9.2" > /dev/null || ret=1 +[ -f ns1/unchanged2.db.jnl ] && ret=1 +[ $ret -eq 0 ] || echo_i "failed" +status=`expr $status + $ret` + +n=`expr $n + 1` +echo_i "check outdated up-to-date journal succeeded (ixfr-from-differences) ($n)" +ret=0 +dig_with_opts -t soa ixfr > dig.out.test$n +grep 'status: NOERROR' dig.out.test$n > /dev/null || ret=1 +grep '2012010902' dig.out.test$n > /dev/null || ret=1 +grep 'zone ixfr/IN: journal rollforward completed successfully: recoverable' ns1/named.run > /dev/null || ret=1 +[ $ret -eq 0 ] || echo_i "failed" +status=`expr $status + $ret` + +n=`expr $n + 1` +echo_i "check outdated journal was updated (ixfr-from-differences) ($n)" +ret=0 +cat -v ns1/ixfr.db.jnl | grep "BIND LOG V9.2" > /dev/null || ret=1 +[ $ret -eq 0 ] || echo_i "failed" +status=`expr $status + $ret` + +n=`expr $n + 1` +echo_i "check journal with mixed headers succeeded (version 1,2,1,2) ($n)" +ret=0 +dig_with_opts -t soa hdr1d1d2d1d2 > dig.out.test$n +grep 'status: NOERROR' dig.out.test$n > /dev/null || ret=1 +grep '2012010905' dig.out.test$n > /dev/null || ret=1 +grep 'zone hdr1d1d2d1d2/IN: journal rollforward completed successfully: recoverable' ns1/named.run > /dev/null || ret=1 +[ $ret -eq 0 ] || echo_i "failed" +status=`expr $status + $ret` + +n=`expr $n + 1` +echo_i "check journal with mixed headers was updated (version 1,2,1,2) ($n)" +ret=0 +[ $($JOURNALPRINT -x ns1/d1212.jnl.saved | grep -c "version 1") -eq 2 ] || ret=1 +[ $($JOURNALPRINT -x ns1/d1212.jnl.saved | grep -c "version 2") -eq 2 ] || ret=1 +[ $($JOURNALPRINT -x ns1/d1212.db.jnl | grep -c "version 1") -eq 0 ] || ret=1 +[ $($JOURNALPRINT -x ns1/d1212.db.jnl | grep -c "version 2") -eq 4 ] || ret=1 +[ $ret -eq 0 ] || echo_i "failed" +status=`expr $status + $ret` + +n=`expr $n + 1` +echo_i "check journal with mixed headers succeeded (version 2,1,2,1) ($n)" +ret=0 +dig_with_opts -t soa hdr1d2d1d2d1 > dig.out.test$n +grep 'status: NOERROR' dig.out.test$n > /dev/null || ret=1 +grep '2012010905' dig.out.test$n > /dev/null || ret=1 +grep 'zone hdr1d2d1d2d1/IN: journal rollforward completed successfully: recoverable' ns1/named.run > /dev/null || ret=1 +[ $ret -eq 0 ] || echo_i "failed" +status=`expr $status + $ret` + +n=`expr $n + 1` +echo_i "check journal with mixed headers was updated (version 2,1,2,1) ($n)" +ret=0 +[ $($JOURNALPRINT -x ns1/d2121.jnl.saved | grep -c "version 1") -eq 2 ] || ret=1 +[ $($JOURNALPRINT -x ns1/d2121.jnl.saved | grep -c "version 2") -eq 2 ] || ret=1 +[ $($JOURNALPRINT -x ns1/d2121.db.jnl | grep -c "version 1") -eq 0 ] || ret=1 +[ $($JOURNALPRINT -x ns1/d2121.db.jnl | grep -c "version 2") -eq 4 ] || ret=1 +[ $ret -eq 0 ] || echo_i "failed" +status=`expr $status + $ret` + +n=`expr $n + 1` +echo_i "check there are no journals left un-updated ($n)" +ret=0 +c1=$(cat -v ns1/*.jnl | grep -c "BIND LOG V9") +c2=$(cat -v ns1/*.jnl | grep -c "BIND LOG V9.2") +[ ${c1} -eq ${c2} ] || ret=1 +[ $ret -eq 0 ] || echo_i "failed" +status=`expr $status + $ret` + +n=`expr $n + 1` +echo_i "check journal downgrade/upgrade ($n)" +ret=0 +cp ns1/changed.db.jnl ns1/temp.jnl +$JOURNALPRINT -d ns1/temp.jnl +[ $($JOURNALPRINT -x ns1/temp.jnl | grep -c "version 1") -eq 1 ] || ret=1 +$JOURNALPRINT -x ns1/temp.jnl | grep -q "Header version = 1" || ret=1 +$JOURNALPRINT -u ns1/temp.jnl +$JOURNALPRINT -x ns1/temp.jnl | grep -q "Header version = 2" || ret=1 +[ $($JOURNALPRINT -x ns1/temp.jnl | grep -c "version 2") -eq 1 ] || ret=1 +[ $ret -eq 0 ] || echo_i "failed" +status=`expr $status + $ret` + +n=`expr $n + 1` +echo_i "check max-journal-size works after journal update ($n)" +ret=0 +# a dump should have been triggered by repairing the journal, +# which would have resulted in the journal already being +# compacted. +[ $(wc -c < ns1/maxjournal.db.jnl) -lt 4000 ] || ret=1 +[ $ret -eq 0 ] || echo_i "failed" +status=`expr $status + $ret` + +n=`expr $n + 1` +echo_i "check max-journal-size works with non-updated journals ($n)" +ret=0 +# journal was not repaired, so it should still be big +[ $(wc -c < ns1/maxjournal2.db.jnl) -gt 12000 ] || ret=1 +# the zone hasn't been dumped yet, so 'rndc sync' should work without +# needing a zone update first. +rndc_with_opts 10.53.0.1 sync maxjournal2 +check_size() ( + [ $(wc -c < ns1/maxjournal2.db.jnl) -lt 4000 ] +) +retry_quiet 10 check_size || ret=1 +[ $ret -eq 0 ] || echo_i "failed" +status=`expr $status + $ret` + +n=`expr $n + 1` +echo_i "check journal index consistency ($n)" +ret=0 +for jnl in ns1/*.jnl; do + $JOURNALPRINT -x $jnl 2>&1 | grep -q "Offset mismatch" && ret=1 +done +[ $ret -eq 0 ] || echo_i "failed" +status=`expr $status + $ret` + +echo_i "exit status: $status" +[ $status -eq 0 ] || exit 1 diff --git a/util/copyrights b/util/copyrights index ac7cb8e8d0..e992de8e8a 100644 --- a/util/copyrights +++ b/util/copyrights @@ -514,6 +514,18 @@ ./bin/tests/system/ixfr/ixfr-stats.good X 2019,2020,2021 ./bin/tests/system/ixfr/setup.sh SH 2001,2004,2007,2011,2012,2013,2014,2016,2018,2019,2020,2021 ./bin/tests/system/ixfr/tests.sh SH 2001,2004,2007,2011,2012,2014,2016,2018,2019,2020,2021 +./bin/tests/system/journal/clean.sh SH 2021 +./bin/tests/system/journal/ns1/changed.ver1.jnl.saved X 2021 +./bin/tests/system/journal/ns1/changed.ver2.jnl.saved X 2021 +./bin/tests/system/journal/ns1/d1212.jnl.saved X 2021 +./bin/tests/system/journal/ns1/d2121.jnl.saved X 2021 +./bin/tests/system/journal/ns1/ixfr.ver1.jnl.saved X 2021 +./bin/tests/system/journal/ns1/maxjournal.jnl.saved X 2021 +./bin/tests/system/journal/ns1/maxjournal2.jnl.saved X 2021 +./bin/tests/system/journal/ns1/unchanged.ver1.jnl.saved X 2021 +./bin/tests/system/journal/ns1/unchanged.ver2.jnl.saved X 2021 +./bin/tests/system/journal/setup.sh SH 2021 +./bin/tests/system/journal/tests.sh SH 2021 ./bin/tests/system/kasp/README TXT.BRIEF 2019,2020,2021 ./bin/tests/system/kasp/clean.sh SH 2019,2020,2021 ./bin/tests/system/kasp/ns2/setup.sh SH 2019,2020,2021 From 82b82bb8210ac11b9ae44966e04f51535ecf6dbd Mon Sep 17 00:00:00 2001 From: Evan Hunt Date: Mon, 22 Feb 2021 19:13:04 -0800 Subject: [PATCH 5/5] CHANGES, release note --- CHANGES | 6 ++++++ doc/notes/notes-current.rst | 17 +++++++++++++++++ 2 files changed, 23 insertions(+) diff --git a/CHANGES b/CHANGES index b131b13d65..26b7f355e4 100644 --- a/CHANGES +++ b/CHANGES @@ -1,3 +1,9 @@ +5593. [bug] Journal files written by older versions of named + can now be read when loading zones so that journal + incompatibility will not cause problems on upgrade. + Outdated journals will be updated to the new format + after loading. [GL #2505] + 5592. [bug] Add globally available thread_id (isc_tid_v) that's incremented for each new thread, but the old thread ids are reused, so the maximum thread_id always diff --git a/doc/notes/notes-current.rst b/doc/notes/notes-current.rst index 1ee8b09025..3d615f149f 100644 --- a/doc/notes/notes-current.rst +++ b/doc/notes/notes-current.rst @@ -64,3 +64,20 @@ Bug Fixes (crash) if the return of stale cached answers was enabled and ``stale-answer-client-timeout`` was applied to a client query in process. This has been fixed. [GL #2503] + +- Zone journal (``.jnl``) files created by versions of ``named`` prior + to 9.16.12 were no longer compatible; this could cause problems when + upgrading if journal files were not synchronized first. This has been + corrected: older journal files can now be read when starting up. When + an old-style journal file is detected, it is updated to the new + format immediately after loading. + + Note that journals created by the current version of ``named`` are not + usable by versions prior to 9.16.12. Before downgrading to a prior + release, users are advised to ensure that all dynamic zones have been + synchronized using ``rndc sync -clean``. + + A journal file's format can be changed manually by running + ``named-journalprint -d`` (downgrade) or ``named-journalprint -u`` + (upgrade). Note that this *must not* be done while ``named`` is + running. [GL #2505]