diff --git a/CHANGES b/CHANGES index b131b13d65..26b7f355e4 100644 --- a/CHANGES +++ b/CHANGES @@ -1,3 +1,9 @@ +5593. [bug] Journal files written by older versions of named + can now be read when loading zones so that journal + incompatibility will not cause problems on upgrade. + Outdated journals will be updated to the new format + after loading. [GL #2505] + 5592. [bug] Add globally available thread_id (isc_tid_v) that's incremented for each new thread, but the old thread ids are reused, so the maximum thread_id always diff --git a/bin/tests/system/Makefile.am b/bin/tests/system/Makefile.am index 92980fa441..feb7598241 100644 --- a/bin/tests/system/Makefile.am +++ b/bin/tests/system/Makefile.am @@ -111,6 +111,7 @@ TESTS += \ inline \ integrity \ hooks \ + journal \ keepalive \ legacy \ limits \ diff --git a/bin/tests/system/journal/clean.sh b/bin/tests/system/journal/clean.sh new file mode 100644 index 0000000000..1b8649595a --- /dev/null +++ b/bin/tests/system/journal/clean.sh @@ -0,0 +1,14 @@ +# Copyright (C) Internet Systems Consortium, Inc. ("ISC") +# +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, you can obtain one at https://mozilla.org/MPL/2.0/. +# +# See the COPYRIGHT file distributed with this work for additional +# information regarding copyright ownership. + +rm -f */named.memstats +rm -f */named.run +rm -f */named.conf +rm -f */*.db */*.jnl +rm -f dig.out* diff --git a/bin/tests/system/journal/ns1/changed.ver1.jnl.saved b/bin/tests/system/journal/ns1/changed.ver1.jnl.saved new file mode 100644 index 0000000000..b449a7dbcc Binary files /dev/null and b/bin/tests/system/journal/ns1/changed.ver1.jnl.saved differ diff --git a/bin/tests/system/journal/ns1/changed.ver2.jnl.saved b/bin/tests/system/journal/ns1/changed.ver2.jnl.saved new file mode 100644 index 0000000000..d2fa199654 Binary files /dev/null and b/bin/tests/system/journal/ns1/changed.ver2.jnl.saved differ diff --git a/bin/tests/system/journal/ns1/d1212.jnl.saved b/bin/tests/system/journal/ns1/d1212.jnl.saved new file mode 100644 index 0000000000..57c149722e Binary files /dev/null and b/bin/tests/system/journal/ns1/d1212.jnl.saved differ diff --git a/bin/tests/system/journal/ns1/d2121.jnl.saved b/bin/tests/system/journal/ns1/d2121.jnl.saved new file mode 100644 index 0000000000..ec21372f44 Binary files /dev/null and b/bin/tests/system/journal/ns1/d2121.jnl.saved differ diff --git a/bin/tests/system/journal/ns1/generic.db.in b/bin/tests/system/journal/ns1/generic.db.in new file mode 100644 index 0000000000..bb75d3ce82 --- /dev/null +++ b/bin/tests/system/journal/ns1/generic.db.in @@ -0,0 +1,16 @@ +; Copyright (C) Internet Systems Consortium, Inc. ("ISC") +; +; This Source Code Form is subject to the terms of the Mozilla Public +; License, v. 2.0. If a copy of the MPL was not distributed with this +; file, You can obtain one at http://mozilla.org/MPL/2.0/. +; +; See the COPYRIGHT file distributed with this work for additional +; information regarding copyright ownership. + + +$TTL 600 +@ SOA ns hostmaster 2012010901 3600 1200 604800 1200 + NS ns +ns A 192.0.2.1 + +addr1 A 10.53.0.1 diff --git a/bin/tests/system/journal/ns1/ixfr.db.in b/bin/tests/system/journal/ns1/ixfr.db.in new file mode 100644 index 0000000000..cee52ca6c5 --- /dev/null +++ b/bin/tests/system/journal/ns1/ixfr.db.in @@ -0,0 +1,17 @@ +; Copyright (C) Internet Systems Consortium, Inc. ("ISC") +; +; This Source Code Form is subject to the terms of the Mozilla Public +; License, v. 2.0. If a copy of the MPL was not distributed with this +; file, You can obtain one at http://mozilla.org/MPL/2.0/. +; +; See the COPYRIGHT file distributed with this work for additional +; information regarding copyright ownership. + + +$TTL 600 +@ SOA ns hostmaster 2012010902 3600 1200 604800 1200 + NS ns +ns A 192.0.2.1 + +addr1 A 10.53.0.1 +addr2 A 10.53.0.2 diff --git a/bin/tests/system/journal/ns1/ixfr.ver1.jnl.saved b/bin/tests/system/journal/ns1/ixfr.ver1.jnl.saved new file mode 100644 index 0000000000..10b5116ef2 Binary files /dev/null and b/bin/tests/system/journal/ns1/ixfr.ver1.jnl.saved differ diff --git a/bin/tests/system/journal/ns1/maxjournal.jnl.saved b/bin/tests/system/journal/ns1/maxjournal.jnl.saved new file mode 100644 index 0000000000..7c79e5c30e Binary files /dev/null and b/bin/tests/system/journal/ns1/maxjournal.jnl.saved differ diff --git a/bin/tests/system/journal/ns1/maxjournal2.jnl.saved b/bin/tests/system/journal/ns1/maxjournal2.jnl.saved new file mode 100644 index 0000000000..e200905955 Binary files /dev/null and b/bin/tests/system/journal/ns1/maxjournal2.jnl.saved differ diff --git a/bin/tests/system/journal/ns1/named.conf.in b/bin/tests/system/journal/ns1/named.conf.in new file mode 100644 index 0000000000..3fdd96b0ed --- /dev/null +++ b/bin/tests/system/journal/ns1/named.conf.in @@ -0,0 +1,90 @@ +/* + * Copyright (C) Internet Systems Consortium, Inc. ("ISC") + * + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. + * + * See the COPYRIGHT file distributed with this work for additional + * information regarding copyright ownership. + */ + +options { + query-source address 10.53.0.1; + notify-source 10.53.0.1; + transfer-source 10.53.0.1; + port @PORT@; + session-keyfile "session.key"; + pid-file "named.pid"; + listen-on { 10.53.0.1; }; + listen-on-v6 { none; }; + dnssec-validation yes; + minimal-responses no; + recursion no; + notify yes; +}; + +key rndc_key { + secret "1234abcd8765"; + algorithm hmac-sha256; +}; + +controls { + inet 10.53.0.1 port @CONTROLPORT@ allow { any; } keys { rndc_key; }; +}; + +zone changed { + type primary; + update-policy local; + file "changed.db"; +}; + +zone unchanged { + type primary; + update-policy local; + file "unchanged.db"; +}; + +zone changed2 { + type primary; + update-policy local; + file "changed2.db"; +}; + +zone unchanged2 { + type primary; + update-policy local; + file "unchanged2.db"; +}; + +zone hdr1d1d2d1d2 { + type primary; + update-policy local; + file "d1212.db"; +}; + +zone hdr1d2d1d2d1 { + type primary; + update-policy local; + file "d2121.db"; +}; + +zone ixfr { + type primary; + ixfr-from-differences yes; + file "ixfr.db"; +}; + +zone maxjournal { + type primary; + max-journal-size 1k; + update-policy local; + file "maxjournal.db"; +}; + +zone maxjournal2 { + type primary; + max-journal-size 1k; + update-policy local; + file "maxjournal2.db"; +}; diff --git a/bin/tests/system/journal/ns1/unchanged.ver1.jnl.saved b/bin/tests/system/journal/ns1/unchanged.ver1.jnl.saved new file mode 100644 index 0000000000..f7885d93dd Binary files /dev/null and b/bin/tests/system/journal/ns1/unchanged.ver1.jnl.saved differ diff --git a/bin/tests/system/journal/ns1/unchanged.ver2.jnl.saved b/bin/tests/system/journal/ns1/unchanged.ver2.jnl.saved new file mode 100644 index 0000000000..d974be4df7 Binary files /dev/null and b/bin/tests/system/journal/ns1/unchanged.ver2.jnl.saved differ diff --git a/bin/tests/system/journal/setup.sh b/bin/tests/system/journal/setup.sh new file mode 100644 index 0000000000..23c963920f --- /dev/null +++ b/bin/tests/system/journal/setup.sh @@ -0,0 +1,42 @@ +#!/bin/sh +# +# Copyright (C) Internet Systems Consortium, Inc. ("ISC") +# +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, you can obtain one at https://mozilla.org/MPL/2.0/. +# +# See the COPYRIGHT file distributed with this work for additional +# information regarding copyright ownership. + +. ../conf.sh + +$SHELL clean.sh + +copy_setports ns1/named.conf.in ns1/named.conf +cp ns1/generic.db.in ns1/changed.db +cp ns1/changed.ver1.jnl.saved ns1/changed.db.jnl + +cp ns1/generic.db.in ns1/unchanged.db +cp ns1/unchanged.ver1.jnl.saved ns1/unchanged.db.jnl + +cp ns1/generic.db.in ns1/changed2.db +cp ns1/changed.ver2.jnl.saved ns1/changed2.db.jnl + +cp ns1/generic.db.in ns1/unchanged2.db +cp ns1/unchanged.ver2.jnl.saved ns1/unchanged2.db.jnl + +cp ns1/ixfr.db.in ns1/ixfr.db +cp ns1/ixfr.ver1.jnl.saved ns1/ixfr.db.jnl + +cp ns1/generic.db.in ns1/d1212.db +cp ns1/d1212.jnl.saved ns1/d1212.db.jnl + +cp ns1/generic.db.in ns1/d2121.db +cp ns1/d2121.jnl.saved ns1/d2121.db.jnl + +cp ns1/generic.db.in ns1/maxjournal.db +cp ns1/maxjournal.jnl.saved ns1/maxjournal.db.jnl + +cp ns1/generic.db.in ns1/maxjournal2.db +cp ns1/maxjournal2.jnl.saved ns1/maxjournal2.db.jnl diff --git a/bin/tests/system/journal/tests.sh b/bin/tests/system/journal/tests.sh new file mode 100644 index 0000000000..7fd12dc14c --- /dev/null +++ b/bin/tests/system/journal/tests.sh @@ -0,0 +1,201 @@ +#!/bin/sh +# +# Copyright (C) Internet Systems Consortium, Inc. ("ISC") +# +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, you can obtain one at https://mozilla.org/MPL/2.0/. +# +# See the COPYRIGHT file distributed with this work for additional +# information regarding copyright ownership. + +. ../conf.sh + +dig_with_opts() { + "$DIG" @10.53.0.1 -p "$PORT" +tcp "$@" +} + +rndc_with_opts() { + "$RNDC" -c ../common/rndc.conf -p "$CONTROLPORT" -s "$@" +} + +status=0 +n=0 + +n=`expr $n + 1` +echo_i "check outdated journal rolled forward (dynamic) ($n)" +ret=0 +dig_with_opts changed soa > dig.out.test$n +grep 'status: NOERROR' dig.out.test$n > /dev/null || ret=1 +grep '2012010902' dig.out.test$n > /dev/null || ret=1 +grep 'zone changed/IN: retried using old journal format' ns1/named.run > /dev/null || ret=1 +[ $ret -eq 0 ] || echo_i "failed" +status=`expr $status + $ret` + +n=`expr $n + 1` +echo_i "check outdated empty journal did not cause an error (dynamic) ($n)" +ret=0 +dig_with_opts unchanged soa > dig.out.test$n +grep 'status: NOERROR' dig.out.test$n > /dev/null || ret=1 +grep '2012010901' dig.out.test$n > /dev/null || ret=1 +[ $ret -eq 0 ] || echo_i "failed" +status=`expr $status + $ret` + +n=`expr $n + 1` +echo_i "check outdated journals were updated or removed (dynamic) ($n)" +ret=0 +cat -v ns1/changed.db.jnl | grep "BIND LOG V9.2" > /dev/null || ret=1 +[ -f ns1/unchanged.db.jnl ] && ret=1 +[ $ret -eq 0 ] || echo_i "failed" +status=`expr $status + $ret` + +n=`expr $n + 1` +echo_i "check updated journal has correct RR count (dynamic) ($n)" +ret=0 +$JOURNALPRINT -x ns1/changed.db.jnl | grep "rrcount 3 " > /dev/null || ret=1 +[ $ret -eq 0 ] || echo_i "failed" +status=`expr $status + $ret` + +n=`expr $n + 1` +echo_i "check new-format journal rolled forward (dynamic) ($n)" +ret=0 +dig_with_opts changed2 soa > dig.out.test$n +grep 'status: NOERROR' dig.out.test$n > /dev/null || ret=1 +grep '2012010902' dig.out.test$n > /dev/null || ret=1 +grep 'zone changed2/IN: retried using old journal format' ns1/named.run > /dev/null && ret=1 +[ $ret -eq 0 ] || echo_i "failed" +status=`expr $status + $ret` + +n=`expr $n + 1` +echo_i "check new-format empty journal did not cause error (dynamic) ($n)" +ret=0 +dig_with_opts unchanged2 soa > dig.out.test$n +grep 'status: NOERROR' dig.out.test$n > /dev/null || ret=1 +grep '2012010901' dig.out.test$n > /dev/null || ret=1 +grep 'zone unchanged2/IN: retried using old journal format' ns1/named.run > /dev/null && ret=1 +[ $ret -eq 0 ] || echo_i "failed" +status=`expr $status + $ret` + +n=`expr $n + 1` +echo_i "check new-format journals were updated or removed (dynamic) ($n)" +ret=0 +cat -v ns1/changed2.db.jnl | grep "BIND LOG V9.2" > /dev/null || ret=1 +[ -f ns1/unchanged2.db.jnl ] && ret=1 +[ $ret -eq 0 ] || echo_i "failed" +status=`expr $status + $ret` + +n=`expr $n + 1` +echo_i "check outdated up-to-date journal succeeded (ixfr-from-differences) ($n)" +ret=0 +dig_with_opts -t soa ixfr > dig.out.test$n +grep 'status: NOERROR' dig.out.test$n > /dev/null || ret=1 +grep '2012010902' dig.out.test$n > /dev/null || ret=1 +grep 'zone ixfr/IN: journal rollforward completed successfully: recoverable' ns1/named.run > /dev/null || ret=1 +[ $ret -eq 0 ] || echo_i "failed" +status=`expr $status + $ret` + +n=`expr $n + 1` +echo_i "check outdated journal was updated (ixfr-from-differences) ($n)" +ret=0 +cat -v ns1/ixfr.db.jnl | grep "BIND LOG V9.2" > /dev/null || ret=1 +[ $ret -eq 0 ] || echo_i "failed" +status=`expr $status + $ret` + +n=`expr $n + 1` +echo_i "check journal with mixed headers succeeded (version 1,2,1,2) ($n)" +ret=0 +dig_with_opts -t soa hdr1d1d2d1d2 > dig.out.test$n +grep 'status: NOERROR' dig.out.test$n > /dev/null || ret=1 +grep '2012010905' dig.out.test$n > /dev/null || ret=1 +grep 'zone hdr1d1d2d1d2/IN: journal rollforward completed successfully: recoverable' ns1/named.run > /dev/null || ret=1 +[ $ret -eq 0 ] || echo_i "failed" +status=`expr $status + $ret` + +n=`expr $n + 1` +echo_i "check journal with mixed headers was updated (version 1,2,1,2) ($n)" +ret=0 +[ $($JOURNALPRINT -x ns1/d1212.jnl.saved | grep -c "version 1") -eq 2 ] || ret=1 +[ $($JOURNALPRINT -x ns1/d1212.jnl.saved | grep -c "version 2") -eq 2 ] || ret=1 +[ $($JOURNALPRINT -x ns1/d1212.db.jnl | grep -c "version 1") -eq 0 ] || ret=1 +[ $($JOURNALPRINT -x ns1/d1212.db.jnl | grep -c "version 2") -eq 4 ] || ret=1 +[ $ret -eq 0 ] || echo_i "failed" +status=`expr $status + $ret` + +n=`expr $n + 1` +echo_i "check journal with mixed headers succeeded (version 2,1,2,1) ($n)" +ret=0 +dig_with_opts -t soa hdr1d2d1d2d1 > dig.out.test$n +grep 'status: NOERROR' dig.out.test$n > /dev/null || ret=1 +grep '2012010905' dig.out.test$n > /dev/null || ret=1 +grep 'zone hdr1d2d1d2d1/IN: journal rollforward completed successfully: recoverable' ns1/named.run > /dev/null || ret=1 +[ $ret -eq 0 ] || echo_i "failed" +status=`expr $status + $ret` + +n=`expr $n + 1` +echo_i "check journal with mixed headers was updated (version 2,1,2,1) ($n)" +ret=0 +[ $($JOURNALPRINT -x ns1/d2121.jnl.saved | grep -c "version 1") -eq 2 ] || ret=1 +[ $($JOURNALPRINT -x ns1/d2121.jnl.saved | grep -c "version 2") -eq 2 ] || ret=1 +[ $($JOURNALPRINT -x ns1/d2121.db.jnl | grep -c "version 1") -eq 0 ] || ret=1 +[ $($JOURNALPRINT -x ns1/d2121.db.jnl | grep -c "version 2") -eq 4 ] || ret=1 +[ $ret -eq 0 ] || echo_i "failed" +status=`expr $status + $ret` + +n=`expr $n + 1` +echo_i "check there are no journals left un-updated ($n)" +ret=0 +c1=$(cat -v ns1/*.jnl | grep -c "BIND LOG V9") +c2=$(cat -v ns1/*.jnl | grep -c "BIND LOG V9.2") +[ ${c1} -eq ${c2} ] || ret=1 +[ $ret -eq 0 ] || echo_i "failed" +status=`expr $status + $ret` + +n=`expr $n + 1` +echo_i "check journal downgrade/upgrade ($n)" +ret=0 +cp ns1/changed.db.jnl ns1/temp.jnl +$JOURNALPRINT -d ns1/temp.jnl +[ $($JOURNALPRINT -x ns1/temp.jnl | grep -c "version 1") -eq 1 ] || ret=1 +$JOURNALPRINT -x ns1/temp.jnl | grep -q "Header version = 1" || ret=1 +$JOURNALPRINT -u ns1/temp.jnl +$JOURNALPRINT -x ns1/temp.jnl | grep -q "Header version = 2" || ret=1 +[ $($JOURNALPRINT -x ns1/temp.jnl | grep -c "version 2") -eq 1 ] || ret=1 +[ $ret -eq 0 ] || echo_i "failed" +status=`expr $status + $ret` + +n=`expr $n + 1` +echo_i "check max-journal-size works after journal update ($n)" +ret=0 +# a dump should have been triggered by repairing the journal, +# which would have resulted in the journal already being +# compacted. +[ $(wc -c < ns1/maxjournal.db.jnl) -lt 4000 ] || ret=1 +[ $ret -eq 0 ] || echo_i "failed" +status=`expr $status + $ret` + +n=`expr $n + 1` +echo_i "check max-journal-size works with non-updated journals ($n)" +ret=0 +# journal was not repaired, so it should still be big +[ $(wc -c < ns1/maxjournal2.db.jnl) -gt 12000 ] || ret=1 +# the zone hasn't been dumped yet, so 'rndc sync' should work without +# needing a zone update first. +rndc_with_opts 10.53.0.1 sync maxjournal2 +check_size() ( + [ $(wc -c < ns1/maxjournal2.db.jnl) -lt 4000 ] +) +retry_quiet 10 check_size || ret=1 +[ $ret -eq 0 ] || echo_i "failed" +status=`expr $status + $ret` + +n=`expr $n + 1` +echo_i "check journal index consistency ($n)" +ret=0 +for jnl in ns1/*.jnl; do + $JOURNALPRINT -x $jnl 2>&1 | grep -q "Offset mismatch" && ret=1 +done +[ $ret -eq 0 ] || echo_i "failed" +status=`expr $status + $ret` + +echo_i "exit status: $status" +[ $status -eq 0 ] || exit 1 diff --git a/bin/tools/named-journalprint.c b/bin/tools/named-journalprint.c index 20b8740021..37cc0ff205 100644 --- a/bin/tools/named-journalprint.c +++ b/bin/tools/named-journalprint.c @@ -13,6 +13,7 @@ #include +#include #include #include #include @@ -23,6 +24,14 @@ #include #include +const char *progname = NULL; + +static void +usage(void) { + fprintf(stderr, "Usage: %s [-dux] journal\n", progname); + exit(1); +} + /* * Setup logging to use stderr. */ @@ -57,20 +66,50 @@ main(int argc, char **argv) { isc_mem_t *mctx = NULL; isc_result_t result; isc_log_t *lctx = NULL; + uint32_t flags = 0U; + char ch; + bool downgrade = false; + bool upgrade = false; - if (argc != 2) { - printf("usage: %s journal\n", argv[0]); - return (1); + progname = argv[0]; + while ((ch = isc_commandline_parse(argc, argv, "dux")) != -1) { + switch (ch) { + case 'd': + downgrade = true; + break; + case 'u': + upgrade = true; + break; + case 'x': + flags |= DNS_JOURNAL_PRINTXHDR; + break; + default: + usage(); + } } - file = argv[1]; + argc -= isc_commandline_index; + argv += isc_commandline_index; + + if (argc != 1) { + usage(); + } + file = argv[0]; isc_mem_create(&mctx); RUNTIME_CHECK(setup_logging(mctx, stderr, &lctx) == ISC_R_SUCCESS); - result = dns_journal_print(mctx, file, stdout); - if (result == DNS_R_NOJOURNAL) { - fprintf(stderr, "%s\n", dns_result_totext(result)); + if (upgrade) { + flags = DNS_JOURNAL_COMPACTALL; + result = dns_journal_compact(mctx, file, 0, flags, 0); + } else if (downgrade) { + flags = DNS_JOURNAL_COMPACTALL | DNS_JOURNAL_VERSION1; + result = dns_journal_compact(mctx, file, 0, flags, 0); + } else { + result = dns_journal_print(mctx, flags, file, stdout); + if (result == DNS_R_NOJOURNAL) { + fprintf(stderr, "%s\n", dns_result_totext(result)); + } } isc_log_destroy(&lctx); isc_mem_detach(&mctx); diff --git a/bin/tools/named-journalprint.rst b/bin/tools/named-journalprint.rst index ffa9ac9577..633bcb628b 100644 --- a/bin/tools/named-journalprint.rst +++ b/bin/tools/named-journalprint.rst @@ -29,13 +29,14 @@ named-journalprint - print zone journal in human-readable form Synopsis ~~~~~~~~ -:program:`named-journalprint` {journal} +:program:`named-journalprint` [**-dux**] {journal} Description ~~~~~~~~~~~ -``named-journalprint`` prints the contents of a zone journal file in a -human-readable form. +``named-journalprint`` scans the contents of a zone journal file, +printing it in a human-readable form, or, optionally, converting it +to a different journal file format. Journal files are automatically created by ``named`` when changes are made to dynamic zones (e.g., by ``nsupdate``). They record each addition @@ -50,6 +51,17 @@ into a human-readable text format. Each line begins with ``add`` or ``del``, to indicate whether the record was added or deleted, and continues with the resource record in master-file format. +The ``-x`` option causes additional data about the journal file to be +printed at the beginning of the output and before each group of changes. + +The ``-u`` (upgrade) and ``-d`` (downgrade) options recreate the journal +file with a modified format version. The existing journal file is +replaced. ``-d`` writes out the journal in the format used by +versions of BIND up to 9.16.11; ``-u`` writes it out in the format used +by versions since 9.16.13. (9.16.12 is omitted due to a journal-formatting +bug in that release.) Note that these options *must not* be used while +``named`` is running. + See Also ~~~~~~~~ diff --git a/doc/man/named-journalprint.1in b/doc/man/named-journalprint.1in index 2c86eec8df..b18f896547 100644 --- a/doc/man/named-journalprint.1in +++ b/doc/man/named-journalprint.1in @@ -32,11 +32,12 @@ level margin: \\n[rst2man-indent\\n[rst2man-indent-level]] .. .SH SYNOPSIS .sp -\fBnamed\-journalprint\fP {journal} +\fBnamed\-journalprint\fP [\fB\-dux\fP] {journal} .SH DESCRIPTION .sp -\fBnamed\-journalprint\fP prints the contents of a zone journal file in a -human\-readable form. +\fBnamed\-journalprint\fP scans the contents of a zone journal file, +printing it in a human\-readable form, or, optionally, converting it +to a different journal file format. .sp Journal files are automatically created by \fBnamed\fP when changes are made to dynamic zones (e.g., by \fBnsupdate\fP). They record each addition @@ -50,6 +51,17 @@ file. into a human\-readable text format. Each line begins with \fBadd\fP or \fBdel\fP, to indicate whether the record was added or deleted, and continues with the resource record in master\-file format. +.sp +The \fB\-x\fP option causes additional data about the journal file to be +printed at the beginning of the output and before each group of changes. +.sp +The \fB\-u\fP (upgrade) and \fB\-d\fP (downgrade) options recreate the journal +file with a modified format version. The existing journal file is +replaced. \fB\-d\fP writes out the journal in the format used by +versions of BIND up to 9.16.11; \fB\-u\fP writes it out in the format used +by versions since 9.16.13. (9.16.12 is omitted due to a journal\-formatting +bug in that release.) Note that these options \fImust not\fP be used while +\fBnamed\fP is running. .SH SEE ALSO .sp \fBnamed(8)\fP, \fBnsupdate(1)\fP, BIND 9 Administrator Reference Manual. diff --git a/doc/notes/notes-current.rst b/doc/notes/notes-current.rst index 1ee8b09025..3d615f149f 100644 --- a/doc/notes/notes-current.rst +++ b/doc/notes/notes-current.rst @@ -64,3 +64,20 @@ Bug Fixes (crash) if the return of stale cached answers was enabled and ``stale-answer-client-timeout`` was applied to a client query in process. This has been fixed. [GL #2503] + +- Zone journal (``.jnl``) files created by versions of ``named`` prior + to 9.16.12 were no longer compatible; this could cause problems when + upgrading if journal files were not synchronized first. This has been + corrected: older journal files can now be read when starting up. When + an old-style journal file is detected, it is updated to the new + format immediately after loading. + + Note that journals created by the current version of ``named`` are not + usable by versions prior to 9.16.12. Before downgrading to a prior + release, users are advised to ensure that all dynamic zones have been + synchronized using ``rndc sync -clean``. + + A journal file's format can be changed manually by running + ``named-journalprint -d`` (downgrade) or ``named-journalprint -u`` + (upgrade). Note that this *must not* be done while ``named`` is + running. [GL #2505] diff --git a/lib/dns/include/dns/events.h b/lib/dns/include/dns/events.h index 50a8d01ac6..894c4dd211 100644 --- a/lib/dns/include/dns/events.h +++ b/lib/dns/include/dns/events.h @@ -79,6 +79,7 @@ #define DNS_EVENT_RPZUPDATED (ISC_EVENTCLASS_DNS + 57) #define DNS_EVENT_STARTUPDATE (ISC_EVENTCLASS_DNS + 58) #define DNS_EVENT_TRYSTALE (ISC_EVENTCLASS_DNS + 59) +#define DNS_EVENT_ZONEFLUSH (ISC_EVENTCLASS_DNS + 60) #define DNS_EVENT_FIRSTEVENT (ISC_EVENTCLASS_DNS + 0) #define DNS_EVENT_LASTEVENT (ISC_EVENTCLASS_DNS + 65535) diff --git a/lib/dns/include/dns/journal.h b/lib/dns/include/dns/journal.h index 8acf9ab44e..e115bbc226 100644 --- a/lib/dns/include/dns/journal.h +++ b/lib/dns/include/dns/journal.h @@ -48,6 +48,13 @@ #define DNS_JOURNAL_SIZE_MAX INT32_MAX #define DNS_JOURNAL_SIZE_MIN 4096 +/*% Print transaction header data */ +#define DNS_JOURNAL_PRINTXHDR 0x0001 + +/*% Rewrite whole journal file instead of compacting */ +#define DNS_JOURNAL_COMPACTALL 0x0001 +#define DNS_JOURNAL_VERSION1 0x0002 + /*** *** Types ***/ @@ -258,12 +265,18 @@ dns_journal_rollforward(isc_mem_t *mctx, dns_db_t *db, unsigned int options, *\li DNS_R_NOJOURNAL when journal does not exist. *\li ISC_R_NOTFOUND when current serial in not in journal. *\li ISC_R_RANGE when current serial in not in journals range. - *\li ISC_R_SUCCESS journal has been applied successfully to database. + *\li DNS_R_UPTODATE when the database was already up to date. + *\li ISC_R_SUCCESS journal has been applied successfully to the + * database without any issues. + *\li DNS_R_RECOVERABLE if successful or up to date, but the journal + * was found to contain at least one outdated transaction header. + * * others */ isc_result_t -dns_journal_print(isc_mem_t *mctx, const char *filename, FILE *file); +dns_journal_print(isc_mem_t *mctx, uint32_t flags, const char *filename, + FILE *file); /* For debugging not general use */ isc_result_t @@ -286,7 +299,7 @@ dns_db_diffx(dns_diff_t *diff, dns_db_t *dba, dns_dbversion_t *dbvera, isc_result_t dns_journal_compact(isc_mem_t *mctx, char *filename, uint32_t serial, - uint32_t target_size); + uint32_t flags, uint32_t target_size); /*%< * Attempt to compact the journal if it is greater that 'target_size'. * Changes from 'serial' onwards will be preserved. If the journal diff --git a/lib/dns/journal.c b/lib/dns/journal.c index bc1ad96314..fad38c0f57 100644 --- a/lib/dns/journal.c +++ b/lib/dns/journal.c @@ -18,6 +18,7 @@ #include #include #include +#include #include #include #include @@ -193,6 +194,11 @@ typedef struct { */ #define JOURNAL_HEADER_SIZE 64 /* Bytes. */ +typedef enum { + XHDR_VERSION1 = 1, + XHDR_VERSION2 = 2, +} xhdr_version_t; + /*% * The on-disk representation of the journal header. * All numbers are stored in big-endian order. @@ -216,7 +222,7 @@ typedef union { } journal_rawheader_t; /*% - * The on-disk representation of the transaction header. + * The on-disk representation of the transaction header, version 2. * There is one of these at the beginning of each transaction. */ typedef struct { @@ -226,6 +232,16 @@ typedef struct { unsigned char serial1[4]; /*%< SOA serial after update. */ } journal_rawxhdr_t; +/*% + * Old-style raw transaction header, version 1, used for backward + * compatibility mode. + */ +typedef struct { + unsigned char size[4]; + unsigned char serial0[4]; + unsigned char serial1[4]; +} journal_rawxhdr_ver1_t; + /*% * The on-disk representation of the RR header. * There is one of these at the beginning of each RR. @@ -275,16 +291,19 @@ typedef struct { * Initial contents to store in the header of a newly created * journal file. * - * The header starts with the magic string ";BIND LOG V9\n" + * The header starts with the magic string ";BIND LOG V9.2\n" * to identify the file as a BIND 9 journal file. An ASCII * identification string is used rather than a binary magic * number to be consistent with BIND 8 (BIND 8 journal files * are ASCII text files). */ -static journal_header_t initial_journal_header = { +static journal_header_t journal_header_ver1 = { ";BIND LOG V9\n", { 0, 0 }, { 0, 0 }, 0, 0, 0 }; +static journal_header_t initial_journal_header = { + ";BIND LOG V9.2\n", { 0, 0 }, { 0, 0 }, 0, 0, 0 +}; #define JOURNAL_EMPTY(h) ((h)->begin.offset == (h)->end.offset) @@ -300,13 +319,19 @@ struct dns_journal { unsigned int magic; /*%< JOUR */ isc_mem_t *mctx; /*%< Memory context */ journal_state_t state; - char *filename; /*%< Journal file name */ - FILE *fp; /*%< File handle */ - isc_offset_t offset; /*%< Current file offset */ - journal_header_t header; /*%< In-core journal header */ - unsigned char *rawindex; /*%< In-core buffer for journal index - * in on-disk format */ - journal_pos_t *index; /*%< In-core journal index */ + xhdr_version_t xhdr_version; /*%< Expected transaction header version */ + bool header_ver1; /*%< Transaction header compatibility + * mode is allowed */ + bool recovered; /*%< A recoverable error was found + * while reading the journal */ + char *filename; /*%< Journal file name */ + FILE *fp; /*%< File handle */ + isc_offset_t offset; /*%< Current file offset */ + journal_xhdr_t curxhdr; /*%< Current transaction header */ + journal_header_t header; /*%< In-core journal header */ + unsigned char *rawindex; /*%< In-core buffer for journal index + * in on-disk format */ + journal_pos_t *index; /*%< In-core journal index */ /*% Current transaction state (when writing). */ struct { @@ -319,10 +344,10 @@ struct dns_journal { struct { /* These define the part of the journal we iterate over. */ journal_pos_t bpos; /*%< Position before first, */ + journal_pos_t cpos; /*%< before current, */ journal_pos_t epos; /*%< and after last transaction */ /* The rest is iterator state. */ - uint32_t current_serial; /*%< Current SOA serial - * */ + uint32_t current_serial; /*%< Current SOA serial */ isc_buffer_t source; /*%< Data from disk */ isc_buffer_t target; /*%< Data from _fromwire check */ dns_decompress_t dctx; /*%< Dummy decompression ctx */ @@ -353,6 +378,7 @@ journal_pos_encode(journal_rawpos_t *raw, journal_pos_t *cooked) { static void journal_header_decode(journal_rawheader_t *raw, journal_header_t *cooked) { INSIST(sizeof(cooked->format) == sizeof(raw->h.format)); + memmove(cooked->format, raw->h.format, sizeof(cooked->format)); journal_pos_decode(&raw->h.begin, &cooked->begin); journal_pos_decode(&raw->h.end, &cooked->end); @@ -366,6 +392,7 @@ journal_header_encode(journal_header_t *cooked, journal_rawheader_t *raw) { unsigned char flags = 0; INSIST(sizeof(cooked->format) == sizeof(raw->h.format)); + memset(raw->pad, 0, sizeof(raw->pad)); memmove(raw->h.format, cooked->format, sizeof(raw->h.format)); journal_pos_encode(&raw->h.begin, &cooked->begin); @@ -432,6 +459,7 @@ journal_write(dns_journal_t *j, void *mem, size_t nbytes) { static isc_result_t journal_fsync(dns_journal_t *j) { isc_result_t result; + result = isc_stdio_flush(j->fp); if (result != ISC_R_SUCCESS) { isc_log_write(JOURNAL_COMMON_LOGARGS, ISC_LOG_ERROR, @@ -452,31 +480,63 @@ journal_fsync(dns_journal_t *j) { /* * Read/write a transaction header at the current file position. */ - static isc_result_t journal_read_xhdr(dns_journal_t *j, journal_xhdr_t *xhdr) { - journal_rawxhdr_t raw; isc_result_t result; - result = journal_read(j, &raw, sizeof(raw)); - if (result != ISC_R_SUCCESS) { - return (result); + + j->it.cpos.offset = j->offset; + + switch (j->xhdr_version) { + case XHDR_VERSION1: { + journal_rawxhdr_ver1_t raw; + result = journal_read(j, &raw, sizeof(raw)); + if (result != ISC_R_SUCCESS) { + return (result); + } + xhdr->size = decode_uint32(raw.size); + xhdr->count = 0; + xhdr->serial0 = decode_uint32(raw.serial0); + xhdr->serial1 = decode_uint32(raw.serial1); + j->curxhdr = *xhdr; + return (ISC_R_SUCCESS); + } + + case XHDR_VERSION2: { + journal_rawxhdr_t raw; + result = journal_read(j, &raw, sizeof(raw)); + if (result != ISC_R_SUCCESS) { + return (result); + } + xhdr->size = decode_uint32(raw.size); + xhdr->count = decode_uint32(raw.count); + xhdr->serial0 = decode_uint32(raw.serial0); + xhdr->serial1 = decode_uint32(raw.serial1); + j->curxhdr = *xhdr; + return (ISC_R_SUCCESS); + } + + default: + return (ISC_R_NOTIMPLEMENTED); } - xhdr->size = decode_uint32(raw.size); - xhdr->count = decode_uint32(raw.count); - xhdr->serial0 = decode_uint32(raw.serial0); - xhdr->serial1 = decode_uint32(raw.serial1); - return (ISC_R_SUCCESS); } static isc_result_t journal_write_xhdr(dns_journal_t *j, uint32_t size, uint32_t count, uint32_t serial0, uint32_t serial1) { - journal_rawxhdr_t raw; - encode_uint32(size, raw.size); - encode_uint32(count, raw.count); - encode_uint32(serial0, raw.serial0); - encode_uint32(serial1, raw.serial1); - return (journal_write(j, &raw, sizeof(raw))); + if (j->header_ver1) { + journal_rawxhdr_ver1_t raw; + encode_uint32(size, raw.size); + encode_uint32(serial0, raw.serial0); + encode_uint32(serial1, raw.serial1); + return (journal_write(j, &raw, sizeof(raw))); + } else { + journal_rawxhdr_t raw; + encode_uint32(size, raw.size); + encode_uint32(count, raw.count); + encode_uint32(serial0, raw.serial0); + encode_uint32(serial1, raw.serial1); + return (journal_write(j, &raw, sizeof(raw))); + } } /* @@ -487,6 +547,7 @@ static isc_result_t journal_read_rrhdr(dns_journal_t *j, journal_rrhdr_t *rrhdr) { journal_rawrrhdr_t raw; isc_result_t result; + result = journal_read(j, &raw, sizeof(raw)); if (result != ISC_R_SUCCESS) { return (result); @@ -496,14 +557,14 @@ journal_read_rrhdr(dns_journal_t *j, journal_rrhdr_t *rrhdr) { } static isc_result_t -journal_file_create(isc_mem_t *mctx, const char *filename) { +journal_file_create(isc_mem_t *mctx, bool downgrade, const char *filename) { FILE *fp = NULL; isc_result_t result; journal_header_t header; journal_rawheader_t rawheader; int index_size = 56; /* XXX configurable */ int size; - void *mem; /* Memory for temporary index image. */ + void *mem = NULL; /* Memory for temporary index image. */ INSIST(sizeof(journal_rawheader_t) == JOURNAL_HEADER_SIZE); @@ -515,7 +576,11 @@ journal_file_create(isc_mem_t *mctx, const char *filename) { return (ISC_R_UNEXPECTED); } - header = initial_journal_header; + if (downgrade) { + header = journal_header_ver1; + } else { + header = initial_journal_header; + } header.index_size = index_size; journal_header_encode(&header, &rawheader); @@ -552,36 +617,28 @@ journal_file_create(isc_mem_t *mctx, const char *filename) { static isc_result_t journal_open(isc_mem_t *mctx, const char *filename, bool writable, bool create, - dns_journal_t **journalp) { + bool downgrade, dns_journal_t **journalp) { FILE *fp = NULL; isc_result_t result; journal_rawheader_t rawheader; dns_journal_t *j; - INSIST(journalp != NULL && *journalp == NULL); + REQUIRE(journalp != NULL && *journalp == NULL); + j = isc_mem_get(mctx, sizeof(*j)); - - j->mctx = NULL; + *j = (dns_journal_t){ .state = JOURNAL_STATE_INVALID, + .filename = isc_mem_strdup(mctx, filename), + .xhdr_version = XHDR_VERSION2 }; isc_mem_attach(mctx, &j->mctx); - j->state = JOURNAL_STATE_INVALID; - j->fp = NULL; - j->filename = isc_mem_strdup(mctx, filename); - j->index = NULL; - j->rawindex = NULL; - - if (j->filename == NULL) { - FAIL(ISC_R_NOMEMORY); - } result = isc_stdio_open(j->filename, writable ? "rb+" : "rb", &fp); - if (result == ISC_R_FILENOTFOUND) { if (create) { isc_log_write(JOURNAL_COMMON_LOGARGS, ISC_LOG_DEBUG(1), "journal file %s does not exist, " "creating it", j->filename); - CHECK(journal_file_create(mctx, filename)); + CHECK(journal_file_create(mctx, downgrade, filename)); /* * Retry. */ @@ -607,9 +664,29 @@ journal_open(isc_mem_t *mctx, const char *filename, bool writable, bool create, CHECK(journal_seek(j, 0)); CHECK(journal_read(j, &rawheader, sizeof(rawheader))); - if (memcmp(rawheader.h.format, initial_journal_header.format, - sizeof(initial_journal_header.format)) != 0) + if (memcmp(rawheader.h.format, journal_header_ver1.format, + sizeof(journal_header_ver1.format)) == 0) { + /* + * The file header says it's the old format, but it + * still might have the new xhdr format because we + * forgot to change the format string when we introduced + * the new xhdr. When we first try to read it, we assume + * it uses the new xhdr format. If that fails, we'll be + * called a second time with compat set to true, in which + * case we can lower xhdr_version to 1 if we find a + * corrupt transaction. + */ + j->header_ver1 = true; + } else if (memcmp(rawheader.h.format, initial_journal_header.format, + sizeof(initial_journal_header.format)) == 0) + { + /* + * File header says this is format version 2; all + * transactions have to match. + */ + j->header_ver1 = false; + } else { isc_log_write(JOURNAL_COMMON_LOGARGS, ISC_LOG_ERROR, "%s: journal format not recognized", j->filename); FAIL(ISC_R_UNEXPECTED); @@ -695,7 +772,8 @@ dns_journal_open(isc_mem_t *mctx, const char *filename, unsigned int mode, create = ((mode & DNS_JOURNAL_CREATE) != 0); writable = ((mode & (DNS_JOURNAL_WRITE | DNS_JOURNAL_CREATE)) != 0); - result = journal_open(mctx, filename, writable, create, journalp); + result = journal_open(mctx, filename, writable, create, false, + journalp); if (result == ISC_R_NOTFOUND) { namelen = strlen(filename); if (namelen > 4U && strcmp(filename + namelen - 4, ".jnl") == 0) @@ -708,7 +786,7 @@ dns_journal_open(isc_mem_t *mctx, const char *filename, unsigned int mode, if (result >= sizeof(backup)) { return (ISC_R_NOSPACE); } - result = journal_open(mctx, backup, writable, writable, + result = journal_open(mctx, backup, writable, writable, false, journalp); } return (result); @@ -795,9 +873,11 @@ ixfr_order(const void *av, const void *bv) { * Other results due to file errors are possible. */ static isc_result_t -journal_next(dns_journal_t *j, journal_pos_t *pos) { +journal_next(dns_journal_t *j, journal_pos_t *pos, bool retry) { isc_result_t result; journal_xhdr_t xhdr; + size_t hdrsize; + REQUIRE(DNS_JOURNAL_VALID(j)); result = journal_seek(j, pos->offset); @@ -808,6 +888,7 @@ journal_next(dns_journal_t *j, journal_pos_t *pos) { if (pos->serial == j->header.end.serial) { return (ISC_R_NOMORE); } + /* * Read the header of the current transaction. * This will return ISC_R_NOMORE if we are at EOF. @@ -820,26 +901,59 @@ journal_next(dns_journal_t *j, journal_pos_t *pos) { /* * Check serial number consistency. */ - if (xhdr.serial0 != pos->serial) { - isc_log_write(JOURNAL_COMMON_LOGARGS, ISC_LOG_ERROR, - "%s: journal file corrupt: " - "expected serial %u, got %u", - j->filename, pos->serial, xhdr.serial0); - return (ISC_R_UNEXPECTED); + if (xhdr.serial0 != pos->serial || + isc_serial_le(xhdr.serial1, xhdr.serial0)) { + if (j->header_ver1 && j->xhdr_version == XHDR_VERSION1 && + xhdr.serial1 == pos->serial && !retry) + { + /* XHDR_VERSION1 -> XHDR_VERSION2 */ + isc_log_write( + JOURNAL_COMMON_LOGARGS, ISC_LOG_DEBUG(3), + "%s: XHDR_VERSION1 -> XHDR_VERSION2 at %u\n", + j->filename, pos->serial); + j->xhdr_version = XHDR_VERSION2; + result = journal_next(j, pos, true); + if (result == ISC_R_SUCCESS) { + j->recovered = true; + } + return (result); + } else if (j->header_ver1 && j->xhdr_version == XHDR_VERSION2 && + xhdr.count == pos->serial && !retry) + { + /* XHDR_VERSION2 -> XHDR_VERSION1 */ + isc_log_write( + JOURNAL_COMMON_LOGARGS, ISC_LOG_DEBUG(3), + "%s: XHDR_VERSION2 -> XHDR_VERSION1 at %u\n", + j->filename, pos->serial); + j->xhdr_version = XHDR_VERSION1; + result = journal_next(j, pos, true); + if (result == ISC_R_SUCCESS) { + j->recovered = true; + } + return (result); + } else { + isc_log_write(JOURNAL_COMMON_LOGARGS, ISC_LOG_ERROR, + "%s: journal file corrupt: " + "expected serial %u, got %u", + j->filename, pos->serial, xhdr.serial0); + return (ISC_R_UNEXPECTED); + } } /* * Check for offset wraparound. */ - if ((isc_offset_t)(pos->offset + sizeof(journal_rawxhdr_t) + - xhdr.size) < pos->offset) - { + hdrsize = (j->xhdr_version == XHDR_VERSION2) + ? sizeof(journal_rawxhdr_t) + : sizeof(journal_rawxhdr_ver1_t); + + if ((isc_offset_t)(pos->offset + hdrsize + xhdr.size) < pos->offset) { isc_log_write(JOURNAL_COMMON_LOGARGS, ISC_LOG_ERROR, "%s: offset too large", j->filename); return (ISC_R_UNEXPECTED); } - pos->offset += sizeof(journal_rawxhdr_t) + xhdr.size; + pos->offset += hdrsize + xhdr.size; pos->serial = xhdr.serial1; return (ISC_R_SUCCESS); } @@ -879,9 +993,11 @@ index_find(dns_journal_t *j, uint32_t serial, journal_pos_t *best_guess) { static void index_add(dns_journal_t *j, journal_pos_t *pos) { unsigned int i; + if (j->index == NULL) { return; } + /* * Search for a vacant position. */ @@ -953,6 +1069,7 @@ static isc_result_t journal_find(dns_journal_t *j, uint32_t serial, journal_pos_t *pos) { isc_result_t result; journal_pos_t current_pos; + REQUIRE(DNS_JOURNAL_VALID(j)); if (DNS_SERIAL_GT(j->header.begin.serial, serial)) { @@ -973,7 +1090,7 @@ journal_find(dns_journal_t *j, uint32_t serial, journal_pos_t *pos) { if (DNS_SERIAL_GT(current_pos.serial, serial)) { return (ISC_R_NOTFOUND); } - result = journal_next(j, ¤t_pos); + result = journal_next(j, ¤t_pos, false); if (result != ISC_R_SUCCESS) { return (result); } @@ -1187,7 +1304,7 @@ dns_journal_commit(dns_journal_t *j) { if (!JOURNAL_EMPTY(&j->header)) { while (!DNS_SERIAL_GT(j->x.pos[1].serial, j->header.begin.serial)) { - CHECK(journal_next(j, &j->header.begin)); + CHECK(journal_next(j, &j->header.begin, false)); } index_invalidate(j, j->x.pos[1].serial); } @@ -1256,6 +1373,7 @@ failure: isc_result_t dns_journal_write_transaction(dns_journal_t *j, dns_diff_t *diff) { isc_result_t result; + CHECK(dns_diff_sort(diff, ixfr_order)); CHECK(dns_journal_begin_transaction(j)); CHECK(dns_journal_writediff(j, diff)); @@ -1267,9 +1385,13 @@ failure: void dns_journal_destroy(dns_journal_t **journalp) { - dns_journal_t *j = *journalp; + dns_journal_t *j = NULL; + + REQUIRE(journalp != NULL); + REQUIRE(DNS_JOURNAL_VALID(*journalp)); + + j = *journalp; *journalp = NULL; - REQUIRE(DNS_JOURNAL_VALID(j)); j->it.result = ISC_R_FAILURE; dns_name_invalidate(&j->it.name); @@ -1346,33 +1468,38 @@ roll_forward(dns_journal_t *j, dns_db_t *db, unsigned int options) { * Locate a journal entry for the current database serial. */ CHECK(journal_find(j, db_serial, &pos)); - /* - * XXX do more drastic things, like marking zone stale, - * if this fails? - */ - /* - * XXXRTH The zone code should probably mark the zone as bad and - * scream loudly into the log if this is a dynamic update - * log reply that failed. - */ end_serial = dns_journal_last_serial(j); + + /* + * If we're reading a version 1 file, scan all the transactions + * to see if the journal needs rewriting: if any outdated + * transaction headers are found, j->recovered will be set. + */ + if (j->header_ver1) { + uint32_t start_serial = dns_journal_first_serial(j); + + CHECK(dns_journal_iter_init(j, start_serial, db_serial, NULL)); + for (result = dns_journal_first_rr(j); result == ISC_R_SUCCESS; + result = dns_journal_next_rr(j)) + { + continue; + } + } + if (db_serial == end_serial) { CHECK(DNS_R_UPTODATE); } CHECK(dns_journal_iter_init(j, db_serial, end_serial, NULL)); - for (result = dns_journal_first_rr(j); result == ISC_R_SUCCESS; result = dns_journal_next_rr(j)) { - dns_name_t *name; - uint32_t ttl; - dns_rdata_t *rdata; + dns_name_t *name = NULL; + dns_rdata_t *rdata = NULL; dns_difftuple_t *tuple = NULL; + uint32_t ttl; - name = NULL; - rdata = NULL; dns_journal_current_rr(j, &name, &ttl, &rdata); if (rdata->type == dns_rdatatype_soa) { @@ -1450,13 +1577,12 @@ failure: isc_result_t dns_journal_rollforward(isc_mem_t *mctx, dns_db_t *db, unsigned int options, const char *filename) { - dns_journal_t *j; + dns_journal_t *j = NULL; isc_result_t result; REQUIRE(DNS_DB_VALID(db)); REQUIRE(filename != NULL); - j = NULL; result = dns_journal_open(mctx, filename, DNS_JOURNAL_READ, &j); if (result == ISC_R_NOTFOUND) { isc_log_write(JOURNAL_DEBUG_LOGARGS(3), "no journal file, but " @@ -1466,20 +1592,26 @@ dns_journal_rollforward(isc_mem_t *mctx, dns_db_t *db, unsigned int options, if (result != ISC_R_SUCCESS) { return (result); } + if (JOURNAL_EMPTY(&j->header)) { - result = DNS_R_UPTODATE; - } else { - result = roll_forward(j, db, options); + CHECK(DNS_R_UPTODATE); } - dns_journal_destroy(&j); + result = roll_forward(j, db, options); + if ((result == ISC_R_SUCCESS || result == DNS_R_UPTODATE) && + j->recovered) { + result = DNS_R_RECOVERABLE; + } +failure: + dns_journal_destroy(&j); return (result); } isc_result_t -dns_journal_print(isc_mem_t *mctx, const char *filename, FILE *file) { - dns_journal_t *j; +dns_journal_print(isc_mem_t *mctx, uint32_t flags, const char *filename, + FILE *file) { + dns_journal_t *j = NULL; isc_buffer_t source; /* Transaction data from disk */ isc_buffer_t target; /* Ditto after _fromwire check */ uint32_t start_serial; /* Database SOA serial */ @@ -1488,23 +1620,34 @@ dns_journal_print(isc_mem_t *mctx, const char *filename, FILE *file) { dns_diff_t diff; unsigned int n_soa = 0; unsigned int n_put = 0; + bool printxhdr = ((flags & DNS_JOURNAL_PRINTXHDR) != 0); REQUIRE(filename != NULL); - j = NULL; result = dns_journal_open(mctx, filename, DNS_JOURNAL_READ, &j); if (result == ISC_R_NOTFOUND) { isc_log_write(JOURNAL_DEBUG_LOGARGS(3), "no journal file"); return (DNS_R_NOJOURNAL); - } - - if (result != ISC_R_SUCCESS) { + } else if (result != ISC_R_SUCCESS) { isc_log_write(JOURNAL_COMMON_LOGARGS, ISC_LOG_ERROR, "journal open failure: %s: %s", isc_result_totext(result), filename); return (result); } + if (printxhdr) { + fprintf(file, "Journal format = %sHeader version = %d\n", + j->header.format + 1, j->header_ver1 ? 1 : 2); + fprintf(file, "Index (size = %u):\n", j->header.index_size); + for (uint32_t i = 0; i < j->header.index_size; i++) { + if (j->index[i].offset == 0) { + fputc('\n', file); + break; + } + fprintf(file, "%lld", (long long)j->index[i].offset); + fputc((i + 1) % 8 == 0 ? '\n' : ' ', file); + } + } if (j->header.serialset) { fprintf(file, "Source serial = %u\n", j->header.sourceserial); } @@ -1526,21 +1669,23 @@ dns_journal_print(isc_mem_t *mctx, const char *filename, FILE *file) { for (result = dns_journal_first_rr(j); result == ISC_R_SUCCESS; result = dns_journal_next_rr(j)) { - dns_name_t *name; - uint32_t ttl; - dns_rdata_t *rdata; + dns_name_t *name = NULL; + dns_rdata_t *rdata = NULL; dns_difftuple_t *tuple = NULL; + static uint32_t i = 0; + bool print = false; + uint32_t ttl; - name = NULL; - rdata = NULL; dns_journal_current_rr(j, &name, &ttl, &rdata); if (rdata->type == dns_rdatatype_soa) { n_soa++; - } - - if (n_soa == 3) { - n_soa = 1; + if (n_soa == 3) { + n_soa = 1; + } + if (n_soa == 1) { + print = printxhdr; + } } if (n_soa == 0) { isc_log_write(JOURNAL_COMMON_LOGARGS, ISC_LOG_ERROR, @@ -1549,12 +1694,29 @@ dns_journal_print(isc_mem_t *mctx, const char *filename, FILE *file) { j->filename); FAIL(ISC_R_UNEXPECTED); } + + if (print) { + fprintf(file, + "Transaction: version %d offset %lld size %u " + "rrcount %u start %u end %u\n", + j->xhdr_version, (long long)j->it.cpos.offset, + j->curxhdr.size, j->curxhdr.count, + j->curxhdr.serial0, j->curxhdr.serial1); + if (j->it.cpos.offset > j->index[i].offset) { + fprintf(file, + "ERROR: Offset mismatch, " + "expected %lld\n", + (long long)j->index[i].offset); + } else if (j->it.cpos.offset == j->index[i].offset) { + i++; + } + } CHECK(dns_difftuple_create( diff.mctx, n_soa == 1 ? DNS_DIFFOP_DEL : DNS_DIFFOP_ADD, name, ttl, rdata, &tuple)); dns_diff_append(&diff, &tuple); - if (++n_put > 100) { + if (++n_put != 0 || printxhdr) { result = dns_diff_print(&diff, file); dns_diff_clear(&diff); n_put = 0; @@ -1568,7 +1730,7 @@ dns_journal_print(isc_mem_t *mctx, const char *filename, FILE *file) { } CHECK(result); - if (n_put != 0) { + if (!printxhdr && n_put != 0) { result = dns_diff_print(&diff, file); dns_diff_clear(&diff); } @@ -1648,7 +1810,7 @@ dns_journal_get_sourceserial(dns_journal_t *j, uint32_t *sourceserial) { */ static isc_result_t -read_one_rr(dns_journal_t *j); +read_one_rr(dns_journal_t *j, bool retry); /* * Make sure the buffer 'b' is has at least 'size' bytes @@ -1706,7 +1868,7 @@ dns_journal_iter_init(dns_journal_t *j, uint32_t begin_serial, size += xhdr.size; count += xhdr.count; - result = journal_next(j, &pos); + result = journal_next(j, &pos, false); if (result == ISC_R_NOMORE) { result = ISC_R_SUCCESS; } @@ -1742,22 +1904,22 @@ dns_journal_first_rr(dns_journal_t *j) { j->it.xsize = 0; /* We have no transaction data yet... */ j->it.xpos = 0; /* ...and haven't used any of it. */ - return (read_one_rr(j)); + return (read_one_rr(j, false)); failure: return (result); } static isc_result_t -read_one_rr(dns_journal_t *j) { +read_one_rr(dns_journal_t *j, bool retry) { isc_result_t result; - dns_rdatatype_t rdtype; dns_rdataclass_t rdclass; unsigned int rdlen; uint32_t ttl; journal_xhdr_t xhdr; journal_rrhdr_t rrhdr; + dns_journal_t save = *j; if (j->offset > j->it.epos.offset) { isc_log_write(JOURNAL_COMMON_LOGARGS, ISC_LOG_ERROR, @@ -1780,7 +1942,34 @@ read_one_rr(dns_journal_t *j) { j->filename); FAIL(ISC_R_UNEXPECTED); } - if (xhdr.serial0 != j->it.current_serial) { + if (xhdr.serial0 != j->it.current_serial || + isc_serial_le(xhdr.serial1, xhdr.serial0)) + { + if (!retry && j->header_ver1 && + j->xhdr_version == XHDR_VERSION2 && + xhdr.count == j->it.current_serial) + { + /* XHDR_VERSION2 -> XHDR_VERSION1 */ + j->xhdr_version = XHDR_VERSION1; + CHECK(journal_seek(j, save.offset)); + result = read_one_rr(j, true); + if (result == ISC_R_SUCCESS) { + j->recovered = true; + } + return (result); + } else if (!retry && j->header_ver1 && + j->xhdr_version == XHDR_VERSION1 && + xhdr.serial1 == j->it.current_serial) + { + /* XHDR_VERSION1 -> XHDR_VERSION2 */ + j->xhdr_version = XHDR_VERSION2; + CHECK(journal_seek(j, save.offset)); + result = read_one_rr(j, true); + if (result == ISC_R_SUCCESS) { + j->recovered = true; + } + return (result); + } isc_log_write(JOURNAL_COMMON_LOGARGS, ISC_LOG_ERROR, "%s: journal file corrupt: " "expected serial %u, got %u", @@ -1871,7 +2060,7 @@ failure: isc_result_t dns_journal_next_rr(dns_journal_t *j) { - j->it.result = read_one_rr(j); + j->it.result = read_one_rr(j, false); return (j->it.result); } @@ -2200,16 +2389,33 @@ failure: return (result); } +static uint32_t +rrcount(char *buf, unsigned int size) { + isc_buffer_t b; + uint32_t rrsize, count = 0; + + isc_buffer_init(&b, buf, size); + isc_buffer_add(&b, size); + while (isc_buffer_remaininglength(&b) > 0) { + rrsize = isc_buffer_getuint32(&b); + INSIST(isc_buffer_remaininglength(&b) >= rrsize); + isc_buffer_forward(&b, rrsize); + count++; + } + + return (count); +} + isc_result_t dns_journal_compact(isc_mem_t *mctx, char *filename, uint32_t serial, - uint32_t target_size) { + uint32_t flags, uint32_t target_size) { unsigned int i; journal_pos_t best_guess; journal_pos_t current_pos; dns_journal_t *j1 = NULL; dns_journal_t *j2 = NULL; journal_rawheader_t rawheader; - unsigned int copy_length; + unsigned int len; size_t namelen; char *buf = NULL; unsigned int size = 0; @@ -2218,6 +2424,8 @@ dns_journal_compact(isc_mem_t *mctx, char *filename, uint32_t serial, char newname[PATH_MAX]; char backup[PATH_MAX]; bool is_backup = false; + bool rewrite = false; + bool downgrade = false; REQUIRE(filename != NULL); @@ -2234,16 +2442,26 @@ dns_journal_compact(isc_mem_t *mctx, char *filename, uint32_t serial, filename); RUNTIME_CHECK(result < sizeof(backup)); - result = journal_open(mctx, filename, false, false, &j1); + result = journal_open(mctx, filename, false, false, false, &j1); if (result == ISC_R_NOTFOUND) { is_backup = true; - result = journal_open(mctx, backup, false, false, &j1); + result = journal_open(mctx, backup, false, false, false, &j1); } if (result != ISC_R_SUCCESS) { return (result); } - if (JOURNAL_EMPTY(&j1->header)) { + /* + * Check whether we need to rewrite the whole journal + * file (for example, to upversion it). + */ + if ((flags & DNS_JOURNAL_COMPACTALL) != 0) { + if ((flags & DNS_JOURNAL_VERSION1) != 0) { + downgrade = true; + } + rewrite = true; + serial = dns_journal_first_serial(j1); + } else if (JOURNAL_EMPTY(&j1->header)) { dns_journal_destroy(&j1); return (ISC_R_SUCCESS); } @@ -2270,12 +2488,13 @@ dns_journal_compact(isc_mem_t *mctx, char *filename, uint32_t serial, /* * See if there is any work to do. */ - if ((uint32_t)j1->header.end.offset < target_size) { + if (!rewrite && (uint32_t)j1->header.end.offset < target_size) { dns_journal_destroy(&j1); return (ISC_R_SUCCESS); } - CHECK(journal_open(mctx, newname, true, true, &j2)); + CHECK(journal_open(mctx, newname, true, true, downgrade, &j2)); + CHECK(journal_seek(j2, indexend)); /* * Remove overhead so space test below can succeed. @@ -2301,7 +2520,7 @@ dns_journal_compact(isc_mem_t *mctx, char *filename, uint32_t serial, current_pos = best_guess; while (current_pos.serial != serial) { - CHECK(journal_next(j1, ¤t_pos)); + CHECK(journal_next(j1, ¤t_pos, false)); if (current_pos.serial == j1->header.end.serial) { break; } @@ -2319,7 +2538,7 @@ dns_journal_compact(isc_mem_t *mctx, char *filename, uint32_t serial, INSIST(best_guess.serial != j1->header.end.serial); if (best_guess.serial != serial) { - CHECK(journal_next(j1, &best_guess)); + CHECK(journal_next(j1, &best_guess, false)); } /* @@ -2327,40 +2546,98 @@ dns_journal_compact(isc_mem_t *mctx, char *filename, uint32_t serial, * we did not reach 'serial'. If not we will just copy * all uncommitted deltas regardless of the size. */ - copy_length = j1->header.end.offset - best_guess.offset; - - if (copy_length != 0) { - /* - * Copy best_guess to end into space just freed. - */ - size = 64 * 1024; - if (copy_length < size) { - size = copy_length; - } - buf = isc_mem_get(mctx, size); - + len = j1->header.end.offset - best_guess.offset; + if (len != 0) { CHECK(journal_seek(j1, best_guess.offset)); - CHECK(journal_seek(j2, indexend)); - for (i = 0; i < copy_length; i += size) { - unsigned int len = (copy_length - i) > size - ? size - : (copy_length - i); - CHECK(journal_read(j1, buf, len)); - CHECK(journal_write(j2, buf, len)); + + /* Prepare new header */ + j2->header.begin.serial = best_guess.serial; + j2->header.begin.offset = indexend; + j2->header.sourceserial = j1->header.sourceserial; + j2->header.serialset = j1->header.serialset; + j2->header.end.serial = j1->header.end.serial; + + /* + * Only use this method if we're rewriting the + * journal to fix outdated transaction headers; + * otherwise we'll copy the whole journal without + * parsing individual deltas below. + */ + while (rewrite && len > 0) { + journal_xhdr_t xhdr; + isc_offset_t offset = j1->offset; + uint32_t count; + + result = journal_read_xhdr(j1, &xhdr); + if (rewrite && result == ISC_R_NOMORE) { + break; + } + CHECK(result); + + /* + * If we're repairing an outdated journal, the + * xhdr format may be wrong. + */ + if (rewrite && + (xhdr.serial0 != serial || + isc_serial_le(xhdr.serial1, xhdr.serial0))) + { + if (j1->xhdr_version == XHDR_VERSION2 && + xhdr.count == serial) { + /* XHDR_VERSION2 -> XHDR_VERSION1 */ + j1->xhdr_version = XHDR_VERSION1; + CHECK(journal_seek(j1, offset)); + CHECK(journal_read_xhdr(j1, &xhdr)); + } else if (j1->xhdr_version == XHDR_VERSION1 && + xhdr.serial1 == serial) { + /* XHDR_VERSION1 -> XHDR_VERSION2 */ + j1->xhdr_version = XHDR_VERSION2; + CHECK(journal_seek(j1, offset)); + CHECK(journal_read_xhdr(j1, &xhdr)); + } + + /* Check again */ + if (xhdr.serial0 != serial || + isc_serial_le(xhdr.serial1, xhdr.serial0)) { + CHECK(ISC_R_UNEXPECTED); + } + } + + size = xhdr.size; + buf = isc_mem_get(mctx, size); + CHECK(journal_read(j1, buf, size)); + + count = rrcount(buf, size); + CHECK(journal_write_xhdr(j2, xhdr.size, count, + xhdr.serial0, xhdr.serial1)); + CHECK(journal_write(j2, buf, size)); + + j2->header.end.offset = j2->offset; + + serial = xhdr.serial1; + + len = j1->header.end.offset - j1->offset; + isc_mem_put(mctx, buf, size); + } + + /* + * If we're not rewriting transaction headers, we can use + * this faster method instead. + */ + if (!rewrite) { + size = ISC_MIN(64 * 1024, len); + buf = isc_mem_get(mctx, size); + for (i = 0; i < len; i += size) { + unsigned int blob = ISC_MIN(size, len - i); + CHECK(journal_read(j1, buf, blob)); + CHECK(journal_write(j2, buf, blob)); + } + + j2->header.end.offset = indexend + len; } CHECK(journal_fsync(j2)); - /* - * Compute new header. - */ - j2->header.begin.serial = best_guess.serial; - j2->header.begin.offset = indexend; - j2->header.end.serial = j1->header.end.serial; - j2->header.end.offset = indexend + copy_length; - j2->header.sourceserial = j1->header.sourceserial; - j2->header.serialset = j1->header.serialset; - /* * Update the journal header. */ @@ -2375,7 +2652,7 @@ dns_journal_compact(isc_mem_t *mctx, char *filename, uint32_t serial, current_pos = j2->header.begin; while (current_pos.serial != j2->header.end.serial) { index_add(j2, ¤t_pos); - CHECK(journal_next(j2, ¤t_pos)); + CHECK(journal_next(j2, ¤t_pos, false)); } /* diff --git a/lib/dns/zone.c b/lib/dns/zone.c index 5b8b1f1ddc..ae21050106 100644 --- a/lib/dns/zone.c +++ b/lib/dns/zone.c @@ -503,7 +503,10 @@ typedef enum { DNS_ZONEFLG_NEEDSTARTUPNOTIFY = 0x80000000U, /*%< need to send out * notify due to the zone * just being loaded for - * the first time. */ + * the first time. */ + DNS_ZONEFLG_FIXJOURNAL = 0x100000000U, /*%< journal file had + * recoverable error, + * needs rewriting */ DNS_ZONEFLG___MAX = UINT64_MAX, /* trick to make the ENUM 64-bit wide */ } dns_zoneflg_t; @@ -890,6 +893,8 @@ static isc_result_t zone_send_securedb(dns_zone_t *zone, dns_db_t *db); static void setrl(isc_ratelimiter_t *rl, unsigned int *rate, unsigned int value); +static void +zone_journal_compact(dns_zone_t *zone, dns_db_t *db, uint32_t serial); #define ENTER zone_debuglog(zone, me, 1, "enter") @@ -4743,6 +4748,7 @@ zone_postload(dns_zone_t *zone, dns_db_t *db, isc_time_t loadtime, uint32_t serial, oldserial, refresh, retry, expire, minimum; isc_time_t now; bool needdump = false; + bool fixjournal = false; bool hasinclude = DNS_ZONE_FLAG(zone, DNS_ZONEFLG_HASINCLUDE); bool nomaster = false; bool had_db = false; @@ -4844,9 +4850,9 @@ zone_postload(dns_zone_t *zone, dns_db_t *db, isc_time_t loadtime, } result = dns_journal_rollforward(zone->mctx, db, options, zone->journal); - if (result != ISC_R_SUCCESS && result != ISC_R_NOTFOUND && - result != DNS_R_UPTODATE && result != DNS_R_NOJOURNAL && - result != ISC_R_RANGE) + if (result != ISC_R_SUCCESS && result != DNS_R_RECOVERABLE && + result != ISC_R_NOTFOUND && result != DNS_R_UPTODATE && + result != DNS_R_NOJOURNAL && result != ISC_R_RANGE) { dns_zone_logc(zone, DNS_LOGCATEGORY_ZONELOAD, ISC_LOG_ERROR, @@ -4867,6 +4873,12 @@ zone_postload(dns_zone_t *zone, dns_db_t *db, isc_time_t loadtime, dns_result_totext(result)); if (result == ISC_R_SUCCESS) { needdump = true; + } else if (result == DNS_R_RECOVERABLE) { + dns_zone_logc(zone, DNS_LOGCATEGORY_ZONELOAD, + ISC_LOG_ERROR, + "retried using old journal format"); + needdump = true; + fixjournal = true; } } @@ -5192,7 +5204,13 @@ zone_postload(dns_zone_t *zone, dns_db_t *db, isc_time_t loadtime, if (zone->type == dns_zone_key) { zone_needdump(zone, 30); } else { - zone_needdump(zone, DNS_DUMP_DELAY); + if (fixjournal) { + DNS_ZONE_SETFLAG(zone, DNS_ZONEFLG_FIXJOURNAL); + zone_journal_compact(zone, zone->db, 0); + zone_needdump(zone, 0); + } else { + zone_needdump(zone, DNS_DUMP_DELAY); + } } } @@ -11435,6 +11453,7 @@ zone_journal_compact(dns_zone_t *zone, dns_db_t *db, uint32_t serial) { int32_t journalsize; dns_dbversion_t *ver = NULL; uint64_t dbsize; + uint32_t options = 0; INSIST(LOCKED_ZONE(zone)); if (inline_raw(zone)) { @@ -11456,9 +11475,16 @@ zone_journal_compact(dns_zone_t *zone, dns_db_t *db, uint32_t serial) { journalsize = (int32_t)dbsize * 2; } } - zone_debuglog(zone, "zone_journal_compact", 1, "target journal size %d", - journalsize); - result = dns_journal_compact(zone->mctx, zone->journal, serial, + if (DNS_ZONE_FLAG(zone, DNS_ZONEFLG_FIXJOURNAL)) { + options |= DNS_JOURNAL_COMPACTALL; + DNS_ZONE_CLRFLAG(zone, DNS_ZONEFLG_FIXJOURNAL); + zone_debuglog(zone, "zone_journal_compact", 1, + "repair full journal"); + } else { + zone_debuglog(zone, "zone_journal_compact", 1, + "target journal size %d", journalsize); + } + result = dns_journal_compact(zone->mctx, zone->journal, serial, options, journalsize); switch (result) { case ISC_R_SUCCESS: @@ -11486,6 +11512,7 @@ dns_zone_flush(dns_zone_t *zone) { DNS_ZONE_SETFLAG(zone, DNS_ZONEFLG_FLUSH); if (DNS_ZONE_FLAG(zone, DNS_ZONEFLG_NEEDDUMP) && zone->masterfile != NULL) { + DNS_ZONE_SETFLAG(zone, DNS_ZONEFLG_NEEDCOMPACT); result = ISC_R_ALREADYRUNNING; dumping = was_dumping(zone); } else { diff --git a/util/copyrights b/util/copyrights index ac7cb8e8d0..e992de8e8a 100644 --- a/util/copyrights +++ b/util/copyrights @@ -514,6 +514,18 @@ ./bin/tests/system/ixfr/ixfr-stats.good X 2019,2020,2021 ./bin/tests/system/ixfr/setup.sh SH 2001,2004,2007,2011,2012,2013,2014,2016,2018,2019,2020,2021 ./bin/tests/system/ixfr/tests.sh SH 2001,2004,2007,2011,2012,2014,2016,2018,2019,2020,2021 +./bin/tests/system/journal/clean.sh SH 2021 +./bin/tests/system/journal/ns1/changed.ver1.jnl.saved X 2021 +./bin/tests/system/journal/ns1/changed.ver2.jnl.saved X 2021 +./bin/tests/system/journal/ns1/d1212.jnl.saved X 2021 +./bin/tests/system/journal/ns1/d2121.jnl.saved X 2021 +./bin/tests/system/journal/ns1/ixfr.ver1.jnl.saved X 2021 +./bin/tests/system/journal/ns1/maxjournal.jnl.saved X 2021 +./bin/tests/system/journal/ns1/maxjournal2.jnl.saved X 2021 +./bin/tests/system/journal/ns1/unchanged.ver1.jnl.saved X 2021 +./bin/tests/system/journal/ns1/unchanged.ver2.jnl.saved X 2021 +./bin/tests/system/journal/setup.sh SH 2021 +./bin/tests/system/journal/tests.sh SH 2021 ./bin/tests/system/kasp/README TXT.BRIEF 2019,2020,2021 ./bin/tests/system/kasp/clean.sh SH 2019,2020,2021 ./bin/tests/system/kasp/ns2/setup.sh SH 2019,2020,2021