Merge branch '2505-journal-compatibility' into 'main'

allow dns_journal_rollforward() to read old journal files

Closes #2505

See merge request isc-projects/bind9!4720

CHANGES

@@ -1,3 +1,9 @@
+5593.	[bug]		Journal files written by older versions of named
+			can now be read when loading zones so that journal
+			incompatibility will not cause problems on upgrade.
+			Outdated journals will be updated to the new format
+			after loading. [GL #2505]
+
 5592.	[bug]		Add globally available thread_id (isc_tid_v) that's
 			incremented for each new thread, but the old thread
 			ids are reused, so the maximum thread_id always

bin/tests/system/Makefile.am

@@ -111,6 +111,7 @@ TESTS +=			\
 	inline			\
 	integrity		\
 	hooks			\
+	journal			\
 	keepalive		\
 	legacy			\
 	limits			\

bin/tests/system/journal/clean.sh

@@ -0,0 +1,14 @@
+# Copyright (C) Internet Systems Consortium, Inc. ("ISC")
+#
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, you can obtain one at https://mozilla.org/MPL/2.0/.
+#
+# See the COPYRIGHT file distributed with this work for additional
+# information regarding copyright ownership.
+
+rm -f */named.memstats
+rm -f */named.run
+rm -f */named.conf
+rm -f */*.db */*.jnl
+rm -f dig.out*

bin/tests/system/journal/ns1/generic.db.in

@@ -0,0 +1,16 @@
+; Copyright (C) Internet Systems Consortium, Inc. ("ISC")
+;
+; This Source Code Form is subject to the terms of the Mozilla Public
+; License, v. 2.0. If a copy of the MPL was not distributed with this
+; file, You can obtain one at http://mozilla.org/MPL/2.0/.
+;
+; See the COPYRIGHT file distributed with this work for additional
+; information regarding copyright ownership.
+
+
+$TTL 600
+@		SOA     ns hostmaster 2012010901 3600 1200 604800 1200
+                NS      ns
+ns		A       192.0.2.1
+
+addr1	A      10.53.0.1

bin/tests/system/journal/ns1/ixfr.db.in

@@ -0,0 +1,17 @@
+; Copyright (C) Internet Systems Consortium, Inc. ("ISC")
+;
+; This Source Code Form is subject to the terms of the Mozilla Public
+; License, v. 2.0. If a copy of the MPL was not distributed with this
+; file, You can obtain one at http://mozilla.org/MPL/2.0/.
+;
+; See the COPYRIGHT file distributed with this work for additional
+; information regarding copyright ownership.
+
+
+$TTL 600
+@		SOA     ns hostmaster 2012010902 3600 1200 604800 1200
+                NS      ns
+ns		A       192.0.2.1
+
+addr1	A      10.53.0.1
+addr2	A      10.53.0.2

bin/tests/system/journal/ns1/named.conf.in

@@ -0,0 +1,90 @@
+/*
+ * Copyright (C) Internet Systems Consortium, Inc. ("ISC")
+ *
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/.
+ *
+ * See the COPYRIGHT file distributed with this work for additional
+ * information regarding copyright ownership.
+ */
+
+options {
+	query-source address 10.53.0.1;
+	notify-source 10.53.0.1;
+	transfer-source 10.53.0.1;
+	port @PORT@;
+	session-keyfile "session.key";
+	pid-file "named.pid";
+	listen-on { 10.53.0.1; };
+	listen-on-v6 { none; };
+	dnssec-validation yes;
+	minimal-responses no;
+	recursion no;
+	notify yes;
+};
+
+key rndc_key {
+	secret "1234abcd8765";
+	algorithm hmac-sha256;
+};
+
+controls {
+	inet 10.53.0.1 port @CONTROLPORT@ allow { any; } keys { rndc_key; };
+};
+
+zone changed {
+	type primary;
+	update-policy local;
+	file "changed.db";
+};
+
+zone unchanged {
+	type primary;
+	update-policy local;
+	file "unchanged.db";
+};
+
+zone changed2 {
+	type primary;
+	update-policy local;
+	file "changed2.db";
+};
+
+zone unchanged2 {
+	type primary;
+	update-policy local;
+	file "unchanged2.db";
+};
+
+zone hdr1d1d2d1d2 {
+	type primary;
+	update-policy local;
+	file "d1212.db";
+};
+
+zone hdr1d2d1d2d1 {
+	type primary;
+	update-policy local;
+	file "d2121.db";
+};
+
+zone ixfr {
+	type primary;
+	ixfr-from-differences yes;
+	file "ixfr.db";
+};
+
+zone maxjournal {
+	type primary;
+	max-journal-size 1k;
+	update-policy local;
+	file "maxjournal.db";
+};
+
+zone maxjournal2 {
+	type primary;
+	max-journal-size 1k;
+	update-policy local;
+	file "maxjournal2.db";
+};

bin/tests/system/journal/setup.sh

@@ -0,0 +1,42 @@
+#!/bin/sh
+#
+# Copyright (C) Internet Systems Consortium, Inc. ("ISC")
+#
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, you can obtain one at https://mozilla.org/MPL/2.0/.
+#
+# See the COPYRIGHT file distributed with this work for additional
+# information regarding copyright ownership.
+
+. ../conf.sh
+
+$SHELL clean.sh
+
+copy_setports ns1/named.conf.in ns1/named.conf
+cp ns1/generic.db.in ns1/changed.db
+cp ns1/changed.ver1.jnl.saved ns1/changed.db.jnl
+
+cp ns1/generic.db.in ns1/unchanged.db
+cp ns1/unchanged.ver1.jnl.saved ns1/unchanged.db.jnl
+
+cp ns1/generic.db.in ns1/changed2.db
+cp ns1/changed.ver2.jnl.saved ns1/changed2.db.jnl
+
+cp ns1/generic.db.in ns1/unchanged2.db
+cp ns1/unchanged.ver2.jnl.saved ns1/unchanged2.db.jnl
+
+cp ns1/ixfr.db.in ns1/ixfr.db
+cp ns1/ixfr.ver1.jnl.saved ns1/ixfr.db.jnl
+
+cp ns1/generic.db.in ns1/d1212.db
+cp ns1/d1212.jnl.saved ns1/d1212.db.jnl
+
+cp ns1/generic.db.in ns1/d2121.db
+cp ns1/d2121.jnl.saved ns1/d2121.db.jnl
+
+cp ns1/generic.db.in ns1/maxjournal.db
+cp ns1/maxjournal.jnl.saved ns1/maxjournal.db.jnl
+
+cp ns1/generic.db.in ns1/maxjournal2.db
+cp ns1/maxjournal2.jnl.saved ns1/maxjournal2.db.jnl

bin/tests/system/journal/tests.sh

@@ -0,0 +1,201 @@
+#!/bin/sh
+#
+# Copyright (C) Internet Systems Consortium, Inc. ("ISC")
+#
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, you can obtain one at https://mozilla.org/MPL/2.0/.
+#
+# See the COPYRIGHT file distributed with this work for additional
+# information regarding copyright ownership.
+
+. ../conf.sh
+
+dig_with_opts() {
+    "$DIG" @10.53.0.1 -p "$PORT" +tcp "$@"
+}
+
+rndc_with_opts() {
+    "$RNDC" -c ../common/rndc.conf -p "$CONTROLPORT" -s "$@"
+}
+
+status=0
+n=0
+
+n=`expr $n + 1`
+echo_i "check outdated journal rolled forward (dynamic) ($n)"
+ret=0
+dig_with_opts changed soa > dig.out.test$n
+grep 'status: NOERROR' dig.out.test$n > /dev/null || ret=1
+grep '2012010902' dig.out.test$n > /dev/null || ret=1
+grep 'zone changed/IN: retried using old journal format' ns1/named.run > /dev/null || ret=1
+[ $ret -eq 0 ] || echo_i "failed"
+status=`expr $status + $ret`
+
+n=`expr $n + 1`
+echo_i "check outdated empty journal did not cause an error (dynamic) ($n)"
+ret=0
+dig_with_opts unchanged soa > dig.out.test$n
+grep 'status: NOERROR' dig.out.test$n > /dev/null || ret=1
+grep '2012010901' dig.out.test$n > /dev/null || ret=1
+[ $ret -eq 0 ] || echo_i "failed"
+status=`expr $status + $ret`
+
+n=`expr $n + 1`
+echo_i "check outdated journals were updated or removed (dynamic) ($n)"
+ret=0
+cat -v ns1/changed.db.jnl | grep "BIND LOG V9.2" > /dev/null || ret=1
+[ -f ns1/unchanged.db.jnl ] && ret=1
+[ $ret -eq 0 ] || echo_i "failed"
+status=`expr $status + $ret`
+
+n=`expr $n + 1`
+echo_i "check updated journal has correct RR count (dynamic) ($n)"
+ret=0
+$JOURNALPRINT -x ns1/changed.db.jnl | grep "rrcount 3 " > /dev/null || ret=1
+[ $ret -eq 0 ] || echo_i "failed"
+status=`expr $status + $ret`
+
+n=`expr $n + 1`
+echo_i "check new-format journal rolled forward (dynamic) ($n)"
+ret=0
+dig_with_opts changed2 soa > dig.out.test$n
+grep 'status: NOERROR' dig.out.test$n > /dev/null || ret=1
+grep '2012010902' dig.out.test$n > /dev/null || ret=1
+grep 'zone changed2/IN: retried using old journal format' ns1/named.run > /dev/null && ret=1
+[ $ret -eq 0 ] || echo_i "failed"
+status=`expr $status + $ret`
+
+n=`expr $n + 1`
+echo_i "check new-format empty journal did not cause error (dynamic) ($n)"
+ret=0
+dig_with_opts unchanged2 soa > dig.out.test$n
+grep 'status: NOERROR' dig.out.test$n > /dev/null || ret=1
+grep '2012010901' dig.out.test$n > /dev/null || ret=1
+grep 'zone unchanged2/IN: retried using old journal format' ns1/named.run > /dev/null && ret=1
+[ $ret -eq 0 ] || echo_i "failed"
+status=`expr $status + $ret`
+
+n=`expr $n + 1`
+echo_i "check new-format journals were updated or removed (dynamic) ($n)"
+ret=0
+cat -v ns1/changed2.db.jnl | grep "BIND LOG V9.2" > /dev/null || ret=1
+[ -f ns1/unchanged2.db.jnl ] && ret=1
+[ $ret -eq 0 ] || echo_i "failed"
+status=`expr $status + $ret`
+
+n=`expr $n + 1`
+echo_i "check outdated up-to-date journal succeeded (ixfr-from-differences) ($n)"
+ret=0
+dig_with_opts -t soa ixfr > dig.out.test$n
+grep 'status: NOERROR' dig.out.test$n > /dev/null || ret=1
+grep '2012010902' dig.out.test$n > /dev/null || ret=1
+grep 'zone ixfr/IN: journal rollforward completed successfully: recoverable' ns1/named.run > /dev/null || ret=1
+[ $ret -eq 0 ] || echo_i "failed"
+status=`expr $status + $ret`
+
+n=`expr $n + 1`
+echo_i "check outdated journal was updated (ixfr-from-differences) ($n)"
+ret=0
+cat -v ns1/ixfr.db.jnl | grep "BIND LOG V9.2" > /dev/null || ret=1
+[ $ret -eq 0 ] || echo_i "failed"
+status=`expr $status + $ret`
+
+n=`expr $n + 1`
+echo_i "check journal with mixed headers succeeded (version 1,2,1,2) ($n)"
+ret=0
+dig_with_opts -t soa hdr1d1d2d1d2 > dig.out.test$n
+grep 'status: NOERROR' dig.out.test$n > /dev/null || ret=1
+grep '2012010905' dig.out.test$n > /dev/null || ret=1
+grep 'zone hdr1d1d2d1d2/IN: journal rollforward completed successfully: recoverable' ns1/named.run > /dev/null || ret=1
+[ $ret -eq 0 ] || echo_i "failed"
+status=`expr $status + $ret`
+
+n=`expr $n + 1`
+echo_i "check journal with mixed headers was updated (version 1,2,1,2) ($n)"
+ret=0
+[ $($JOURNALPRINT -x ns1/d1212.jnl.saved | grep -c "version 1") -eq 2 ] || ret=1
+[ $($JOURNALPRINT -x ns1/d1212.jnl.saved | grep -c "version 2") -eq 2 ] || ret=1
+[ $($JOURNALPRINT -x ns1/d1212.db.jnl | grep -c "version 1") -eq 0 ] || ret=1
+[ $($JOURNALPRINT -x ns1/d1212.db.jnl | grep -c "version 2") -eq 4 ] || ret=1
+[ $ret -eq 0 ] || echo_i "failed"
+status=`expr $status + $ret`
+
+n=`expr $n + 1`
+echo_i "check journal with mixed headers succeeded (version 2,1,2,1) ($n)"
+ret=0
+dig_with_opts -t soa hdr1d2d1d2d1 > dig.out.test$n
+grep 'status: NOERROR' dig.out.test$n > /dev/null || ret=1
+grep '2012010905' dig.out.test$n > /dev/null || ret=1
+grep 'zone hdr1d2d1d2d1/IN: journal rollforward completed successfully: recoverable' ns1/named.run > /dev/null || ret=1
+[ $ret -eq 0 ] || echo_i "failed"
+status=`expr $status + $ret`
+
+n=`expr $n + 1`
+echo_i "check journal with mixed headers was updated (version 2,1,2,1) ($n)"
+ret=0
+[ $($JOURNALPRINT -x ns1/d2121.jnl.saved | grep -c "version 1") -eq 2 ] || ret=1
+[ $($JOURNALPRINT -x ns1/d2121.jnl.saved | grep -c "version 2") -eq 2 ] || ret=1
+[ $($JOURNALPRINT -x ns1/d2121.db.jnl | grep -c "version 1") -eq 0 ] || ret=1
+[ $($JOURNALPRINT -x ns1/d2121.db.jnl | grep -c "version 2") -eq 4 ] || ret=1
+[ $ret -eq 0 ] || echo_i "failed"
+status=`expr $status + $ret`
+
+n=`expr $n + 1`
+echo_i "check there are no journals left un-updated ($n)"
+ret=0
+c1=$(cat -v ns1/*.jnl | grep -c "BIND LOG V9")
+c2=$(cat -v ns1/*.jnl | grep -c "BIND LOG V9.2")
+[ ${c1} -eq ${c2} ] || ret=1
+[ $ret -eq 0 ] || echo_i "failed"
+status=`expr $status + $ret`
+
+n=`expr $n + 1`
+echo_i "check journal downgrade/upgrade ($n)"
+ret=0
+cp ns1/changed.db.jnl ns1/temp.jnl
+$JOURNALPRINT -d ns1/temp.jnl
+[ $($JOURNALPRINT -x ns1/temp.jnl | grep -c "version 1") -eq 1 ] || ret=1
+$JOURNALPRINT -x ns1/temp.jnl | grep -q "Header version = 1" || ret=1
+$JOURNALPRINT -u ns1/temp.jnl
+$JOURNALPRINT -x ns1/temp.jnl | grep -q "Header version = 2" || ret=1
+[ $($JOURNALPRINT -x ns1/temp.jnl | grep -c "version 2") -eq 1 ] || ret=1
+[ $ret -eq 0 ] || echo_i "failed"
+status=`expr $status + $ret`
+
+n=`expr $n + 1`
+echo_i "check max-journal-size works after journal update ($n)"
+ret=0
+# a dump should have been triggered by repairing the journal,
+# which would have resulted in the journal already being
+# compacted.
+[ $(wc -c < ns1/maxjournal.db.jnl) -lt 4000 ] || ret=1
+[ $ret -eq 0 ] || echo_i "failed"
+status=`expr $status + $ret`
+
+n=`expr $n + 1`
+echo_i "check max-journal-size works with non-updated journals ($n)"
+ret=0
+# journal was not repaired, so it should still be big
+[ $(wc -c < ns1/maxjournal2.db.jnl) -gt 12000 ] || ret=1
+# the zone hasn't been dumped yet, so 'rndc sync' should work without
+# needing a zone update first.
+rndc_with_opts 10.53.0.1 sync maxjournal2
+check_size() (
+    [ $(wc -c < ns1/maxjournal2.db.jnl) -lt 4000 ]
+)
+retry_quiet 10 check_size || ret=1
+[ $ret -eq 0 ] || echo_i "failed"
+status=`expr $status + $ret`
+
+n=`expr $n + 1`
+echo_i "check journal index consistency ($n)"
+ret=0
+for jnl in ns1/*.jnl; do
+    $JOURNALPRINT -x $jnl 2>&1 | grep -q "Offset mismatch" && ret=1
+done
+[ $ret -eq 0 ] || echo_i "failed"
+status=`expr $status + $ret`
+
+echo_i "exit status: $status"
+[ $status -eq 0 ] || exit 1

bin/tools/named-journalprint.c

@@ -13,6 +13,7 @@
 
 #include <stdlib.h>
 
+#include <isc/commandline.h>
 #include <isc/log.h>
 #include <isc/mem.h>
 #include <isc/print.h>
@@ -23,6 +24,14 @@
 #include <dns/result.h>
 #include <dns/types.h>
 
+const char *progname = NULL;
+
+static void
+usage(void) {
+	fprintf(stderr, "Usage: %s [-dux] journal\n", progname);
+	exit(1);
+}
+
 /*
  * Setup logging to use stderr.
  */
@@ -57,20 +66,50 @@ main(int argc, char **argv) {
 	isc_mem_t *mctx = NULL;
 	isc_result_t result;
 	isc_log_t *lctx = NULL;
+	uint32_t flags = 0U;
+	char ch;
+	bool downgrade = false;
+	bool upgrade = false;
 
-	if (argc != 2) {
-		printf("usage: %s journal\n", argv[0]);
-		return (1);
+	progname = argv[0];
+	while ((ch = isc_commandline_parse(argc, argv, "dux")) != -1) {
+		switch (ch) {
+		case 'd':
+			downgrade = true;
+			break;
+		case 'u':
+			upgrade = true;
+			break;
+		case 'x':
+			flags |= DNS_JOURNAL_PRINTXHDR;
+			break;
+		default:
+			usage();
+		}
 	}
 
-	file = argv[1];
+	argc -= isc_commandline_index;
+	argv += isc_commandline_index;
+
+	if (argc != 1) {
+		usage();
+	}
+	file = argv[0];
 
 	isc_mem_create(&mctx);
 	RUNTIME_CHECK(setup_logging(mctx, stderr, &lctx) == ISC_R_SUCCESS);
 
-	result = dns_journal_print(mctx, file, stdout);
-	if (result == DNS_R_NOJOURNAL) {
-		fprintf(stderr, "%s\n", dns_result_totext(result));
+	if (upgrade) {
+		flags = DNS_JOURNAL_COMPACTALL;
+		result = dns_journal_compact(mctx, file, 0, flags, 0);
+	} else if (downgrade) {
+		flags = DNS_JOURNAL_COMPACTALL | DNS_JOURNAL_VERSION1;
+		result = dns_journal_compact(mctx, file, 0, flags, 0);
+	} else {
+		result = dns_journal_print(mctx, flags, file, stdout);
+		if (result == DNS_R_NOJOURNAL) {
+			fprintf(stderr, "%s\n", dns_result_totext(result));
+		}
 	}
 	isc_log_destroy(&lctx);
 	isc_mem_detach(&mctx);

bin/tools/named-journalprint.rst

@@ -29,13 +29,14 @@ named-journalprint - print zone journal in human-readable form
 Synopsis
 ~~~~~~~~
 
-:program:`named-journalprint` {journal}
+:program:`named-journalprint` [**-dux**] {journal}
 
 Description
 ~~~~~~~~~~~
 
-``named-journalprint`` prints the contents of a zone journal file in a
-human-readable form.
+``named-journalprint`` scans the contents of a zone journal file,
+printing it in a human-readable form, or, optionally, converting it
+to a different journal file format.
 
 Journal files are automatically created by ``named`` when changes are
 made to dynamic zones (e.g., by ``nsupdate``). They record each addition
@@ -50,6 +51,17 @@ into a human-readable text format. Each line begins with ``add`` or ``del``,
 to indicate whether the record was added or deleted, and continues with
 the resource record in master-file format.
 
+The ``-x`` option causes additional data about the journal file to be
+printed at the beginning of the output and before each group of changes.
+
+The ``-u`` (upgrade) and ``-d`` (downgrade) options recreate the journal
+file with a modified format version.  The existing journal file is
+replaced.  ``-d`` writes out the journal in the format used by
+versions of BIND up to 9.16.11; ``-u`` writes it out in the format used
+by versions since 9.16.13. (9.16.12 is omitted due to a journal-formatting
+bug in that release.) Note that these options *must not* be used while
+``named`` is running.
+
 See Also
 ~~~~~~~~
 

doc/man/named-journalprint.1in

@@ -32,11 +32,12 @@ level margin: \\n[rst2man-indent\\n[rst2man-indent-level]]
 ..
 .SH SYNOPSIS
 .sp
-\fBnamed\-journalprint\fP {journal}
+\fBnamed\-journalprint\fP [\fB\-dux\fP] {journal}
 .SH DESCRIPTION
 .sp
-\fBnamed\-journalprint\fP prints the contents of a zone journal file in a
-human\-readable form.
+\fBnamed\-journalprint\fP scans the contents of a zone journal file,
+printing it in a human\-readable form, or, optionally, converting it
+to a different journal file format.
 .sp
 Journal files are automatically created by \fBnamed\fP when changes are
 made to dynamic zones (e.g., by \fBnsupdate\fP). They record each addition
@@ -50,6 +51,17 @@ file.
 into a human\-readable text format. Each line begins with \fBadd\fP or \fBdel\fP,
 to indicate whether the record was added or deleted, and continues with
 the resource record in master\-file format.
+.sp
+The \fB\-x\fP option causes additional data about the journal file to be
+printed at the beginning of the output and before each group of changes.
+.sp
+The \fB\-u\fP (upgrade) and \fB\-d\fP (downgrade) options recreate the journal
+file with a modified format version.  The existing journal file is
+replaced.  \fB\-d\fP writes out the journal in the format used by
+versions of BIND up to 9.16.11; \fB\-u\fP writes it out in the format used
+by versions since 9.16.13. (9.16.12 is omitted due to a journal\-formatting
+bug in that release.) Note that these options \fImust not\fP be used while
+\fBnamed\fP is running.
 .SH SEE ALSO
 .sp
 \fBnamed(8)\fP, \fBnsupdate(1)\fP, BIND 9 Administrator Reference Manual.

doc/notes/notes-current.rst

@@ -64,3 +64,20 @@ Bug Fixes
   (crash) if the return of stale cached answers was enabled and
   ``stale-answer-client-timeout`` was applied to a client query in process.
   This has been fixed. [GL #2503]
+
+- Zone journal (``.jnl``) files created by versions of ``named`` prior
+  to 9.16.12 were no longer compatible; this could cause problems when
+  upgrading if journal files were not synchronized first.  This has been
+  corrected: older journal files can now be read when starting up.  When
+  an old-style journal file is detected, it is updated to the new
+  format immediately after loading.
+
+  Note that journals created by the current version of ``named`` are not
+  usable by versions prior to 9.16.12. Before downgrading to a prior
+  release, users are advised to ensure that all dynamic zones have been
+  synchronized using ``rndc sync -clean``.
+
+  A journal file's format can be changed manually by running
+  ``named-journalprint -d`` (downgrade) or ``named-journalprint -u``
+  (upgrade). Note that this *must not* be done while ``named`` is
+  running.  [GL #2505]

lib/dns/include/dns/events.h

@@ -79,6 +79,7 @@
 #define DNS_EVENT_RPZUPDATED	     (ISC_EVENTCLASS_DNS + 57)
 #define DNS_EVENT_STARTUPDATE	     (ISC_EVENTCLASS_DNS + 58)
 #define DNS_EVENT_TRYSTALE	     (ISC_EVENTCLASS_DNS + 59)
+#define DNS_EVENT_ZONEFLUSH	     (ISC_EVENTCLASS_DNS + 60)
 
 #define DNS_EVENT_FIRSTEVENT (ISC_EVENTCLASS_DNS + 0)
 #define DNS_EVENT_LASTEVENT  (ISC_EVENTCLASS_DNS + 65535)

lib/dns/include/dns/journal.h

@@ -48,6 +48,13 @@
 #define DNS_JOURNAL_SIZE_MAX INT32_MAX
 #define DNS_JOURNAL_SIZE_MIN 4096
 
+/*% Print transaction header data */
+#define DNS_JOURNAL_PRINTXHDR 0x0001
+
+/*% Rewrite whole journal file instead of compacting */
+#define DNS_JOURNAL_COMPACTALL 0x0001
+#define DNS_JOURNAL_VERSION1   0x0002
+
 /***
  *** Types
  ***/
@@ -258,12 +265,18 @@ dns_journal_rollforward(isc_mem_t *mctx, dns_db_t *db, unsigned int options,
  *\li	DNS_R_NOJOURNAL when journal does not exist.
  *\li	ISC_R_NOTFOUND when current serial in not in journal.
  *\li	ISC_R_RANGE when current serial in not in journals range.
- *\li	ISC_R_SUCCESS journal has been applied successfully to database.
+ *\li	DNS_R_UPTODATE when the database was already up to date.
+ *\li	ISC_R_SUCCESS journal has been applied successfully to the
+ *      database without any issues.
+ *\li	DNS_R_RECOVERABLE if successful or up to date, but the journal
+ *      was found to contain at least one outdated transaction header.
+ *
  *	others
  */
 
 isc_result_t
-dns_journal_print(isc_mem_t *mctx, const char *filename, FILE *file);
+dns_journal_print(isc_mem_t *mctx, uint32_t flags, const char *filename,
+		  FILE *file);
 /* For debugging not general use */
 
 isc_result_t
@@ -286,7 +299,7 @@ dns_db_diffx(dns_diff_t *diff, dns_db_t *dba, dns_dbversion_t *dbvera,
 
 isc_result_t
 dns_journal_compact(isc_mem_t *mctx, char *filename, uint32_t serial,
-		    uint32_t target_size);
+		    uint32_t flags, uint32_t target_size);
 /*%<
  * Attempt to compact the journal if it is greater that 'target_size'.
  * Changes from 'serial' onwards will be preserved.  If the journal

lib/dns/zone.c

@@ -503,7 +503,10 @@ typedef enum {
 	DNS_ZONEFLG_NEEDSTARTUPNOTIFY = 0x80000000U, /*%< need to send out
 						      * notify due to the zone
 						      * just being loaded for
-						      * the first time.  */
+						      * the first time. */
+	DNS_ZONEFLG_FIXJOURNAL = 0x100000000U,	     /*%< journal file had
+						      * recoverable error,
+						      * needs rewriting */
 	DNS_ZONEFLG___MAX = UINT64_MAX, /* trick to make the ENUM 64-bit wide */
 } dns_zoneflg_t;
 
@@ -890,6 +893,8 @@ static isc_result_t
 zone_send_securedb(dns_zone_t *zone, dns_db_t *db);
 static void
 setrl(isc_ratelimiter_t *rl, unsigned int *rate, unsigned int value);
+static void
+zone_journal_compact(dns_zone_t *zone, dns_db_t *db, uint32_t serial);
 
 #define ENTER zone_debuglog(zone, me, 1, "enter")
 
@@ -4743,6 +4748,7 @@ zone_postload(dns_zone_t *zone, dns_db_t *db, isc_time_t loadtime,
 	uint32_t serial, oldserial, refresh, retry, expire, minimum;
 	isc_time_t now;
 	bool needdump = false;
+	bool fixjournal = false;
 	bool hasinclude = DNS_ZONE_FLAG(zone, DNS_ZONEFLG_HASINCLUDE);
 	bool nomaster = false;
 	bool had_db = false;
@@ -4844,9 +4850,9 @@ zone_postload(dns_zone_t *zone, dns_db_t *db, isc_time_t loadtime,
 		}
 		result = dns_journal_rollforward(zone->mctx, db, options,
 						 zone->journal);
-		if (result != ISC_R_SUCCESS && result != ISC_R_NOTFOUND &&
-		    result != DNS_R_UPTODATE && result != DNS_R_NOJOURNAL &&
-		    result != ISC_R_RANGE)
+		if (result != ISC_R_SUCCESS && result != DNS_R_RECOVERABLE &&
+		    result != ISC_R_NOTFOUND && result != DNS_R_UPTODATE &&
+		    result != DNS_R_NOJOURNAL && result != ISC_R_RANGE)
 		{
 			dns_zone_logc(zone, DNS_LOGCATEGORY_ZONELOAD,
 				      ISC_LOG_ERROR,
@@ -4867,6 +4873,12 @@ zone_postload(dns_zone_t *zone, dns_db_t *db, isc_time_t loadtime,
 			      dns_result_totext(result));
 		if (result == ISC_R_SUCCESS) {
 			needdump = true;
+		} else if (result == DNS_R_RECOVERABLE) {
+			dns_zone_logc(zone, DNS_LOGCATEGORY_ZONELOAD,
+				      ISC_LOG_ERROR,
+				      "retried using old journal format");
+			needdump = true;
+			fixjournal = true;
 		}
 	}
 
@@ -5192,7 +5204,13 @@ zone_postload(dns_zone_t *zone, dns_db_t *db, isc_time_t loadtime,
 		if (zone->type == dns_zone_key) {
 			zone_needdump(zone, 30);
 		} else {
-			zone_needdump(zone, DNS_DUMP_DELAY);
+			if (fixjournal) {
+				DNS_ZONE_SETFLAG(zone, DNS_ZONEFLG_FIXJOURNAL);
+				zone_journal_compact(zone, zone->db, 0);
+				zone_needdump(zone, 0);
+			} else {
+				zone_needdump(zone, DNS_DUMP_DELAY);
+			}
 		}
 	}
 
@@ -11435,6 +11453,7 @@ zone_journal_compact(dns_zone_t *zone, dns_db_t *db, uint32_t serial) {
 	int32_t journalsize;
 	dns_dbversion_t *ver = NULL;
 	uint64_t dbsize;
+	uint32_t options = 0;
 
 	INSIST(LOCKED_ZONE(zone));
 	if (inline_raw(zone)) {
@@ -11456,9 +11475,16 @@ zone_journal_compact(dns_zone_t *zone, dns_db_t *db, uint32_t serial) {
 			journalsize = (int32_t)dbsize * 2;
 		}
 	}
-	zone_debuglog(zone, "zone_journal_compact", 1, "target journal size %d",
-		      journalsize);
-	result = dns_journal_compact(zone->mctx, zone->journal, serial,
+	if (DNS_ZONE_FLAG(zone, DNS_ZONEFLG_FIXJOURNAL)) {
+		options |= DNS_JOURNAL_COMPACTALL;
+		DNS_ZONE_CLRFLAG(zone, DNS_ZONEFLG_FIXJOURNAL);
+		zone_debuglog(zone, "zone_journal_compact", 1,
+			      "repair full journal");
+	} else {
+		zone_debuglog(zone, "zone_journal_compact", 1,
+			      "target journal size %d", journalsize);
+	}
+	result = dns_journal_compact(zone->mctx, zone->journal, serial, options,
 				     journalsize);
 	switch (result) {
 	case ISC_R_SUCCESS:
@@ -11486,6 +11512,7 @@ dns_zone_flush(dns_zone_t *zone) {
 	DNS_ZONE_SETFLAG(zone, DNS_ZONEFLG_FLUSH);
 	if (DNS_ZONE_FLAG(zone, DNS_ZONEFLG_NEEDDUMP) &&
 	    zone->masterfile != NULL) {
+		DNS_ZONE_SETFLAG(zone, DNS_ZONEFLG_NEEDCOMPACT);
 		result = ISC_R_ALREADYRUNNING;
 		dumping = was_dumping(zone);
 	} else {

util/copyrights

@@ -514,6 +514,18 @@
 ./bin/tests/system/ixfr/ixfr-stats.good		X	2019,2020,2021
 ./bin/tests/system/ixfr/setup.sh		SH	2001,2004,2007,2011,2012,2013,2014,2016,2018,2019,2020,2021
 ./bin/tests/system/ixfr/tests.sh		SH	2001,2004,2007,2011,2012,2014,2016,2018,2019,2020,2021
+./bin/tests/system/journal/clean.sh		SH	2021
+./bin/tests/system/journal/ns1/changed.ver1.jnl.saved	X	2021
+./bin/tests/system/journal/ns1/changed.ver2.jnl.saved	X	2021
+./bin/tests/system/journal/ns1/d1212.jnl.saved	X	2021
+./bin/tests/system/journal/ns1/d2121.jnl.saved	X	2021
+./bin/tests/system/journal/ns1/ixfr.ver1.jnl.saved	X	2021
+./bin/tests/system/journal/ns1/maxjournal.jnl.saved	X	2021
+./bin/tests/system/journal/ns1/maxjournal2.jnl.saved	X	2021
+./bin/tests/system/journal/ns1/unchanged.ver1.jnl.saved	X	2021
+./bin/tests/system/journal/ns1/unchanged.ver2.jnl.saved	X	2021
+./bin/tests/system/journal/setup.sh		SH	2021
+./bin/tests/system/journal/tests.sh		SH	2021
 ./bin/tests/system/kasp/README			TXT.BRIEF	2019,2020,2021
 ./bin/tests/system/kasp/clean.sh		SH	2019,2020,2021
 ./bin/tests/system/kasp/ns2/setup.sh		SH	2019,2020,2021