Merge branch '2505-journal-compatibility' into 'main'

allow dns_journal_rollforward() to read old journal files

Closes #2505

See merge request isc-projects/bind9!4720

CHANGES

@@ -1,3 +1,9 @@
+5593.	[bug]		Journal files written by older versions of named
+			can now be read when loading zones so that journal
+			incompatibility will not cause problems on upgrade.
+			Outdated journals will be updated to the new format
+			after loading. [GL #2505]
+
 5592.	[bug]		Add globally available thread_id (isc_tid_v) that's
 			incremented for each new thread, but the old thread
 			ids are reused, so the maximum thread_id always

bin/tests/system/Makefile.am

@@ -111,6 +111,7 @@ TESTS +=			\
 	inline			\
 	integrity		\
 	hooks			\
+	journal			\
 	keepalive		\
 	legacy			\
 	limits			\

bin/tests/system/journal/clean.sh

@@ -0,0 +1,14 @@
+# Copyright (C) Internet Systems Consortium, Inc. ("ISC")
+#
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, you can obtain one at https://mozilla.org/MPL/2.0/.
+#
+# See the COPYRIGHT file distributed with this work for additional
+# information regarding copyright ownership.
+
+rm -f */named.memstats
+rm -f */named.run
+rm -f */named.conf
+rm -f */*.db */*.jnl
+rm -f dig.out*

bin/tests/system/journal/ns1/generic.db.in

@@ -0,0 +1,16 @@
+; Copyright (C) Internet Systems Consortium, Inc. ("ISC")
+;
+; This Source Code Form is subject to the terms of the Mozilla Public
+; License, v. 2.0. If a copy of the MPL was not distributed with this
+; file, You can obtain one at http://mozilla.org/MPL/2.0/.
+;
+; See the COPYRIGHT file distributed with this work for additional
+; information regarding copyright ownership.
+
+
+$TTL 600
+@		SOA     ns hostmaster 2012010901 3600 1200 604800 1200
+                NS      ns
+ns		A       192.0.2.1
+
+addr1	A      10.53.0.1

bin/tests/system/journal/ns1/ixfr.db.in

@@ -0,0 +1,17 @@
+; Copyright (C) Internet Systems Consortium, Inc. ("ISC")
+;
+; This Source Code Form is subject to the terms of the Mozilla Public
+; License, v. 2.0. If a copy of the MPL was not distributed with this
+; file, You can obtain one at http://mozilla.org/MPL/2.0/.
+;
+; See the COPYRIGHT file distributed with this work for additional
+; information regarding copyright ownership.
+
+
+$TTL 600
+@		SOA     ns hostmaster 2012010902 3600 1200 604800 1200
+                NS      ns
+ns		A       192.0.2.1
+
+addr1	A      10.53.0.1
+addr2	A      10.53.0.2

bin/tests/system/journal/ns1/named.conf.in

@@ -0,0 +1,90 @@
+/*
+ * Copyright (C) Internet Systems Consortium, Inc. ("ISC")
+ *
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/.
+ *
+ * See the COPYRIGHT file distributed with this work for additional
+ * information regarding copyright ownership.
+ */
+
+options {
+	query-source address 10.53.0.1;
+	notify-source 10.53.0.1;
+	transfer-source 10.53.0.1;
+	port @PORT@;
+	session-keyfile "session.key";
+	pid-file "named.pid";
+	listen-on { 10.53.0.1; };
+	listen-on-v6 { none; };
+	dnssec-validation yes;
+	minimal-responses no;
+	recursion no;
+	notify yes;
+};
+
+key rndc_key {
+	secret "1234abcd8765";
+	algorithm hmac-sha256;
+};
+
+controls {
+	inet 10.53.0.1 port @CONTROLPORT@ allow { any; } keys { rndc_key; };
+};
+
+zone changed {
+	type primary;
+	update-policy local;
+	file "changed.db";
+};
+
+zone unchanged {
+	type primary;
+	update-policy local;
+	file "unchanged.db";
+};
+
+zone changed2 {
+	type primary;
+	update-policy local;
+	file "changed2.db";
+};
+
+zone unchanged2 {
+	type primary;
+	update-policy local;
+	file "unchanged2.db";
+};
+
+zone hdr1d1d2d1d2 {
+	type primary;
+	update-policy local;
+	file "d1212.db";
+};
+
+zone hdr1d2d1d2d1 {
+	type primary;
+	update-policy local;
+	file "d2121.db";
+};
+
+zone ixfr {
+	type primary;
+	ixfr-from-differences yes;
+	file "ixfr.db";
+};
+
+zone maxjournal {
+	type primary;
+	max-journal-size 1k;
+	update-policy local;
+	file "maxjournal.db";
+};
+
+zone maxjournal2 {
+	type primary;
+	max-journal-size 1k;
+	update-policy local;
+	file "maxjournal2.db";
+};

bin/tests/system/journal/setup.sh

@@ -0,0 +1,42 @@
+#!/bin/sh
+#
+# Copyright (C) Internet Systems Consortium, Inc. ("ISC")
+#
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, you can obtain one at https://mozilla.org/MPL/2.0/.
+#
+# See the COPYRIGHT file distributed with this work for additional
+# information regarding copyright ownership.
+
+. ../conf.sh
+
+$SHELL clean.sh
+
+copy_setports ns1/named.conf.in ns1/named.conf
+cp ns1/generic.db.in ns1/changed.db
+cp ns1/changed.ver1.jnl.saved ns1/changed.db.jnl
+
+cp ns1/generic.db.in ns1/unchanged.db
+cp ns1/unchanged.ver1.jnl.saved ns1/unchanged.db.jnl
+
+cp ns1/generic.db.in ns1/changed2.db
+cp ns1/changed.ver2.jnl.saved ns1/changed2.db.jnl
+
+cp ns1/generic.db.in ns1/unchanged2.db
+cp ns1/unchanged.ver2.jnl.saved ns1/unchanged2.db.jnl
+
+cp ns1/ixfr.db.in ns1/ixfr.db
+cp ns1/ixfr.ver1.jnl.saved ns1/ixfr.db.jnl
+
+cp ns1/generic.db.in ns1/d1212.db
+cp ns1/d1212.jnl.saved ns1/d1212.db.jnl
+
+cp ns1/generic.db.in ns1/d2121.db
+cp ns1/d2121.jnl.saved ns1/d2121.db.jnl
+
+cp ns1/generic.db.in ns1/maxjournal.db
+cp ns1/maxjournal.jnl.saved ns1/maxjournal.db.jnl
+
+cp ns1/generic.db.in ns1/maxjournal2.db
+cp ns1/maxjournal2.jnl.saved ns1/maxjournal2.db.jnl

bin/tests/system/journal/tests.sh

@@ -0,0 +1,201 @@
+#!/bin/sh
+#
+# Copyright (C) Internet Systems Consortium, Inc. ("ISC")
+#
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, you can obtain one at https://mozilla.org/MPL/2.0/.
+#
+# See the COPYRIGHT file distributed with this work for additional
+# information regarding copyright ownership.
+
+. ../conf.sh
+
+dig_with_opts() {
+    "$DIG" @10.53.0.1 -p "$PORT" +tcp "$@"
+}
+
+rndc_with_opts() {
+    "$RNDC" -c ../common/rndc.conf -p "$CONTROLPORT" -s "$@"
+}
+
+status=0
+n=0
+
+n=`expr $n + 1`
+echo_i "check outdated journal rolled forward (dynamic) ($n)"
+ret=0
+dig_with_opts changed soa > dig.out.test$n
+grep 'status: NOERROR' dig.out.test$n > /dev/null || ret=1
+grep '2012010902' dig.out.test$n > /dev/null || ret=1
+grep 'zone changed/IN: retried using old journal format' ns1/named.run > /dev/null || ret=1
+[ $ret -eq 0 ] || echo_i "failed"
+status=`expr $status + $ret`
+
+n=`expr $n + 1`
+echo_i "check outdated empty journal did not cause an error (dynamic) ($n)"
+ret=0
+dig_with_opts unchanged soa > dig.out.test$n
+grep 'status: NOERROR' dig.out.test$n > /dev/null || ret=1
+grep '2012010901' dig.out.test$n > /dev/null || ret=1
+[ $ret -eq 0 ] || echo_i "failed"
+status=`expr $status + $ret`
+
+n=`expr $n + 1`
+echo_i "check outdated journals were updated or removed (dynamic) ($n)"
+ret=0
+cat -v ns1/changed.db.jnl | grep "BIND LOG V9.2" > /dev/null || ret=1
+[ -f ns1/unchanged.db.jnl ] && ret=1
+[ $ret -eq 0 ] || echo_i "failed"
+status=`expr $status + $ret`
+
+n=`expr $n + 1`
+echo_i "check updated journal has correct RR count (dynamic) ($n)"
+ret=0
+$JOURNALPRINT -x ns1/changed.db.jnl | grep "rrcount 3 " > /dev/null || ret=1
+[ $ret -eq 0 ] || echo_i "failed"
+status=`expr $status + $ret`
+
+n=`expr $n + 1`
+echo_i "check new-format journal rolled forward (dynamic) ($n)"
+ret=0
+dig_with_opts changed2 soa > dig.out.test$n
+grep 'status: NOERROR' dig.out.test$n > /dev/null || ret=1
+grep '2012010902' dig.out.test$n > /dev/null || ret=1
+grep 'zone changed2/IN: retried using old journal format' ns1/named.run > /dev/null && ret=1
+[ $ret -eq 0 ] || echo_i "failed"
+status=`expr $status + $ret`
+
+n=`expr $n + 1`
+echo_i "check new-format empty journal did not cause error (dynamic) ($n)"
+ret=0
+dig_with_opts unchanged2 soa > dig.out.test$n
+grep 'status: NOERROR' dig.out.test$n > /dev/null || ret=1
+grep '2012010901' dig.out.test$n > /dev/null || ret=1
+grep 'zone unchanged2/IN: retried using old journal format' ns1/named.run > /dev/null && ret=1
+[ $ret -eq 0 ] || echo_i "failed"
+status=`expr $status + $ret`
+
+n=`expr $n + 1`
+echo_i "check new-format journals were updated or removed (dynamic) ($n)"
+ret=0
+cat -v ns1/changed2.db.jnl | grep "BIND LOG V9.2" > /dev/null || ret=1
+[ -f ns1/unchanged2.db.jnl ] && ret=1
+[ $ret -eq 0 ] || echo_i "failed"
+status=`expr $status + $ret`
+
+n=`expr $n + 1`
+echo_i "check outdated up-to-date journal succeeded (ixfr-from-differences) ($n)"
+ret=0
+dig_with_opts -t soa ixfr > dig.out.test$n
+grep 'status: NOERROR' dig.out.test$n > /dev/null || ret=1
+grep '2012010902' dig.out.test$n > /dev/null || ret=1
+grep 'zone ixfr/IN: journal rollforward completed successfully: recoverable' ns1/named.run > /dev/null || ret=1
+[ $ret -eq 0 ] || echo_i "failed"
+status=`expr $status + $ret`
+
+n=`expr $n + 1`
+echo_i "check outdated journal was updated (ixfr-from-differences) ($n)"
+ret=0
+cat -v ns1/ixfr.db.jnl | grep "BIND LOG V9.2" > /dev/null || ret=1
+[ $ret -eq 0 ] || echo_i "failed"
+status=`expr $status + $ret`
+
+n=`expr $n + 1`
+echo_i "check journal with mixed headers succeeded (version 1,2,1,2) ($n)"
+ret=0
+dig_with_opts -t soa hdr1d1d2d1d2 > dig.out.test$n
+grep 'status: NOERROR' dig.out.test$n > /dev/null || ret=1
+grep '2012010905' dig.out.test$n > /dev/null || ret=1
+grep 'zone hdr1d1d2d1d2/IN: journal rollforward completed successfully: recoverable' ns1/named.run > /dev/null || ret=1
+[ $ret -eq 0 ] || echo_i "failed"
+status=`expr $status + $ret`
+
+n=`expr $n + 1`
+echo_i "check journal with mixed headers was updated (version 1,2,1,2) ($n)"
+ret=0
+[ $($JOURNALPRINT -x ns1/d1212.jnl.saved | grep -c "version 1") -eq 2 ] || ret=1
+[ $($JOURNALPRINT -x ns1/d1212.jnl.saved | grep -c "version 2") -eq 2 ] || ret=1
+[ $($JOURNALPRINT -x ns1/d1212.db.jnl | grep -c "version 1") -eq 0 ] || ret=1
+[ $($JOURNALPRINT -x ns1/d1212.db.jnl | grep -c "version 2") -eq 4 ] || ret=1
+[ $ret -eq 0 ] || echo_i "failed"
+status=`expr $status + $ret`
+
+n=`expr $n + 1`
+echo_i "check journal with mixed headers succeeded (version 2,1,2,1) ($n)"
+ret=0
+dig_with_opts -t soa hdr1d2d1d2d1 > dig.out.test$n
+grep 'status: NOERROR' dig.out.test$n > /dev/null || ret=1
+grep '2012010905' dig.out.test$n > /dev/null || ret=1
+grep 'zone hdr1d2d1d2d1/IN: journal rollforward completed successfully: recoverable' ns1/named.run > /dev/null || ret=1
+[ $ret -eq 0 ] || echo_i "failed"
+status=`expr $status + $ret`
+
+n=`expr $n + 1`
+echo_i "check journal with mixed headers was updated (version 2,1,2,1) ($n)"
+ret=0
+[ $($JOURNALPRINT -x ns1/d2121.jnl.saved | grep -c "version 1") -eq 2 ] || ret=1
+[ $($JOURNALPRINT -x ns1/d2121.jnl.saved | grep -c "version 2") -eq 2 ] || ret=1
+[ $($JOURNALPRINT -x ns1/d2121.db.jnl | grep -c "version 1") -eq 0 ] || ret=1
+[ $($JOURNALPRINT -x ns1/d2121.db.jnl | grep -c "version 2") -eq 4 ] || ret=1
+[ $ret -eq 0 ] || echo_i "failed"
+status=`expr $status + $ret`
+
+n=`expr $n + 1`
+echo_i "check there are no journals left un-updated ($n)"
+ret=0
+c1=$(cat -v ns1/*.jnl | grep -c "BIND LOG V9")
+c2=$(cat -v ns1/*.jnl | grep -c "BIND LOG V9.2")
+[ ${c1} -eq ${c2} ] || ret=1
+[ $ret -eq 0 ] || echo_i "failed"
+status=`expr $status + $ret`
+
+n=`expr $n + 1`
+echo_i "check journal downgrade/upgrade ($n)"
+ret=0
+cp ns1/changed.db.jnl ns1/temp.jnl
+$JOURNALPRINT -d ns1/temp.jnl
+[ $($JOURNALPRINT -x ns1/temp.jnl | grep -c "version 1") -eq 1 ] || ret=1
+$JOURNALPRINT -x ns1/temp.jnl | grep -q "Header version = 1" || ret=1
+$JOURNALPRINT -u ns1/temp.jnl
+$JOURNALPRINT -x ns1/temp.jnl | grep -q "Header version = 2" || ret=1
+[ $($JOURNALPRINT -x ns1/temp.jnl | grep -c "version 2") -eq 1 ] || ret=1
+[ $ret -eq 0 ] || echo_i "failed"
+status=`expr $status + $ret`
+
+n=`expr $n + 1`
+echo_i "check max-journal-size works after journal update ($n)"
+ret=0
+# a dump should have been triggered by repairing the journal,
+# which would have resulted in the journal already being
+# compacted.
+[ $(wc -c < ns1/maxjournal.db.jnl) -lt 4000 ] || ret=1
+[ $ret -eq 0 ] || echo_i "failed"
+status=`expr $status + $ret`
+
+n=`expr $n + 1`
+echo_i "check max-journal-size works with non-updated journals ($n)"
+ret=0
+# journal was not repaired, so it should still be big
+[ $(wc -c < ns1/maxjournal2.db.jnl) -gt 12000 ] || ret=1
+# the zone hasn't been dumped yet, so 'rndc sync' should work without
+# needing a zone update first.
+rndc_with_opts 10.53.0.1 sync maxjournal2
+check_size() (
+    [ $(wc -c < ns1/maxjournal2.db.jnl) -lt 4000 ]
+)
+retry_quiet 10 check_size || ret=1
+[ $ret -eq 0 ] || echo_i "failed"
+status=`expr $status + $ret`
+
+n=`expr $n + 1`
+echo_i "check journal index consistency ($n)"
+ret=0
+for jnl in ns1/*.jnl; do
+    $JOURNALPRINT -x $jnl 2>&1 | grep -q "Offset mismatch" && ret=1
+done
+[ $ret -eq 0 ] || echo_i "failed"
+status=`expr $status + $ret`
+
+echo_i "exit status: $status"
+[ $status -eq 0 ] || exit 1

bin/tools/named-journalprint.c

@@ -13,6 +13,7 @@
 
 #include <stdlib.h>
 
+#include <isc/commandline.h>
 #include <isc/log.h>
 #include <isc/mem.h>
 #include <isc/print.h>
@@ -23,6 +24,14 @@
 #include <dns/result.h>
 #include <dns/types.h>
 
+const char *progname = NULL;
+
+static void
+usage(void) {
+	fprintf(stderr, "Usage: %s [-dux] journal\n", progname);
+	exit(1);
+}
+
 /*
  * Setup logging to use stderr.
  */
@@ -57,21 +66,51 @@ main(int argc, char **argv) {
 	isc_mem_t *mctx = NULL;
 	isc_result_t result;
 	isc_log_t *lctx = NULL;
+	uint32_t flags = 0U;
+	char ch;
+	bool downgrade = false;
+	bool upgrade = false;
 
-	if (argc != 2) {
-		printf("usage: %s journal\n", argv[0]);
-		return (1);
+	progname = argv[0];
+	while ((ch = isc_commandline_parse(argc, argv, "dux")) != -1) {
+		switch (ch) {
+		case 'd':
+			downgrade = true;
+			break;
+		case 'u':
+			upgrade = true;
+			break;
+		case 'x':
+			flags |= DNS_JOURNAL_PRINTXHDR;
+			break;
+		default:
+			usage();
+		}
 	}
 
-	file = argv[1];
+	argc -= isc_commandline_index;
+	argv += isc_commandline_index;
+
+	if (argc != 1) {
+		usage();
+	}
+	file = argv[0];
 
 	isc_mem_create(&mctx);
 	RUNTIME_CHECK(setup_logging(mctx, stderr, &lctx) == ISC_R_SUCCESS);
 
-	result = dns_journal_print(mctx, file, stdout);
+	if (upgrade) {
+		flags = DNS_JOURNAL_COMPACTALL;
+		result = dns_journal_compact(mctx, file, 0, flags, 0);
+	} else if (downgrade) {
+		flags = DNS_JOURNAL_COMPACTALL | DNS_JOURNAL_VERSION1;
+		result = dns_journal_compact(mctx, file, 0, flags, 0);
+	} else {
+		result = dns_journal_print(mctx, flags, file, stdout);
 		if (result == DNS_R_NOJOURNAL) {
 			fprintf(stderr, "%s\n", dns_result_totext(result));
 		}
+	}
 	isc_log_destroy(&lctx);
 	isc_mem_detach(&mctx);
 	return (result != ISC_R_SUCCESS ? 1 : 0);

bin/tools/named-journalprint.rst

@@ -29,13 +29,14 @@ named-journalprint - print zone journal in human-readable form
 Synopsis
 ~~~~~~~~
 
-:program:`named-journalprint` {journal}
+:program:`named-journalprint` [**-dux**] {journal}
 
 Description
 ~~~~~~~~~~~
 
-``named-journalprint`` prints the contents of a zone journal file in a
-human-readable form.
+``named-journalprint`` scans the contents of a zone journal file,
+printing it in a human-readable form, or, optionally, converting it
+to a different journal file format.
 
 Journal files are automatically created by ``named`` when changes are
 made to dynamic zones (e.g., by ``nsupdate``). They record each addition
@@ -50,6 +51,17 @@ into a human-readable text format. Each line begins with ``add`` or ``del``,
 to indicate whether the record was added or deleted, and continues with
 the resource record in master-file format.
 
+The ``-x`` option causes additional data about the journal file to be
+printed at the beginning of the output and before each group of changes.
+
+The ``-u`` (upgrade) and ``-d`` (downgrade) options recreate the journal
+file with a modified format version.  The existing journal file is
+replaced.  ``-d`` writes out the journal in the format used by
+versions of BIND up to 9.16.11; ``-u`` writes it out in the format used
+by versions since 9.16.13. (9.16.12 is omitted due to a journal-formatting
+bug in that release.) Note that these options *must not* be used while
+``named`` is running.
+
 See Also
 ~~~~~~~~
 

doc/man/named-journalprint.1in

@@ -32,11 +32,12 @@ level margin: \\n[rst2man-indent\\n[rst2man-indent-level]]
 ..
 .SH SYNOPSIS
 .sp
-\fBnamed\-journalprint\fP {journal}
+\fBnamed\-journalprint\fP [\fB\-dux\fP] {journal}
 .SH DESCRIPTION
 .sp
-\fBnamed\-journalprint\fP prints the contents of a zone journal file in a
-human\-readable form.
+\fBnamed\-journalprint\fP scans the contents of a zone journal file,
+printing it in a human\-readable form, or, optionally, converting it
+to a different journal file format.
 .sp
 Journal files are automatically created by \fBnamed\fP when changes are
 made to dynamic zones (e.g., by \fBnsupdate\fP). They record each addition
@@ -50,6 +51,17 @@ file.
 into a human\-readable text format. Each line begins with \fBadd\fP or \fBdel\fP,
 to indicate whether the record was added or deleted, and continues with
 the resource record in master\-file format.
+.sp
+The \fB\-x\fP option causes additional data about the journal file to be
+printed at the beginning of the output and before each group of changes.
+.sp
+The \fB\-u\fP (upgrade) and \fB\-d\fP (downgrade) options recreate the journal
+file with a modified format version.  The existing journal file is
+replaced.  \fB\-d\fP writes out the journal in the format used by
+versions of BIND up to 9.16.11; \fB\-u\fP writes it out in the format used
+by versions since 9.16.13. (9.16.12 is omitted due to a journal\-formatting
+bug in that release.) Note that these options \fImust not\fP be used while
+\fBnamed\fP is running.
 .SH SEE ALSO
 .sp
 \fBnamed(8)\fP, \fBnsupdate(1)\fP, BIND 9 Administrator Reference Manual.

doc/notes/notes-current.rst

@@ -64,3 +64,20 @@ Bug Fixes
   (crash) if the return of stale cached answers was enabled and
   ``stale-answer-client-timeout`` was applied to a client query in process.
   This has been fixed. [GL #2503]
+
+- Zone journal (``.jnl``) files created by versions of ``named`` prior
+  to 9.16.12 were no longer compatible; this could cause problems when
+  upgrading if journal files were not synchronized first.  This has been
+  corrected: older journal files can now be read when starting up.  When
+  an old-style journal file is detected, it is updated to the new
+  format immediately after loading.
+
+  Note that journals created by the current version of ``named`` are not
+  usable by versions prior to 9.16.12. Before downgrading to a prior
+  release, users are advised to ensure that all dynamic zones have been
+  synchronized using ``rndc sync -clean``.
+
+  A journal file's format can be changed manually by running
+  ``named-journalprint -d`` (downgrade) or ``named-journalprint -u``
+  (upgrade). Note that this *must not* be done while ``named`` is
+  running.  [GL #2505]

lib/dns/include/dns/events.h

@@ -79,6 +79,7 @@
 #define DNS_EVENT_RPZUPDATED	     (ISC_EVENTCLASS_DNS + 57)
 #define DNS_EVENT_STARTUPDATE	     (ISC_EVENTCLASS_DNS + 58)
 #define DNS_EVENT_TRYSTALE	     (ISC_EVENTCLASS_DNS + 59)
+#define DNS_EVENT_ZONEFLUSH	     (ISC_EVENTCLASS_DNS + 60)
 
 #define DNS_EVENT_FIRSTEVENT (ISC_EVENTCLASS_DNS + 0)
 #define DNS_EVENT_LASTEVENT  (ISC_EVENTCLASS_DNS + 65535)

lib/dns/include/dns/journal.h

@@ -48,6 +48,13 @@
 #define DNS_JOURNAL_SIZE_MAX INT32_MAX
 #define DNS_JOURNAL_SIZE_MIN 4096
 
+/*% Print transaction header data */
+#define DNS_JOURNAL_PRINTXHDR 0x0001
+
+/*% Rewrite whole journal file instead of compacting */
+#define DNS_JOURNAL_COMPACTALL 0x0001
+#define DNS_JOURNAL_VERSION1   0x0002
+
 /***
  *** Types
  ***/
@@ -258,12 +265,18 @@ dns_journal_rollforward(isc_mem_t *mctx, dns_db_t *db, unsigned int options,
  *\li	DNS_R_NOJOURNAL when journal does not exist.
  *\li	ISC_R_NOTFOUND when current serial in not in journal.
  *\li	ISC_R_RANGE when current serial in not in journals range.
- *\li	ISC_R_SUCCESS journal has been applied successfully to database.
+ *\li	DNS_R_UPTODATE when the database was already up to date.
+ *\li	ISC_R_SUCCESS journal has been applied successfully to the
+ *      database without any issues.
+ *\li	DNS_R_RECOVERABLE if successful or up to date, but the journal
+ *      was found to contain at least one outdated transaction header.
+ *
  *	others
  */
 
 isc_result_t
-dns_journal_print(isc_mem_t *mctx, const char *filename, FILE *file);
+dns_journal_print(isc_mem_t *mctx, uint32_t flags, const char *filename,
+		  FILE *file);
 /* For debugging not general use */
 
 isc_result_t
@@ -286,7 +299,7 @@ dns_db_diffx(dns_diff_t *diff, dns_db_t *dba, dns_dbversion_t *dbvera,
 
 isc_result_t
 dns_journal_compact(isc_mem_t *mctx, char *filename, uint32_t serial,
-		    uint32_t target_size);
+		    uint32_t flags, uint32_t target_size);
 /*%<
  * Attempt to compact the journal if it is greater that 'target_size'.
  * Changes from 'serial' onwards will be preserved.  If the journal

lib/dns/journal.c

@@ -18,6 +18,7 @@
 #include <isc/file.h>
 #include <isc/mem.h>
 #include <isc/print.h>
+#include <isc/serial.h>
 #include <isc/stdio.h>
 #include <isc/string.h>
 #include <isc/util.h>
@@ -193,6 +194,11 @@ typedef struct {
  */
 #define JOURNAL_HEADER_SIZE 64 /* Bytes. */
 
+typedef enum {
+	XHDR_VERSION1 = 1,
+	XHDR_VERSION2 = 2,
+} xhdr_version_t;
+
 /*%
  * The on-disk representation of the journal header.
  * All numbers are stored in big-endian order.
@@ -216,7 +222,7 @@ typedef union {
 } journal_rawheader_t;
 
 /*%
- * The on-disk representation of the transaction header.
+ * The on-disk representation of the transaction header, version 2.
  * There is one of these at the beginning of each transaction.
  */
 typedef struct {
@@ -226,6 +232,16 @@ typedef struct {
 	unsigned char serial1[4]; /*%< SOA serial after update. */
 } journal_rawxhdr_t;
 
+/*%
+ * Old-style raw transaction header, version 1, used for backward
+ * compatibility mode.
+ */
+typedef struct {
+	unsigned char size[4];
+	unsigned char serial0[4];
+	unsigned char serial1[4];
+} journal_rawxhdr_ver1_t;
+
 /*%
  * The on-disk representation of the RR header.
  * There is one of these at the beginning of each RR.
@@ -275,16 +291,19 @@ typedef struct {
  * Initial contents to store in the header of a newly created
  * journal file.
  *
- * The header starts with the magic string ";BIND LOG V9\n"
+ * The header starts with the magic string ";BIND LOG V9.2\n"
  * to identify the file as a BIND 9 journal file.  An ASCII
  * identification string is used rather than a binary magic
  * number to be consistent with BIND 8 (BIND 8 journal files
  * are ASCII text files).
  */
 
-static journal_header_t initial_journal_header = {
+static journal_header_t journal_header_ver1 = {
 	";BIND LOG V9\n", { 0, 0 }, { 0, 0 }, 0, 0, 0
 };
+static journal_header_t initial_journal_header = {
+	";BIND LOG V9.2\n", { 0, 0 }, { 0, 0 }, 0, 0, 0
+};
 
 #define JOURNAL_EMPTY(h) ((h)->begin.offset == (h)->end.offset)
 
@@ -300,9 +319,15 @@ struct dns_journal {
 	unsigned int magic; /*%< JOUR */
 	isc_mem_t *mctx;    /*%< Memory context */
 	journal_state_t state;
+	xhdr_version_t xhdr_version; /*%< Expected transaction header version */
+	bool header_ver1;	     /*%< Transaction header compatibility
+				      *   mode is allowed */
+	bool recovered;		     /*%< A recoverable error was found
+				      *   while reading the journal */
 	char *filename;		     /*%< Journal file name */
 	FILE *fp;		     /*%< File handle */
 	isc_offset_t offset;	     /*%< Current file offset */
+	journal_xhdr_t curxhdr;	     /*%< Current transaction header */
 	journal_header_t header;     /*%< In-core journal header */
 	unsigned char *rawindex;     /*%< In-core buffer for journal index
 				      * in on-disk format */
@@ -319,10 +344,10 @@ struct dns_journal {
 	struct {
 		/* These define the part of the journal we iterate over. */
 		journal_pos_t bpos; /*%< Position before first, */
+		journal_pos_t cpos; /*%< before current, */
 		journal_pos_t epos; /*%< and after last transaction */
 		/* The rest is iterator state. */
-		uint32_t current_serial; /*%< Current SOA serial
-					  * */
+		uint32_t current_serial; /*%< Current SOA serial */
 		isc_buffer_t source;	 /*%< Data from disk */
 		isc_buffer_t target;	 /*%< Data from _fromwire check */
 		dns_decompress_t dctx;	 /*%< Dummy decompression ctx */
@@ -353,6 +378,7 @@ journal_pos_encode(journal_rawpos_t *raw, journal_pos_t *cooked) {
 static void
 journal_header_decode(journal_rawheader_t *raw, journal_header_t *cooked) {
 	INSIST(sizeof(cooked->format) == sizeof(raw->h.format));
+
 	memmove(cooked->format, raw->h.format, sizeof(cooked->format));
 	journal_pos_decode(&raw->h.begin, &cooked->begin);
 	journal_pos_decode(&raw->h.end, &cooked->end);
@@ -366,6 +392,7 @@ journal_header_encode(journal_header_t *cooked, journal_rawheader_t *raw) {
 	unsigned char flags = 0;
 
 	INSIST(sizeof(cooked->format) == sizeof(raw->h.format));
+
 	memset(raw->pad, 0, sizeof(raw->pad));
 	memmove(raw->h.format, cooked->format, sizeof(raw->h.format));
 	journal_pos_encode(&raw->h.begin, &cooked->begin);
@@ -432,6 +459,7 @@ journal_write(dns_journal_t *j, void *mem, size_t nbytes) {
 static isc_result_t
 journal_fsync(dns_journal_t *j) {
 	isc_result_t result;
+
 	result = isc_stdio_flush(j->fp);
 	if (result != ISC_R_SUCCESS) {
 		isc_log_write(JOURNAL_COMMON_LOGARGS, ISC_LOG_ERROR,
@@ -452,11 +480,29 @@ journal_fsync(dns_journal_t *j) {
 /*
  * Read/write a transaction header at the current file position.
  */
-
 static isc_result_t
 journal_read_xhdr(dns_journal_t *j, journal_xhdr_t *xhdr) {
-	journal_rawxhdr_t raw;
 	isc_result_t result;
+
+	j->it.cpos.offset = j->offset;
+
+	switch (j->xhdr_version) {
+	case XHDR_VERSION1: {
+		journal_rawxhdr_ver1_t raw;
+		result = journal_read(j, &raw, sizeof(raw));
+		if (result != ISC_R_SUCCESS) {
+			return (result);
+		}
+		xhdr->size = decode_uint32(raw.size);
+		xhdr->count = 0;
+		xhdr->serial0 = decode_uint32(raw.serial0);
+		xhdr->serial1 = decode_uint32(raw.serial1);
+		j->curxhdr = *xhdr;
+		return (ISC_R_SUCCESS);
+	}
+
+	case XHDR_VERSION2: {
+		journal_rawxhdr_t raw;
 		result = journal_read(j, &raw, sizeof(raw));
 		if (result != ISC_R_SUCCESS) {
 			return (result);
@@ -465,18 +511,32 @@ journal_read_xhdr(dns_journal_t *j, journal_xhdr_t *xhdr) {
 		xhdr->count = decode_uint32(raw.count);
 		xhdr->serial0 = decode_uint32(raw.serial0);
 		xhdr->serial1 = decode_uint32(raw.serial1);
+		j->curxhdr = *xhdr;
 		return (ISC_R_SUCCESS);
+	}
+
+	default:
+		return (ISC_R_NOTIMPLEMENTED);
+	}
 }
 
 static isc_result_t
 journal_write_xhdr(dns_journal_t *j, uint32_t size, uint32_t count,
 		   uint32_t serial0, uint32_t serial1) {
+	if (j->header_ver1) {
+		journal_rawxhdr_ver1_t raw;
+		encode_uint32(size, raw.size);
+		encode_uint32(serial0, raw.serial0);
+		encode_uint32(serial1, raw.serial1);
+		return (journal_write(j, &raw, sizeof(raw)));
+	} else {
 		journal_rawxhdr_t raw;
 		encode_uint32(size, raw.size);
 		encode_uint32(count, raw.count);
 		encode_uint32(serial0, raw.serial0);
 		encode_uint32(serial1, raw.serial1);
 		return (journal_write(j, &raw, sizeof(raw)));
+	}
 }
 
 /*
@@ -487,6 +547,7 @@ static isc_result_t
 journal_read_rrhdr(dns_journal_t *j, journal_rrhdr_t *rrhdr) {
 	journal_rawrrhdr_t raw;
 	isc_result_t result;
+
 	result = journal_read(j, &raw, sizeof(raw));
 	if (result != ISC_R_SUCCESS) {
 		return (result);
@@ -496,14 +557,14 @@ journal_read_rrhdr(dns_journal_t *j, journal_rrhdr_t *rrhdr) {
 }
 
 static isc_result_t
-journal_file_create(isc_mem_t *mctx, const char *filename) {
+journal_file_create(isc_mem_t *mctx, bool downgrade, const char *filename) {
 	FILE *fp = NULL;
 	isc_result_t result;
 	journal_header_t header;
 	journal_rawheader_t rawheader;
 	int index_size = 56; /* XXX configurable */
 	int size;
-	void *mem; /* Memory for temporary index image. */
+	void *mem = NULL; /* Memory for temporary index image. */
 
 	INSIST(sizeof(journal_rawheader_t) == JOURNAL_HEADER_SIZE);
 
@@ -515,7 +576,11 @@ journal_file_create(isc_mem_t *mctx, const char *filename) {
 		return (ISC_R_UNEXPECTED);
 	}
 
+	if (downgrade) {
+		header = journal_header_ver1;
+	} else {
 		header = initial_journal_header;
+	}
 	header.index_size = index_size;
 	journal_header_encode(&header, &rawheader);
 
@@ -552,36 +617,28 @@ journal_file_create(isc_mem_t *mctx, const char *filename) {
 
 static isc_result_t
 journal_open(isc_mem_t *mctx, const char *filename, bool writable, bool create,
-	     dns_journal_t **journalp) {
+	     bool downgrade, dns_journal_t **journalp) {
 	FILE *fp = NULL;
 	isc_result_t result;
 	journal_rawheader_t rawheader;
 	dns_journal_t *j;
 
-	INSIST(journalp != NULL && *journalp == NULL);
+	REQUIRE(journalp != NULL && *journalp == NULL);
+
 	j = isc_mem_get(mctx, sizeof(*j));
-
-	j->mctx = NULL;
+	*j = (dns_journal_t){ .state = JOURNAL_STATE_INVALID,
+			      .filename = isc_mem_strdup(mctx, filename),
+			      .xhdr_version = XHDR_VERSION2 };
 	isc_mem_attach(mctx, &j->mctx);
-	j->state = JOURNAL_STATE_INVALID;
-	j->fp = NULL;
-	j->filename = isc_mem_strdup(mctx, filename);
-	j->index = NULL;
-	j->rawindex = NULL;
-
-	if (j->filename == NULL) {
-		FAIL(ISC_R_NOMEMORY);
-	}
 
 	result = isc_stdio_open(j->filename, writable ? "rb+" : "rb", &fp);
-
 	if (result == ISC_R_FILENOTFOUND) {
 		if (create) {
 			isc_log_write(JOURNAL_COMMON_LOGARGS, ISC_LOG_DEBUG(1),
 				      "journal file %s does not exist, "
 				      "creating it",
 				      j->filename);
-			CHECK(journal_file_create(mctx, filename));
+			CHECK(journal_file_create(mctx, downgrade, filename));
 			/*
 			 * Retry.
 			 */
@@ -607,9 +664,29 @@ journal_open(isc_mem_t *mctx, const char *filename, bool writable, bool create,
 	CHECK(journal_seek(j, 0));
 	CHECK(journal_read(j, &rawheader, sizeof(rawheader)));
 
-	if (memcmp(rawheader.h.format, initial_journal_header.format,
-		   sizeof(initial_journal_header.format)) != 0)
+	if (memcmp(rawheader.h.format, journal_header_ver1.format,
+		   sizeof(journal_header_ver1.format)) == 0)
 	{
+		/*
+		 * The file header says it's the old format, but it
+		 * still might have the new xhdr format because we
+		 * forgot to change the format string when we introduced
+		 * the new xhdr.  When we first try to read it, we assume
+		 * it uses the new xhdr format. If that fails, we'll be
+		 * called a second time with compat set to true, in which
+		 * case we can lower xhdr_version to 1 if we find a
+		 * corrupt transaction.
+		 */
+		j->header_ver1 = true;
+	} else if (memcmp(rawheader.h.format, initial_journal_header.format,
+			  sizeof(initial_journal_header.format)) == 0)
+	{
+		/*
+		 * File header says this is format version 2; all
+		 * transactions have to match.
+		 */
+		j->header_ver1 = false;
+	} else {
 		isc_log_write(JOURNAL_COMMON_LOGARGS, ISC_LOG_ERROR,
 			      "%s: journal format not recognized", j->filename);
 		FAIL(ISC_R_UNEXPECTED);
@@ -695,7 +772,8 @@ dns_journal_open(isc_mem_t *mctx, const char *filename, unsigned int mode,
 	create = ((mode & DNS_JOURNAL_CREATE) != 0);
 	writable = ((mode & (DNS_JOURNAL_WRITE | DNS_JOURNAL_CREATE)) != 0);
 
-	result = journal_open(mctx, filename, writable, create, journalp);
+	result = journal_open(mctx, filename, writable, create, false,
+			      journalp);
 	if (result == ISC_R_NOTFOUND) {
 		namelen = strlen(filename);
 		if (namelen > 4U && strcmp(filename + namelen - 4, ".jnl") == 0)
@@ -708,7 +786,7 @@ dns_journal_open(isc_mem_t *mctx, const char *filename, unsigned int mode,
 		if (result >= sizeof(backup)) {
 			return (ISC_R_NOSPACE);
 		}
-		result = journal_open(mctx, backup, writable, writable,
+		result = journal_open(mctx, backup, writable, writable, false,
 				      journalp);
 	}
 	return (result);
@@ -795,9 +873,11 @@ ixfr_order(const void *av, const void *bv) {
  *    Other results due to file errors are possible.
  */
 static isc_result_t
-journal_next(dns_journal_t *j, journal_pos_t *pos) {
+journal_next(dns_journal_t *j, journal_pos_t *pos, bool retry) {
 	isc_result_t result;
 	journal_xhdr_t xhdr;
+	size_t hdrsize;
+
 	REQUIRE(DNS_JOURNAL_VALID(j));
 
 	result = journal_seek(j, pos->offset);
@@ -808,6 +888,7 @@ journal_next(dns_journal_t *j, journal_pos_t *pos) {
 	if (pos->serial == j->header.end.serial) {
 		return (ISC_R_NOMORE);
 	}
+
 	/*
 	 * Read the header of the current transaction.
 	 * This will return ISC_R_NOMORE if we are at EOF.
@@ -820,26 +901,59 @@ journal_next(dns_journal_t *j, journal_pos_t *pos) {
 	/*
 	 * Check serial number consistency.
 	 */
-	if (xhdr.serial0 != pos->serial) {
+	if (xhdr.serial0 != pos->serial ||
+	    isc_serial_le(xhdr.serial1, xhdr.serial0)) {
+		if (j->header_ver1 && j->xhdr_version == XHDR_VERSION1 &&
+		    xhdr.serial1 == pos->serial && !retry)
+		{
+			/* XHDR_VERSION1 -> XHDR_VERSION2 */
+			isc_log_write(
+				JOURNAL_COMMON_LOGARGS, ISC_LOG_DEBUG(3),
+				"%s: XHDR_VERSION1 -> XHDR_VERSION2 at %u\n",
+				j->filename, pos->serial);
+			j->xhdr_version = XHDR_VERSION2;
+			result = journal_next(j, pos, true);
+			if (result == ISC_R_SUCCESS) {
+				j->recovered = true;
+			}
+			return (result);
+		} else if (j->header_ver1 && j->xhdr_version == XHDR_VERSION2 &&
+			   xhdr.count == pos->serial && !retry)
+		{
+			/* XHDR_VERSION2 -> XHDR_VERSION1 */
+			isc_log_write(
+				JOURNAL_COMMON_LOGARGS, ISC_LOG_DEBUG(3),
+				"%s: XHDR_VERSION2 -> XHDR_VERSION1 at %u\n",
+				j->filename, pos->serial);
+			j->xhdr_version = XHDR_VERSION1;
+			result = journal_next(j, pos, true);
+			if (result == ISC_R_SUCCESS) {
+				j->recovered = true;
+			}
+			return (result);
+		} else {
 			isc_log_write(JOURNAL_COMMON_LOGARGS, ISC_LOG_ERROR,
 				      "%s: journal file corrupt: "
 				      "expected serial %u, got %u",
 				      j->filename, pos->serial, xhdr.serial0);
 			return (ISC_R_UNEXPECTED);
 		}
+	}
 
 	/*
 	 * Check for offset wraparound.
 	 */
-	if ((isc_offset_t)(pos->offset + sizeof(journal_rawxhdr_t) +
-			   xhdr.size) < pos->offset)
-	{
+	hdrsize = (j->xhdr_version == XHDR_VERSION2)
+			  ? sizeof(journal_rawxhdr_t)
+			  : sizeof(journal_rawxhdr_ver1_t);
+
+	if ((isc_offset_t)(pos->offset + hdrsize + xhdr.size) < pos->offset) {
 		isc_log_write(JOURNAL_COMMON_LOGARGS, ISC_LOG_ERROR,
 			      "%s: offset too large", j->filename);
 		return (ISC_R_UNEXPECTED);
 	}
 
-	pos->offset += sizeof(journal_rawxhdr_t) + xhdr.size;
+	pos->offset += hdrsize + xhdr.size;
 	pos->serial = xhdr.serial1;
 	return (ISC_R_SUCCESS);
 }
@@ -879,9 +993,11 @@ index_find(dns_journal_t *j, uint32_t serial, journal_pos_t *best_guess) {
 static void
 index_add(dns_journal_t *j, journal_pos_t *pos) {
 	unsigned int i;
+
 	if (j->index == NULL) {
 		return;
 	}
+
 	/*
 	 * Search for a vacant position.
 	 */
@@ -953,6 +1069,7 @@ static isc_result_t
 journal_find(dns_journal_t *j, uint32_t serial, journal_pos_t *pos) {
 	isc_result_t result;
 	journal_pos_t current_pos;
+
 	REQUIRE(DNS_JOURNAL_VALID(j));
 
 	if (DNS_SERIAL_GT(j->header.begin.serial, serial)) {
@@ -973,7 +1090,7 @@ journal_find(dns_journal_t *j, uint32_t serial, journal_pos_t *pos) {
 		if (DNS_SERIAL_GT(current_pos.serial, serial)) {
 			return (ISC_R_NOTFOUND);
 		}
-		result = journal_next(j, &current_pos);
+		result = journal_next(j, &current_pos, false);
 		if (result != ISC_R_SUCCESS) {
 			return (result);
 		}
@@ -1187,7 +1304,7 @@ dns_journal_commit(dns_journal_t *j) {
 	if (!JOURNAL_EMPTY(&j->header)) {
 		while (!DNS_SERIAL_GT(j->x.pos[1].serial,
 				      j->header.begin.serial)) {
-			CHECK(journal_next(j, &j->header.begin));
+			CHECK(journal_next(j, &j->header.begin, false));
 		}
 		index_invalidate(j, j->x.pos[1].serial);
 	}
@@ -1256,6 +1373,7 @@ failure:
 isc_result_t
 dns_journal_write_transaction(dns_journal_t *j, dns_diff_t *diff) {
 	isc_result_t result;
+
 	CHECK(dns_diff_sort(diff, ixfr_order));
 	CHECK(dns_journal_begin_transaction(j));
 	CHECK(dns_journal_writediff(j, diff));
@@ -1267,9 +1385,13 @@ failure:
 
 void
 dns_journal_destroy(dns_journal_t **journalp) {
-	dns_journal_t *j = *journalp;
+	dns_journal_t *j = NULL;
+
+	REQUIRE(journalp != NULL);
+	REQUIRE(DNS_JOURNAL_VALID(*journalp));
+
+	j = *journalp;
 	*journalp = NULL;
-	REQUIRE(DNS_JOURNAL_VALID(j));
 
 	j->it.result = ISC_R_FAILURE;
 	dns_name_invalidate(&j->it.name);
@@ -1346,33 +1468,38 @@ roll_forward(dns_journal_t *j, dns_db_t *db, unsigned int options) {
 	 * Locate a journal entry for the current database serial.
 	 */
 	CHECK(journal_find(j, db_serial, &pos));
-	/*
-	 * XXX do more drastic things, like marking zone stale,
-	 * if this fails?
-	 */
-	/*
-	 * XXXRTH  The zone code should probably mark the zone as bad and
-	 *         scream loudly into the log if this is a dynamic update
-	 *	   log reply that failed.
-	 */
 
 	end_serial = dns_journal_last_serial(j);
+
+	/*
+	 * If we're reading a version 1 file, scan all the transactions
+	 * to see if the journal needs rewriting: if any outdated
+	 * transaction headers are found, j->recovered will be set.
+	 */
+	if (j->header_ver1) {
+		uint32_t start_serial = dns_journal_first_serial(j);
+
+		CHECK(dns_journal_iter_init(j, start_serial, db_serial, NULL));
+		for (result = dns_journal_first_rr(j); result == ISC_R_SUCCESS;
+		     result = dns_journal_next_rr(j))
+		{
+			continue;
+		}
+	}
+
 	if (db_serial == end_serial) {
 		CHECK(DNS_R_UPTODATE);
 	}
 
 	CHECK(dns_journal_iter_init(j, db_serial, end_serial, NULL));
-
 	for (result = dns_journal_first_rr(j); result == ISC_R_SUCCESS;
 	     result = dns_journal_next_rr(j))
 	{
-		dns_name_t *name;
-		uint32_t ttl;
-		dns_rdata_t *rdata;
+		dns_name_t *name = NULL;
+		dns_rdata_t *rdata = NULL;
 		dns_difftuple_t *tuple = NULL;
+		uint32_t ttl;
 
-		name = NULL;
-		rdata = NULL;
 		dns_journal_current_rr(j, &name, &ttl, &rdata);
 
 		if (rdata->type == dns_rdatatype_soa) {
@@ -1450,13 +1577,12 @@ failure:
 isc_result_t
 dns_journal_rollforward(isc_mem_t *mctx, dns_db_t *db, unsigned int options,
 			const char *filename) {
-	dns_journal_t *j;
+	dns_journal_t *j = NULL;
 	isc_result_t result;
 
 	REQUIRE(DNS_DB_VALID(db));
 	REQUIRE(filename != NULL);
 
-	j = NULL;
 	result = dns_journal_open(mctx, filename, DNS_JOURNAL_READ, &j);
 	if (result == ISC_R_NOTFOUND) {
 		isc_log_write(JOURNAL_DEBUG_LOGARGS(3), "no journal file, but "
@@ -1466,20 +1592,26 @@ dns_journal_rollforward(isc_mem_t *mctx, dns_db_t *db, unsigned int options,
 	if (result != ISC_R_SUCCESS) {
 		return (result);
 	}
+
 	if (JOURNAL_EMPTY(&j->header)) {
-		result = DNS_R_UPTODATE;
-	} else {
-		result = roll_forward(j, db, options);
+		CHECK(DNS_R_UPTODATE);
 	}
 
-	dns_journal_destroy(&j);
+	result = roll_forward(j, db, options);
+	if ((result == ISC_R_SUCCESS || result == DNS_R_UPTODATE) &&
+	    j->recovered) {
+		result = DNS_R_RECOVERABLE;
+	}
 
+failure:
+	dns_journal_destroy(&j);
 	return (result);
 }
 
 isc_result_t
-dns_journal_print(isc_mem_t *mctx, const char *filename, FILE *file) {
-	dns_journal_t *j;
+dns_journal_print(isc_mem_t *mctx, uint32_t flags, const char *filename,
+		  FILE *file) {
+	dns_journal_t *j = NULL;
 	isc_buffer_t source;   /* Transaction data from disk */
 	isc_buffer_t target;   /* Ditto after _fromwire check */
 	uint32_t start_serial; /* Database SOA serial */
@@ -1488,23 +1620,34 @@ dns_journal_print(isc_mem_t *mctx, const char *filename, FILE *file) {
 	dns_diff_t diff;
 	unsigned int n_soa = 0;
 	unsigned int n_put = 0;
+	bool printxhdr = ((flags & DNS_JOURNAL_PRINTXHDR) != 0);
 
 	REQUIRE(filename != NULL);
 
-	j = NULL;
 	result = dns_journal_open(mctx, filename, DNS_JOURNAL_READ, &j);
 	if (result == ISC_R_NOTFOUND) {
 		isc_log_write(JOURNAL_DEBUG_LOGARGS(3), "no journal file");
 		return (DNS_R_NOJOURNAL);
-	}
-
-	if (result != ISC_R_SUCCESS) {
+	} else if (result != ISC_R_SUCCESS) {
 		isc_log_write(JOURNAL_COMMON_LOGARGS, ISC_LOG_ERROR,
 			      "journal open failure: %s: %s",
 			      isc_result_totext(result), filename);
 		return (result);
 	}
 
+	if (printxhdr) {
+		fprintf(file, "Journal format = %sHeader version = %d\n",
+			j->header.format + 1, j->header_ver1 ? 1 : 2);
+		fprintf(file, "Index (size = %u):\n", j->header.index_size);
+		for (uint32_t i = 0; i < j->header.index_size; i++) {
+			if (j->index[i].offset == 0) {
+				fputc('\n', file);
+				break;
+			}
+			fprintf(file, "%lld", (long long)j->index[i].offset);
+			fputc((i + 1) % 8 == 0 ? '\n' : ' ', file);
+		}
+	}
 	if (j->header.serialset) {
 		fprintf(file, "Source serial = %u\n", j->header.sourceserial);
 	}
@@ -1526,22 +1669,24 @@ dns_journal_print(isc_mem_t *mctx, const char *filename, FILE *file) {
 	for (result = dns_journal_first_rr(j); result == ISC_R_SUCCESS;
 	     result = dns_journal_next_rr(j))
 	{
-		dns_name_t *name;
-		uint32_t ttl;
-		dns_rdata_t *rdata;
+		dns_name_t *name = NULL;
+		dns_rdata_t *rdata = NULL;
 		dns_difftuple_t *tuple = NULL;
+		static uint32_t i = 0;
+		bool print = false;
+		uint32_t ttl;
 
-		name = NULL;
-		rdata = NULL;
 		dns_journal_current_rr(j, &name, &ttl, &rdata);
 
 		if (rdata->type == dns_rdatatype_soa) {
 			n_soa++;
-		}
-
 			if (n_soa == 3) {
 				n_soa = 1;
 			}
+			if (n_soa == 1) {
+				print = printxhdr;
+			}
+		}
 		if (n_soa == 0) {
 			isc_log_write(JOURNAL_COMMON_LOGARGS, ISC_LOG_ERROR,
 				      "%s: journal file corrupt: missing "
@@ -1549,12 +1694,29 @@ dns_journal_print(isc_mem_t *mctx, const char *filename, FILE *file) {
 				      j->filename);
 			FAIL(ISC_R_UNEXPECTED);
 		}
+
+		if (print) {
+			fprintf(file,
+				"Transaction: version %d offset %lld size %u "
+				"rrcount %u start %u end %u\n",
+				j->xhdr_version, (long long)j->it.cpos.offset,
+				j->curxhdr.size, j->curxhdr.count,
+				j->curxhdr.serial0, j->curxhdr.serial1);
+			if (j->it.cpos.offset > j->index[i].offset) {
+				fprintf(file,
+					"ERROR: Offset mismatch, "
+					"expected %lld\n",
+					(long long)j->index[i].offset);
+			} else if (j->it.cpos.offset == j->index[i].offset) {
+				i++;
+			}
+		}
 		CHECK(dns_difftuple_create(
 			diff.mctx, n_soa == 1 ? DNS_DIFFOP_DEL : DNS_DIFFOP_ADD,
 			name, ttl, rdata, &tuple));
 		dns_diff_append(&diff, &tuple);
 
-		if (++n_put > 100) {
+		if (++n_put != 0 || printxhdr) {
 			result = dns_diff_print(&diff, file);
 			dns_diff_clear(&diff);
 			n_put = 0;
@@ -1568,7 +1730,7 @@ dns_journal_print(isc_mem_t *mctx, const char *filename, FILE *file) {
 	}
 	CHECK(result);
 
-	if (n_put != 0) {
+	if (!printxhdr && n_put != 0) {
 		result = dns_diff_print(&diff, file);
 		dns_diff_clear(&diff);
 	}
@@ -1648,7 +1810,7 @@ dns_journal_get_sourceserial(dns_journal_t *j, uint32_t *sourceserial) {
  */
 
 static isc_result_t
-read_one_rr(dns_journal_t *j);
+read_one_rr(dns_journal_t *j, bool retry);
 
 /*
  * Make sure the buffer 'b' is has at least 'size' bytes
@@ -1706,7 +1868,7 @@ dns_journal_iter_init(dns_journal_t *j, uint32_t begin_serial,
 			size += xhdr.size;
 			count += xhdr.count;
 
-			result = journal_next(j, &pos);
+			result = journal_next(j, &pos, false);
 			if (result == ISC_R_NOMORE) {
 				result = ISC_R_SUCCESS;
 			}
@@ -1742,22 +1904,22 @@ dns_journal_first_rr(dns_journal_t *j) {
 	j->it.xsize = 0; /* We have no transaction data yet... */
 	j->it.xpos = 0;	 /* ...and haven't used any of it. */
 
-	return (read_one_rr(j));
+	return (read_one_rr(j, false));
 
 failure:
 	return (result);
 }
 
 static isc_result_t
-read_one_rr(dns_journal_t *j) {
+read_one_rr(dns_journal_t *j, bool retry) {
 	isc_result_t result;
-
 	dns_rdatatype_t rdtype;
 	dns_rdataclass_t rdclass;
 	unsigned int rdlen;
 	uint32_t ttl;
 	journal_xhdr_t xhdr;
 	journal_rrhdr_t rrhdr;
+	dns_journal_t save = *j;
 
 	if (j->offset > j->it.epos.offset) {
 		isc_log_write(JOURNAL_COMMON_LOGARGS, ISC_LOG_ERROR,
@@ -1780,7 +1942,34 @@ read_one_rr(dns_journal_t *j) {
 				      j->filename);
 			FAIL(ISC_R_UNEXPECTED);
 		}
-		if (xhdr.serial0 != j->it.current_serial) {
+		if (xhdr.serial0 != j->it.current_serial ||
+		    isc_serial_le(xhdr.serial1, xhdr.serial0))
+		{
+			if (!retry && j->header_ver1 &&
+			    j->xhdr_version == XHDR_VERSION2 &&
+			    xhdr.count == j->it.current_serial)
+			{
+				/* XHDR_VERSION2 -> XHDR_VERSION1 */
+				j->xhdr_version = XHDR_VERSION1;
+				CHECK(journal_seek(j, save.offset));
+				result = read_one_rr(j, true);
+				if (result == ISC_R_SUCCESS) {
+					j->recovered = true;
+				}
+				return (result);
+			} else if (!retry && j->header_ver1 &&
+				   j->xhdr_version == XHDR_VERSION1 &&
+				   xhdr.serial1 == j->it.current_serial)
+			{
+				/* XHDR_VERSION1 -> XHDR_VERSION2 */
+				j->xhdr_version = XHDR_VERSION2;
+				CHECK(journal_seek(j, save.offset));
+				result = read_one_rr(j, true);
+				if (result == ISC_R_SUCCESS) {
+					j->recovered = true;
+				}
+				return (result);
+			}
 			isc_log_write(JOURNAL_COMMON_LOGARGS, ISC_LOG_ERROR,
 				      "%s: journal file corrupt: "
 				      "expected serial %u, got %u",
@@ -1871,7 +2060,7 @@ failure:
 
 isc_result_t
 dns_journal_next_rr(dns_journal_t *j) {
-	j->it.result = read_one_rr(j);
+	j->it.result = read_one_rr(j, false);
 	return (j->it.result);
 }
 
@@ -2200,16 +2389,33 @@ failure:
 	return (result);
 }
 
+static uint32_t
+rrcount(char *buf, unsigned int size) {
+	isc_buffer_t b;
+	uint32_t rrsize, count = 0;
+
+	isc_buffer_init(&b, buf, size);
+	isc_buffer_add(&b, size);
+	while (isc_buffer_remaininglength(&b) > 0) {
+		rrsize = isc_buffer_getuint32(&b);
+		INSIST(isc_buffer_remaininglength(&b) >= rrsize);
+		isc_buffer_forward(&b, rrsize);
+		count++;
+	}
+
+	return (count);
+}
+
 isc_result_t
 dns_journal_compact(isc_mem_t *mctx, char *filename, uint32_t serial,
-		    uint32_t target_size) {
+		    uint32_t flags, uint32_t target_size) {
 	unsigned int i;
 	journal_pos_t best_guess;
 	journal_pos_t current_pos;
 	dns_journal_t *j1 = NULL;
 	dns_journal_t *j2 = NULL;
 	journal_rawheader_t rawheader;
-	unsigned int copy_length;
+	unsigned int len;
 	size_t namelen;
 	char *buf = NULL;
 	unsigned int size = 0;
@@ -2218,6 +2424,8 @@ dns_journal_compact(isc_mem_t *mctx, char *filename, uint32_t serial,
 	char newname[PATH_MAX];
 	char backup[PATH_MAX];
 	bool is_backup = false;
+	bool rewrite = false;
+	bool downgrade = false;
 
 	REQUIRE(filename != NULL);
 
@@ -2234,16 +2442,26 @@ dns_journal_compact(isc_mem_t *mctx, char *filename, uint32_t serial,
 			  filename);
 	RUNTIME_CHECK(result < sizeof(backup));
 
-	result = journal_open(mctx, filename, false, false, &j1);
+	result = journal_open(mctx, filename, false, false, false, &j1);
 	if (result == ISC_R_NOTFOUND) {
 		is_backup = true;
-		result = journal_open(mctx, backup, false, false, &j1);
+		result = journal_open(mctx, backup, false, false, false, &j1);
 	}
 	if (result != ISC_R_SUCCESS) {
 		return (result);
 	}
 
-	if (JOURNAL_EMPTY(&j1->header)) {
+	/*
+	 * Check whether we need to rewrite the whole journal
+	 * file (for example, to upversion it).
+	 */
+	if ((flags & DNS_JOURNAL_COMPACTALL) != 0) {
+		if ((flags & DNS_JOURNAL_VERSION1) != 0) {
+			downgrade = true;
+		}
+		rewrite = true;
+		serial = dns_journal_first_serial(j1);
+	} else if (JOURNAL_EMPTY(&j1->header)) {
 		dns_journal_destroy(&j1);
 		return (ISC_R_SUCCESS);
 	}
@@ -2270,12 +2488,13 @@ dns_journal_compact(isc_mem_t *mctx, char *filename, uint32_t serial,
 	/*
 	 * See if there is any work to do.
 	 */
-	if ((uint32_t)j1->header.end.offset < target_size) {
+	if (!rewrite && (uint32_t)j1->header.end.offset < target_size) {
 		dns_journal_destroy(&j1);
 		return (ISC_R_SUCCESS);
 	}
 
-	CHECK(journal_open(mctx, newname, true, true, &j2));
+	CHECK(journal_open(mctx, newname, true, true, downgrade, &j2));
+	CHECK(journal_seek(j2, indexend));
 
 	/*
 	 * Remove overhead so space test below can succeed.
@@ -2301,7 +2520,7 @@ dns_journal_compact(isc_mem_t *mctx, char *filename, uint32_t serial,
 
 	current_pos = best_guess;
 	while (current_pos.serial != serial) {
-		CHECK(journal_next(j1, &current_pos));
+		CHECK(journal_next(j1, &current_pos, false));
 		if (current_pos.serial == j1->header.end.serial) {
 			break;
 		}
@@ -2319,7 +2538,7 @@ dns_journal_compact(isc_mem_t *mctx, char *filename, uint32_t serial,
 
 	INSIST(best_guess.serial != j1->header.end.serial);
 	if (best_guess.serial != serial) {
-		CHECK(journal_next(j1, &best_guess));
+		CHECK(journal_next(j1, &best_guess, false));
 	}
 
 	/*
@@ -2327,40 +2546,98 @@ dns_journal_compact(isc_mem_t *mctx, char *filename, uint32_t serial,
 	 * we did not reach 'serial'.  If not we will just copy
 	 * all uncommitted deltas regardless of the size.
 	 */
-	copy_length = j1->header.end.offset - best_guess.offset;
-
-	if (copy_length != 0) {
-		/*
-		 * Copy best_guess to end into space just freed.
-		 */
-		size = 64 * 1024;
-		if (copy_length < size) {
-			size = copy_length;
-		}
-		buf = isc_mem_get(mctx, size);
-
+	len = j1->header.end.offset - best_guess.offset;
+	if (len != 0) {
 		CHECK(journal_seek(j1, best_guess.offset));
-		CHECK(journal_seek(j2, indexend));
-		for (i = 0; i < copy_length; i += size) {
-			unsigned int len = (copy_length - i) > size
-						   ? size
-						   : (copy_length - i);
-			CHECK(journal_read(j1, buf, len));
-			CHECK(journal_write(j2, buf, len));
+
+		/* Prepare new header */
+		j2->header.begin.serial = best_guess.serial;
+		j2->header.begin.offset = indexend;
+		j2->header.sourceserial = j1->header.sourceserial;
+		j2->header.serialset = j1->header.serialset;
+		j2->header.end.serial = j1->header.end.serial;
+
+		/*
+		 * Only use this method if we're rewriting the
+		 * journal to fix outdated transaction headers;
+		 * otherwise we'll copy the whole journal without
+		 * parsing individual deltas below.
+		 */
+		while (rewrite && len > 0) {
+			journal_xhdr_t xhdr;
+			isc_offset_t offset = j1->offset;
+			uint32_t count;
+
+			result = journal_read_xhdr(j1, &xhdr);
+			if (rewrite && result == ISC_R_NOMORE) {
+				break;
+			}
+			CHECK(result);
+
+			/*
+			 * If we're repairing an outdated journal, the
+			 * xhdr format may be wrong.
+			 */
+			if (rewrite &&
+			    (xhdr.serial0 != serial ||
+			     isc_serial_le(xhdr.serial1, xhdr.serial0)))
+			{
+				if (j1->xhdr_version == XHDR_VERSION2 &&
+				    xhdr.count == serial) {
+					/* XHDR_VERSION2 -> XHDR_VERSION1 */
+					j1->xhdr_version = XHDR_VERSION1;
+					CHECK(journal_seek(j1, offset));
+					CHECK(journal_read_xhdr(j1, &xhdr));
+				} else if (j1->xhdr_version == XHDR_VERSION1 &&
+					   xhdr.serial1 == serial) {
+					/* XHDR_VERSION1 -> XHDR_VERSION2 */
+					j1->xhdr_version = XHDR_VERSION2;
+					CHECK(journal_seek(j1, offset));
+					CHECK(journal_read_xhdr(j1, &xhdr));
+				}
+
+				/* Check again */
+				if (xhdr.serial0 != serial ||
+				    isc_serial_le(xhdr.serial1, xhdr.serial0)) {
+					CHECK(ISC_R_UNEXPECTED);
+				}
+			}
+
+			size = xhdr.size;
+			buf = isc_mem_get(mctx, size);
+			CHECK(journal_read(j1, buf, size));
+
+			count = rrcount(buf, size);
+			CHECK(journal_write_xhdr(j2, xhdr.size, count,
+						 xhdr.serial0, xhdr.serial1));
+			CHECK(journal_write(j2, buf, size));
+
+			j2->header.end.offset = j2->offset;
+
+			serial = xhdr.serial1;
+
+			len = j1->header.end.offset - j1->offset;
+			isc_mem_put(mctx, buf, size);
+		}
+
+		/*
+		 * If we're not rewriting transaction headers, we can use
+		 * this faster method instead.
+		 */
+		if (!rewrite) {
+			size = ISC_MIN(64 * 1024, len);
+			buf = isc_mem_get(mctx, size);
+			for (i = 0; i < len; i += size) {
+				unsigned int blob = ISC_MIN(size, len - i);
+				CHECK(journal_read(j1, buf, blob));
+				CHECK(journal_write(j2, buf, blob));
+			}
+
+			j2->header.end.offset = indexend + len;
 		}
 
 		CHECK(journal_fsync(j2));
 
-		/*
-		 * Compute new header.
-		 */
-		j2->header.begin.serial = best_guess.serial;
-		j2->header.begin.offset = indexend;
-		j2->header.end.serial = j1->header.end.serial;
-		j2->header.end.offset = indexend + copy_length;
-		j2->header.sourceserial = j1->header.sourceserial;
-		j2->header.serialset = j1->header.serialset;
-
 		/*
 		 * Update the journal header.
 		 */
@@ -2375,7 +2652,7 @@ dns_journal_compact(isc_mem_t *mctx, char *filename, uint32_t serial,
 		current_pos = j2->header.begin;
 		while (current_pos.serial != j2->header.end.serial) {
 			index_add(j2, &current_pos);
-			CHECK(journal_next(j2, &current_pos));
+			CHECK(journal_next(j2, &current_pos, false));
 		}
 
 		/*

lib/dns/zone.c

@@ -504,6 +504,9 @@ typedef enum {
 						      * notify due to the zone
 						      * just being loaded for
 						      * the first time. */
+	DNS_ZONEFLG_FIXJOURNAL = 0x100000000U,	     /*%< journal file had
+						      * recoverable error,
+						      * needs rewriting */
 	DNS_ZONEFLG___MAX = UINT64_MAX, /* trick to make the ENUM 64-bit wide */
 } dns_zoneflg_t;
 
@@ -890,6 +893,8 @@ static isc_result_t
 zone_send_securedb(dns_zone_t *zone, dns_db_t *db);
 static void
 setrl(isc_ratelimiter_t *rl, unsigned int *rate, unsigned int value);
+static void
+zone_journal_compact(dns_zone_t *zone, dns_db_t *db, uint32_t serial);
 
 #define ENTER zone_debuglog(zone, me, 1, "enter")
 
@@ -4743,6 +4748,7 @@ zone_postload(dns_zone_t *zone, dns_db_t *db, isc_time_t loadtime,
 	uint32_t serial, oldserial, refresh, retry, expire, minimum;
 	isc_time_t now;
 	bool needdump = false;
+	bool fixjournal = false;
 	bool hasinclude = DNS_ZONE_FLAG(zone, DNS_ZONEFLG_HASINCLUDE);
 	bool nomaster = false;
 	bool had_db = false;
@@ -4844,9 +4850,9 @@ zone_postload(dns_zone_t *zone, dns_db_t *db, isc_time_t loadtime,
 		}
 		result = dns_journal_rollforward(zone->mctx, db, options,
 						 zone->journal);
-		if (result != ISC_R_SUCCESS && result != ISC_R_NOTFOUND &&
-		    result != DNS_R_UPTODATE && result != DNS_R_NOJOURNAL &&
-		    result != ISC_R_RANGE)
+		if (result != ISC_R_SUCCESS && result != DNS_R_RECOVERABLE &&
+		    result != ISC_R_NOTFOUND && result != DNS_R_UPTODATE &&
+		    result != DNS_R_NOJOURNAL && result != ISC_R_RANGE)
 		{
 			dns_zone_logc(zone, DNS_LOGCATEGORY_ZONELOAD,
 				      ISC_LOG_ERROR,
@@ -4867,6 +4873,12 @@ zone_postload(dns_zone_t *zone, dns_db_t *db, isc_time_t loadtime,
 			      dns_result_totext(result));
 		if (result == ISC_R_SUCCESS) {
 			needdump = true;
+		} else if (result == DNS_R_RECOVERABLE) {
+			dns_zone_logc(zone, DNS_LOGCATEGORY_ZONELOAD,
+				      ISC_LOG_ERROR,
+				      "retried using old journal format");
+			needdump = true;
+			fixjournal = true;
 		}
 	}
 
@@ -5191,10 +5203,16 @@ zone_postload(dns_zone_t *zone, dns_db_t *db, isc_time_t loadtime,
 	if (needdump) {
 		if (zone->type == dns_zone_key) {
 			zone_needdump(zone, 30);
+		} else {
+			if (fixjournal) {
+				DNS_ZONE_SETFLAG(zone, DNS_ZONEFLG_FIXJOURNAL);
+				zone_journal_compact(zone, zone->db, 0);
+				zone_needdump(zone, 0);
 			} else {
 				zone_needdump(zone, DNS_DUMP_DELAY);
 			}
 		}
+	}
 
 	if (zone->task != NULL) {
 		if (zone->type == dns_zone_master) {
@@ -11435,6 +11453,7 @@ zone_journal_compact(dns_zone_t *zone, dns_db_t *db, uint32_t serial) {
 	int32_t journalsize;
 	dns_dbversion_t *ver = NULL;
 	uint64_t dbsize;
+	uint32_t options = 0;
 
 	INSIST(LOCKED_ZONE(zone));
 	if (inline_raw(zone)) {
@@ -11456,9 +11475,16 @@ zone_journal_compact(dns_zone_t *zone, dns_db_t *db, uint32_t serial) {
 			journalsize = (int32_t)dbsize * 2;
 		}
 	}
-	zone_debuglog(zone, "zone_journal_compact", 1, "target journal size %d",
-		      journalsize);
-	result = dns_journal_compact(zone->mctx, zone->journal, serial,
+	if (DNS_ZONE_FLAG(zone, DNS_ZONEFLG_FIXJOURNAL)) {
+		options |= DNS_JOURNAL_COMPACTALL;
+		DNS_ZONE_CLRFLAG(zone, DNS_ZONEFLG_FIXJOURNAL);
+		zone_debuglog(zone, "zone_journal_compact", 1,
+			      "repair full journal");
+	} else {
+		zone_debuglog(zone, "zone_journal_compact", 1,
+			      "target journal size %d", journalsize);
+	}
+	result = dns_journal_compact(zone->mctx, zone->journal, serial, options,
 				     journalsize);
 	switch (result) {
 	case ISC_R_SUCCESS:
@@ -11486,6 +11512,7 @@ dns_zone_flush(dns_zone_t *zone) {
 	DNS_ZONE_SETFLAG(zone, DNS_ZONEFLG_FLUSH);
 	if (DNS_ZONE_FLAG(zone, DNS_ZONEFLG_NEEDDUMP) &&
 	    zone->masterfile != NULL) {
+		DNS_ZONE_SETFLAG(zone, DNS_ZONEFLG_NEEDCOMPACT);
 		result = ISC_R_ALREADYRUNNING;
 		dumping = was_dumping(zone);
 	} else {

util/copyrights

@@ -514,6 +514,18 @@
 ./bin/tests/system/ixfr/ixfr-stats.good		X	2019,2020,2021
 ./bin/tests/system/ixfr/setup.sh		SH	2001,2004,2007,2011,2012,2013,2014,2016,2018,2019,2020,2021
 ./bin/tests/system/ixfr/tests.sh		SH	2001,2004,2007,2011,2012,2014,2016,2018,2019,2020,2021
+./bin/tests/system/journal/clean.sh		SH	2021
+./bin/tests/system/journal/ns1/changed.ver1.jnl.saved	X	2021
+./bin/tests/system/journal/ns1/changed.ver2.jnl.saved	X	2021
+./bin/tests/system/journal/ns1/d1212.jnl.saved	X	2021
+./bin/tests/system/journal/ns1/d2121.jnl.saved	X	2021
+./bin/tests/system/journal/ns1/ixfr.ver1.jnl.saved	X	2021
+./bin/tests/system/journal/ns1/maxjournal.jnl.saved	X	2021
+./bin/tests/system/journal/ns1/maxjournal2.jnl.saved	X	2021
+./bin/tests/system/journal/ns1/unchanged.ver1.jnl.saved	X	2021
+./bin/tests/system/journal/ns1/unchanged.ver2.jnl.saved	X	2021
+./bin/tests/system/journal/setup.sh		SH	2021
+./bin/tests/system/journal/tests.sh		SH	2021
 ./bin/tests/system/kasp/README			TXT.BRIEF	2019,2020,2021
 ./bin/tests/system/kasp/clean.sh		SH	2019,2020,2021
 ./bin/tests/system/kasp/ns2/setup.sh		SH	2019,2020,2021