[#3770] Update dhcp4_options in schema

Change dhcp4_options.client_classes NULL values with '[  ]'
Change dhcp4_options.client_classes column to NOT NULL

modified:   src/bin/admin/tests/mysql_tests.sh.in
modified:   src/bin/admin/tests/pgsql_tests.sh.in
modified:   src/lib/mysql/mysql_constants.h
modified:   src/lib/pgsql/pgsql_connection.h
modified:   src/share/database/scripts/mysql/dhcpdb_create.mysql
modified:   src/share/database/scripts/mysql/meson.build
new file:   src/share/database/scripts/mysql/upgrade_030_to_031.sh.in
modified:   src/share/database/scripts/pgsql/dhcpdb_create.pgsql
modified:   src/share/database/scripts/pgsql/meson.build
new file:   src/share/database/scripts/pgsql/upgrade_029_to_030.sh.in

src/bin/admin/tests/mysql_tests.sh.in

@@ -157,7 +157,7 @@ mysql_db_version_test() {
     run_command \
         "${kea_admin}" db-version mysql -u "${db_user}" -p "${db_password}" -n "${db_name}"
     version="${OUTPUT}"
-    assert_str_eq "30.0" "${version}" "Expected kea-admin to return %s, returned value was %s"
+    assert_str_eq "31.0" "${version}" "Expected kea-admin to return %s, returned value was %s"
 
     # Let's wipe the whole database
     mysql_wipe
@@ -1157,6 +1157,18 @@ mysql_upgrade_29_to_30_test() {
     assert_str_eq 'NULL' "${OUTPUT}" "${query}: expected output %s, returned %s"
 }
 
+mysql_upgrade_30_to_31_test() {
+    # Check dhcp4.client_classes now needs a value
+    qry="SET @disable_audit = 1;\
+        INSERT INTO dhcp4_options (scope_id, code, space, formatted_value, modification_ts) \
+        VALUES (0, '223', 'dhcp4', 'foo' , current_timestamp );";
+
+    # Should fail with "ERROR 1364 (HY000): Field 'client_classes' doesn't have a default value"
+    run_command \
+        mysql_execute "${qry}"
+    assert_eq 1 "${EXIT_CODE}" "${qry}: expected %d, returned %d"
+}
+
 mysql_upgrade_test() {
 
     test_start "mysql.upgrade"
@@ -1178,7 +1190,7 @@ mysql_upgrade_test() {
 
     # Verify that the upgraded schema reports the latest version.
     version=$("${kea_admin}" db-version mysql -u "${db_user}" -p "${db_password}" -n "${db_name}" -d "${db_scripts_dir}")
-    assert_str_eq "30.0" "${version}" "Expected kea-admin to return %s, returned value was %s"
+    assert_str_eq "31.0" "${version}" "Expected kea-admin to return %s, returned value was %s"
 
     # Let's check that the new tables are indeed there.
 
@@ -1773,7 +1785,8 @@ insert into hosts(dhcp_identifier, dhcp_identifier_type, dhcp4_subnet_id, ipv4_a
     qry="\
 SET @disable_audit = 1;\
 INSERT INTO dhcp4_client_class(name, modification_ts) VALUES ('foo', now());\
-INSERT INTO dhcp4_options(code, scope_id, dhcp_client_class, modification_ts) VALUES (222, 2, 'foo', now());\
+INSERT INTO dhcp4_options(code, scope_id, dhcp_client_class, modification_ts, client_classes)\
+            VALUES (222, 2, 'foo', now(), '[  ]');\
 INSERT INTO dhcp6_client_class(name, modification_ts) VALUES ('foo', now());\
 INSERT INTO dhcp6_options(code, scope_id, dhcp_client_class, modification_ts) VALUES (222, 2, 'foo', now());\
 SET @disable_audit = 0"
@@ -1865,6 +1878,9 @@ SET @disable_audit = 0"
     # Check upgrade from 29.0 to 30.0.
     mysql_upgrade_29_to_30_test
 
+    # Check upgrade from 29.0 to 30.0.
+    mysql_upgrade_30_to_31_test
+
     # Let's wipe the whole database
     mysql_wipe
 
@@ -3965,6 +3981,56 @@ mysql_migrate_client_class_test() {
     test_finish 0
 }
 
+# Verifies migration of dhcp4_options.client_classes column to not null.
+mysql_migrate_dhcp4_options_client_classes() {
+    test_start "mysql.mysql_migrate_dhcp4_options_client_classes"
+
+    # Let's wipe the whole database
+    mysql_wipe
+
+    # We need to create an older database with lease data so we can
+    # verify the upgrade mechanisms which prepopulate the lease stat
+    # tables.
+    #
+    # Initialize database to schema 1.0.
+    mysql_execute_script "@abs_top_srcdir@/src/bin/admin/tests/dhcpdb_create_1.0.mysql"
+    assert_eq 0 "${EXIT_CODE}" "cannot initialize 1.0 database, expected exit code: %d, actual: %d"
+
+    # Now upgrade to schema 30.0
+    mysql_upgrade_schema_to_version 30.0
+
+    # Now insert options.
+    sql="\
+    set @disable_audit = 1;\
+    insert into dhcp4_options \
+        (scope_id, code, space, formatted_value, modification_ts) \
+        values (0, 222, 'dhcp4', 'foo' , current_timestamp); \
+    insert into dhcp4_options \
+        (scope_id, code, space, formatted_value, client_classes, modification_ts) \
+        values (0, 223, 'dhcp4', 'bar' , '[ \"cc-one\" ]', current_timestamp);"
+
+    run_statement "insert option" "$sql"
+
+    # Verify the inserted record counts.
+    qry="select count(*) from dhcp4_options;"
+    run_statement "#get 4_option_count before update" "$qry" 2
+
+    # Upgrade to schema 31.0
+    mysql_upgrade_schema_to_version 31.0
+
+    # Verify the migrated records.
+    qry="select code from dhcp4_options where client_classes = '[  ]';"
+    run_statement "#get 4_option_222 after update" "$qry" 222 
+
+    qry="select code from dhcp4_options where client_classes = '[ \"cc-one\" ]';"
+    run_statement "#get 4_option_223 after update" "$qry" 223 
+
+    # Let's wipe the whole database
+    mysql_wipe
+
+    test_finish 0
+}
+
 # Run tests.
 mysql_db_init_test
 mysql_host_reservation_init_test
@@ -4000,3 +4066,4 @@ mysql_reservation_mode_out_of_pool_parameters_test
 mysql_migrate_opt_record_type
 mysql_remove_control_socket_parameters_test
 mysql_migrate_client_class_test
+mysql_migrate_dhcp4_options_client_classes

src/bin/admin/tests/pgsql_tests.sh.in

@@ -158,7 +158,7 @@ pgsql_db_version_test() {
     run_command \
         "${kea_admin}" db-version pgsql -u "${db_user}" -p "${db_password}" -n "${db_name}"
     version="${OUTPUT}"
-    assert_str_eq "29.0" "${version}" "Expected kea-admin to return %s, returned value was %s"
+    assert_str_eq "30.0" "${version}" "Expected kea-admin to return %s, returned value was %s"
 
     # Let's wipe the whole database
     pgsql_wipe
@@ -1129,6 +1129,19 @@ pgsql_upgrade_28_to_29_test() {
     assert_str_eq 'registered' "${OUTPUT}" "${query}: expected output %s, returned %s"
 }
 
+pgsql_upgrade_29_to_30_test() {
+    # Check dhcp4.client_classes now needs a value
+    qry="SELECT set_config('kea.disable_audit', 'true', false);\
+        INSERT INTO dhcp4_options (scope_id, code, space, formatted_value, modification_ts) \
+        VALUES (0, '223', 'dhcp4', 'foo' , current_timestamp );";
+
+    # Should fail with "# ERROR:  null value in column "client_classes" of relation "dhcp4_options"
+    # violates not-null constraint"
+    run_command \
+        pgsql_execute "${qry}"
+    assert_eq 3 "${EXIT_CODE}" "${qry}: expected %d, returned %d"
+}
+
 pgsql_upgrade_test() {
     test_start "pgsql.upgrade"
 
@@ -1147,7 +1160,7 @@ pgsql_upgrade_test() {
 
     # Verify upgraded schema reports the latest version.
     version=$("${kea_admin}" db-version pgsql -u "${db_user}" -p "${db_password}" -n "${db_name}" -d "${db_scripts_dir}")
-    assert_str_eq "29.0" "${version}" 'Expected kea-admin to return %s, returned value was %s'
+    assert_str_eq "30.0" "${version}" 'Expected kea-admin to return %s, returned value was %s'
 
     # Check 1.0 to 2.0 upgrade
     pgsql_upgrade_1_0_to_2_0_test
@@ -1227,6 +1240,10 @@ pgsql_upgrade_test() {
     # Check 28 to 29 upgrade
     pgsql_upgrade_28_to_29_test
 
+    # Check 29 to 30 upgrade
+    pgsql_upgrade_29_to_30_test
+
+
     # Let's wipe the whole database
     pgsql_wipe
 
@@ -3106,6 +3123,54 @@ pgsql_migrate_client_class_test() {
     test_finish 0
 }
 
+# Verifies migration of dhcp4_options.client_classes column to not null.
+pgsql_migrate_dhcp4_options_client_classes() {
+    test_start "pgsql.pgsql_migrate_dhcp4_options_client_classes"
+
+    # Let's wipe the whole database
+    pgsql_wipe
+
+    # We need to create an older database with lease data so we can
+    # verify the upgrade mechanisms which prepopulate the lease stat
+    # tables.
+    #
+    # Initialize database to schema 1.0.
+    pgsql_execute_script "@abs_top_srcdir@/src/bin/admin/tests/dhcpdb_create_1.0.pgsql"
+    assert_eq 0 "${EXIT_CODE}" "cannot initialize 1.0 database, expected exit code: %d, actual: %d"
+
+    # Now upgrade to schema 29.0
+    pgsql_upgrade_schema_to_version 29.0
+
+    # Now insert options.
+    sql="\
+    select set_config('kea.disable_audit', 'true', false);\
+    insert into dhcp4_options (scope_id, code, space, formatted_value, modification_ts) \
+        values (0, '222', 'dhcp4', 'foo' , current_timestamp); \
+    insert into dhcp4_options (scope_id, code, space, formatted_value, client_classes, modification_ts) \
+        values (0, '223', 'dhcp4', 'bar' , '[ \"cc-one\" ]', current_timestamp); \
+    select set_config('kea.disable_audit', 'false', false);"
+
+    run_statement "insert option" "$sql"
+
+    # Verify the inserted record counts.
+    qry="select count(*) from dhcp4_options;"
+    run_statement "#get 4_option_count before update" "$qry" 2
+
+    # Upgrade to schema 30.0
+    pgsql_upgrade_schema_to_version 30.0
+
+    # Verify the migrated records.
+    qry="select code from dhcp4_options where client_classes = '[  ]';"
+    run_statement "#get 4_option_foo after update" "$qry" 222 
+
+    qry="select code from dhcp4_options where client_classes = '[ \"cc-one\" ]';"
+    run_statement "#get 4_option_foo after update" "$qry" 223 
+
+    # Let's wipe the whole database
+    pgsql_wipe
+
+    test_finish 0
+}
 
 # Run tests.
 pgsql_db_init_test
@@ -3136,3 +3201,4 @@ pgsql_reservation_mode_out_of_pool_parameters_test
 pgsql_migrate_opt_record_type
 pgsql_remove_control_socket_parameters_test
 pgsql_migrate_client_class_test
+pgsql_migrate_dhcp4_options_client_classes

src/lib/mysql/mysql_constants.h

@@ -52,7 +52,7 @@ const int MLM_MYSQL_FETCH_FAILURE = 0;
 
 /// @name Current database schema version values.
 //@{
-const uint32_t MYSQL_SCHEMA_VERSION_MAJOR = 30;
+const uint32_t MYSQL_SCHEMA_VERSION_MAJOR = 31;
 const uint32_t MYSQL_SCHEMA_VERSION_MINOR = 0;
 
 //@}

src/lib/pgsql/pgsql_connection.h

@@ -18,7 +18,7 @@ namespace isc {
 namespace db {
 
 /// @brief Define the PostgreSQL backend version.
-const uint32_t PGSQL_SCHEMA_VERSION_MAJOR = 29;
+const uint32_t PGSQL_SCHEMA_VERSION_MAJOR = 30;
 const uint32_t PGSQL_SCHEMA_VERSION_MINOR = 0;
 
 // Maximum number of parameters that can be used a statement

src/share/database/scripts/mysql/dhcpdb_create.mysql

@@ -6436,6 +6436,19 @@ UPDATE schema_version
 
 -- This line concludes the schema upgrade to version 30.0.
 
+-- This line starts the schema upgrade to version 31.0.
+
+SET @disable_audit = 1;
+UPDATE dhcp4_options SET client_classes = '[  ]' WHERE client_classes IS NULL;
+ALTER TABLE dhcp4_options MODIFY COLUMN client_classes LONGTEXT NOT NULL;
+SET @disable_audit = 0;
+
+-- Update the schema version number.
+UPDATE schema_version
+    SET version = '31', minor = '0';
+
+-- This line concludes the schema upgrade to version 31.0.
+
 # Notes:
 #
 # Indexes

src/share/database/scripts/mysql/meson.build

@@ -71,6 +71,7 @@ upgrade_scripts = [
     'upgrade_027_to_028.sh',
     'upgrade_028_to_029.sh',
     'upgrade_029_to_030.sh',
+    'upgrade_030_to_031.sh',
 ]
 list = run_command(
     GRABBER,

src/share/database/scripts/mysql/upgrade_030_to_031.sh.in

@@ -0,0 +1,69 @@
+#!/bin/sh
+
+# Copyright (C) 2025 Internet Systems Consortium, Inc. ("ISC") #
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+# Exit with error if commands exit with non-zero and if undefined variables are
+# used.
+set -eu
+
+# shellcheck disable=SC2034
+# SC2034: ... appears unused. Verify use (or export if used externally).
+prefix="@prefix@"
+
+# Include utilities based on location of this script. Check for sources first,
+# so that the unexpected situations with weird paths fall on the default
+# case of installed.
+script_path=$(cd "$(dirname "${0}")" && pwd)
+if test "${script_path}" = "@abs_top_builddir@/src/share/database/scripts/mysql"; then
+    # shellcheck source=./src/bin/admin/admin-utils.sh.in
+    . "@abs_top_builddir@/src/bin/admin/admin-utils.sh"
+else
+    # shellcheck source=./src/bin/admin/admin-utils.sh.in
+    . "@datarootdir@/@PACKAGE_NAME@/scripts/admin-utils.sh"
+fi
+
+# Check only major version to allow for intermediary backported schema changes.
+version=$(mysql_version "${@}" | cut -d '.' -f 1)
+if test "${version}" != '30'; then
+    printf 'This script upgrades 30.* to 31.0. '
+    printf 'Reported version is %s. Skipping upgrade.\n' "${version}"
+    exit 0
+fi
+
+# Get the schema name from database argument. We need this to
+# query information_schema for the right database.
+for arg in "${@}"
+do
+    if ! printf '%s' "${arg}" | grep -Eq -- '^--'
+    then
+        schema="$arg"
+        break
+    fi
+done
+
+# Make sure we have the schema.
+if [ -z "$schema" ]
+then
+    printf "Could not find database schema name in cmd line args: %s\n" "${*}"
+    exit 255
+fi
+
+mysql "$@" <<EOF
+
+-- This line starts the schema upgrade to version 31.0.
+
+SET @disable_audit = 1;
+UPDATE dhcp4_options SET client_classes = '[  ]' WHERE client_classes IS NULL;
+ALTER TABLE dhcp4_options MODIFY COLUMN client_classes LONGTEXT NOT NULL;
+SET @disable_audit = 0;
+
+-- Update the schema version number.
+UPDATE schema_version
+    SET version = '31', minor = '0';
+
+-- This line concludes the schema upgrade to version 31.0.
+
+EOF

src/share/database/scripts/pgsql/dhcpdb_create.pgsql

@@ -6711,6 +6711,19 @@ UPDATE schema_version
 
 -- This line concludes the schema upgrade to version 29.0.
 
+-- This line starts the schema upgrade to version 30.0.
+
+SELECT set_config('kea.disable_audit', 'true', false);
+UPDATE dhcp4_options SET client_classes = '[  ]' WHERE client_classes IS NULL;
+ALTER TABLE dhcp4_options ALTER COLUMN client_classes SET NOT NULL;
+SELECT set_config('kea.disable_audit', 'false', false);
+
+-- Update the schema version number.
+UPDATE schema_version
+    SET version = '30', minor = '0';
+
+-- This line concludes the schema upgrade to version 30.0.
+
 -- Commit the script transaction.
 COMMIT;
 

src/share/database/scripts/pgsql/meson.build

@@ -65,6 +65,7 @@ upgrade_scripts = [
     'upgrade_026_to_027.sh',
     'upgrade_027_to_028.sh',
     'upgrade_028_to_029.sh',
+    'upgrade_029_to_030.sh',
 ]
 list = run_command(
     GRABBER,

src/share/database/scripts/pgsql/upgrade_029_to_030.sh.in

@@ -0,0 +1,56 @@
+#!/bin/sh
+
+# Copyright (C) 2024-2025 Internet Systems Consortium, Inc. ("ISC")
+#
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+# Exit with error if commands exit with non-zero and if undefined variables are
+# used.
+set -eu
+
+# shellcheck disable=SC2034
+# SC2034: ... appears unused. Verify use (or export if used externally).
+prefix="@prefix@"
+
+# Include utilities based on location of this script. Check for sources first,
+# so that the unexpected situations with weird paths fall on the default
+# case of installed.
+script_path=$(cd "$(dirname "${0}")" && pwd)
+if test "${script_path}" = "@abs_top_builddir@/src/share/database/scripts/pgsql"; then
+    # shellcheck source=./src/bin/admin/admin-utils.sh.in
+    . "@abs_top_builddir@/src/bin/admin/admin-utils.sh"
+else
+    # shellcheck source=./src/bin/admin/admin-utils.sh.in
+    . "@datarootdir@/@PACKAGE_NAME@/scripts/admin-utils.sh"
+fi
+
+# Check only major version to allow for intermediary backported schema changes.
+version=$(pgsql_version "${@}" | cut -d '.' -f 1)
+if test "${version}" != '29'; then
+    printf 'This script upgrades 29.* to 30.0. '
+    printf 'Reported version is %s. Skipping upgrade.\n' "${version}"
+    exit 0
+fi
+
+psql "$@" >/dev/null <<EOF
+START TRANSACTION;
+
+-- This line starts the schema upgrade to version 30.0.
+
+SELECT set_config('kea.disable_audit', 'true', false);
+UPDATE dhcp4_options SET client_classes = '[  ]' WHERE client_classes IS NULL;
+ALTER TABLE dhcp4_options ALTER COLUMN client_classes SET NOT NULL;
+SELECT set_config('kea.disable_audit', 'false', false);
+
+-- Update the schema version number.
+UPDATE schema_version
+    SET version = '30', minor = '0';
+
+-- This line concludes the schema upgrade to version 30.0.
+
+-- Commit the script transaction.
+COMMIT;
+
+EOF