kea/src/lib/testutils/dhcp_test_lib.sh.in

#!/bin/sh

# Copyright (C) 2014-2020 Internet Systems Consortium, Inc. ("ISC")
#
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.

# shellcheck disable=SC1091
# SC1091: Not following: ... was not specified as input (see shellcheck -x).

# shellcheck disable=SC2034
# SC2034: ... appears unused. Verify use (or export if used externally).

# shellcheck disable=SC2039
# SC2039: In POSIX sh, 'local' is undefined.

# shellcheck disable=SC2153
# SC2153: Possible misspelling: ... may not be assigned, but ... is.

# shellcheck disable=SC2154
# SC2154: bin_path is referenced but not assigned.

# Exit with error if commands exit with non-zero and if undefined variables are
# used.
set -eu

# Include XML reporting library.
. "@abs_top_builddir@/src/lib/testutils/xml_reporting_test_lib.sh"

prefix="@prefix@"

# Expected version
EXPECTED_VERSION="@PACKAGE_VERSION@"

# A list of Kea processes, mainly used by the cleanup functions.
KEA_PROCS="kea-dhcp4 kea-dhcp6 kea-dhcp-ddns kea-ctrl-agent"

### Colors ###

if test -t 1; then
  green='\033[92m'
  red='\033[91m'
  reset='\033[0m'
fi

### Logging functions ###

# Prints error message.
test_lib_error() {
    local s=${1}            # Error message.
    local no_new_line=${2}  # If specified, the message not terminated with
                            # new line.
    printf "ERROR/test_lib: %s" "${s}"
    if [ -z "${no_new_line}" ]; then
        printf '\n'
    fi
}

# Prints info message.
test_lib_info() {
    local s=${1}            # Info message.
    local no_new_line=${2}  # If specified, the message is not terminated with
                            # new line.
    printf "INFO/test_lib: %s" "${s}"
    if [ -z "${no_new_line}" ]; then
        printf '\n'
    fi
}

### Assertions ###

# Assertion that checks if two numbers are equal.
# If numbers are not equal, the mismatched values are presented and the
# detailed error is printed. The detailed error must use the printf
# formatting like this:
#    "Expected that some value 1 %d is equal to some other value %d".
assert_eq() {
    val1=${1}         # Reference value
    val2=${2}         # Tested value
    detailed_err=${3} # Detailed error format string
    # If nothing found, present an error an exit.
    if [ "${val1}" -ne "${val2}" ]; then
        printf "Assertion failure: %s != %s, for val1=%s, val2=%s\n" \
            "${val1}" "${val2}" "${val1}" "${val2}"
        # shellcheck disable=SC2059
        # SC2059: Don't use variables in the printf format string. Use printf '..%s..' "$foo"
        ERROR=$(printf "${detailed_err}" "${val1}" "${val2}")
        printf '%s\n%s\n' "${ERROR}" "${OUTPUT}" >&2
        clean_exit 1
    fi
}

# Assertion that checks if two strings are equal.
# If numbers are not equal, the mismatched values are presented and the
# detailed error is printed. The detailed error must use the printf
# formatting like this:
#    "Expected that some value 1 %d is equal to some other value %d".
assert_str_eq() {
    val1=${1}         # Reference value
    val2=${2}         # Tested value
    detailed_err=${3} # Detailed error format string
    # If nothing found, present an error an exit.
    if [ "${val1}" != "${val2}" ]; then
        printf 'Assertion failure: %s != %s, for val1=%s, val2=%s\n' \
            "${val1}" "${val2}" "${val1}" "${val2}"
        # shellcheck disable=SC2059
        # SC2059: SC2059: Don't use variables in the printf format string. Use printf '..%s..' "$foo".
        ERROR=$(printf "${detailed_err}" "${val1}" "${val2}")
        printf '%s\n%s\n' "${ERROR}" "${OUTPUT}" >&2
        clean_exit 1
    fi
}

# Assertion that checks if one string contains another string.
# If assertion fails, both strings are displayed and the detailed
# error is printed. The detailed error must use the printf formatting
# like this:
#    "Expected some string to contain this string: %s".
assert_string_contains() {
    pattern="${1}"      # Substring or awk pattern
    text="${2}"         # Text to be searched for substring
    detailed_err="${3}" # Detailed error format string
    # Search for a pattern
    match=$( printf "%s" "${text}" | awk /"${pattern}"/ )
    # If nothing found, present an error and exit.
    if [ -z "${match}" ]; then
        ERROR=$(printf \
"Assertion failure:
\"%s\"

does not contain pattern:
\"%s\"

${detailed_err}
" "${text}" "${pattern}" "${pattern}")
        printf '%s\n%s\n' "${ERROR}" "${OUTPUT}" >&2
        clean_exit 1
    fi
}

# Runs all the given arguments as a single command. Maintains quoting. Places
# output in ${OUTPUT} and exit code in ${EXIT_CODE}. Does not support pipes and
# redirections. Support for them could be added through eval and single
# parameter assignment, but eval is not recommended.
# shellcheck disable=SC2034
# SC2034: ... appears unused. Verify use (or export if used externally).
run_command() {
    if test -n "${DEBUG+x}"; then
        printf '%s\n' "${*}" >&2
    fi
    set +e
    OUTPUT=$("${@}")
    EXIT_CODE=${?}
    set -e
}

# Enable traps to print FAILED status when a command fails unexpectedly or when
# the user sends a SIGINT. Used in `test_start`.
traps_on() {
    for t in HUP INT QUIT KILL TERM EXIT; do
        # shellcheck disable=SC2064
        # SC2064: Use single quotes, otherwise this expands now rather than when signalled.
        # reason: we want ${red-} and ${reset-} to expand here, at trap-time
        # they will be empty or have other values
        trap "
            exit_code=\${?}
            printf '${red-}[  FAILED  ]${reset-} %s (exit code: %d)\n' \
                \"\${TEST_NAME}\" \"\${exit_code}\"
        " "${t}"
    done
}

# Disable traps so that a double status is not printed. Used in `test_finish`
# after the status has been printed explicitly.
traps_off() {
    for t in HUP INT QUIT KILL TERM EXIT; do
        trap - "${t}"
    done
}

# Print UNIX time with millisecond resolution.
get_current_time() {
    local time
    time=$(date +%s%3N)

    # In some systems, particularly BSD-based, `+%3N` millisecond resolution is
    # not supported. It instead prints the literal '3N', but we check for any
    # alphabetical character. If we do find one, revert to second resolution and
    # convert to milliseconds.
    if printf '%s' "${time}" | grep -E '[A-Za-z]' > /dev/null 2>&1; then
        time=$(date +%s)
        time=$((1000 * time))
    fi

    printf '%s' "${time}"
}

# Begins a test by printing its name.
test_start() {
    TEST_NAME=${1}
    if [ -z "${TEST_NAME}" ]; then
        test_lib_error "test_start requires test name as an argument"
        clean_exit 1
    fi

    # Set traps first to fail if something goes wrong.
    traps_on

    # Announce test start.
    printf "${green-}[ RUN      ]${reset-} %s\n" "${TEST_NAME}"

    # Reset variables that are used in error reporting.
    ERROR=
    OUTPUT=

    # Start timer in milliseconds.
    START_TIME=$(get_current_time)
}

# Prints test result an cleans up after the test.
test_finish() {
    # Exit code to be returned by the exit function
    local exit_code=${1}

    # Stop timer and set duration.
    FINISH_TIME=$(get_current_time)
    local duration
    duration=$((FINISH_TIME - START_TIME))

    # Add the test result to the XML.
    report_test_result_in_xml "${TEST_NAME}" "${exit_code}" "${duration}"

    if [ "${exit_code}" -eq 0 ]; then
        cleanup
        printf "${green-}[       OK ]${reset-} %s\n" "${TEST_NAME}"
    else
        # Dump log file for debugging purposes if specified and exists.
        # Otherwise the code below would simply call cat.
        # Use ${var+x} to test if ${var} is defined.
        if test -n "${LOG_FILE+x}" && test -s "${LOG_FILE}"; then
            printf 'Log file dump:\n'
            cat "${LOG_FILE}"
        fi
        cleanup
        printf "${red-}[  FAILED  ]${reset-} %s\n" "${TEST_NAME}"
    fi

    # Reset traps.
    traps_off

    # Explicitly return ${exit_code}. The effect should be for `make check` to
    # return with the exit same code or at least another non-zero exit code thus
    # reporting a failure.
    return "${exit_code}"
}

# Stores the configuration specified as a parameter in the configuration
# file which name has been set in the ${CFG_FILE} variable.
create_config() {
    local cfg="${1}"  # Configuration string.
    if [ -z "${CFG_FILE}" ]; then
        test_lib_error "create_config requires CFG_FILE variable be set"
        clean_exit 1

    elif [ -z "${cfg}" ]; then
        test_lib_error "create_config requires argument holding a configuration"
        clean_exit 1
    fi
    printf 'Creating Kea configuration file: %s.\n' "${CFG_FILE}"
    printf '%b' "${cfg}" > "${CFG_FILE}"
}

# Stores the DHCP4 configuration specified as a parameter in the
# configuration file which name has been set in the ${DHCP4_CFG_FILE}
# variable.
create_dhcp4_config() {
    local cfg="${1}"  # Configuration string.
    if [ -z "${DHCP4_CFG_FILE}" ]; then
        test_lib_error "create_dhcp4_config requires DHCP4_CFG_FILE \
variable be set"
        clean_exit 1

    elif [ -z "${cfg}" ]; then
        test_lib_error "create_dhcp4_config requires argument holding a \
configuration"
        clean_exit 1
    fi
    printf 'Creating Dhcp4 configuration file: %s.\n' "${DHCP4_CFG_FILE}"
    printf '%b' "${cfg}" > "${DHCP4_CFG_FILE}"
}

# Stores the DHCP6 configuration specified as a parameter in the
# configuration file which name has been set in the ${DHCP6_CFG_FILE}
# variable.
create_dhcp6_config() {
    local cfg="${1}"  # Configuration string.
    if [ -z "${DHCP6_CFG_FILE}" ]; then
        test_lib_error "create_dhcp6_config requires DHCP6_CFG_FILE \
variable be set"
        clean_exit 1

    elif [ -z "${cfg}" ]; then
        test_lib_error "create_dhcp6_config requires argument holding a \
configuration"
        clean_exit 1
    fi
    printf 'Creating Dhcp6 configuration file: %s.\n' "${DHCP6_CFG_FILE}"
    printf '%b' "${cfg}" > "${DHCP6_CFG_FILE}"
}

# Stores the D2 configuration specified as a parameter in the
# configuration file which name has been set in the ${D2_CFG_FILE}
# variable.
create_d2_config() {
    local cfg="${1}"  # Configuration string.
    if [ -z "${D2_CFG_FILE}" ]; then
        test_lib_error "create_d2_config requires D2_CFG_FILE \
variable be set"
        clean_exit 1

    elif [ -z "${cfg}" ]; then
        test_lib_error "create_d2_config requires argument holding a \
configuration"
        clean_exit 1
    fi
    printf 'Creating D2 configuration file: %s.\n' "${D2_CFG_FILE}"
    printf '%b' "${cfg}" > "${D2_CFG_FILE}"
}

# Stores the CA configuration specified as a parameter in the
# configuration file which name has been set in the ${CA_CFG_FILE}
# variable.
create_ca_config() {
    local cfg="${1}"  # Configuration string.
    if [ -z "${CA_CFG_FILE}" ]; then
        test_lib_error "create_ca_config requires CA_CFG_FILE \
variable be set"
        clean_exit 1

    elif [ -z "${cfg}" ]; then
        test_lib_error "create_ca_config requires argument holding a \
configuration"
        clean_exit 1
    fi
    printf 'Creating Ca configuration file: %s.\n' "${CA_CFG_FILE}"
    printf '%b' "${cfg}" > "${CA_CFG_FILE}"
}

# Stores the NC configuration specified as a parameter in the
# configuration file which name has been set in the ${NC_CFG_FILE}
# variable.
create_nc_config() {
    local cfg="${1}"  # Configuration string.
    if [ -z "${NC_CFG_FILE}" ]; then
        test_lib_error "create_nc_config requires NC_CFG_FILE \
variable be set"
        clean_exit 1

    elif [ -z "${cfg}" ]; then
        test_lib_error "create_nc_config requires argument holding a \
configuration"
        clean_exit 1
    fi
    printf 'Creating Nc configuration file: %s.\n' "${NC_CFG_FILE}"
    printf '%b' "${cfg}" > "${NC_CFG_FILE}"
}

# Stores the keactrl configuration specified as a parameter in the
# configuration file which name has been set in the ${KEACTRL_CFG_FILE}
# variable.
create_keactrl_config() {
    local cfg="${1}" # Configuration string.
    if [ -z "${KEACTRL_CFG_FILE}" ]; then
        test_lib_error "create_keactrl_config requires KEACTRL_CFG_FILE \
variable be set"
        clean_exit 1

    elif [ -z "${cfg}" ]; then
        test_lib_error "create_keactrl_config requires argument holding a \
configuration"
        clean_exit 1
    fi
    printf 'Creating keactrl configuration file: %s.\n' "${KEACTRL_CFG_FILE}"
    printf '%b' "${cfg}" > "${KEACTRL_CFG_FILE}"
}

# Sets Kea logger to write to the file specified by the global value
# ${LOG_FILE}.
set_logger() {
    if [ -z "${LOG_FILE}" ]; then
        test_lib_error "set_logger requires LOG_FILE variable be set"
        clean_exit 1
    fi
    printf 'Kea log will be stored in %s.\n' "${LOG_FILE}"
    export KEA_LOGGER_DESTINATION=${LOG_FILE}
}

# PID file path is by default <kea-install-dir>/var/run/kea, but can be
# overridden by the environmental variable.
PID_FILE_PATH="@runstatedir@/@PACKAGE@/"
# Use ${var+x} to test if ${var} is defined.
if test -n "${KEA_PIDFILE_DIR+x}" && test -n "${KEA_PIDFILE_DIR}"; then
    PID_FILE_PATH="${KEA_PIDFILE_DIR}"
fi

# Checks if specified process is running.
#
# This function uses PID file to obtain the PID and then calls
# 'kill -0 <pid>' to check if the process is alive.
# The PID files are expected to be located in the ${PID_FILE_PATH},
# and their names should match the following pattern:
# <cfg_file_name>.<proc_name>.pid. If the <cfg_file_name> is not
# specified a 'test_config' is used by default.
#
# Return value:
#   _GET_PID: holds a PID if process is running
#   _GET_PIDS_NUM: holds 1 if process is running, 0 otherwise
get_pid() {
    local proc_name=${1}        # Process name
    local cfg_file_name=${2-}   # Configuration file name without extension.

    # PID file name includes process name. The process name is required.
    if [ -z "${proc_name}" ]; then
        test_lib_error "get_pid requires process name"
        clean_exit 1
    fi

    # PID file name includes server configuration file name. For most of
    # the tests it is 'test-config' (excluding .json extension). It is
    # possible to specify custom name if required.
    if [ -z "${cfg_file_name}" ]; then
        cfg_file_name="test_config"
    fi

    # Get the absolute location of the PID file for the specified process
    # name.
    abs_pidfile_path="${PID_FILE_PATH}/${cfg_file_name}.${proc_name}.pid"
    _GET_PID=0
    _GET_PIDS_NUM=0

    # If the PID file exists, get the PID and see if the process is alive.
    if [ -e "${abs_pidfile_path}" ]; then
        pid=$(cat "${abs_pidfile_path}")
        if kill -0 "${pid}" > /dev/null 2>&1; then
            _GET_PID=${pid}
            _GET_PIDS_NUM=1
        fi
    fi
}

# Get the name of the process identified by PID.
get_process_name() {
    local pid=${1}
    ps "${pid}" | tr -s ' ' | cut -d ' ' -f 6- | head -n 2 | tail -n 1
}

# Wait for file to be created.
wait_for_file() {
    local file=${1}
    timeout='4' # seconds
    deadline=$(($(date +%s) + timeout))
    while ! test -f "${file}"; do
        if test "${deadline}" -lt "$(date +%s)"; then
            # Time is up.
            printf 'ERROR: file "%s" was not created in time.\n' "${file}" >&2
            return 1
        fi
        printf 'Waiting for file "%s" to be created...\n' "${file}"
        sleep 1
    done
}

# Wait for process identified by PID to die.
wait_for_process_to_die() {
    local pid=${1}
    timeout='4' # seconds
    deadline=$(($(date +%s) + timeout))
    while ps "${pid}" >/dev/null; do
        if test "${deadline}" -lt "$(date +%s)"; then
            # Time is up.
            printf 'ERROR: %s does not want to die.\n' "$(get_process_name "${pid}")" >&2
            return 1
        fi
        printf 'Waiting for %s to die...\n' "$(get_process_name "${pid}")"
        sleep 1
    done
}

# Kills processes specified by name.
#
# This function kills all processes having a specified name.
# It uses 'pgrep' to obtain pids of those processes.
# This function should be used when identifying process by
# the value in its PID file is not relevant.
#
# Linux limitation for pgrep: The process name used for matching is
# limited to the 15 characters. If you call this with long process
# names, add this before pgrep:
# proc_name=$(printf '%s' "${proc_name}" | cut -c1-15)
kill_processes_by_name() {
    local proc_name=${1} # Process name
    if [ -z "${proc_name}" ]; then
        test_lib_error "get_pids requires process name"
        clean_exit 1
    fi
    # Obtain PIDs of running processes.
    local pids
    pids=$(pgrep "${proc_name}" || true)
    # For each PID found, send kill signal.
    for pid in ${pids}
    do
        printf 'Shutting down Kea process %s having pid %d.\n' "${proc_name}" "${pid}"
        kill -9 "${pid}"
    done
}

# Returns the number of occurrences of the Kea log message in the log file.
# Return value:
#   _GET_LOG_MESSAGES: number of log message occurrences.
get_log_messages() {
    local msg="${1}"  # Message id, e.g. DHCP6_SHUTDOWN
    if [ -z "${msg}" ]; then
        test_lib_error "get_log_messages require message identifier"
        clean_exit 1
    fi
    _GET_LOG_MESSAGES=0
    # If log file is not present, the number of occurrences is 0.
    # Use ${var+x} to test if ${var} is defined.
    if test -n "${LOG_FILE+x}" && test -s "${LOG_FILE}"; then
        # Grep log file for the logger message occurrences and remove
        # whitespaces, if any.
        _GET_LOG_MESSAGES=$(grep -Fo "${msg}" "${LOG_FILE}" | wc -w | tr -d " ")
    fi
}

# Returns the number of server configurations performed so far. Also
# returns the number of configuration errors.
# Return values:
#   _GET_RECONFIGS: number of configurations so far.
#   _GET_RECONFIG_ERRORS: number of configuration errors.
get_reconfigs() {
    # Grep log file for CONFIG_COMPLETE occurrences. There should
    # be one occurrence per (re)configuration.
    _GET_RECONFIGS=$(grep -Fo CONFIG_COMPLETE "${LOG_FILE}" | wc -w)
    # Grep log file for CONFIG_LOAD_FAIL to check for configuration
    # failures.
    _GET_RECONFIG_ERRORS=$(grep -Fo CONFIG_LOAD_FAIL "${LOG_FILE}" | wc -w)
    # Remove whitespaces
    ${_GET_RECONFIGS##*[! ]}
    ${_GET_RECONFIG_ERRORS##*[! ]}
}

# Remove the given directories or files if they exist.
remove_if_exists() {
    while test ${#} -gt 0; do
        if test -e "${1}"; then
            rm -rf "${1}"
        fi
        shift
    done
}

# Performs cleanup after test.
# It shuts down running Kea processes and removes temporary files.
# The location of the log file and the configuration files should be set
# in the ${LOG_FILE}, ${CFG_FILE} and ${KEACTRL_CFG_FILE} variables
# respectively, prior to calling this function.
cleanup() {
    # If there is no KEA_PROCS set, just return
    if [ -z "${KEA_PROCS}" ]; then
        return
    fi

    # KEA_PROCS holds the name of all Kea processes. Shut down each
    # of them if running.
    for proc_name in ${KEA_PROCS}
    do
        get_pid "${proc_name}"
        # Shut down running Kea process.
        if [ "${_GET_PIDS_NUM}" -ne 0 ]; then
            printf 'Shutting down Kea process having pid %d.\n' "${_GET_PID}"
            kill -9 "${_GET_PID}"
        fi
    done

    # Kill any running LFC processes. Even though 'kea-lfc' creates PID
    # file we rather want to use 'pgrep' to find the process PID, because
    # kea-lfc execution is not controlled from the test and thus there
    # is possibility that process is already/still running but the PID
    # file doesn't exist for it. As a result, the process will not
    # be killed. This is not a problem for other processes because
    # tests control launching them and monitor when they are shut down.
    kill_processes_by_name "kea-lfc"

    # Remove temporary files.
    remove_if_exists \
        "${CA_CFG_FILE-}" \
        "${CFG_FILE-}" \
        "${D2_CFG_FILE-}" \
        "${DHCP4_CFG_FILE-}" \
        "${DHCP6_CFG_FILE-}" \
        "${KEACTRL_CFG_FILE-}" \
        "${NC_CFG_FILE-}"

    # Use ${var+x} to test if ${var} is defined.
    if test -n "${LOG_FILE+x}" && test -n "${LOG_FILE}"; then
        rm -rf "${LOG_FILE}"
        rm -rf "${LOG_FILE}.lock"
    fi
    # Use asterisk to remove all files starting with the given name,
    # in case the LFC has been run. LFC creates files with postfixes
    # appended to the lease file name.
    if test -n "${LEASE_FILE+x}" && test -n "${LEASE_FILE}"; then
        rm -rf "${LEASE_FILE}"*
    fi
}

# Exists the test in the clean way.
# It performs the cleanup and prints whether the test has passed or failed.
# If a test fails, the Kea log is dumped.
clean_exit() {
    exit_code=${1}  # Exit code to be returned by the exit function.
    case ${exit_code} in
        ''|*[!0-9]*)
            test_lib_error "argument passed to clean_exit must be a number" ;;
    esac
    # Print test result and perform a cleanup
    test_finish "${exit_code}"
    exit "${exit_code}"
}

# Starts Kea process in background using a configuration file specified
# in the global variable ${CFG_FILE}.
start_kea() {
    local bin=${1}
    if [ -z "${bin}" ]; then
        test_lib_error "binary name must be specified for start_kea"
        clean_exit 1
    fi
    printf "Running command %s.\n" "\"${bin} -c ${CFG_FILE}\""
    "${bin}" -c "${CFG_FILE}" &
}

# Waits with timeout for Kea to start.
# This function repeatedly checks if the Kea log file has been created
# and is non-empty. If it is, the function assumes that Kea has started.
# It doesn't check the contents of the log file though.
# If the log file doesn't exist the function sleeps for a second and
# checks again. This is repeated until timeout is reached or non-empty
# log file is found. If timeout is reached, the function reports an
# error.
# Return value:
#    _WAIT_FOR_KEA: 0 if Kea hasn't started, 1 otherwise
wait_for_kea() {
    local timeout=${1} # Desired timeout in seconds.
    case ${timeout} in
        ''|*[!0-9]*)
            test_lib_error "argument passed to wait_for_kea must be a number"
            clean_exit 1 ;;
    esac
    local loops=0 # Loops counter
    _WAIT_FOR_KEA=0
    test_lib_info "wait_for_kea " "skip-new-line"
    while [ ! -s "${LOG_FILE}" ] && [ "${loops}" -le "${timeout}" ]; do
        printf "."
        sleep 1
        loops=$(( loops + 1 ))
    done
    printf '\n'
    if [ "${loops}" -le "${timeout}" ]; then
        _WAIT_FOR_KEA=1
    fi
}

# Waits for a specific message to occur in the Kea log file.
# This function is called when the test expects specific message
# to show up in the log file as a result of some action that has
# been taken. Typically, the test expects that the message
# is logged when the SIGHUP or SIGTERM signal has been sent to the
# Kea process.
# This function waits a specified number of seconds for the number
# of message occurrences to show up. If the expected number of
# message doesn't occur, the error status is returned.
# Return value:
#    _WAIT_FOR_MESSAGE: 0 if the message hasn't occurred, 1 otherwise.
wait_for_message() {
    local timeout=${1}     # Expected timeout value in seconds.
    local message="${2}"   # Expected message id.
    local occurrences=${3} # Number of expected occurrences.

    # Validate timeout
    case ${timeout} in
        ''|*[!0-9]*)
            test_lib_error "argument timeout passed to wait_for_message must \
be a number"
        clean_exit 1 ;;
    esac

    # Validate message
    if [ -z "${message}" ]; then
        test_lib_error "message id is a required argument for wait_for_message"
        clean_exit 1
    fi

    # Validate occurrences
    case ${occurrences} in
        ''|*[!0-9]*)
            test_lib_error "argument occurrences passed to wait_for_message \
must be a number"
        clean_exit 1 ;;
    esac

    local loops=0          # Number of loops performed so far.
    _WAIT_FOR_MESSAGE=0
    test_lib_info "wait_for_message ${message}: " "skip-new-line"
    # Check if log file exists and if we reached timeout.
    while [ "${loops}" -le "${timeout}" ]; do
        printf "."
        # Check if the message has been logged.
        get_log_messages "${message}"
        if [ "${_GET_LOG_MESSAGES}" -ge "${occurrences}" ]; then
            printf '\n'
            _WAIT_FOR_MESSAGE=1
            return
        fi
        # Message not recorded. Keep going.
        sleep 1
        loops=$(( loops + 1 ))
    done
    printf '\n'
    # Timeout.
}

# Waits for server to be down.
# Return value:
#    _WAIT_FOR_SERVER_DOWN: 1 if server is down, 0 if timeout occurred and the
#                             server is still running.
wait_for_server_down() {
    local timeout=${1}    # Timeout specified in seconds.
    local proc_name=${2}  # Server process name.

    case ${timeout} in
        ''|*[!0-9]*)
            test_lib_error "argument passed to wait_for_server_down must be a number"
            clean_exit 1 ;;
    esac
    local loops=0 # Loops counter
    _WAIT_FOR_SERVER_DOWN=0
    test_lib_info "wait_for_server_down ${proc_name}: " "skip-new-line"
    while [ "${loops}" -le "${timeout}" ]; do
        printf "."
        get_pid "${proc_name}"
        if [ "${_GET_PIDS_NUM}" -eq 0 ]; then
            printf '\n'
            _WAIT_FOR_SERVER_DOWN=1
            return
        fi
        sleep 1
        loops=$(( loops + 1 ))
    done
    printf '\n'
}

# Sends specified signal to the Kea process.
send_signal() {
    local sig=${1}       # Signal number.
    local proc_name=${2} # Process name

    # Validate signal
    case ${sig} in
        ''|*[!0-9]*)
            test_lib_error "signal number passed to send_signal \
must be a number"
        clean_exit 1 ;;
    esac
    # Validate process name
    if [ -z "${proc_name}" ]; then
        test_lib_error "send_signal requires process name be passed as argument"
        clean_exit 1
    fi
    # Get Kea pid.
    get_pid "${proc_name}"
    if [ "${_GET_PIDS_NUM}" -ne 1 ]; then
        printf "ERROR: expected one Kea process to be started.\
 Found %d processes started.\n" ${_GET_PIDS_NUM}
        clean_exit 1
    fi
    printf "Sending signal %s to Kea process (pid=%s).\n" "${sig}" "${_GET_PID}"
    # Actually send a signal.
    kill "-${sig}" "${_GET_PID}"
}

# Verifies that a server is up running by its PID file
# The PID file is constructed from the given config file name and
# binary name.  If it exists and the PID it contains refers to a
# live process it sets _SERVER_PID_FILE and _SERVER_PID to the
# corresponding values.  Otherwise, it emits an error and exits.
verify_server_pid() {
    local bin_name="${1}" # binary name of the server
    local cfg_file="${2}" # config file name

    # We will construct the PID file name based on the server config
    # and binary name
    if [ -z "${bin_name}" ]; then
        test_lib_error "verify_server_pid requires binary name"
        clean_exit 1
    fi

    if [ -z "${cfg_file}" ]; then
        test_lib_error "verify_server_pid requires config file name"
        clean_exit 1
    fi

    # Only the file name portion of the config file is used, try and
    # extract it. NOTE if this "algorithm" changes this code will need
    # to be updated.
    fname=$(basename "${cfg_file}")
    fname=$(echo "${fname}" | cut -f1 -d'.')

    if [ -z "${fname}" ]; then
        test_lib_error "verify_server_pid could not extract config name"
        clean_exit 1
    fi

    # Now we can build the name:
    pid_file="$KEA_PIDFILE_DIR/$fname.$bin_name.pid"

    if [ ! -e "${pid_file}" ]; then
        printf "ERROR: PID file:[%s] does not exist\n" "${pid_file}"
        clean_exit 1
    fi

    # File exists, does its PID point to a live process?
    pid=$(cat "${pid_file}")
    if ! kill -0 "${pid}"; then
        printf "ERROR: PID file:[%s] exists but PID:[%d] does not\n" \
               "${pid_file}" "${pid}"
        clean_exit 1
    fi

    # Make the values accessible to the caller
    _SERVER_PID="${pid}"
    _SERVER_PID_FILE="${pid_file}"
}

# This test verifies that the binary is reporting its version properly.
version_test() {
    test_name=${1}      # Test name
    long_version=${2-}  # Test long version?

    # Log the start of the test and print test name.
    test_start "${test_name}"

    # Remove dangling Kea instances and remove log files.
    cleanup

    # If set to anything other than empty string, reset it to the long version
    # parameter.
    if test -n "${long_version}"; then
        long_version='--version'
    fi

    # Keep ${long_version} unquoted so that it is not included as an empty
    # string if not given as argument.
    for v in -v ${long_version}; do
        run_command \
            "${bin_path}/${bin}" "${v}"

        if test "${OUTPUT}" != "${EXPECTED_VERSION}"; then
            printf 'ERROR: Expected version "%s", got "%s" when calling "%s"\n' \
                "${EXPECTED_VERSION}" "${OUTPUT}" "${bin} ${v}"
            test_finish 1
        fi
    done

    test_finish 0
}

# This test verifies that the server is using logger variable
# KEA_LOCKFILE_DIR properly (it should be used to point out to the directory,
# where lockfile should be created. Also, "none" value means to not create
# the lockfile at all).
logger_vars_test() {
    test_name=${1}  # Test name

    # Log the start of the test and print test name.
    test_start "${test_name}"
    # Remove dangling Kea instances and remove log files.
    cleanup

    # Create bogus configuration file. We don't really want the server to start,
    # just want it to log something and die. Empty config is an easy way to
    # enforce that behavior.
    create_config "{ }"
    printf "Please ignore any config error messages.\n"

    # Remember old KEA_LOCKFILE_DIR
    KEA_LOCKFILE_DIR_OLD=${KEA_LOCKFILE_DIR}

    # Set lockfile directory to current directory.
    KEA_LOCKFILE_DIR=.

    # Start Kea.
    start_kea "${bin_path}/${bin}"

    # Wait for Kea to process the invalid configuration and die.
    sleep 1

    # Check if it is still running. It should have terminated.
    get_pid "${bin}"
    if [ "${_GET_PIDS_NUM}" -ne 0 ]; then
        printf 'ERROR: expected Kea process to not start. '
        printf 'Found %d processes running.\n' "${_GET_PIDS_NUM}"

        # Revert to the old KEA_LOCKFILE_DIR value
        KEA_LOCKFILE_DIR=${KEA_LOCKFILE_DIR_OLD}
        clean_exit 1
    fi

    if [ ! -f "./logger_lockfile" ]; then
        printf 'ERROR: Expect %s to create logger_lockfile in the ' "${bin}"
        printf 'current directory, but no such file exists.\n'

        # Revert to the old KEA_LOCKFILE_DIR value
        KEA_LOCKFILE_DIR=${KEA_LOCKFILE_DIR__OLD}
        clean_exit 1
    fi

    # Remove the lock file
    rm -f ./logger_lockfile

    # Tell Kea to NOT create logfiles at all
    KEA_LOCKFILE_DIR="none"

    # Start Kea.
    start_kea "${bin_path}/${bin}"

    # Wait for Kea to process the invalid configuration and die.
    sleep 1

    # Check if it is still running. It should have terminated.
    get_pid "${bin}"
    if [ "${_GET_PIDS_NUM}" -ne 0 ]; then
        printf 'ERROR: expected Kea process to not start. '
        printf 'Found %d processes running.\n' "${_GET_PIDS_NUM}"

        # Revert to the old KEA_LOCKFILE_DIR value
        KEA_LOCKFILE_DIR=${KEA_LOCKFILE_DIR_OLD}

        clean_exit 1
    fi

    if [ -f "./logger_lockfile" ]; then
        printf 'ERROR: Expect %s to NOT create logger_lockfile in the ' "${bin}"
        printf 'current directory, but the file exists.\n'

        # Revert to the old KEA_LOCKFILE_DIR value
        KEA_LOCKFILE_DIR=${KEA_LOCKFILE_DIR_OLD}

        clean_exit 1
    fi

    # Revert to the old KEA_LOCKFILE_DIR value
    printf 'Reverting KEA_LOCKFILE_DIR to %s\n' "${KEA_LOCKFILE_DIR_OLD}"
    KEA_LOCKFILE_DIR=${KEA_LOCKFILE_DIR_OLD}

    test_finish 0
}

# This test verifies server PID file management
# 1. It verifies that upon startup, the server creates a PID file
# 2. It verifies the an attempt to start a second instance fails
# due to pre-existing PID File/PID detection
server_pid_file_test() {
    local server_cfg="${1}"
    local log_id="${2}"

    # Log the start of the test and print test name.
    test_start "${bin}.server_pid_file_test"
    # Remove dangling DHCP4 instances and remove log files.
    cleanup
    # Create new configuration file.
    create_config "${CONFIG}"
    # Instruct server to log to the specific file.
    set_logger
    # Start server
    start_kea "${bin_path}/${bin}"
    # Wait up to 20s for server to start.
    wait_for_kea 20
    if [ "${_WAIT_FOR_KEA}" -eq 0 ]; then
        printf 'ERROR: timeout waiting for %s to start.\n' "${bin}"
        clean_exit 1
    fi

    # Verify server is still running
    verify_server_pid "${bin}" "${CFG_FILE}"

    printf 'PID file is [%s],  PID is [%d]' "${_SERVER_PID_FILE}" "${_SERVER_PID}"

    # Now try to start a second one
    start_kea "${bin_path}/${bin}"

    wait_for_message 10 "${log_id}" 1
    if [ "${_WAIT_FOR_MESSAGE}" -eq 0 ]; then
        printf 'ERROR: Second %s instance started? ' "${bin}"
        printf 'PID conflict not reported.\n'
        clean_exit 1
    fi

    # Verify server is still running
    verify_server_pid "${bin}" "${CFG_FILE}"

    # All ok. Shut down the server and exit.
    test_finish 0
}