diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index e283bd2e56..39e7cd7396 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -43,6 +43,7 @@ shellcheck: - SCRIPTS+="src/lib/log/tests/logger_lock_test.sh.in " - SCRIPTS+="src/lib/log/tests/severity_test.sh.in " - SCRIPTS+="src/lib/testutils/dhcp_test_lib.sh.in " + - SCRIPTS+="src/lib/testutils/xml_reporting_test_lib.sh.in " - SCRIPTS+="src/lib/util/tests/process_spawn_app.sh.in " - SCRIPTS+="src/share/database/scripts/cql/upgrade_1.0_to_2.0.sh.in " - SCRIPTS+="src/share/database/scripts/cql/upgrade_2.0_to_3.0.sh.in " diff --git a/configure.ac b/configure.ac index 50cec079d6..a5883b0a03 100755 --- a/configure.ac +++ b/configure.ac @@ -1722,6 +1722,7 @@ AC_CONFIG_FILES([Makefile src/lib/stats/tests/Makefile src/lib/testutils/Makefile src/lib/testutils/dhcp_test_lib.sh + src/lib/testutils/xml_reporting_test_lib.sh src/lib/util/Makefile src/lib/util/io/Makefile src/lib/util/python/Makefile diff --git a/src/bin/admin/tests/mysql_tests.sh.in b/src/bin/admin/tests/mysql_tests.sh.in index 8aaadb0d2c..600c36e0cc 100644 --- a/src/bin/admin/tests/mysql_tests.sh.in +++ b/src/bin/admin/tests/mysql_tests.sh.in @@ -1064,7 +1064,7 @@ mysql_lease6_stat_test() { # lease<4/6>_stat tables will be populated based on existing # leases and that the stat triggers work properly. mysql_lease_stat_upgrade_test() { - test_start "my_sql_lease_stat_upgrade_test" + test_start "mysql.lease_stat_upgrade_test" # Let's wipe the whole database mysql_wipe @@ -1208,7 +1208,7 @@ mysql_lease_stat_upgrade_test() { } mysql_lease_stat_recount_test() { - test_start "my_sql_lease_stat_recount_test" + test_start "mysql.lease_stat_recount_test" # Let's wipe the whole database mysql_wipe diff --git a/src/lib/testutils/.gitignore b/src/lib/testutils/.gitignore index 17933cf0b6..077de8d922 100644 --- a/src/lib/testutils/.gitignore +++ b/src/lib/testutils/.gitignore @@ -1 +1,2 @@ /dhcp_test_lib.sh +/xml_reporting_test_lib.sh diff --git a/src/lib/testutils/Makefile.am b/src/lib/testutils/Makefile.am index 756b9cfc57..30332da548 100644 --- a/src/lib/testutils/Makefile.am +++ b/src/lib/testutils/Makefile.am @@ -2,7 +2,7 @@ AM_CPPFLAGS = -I$(top_srcdir)/src/lib -I$(top_builddir)/src/lib AM_CPPFLAGS += $(BOOST_INCLUDES) AM_CXXFLAGS=$(KEA_CXXFLAGS) -noinst_SCRIPTS = dhcp_test_lib.sh +noinst_SCRIPTS = dhcp_test_lib.sh xml_reporting_test_lib.sh if HAVE_GTEST noinst_LTLIBRARIES = libkea-testutils.la @@ -22,8 +22,8 @@ libkea_testutils_la_LIBADD += $(top_builddir)/src/lib/dns/libkea-dns++.la endif # Include common libraries being used by shell-based tests. -SHLIBS = dhcp_test_lib.sh.in +SHLIBS = dhcp_test_lib.sh.in xml_reporting_test_lib.sh.in EXTRA_DIST = $(SHLIBS) -CLEANFILES = dhcp_test_lib.sh +CLEANFILES = dhcp_test_lib.sh xml_reporting_test_lib.sh diff --git a/src/lib/testutils/dhcp_test_lib.sh.in b/src/lib/testutils/dhcp_test_lib.sh.in index 44fbe503be..7183372b20 100644 --- a/src/lib/testutils/dhcp_test_lib.sh.in +++ b/src/lib/testutils/dhcp_test_lib.sh.in @@ -6,6 +6,9 @@ # License, v. 2.0. If a copy of the MPL was not distributed with this # file, You can obtain one at http://mozilla.org/MPL/2.0/. +# shellcheck disable=SC1091 +# SC1091: Not following: ... was not specified as input (see shellcheck -x). + # shellcheck disable=SC2034 # SC2034: ... appears unused. Verify use (or export if used externally). @@ -22,6 +25,9 @@ # used. set -eu +# Include XML reporting library. +. "@abs_top_builddir@/src/lib/testutils/xml_reporting_test_lib.sh" + prefix="@prefix@" # Expected version @@ -178,6 +184,23 @@ traps_off() { done } +# Print UNIX time with millisecond resolution. +get_current_time() { + local time + time=$(date +%s%3N) + + # In some systems, particularly BSD-based, `+%3N` millisecond resolution is + # not supported. It instead prints the literal '3N', but we check for any + # alphabetical character. If we do find one, revert to second resolution and + # convert to milliseconds. + if printf '%s' "${time}" | grep -E '[A-Za-z]' > /dev/null 2>&1; then + time=$(date +%s) + time=$((1000 * time)) + fi + + printf '%s' "${time}" +} + # Begins a test by printing its name. test_start() { TEST_NAME=${1} @@ -191,11 +214,24 @@ test_start() { # Announce test start. printf "${green-}[ RUN ]${reset-} %s\n" "${TEST_NAME}" + + # Start timer in milliseconds. + START_TIME=$(get_current_time) } # Prints test result an cleans up after the test. test_finish() { - local exit_code=${1} # Exit code to be returned by the exit function. + # Exit code to be returned by the exit function + local exit_code=${1} + + # Stop timer and set duration. + FINISH_TIME=$(get_current_time) + local duration + duration=$((FINISH_TIME - START_TIME)) + + # Add the test result to the XML. + report_test_result_in_xml "${TEST_NAME}" "${exit_code}" "${duration}" + if [ "${exit_code}" -eq 0 ]; then cleanup printf "${green-}[ OK ]${reset-} %s\n" "${TEST_NAME}" diff --git a/src/lib/testutils/xml_reporting_test_lib.sh.in b/src/lib/testutils/xml_reporting_test_lib.sh.in new file mode 100644 index 0000000000..1605c21b7d --- /dev/null +++ b/src/lib/testutils/xml_reporting_test_lib.sh.in @@ -0,0 +1,327 @@ +#!/bin/sh + +# Copyright (C) 2020 Internet Systems Consortium, Inc. ("ISC") +# +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +# shellcheck disable=SC2039 +# SC2039: In POSIX sh, 'local' is undefined. + +# Exit with error if commands exit with non-zero and if undefined variables are +# used. +set -eu + +############################### Public functions ############################### + +# Add an entry to the XML test report. +report_test_result_in_xml() { + # If GTEST_OUTPUT is not defined... + if ! test -n "${GTEST_OUTPUT+x}"; then + # There is nowhere to report. + return + fi + + # Declarations + local test_name=${1}; shift + local exit_code=${1}; shift + local duration=${1}; shift # milliseconds + local now + local test_case + local test_suite + local xml + now=$(date '+%FT%H:%M:%S') + test_suite=$(printf '%s' "${test_name}" | cut -d '.' -f 1) + test_case=$(printf '%s' "${test_name}" | cut -d '.' -f 2-) + + # Strip the 'xml:' at the start of GTEST_OUTPUT if it is there. + xml="${GTEST_OUTPUT}" + if test "$(printf '%s' "${xml}" | cut -c 1-4)" = 'xml:'; then + xml=$(printf '%s' "${xml}" | cut -c 5-) + fi + xml="${xml}/${test_suite}.sh.xml" + + # Convert to seconds, but keep the millisecond precision. + duration=$(_calculate "${duration} / 1000.0") + + # For test suites that have a single test case and no name for the test + # case, name the test case after the test suite. + if test -z "${test_case}"; then + test_case="${test_suite}" + fi + + # Determine result based on exit code. Googletest seems to omit the failed + # tests, instead we are explicitly adding them with a 'failed' result. + local result + if test "${exit_code}" -eq 0; then + result='completed' + else + result='failed' + fi + + _create_xml "${xml}" "${now}" + + _add_test_suite "${test_suite}" "${xml}" "${now}" + + _add_test_case "${test_suite}" "${test_case}" "${duration}" "${result}" \ + "${xml}" "${now}" +} + +############################## Private functions ############################### + +# Add ${string} after ${reference} in ${file}. +_add_after() { + local string=${1}; shift + local reference=${1}; shift + local file=${1}; shift + + # Escape all slashes. + string=$(printf '%s' "${string}" | sed 's#\/#\\\/#g') + reference=$(printf '%s' "${reference}" | sed 's#\/#\\\/#g') + + # Escape all spaces. Only trailing spaces need escaped, but that's harder + # and this still empirically works. + string=$(printf '%s' "${string}" | sed 's#\ #\\\ #g') + reference=$(printf '%s' "${reference}" | sed 's#\ #\\\ #g') + + # Add ${string} after ${reference} in ${file}. + # The "\\" followed by newline is for BSD support. + sed "/${reference}/a\\ +${string} +" "${file}" > "${file}.tmp" + mv "${file}.tmp" "${file}" +} + +# Add ${string} before ${reference} in ${file}. +_add_before() { + local string=${1}; shift + local reference=${1}; shift + local file=${1}; shift + + # Get the line number of the reference line. + local line_number + line_number=$(grep -Fn "${reference}" "${file}" | cut -d ':' -f 1) + + # Escape all slashes. + string=$(printf '%s' "${string}" | sed 's#\/#\\\/#g') + reference=$(printf '%s' "${reference}" | sed 's#\/#\\\/#g') + + # Escape all spaces. Only trailing spaces need escaped, but that's harder + # and this still empirically works. + string=$(printf '%s' "${string}" | sed 's#\ #\\\ #g') + reference=$(printf '%s' "${reference}" | sed 's#\ #\\\ #g') + + # Add ${string} before ${reference} in ${file}. + # The "\\" followed by newline is for BSD support. + sed "${line_number}i\\ +${string} +" "${file}" > "${file}.tmp" + mv "${file}.tmp" "${file}" +} + +# Add test result if not in file. +_add_test_case() { + local test_suite=${1}; shift + local test_case=${1}; shift + local result=${1}; shift + local duration=${1}; shift + local xml=${1}; shift + local now=${1}; shift + + # Create the test XML tag. + local test_case_line + test_case_line=$(printf ' ' \ + "${test_case}" "${duration}" "${result}" "${now}" "${test_suite}") + + # Add this test case to all the other test cases and sort them. + local all_test_cases + all_test_cases=$(_print_lines_between_matching_patterns \ + " ' "${xml}") + all_test_cases=$(printf '%s\n%s' "${all_test_cases}" "${test_case_line}") + + _update_test_suite_metrics "${test_suite}" "${all_test_cases}" "${xml}" "${now}" + + # Find the test following this one. + local following_line + following_line=$(printf '%s' "${all_test_cases}" | \ + grep -A1 -F "${test_case_line}" | \ + grep -Fv "${test_case_line}" || true) + if test -n "${following_line}"; then + # If found, add it before. + _add_before "${test_case_line}" "${following_line}" "${xml}" + return + fi + + # Find the test before this one. + local previous_line + previous_line=$(printf '%s' "${all_test_cases}" | \ + grep -B1 -F "${test_case_line}" | \ + grep -Fv "${test_case_line}" || true) + if test -n "${previous_line}"; then + # If found, add it after. + _add_after "${test_case_line}" "${previous_line}" "${xml}" + return + fi + + # If neither were found, add it as the first test case following the test + # suite line. + _add_after "${test_case_line}" " /dev/null 2>&1; then + return + fi + + # Create the test suite XML tag. + local test_suite_line + test_suite_line=$(printf ' ' \ + "${test_suite}" "${now}") + + # Add this test suite to all the other test suites and sort them. + local all_test_suites + all_test_suites=$(printf '%s\n%s' " ${test_suite_line}" \ + "$(grep -E ' ' "${xml}")") + + # Find the test suite following this one. + local following_line + following_line=$(printf '%s' "${all_test_suites}" | \ + grep -A1 -F "${test_suite_line}" | \ + grep -Fv "${test_suite_line}" || true) + + # Add the test suite tag to the XML. + _add_before "${test_suite_line}" "${following_line}" "${xml}" + _add_after ' ' "${test_suite_line}" "${xml}" +} + +# Calculate the given mathematical expression and print it in a format that +# matches googletest's time in the XML attribute time="..." which is seconds +# rounded to 3 decimals. +_calculate() { + local expression=${1} + local result + result=$(echo " + + # 3 decimals maximum + scale = 3 + + # Store result. + result = ${expression} + + # If subunit, print the leading zero. + if (0 < result && result < 1) print 0 + + # Print the result. + print result + + " | bc) + + # Delete trailing zeros and trailing dot. + if printf '%s' "${result}" | grep -F '.' > /dev/null 2>&1; then + result=$(printf '%s' "${result}" | sed 's/0*$//g' | sed 's/\.$//g') + fi + + printf '%s' "${result}" +} + +# Create XML with header and top-level tags if the file doesn't exist. +_create_xml() { + # If file exists and we have set GTEST_OUTPUT_CREATED previously, then there + # is nothing to do. + if test -f "${xml}" && test -n "${GTEST_OUTPUT_CREATED+x}"; then + return; + fi + + local xml=${1}; shift + local now=${1}; shift + + mkdir -p "$(dirname "${xml}")" + printf \ +' + + +' "${now}" > "${xml}" + + # GTEST_OUTPUT_CREATED is not a googletest variable, but our way of allowing + # to overwrite XMLs created in a previous test run. The lifetime of + # GTEST_OUTPUT_CREATED is extended to the oldest ancestor file who has + # sourced this script i.e. the *_test.sh file. So it gets lost from one + # *_test.sh to another. The consensus that need to be kept so that this + # works correctly are: + # * Needless to say, don't set this variable on your own. + # * Always call these scripts directly or through `make check`. + # Never source test files e.g. `source memfile_tests.sh` or + # `. memfile_tests.sh`. + # * The ${xml} passed here must be deterministically and uniquely + # attributed to the *_test.sh. At the time of this writing, ${xml} is the + # part of the name before the dot. So for example, for memfile, all tests + # should start with the same thing e.g. `memfile.*`. + export GTEST_OUTPUT_CREATED=true +} + +# Print the lines between two matching regex patterns from a file. Excludes the +# lines that contain the patterns themselves. Matches only the first occurence. +_print_lines_between_matching_patterns() { + local start_pattern=${1}; shift + local end_pattern=${1}; shift + local file=${1}; shift + + # Escape all slashes. + start_pattern=$(printf '%s' "${start_pattern}" | sed 's#\/#\\\/#g') + end_pattern=$(printf '%s' "${end_pattern}" | sed 's#\/#\\\/#g') + + # Print with sed. + sed -n "/${start_pattern}/,/${end_pattern}/p;/${end_pattern}/q" "${file}" \ + | sed '$d' | tail -n +2 +} + +# Update the test suite XML attributes with metrics collected from the child +# test cases. +_update_test_suite_metrics() { + local test_suite=${1}; shift + local all_test_cases=${1}; shift + local xml=${1}; shift + local now=${1}; shift + + # Get the metrics on the parent test suite. + local duration + local durations_summed + local failures + local tests + tests=$(echo "${all_test_cases}" | wc -l) + failures=$(printf '%s' "${all_test_cases}" \ + | grep -Fc 'result="failed"' || true) + durations_summed=$(printf '%s' "${all_test_cases}" \ + | grep -Eo 'time="[0-9.]+"' | cut -d '"' -f 2 | xargs | sed 's/ / + /g') + duration=$(_calculate "${durations_summed}") + + # Create the test suite XML tag. + local test_suite_line + test_suite_line=$(printf ' ' \ + "${test_suite}" "${tests}" "${failures}" "${duration}" "${now}") + + # Update the test suite with the collected metrics. + sed "s# #${test_suite_line}#g" \ + "${xml}" > "${xml}.tmp" + mv "${xml}.tmp" "${xml}" + + # Create the test suites XML tag. + local test_suites_line + test_suites_line=$(printf '' \ + "${tests}" "${failures}" "${duration}" "${now}") + + # Update the test suites with the collected metrics. + sed "s##${test_suites_line}#g" \ + "${xml}" > "${xml}.tmp" + mv "${xml}.tmp" "${xml}" +}