From b66fb31dcbaeec4e0d4486a468666b40a25cf5e7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nicki=20K=C5=99=C3=AD=C5=BEek?= Date: Tue, 12 Nov 2024 10:07:02 +0100 Subject: [PATCH] Ensure pytest runner get proper outcome from flaky reruns When a test is re-run by the flaky plugin, the TestReport outcomes collected in the pytest_runtest_makereport() hook should be overriden. Each of the setup/call/teardown phases is reported again and since we care about the overall outcome, their respective results should be overriden so that only the outcome from the final test (re)run gets reported. Prior to this change, it lead to a situation where an extra_artifact generated during the test might be ignored. This was caused because the check was skipped, since the test was incorrectly considered as "failed" in the case where the test would fail on the first run, but pass on a subsequent flaky rerun. --- bin/tests/system/conftest.py | 23 ++++++++++++++++------- 1 file changed, 16 insertions(+), 7 deletions(-) diff --git a/bin/tests/system/conftest.py b/bin/tests/system/conftest.py index e5cbd3f326..919a188a96 100644 --- a/bin/tests/system/conftest.py +++ b/bin/tests/system/conftest.py @@ -134,16 +134,25 @@ def pytest_collection_modifyitems(items): class NodeResult: def __init__(self, report=None): - self.outcome = None - self.messages = [] + self._outcomes = {} + self.messages = {} if report is not None: self.update(report) def update(self, report): - if self.outcome is None or report.outcome != "passed": - self.outcome = report.outcome - if report.longreprtext: - self.messages.append(report.longreprtext) + # Allow the same nodeid/when to be overriden. This only happens when + # the test is re-run with flaky plugin. In that case, we want the + # latest result to override any previous results. + key = (report.nodeid, report.when) + self._outcomes[key] = report.outcome + self.messages[key] = report.longreprtext + + @property + def outcome(self): + for outcome in self._outcomes.values(): + if outcome != "passed": + return outcome + return "passed" @pytest.hookimpl(tryfirst=True, hookwrapper=True) @@ -347,7 +356,7 @@ def system_test_dir(request, system_test_name, expected_artifacts): messages = [] for node, result in test_results.items(): isctest.log.debug("%s %s", result.outcome.upper(), node) - messages.extend(result.messages) + messages.extend(result.messages.values()) for message in messages: isctest.log.debug("\n" + message) failed = any(res.outcome == "failed" for res in test_results.values())