From 63e3f72bd5321e62b8a49d55e632cc9dd85605e5 Mon Sep 17 00:00:00 2001
From: Jan Kiene <jan.kiene@iis.fraunhofer.de>
Date: Mon, 4 May 2026 15:29:41 +0200
Subject: [PATCH 1/2] raise exception instead of pytest.fail for non-zero ret
 codes

this should fix the error reporting in html report
---
 tests/conftest.py | 33 +++++++++------------------------
 1 file changed, 9 insertions(+), 24 deletions(-)

diff --git a/tests/conftest.py b/tests/conftest.py
index 258fd8049..05bbdd85c 100644
--- a/tests/conftest.py
+++ b/tests/conftest.py
@@ -680,19 +680,14 @@ class EncoderFrontend:
             stderr_str = textwrap.indent(self.stderr, prefix="\t")
             log_dbg_msg(f"{self._type} encoder stderr:\n{stderr_str}")
         if self.returncode:
-            pytest.fail(
+            raise RuntimeError(
                 f"{self._type} encoder terminated with a non-0 return code: {self.returncode}"
             )
         if self.stderr and "UndefinedBehaviorSanitizer" in self.stderr:
-            pytest.fail("Undefined Behaviour runtime error encountered")
+            raise RuntimeError("Undefined Behaviour runtime error encountered")
 
     def _check_run(self):
-        if self.returncode is not None:
-            if self.returncode:
-                pytest.fail(
-                    f"{self._type} encoder terminated with a non-0 return code: {self.returncode}"
-                )
-        else:
+        if self.returncode is None:
             logger.warning("%s encoder was set-up, but not run", self._type)
         # next assert is not OK since stderr contains messages even when encoding was successful
         # assert not self.stderr, self._type + " encoder stderr is not empty"
@@ -964,19 +959,14 @@ class DecoderFrontend:
             stderr_str = textwrap.indent(self.stderr, prefix="\t")
             log_dbg_msg(f"{self._type} decoder stderr:\n{stderr_str}")
         if self.returncode:
-            pytest.fail(
+            raise RuntimeError(
                 f"{self._type} decoder terminated with a non-0 return code: {self.returncode}"
             )
         if self.stderr and "UndefinedBehaviorSanitizer" in self.stderr:
-            pytest.fail("Undefined Behaviour runtime error encountered")
+            raise RuntimeError("Undefined Behaviour runtime error encountered")
 
     def _check_run(self):
-        if self.returncode is not None:
-            if self.returncode:
-                pytest.fail(
-                    f"{self._type} decoder terminated with a non-0 return code: {self.returncode}"
-                )
-        else:
+        if self.returncode is None:
             logger.warning("%s decoder was set-up, but not run", self._type)
         # next assert is not OK since stderr contains messages even when decoding was successful
         # assert not self.stderr, self._type + " decoder stderr is not empty"
@@ -1466,19 +1456,14 @@ class PostRendFrontend:
             stderr_str = textwrap.indent(self.stderr, prefix="\t")
             log_dbg_msg(f"{self._type} post-rend stderr:\n{stderr_str}")
         if self.returncode:
-            pytest.fail(
+            raise RuntimeError(
                 f"{self._type} post-rend terminated with a non-0 return code: {self.returncode}"
             )
         if self.stderr and "UndefinedBehaviorSanitizer" in self.stderr:
-            pytest.fail("Undefined Behaviour runtime error encountered")
+            raise RuntimeError("Undefined Behaviour runtime error encountered")
 
     def _check_run(self):
-        if self.returncode is not None:
-            if self.returncode:
-                pytest.fail(
-                    f"{self._type} post-rend terminated with a non-0 return code: {self.returncode}"
-                )
-        else:
+        if self.returncode is None:
             logger.warning("%s post-rend was set-up, but not run", self._type)
         # next assert is not OK since stderr contains messages even when decoding was successful
         # assert not self.stderr, self._type + " decoder stderr is not empty"
-- 
GitLab


From 892a90f67958f701992e1989668fd1460ad29ffb Mon Sep 17 00:00:00 2001
From: Jan Kiene <jan.kiene@iis.fraunhofer.de>
Date: Mon, 4 May 2026 15:56:20 +0200
Subject: [PATCH 2/2] change testinfo.when to "setup" to cause "ERROR" on non-0
 ret codes

---
 tests/conftest.py | 18 +++++++-----------
 1 file changed, 7 insertions(+), 11 deletions(-)

diff --git a/tests/conftest.py b/tests/conftest.py
index 05bbdd85c..edfb77e7d 100644
--- a/tests/conftest.py
+++ b/tests/conftest.py
@@ -453,25 +453,21 @@ def dut_encoder_path(request) -> str:
 def test_info(request):
     yield request
 
-    # Check for errors during teardown to report error instead of failure
-    if hasattr(request, "error"):
-        pytest.fail(request.error)
-
 
 @pytest.hookimpl(hookwrapper=True)
 def pytest_runtest_makereport(item, call):
-    # Use hook to catch exceptions
     outcome = yield
     report = outcome.get_result()
-    test_info = item.funcargs.get("test_info", None)
 
     if call.excinfo is not None and report.when == "call":
-        # Make sure exception is not due to a skipped or failed test (allowed exceptions)
-        type = call.excinfo.type
+        exc_type = call.excinfo.type
         xfail = hasattr(report, "wasxfail")
-        if type not in [Skipped, Failed] and not xfail:
-            # Capture exception in test_info
-            test_info.error = str(call.excinfo.value)
+        if exc_type not in [Skipped, Failed] and not xfail:
+            # pytest-html v4 (_is_error in basereport.py) only shows "Error" when
+            # report.when is "setup", "teardown", or "collect" — never for "call".
+            # Reclassify this call-phase crash as a setup-phase error so the HTML
+            # report shows ERROR instead of FAILURE.
+            report.when = "setup"
 
 
 @pytest.fixture(scope="session")
-- 
GitLab