From eaeb63310e9d8e1d3d7acc26b686cb87033ae91c Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Mon, 17 Jul 2023 17:31:23 +0200 Subject: [PATCH 1/5] add --selection_be_md5_file parameter --- tests/codec_be_on_mr_selection/__init__.py | 28 +++++++++++++++------- tests/conftest.py | 11 +++++++++ tests/testconfig.py | 2 ++ 3 files changed, 32 insertions(+), 9 deletions(-) diff --git a/tests/codec_be_on_mr_selection/__init__.py b/tests/codec_be_on_mr_selection/__init__.py index 34a307a739..9adfe47b44 100644 --- a/tests/codec_be_on_mr_selection/__init__.py +++ b/tests/codec_be_on_mr_selection/__init__.py @@ -35,6 +35,7 @@ import filecmp from pathlib import Path import subprocess from .constants import OUTPUT_MODES_AND_OPTIONS_FOR_EXPERIMENT, DTX_ON, FER_5PERC +from ..testconfig import MD5_REF_DICT HERE = Path(__file__).parent # set environment variables in CI job @@ -98,8 +99,20 @@ def apply_error_pattern_on_bitstream( subprocess.run(cmd) -def files_equal(dut_file, ref_file): - return filecmp.cmp(dut_file, ref_file) +def is_be_to_ref(dut_file): + """ + Check bitexactness either by comparing files directly or by comparing MD5 sums + """ + if MD5_REF_DICT == dict(): + ref_file = REF_PATH.joinpath(dut_file.name) + is_be = filecmp.cmp(dut_file, ref_file) + else: + md5_ref = MD5_REF_DICT[dut_file.name] + cmd = f"powershell.exe (Get-FileHash {str(dut_file)} -Algorithm MD5).Hash" + md5_dut = subprocess.check_output(cmd, shell=True).decode().splitlines()[-1] + is_be = md5_ref == md5_dut + + return is_be def run_check( @@ -147,9 +160,8 @@ def run_check( add_option_list=options + [str(f) for f in metadata], ) - ref_bitstream = REF_PATH.joinpath(dut_bitstream.name) - if not is_ref_creation and not files_equal(dut_bitstream, ref_bitstream): - pytest.fail("Bitstream file differs from reference") + if not is_be_to_ref(dut_bitstream): + pytest.fail(f"Bitstream file differs from reference") dut_bitstream_to_decoder = dut_bitstream if error_pattern is not None: @@ -168,12 +180,10 @@ def run_check( # this should not be a problem as both the reference and the tdut output was generated by the codec, so # diverging headers should also indicate a problem - still, keep in mind if something bogus happens if not is_ref_creation: - ref_output = REF_PATH.joinpath(dut_output.name) - if not files_equal(dut_output, ref_output): + if not is_be_to_ref(dut_output): pytest.fail("Decoder output differs from reference") for md in metadata: md_suffix = "".join(md.suffixes) dut_md = DUT_PATH.joinpath(dut_output.with_suffix(md_suffix).name) - ref_md = REF_PATH.joinpath(dut_output.with_suffix(md_suffix).name) - if not files_equal(dut_md, ref_md): + if not is_be_to_ref(dut_md): pytest.fail("Metadata file {md.name} differs from reference") diff --git a/tests/conftest.py b/tests/conftest.py index 12b62caccd..365245e6f3 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -133,6 +133,13 @@ def pytest_addoption(parser): " Use --keep_files to prevent these deletions.", ) + parser.addoption( + "--selection_be_md5_file", + type=Path, + default=Path(), + help="Path to file with md5 sums for the reference signals of the selection-BE test" + ) + @pytest.fixture(scope="session", autouse=True) def update_ref(request): @@ -513,3 +520,7 @@ def pytest_configure(config): ) if config.option.param_file: testconfig.PARAM_FILE = config.option.param_file + if config.option.selection_be_md5_file: + md5_file_path = config.option.selection_be_md5_file + with open(md5_file_path) as f: + testconfig.MD5_REF_DICT = {line.split()[0]: line.split()[1] for line in f.readlines()} \ No newline at end of file diff --git a/tests/testconfig.py b/tests/testconfig.py index 12d170ba8d..1dbfbb8403 100644 --- a/tests/testconfig.py +++ b/tests/testconfig.py @@ -35,3 +35,5 @@ To configure test modules. """ PARAM_FILE = "scripts/config/self_test.prm" + +MD5_REF_DICT = dict() \ No newline at end of file -- GitLab From 768916189f3cbab6da53c2f86aafc60565794ace Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Mon, 17 Jul 2023 17:51:14 +0200 Subject: [PATCH 2/5] re-use keep_files flag in selection-BE test --- tests/codec_be_on_mr_selection/__init__.py | 6 ++++++ tests/codec_be_on_mr_selection/test_experiments.py | 6 ++++++ 2 files changed, 12 insertions(+) diff --git a/tests/codec_be_on_mr_selection/__init__.py b/tests/codec_be_on_mr_selection/__init__.py index 9adfe47b44..b7f4cd8efe 100644 --- a/tests/codec_be_on_mr_selection/__init__.py +++ b/tests/codec_be_on_mr_selection/__init__.py @@ -126,6 +126,7 @@ def run_check( decoder_frontend, is_ref_creation, input_file_num=None, + keep_files=True, ): sampling_rate = 48 output_mode, options = OUTPUT_MODES_AND_OPTIONS_FOR_EXPERIMENT[experiment] @@ -182,8 +183,13 @@ def run_check( if not is_ref_creation: if not is_be_to_ref(dut_output): pytest.fail("Decoder output differs from reference") + elif not keep_files: + os.remove(dut_output) + os.remove(dut_bitstream) for md in metadata: md_suffix = "".join(md.suffixes) dut_md = DUT_PATH.joinpath(dut_output.with_suffix(md_suffix).name) if not is_be_to_ref(dut_md): pytest.fail("Metadata file {md.name} differs from reference") + elif not keep_files: + os.remove(dut_md) diff --git a/tests/codec_be_on_mr_selection/test_experiments.py b/tests/codec_be_on_mr_selection/test_experiments.py index a0a4aa780a..5b2f516893 100644 --- a/tests/codec_be_on_mr_selection/test_experiments.py +++ b/tests/codec_be_on_mr_selection/test_experiments.py @@ -52,6 +52,7 @@ def test_p800( dut_encoder_frontend, dut_decoder_frontend, update_ref, + keep_files, ): run_check( experiment, @@ -63,6 +64,7 @@ def test_p800( dut_encoder_frontend, dut_decoder_frontend, update_ref == 1, + keep_files=keep_files, ) @@ -79,6 +81,7 @@ def test_bs1534_no_masa( dut_encoder_frontend, dut_decoder_frontend, update_ref, + keep_files, ): category = "" run_check( @@ -92,6 +95,7 @@ def test_bs1534_no_masa( dut_decoder_frontend, update_ref == 1, input_file_num=input_file_num, + keep_files=keep_files, ) @@ -111,6 +115,7 @@ def test_bs1534_masa( dut_encoder_frontend, dut_decoder_frontend, update_ref, + keep_files, ): run_check( experiment, @@ -123,4 +128,5 @@ def test_bs1534_masa( dut_decoder_frontend, update_ref == 1, input_file_num=input_file_num, + keep_files=keep_files, ) -- GitLab From 1c2839543a29af0b1330d3600cb09d77429daa3c Mon Sep 17 00:00:00 2001 From: knj Date: Tue, 18 Jul 2023 11:43:40 +0200 Subject: [PATCH 3/5] remove default for MD5 file to fix other tests --- tests/conftest.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/tests/conftest.py b/tests/conftest.py index 365245e6f3..2540d7f065 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -136,7 +136,6 @@ def pytest_addoption(parser): parser.addoption( "--selection_be_md5_file", type=Path, - default=Path(), help="Path to file with md5 sums for the reference signals of the selection-BE test" ) @@ -523,4 +522,4 @@ def pytest_configure(config): if config.option.selection_be_md5_file: md5_file_path = config.option.selection_be_md5_file with open(md5_file_path) as f: - testconfig.MD5_REF_DICT = {line.split()[0]: line.split()[1] for line in f.readlines()} \ No newline at end of file + testconfig.MD5_REF_DICT = {line.split()[0]: line.split()[1] for line in f.readlines()} -- GitLab From fb1bd5b05d82b55e866c30124ad7ffd7e8a37e53 Mon Sep 17 00:00:00 2001 From: knj Date: Tue, 18 Jul 2023 14:49:02 +0200 Subject: [PATCH 4/5] add check for windows when using MD5 comparison --- tests/conftest.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/conftest.py b/tests/conftest.py index 2540d7f065..ecca54374a 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -521,5 +521,7 @@ def pytest_configure(config): testconfig.PARAM_FILE = config.option.param_file if config.option.selection_be_md5_file: md5_file_path = config.option.selection_be_md5_file + if not platform.system == "Windows": + raise NotImplementedError("MD5 comparison is currently hardcoded for windows") with open(md5_file_path) as f: testconfig.MD5_REF_DICT = {line.split()[0]: line.split()[1] for line in f.readlines()} -- GitLab From a989e6844734598e1bb90cf7df94f5cd2930f996 Mon Sep 17 00:00:00 2001 From: Charles Kinuthia Date: Thu, 20 Jul 2023 17:18:32 +0200 Subject: [PATCH 5/5] [fix] add missing paranthesis --- tests/conftest.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/conftest.py b/tests/conftest.py index ecca54374a..e4de453b0e 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -521,7 +521,7 @@ def pytest_configure(config): testconfig.PARAM_FILE = config.option.param_file if config.option.selection_be_md5_file: md5_file_path = config.option.selection_be_md5_file - if not platform.system == "Windows": + if not platform.system() == "Windows": raise NotImplementedError("MD5 comparison is currently hardcoded for windows") with open(md5_file_path) as f: testconfig.MD5_REF_DICT = {line.split()[0]: line.split()[1] for line in f.readlines()} -- GitLab