diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 3db54afcaa264f31b5c2f73fe9fd9c7a13346cb5..711c3c5284e772164c9f305356d4995e2abcad98 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1270,14 +1270,14 @@ ivas-conformance: - mkdir testvec/bin - cp -r tests/renderer/data testvec/testv/renderer/data - cp -r tests/renderer/cut testvec/testv/renderer/ref - - cp tests/test_26252.py testvec + - cp -r conformance-test testvec/ - cp Readme_IVAS_dec.txt Readme_IVAS_enc.txt Readme_IVAS_rend.txt testvec - cp IVAS_cod IVAS_dec IVAS_rend testvec/bin # Test run generated scripts in testvec - cd testvec - exit_code=0 - - python3 -m pytest tests/test_26252.py --junitxml=report-junit.xml --html=report.html --self-contained-html || exit_code=$? + - python3 -m pytest conformance-test/test_26252.py --encoder_path bin/IVAS_cod --decoder_path bin/IVAS_dec --renderer_path bin/IVAS_rend --junit-xml=report-junit.xml --html=report.html --self-contained-html || exit_code=$? - zero_errors=$(cat report-junit.xml | grep -c 'errors="0"') || true - *merge-request-comparison-check diff --git a/conformance-test/conftest.py b/conformance-test/conftest.py new file mode 100644 index 0000000000000000000000000000000000000000..0c5ca7606285485dd1abfc44794c2a792a85a1f0 --- /dev/null +++ b/conformance-test/conftest.py @@ -0,0 +1,71 @@ +__copyright__ = """ +(C) 2022-2023 IVAS codec Public Collaboration with portions copyright Dolby International AB, Ericsson AB, +Fraunhofer-Gesellschaft zur Foerderung der angewandten Forschung e.V., Huawei Technologies Co. LTD., +Koninklijke Philips N.V., Nippon Telegraph and Telephone Corporation, Nokia Technologies Oy, Orange, +Panasonic Holdings Corporation, Qualcomm Technologies, Inc., VoiceAge Corporation, and other +contributors to this repository. All Rights Reserved. + +This software is protected by copyright law and by international treaties. +The IVAS codec Public Collaboration consisting of Dolby International AB, Ericsson AB, +Fraunhofer-Gesellschaft zur Foerderung der angewandten Forschung e.V., Huawei Technologies Co. LTD., +Koninklijke Philips N.V., Nippon Telegraph and Telephone Corporation, Nokia Technologies Oy, Orange, +Panasonic Holdings Corporation, Qualcomm Technologies, Inc., VoiceAge Corporation, and other +contributors to this repository retain full ownership rights in their respective contributions in +the software. This notice grants no license of any kind, including but not limited to patent +license, nor is any license granted by implication, estoppel or otherwise. + +Contributors are required to enter into the IVAS codec Public Collaboration agreement before making +contributions. + +This software is provided "AS IS", without any express or implied warranties. The software is in the +development stage. It is intended exclusively for experts who have experience with such software and +solely for the purpose of inspection. All implied warranties of non-infringement, merchantability +and fitness for a particular purpose are hereby disclaimed and excluded. + +Any dispute, controversy or claim arising under or in relation to providing this software shall be +submitted to and settled by the final, binding jurisdiction of the courts of Munich, Germany in +accordance with the laws of the Federal Republic of Germany excluding its conflict of law rules and +the United Nations Convention on Contracts on the International Sales of Goods. +""" + +import pytest +import pathlib + + +def pytest_addoption(parser): + parser.addoption( + "--encoder_path", + action="store", + help="path to encoder binary IVAS_cod(.exe)", + required=True, + type=pathlib.Path, + ) + parser.addoption( + "--decoder_path", + action="store", + help="path to decoder binary IVAS_dec(.exe)", + required=True, + type=pathlib.Path, + ) + parser.addoption( + "--renderer_path", + action="store", + help="path to renderer binary IVAS_rend(.exe)", + required=True, + type=pathlib.Path, + ) + + +@pytest.fixture(scope="session") +def encoder_path(request) -> str: + return str(request.config.option.encoder_path.absolute()) + + +@pytest.fixture(scope="session") +def decoder_path(request) -> str: + return str(request.config.option.decoder_path.absolute()) + + +@pytest.fixture(scope="session") +def renderer_path(request) -> str: + return str(request.config.option.renderer_path.absolute()) diff --git a/conformance-test/pytest.ini b/conformance-test/pytest.ini new file mode 100644 index 0000000000000000000000000000000000000000..6d179f0724d6ccc6a27b19e8e6bfd4e5cc751065 --- /dev/null +++ b/conformance-test/pytest.ini @@ -0,0 +1,15 @@ +# pytest.ini +[pytest] +# TODO remove ignore after tests are harmonized +addopts = -ra --tb=short --basetemp=./tmp -n auto -v +# Write captured system-out log messages to JUnit report. +junit_logging = system-out +# Do not capture log information for passing tests to JUnit report. +junit_log_passing_tests = False +junit_duration_report = call +junit_family = xunit1 +log_file_level = DEBUG +log_format = %(asctime)s %(levelname)s %(message)s +log_date_format = %Y-%m-%d %H:%M:%S +# for pytest-html report: do not log environment variables from the runners +environment_table_redact_list = .* diff --git a/tests/test_26252.py b/conformance-test/test_26252.py similarity index 94% rename from tests/test_26252.py rename to conformance-test/test_26252.py index f91da1bbb21571e2d8cb8de156764e732a446972..a691aa7df9b5f88a112063f89070ae9c955dbb9d 100644 --- a/tests/test_26252.py +++ b/conformance-test/test_26252.py @@ -98,19 +98,19 @@ for s in scripts: subprocess.run(proc.split()) @pytest.mark.parametrize("test_tag", list(test_dict.keys())) -def test_26252(test_tag): +def test_26252(test_tag, encoder_path, decoder_path, renderer_path): enc_opts, dec_opts, rend_opts, diff_opts, testv_path, ref_path, cut_path = test_dict[test_tag] if enc_opts: enc_opts = replace_paths(enc_opts, testv_path, ref_path, cut_path) - subprocess.run(["./bin/IVAS_cod"] + enc_opts.split()[1:], check = True) + subprocess.run([encoder_path] + enc_opts.split()[1:], check = True) if dec_opts: dec_opts = replace_paths(dec_opts, testv_path, ref_path, cut_path) - subprocess.run(["./bin/IVAS_dec"] + dec_opts.split()[1:], check = True) + subprocess.run([decoder_path] + dec_opts.split()[1:], check = True) if rend_opts: rend_opts = replace_paths(rend_opts, testv_path, ref_path, cut_path) - subprocess.run(["./bin/IVAS_rend"] + rend_opts.split()[1:], check = True) + subprocess.run([renderer_path] + rend_opts.split()[1:], check = True) diff_opts = replace_paths(diff_opts, testv_path, ref_path, cut_path) if ';' in diff_opts: