From da44706123223723fb0a4c92c614eb57e061d7c7 Mon Sep 17 00:00:00 2001 From: Archit Tamarapu Date: Fri, 27 Oct 2023 10:28:44 +0200 Subject: [PATCH 01/13] update renderer tests and gitlab-ci configuration to accomodate a change in arguments with ref-using-main --- .gitlab-ci.yml | 11 +- tests/renderer/README.md | 21 +- tests/renderer/conftest.py | 44 ++ tests/renderer/test_renderer.py | 512 +++++++++--------- tests/renderer/test_renderer_be_comparison.py | 328 ----------- tests/renderer/utils.py | 97 ++-- 6 files changed, 370 insertions(+), 643 deletions(-) create mode 100644 tests/renderer/conftest.py delete mode 100644 tests/renderer/test_renderer_be_comparison.py diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 2c15519a84..816bbced0c 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -636,17 +636,22 @@ renderer-pytest-on-merge-request: - make -j IVAS_rend - mv IVAS_rend IVAS_rend_ref + ### If ref_using_main is not set, checkout the source branch to use scripts and input from there + - if [ $ref_using_main == 0 ]; then git checkout $source_branch_commit_sha; fi + - exit_code=0 + - python3 -m pytest -q --log-level ERROR -n auto -rA --html=report.html --self-contained-html --junit-xml=report-junit.xml tests/renderer/test_renderer_be_comparison.py --create_ref || exit_code=$? + # back to source branch - git checkout $source_branch_commit_sha - make clean - make -j IVAS_rend - ### Run test using scripts and input from main - - if [ $ref_using_main == 1 ]; then git checkout $target_commit; fi + ### Run test using branch scripts and input + - if [ $ref_using_main == 1 ]; then git checkout $source_branch_commit_sha; fi # run test - exit_code=0 - - python3 -m pytest -q --log-level ERROR -n auto -rA --html=report.html --self-contained-html --junit-xml=report-junit.xml tests/renderer/test_renderer_be_comparison.py || exit_code=$? + - python3 -m pytest -q --log-level ERROR -n auto -rA --html=report.html --self-contained-html --junit-xml=report-junit.xml tests/renderer/test_renderer_be_comparison.py --create_cut || exit_code=$? - zero_errors=$(cat report-junit.xml | grep -c 'errors="0"') || true - *merge-request-comparison-check diff --git a/tests/renderer/README.md b/tests/renderer/README.md index 513bd09911..fe0223a11b 100644 --- a/tests/renderer/README.md +++ b/tests/renderer/README.md @@ -8,26 +8,35 @@ See also the [contribution page](https://forge.3gpp.org/rep/ivas-codec-pc/ivas-c ### Run tests with: +## Smoke test: + `python3 -m pytest -q -n auto tests/renderer/test_renderer.py` +## Comparison test: + +`python3 -m pytest -q -n auto tests/renderer/test_renderer.py --create_ref` +(note: requires IVAS_rend_ref in root!) + +`python3 -m pytest -q -n auto tests/renderer/test_renderer.py --create_cut` + ### Important flags (see [pytest docs](https://docs.pytest.org/en/7.2.x/) for more information): -* `-k` flag can filter test cases, e.g. `-k "test_ism_binaural_static"` -* `-rA` reports ALL (pass, xpass, xfail, fail) instead of the default behaviour of reporting only failed tests\ -this option will also report captured logs, **required for obtaining the commandline of testcases that pass or xfail** -* `--last-failed` re-runs only the cases that failed in the last test run -* `--collect-only` is useful when adding new testcases to check if argument parametrization is working correctly +- `-k` flag can filter test cases, e.g. `-k "test_ism_binaural_static"` +- `-rA` reports ALL (pass, xpass, xfail, fail) instead of the default behaviour of reporting only failed tests\ + this option will also report captured logs, **required for obtaining the commandline of testcases that pass or xfail** +- `--last-failed` re-runs only the cases that failed in the last test run +- `--collect-only` is useful when adding new testcases to check if argument parametrization is working correctly ### Directory tree ``` . ├── compare_audio.py -> Python implementation of CompAudio, used for comparisons in tests +├── conftest.py -> Pytest configuration (enable commandline argument ingestion) ├── constants.py -> Important paths, formats, metadata files and commandline templates ├── cut -> Default location for output files for test conditions ├── data -> Input test vectors ├── ref -> Default location for output files for reference conditions -├── test_renderer_be_comparison.py -> Tests for CI Merge Request pipeline to compare renderer bit-exactness ├── test_renderer.py -> Runs the renderer for all modes └── utils.py -> Wrapper functions for executables for use in testcases ``` diff --git a/tests/renderer/conftest.py b/tests/renderer/conftest.py new file mode 100644 index 0000000000..4668d742db --- /dev/null +++ b/tests/renderer/conftest.py @@ -0,0 +1,44 @@ +#!/usr/bin/env python3 + +""" + (C) 2022-2023 IVAS codec Public Collaboration with portions copyright Dolby International AB, Ericsson AB, + Fraunhofer-Gesellschaft zur Foerderung der angewandten Forschung e.V., Huawei Technologies Co. LTD., + Koninklijke Philips N.V., Nippon Telegraph and Telephone Corporation, Nokia Technologies Oy, Orange, + Panasonic Holdings Corporation, Qualcomm Technologies, Inc., VoiceAge Corporation, and other + contributors to this repository. All Rights Reserved. + + This software is protected by copyright law and by international treaties. + The IVAS codec Public Collaboration consisting of Dolby International AB, Ericsson AB, + Fraunhofer-Gesellschaft zur Foerderung der angewandten Forschung e.V., Huawei Technologies Co. LTD., + Koninklijke Philips N.V., Nippon Telegraph and Telephone Corporation, Nokia Technologies Oy, Orange, + Panasonic Holdings Corporation, Qualcomm Technologies, Inc., VoiceAge Corporation, and other + contributors to this repository retain full ownership rights in their respective contributions in + the software. This notice grants no license of any kind, including but not limited to patent + license, nor is any license granted by implication, estoppel or otherwise. + + Contributors are required to enter into the IVAS codec Public Collaboration agreement before making + contributions. + + This software is provided "AS IS", without any express or implied warranties. The software is in the + development stage. It is intended exclusively for experts who have experience with such software and + solely for the purpose of inspection. All implied warranties of non-infringement, merchantability + and fitness for a particular purpose are hereby disclaimed and excluded. + + Any dispute, controversy or claim arising under or in relation to providing this software shall be + submitted to and settled by the final, binding jurisdiction of the courts of Munich, Germany in + accordance with the laws of the Federal Republic of Germany excluding its conflict of law rules and + the United Nations Convention on Contracts on the International Sales of Goods. +""" + + +def pytest_addoption(parser): + parser.addoption( + "--create_ref", + action="store_true", + default=False, + ) + parser.addoption( + "--create_cut", + action="store_true", + default=False, + ) diff --git a/tests/renderer/test_renderer.py b/tests/renderer/test_renderer.py index 57a54f6d4e..4ea7a289bb 100644 --- a/tests/renderer/test_renderer.py +++ b/tests/renderer/test_renderer.py @@ -34,6 +34,12 @@ import pytest from .utils import * +############################################################################## +# Bit-exactness tests +# +# These tests are run in REF and CUT mode and the outputs are compared for +# bit-exactness +############################################################################## """ Ambisonics """ @@ -42,9 +48,9 @@ from .utils import * @pytest.mark.parametrize("framing_5ms", FRAMING_5MS_TO_TEST) def test_ambisonics(test_info, in_fmt, out_fmt, framing_5ms): run_renderer( + test_info, in_fmt, out_fmt, - test_case_name=test_info.node.name, framing_5ms=(framing_5ms == "5ms"), ) @@ -53,11 +59,10 @@ def test_ambisonics(test_info, in_fmt, out_fmt, framing_5ms): @pytest.mark.parametrize("in_fmt", INPUT_FORMATS_AMBI) @pytest.mark.parametrize("framing_5ms", FRAMING_5MS_TO_TEST) def test_ambisonics_binaural_static(test_info, in_fmt, out_fmt, framing_5ms): - run_renderer( + test_info, in_fmt, out_fmt, - test_case_name=test_info.node.name, framing_5ms=(framing_5ms == "5ms"), ) @@ -69,16 +74,248 @@ def test_ambisonics_binaural_static(test_info, in_fmt, out_fmt, framing_5ms): def test_ambisonics_binaural_headrotation( test_info, in_fmt, out_fmt, trj_file, framing_5ms ): + run_renderer( + test_info, + in_fmt, + out_fmt, + trj_file=HR_TRAJECTORY_DIR.joinpath(f"{trj_file}.csv"), + framing_5ms=(framing_5ms == "5ms"), + ) + +""" Multichannel """ + + +@pytest.mark.parametrize("out_fmt", OUTPUT_FORMATS[2:]) +@pytest.mark.parametrize("in_fmt", INPUT_FORMATS_MC) +@pytest.mark.parametrize("framing_5ms", FRAMING_5MS_TO_TEST) +def test_multichannel(test_info, in_fmt, out_fmt, framing_5ms): run_renderer( + test_info, + in_fmt, + out_fmt, + framing_5ms=(framing_5ms == "5ms"), + ) + + +@pytest.mark.parametrize("out_fmt", OUTPUT_FORMATS_BINAURAL) +@pytest.mark.parametrize("in_fmt", INPUT_FORMATS_MC) +@pytest.mark.parametrize("framing_5ms", FRAMING_5MS_TO_TEST) +def test_multichannel_binaural_static(test_info, in_fmt, out_fmt, framing_5ms): + if in_fmt in ["MONO", "STEREO"]: + pytest.skip("MONO or STEREO to Binaural rendering unsupported") + + run_renderer( + test_info, + in_fmt, + out_fmt, + framing_5ms=(framing_5ms == "5ms"), + ) + + +@pytest.mark.parametrize("trj_file", HR_TRAJECTORIES_TO_TEST) +@pytest.mark.parametrize("out_fmt", OUTPUT_FORMATS_BINAURAL) +@pytest.mark.parametrize("in_fmt", INPUT_FORMATS_MC) +@pytest.mark.parametrize("framing_5ms", FRAMING_5MS_TO_TEST) +def test_multichannel_binaural_headrotation( + test_info, in_fmt, out_fmt, trj_file, framing_5ms +): + if in_fmt in ["MONO", "STEREO"]: + pytest.skip("MONO or STEREO to Binaural rendering unsupported") + + run_renderer( + test_info, in_fmt, out_fmt, - test_case_name=test_info.node.name, trj_file=HR_TRAJECTORY_DIR.joinpath(f"{trj_file}.csv"), framing_5ms=(framing_5ms == "5ms"), ) +""" ISM """ + + +@pytest.mark.parametrize("out_fmt", OUTPUT_FORMATS[2:]) +@pytest.mark.parametrize("in_fmt", INPUT_FORMATS_ISM) +@pytest.mark.parametrize("framing_5ms", FRAMING_5MS_TO_TEST) +def test_ism(test_info, in_fmt, out_fmt, framing_5ms): + run_renderer( + test_info, + in_fmt, + out_fmt, + in_meta_files=FORMAT_TO_METADATA_FILES[in_fmt], + framing_5ms=(framing_5ms == "5ms"), + ) + + +@pytest.mark.parametrize("out_fmt", OUTPUT_FORMATS_BINAURAL) +@pytest.mark.parametrize("in_fmt", INPUT_FORMATS_ISM) +@pytest.mark.parametrize("framing_5ms", FRAMING_5MS_TO_TEST) +def test_ism_binaural_static(test_info, in_fmt, out_fmt, framing_5ms): + try: + in_meta_files = FORMAT_TO_METADATA_FILES[in_fmt] + except KeyError: + in_meta_files = None + + run_renderer( + test_info, + in_fmt, + out_fmt, + in_meta_files=in_meta_files, + framing_5ms=(framing_5ms == "5ms"), + ) + + +@pytest.mark.parametrize("trj_file", HR_TRAJECTORIES_TO_TEST) +@pytest.mark.parametrize("out_fmt", OUTPUT_FORMATS_BINAURAL) +@pytest.mark.parametrize("in_fmt", INPUT_FORMATS_ISM) +@pytest.mark.parametrize("framing_5ms", FRAMING_5MS_TO_TEST) +def test_ism_binaural_headrotation(test_info, in_fmt, out_fmt, trj_file, framing_5ms): + try: + in_meta_files = FORMAT_TO_METADATA_FILES[in_fmt] + except KeyError: + in_meta_files = None + + run_renderer( + test_info, + in_fmt, + out_fmt, + trj_file=HR_TRAJECTORY_DIR.joinpath(f"{trj_file}.csv"), + in_meta_files=in_meta_files, + framing_5ms=(framing_5ms == "5ms"), + ) + + +""" MASA """ + + +@pytest.mark.parametrize("out_fmt", OUTPUT_FORMATS) +@pytest.mark.parametrize("in_fmt", INPUT_FORMATS_MASA) +@pytest.mark.parametrize("framing_5ms", FRAMING_5MS_TO_TEST) +def test_masa(test_info, in_fmt, out_fmt, framing_5ms): + run_renderer( + test_info, + in_fmt, + out_fmt, + in_meta_files=FORMAT_TO_METADATA_FILES[in_fmt], + framing_5ms=(framing_5ms == "5ms"), + ) + + +""" Custom loudspeaker layouts """ + + +@pytest.mark.parametrize("out_fmt", OUTPUT_FORMATS[2:]) +@pytest.mark.parametrize("in_layout", CUSTOM_LS_TO_TEST) +@pytest.mark.parametrize("framing_5ms", FRAMING_5MS_TO_TEST) +def test_custom_ls_input(test_info, in_layout, out_fmt, framing_5ms): + run_renderer( + test_info, + CUSTOM_LAYOUT_DIR.joinpath(f"{in_layout}.txt"), + out_fmt, + framing_5ms=(framing_5ms == "5ms"), + ) + + +@pytest.mark.parametrize("out_fmt", CUSTOM_LS_TO_TEST) +@pytest.mark.parametrize("in_fmt", OUTPUT_FORMATS[2:]) +def test_custom_ls_output(test_info, in_fmt, out_fmt): + run_renderer( + test_info, + in_fmt, + CUSTOM_LAYOUT_DIR.joinpath(f"{out_fmt}.txt"), + ) + + +@pytest.mark.parametrize("out_fmt", CUSTOM_LS_TO_TEST) +@pytest.mark.parametrize("in_fmt", CUSTOM_LS_TO_TEST) +def test_custom_ls_input_output(test_info, in_fmt, out_fmt): + run_renderer( + test_info, + CUSTOM_LAYOUT_DIR.joinpath(f"{in_fmt}.txt"), + CUSTOM_LAYOUT_DIR.joinpath(f"{out_fmt}.txt"), + ) + + +@pytest.mark.parametrize("out_fmt", OUTPUT_FORMATS_BINAURAL) +@pytest.mark.parametrize("in_layout", CUSTOM_LS_TO_TEST) +@pytest.mark.parametrize("framing_5ms", FRAMING_5MS_TO_TEST) +def test_custom_ls_input_binaural(test_info, in_layout, out_fmt, framing_5ms): + run_renderer( + test_info, + CUSTOM_LAYOUT_DIR.joinpath(f"{in_layout}.txt"), + out_fmt, + framing_5ms=(framing_5ms == "5ms"), + ) + + +@pytest.mark.parametrize("trj_file", HR_TRAJECTORIES_TO_TEST) +@pytest.mark.parametrize("out_fmt", OUTPUT_FORMATS_BINAURAL) +@pytest.mark.parametrize("in_layout", CUSTOM_LS_TO_TEST) +@pytest.mark.parametrize("framing_5ms", FRAMING_5MS_TO_TEST) +def test_custom_ls_input_binaural_headrotation( + test_info, in_layout, out_fmt, trj_file, framing_5ms +): + run_renderer( + test_info, + CUSTOM_LAYOUT_DIR.joinpath(f"{in_layout}.txt"), + out_fmt, + trj_file=HR_TRAJECTORY_DIR.joinpath(f"{trj_file}.csv"), + framing_5ms=(framing_5ms == "5ms"), + ) + + +""" Metadata / scene description input """ + + +@pytest.mark.parametrize("out_fmt", OUTPUT_FORMATS[2:]) +@pytest.mark.parametrize("in_fmt", METADATA_SCENES_TO_TEST) +@pytest.mark.parametrize("framing_5ms", FRAMING_5MS_TO_TEST) +def test_metadata(test_info, in_fmt, out_fmt, framing_5ms): + run_renderer( + test_info, + "META", + out_fmt, + metadata_input=TEST_VECTOR_DIR.joinpath(f"{in_fmt}.txt"), + framing_5ms=(framing_5ms == "5ms"), + ) + + +""" non diegetic pan """ + + +@pytest.mark.parametrize("out_fmt", ["STEREO"]) +@pytest.mark.parametrize("in_fmt", ["MONO"]) +@pytest.mark.parametrize("non_diegetic_pan", ["0", "-30", "45", "90", "-90"]) +def test_non_diegetic_pan_static(test_info, in_fmt, out_fmt, non_diegetic_pan): + run_renderer( + test_info, + in_fmt, + out_fmt, + non_diegetic_pan=non_diegetic_pan, + ) + + +@pytest.mark.parametrize("out_fmt", ["STEREO"]) +@pytest.mark.parametrize("in_fmt", ["ISM1"]) +@pytest.mark.parametrize("non_diegetic_pan", ["0", "-30", "45", "90", "-90"]) +def test_non_diegetic_pan_ism_static(test_info, in_fmt, out_fmt, non_diegetic_pan): + run_renderer( + test_info, + in_fmt, + out_fmt, + non_diegetic_pan=non_diegetic_pan, + ) + + +############################################################################## +# Smoke tests +# +# These tests are run only for the smoke test and do not perform a +# bit-exactness comparison between REF and CUT, but between CUT1 and CUT2 +############################################################################## + + # Test compares rendering with just a trajectory file against rendering with a trajectory file + a zero ref rotation. # These should be binary equivalent. @pytest.mark.parametrize("trj_file", HR_TRAJECTORIES_TO_TEST) @@ -87,6 +324,9 @@ def test_ambisonics_binaural_headrotation( def test_ambisonics_binaural_headrotation_refrotzero( test_info, in_fmt, out_fmt, trj_file ): + if test_info.config.option.create_ref or test_info.config.option.create_cut: + pytest.skip("OTR tests only run for smoke test") + compare_renderer_args( test_info, in_fmt, @@ -110,6 +350,9 @@ def test_ambisonics_binaural_headrotation_refrotzero( @pytest.mark.parametrize("out_fmt", OUTPUT_FORMATS_BINAURAL) @pytest.mark.parametrize("in_fmt", INPUT_FORMATS_AMBI) def test_ambisonics_binaural_headrotation_refrotequal(test_info, in_fmt, out_fmt): + if test_info.config.option.create_ref or test_info.config.option.create_cut: + pytest.skip("OTR tests only run for smoke test") + compare_renderer_args( test_info, in_fmt, @@ -140,6 +383,9 @@ def test_ambisonics_binaural_headrotation_refrotequal(test_info, in_fmt, out_fmt def test_ambisonics_binaural_headrotation_refveczero( test_info, in_fmt, out_fmt, trj_file ): + if test_info.config.option.create_ref or test_info.config.option.create_cut: + pytest.skip("OTR tests only run for smoke test") + compare_renderer_args( test_info, in_fmt, @@ -164,6 +410,9 @@ def test_ambisonics_binaural_headrotation_refveczero( @pytest.mark.parametrize("out_fmt", OUTPUT_FORMATS_BINAURAL) @pytest.mark.parametrize("in_fmt", INPUT_FORMATS_AMBI) def test_ambisonics_binaural_headrotation_refvecequal(test_info, in_fmt, out_fmt): + if test_info.config.option.create_ref or test_info.config.option.create_cut: + pytest.skip("OTR tests only run for smoke test") + # TODO revert if in_fmt == "HOA3" and out_fmt == "BINAURAL_ROOM_REVERB": pytest.xfail("WIP : minor differences to be resolved") @@ -195,6 +444,9 @@ def test_ambisonics_binaural_headrotation_refvecequal(test_info, in_fmt, out_fmt @pytest.mark.parametrize("out_fmt", OUTPUT_FORMATS_BINAURAL) @pytest.mark.parametrize("in_fmt", INPUT_FORMATS_AMBI) def test_ambisonics_binaural_headrotation_refvec_rotating(test_info, in_fmt, out_fmt): + if test_info.config.option.create_ref or test_info.config.option.create_cut: + pytest.skip("OTR tests only run for smoke test") + # TODO revert if in_fmt == "HOA2" and out_fmt == "BINAURAL_ROOM_REVERB": pytest.xfail("WIP : minor differences to be resolved") @@ -231,6 +483,9 @@ def test_ambisonics_binaural_headrotation_refvec_rotating(test_info, in_fmt, out def test_ambisonics_binaural_headrotation_refvec_rotating_fixed_pos_offset( test_info, in_fmt, out_fmt ): + if test_info.config.option.create_ref or test_info.config.option.create_cut: + pytest.skip("OTR tests only run for smoke test") + compare_renderer_args( test_info, in_fmt, @@ -262,6 +517,9 @@ def test_ambisonics_binaural_headrotation_refvec_rotating_fixed_pos_offset( def test_ambisonics_binaural_headrotation_refveclev_vs_refvec( test_info, in_fmt, out_fmt ): + if test_info.config.option.create_ref or test_info.config.option.create_cut: + pytest.skip("OTR tests only run for smoke test") + compare_renderer_args( test_info, in_fmt, @@ -282,64 +540,6 @@ def test_ambisonics_binaural_headrotation_refveclev_vs_refvec( ) -""" Multichannel """ - - -@pytest.mark.parametrize("out_fmt", OUTPUT_FORMATS[2:]) -@pytest.mark.parametrize("in_fmt", INPUT_FORMATS_MC) -@pytest.mark.parametrize("framing_5ms", FRAMING_5MS_TO_TEST) -def test_multichannel(test_info, in_fmt, out_fmt, framing_5ms): - run_renderer( - in_fmt, - out_fmt, - test_case_name=test_info.node.name, - framing_5ms=(framing_5ms == "5ms"), - ) - - -@pytest.mark.parametrize("out_fmt", OUTPUT_FORMATS_BINAURAL) -@pytest.mark.parametrize("in_fmt", INPUT_FORMATS_MC) -@pytest.mark.parametrize("framing_5ms", FRAMING_5MS_TO_TEST) -def test_multichannel_binaural_static(test_info, in_fmt, out_fmt, framing_5ms): - if in_fmt in ["MONO", "STEREO"]: - pytest.skip("MONO or STEREO to Binaural rendering unsupported") - - run_renderer( - in_fmt, - out_fmt, - test_case_name=test_info.node.name, - framing_5ms=(framing_5ms == "5ms"), - ) - - -@pytest.mark.parametrize("trj_file", HR_TRAJECTORIES_TO_TEST) -@pytest.mark.parametrize("out_fmt", OUTPUT_FORMATS_BINAURAL) -@pytest.mark.parametrize("in_fmt", INPUT_FORMATS_MC) -@pytest.mark.parametrize("framing_5ms", FRAMING_5MS_TO_TEST) -def test_multichannel_binaural_headrotation( - test_info, in_fmt, out_fmt, trj_file, framing_5ms -): - if in_fmt in ["MONO", "STEREO"]: - pytest.skip("MONO or STEREO to Binaural rendering unsupported") - - if (in_fmt == "5_1" or in_fmt == "7_1") and out_fmt == "BINAURAL": - run_renderer( - in_fmt, - out_fmt, - test_case_name=test_info.node.name, - trj_file=HR_TRAJECTORY_DIR.joinpath(f"{trj_file}.csv"), - framing_5ms=(framing_5ms == "5ms"), - ) - else: - run_renderer( - in_fmt, - out_fmt, - test_case_name=test_info.node.name, - trj_file=HR_TRAJECTORY_DIR.joinpath(f"{trj_file}.csv"), - framing_5ms=(framing_5ms == "5ms"), - ) - - # This test compares rendering with: # ref: a head rotation trajectory with elevation (OTR=NONE) # cut: a static head rotation and a reference position trajectory which moves @@ -347,6 +547,9 @@ def test_multichannel_binaural_headrotation( @pytest.mark.parametrize("out_fmt", OUTPUT_FORMATS_BINAURAL) @pytest.mark.parametrize("in_fmt", INPUT_FORMATS_MC) def test_multichannel_binaural_headrotation_refvec_rotating(test_info, in_fmt, out_fmt): + if test_info.config.option.create_ref or test_info.config.option.create_cut: + pytest.skip("OTR tests only run for smoke test") + if in_fmt in ["MONO", "STEREO"]: pytest.skip("MONO or STEREO to Binaural rendering unsupported") @@ -371,62 +574,6 @@ def test_multichannel_binaural_headrotation_refvec_rotating(test_info, in_fmt, o ) -""" ISM """ - - -@pytest.mark.parametrize("out_fmt", OUTPUT_FORMATS[2:]) -@pytest.mark.parametrize("in_fmt", INPUT_FORMATS_ISM) -@pytest.mark.parametrize("framing_5ms", FRAMING_5MS_TO_TEST) -def test_ism(test_info, in_fmt, out_fmt, framing_5ms): - run_renderer( - in_fmt, - out_fmt, - test_case_name=test_info.node.name, - in_meta_files=FORMAT_TO_METADATA_FILES[in_fmt], - framing_5ms=(framing_5ms == "5ms"), - ) - - -@pytest.mark.parametrize("out_fmt", OUTPUT_FORMATS_BINAURAL) -@pytest.mark.parametrize("in_fmt", INPUT_FORMATS_ISM) -@pytest.mark.parametrize("framing_5ms", FRAMING_5MS_TO_TEST) -def test_ism_binaural_static(test_info, in_fmt, out_fmt, framing_5ms): - - try: - in_meta_files = FORMAT_TO_METADATA_FILES[in_fmt] - except: - in_meta_files = None - - run_renderer( - in_fmt, - out_fmt, - test_case_name=test_info.node.name, - in_meta_files=in_meta_files, - framing_5ms=(framing_5ms == "5ms"), - ) - - -@pytest.mark.parametrize("trj_file", HR_TRAJECTORIES_TO_TEST) -@pytest.mark.parametrize("out_fmt", OUTPUT_FORMATS_BINAURAL) -@pytest.mark.parametrize("in_fmt", INPUT_FORMATS_ISM) -@pytest.mark.parametrize("framing_5ms", FRAMING_5MS_TO_TEST) -def test_ism_binaural_headrotation(test_info, in_fmt, out_fmt, trj_file, framing_5ms): - - try: - in_meta_files = FORMAT_TO_METADATA_FILES[in_fmt] - except: - in_meta_files = None - - run_renderer( - in_fmt, - out_fmt, - test_case_name=test_info.node.name, - trj_file=HR_TRAJECTORY_DIR.joinpath(f"{trj_file}.csv"), - in_meta_files=in_meta_files, - framing_5ms=(framing_5ms == "5ms"), - ) - - # This test compares rendering with: # ref: a head rotation trajectory with elevation (OTR=NONE) # cut: a static head rotation and a reference position trajectory which moves @@ -434,9 +581,12 @@ def test_ism_binaural_headrotation(test_info, in_fmt, out_fmt, trj_file, framing @pytest.mark.parametrize("out_fmt", OUTPUT_FORMATS_BINAURAL) @pytest.mark.parametrize("in_fmt", INPUT_FORMATS_ISM) def test_ism_binaural_headrotation_refvec_rotating(test_info, in_fmt, out_fmt): + if test_info.config.option.create_ref or test_info.config.option.create_cut: + pytest.skip("OTR tests only run for smoke test") + try: in_meta_files = FORMAT_TO_METADATA_FILES[in_fmt] - except: + except KeyError: in_meta_files = None compare_renderer_args( @@ -460,127 +610,3 @@ def test_ism_binaural_headrotation_refvec_rotating(test_info, in_fmt, out_fmt): "framing_5ms": "5ms", }, ) - - -""" MASA """ - - -@pytest.mark.parametrize("out_fmt", OUTPUT_FORMATS) -@pytest.mark.parametrize("in_fmt", INPUT_FORMATS_MASA) -@pytest.mark.parametrize("framing_5ms", FRAMING_5MS_TO_TEST) -def test_masa(test_info, in_fmt, out_fmt, framing_5ms): - run_renderer( - in_fmt, - out_fmt, - test_case_name=test_info.node.name, - in_meta_files=FORMAT_TO_METADATA_FILES[in_fmt], - framing_5ms=(framing_5ms == "5ms"), - ) - - -""" Custom loudspeaker layouts """ - - -@pytest.mark.parametrize("out_fmt", OUTPUT_FORMATS[2:]) -@pytest.mark.parametrize("in_layout", CUSTOM_LS_TO_TEST) -@pytest.mark.parametrize("framing_5ms", FRAMING_5MS_TO_TEST) -def test_custom_ls_input(test_info, in_layout, out_fmt, framing_5ms): - run_renderer( - CUSTOM_LAYOUT_DIR.joinpath(f"{in_layout}.txt"), - out_fmt, - test_case_name=test_info.node.name, - framing_5ms=(framing_5ms == "5ms"), - ) - - -@pytest.mark.parametrize("out_fmt", CUSTOM_LS_TO_TEST) -@pytest.mark.parametrize("in_fmt", OUTPUT_FORMATS[2:]) -def test_custom_ls_output(test_info, in_fmt, out_fmt): - run_renderer( - in_fmt, - CUSTOM_LAYOUT_DIR.joinpath(f"{out_fmt}.txt"), - test_case_name=test_info.node.name, - ) - - -@pytest.mark.parametrize("out_fmt", CUSTOM_LS_TO_TEST) -@pytest.mark.parametrize("in_fmt", CUSTOM_LS_TO_TEST) -def test_custom_ls_input_output(test_info, in_fmt, out_fmt): - run_renderer( - CUSTOM_LAYOUT_DIR.joinpath(f"{in_fmt}.txt"), - CUSTOM_LAYOUT_DIR.joinpath(f"{out_fmt}.txt"), - test_case_name=test_info.node.name, - ) - - -@pytest.mark.parametrize("out_fmt", OUTPUT_FORMATS_BINAURAL) -@pytest.mark.parametrize("in_layout", CUSTOM_LS_TO_TEST) -@pytest.mark.parametrize("framing_5ms", FRAMING_5MS_TO_TEST) -def test_custom_ls_input_binaural(test_info, in_layout, out_fmt, framing_5ms): - - run_renderer( - CUSTOM_LAYOUT_DIR.joinpath(f"{in_layout}.txt"), - out_fmt, - framing_5ms=(framing_5ms == "5ms"), - test_case_name=test_info.node.name, - ) - - -@pytest.mark.parametrize("trj_file", HR_TRAJECTORIES_TO_TEST) -@pytest.mark.parametrize("out_fmt", OUTPUT_FORMATS_BINAURAL) -@pytest.mark.parametrize("in_layout", CUSTOM_LS_TO_TEST) -@pytest.mark.parametrize("framing_5ms", FRAMING_5MS_TO_TEST) -def test_custom_ls_input_binaural_headrotation( - test_info, in_layout, out_fmt, trj_file, framing_5ms -): - - run_renderer( - CUSTOM_LAYOUT_DIR.joinpath(f"{in_layout}.txt"), - out_fmt, - test_case_name=test_info.node.name, - trj_file=HR_TRAJECTORY_DIR.joinpath(f"{trj_file}.csv"), - framing_5ms=(framing_5ms == "5ms"), - ) - - -""" Metadata / scene description input """ - - -@pytest.mark.parametrize("out_fmt", OUTPUT_FORMATS[2:]) -@pytest.mark.parametrize("in_fmt", METADATA_SCENES_TO_TEST) -@pytest.mark.parametrize("framing_5ms", FRAMING_5MS_TO_TEST) -def test_metadata(test_info, in_fmt, out_fmt, framing_5ms): - run_renderer( - "META", - out_fmt, - test_case_name=test_info.node.name, - metadata_input=TEST_VECTOR_DIR.joinpath(f"{in_fmt}.txt"), - framing_5ms=(framing_5ms == "5ms"), - ) - - -""" non diegetic pan """ - - -@pytest.mark.parametrize("out_fmt", ["STEREO"]) -@pytest.mark.parametrize("in_fmt", ["MONO"]) -@pytest.mark.parametrize("non_diegetic_pan", ["0", "-30", "45", "90", "-90"]) -def test_non_diegetic_pan_static(test_info, in_fmt, out_fmt, non_diegetic_pan): - run_renderer( - in_fmt, - out_fmt, - test_case_name=test_info.node.name, - non_diegetic_pan=non_diegetic_pan, - ) - - -@pytest.mark.parametrize("out_fmt", ["STEREO"]) -@pytest.mark.parametrize("in_fmt", ["ISM1"]) -@pytest.mark.parametrize("non_diegetic_pan", ["0", "-30", "45", "90", "-90"]) -def test_non_diegetic_pan_ism_static(test_info, in_fmt, out_fmt, non_diegetic_pan): - run_renderer( - in_fmt, - out_fmt, - test_case_name=test_info.node.name, - non_diegetic_pan=non_diegetic_pan, - ) diff --git a/tests/renderer/test_renderer_be_comparison.py b/tests/renderer/test_renderer_be_comparison.py deleted file mode 100644 index 56ffe5470d..0000000000 --- a/tests/renderer/test_renderer_be_comparison.py +++ /dev/null @@ -1,328 +0,0 @@ -#!/usr/bin/env python3 - -""" - (C) 2022-2023 IVAS codec Public Collaboration with portions copyright Dolby International AB, Ericsson AB, - Fraunhofer-Gesellschaft zur Foerderung der angewandten Forschung e.V., Huawei Technologies Co. LTD., - Koninklijke Philips N.V., Nippon Telegraph and Telephone Corporation, Nokia Technologies Oy, Orange, - Panasonic Holdings Corporation, Qualcomm Technologies, Inc., VoiceAge Corporation, and other - contributors to this repository. All Rights Reserved. - - This software is protected by copyright law and by international treaties. - The IVAS codec Public Collaboration consisting of Dolby International AB, Ericsson AB, - Fraunhofer-Gesellschaft zur Foerderung der angewandten Forschung e.V., Huawei Technologies Co. LTD., - Koninklijke Philips N.V., Nippon Telegraph and Telephone Corporation, Nokia Technologies Oy, Orange, - Panasonic Holdings Corporation, Qualcomm Technologies, Inc., VoiceAge Corporation, and other - contributors to this repository retain full ownership rights in their respective contributions in - the software. This notice grants no license of any kind, including but not limited to patent - license, nor is any license granted by implication, estoppel or otherwise. - - Contributors are required to enter into the IVAS codec Public Collaboration agreement before making - contributions. - - This software is provided "AS IS", without any express or implied warranties. The software is in the - development stage. It is intended exclusively for experts who have experience with such software and - solely for the purpose of inspection. All implied warranties of non-infringement, merchantability - and fitness for a particular purpose are hereby disclaimed and excluded. - - Any dispute, controversy or claim arising under or in relation to providing this software shall be - submitted to and settled by the final, binding jurisdiction of the courts of Munich, Germany in - accordance with the laws of the Federal Republic of Germany excluding its conflict of law rules and - the United Nations Convention on Contracts on the International Sales of Goods. -""" - -import pytest - -from .utils import * - -""" Ambisonics """ - - -@pytest.mark.parametrize("out_fmt", OUTPUT_FORMATS) -@pytest.mark.parametrize("in_fmt", INPUT_FORMATS_AMBI) -def test_ambisonics(test_info, in_fmt, out_fmt): - compare_renderer_vs_mergetarget(test_info, in_fmt, out_fmt, is_comparetest=True) - - -@pytest.mark.parametrize("out_fmt", OUTPUT_FORMATS_BINAURAL) -@pytest.mark.parametrize("in_fmt", INPUT_FORMATS_AMBI) -@pytest.mark.parametrize("framing_5ms", FRAMING_5MS_TO_TEST) -def test_ambisonics_binaural_static(test_info, in_fmt, out_fmt, framing_5ms): - compare_renderer_vs_mergetarget( - test_info, - in_fmt, - out_fmt, - framing_5ms=(framing_5ms == "5ms"), - is_comparetest=True, - ) - - -@pytest.mark.parametrize("trj_file", HR_TRAJECTORIES_TO_TEST) -@pytest.mark.parametrize("out_fmt", OUTPUT_FORMATS_BINAURAL) -@pytest.mark.parametrize("in_fmt", INPUT_FORMATS_AMBI) -@pytest.mark.parametrize("framing_5ms", FRAMING_5MS_TO_TEST) -def test_ambisonics_binaural_headrotation( - test_info, in_fmt, out_fmt, trj_file, framing_5ms -): - compare_renderer_vs_mergetarget( - test_info, - in_fmt, - out_fmt, - trj_file=HR_TRAJECTORY_DIR.joinpath(f"{trj_file}.csv"), - framing_5ms=(framing_5ms == "5ms"), - is_comparetest=True, - ) - - -""" Multichannel """ - - -@pytest.mark.parametrize("out_fmt", OUTPUT_FORMATS) -@pytest.mark.parametrize("in_fmt", INPUT_FORMATS_MC) -def test_multichannel(test_info, in_fmt, out_fmt): - compare_renderer_vs_mergetarget(test_info, in_fmt, out_fmt, is_comparetest=True) - - -@pytest.mark.parametrize("out_fmt", OUTPUT_FORMATS_BINAURAL) -@pytest.mark.parametrize("in_fmt", INPUT_FORMATS_MC) -@pytest.mark.parametrize("framing_5ms", FRAMING_5MS_TO_TEST) -def test_multichannel_binaural_static(test_info, in_fmt, out_fmt, framing_5ms): - if in_fmt in ["MONO", "STEREO"]: - pytest.skip("MONO or STEREO to Binaural rendering unsupported") - - compare_renderer_vs_mergetarget( - test_info, - in_fmt, - out_fmt, - framing_5ms=(framing_5ms == "5ms"), - is_comparetest=True, - ) - - -@pytest.mark.parametrize("trj_file", HR_TRAJECTORIES_TO_TEST) -@pytest.mark.parametrize("out_fmt", OUTPUT_FORMATS_BINAURAL) -@pytest.mark.parametrize("in_fmt", INPUT_FORMATS_MC) -@pytest.mark.parametrize("framing_5ms", FRAMING_5MS_TO_TEST) -def test_multichannel_binaural_headrotation( - test_info, in_fmt, out_fmt, trj_file, framing_5ms -): - if in_fmt in ["MONO", "STEREO"]: - pytest.skip("MONO or STEREO to Binaural rendering unsupported") - - compare_renderer_vs_mergetarget( - test_info, - in_fmt, - out_fmt, - trj_file=HR_TRAJECTORY_DIR.joinpath(f"{trj_file}.csv"), - framing_5ms=(framing_5ms == "5ms"), - is_comparetest=True, - ) - - -""" ISM """ - - -@pytest.mark.parametrize("out_fmt", OUTPUT_FORMATS) -@pytest.mark.parametrize("in_fmt", INPUT_FORMATS_ISM) -def test_ism(test_info, in_fmt, out_fmt): - compare_renderer_vs_mergetarget( - test_info, in_fmt, out_fmt, in_meta_files=FORMAT_TO_METADATA_FILES[in_fmt] - ) - - -@pytest.mark.parametrize("out_fmt", OUTPUT_FORMATS_BINAURAL) -@pytest.mark.parametrize("in_fmt", INPUT_FORMATS_ISM) -@pytest.mark.parametrize("framing_5ms", FRAMING_5MS_TO_TEST) -def test_ism_binaural_static(test_info, in_fmt, out_fmt, framing_5ms): - - try: - in_meta_files = FORMAT_TO_METADATA_FILES[in_fmt] - except: - in_meta_files = None - - compare_renderer_vs_mergetarget( - test_info, - in_fmt, - out_fmt, - in_meta_files=in_meta_files, - framing_5ms=(framing_5ms == "5ms"), - is_comparetest=True, - ) - - -@pytest.mark.parametrize("trj_file", HR_TRAJECTORIES_TO_TEST) -@pytest.mark.parametrize("out_fmt", OUTPUT_FORMATS_BINAURAL) -@pytest.mark.parametrize("in_fmt", INPUT_FORMATS_ISM) -@pytest.mark.parametrize("framing_5ms", FRAMING_5MS_TO_TEST) -def test_ism_binaural_headrotation(test_info, in_fmt, out_fmt, trj_file, framing_5ms): - - try: - in_meta_files = FORMAT_TO_METADATA_FILES[in_fmt] - except: - in_meta_files = None - - compare_renderer_vs_mergetarget( - test_info, - in_fmt, - out_fmt, - trj_file=HR_TRAJECTORY_DIR.joinpath(f"{trj_file}.csv"), - in_meta_files=in_meta_files, - framing_5ms=(framing_5ms == "5ms"), - is_comparetest=True, - ) - - -""" MASA """ - - -@pytest.mark.parametrize("out_fmt", OUTPUT_FORMATS) -@pytest.mark.parametrize("in_fmt", INPUT_FORMATS_MASA) -def test_masa(test_info, in_fmt, out_fmt): - compare_renderer_vs_mergetarget( - test_info, in_fmt, out_fmt, in_meta_files=FORMAT_TO_METADATA_FILES[in_fmt] - ) - - -@pytest.mark.parametrize("out_fmt", OUTPUT_FORMATS_BINAURAL) -@pytest.mark.parametrize("in_fmt", INPUT_FORMATS_MASA) -@pytest.mark.parametrize("framing_5ms", FRAMING_5MS_TO_TEST) -def test_masa_binaural_static(test_info, in_fmt, out_fmt, framing_5ms): - if out_fmt in ["BINAURAL_ROOM_IR", "BINAURAL_ROOM_REVERB"]: - pytest.skip("Skipping binaural room outputs for MASA as unimplemented.") - - compare_renderer_vs_mergetarget( - test_info, - in_fmt, - out_fmt, - in_meta_files=FORMAT_TO_METADATA_FILES[in_fmt], - framing_5ms=(framing_5ms == "5ms"), - ) - - -@pytest.mark.parametrize("trj_file", HR_TRAJECTORIES_TO_TEST) -@pytest.mark.parametrize("out_fmt", OUTPUT_FORMATS_BINAURAL) -@pytest.mark.parametrize("in_fmt", INPUT_FORMATS_MASA) -@pytest.mark.parametrize("framing_5ms", FRAMING_5MS_TO_TEST) -def test_masa_binaural_headrotation(test_info, in_fmt, out_fmt, trj_file, framing_5ms): - if out_fmt in ["BINAURAL_ROOM_IR", "BINAURAL_ROOM_REVERB"]: - pytest.skip("Skipping binaural room outputs for MASA as unimplemented.") - - compare_renderer_vs_mergetarget( - test_info, - in_fmt, - out_fmt, - trj_file=HR_TRAJECTORY_DIR.joinpath(f"{trj_file}.csv"), - in_meta_files=FORMAT_TO_METADATA_FILES[in_fmt], - framing_5ms=(framing_5ms == "5ms"), - ) - - -@pytest.mark.parametrize("in_fmt", METADATA_SCENES_TO_TEST_MASA_PREREND) -def test_masa_prerend(test_info, in_fmt): - compare_renderer_vs_mergetarget_plus_meta( - test_info, - "META", - "MASA2", - metadata_input=TEST_VECTOR_DIR.joinpath(f"{in_fmt}.txt"), - ) - - -""" Custom loudspeaker layouts """ - - -@pytest.mark.parametrize("out_fmt", OUTPUT_FORMATS) -@pytest.mark.parametrize("in_layout", CUSTOM_LS_TO_TEST) -def test_custom_ls_input(test_info, in_layout, out_fmt): - compare_renderer_vs_mergetarget( - test_info, - CUSTOM_LAYOUT_DIR.joinpath(f"{in_layout}.txt"), - out_fmt, - is_comparetest=True, - ) - - -@pytest.mark.parametrize("out_fmt", CUSTOM_LS_TO_TEST) -@pytest.mark.parametrize("in_fmt", OUTPUT_FORMATS) -def test_custom_ls_output(test_info, in_fmt, out_fmt): - compare_renderer_vs_mergetarget( - test_info, - in_fmt, - CUSTOM_LAYOUT_DIR.joinpath(f"{out_fmt}.txt"), - is_comparetest=True, - ) - - -@pytest.mark.parametrize("out_fmt", CUSTOM_LS_TO_TEST) -@pytest.mark.parametrize("in_fmt", CUSTOM_LS_TO_TEST) -def test_custom_ls_input_output(test_info, in_fmt, out_fmt): - compare_renderer_vs_mergetarget( - test_info, - CUSTOM_LAYOUT_DIR.joinpath(f"{in_fmt}.txt"), - CUSTOM_LAYOUT_DIR.joinpath(f"{out_fmt}.txt"), - is_comparetest=True, - ) - - -@pytest.mark.parametrize("out_fmt", OUTPUT_FORMATS_BINAURAL) -@pytest.mark.parametrize("in_layout", CUSTOM_LS_TO_TEST) -@pytest.mark.parametrize("framing_5ms", FRAMING_5MS_TO_TEST) -def test_custom_ls_input_binaural(test_info, in_layout, out_fmt, framing_5ms): - compare_renderer_vs_mergetarget( - test_info, - CUSTOM_LAYOUT_DIR.joinpath(f"{in_layout}.txt"), - out_fmt, - framing_5ms=(framing_5ms == "5ms"), - is_comparetest=True, - ) - - -@pytest.mark.parametrize("trj_file", HR_TRAJECTORIES_TO_TEST) -@pytest.mark.parametrize("out_fmt", OUTPUT_FORMATS_BINAURAL) -@pytest.mark.parametrize("in_layout", CUSTOM_LS_TO_TEST) -@pytest.mark.parametrize("framing_5ms", FRAMING_5MS_TO_TEST) -def test_custom_ls_input_binaural_headrotation( - test_info, in_layout, out_fmt, trj_file, framing_5ms -): - compare_renderer_vs_mergetarget( - test_info, - CUSTOM_LAYOUT_DIR.joinpath(f"{in_layout}.txt"), - out_fmt, - trj_file=HR_TRAJECTORY_DIR.joinpath(f"{trj_file}.csv"), - framing_5ms=(framing_5ms == "5ms"), - is_comparetest=True, - ) - - -""" Metadata / scene description input """ - - -@pytest.mark.parametrize("out_fmt", OUTPUT_FORMATS) -@pytest.mark.parametrize("in_fmt", METADATA_SCENES_TO_TEST) -@pytest.mark.parametrize("framing_5ms", FRAMING_5MS_TO_TEST) -def test_metadata(test_info, in_fmt, out_fmt, framing_5ms): - compare_renderer_vs_mergetarget( - test_info, - "META", - out_fmt, - metadata_input=TEST_VECTOR_DIR.joinpath(f"{in_fmt}.txt"), - framing_5ms=(framing_5ms == "5ms"), - is_comparetest=True, - ) - - -""" non diegetic pan """ - - -# @pytest.mark.parametrize("out_fmt", ["STEREO"]) -# @pytest.mark.parametrize("in_fmt", ["MONO"]) -# @pytest.mark.parametrize("non_diegetic_pan", ["0", "-0.2", "0.5", "1", "-1"]) -# def test_non_diegetic_pan_static(test_info, in_fmt, out_fmt, non_diegetic_pan): -# compare_renderer_vs_mergetarget( -# test_info, in_fmt, out_fmt, non_diegetic_pan=non_diegetic_pan) - - -# @pytest.mark.parametrize("out_fmt", ["STEREO"]) -# @pytest.mark.parametrize("in_fmt", ["ISM1"]) -# @pytest.mark.parametrize("non_diegetic_pan", ["0", "-0.2", "0.5", "1", "-1"]) -# def test_non_diegetic_pan_ism_static(test_info, in_fmt, out_fmt, non_diegetic_pan): -# compare_renderer_vs_mergetarget( -# test_info, in_fmt, out_fmt, non_diegetic_pan=non_diegetic_pan) diff --git a/tests/renderer/utils.py b/tests/renderer/utils.py index 80b4202ef2..ad882039d5 100644 --- a/tests/renderer/utils.py +++ b/tests/renderer/utils.py @@ -30,14 +30,11 @@ the United Nations Convention on Contracts on the International Sales of Goods. """ -import filecmp import logging import os import subprocess as sp import sys -from pathlib import Path -from tempfile import TemporaryDirectory -from typing import Dict, Optional, Tuple +from typing import Dict, Optional import numpy as np import pytest @@ -96,6 +93,7 @@ def check_BE( def run_renderer( + test_info, in_fmt: str, out_fmt: str, metadata_input: Optional[str] = None, @@ -107,13 +105,10 @@ def run_renderer( refvec_file: Optional[str] = None, refveclev_file: Optional[str] = None, config_file: Optional[str] = None, - output_path_base: str = OUTPUT_PATH_CUT, binary_suffix: str = "", - is_comparetest: Optional[bool] = False, framing_5ms: Optional[bool] = False, - test_case_name: Optional[str] = None, ) -> str: - """CuT creation with standalone renderer""" + # prepare arguments and filepaths if trj_file is not None: trj_name = f"_{trj_file.stem}" else: @@ -149,7 +144,13 @@ def run_renderer( else: out_name = out_fmt - if is_comparetest: + if test_info.config.option.create_ref: + output_path_base = OUTPUT_PATH_REF + else: + output_path_base = OUTPUT_PATH_CUT + + # if in REF or CUT creation mode use the comparetestv + if test_info.config.option.create_ref or test_info.config.option.create_cut: FORMAT_TO_FILE = FORMAT_TO_FILE_COMPARETEST else: FORMAT_TO_FILE = FORMAT_TO_FILE_SMOKETEST @@ -164,11 +165,9 @@ def run_renderer( in_file = FORMAT_TO_FILE[in_fmt] in_name = in_fmt - out_file = str( - output_path_base.joinpath( - f"{in_name}_to_{out_name}{trj_name}{non_diegetic_pan}{refrot_name}{refvec_name}{refveclev_name}{config_name}{framing_name}{name_extension}.wav" - ) - ) + out_file_stem = f"{in_name}_to_{out_name}{trj_name}{non_diegetic_pan}{refrot_name}{refvec_name}{refveclev_name}{config_name}{framing_name}{name_extension}.wav" + + out_file = str(output_path_base.joinpath(out_file_stem)) cmd = RENDERER_CMD[:] cmd[2] = str(in_file) @@ -176,6 +175,8 @@ def run_renderer( cmd[6] = str(out_file) cmd[8] = str(out_fmt) + if test_info.config.option.create_ref: + cmd[0] += "_ref" cmd[0] += binary_suffix if in_meta_files is not None: @@ -206,77 +207,47 @@ def run_renderer( # Set env variables for UBSAN env = os.environ.copy() - if test_case_name and "UBSAN_OPTIONS" in env.keys(): + if test_info.node.name and "UBSAN_OPTIONS" in env.keys(): env["UBSAN_OPTIONS"] = ( - env["UBSAN_OPTIONS"] + f",log_path=usan_log_{test_case_name}" + env["UBSAN_OPTIONS"] + f",log_path=usan_log_{test_info.node.name}" ) + # run the renderer run_cmd(cmd, env) - return out_file + if test_info.config.option.create_cut: + # CUT creation mode will run a comparison with REF + out_file_ref = str(OUTPUT_PATH_REF.joinpath(out_file_stem)) + try: + ref, ref_fs = pyaudio3dtools.audiofile.readfile(out_file_ref) + except FileNotFoundError: + pytest.fail( + f"Reference vector not found! Ensure they were created with the --create_ref argument.\n{out_file_ref}" + ) -def compare_renderer_vs_mergetarget(test_info, in_fmt, out_fmt, **kwargs): - out_file_ref = run_renderer( - in_fmt, - out_fmt, - test_case_name=test_info.node.name, - binary_suffix=BIN_SUFFIX_MERGETARGET, - output_path_base=OUTPUT_PATH_REF, - **kwargs, - ) - ref, ref_fs = pyaudio3dtools.audiofile.readfile(out_file_ref) - out_file_cut = run_renderer( - in_fmt, out_fmt, test_case_name=test_info.node.name, **kwargs - ) - cut, cut_fs = pyaudio3dtools.audiofile.readfile(out_file_cut) - check_BE(test_info, ref, ref_fs, cut, cut_fs) + cut, cut_fs = pyaudio3dtools.audiofile.readfile(out_file) + check_BE(test_info, ref, ref_fs, cut, cut_fs) -def compare_renderer_vs_pyscripts(test_info, in_fmt, out_fmt, **kwargs): - ref, ref_fs = run_pyscripts(in_fmt, out_fmt, **kwargs) - out_file_cut = run_renderer( - in_fmt, out_fmt, test_case_name=test_info.node.name, **kwargs - ) - cut, cut_fs = pyaudio3dtools.audiofile.readfile(out_file_cut) - check_BE(test_info, ref, ref_fs, cut, cut_fs) + return out_file def compare_renderer_args( test_info, in_fmt, out_fmt, ref_kwargs: Dict, cut_kwargs: Dict ): out_file_ref = run_renderer( - in_fmt, out_fmt, test_case_name=test_info.node.name, **ref_kwargs - ) - ref, ref_fs = pyaudio3dtools.audiofile.readfile(out_file_ref) - out_file_cut = run_renderer( - in_fmt, out_fmt, test_case_name=test_info.node.name, **cut_kwargs - ) - cut, cut_fs = pyaudio3dtools.audiofile.readfile(out_file_cut) - check_BE(test_info, ref, ref_fs, cut, cut_fs) - - -def compare_renderer_vs_mergetarget_plus_meta(test_info, in_fmt, out_fmt, metadata_input, **kwargs): - out_file_ref = run_renderer( + test_info, in_fmt, out_fmt, - metadata_input=metadata_input, - test_case_name=test_info.node.name, - binary_suffix=BIN_SUFFIX_MERGETARGET, - output_path_base=OUTPUT_PATH_REF, - **kwargs, + **ref_kwargs, ) ref, ref_fs = pyaudio3dtools.audiofile.readfile(out_file_ref) out_file_cut = run_renderer( + test_info, in_fmt, out_fmt, - metadata_input=metadata_input, - test_case_name=test_info.node.name, - **kwargs + **cut_kwargs, ) cut, cut_fs = pyaudio3dtools.audiofile.readfile(out_file_cut) - meta_file_ref = out_file_ref + ".met" - meta_file_cut = out_file_cut + ".met" check_BE(test_info, ref, ref_fs, cut, cut_fs) - if not filecmp.cmp(meta_file_cut, meta_file_ref): - pytest.fail("Metadata file differs from reference") -- GitLab From f56f640b4a54e7b647214f06b1f9dabfabf4f53e Mon Sep 17 00:00:00 2001 From: Archit Tamarapu Date: Fri, 27 Oct 2023 13:36:06 +0200 Subject: [PATCH 02/13] - restore MASA tests that were present only in BE-comparetest - fix MONO and STEREO not being tested in smoketests (were only tested for comparetest) --- tests/renderer/test_renderer.py | 56 +++++++++++++++++++++++++++++---- 1 file changed, 50 insertions(+), 6 deletions(-) diff --git a/tests/renderer/test_renderer.py b/tests/renderer/test_renderer.py index 4ea7a289bb..ade00ecbf9 100644 --- a/tests/renderer/test_renderer.py +++ b/tests/renderer/test_renderer.py @@ -43,7 +43,7 @@ from .utils import * """ Ambisonics """ -@pytest.mark.parametrize("out_fmt", OUTPUT_FORMATS[2:]) +@pytest.mark.parametrize("out_fmt", OUTPUT_FORMATS) @pytest.mark.parametrize("in_fmt", INPUT_FORMATS_AMBI) @pytest.mark.parametrize("framing_5ms", FRAMING_5MS_TO_TEST) def test_ambisonics(test_info, in_fmt, out_fmt, framing_5ms): @@ -86,7 +86,7 @@ def test_ambisonics_binaural_headrotation( """ Multichannel """ -@pytest.mark.parametrize("out_fmt", OUTPUT_FORMATS[2:]) +@pytest.mark.parametrize("out_fmt", OUTPUT_FORMATS) @pytest.mark.parametrize("in_fmt", INPUT_FORMATS_MC) @pytest.mark.parametrize("framing_5ms", FRAMING_5MS_TO_TEST) def test_multichannel(test_info, in_fmt, out_fmt, framing_5ms): @@ -135,7 +135,7 @@ def test_multichannel_binaural_headrotation( """ ISM """ -@pytest.mark.parametrize("out_fmt", OUTPUT_FORMATS[2:]) +@pytest.mark.parametrize("out_fmt", OUTPUT_FORMATS) @pytest.mark.parametrize("in_fmt", INPUT_FORMATS_ISM) @pytest.mark.parametrize("framing_5ms", FRAMING_5MS_TO_TEST) def test_ism(test_info, in_fmt, out_fmt, framing_5ms): @@ -202,10 +202,54 @@ def test_masa(test_info, in_fmt, out_fmt, framing_5ms): ) +@pytest.mark.parametrize("out_fmt", OUTPUT_FORMATS_BINAURAL) +@pytest.mark.parametrize("in_fmt", INPUT_FORMATS_MASA) +@pytest.mark.parametrize("framing_5ms", FRAMING_5MS_TO_TEST) +def test_masa_binaural_static(test_info, in_fmt, out_fmt, framing_5ms): + if out_fmt in ["BINAURAL_ROOM_IR", "BINAURAL_ROOM_REVERB"]: + pytest.skip("Skipping binaural room outputs for MASA as unimplemented.") + + run_renderer( + test_info, + in_fmt, + out_fmt, + in_meta_files=FORMAT_TO_METADATA_FILES[in_fmt], + framing_5ms=(framing_5ms == "5ms"), + ) + + +@pytest.mark.parametrize("trj_file", HR_TRAJECTORIES_TO_TEST) +@pytest.mark.parametrize("out_fmt", OUTPUT_FORMATS_BINAURAL) +@pytest.mark.parametrize("in_fmt", INPUT_FORMATS_MASA) +@pytest.mark.parametrize("framing_5ms", FRAMING_5MS_TO_TEST) +def test_masa_binaural_headrotation(test_info, in_fmt, out_fmt, trj_file, framing_5ms): + if out_fmt in ["BINAURAL_ROOM_IR", "BINAURAL_ROOM_REVERB"]: + pytest.skip("Skipping binaural room outputs for MASA as unimplemented.") + + run_renderer( + test_info, + in_fmt, + out_fmt, + trj_file=HR_TRAJECTORY_DIR.joinpath(f"{trj_file}.csv"), + in_meta_files=FORMAT_TO_METADATA_FILES[in_fmt], + framing_5ms=(framing_5ms == "5ms"), + ) + + +@pytest.mark.parametrize("in_fmt", METADATA_SCENES_TO_TEST_MASA_PREREND) +def test_masa_prerend(test_info, in_fmt): + run_renderer( + test_info, + "META", + "MASA2", + metadata_input=TEST_VECTOR_DIR.joinpath(f"{in_fmt}.txt"), + ) + + """ Custom loudspeaker layouts """ -@pytest.mark.parametrize("out_fmt", OUTPUT_FORMATS[2:]) +@pytest.mark.parametrize("out_fmt", OUTPUT_FORMATS) @pytest.mark.parametrize("in_layout", CUSTOM_LS_TO_TEST) @pytest.mark.parametrize("framing_5ms", FRAMING_5MS_TO_TEST) def test_custom_ls_input(test_info, in_layout, out_fmt, framing_5ms): @@ -218,7 +262,7 @@ def test_custom_ls_input(test_info, in_layout, out_fmt, framing_5ms): @pytest.mark.parametrize("out_fmt", CUSTOM_LS_TO_TEST) -@pytest.mark.parametrize("in_fmt", OUTPUT_FORMATS[2:]) +@pytest.mark.parametrize("in_fmt", OUTPUT_FORMATS) def test_custom_ls_output(test_info, in_fmt, out_fmt): run_renderer( test_info, @@ -268,7 +312,7 @@ def test_custom_ls_input_binaural_headrotation( """ Metadata / scene description input """ -@pytest.mark.parametrize("out_fmt", OUTPUT_FORMATS[2:]) +@pytest.mark.parametrize("out_fmt", OUTPUT_FORMATS) @pytest.mark.parametrize("in_fmt", METADATA_SCENES_TO_TEST) @pytest.mark.parametrize("framing_5ms", FRAMING_5MS_TO_TEST) def test_metadata(test_info, in_fmt, out_fmt, framing_5ms): -- GitLab From af46627bea5d15ad008165cb29fa7853564f95fc Mon Sep 17 00:00:00 2001 From: Archit Tamarapu Date: Fri, 27 Oct 2023 14:30:02 +0200 Subject: [PATCH 03/13] update smoke test vectors and add new ones for OSBA --- tests/renderer/README.md | 12 ++++---- tests/renderer/constants.py | 28 +++++++++++++------ .../data/spectral_test_13ch_48kHz.wav | 3 ++ .../data/spectral_test_17ch_48kHz.wav | 3 ++ .../data/spectral_test_18ch_48kHz.wav | 3 ++ .../data/spectral_test_19ch_48kHz.wav | 3 ++ .../data/spectral_test_20ch_48kHz.wav | 3 ++ .../renderer/data/spectral_test_7ch_48kHz.wav | 3 ++ tests/renderer/utils.py | 10 +++---- 9 files changed, 50 insertions(+), 18 deletions(-) create mode 100644 tests/renderer/data/spectral_test_13ch_48kHz.wav create mode 100644 tests/renderer/data/spectral_test_17ch_48kHz.wav create mode 100644 tests/renderer/data/spectral_test_18ch_48kHz.wav create mode 100644 tests/renderer/data/spectral_test_19ch_48kHz.wav create mode 100644 tests/renderer/data/spectral_test_20ch_48kHz.wav create mode 100644 tests/renderer/data/spectral_test_7ch_48kHz.wav diff --git a/tests/renderer/README.md b/tests/renderer/README.md index fe0223a11b..713821c5d3 100644 --- a/tests/renderer/README.md +++ b/tests/renderer/README.md @@ -10,14 +10,16 @@ See also the [contribution page](https://forge.3gpp.org/rep/ivas-codec-pc/ivas-c ## Smoke test: -`python3 -m pytest -q -n auto tests/renderer/test_renderer.py` +```bash +python3 -m pytest -q -n auto tests/renderer/test_renderer.py +``` ## Comparison test: -`python3 -m pytest -q -n auto tests/renderer/test_renderer.py --create_ref` -(note: requires IVAS_rend_ref in root!) - -`python3 -m pytest -q -n auto tests/renderer/test_renderer.py --create_cut` +```bash +python3 -m pytest -q -n auto tests/renderer/test_renderer.py --create_ref # requires IVAS_rend_ref in root! +python3 -m pytest -q -n auto tests/renderer/test_renderer.py --create_cut +``` ### Important flags (see [pytest docs](https://docs.pytest.org/en/7.2.x/) for more information): diff --git a/tests/renderer/constants.py b/tests/renderer/constants.py index 345006e9f7..39005d7a86 100644 --- a/tests/renderer/constants.py +++ b/tests/renderer/constants.py @@ -74,13 +74,19 @@ NCHAN_TO_FILE = { 4: TEST_VECTOR_DIR.joinpath("spectral_test_4ch_48kHz.wav"), 5: TEST_VECTOR_DIR.joinpath("spectral_test_5ch_48kHz.wav"), 6: TEST_VECTOR_DIR.joinpath("spectral_test_6ch_48kHz.wav"), + 7: TEST_VECTOR_DIR.joinpath("spectral_test_7ch_48kHz.wav"), 8: TEST_VECTOR_DIR.joinpath("spectral_test_8ch_48kHz.wav"), 9: TEST_VECTOR_DIR.joinpath("spectral_test_9ch_48kHz.wav"), 10: TEST_VECTOR_DIR.joinpath("spectral_test_10ch_48kHz.wav"), 11: TEST_VECTOR_DIR.joinpath("spectral_test_11ch_48kHz.wav"), 12: TEST_VECTOR_DIR.joinpath("spectral_test_12ch_48kHz.wav"), + 13: TEST_VECTOR_DIR.joinpath("spectral_test_13ch_48kHz.wav"), 15: TEST_VECTOR_DIR.joinpath("spectral_test_15ch_48kHz.wav"), 16: TEST_VECTOR_DIR.joinpath("spectral_test_16ch_48kHz.wav"), + 17: TEST_VECTOR_DIR.joinpath("spectral_test_17ch_48kHz.wav"), + 18: TEST_VECTOR_DIR.joinpath("spectral_test_18ch_48kHz.wav"), + 19: TEST_VECTOR_DIR.joinpath("spectral_test_19ch_48kHz.wav"), + 20: TEST_VECTOR_DIR.joinpath("spectral_test_20ch_48kHz.wav"), } FORMAT_TO_FILE_SMOKETEST = { @@ -99,14 +105,20 @@ FORMAT_TO_FILE_SMOKETEST = { "ISM3": NCHAN_TO_FILE[3], "ISM4": NCHAN_TO_FILE[4], "NDP_ISM4": NCHAN_TO_FILE[4], - # "ISM1": TEST_VECTOR_DIR.joinpath("spectral_test_ism1.txt"), - # "ISM2": TEST_VECTOR_DIR.joinpath("spectral_test_ism2.txt"), - # "ISM3": TEST_VECTOR_DIR.joinpath("spectral_test_ism3.txt"), - # "ISM4": TEST_VECTOR_DIR.joinpath("spectral_test_ism4.txt"), - # "MASA1": NCHAN_TO_FILE[1], - # "MASA2": NCHAN_TO_FILE[2], - "MASA1": TESTV_DIR.joinpath("stv1MASA1TC48c.wav"), - "MASA2": TESTV_DIR.joinpath("stv2MASA2TC48c.wav"), + "MASA1": NCHAN_TO_FILE[1], + "MASA2": NCHAN_TO_FILE[2], + "OSBA_1_1": NCHAN_TO_FILE[5], + "OSBA_2_1": NCHAN_TO_FILE[6], + "OSBA_3_1": NCHAN_TO_FILE[7], + "OSBA_4_1": NCHAN_TO_FILE[8], + "OSBA_1_2": NCHAN_TO_FILE[10], + "OSBA_2_2": NCHAN_TO_FILE[11], + "OSBA_3_2": NCHAN_TO_FILE[12], + "OSBA_4_2": NCHAN_TO_FILE[13], + "OSBA_1_3": NCHAN_TO_FILE[17], + "OSBA_2_3": NCHAN_TO_FILE[18], + "OSBA_3_3": NCHAN_TO_FILE[19], + "OSBA_4_3": NCHAN_TO_FILE[20], "META": TEST_VECTOR_DIR.joinpath("mixed_scene.txt"), "16ch_8+4+4": NCHAN_TO_FILE[16], "4d4": NCHAN_TO_FILE[8], diff --git a/tests/renderer/data/spectral_test_13ch_48kHz.wav b/tests/renderer/data/spectral_test_13ch_48kHz.wav new file mode 100644 index 0000000000..31719ea091 --- /dev/null +++ b/tests/renderer/data/spectral_test_13ch_48kHz.wav @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0bad70578fa0882caf7fc09718005d188bec65419937df3c93a2b6d27eaca09c +size 1248044 diff --git a/tests/renderer/data/spectral_test_17ch_48kHz.wav b/tests/renderer/data/spectral_test_17ch_48kHz.wav new file mode 100644 index 0000000000..fea9dfc9e3 --- /dev/null +++ b/tests/renderer/data/spectral_test_17ch_48kHz.wav @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cfc3d214fc0c3b480a4e69ee4bc20632dfa7e14d18c739cb441c4407e0aa26da +size 1632044 diff --git a/tests/renderer/data/spectral_test_18ch_48kHz.wav b/tests/renderer/data/spectral_test_18ch_48kHz.wav new file mode 100644 index 0000000000..ef088d567c --- /dev/null +++ b/tests/renderer/data/spectral_test_18ch_48kHz.wav @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9461a003e8a75cc389842b704f5523f8ad61125c2b13aa8bdbf99d5b4ab357ae +size 1728044 diff --git a/tests/renderer/data/spectral_test_19ch_48kHz.wav b/tests/renderer/data/spectral_test_19ch_48kHz.wav new file mode 100644 index 0000000000..757617df79 --- /dev/null +++ b/tests/renderer/data/spectral_test_19ch_48kHz.wav @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:dccd5e3b49ef78f1887a32fff3946cc45e7d66f82785f49555b217288a13524d +size 1824044 diff --git a/tests/renderer/data/spectral_test_20ch_48kHz.wav b/tests/renderer/data/spectral_test_20ch_48kHz.wav new file mode 100644 index 0000000000..30e986372c --- /dev/null +++ b/tests/renderer/data/spectral_test_20ch_48kHz.wav @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:39ce9d81bdd6c15196fd6a7e43ae0012a014245f10e66590ebe375ccd0aff6cd +size 1920044 diff --git a/tests/renderer/data/spectral_test_7ch_48kHz.wav b/tests/renderer/data/spectral_test_7ch_48kHz.wav new file mode 100644 index 0000000000..8edc91421e --- /dev/null +++ b/tests/renderer/data/spectral_test_7ch_48kHz.wav @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4eb0a56b818be22c7092006af364f2b58a0513330d60c3e0e2977a5f542f1e47 +size 672044 diff --git a/tests/renderer/utils.py b/tests/renderer/utils.py index ad882039d5..99d3f669c7 100644 --- a/tests/renderer/utils.py +++ b/tests/renderer/utils.py @@ -43,7 +43,7 @@ from .compare_audio import compare_audio_arrays from .constants import * sys.path.append(SCRIPTS_DIR) -import pyaudio3dtools +from pyaudio3dtools.audiofile import readfile # fixture returns test information, enabling per-testcase SNR @@ -220,13 +220,13 @@ def run_renderer( out_file_ref = str(OUTPUT_PATH_REF.joinpath(out_file_stem)) try: - ref, ref_fs = pyaudio3dtools.audiofile.readfile(out_file_ref) + ref, ref_fs = readfile(out_file_ref) except FileNotFoundError: pytest.fail( f"Reference vector not found! Ensure they were created with the --create_ref argument.\n{out_file_ref}" ) - cut, cut_fs = pyaudio3dtools.audiofile.readfile(out_file) + cut, cut_fs = readfile(out_file) check_BE(test_info, ref, ref_fs, cut, cut_fs) @@ -242,12 +242,12 @@ def compare_renderer_args( out_fmt, **ref_kwargs, ) - ref, ref_fs = pyaudio3dtools.audiofile.readfile(out_file_ref) + ref, ref_fs = readfile(out_file_ref) out_file_cut = run_renderer( test_info, in_fmt, out_fmt, **cut_kwargs, ) - cut, cut_fs = pyaudio3dtools.audiofile.readfile(out_file_cut) + cut, cut_fs = readfile(out_file_cut) check_BE(test_info, ref, ref_fs, cut, cut_fs) -- GitLab From 28cd7f9007fa8260be8e6a627d80ba30e8a5e596 Mon Sep 17 00:00:00 2001 From: Archit Tamarapu Date: Fri, 27 Oct 2023 14:30:39 +0200 Subject: [PATCH 04/13] update split rendering tests as well --- .gitlab-ci.yml | 10 +- tests/split_rendering/README.md | 5 +- tests/split_rendering/conftest.py | 44 +++ tests/split_rendering/test_split_rendering.py | 45 ++- .../test_split_rendering_be_comparison.py | 304 ------------------ tests/split_rendering/utils.py | 79 ++++- 6 files changed, 155 insertions(+), 332 deletions(-) create mode 100644 tests/split_rendering/conftest.py delete mode 100644 tests/split_rendering/test_split_rendering_be_comparison.py diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 816bbced0c..f2c3369781 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -727,6 +727,12 @@ split-rendering-pytest-on-merge-request: - mv IVAS_dec IVAS_dec_ref - mv IVAS_rend IVAS_rend_ref + ### If ref_using_main is not set, checkout the source branch to use scripts and input from there + - if [ $ref_using_main == 0 ]; then git restore lib_com/options.h; fi # Revert changes back before checking out another branch to avoid conflicts + - if [ $ref_using_main == 0 ]; then git checkout $source_branch_commit_sha; fi + - exit_code=0 + - python3 -m pytest -q --log-level ERROR -n auto -rA --html=report.html --self-contained-html --junit-xml=report-junit.xml tests/split_rendering/test_split_rendering_be_comparison.py --create_ref || exit_code=$? + # back to source branch - git restore lib_com/options.h # Revert changes back before checking out another branch to avoid conflicts - git checkout $source_branch_commit_sha @@ -736,11 +742,11 @@ split-rendering-pytest-on-merge-request: ### Run test using scripts and input from main - if [ $ref_using_main == 1 ]; then git restore lib_com/options.h; fi # Revert changes back before checking out another branch to avoid conflicts - - if [ $ref_using_main == 1 ]; then git checkout $target_commit; fi + - if [ $ref_using_main == 1 ]; then git checkout $source_branch_commit_sha; fi # run test - exit_code=0 - - python3 -m pytest -q --log-level ERROR -n auto -rA --html=report.html --self-contained-html --junit-xml=report-junit.xml tests/split_rendering/test_split_rendering_be_comparison.py || exit_code=$? + - python3 -m pytest -q --log-level ERROR -n auto -rA --html=report.html --self-contained-html --junit-xml=report-junit.xml tests/split_rendering/test_split_rendering_be_comparison.py --create_cut || exit_code=$? - zero_errors=$(cat report-junit.xml | grep -c 'errors="0"') || true - *merge-request-comparison-check diff --git a/tests/split_rendering/README.md b/tests/split_rendering/README.md index d50bc56496..0ff6b8dd0c 100644 --- a/tests/split_rendering/README.md +++ b/tests/split_rendering/README.md @@ -63,7 +63,7 @@ ivas-codec ├── tests | ├── split_rendering | | ├── test_split_rendering.py -> split rendering smoke tests -| | ├── test_split_rendering_be_comparison.py -> split rendering BE comparison tests +| | ├── conftest.py -> test configuration | | ├── constants.py -> split rendering constants | | ├── utils.py -> split rendering functions | | └── renderer_configs -> directory of render config files @@ -82,7 +82,8 @@ python3 -m pytest -n auto -rA tests/split_rendering/test_split_rendering.py ### BE Comparison Tests ```bash -python3 -m pytest -n auto -rA tests/split_rendering/test_split_rendering_be_comparison.py +python3 -m pytest -n auto -rA tests/split_rendering/test_split_rendering.py --create_ref # requires _ref binaries in root! +python3 -m pytest -n auto -rA tests/split_rendering/test_split_rendering.py --create_cut ``` ## Useful pytest Options diff --git a/tests/split_rendering/conftest.py b/tests/split_rendering/conftest.py new file mode 100644 index 0000000000..4668d742db --- /dev/null +++ b/tests/split_rendering/conftest.py @@ -0,0 +1,44 @@ +#!/usr/bin/env python3 + +""" + (C) 2022-2023 IVAS codec Public Collaboration with portions copyright Dolby International AB, Ericsson AB, + Fraunhofer-Gesellschaft zur Foerderung der angewandten Forschung e.V., Huawei Technologies Co. LTD., + Koninklijke Philips N.V., Nippon Telegraph and Telephone Corporation, Nokia Technologies Oy, Orange, + Panasonic Holdings Corporation, Qualcomm Technologies, Inc., VoiceAge Corporation, and other + contributors to this repository. All Rights Reserved. + + This software is protected by copyright law and by international treaties. + The IVAS codec Public Collaboration consisting of Dolby International AB, Ericsson AB, + Fraunhofer-Gesellschaft zur Foerderung der angewandten Forschung e.V., Huawei Technologies Co. LTD., + Koninklijke Philips N.V., Nippon Telegraph and Telephone Corporation, Nokia Technologies Oy, Orange, + Panasonic Holdings Corporation, Qualcomm Technologies, Inc., VoiceAge Corporation, and other + contributors to this repository retain full ownership rights in their respective contributions in + the software. This notice grants no license of any kind, including but not limited to patent + license, nor is any license granted by implication, estoppel or otherwise. + + Contributors are required to enter into the IVAS codec Public Collaboration agreement before making + contributions. + + This software is provided "AS IS", without any express or implied warranties. The software is in the + development stage. It is intended exclusively for experts who have experience with such software and + solely for the purpose of inspection. All implied warranties of non-infringement, merchantability + and fitness for a particular purpose are hereby disclaimed and excluded. + + Any dispute, controversy or claim arising under or in relation to providing this software shall be + submitted to and settled by the final, binding jurisdiction of the courts of Munich, Germany in + accordance with the laws of the Federal Republic of Germany excluding its conflict of law rules and + the United Nations Convention on Contracts on the International Sales of Goods. +""" + + +def pytest_addoption(parser): + parser.addoption( + "--create_ref", + action="store_true", + default=False, + ) + parser.addoption( + "--create_cut", + action="store_true", + default=False, + ) diff --git a/tests/split_rendering/test_split_rendering.py b/tests/split_rendering/test_split_rendering.py index 51156ec7f8..ccb8b86ea3 100644 --- a/tests/split_rendering/test_split_rendering.py +++ b/tests/split_rendering/test_split_rendering.py @@ -82,12 +82,12 @@ def test_ambisonics_full_chain_split( pre_trajectory = post_trajectory.with_stem(f"{post_trajectory.stem}_delayed") run_full_chain_split_rendering( + test_info, in_fmt=in_fmt, bitrate=bitrate, render_config=RENDER_CFG_DIR.joinpath(f"{render_config}.txt"), pre_trajectory=pre_trajectory, post_trajectory=post_trajectory, - output_path_base=OUTPUT_PATH_CUT, ) @@ -101,11 +101,11 @@ def test_ambisonics_external_split(test_info, in_fmt, render_config, trajectory) pre_trajectory = post_trajectory.with_stem(f"{post_trajectory.stem}_delayed") run_external_split_rendering( + test_info, in_fmt=in_fmt, render_config=RENDER_CFG_DIR.joinpath(f"{render_config}.txt"), pre_trajectory=pre_trajectory, post_trajectory=post_trajectory, - output_path_base=OUTPUT_PATH_CUT, ) @@ -125,12 +125,12 @@ def test_multichannel_full_chain_split( pre_trajectory = post_trajectory.with_stem(f"{post_trajectory.stem}_delayed") run_full_chain_split_rendering( + test_info, in_fmt=in_fmt, bitrate=bitrate, render_config=RENDER_CFG_DIR.joinpath(f"{render_config}.txt"), pre_trajectory=pre_trajectory, post_trajectory=post_trajectory, - output_path_base=OUTPUT_PATH_CUT, ) @@ -144,11 +144,11 @@ def test_multichannel_external_split(test_info, in_fmt, render_config, trajector pre_trajectory = post_trajectory.with_stem(f"{post_trajectory.stem}_delayed") run_external_split_rendering( + test_info, in_fmt=in_fmt, render_config=RENDER_CFG_DIR.joinpath(f"{render_config}.txt"), pre_trajectory=pre_trajectory, post_trajectory=post_trajectory, - output_path_base=OUTPUT_PATH_CUT, ) @@ -166,12 +166,12 @@ def test_ism_full_chain_split(test_info, in_fmt, bitrate, render_config, traject pre_trajectory = post_trajectory.with_stem(f"{post_trajectory.stem}_delayed") run_full_chain_split_rendering( + test_info, in_fmt=in_fmt, bitrate=bitrate, render_config=RENDER_CFG_DIR.joinpath(f"{render_config}.txt"), pre_trajectory=pre_trajectory, post_trajectory=post_trajectory, - output_path_base=OUTPUT_PATH_CUT, ) @@ -185,11 +185,11 @@ def test_ism_external_split(test_info, in_fmt, render_config, trajectory): pre_trajectory = post_trajectory.with_stem(f"{post_trajectory.stem}_delayed") run_external_split_rendering( + test_info, in_fmt=in_fmt, render_config=RENDER_CFG_DIR.joinpath(f"{render_config}.txt"), pre_trajectory=pre_trajectory, post_trajectory=post_trajectory, - output_path_base=OUTPUT_PATH_CUT, ) @@ -207,12 +207,12 @@ def test_masa_full_chain_split(test_info, in_fmt, bitrate, render_config, trajec pre_trajectory = post_trajectory.with_stem(f"{post_trajectory.stem}_delayed") run_full_chain_split_rendering( + test_info, in_fmt=in_fmt, bitrate=bitrate, render_config=RENDER_CFG_DIR.joinpath(f"{render_config}.txt"), pre_trajectory=pre_trajectory, post_trajectory=post_trajectory, - output_path_base=OUTPUT_PATH_CUT, ) @@ -226,11 +226,34 @@ def test_masa_external_split(test_info, in_fmt, render_config, trajectory): pre_trajectory = post_trajectory.with_stem(f"{post_trajectory.stem}_delayed") run_external_split_rendering( + test_info, in_fmt=in_fmt, render_config=RENDER_CFG_DIR.joinpath(f"{render_config}.txt"), pre_trajectory=pre_trajectory, post_trajectory=post_trajectory, - output_path_base=OUTPUT_PATH_CUT, + ) + + +""" OSBA """ + + +@pytest.mark.parametrize("trajectory", SPLIT_REND_HR_TRAJECTORIES_TO_TEST) +@pytest.mark.parametrize("render_config", RENDERER_CONFIGS_TO_TEST_OSBA) +@pytest.mark.parametrize("bitrate", IVAS_BITRATES_OSBA) +@pytest.mark.parametrize("in_fmt", INPUT_FORMATS_OSBA_SPLIT_REND) +def test_osba_full_chain_split(test_info, in_fmt, bitrate, render_config, trajectory): + check_xfail(test_info, in_fmt, render_config, bitrate) + + post_trajectory = HR_TRAJECTORY_DIR.joinpath(f"{trajectory}.csv") + pre_trajectory = post_trajectory.with_stem(f"{post_trajectory.stem}_delayed") + + run_full_chain_split_rendering( + test_info, + in_fmt=in_fmt, + bitrate=bitrate, + render_config=RENDER_CFG_DIR.joinpath(f"{render_config}.txt"), + pre_trajectory=pre_trajectory, + post_trajectory=post_trajectory, ) @@ -248,11 +271,11 @@ def test_post_rend_plc(test_info, in_fmt, render_config, trajectory, error_patte pre_trajectory = post_trajectory.with_stem(f"{post_trajectory.stem}_delayed") run_external_split_rendering( + test_info, in_fmt=in_fmt, render_config=RENDER_CFG_DIR.joinpath(f"{render_config}.txt"), pre_trajectory=pre_trajectory, post_trajectory=post_trajectory, - output_path_base=OUTPUT_PATH_CUT, plc_error_pattern=ERROR_PATTERNS_DIR.joinpath(f"{error_pattern}.ep"), ) @@ -277,12 +300,12 @@ def test_full_chain_split_pcm(test_info, in_fmt, bitrate, render_config): pre_trajectory = post_trajectory.with_stem(f"{post_trajectory.stem}_delayed") run_full_chain_split_rendering( + test_info, in_fmt=in_fmt, bitrate=bitrate, render_config=RENDER_CFG_DIR.joinpath(f"{render_config}.txt"), pre_trajectory=pre_trajectory, post_trajectory=post_trajectory, - output_path_base=OUTPUT_PATH_CUT, renderer_fmt="BINAURAL_SPLIT_PCM", ) @@ -303,10 +326,10 @@ def test_external_split_pcm(test_info, in_fmt, render_config): pre_trajectory = post_trajectory.with_stem(f"{post_trajectory.stem}_delayed") run_external_split_rendering( + test_info, in_fmt=in_fmt, render_config=RENDER_CFG_DIR.joinpath(f"{render_config}.txt"), pre_trajectory=pre_trajectory, post_trajectory=post_trajectory, - output_path_base=OUTPUT_PATH_CUT, renderer_fmt="BINAURAL_SPLIT_PCM", ) diff --git a/tests/split_rendering/test_split_rendering_be_comparison.py b/tests/split_rendering/test_split_rendering_be_comparison.py deleted file mode 100644 index 957ed7f043..0000000000 --- a/tests/split_rendering/test_split_rendering_be_comparison.py +++ /dev/null @@ -1,304 +0,0 @@ -#!/usr/bin/env python3 - -""" - (C) 2022-2023 IVAS codec Public Collaboration with portions copyright Dolby International AB, Ericsson AB, - Fraunhofer-Gesellschaft zur Foerderung der angewandten Forschung e.V., Huawei Technologies Co. LTD., - Koninklijke Philips N.V., Nippon Telegraph and Telephone Corporation, Nokia Technologies Oy, Orange, - Panasonic Holdings Corporation, Qualcomm Technologies, Inc., VoiceAge Corporation, and other - contributors to this repository. All Rights Reserved. - - This software is protected by copyright law and by international treaties. - The IVAS codec Public Collaboration consisting of Dolby International AB, Ericsson AB, - Fraunhofer-Gesellschaft zur Foerderung der angewandten Forschung e.V., Huawei Technologies Co. LTD., - Koninklijke Philips N.V., Nippon Telegraph and Telephone Corporation, Nokia Technologies Oy, Orange, - Panasonic Holdings Corporation, Qualcomm Technologies, Inc., VoiceAge Corporation, and other - contributors to this repository retain full ownership rights in their respective contributions in - the software. This notice grants no license of any kind, including but not limited to patent - license, nor is any license granted by implication, estoppel or otherwise. - - Contributors are required to enter into the IVAS codec Public Collaboration agreement before making - contributions. - - This software is provided "AS IS", without any express or implied warranties. The software is in the - development stage. It is intended exclusively for experts who have experience with such software and - solely for the purpose of inspection. All implied warranties of non-infringement, merchantability - and fitness for a particular purpose are hereby disclaimed and excluded. - - Any dispute, controversy or claim arising under or in relation to providing this software shall be - submitted to and settled by the final, binding jurisdiction of the courts of Munich, Germany in - accordance with the laws of the Federal Republic of Germany excluding its conflict of law rules and - the United Nations Convention on Contracts on the International Sales of Goods. -""" - -import pytest - -from tests.split_rendering.test_split_rendering import check_xfail -from tests.split_rendering.utils import * - -""" Ambisonics """ - - -@pytest.mark.parametrize("trajectory", SPLIT_REND_HR_TRAJECTORIES_TO_TEST) -@pytest.mark.parametrize("render_config", RENDERER_CONFIGS_TO_TEST_AMBI) -@pytest.mark.parametrize("bitrate", IVAS_BITRATES_AMBI) -@pytest.mark.parametrize("in_fmt", INPUT_FORMATS_AMBI_SPLIT_REND) -def test_ambisonics_full_chain_split( - test_info, in_fmt, bitrate, render_config, trajectory -): - check_xfail(test_info, in_fmt, render_config, bitrate) - - post_trajectory = HR_TRAJECTORY_DIR.joinpath(f"{trajectory}.csv") - pre_trajectory = post_trajectory.with_stem(f"{post_trajectory.stem}_delayed") - - compare_full_chain_split_args( - test_info, - in_fmt=in_fmt, - bitrate=bitrate, - render_config=RENDER_CFG_DIR.joinpath(f"{render_config}.txt"), - pre_trajectory=pre_trajectory, - post_trajectory=post_trajectory, - ) - - -@pytest.mark.parametrize("trajectory", SPLIT_REND_HR_TRAJECTORIES_TO_TEST) -@pytest.mark.parametrize("render_config", RENDERER_CONFIGS_TO_TEST_AMBI) -@pytest.mark.parametrize("in_fmt", INPUT_FORMATS_AMBI_SPLIT_REND) -def test_ambisonics_external_split(test_info, in_fmt, render_config, trajectory): - check_xfail(test_info, in_fmt, render_config) - - post_trajectory = HR_TRAJECTORY_DIR.joinpath(f"{trajectory}.csv") - pre_trajectory = post_trajectory.with_stem(f"{post_trajectory.stem}_delayed") - - compare_external_split_args( - test_info, - in_fmt=in_fmt, - render_config=RENDER_CFG_DIR.joinpath(f"{render_config}.txt"), - pre_trajectory=pre_trajectory, - post_trajectory=post_trajectory, - ) - - -""" Multichannel """ - - -@pytest.mark.parametrize("trajectory", SPLIT_REND_HR_TRAJECTORIES_TO_TEST) -@pytest.mark.parametrize("render_config", RENDERER_CONFIGS_TO_TEST_MC) -@pytest.mark.parametrize("bitrate", IVAS_BITRATES_MC) -@pytest.mark.parametrize("in_fmt", INPUT_FORMATS_MC_SPLIT_REND) -def test_multichannel_full_chain_split( - test_info, in_fmt, bitrate, render_config, trajectory -): - check_xfail(test_info, in_fmt, render_config, bitrate) - - post_trajectory = HR_TRAJECTORY_DIR.joinpath(f"{trajectory}.csv") - pre_trajectory = post_trajectory.with_stem(f"{post_trajectory.stem}_delayed") - - compare_full_chain_split_args( - test_info, - in_fmt=in_fmt, - bitrate=bitrate, - render_config=RENDER_CFG_DIR.joinpath(f"{render_config}.txt"), - pre_trajectory=pre_trajectory, - post_trajectory=post_trajectory, - ) - - -@pytest.mark.parametrize("trajectory", SPLIT_REND_HR_TRAJECTORIES_TO_TEST) -@pytest.mark.parametrize("render_config", RENDERER_CONFIGS_TO_TEST_MC) -@pytest.mark.parametrize("in_fmt", INPUT_FORMATS_MC_SPLIT_REND) -def test_multichannel_external_split(test_info, in_fmt, render_config, trajectory): - check_xfail(test_info, in_fmt, render_config) - - post_trajectory = HR_TRAJECTORY_DIR.joinpath(f"{trajectory}.csv") - pre_trajectory = post_trajectory.with_stem(f"{post_trajectory.stem}_delayed") - - compare_external_split_args( - test_info, - in_fmt=in_fmt, - render_config=RENDER_CFG_DIR.joinpath(f"{render_config}.txt"), - pre_trajectory=pre_trajectory, - post_trajectory=post_trajectory, - ) - - -""" ISM """ - - -@pytest.mark.parametrize("trajectory", SPLIT_REND_HR_TRAJECTORIES_TO_TEST) -@pytest.mark.parametrize("render_config", RENDERER_CONFIGS_TO_TEST_ISM) -@pytest.mark.parametrize("bitrate", IVAS_BITRATES_ISM) -@pytest.mark.parametrize("in_fmt", INPUT_FORMATS_ISM_SPLIT_REND) -def test_ism_full_chain_split(test_info, in_fmt, bitrate, render_config, trajectory): - check_xfail(test_info, in_fmt, render_config, bitrate) - - post_trajectory = HR_TRAJECTORY_DIR.joinpath(f"{trajectory}.csv") - pre_trajectory = post_trajectory.with_stem(f"{post_trajectory.stem}_delayed") - - compare_full_chain_split_args( - test_info, - in_fmt=in_fmt, - bitrate=bitrate, - render_config=RENDER_CFG_DIR.joinpath(f"{render_config}.txt"), - pre_trajectory=pre_trajectory, - post_trajectory=post_trajectory, - ) - - -@pytest.mark.parametrize("trajectory", SPLIT_REND_HR_TRAJECTORIES_TO_TEST) -@pytest.mark.parametrize("render_config", RENDERER_CONFIGS_TO_TEST_ISM) -@pytest.mark.parametrize("in_fmt", INPUT_FORMATS_ISM_SPLIT_REND) -def test_ism_external_split(test_info, in_fmt, render_config, trajectory): - check_xfail(test_info, in_fmt, render_config) - - post_trajectory = HR_TRAJECTORY_DIR.joinpath(f"{trajectory}.csv") - pre_trajectory = post_trajectory.with_stem(f"{post_trajectory.stem}_delayed") - - compare_external_split_args( - test_info, - in_fmt=in_fmt, - render_config=RENDER_CFG_DIR.joinpath(f"{render_config}.txt"), - pre_trajectory=pre_trajectory, - post_trajectory=post_trajectory, - ) - - -""" MASA """ - - -@pytest.mark.parametrize("trajectory", SPLIT_REND_HR_TRAJECTORIES_TO_TEST) -@pytest.mark.parametrize("render_config", RENDERER_CONFIGS_TO_TEST_MASA) -@pytest.mark.parametrize("bitrate", IVAS_BITRATES_MASA) -@pytest.mark.parametrize("in_fmt", INPUT_FORMATS_MASA_SPLIT_REND) -def test_masa_full_chain_split(test_info, in_fmt, bitrate, render_config, trajectory): - check_xfail(test_info, in_fmt, render_config, bitrate) - - post_trajectory = HR_TRAJECTORY_DIR.joinpath(f"{trajectory}.csv") - pre_trajectory = post_trajectory.with_stem(f"{post_trajectory.stem}_delayed") - - compare_full_chain_split_args( - test_info, - in_fmt=in_fmt, - bitrate=bitrate, - render_config=RENDER_CFG_DIR.joinpath(f"{render_config}.txt"), - pre_trajectory=pre_trajectory, - post_trajectory=post_trajectory, - ) - - -@pytest.mark.parametrize("trajectory", SPLIT_REND_HR_TRAJECTORIES_TO_TEST) -@pytest.mark.parametrize("render_config", RENDERER_CONFIGS_TO_TEST_MASA) -@pytest.mark.parametrize("in_fmt", INPUT_FORMATS_MASA_SPLIT_REND) -def test_masa_external_split(test_info, in_fmt, render_config, trajectory): - check_xfail(test_info, in_fmt, render_config) - - post_trajectory = HR_TRAJECTORY_DIR.joinpath(f"{trajectory}.csv") - pre_trajectory = post_trajectory.with_stem(f"{post_trajectory.stem}_delayed") - - compare_external_split_args( - test_info, - in_fmt=in_fmt, - render_config=RENDER_CFG_DIR.joinpath(f"{render_config}.txt"), - pre_trajectory=pre_trajectory, - post_trajectory=post_trajectory, - ) - - -""" OSBA """ - - -@pytest.mark.parametrize("trajectory", SPLIT_REND_HR_TRAJECTORIES_TO_TEST) -@pytest.mark.parametrize("render_config", RENDERER_CONFIGS_TO_TEST_OSBA) -@pytest.mark.parametrize("bitrate", IVAS_BITRATES_OSBA) -@pytest.mark.parametrize("in_fmt", INPUT_FORMATS_OSBA_SPLIT_REND) -def test_osba_full_chain_split(test_info, in_fmt, bitrate, render_config, trajectory): - check_xfail(test_info, in_fmt, render_config, bitrate) - - post_trajectory = HR_TRAJECTORY_DIR.joinpath(f"{trajectory}.csv") - pre_trajectory = post_trajectory.with_stem(f"{post_trajectory.stem}_delayed") - - compare_full_chain_split_args( - test_info, - in_fmt=in_fmt, - bitrate=bitrate, - render_config=RENDER_CFG_DIR.joinpath(f"{render_config}.txt"), - pre_trajectory=pre_trajectory, - post_trajectory=post_trajectory, - ) - - -""" PLC """ - - -@pytest.mark.parametrize("error_pattern", PLC_ERROR_PATTERNS) -@pytest.mark.parametrize("trajectory", SPLIT_REND_HR_TRAJECTORIES_TO_TEST) -@pytest.mark.parametrize("render_config", RENDERER_CONFIGS_TO_TEST_PLC) -@pytest.mark.parametrize("in_fmt", INPUT_FORMATS_AMBI_SPLIT_REND[-1:]) -def test_post_rend_plc(test_info, in_fmt, render_config, trajectory, error_pattern): - check_xfail(test_info, in_fmt, render_config) - - post_trajectory = HR_TRAJECTORY_DIR.joinpath(f"{trajectory}.csv") - pre_trajectory = post_trajectory.with_stem(f"{post_trajectory.stem}_delayed") - - compare_external_split_args( - test_info, - in_fmt=in_fmt, - render_config=RENDER_CFG_DIR.joinpath(f"{render_config}.txt"), - pre_trajectory=pre_trajectory, - post_trajectory=post_trajectory, - plc_error_pattern=ERROR_PATTERNS_DIR.joinpath(f"{error_pattern}.ep"), - ) - - -""" BINAURAL_SPLIT_PCM """ - - -full_chain_split_pcm_params = [ - ("HOA3", "96000", "split_renderer_config_1dof_512k_default"), - ("7_1_4", "512000", "split_renderer_config_3dofhq_512k_lc3plus"), - ("ISM4", "384000", "split_renderer_config_2dof_768k_default"), - # ("MASA2", "256000", "split_renderer_config_3dof_384k_lcld"), -] - - -@pytest.mark.parametrize("in_fmt,bitrate,render_config", full_chain_split_pcm_params) -def test_full_chain_split_pcm(test_info, in_fmt, bitrate, render_config): - check_xfail(test_info, in_fmt, render_config, bitrate) - - trajectory = SPLIT_REND_HR_TRAJECTORIES_TO_TEST[0] - post_trajectory = HR_TRAJECTORY_DIR.joinpath(f"{trajectory}.csv") - pre_trajectory = post_trajectory.with_stem(f"{post_trajectory.stem}_delayed") - - compare_full_chain_split_args( - test_info, - in_fmt=in_fmt, - bitrate=bitrate, - render_config=RENDER_CFG_DIR.joinpath(f"{render_config}.txt"), - pre_trajectory=pre_trajectory, - post_trajectory=post_trajectory, - renderer_fmt="BINAURAL_SPLIT_PCM", - ) - - -external_split_pcm_params = [ - ("FOA", "split_renderer_config_3dofhq_512k_lc3plus"), - ("5_1", "split_renderer_config_1dof_768k_default"), - ("ISM1", "split_renderer_config_3dof_384k_lcld"), -] - - -@pytest.mark.parametrize("in_fmt,render_config", external_split_pcm_params) -def test_external_split_pcm(test_info, in_fmt, render_config): - check_xfail(test_info, in_fmt, render_config) - - trajectory = SPLIT_REND_HR_TRAJECTORIES_TO_TEST[0] - post_trajectory = HR_TRAJECTORY_DIR.joinpath(f"{trajectory}.csv") - pre_trajectory = post_trajectory.with_stem(f"{post_trajectory.stem}_delayed") - - compare_external_split_args( - test_info, - in_fmt=in_fmt, - render_config=RENDER_CFG_DIR.joinpath(f"{render_config}.txt"), - pre_trajectory=pre_trajectory, - post_trajectory=post_trajectory, - renderer_fmt="BINAURAL_SPLIT_PCM", - ) diff --git a/tests/split_rendering/utils.py b/tests/split_rendering/utils.py index eab37c3c40..06f7e985cb 100644 --- a/tests/split_rendering/utils.py +++ b/tests/split_rendering/utils.py @@ -36,6 +36,7 @@ from tempfile import TemporaryDirectory from typing import Tuple import numpy as np +import pytest from tests.renderer.utils import check_BE, run_cmd, test_info from tests.split_rendering.constants import * @@ -65,16 +66,15 @@ def truncate_signal( def run_full_chain_split_rendering( + test_info, in_fmt: str, bitrate: str, render_config: Path, pre_trajectory: Path, post_trajectory: Path, - output_path_base: Path, renderer_fmt: str = "BINAURAL_SPLIT_CODED", binary_suffix: str = "", - is_comparetest: bool = False, -) -> Tuple[np.ndarray, int]: +) -> str: """ Runs the full split rendering chain consisting of the IVAS encoder, decoder and split renderer @@ -86,9 +86,14 @@ def run_full_chain_split_rendering( split_bitstream = tmp_dir.joinpath("split.bit") if renderer_fmt == "BINAURAL_SPLIT_PCM": split_md_file = tmp_dir.joinpath("split_md.bin") - out_file = output_path_base.joinpath( - f"{in_fmt}_{bitrate}bps_{renderer_fmt}_{pre_trajectory.stem}_split_full_{post_trajectory.stem}_config_{render_config.stem}.wav" - ) + out_file_stem = f"{in_fmt}_{bitrate}bps_{renderer_fmt}_{pre_trajectory.stem}_split_full_{post_trajectory.stem}_config_{render_config.stem}.wav" + + if test_info.config.option.create_ref: + output_path_base = OUTPUT_PATH_REF + else: + output_path_base = OUTPUT_PATH_CUT + + out_file = output_path_base.joinpath(out_file_stem) # check for metadata files if in_fmt.upper().startswith("OSBA"): @@ -102,9 +107,12 @@ def run_full_chain_split_rendering( # run encoder cmd = SPLIT_PRE_COD_CMD[:] + if test_info.config.option.create_ref: + cmd[0] += "_ref" cmd[0] += binary_suffix cmd[1] = bitrate - if is_comparetest: + # if in REF or CUT creation mode use the comparetestv + if test_info.config.option.create_ref or test_info.config.option.create_cut: in_file = FORMAT_TO_FILE_COMPARETEST[in_fmt] truncate_signal(in_file, cut_in_file) @@ -123,6 +131,8 @@ def run_full_chain_split_rendering( # decode to split-rendering bitstream cmd = SPLIT_PRE_DEC_CMD[:] + if test_info.config.option.create_ref: + cmd[0] += "_ref" cmd[0] += binary_suffix cmd[2] = str(pre_trajectory) cmd[4] = str(render_config) @@ -138,6 +148,8 @@ def run_full_chain_split_rendering( # run split renderer cmd = SPLIT_POST_REND_CMD[:] + if test_info.config.option.create_ref: + cmd[0] += "_ref" cmd[0] += binary_suffix cmd[4] = str(split_bitstream) cmd[6] = renderer_fmt @@ -149,15 +161,30 @@ def run_full_chain_split_rendering( run_cmd(cmd) - return readfile(out_file) + if test_info.config.option.create_cut: + # CUT creation mode will run a comparison with REF + out_file_ref = str(OUTPUT_PATH_REF.joinpath(out_file_stem)) + + try: + ref, ref_fs = readfile(out_file_ref) + except FileNotFoundError: + pytest.fail( + f"Reference vector not found! Ensure they were created with the --create_ref argument.\n{out_file_ref}" + ) + + cut, cut_fs = readfile(out_file) + + check_BE(test_info, ref, ref_fs, cut, cut_fs) + + return out_file def run_external_split_rendering( + test_info, in_fmt: str, render_config: Path, pre_trajectory: Path, post_trajectory: Path, - output_path_base: Path, plc_error_pattern: Path = None, renderer_fmt: str = "BINAURAL_SPLIT_CODED", binary_suffix: str = "", @@ -174,9 +201,15 @@ def run_external_split_rendering( split_bitstream = tmp_dir.joinpath("split.bit") if renderer_fmt == "BINAURAL_SPLIT_PCM": split_md_file = tmp_dir.joinpath("split_md.bin") - out_file = output_path_base.joinpath( - f"{in_fmt}_{renderer_fmt}_{pre_trajectory.stem}_split_ext_{post_trajectory.stem}_config_{render_config.stem}.wav" - ) + out_file_stem = f"{in_fmt}_{renderer_fmt}_{pre_trajectory.stem}_split_ext_{post_trajectory.stem}_config_{render_config.stem}.wav" + + if test_info.config.option.create_ref: + output_path_base = OUTPUT_PATH_REF + else: + output_path_base = OUTPUT_PATH_CUT + + out_file = output_path_base.joinpath(out_file_stem) + if plc_error_pattern: out_file = out_file.with_stem( f"{out_file.stem}_plc_{plc_error_pattern.stem}" @@ -190,6 +223,9 @@ def run_external_split_rendering( # generate split-rendering bitstream cmd = SPLIT_PRE_REND_CMD[:] + + if test_info.config.option.create_ref: + cmd[0] += "_ref" cmd[0] += binary_suffix cmd[4] = str(render_config) if is_comparetest: @@ -215,6 +251,8 @@ def run_external_split_rendering( # run split renderer cmd = SPLIT_POST_REND_CMD[:] + if test_info.config.option.create_ref: + cmd[0] += "_ref" cmd[0] += binary_suffix cmd[4] = str(split_bitstream) cmd[6] = renderer_fmt @@ -229,7 +267,22 @@ def run_external_split_rendering( run_cmd(cmd) - return readfile(out_file) + if test_info.config.option.create_cut: + # CUT creation mode will run a comparison with REF + out_file_ref = str(OUTPUT_PATH_REF.joinpath(out_file_stem)) + + try: + ref, ref_fs = readfile(out_file_ref) + except FileNotFoundError: + pytest.fail( + f"Reference vector not found! Ensure they were created with the --create_ref argument.\n{out_file_ref}" + ) + + cut, cut_fs = readfile(out_file) + + check_BE(test_info, ref, ref_fs, cut, cut_fs) + + return out_file def compare_full_chain_split_args( -- GitLab From 795433520f0249c7d05f4666020660c05b7ba5a0 Mon Sep 17 00:00:00 2001 From: Archit Tamarapu Date: Fri, 27 Oct 2023 14:34:22 +0200 Subject: [PATCH 05/13] restore output MASA metadata BE comparison --- tests/renderer/utils.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/tests/renderer/utils.py b/tests/renderer/utils.py index 99d3f669c7..400e1613c6 100644 --- a/tests/renderer/utils.py +++ b/tests/renderer/utils.py @@ -30,6 +30,7 @@ the United Nations Convention on Contracts on the International Sales of Goods. """ +import filecmp import logging import os import subprocess as sp @@ -229,6 +230,13 @@ def run_renderer( cut, cut_fs = readfile(out_file) check_BE(test_info, ref, ref_fs, cut, cut_fs) + + # compare metadata files in case of MASA prerendering + if "MASA" in out_fmt: + meta_file_ref = out_file_ref + ".met" + meta_file_cut = out_file + ".met" + if not filecmp.cmp(meta_file_cut, meta_file_ref): + pytest.fail("Metadata file differs from reference") return out_file -- GitLab From ec7ad0459c30c43d6bfa11332ddbea0230b64c34 Mon Sep 17 00:00:00 2001 From: Archit Tamarapu Date: Fri, 27 Oct 2023 14:37:32 +0200 Subject: [PATCH 06/13] lint + format --- tests/renderer/utils.py | 2 +- tests/split_rendering/constants.py | 9 --------- 2 files changed, 1 insertion(+), 10 deletions(-) diff --git a/tests/renderer/utils.py b/tests/renderer/utils.py index 400e1613c6..a4fafe2ae0 100644 --- a/tests/renderer/utils.py +++ b/tests/renderer/utils.py @@ -230,7 +230,7 @@ def run_renderer( cut, cut_fs = readfile(out_file) check_BE(test_info, ref, ref_fs, cut, cut_fs) - + # compare metadata files in case of MASA prerendering if "MASA" in out_fmt: meta_file_ref = out_file_ref + ".met" diff --git a/tests/split_rendering/constants.py b/tests/split_rendering/constants.py index 2c0af34ade..e73247cb91 100644 --- a/tests/split_rendering/constants.py +++ b/tests/split_rendering/constants.py @@ -30,7 +30,6 @@ the United Nations Convention on Contracts on the International Sales of Goods. """ -import sys from pathlib import Path """ Set up paths """ @@ -48,14 +47,6 @@ TESTV_DIR = SCRIPTS_DIR.joinpath("testv") BIN_SUFFIX_MERGETARGET = "_ref" -from tests.renderer.constants import ( - CUSTOM_LS_TO_TEST, - FORMAT_TO_FILE_COMPARETEST, - FORMAT_TO_FILE_SMOKETEST, - FORMAT_TO_METADATA_FILES, - METADATA_SCENES_TO_TEST, -) - """ Renderer configurations """ RENDERER_CONFIGS_DEFAULT_CODEC = [ str(cfg.stem) for cfg in RENDER_CFG_DIR.glob("*_default.txt") -- GitLab From 19e10c7ba5b0d51f73ba72ed8dfa5466ab51cbb4 Mon Sep 17 00:00:00 2001 From: Archit Tamarapu Date: Fri, 27 Oct 2023 15:24:04 +0200 Subject: [PATCH 07/13] [ci] use correct script in tests and update coverage test too --- .gitlab-ci.yml | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index f2c3369781..72ad63fda2 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -639,7 +639,7 @@ renderer-pytest-on-merge-request: ### If ref_using_main is not set, checkout the source branch to use scripts and input from there - if [ $ref_using_main == 0 ]; then git checkout $source_branch_commit_sha; fi - exit_code=0 - - python3 -m pytest -q --log-level ERROR -n auto -rA --html=report.html --self-contained-html --junit-xml=report-junit.xml tests/renderer/test_renderer_be_comparison.py --create_ref || exit_code=$? + - python3 -m pytest -q --log-level ERROR -n auto -rA --html=report.html --self-contained-html --junit-xml=report-junit.xml tests/renderer/test_renderer.py --create_ref || exit_code=$? # back to source branch - git checkout $source_branch_commit_sha @@ -651,7 +651,7 @@ renderer-pytest-on-merge-request: # run test - exit_code=0 - - python3 -m pytest -q --log-level ERROR -n auto -rA --html=report.html --self-contained-html --junit-xml=report-junit.xml tests/renderer/test_renderer_be_comparison.py --create_cut || exit_code=$? + - python3 -m pytest -q --log-level ERROR -n auto -rA --html=report.html --self-contained-html --junit-xml=report-junit.xml tests/renderer/test_renderer_.py --create_cut || exit_code=$? - zero_errors=$(cat report-junit.xml | grep -c 'errors="0"') || true - *merge-request-comparison-check @@ -731,7 +731,7 @@ split-rendering-pytest-on-merge-request: - if [ $ref_using_main == 0 ]; then git restore lib_com/options.h; fi # Revert changes back before checking out another branch to avoid conflicts - if [ $ref_using_main == 0 ]; then git checkout $source_branch_commit_sha; fi - exit_code=0 - - python3 -m pytest -q --log-level ERROR -n auto -rA --html=report.html --self-contained-html --junit-xml=report-junit.xml tests/split_rendering/test_split_rendering_be_comparison.py --create_ref || exit_code=$? + - python3 -m pytest -q --log-level ERROR -n auto -rA --html=report.html --self-contained-html --junit-xml=report-junit.xml tests/split_rendering/test_split_rendering.py --create_ref || exit_code=$? # back to source branch - git restore lib_com/options.h # Revert changes back before checking out another branch to avoid conflicts @@ -746,7 +746,7 @@ split-rendering-pytest-on-merge-request: # run test - exit_code=0 - - python3 -m pytest -q --log-level ERROR -n auto -rA --html=report.html --self-contained-html --junit-xml=report-junit.xml tests/split_rendering/test_split_rendering_be_comparison.py --create_cut || exit_code=$? + - python3 -m pytest -q --log-level ERROR -n auto -rA --html=report.html --self-contained-html --junit-xml=report-junit.xml tests/split_rendering/test_split_rendering.py --create_cut || exit_code=$? - zero_errors=$(cat report-junit.xml | grep -c 'errors="0"') || true - *merge-request-comparison-check @@ -1542,7 +1542,8 @@ coverage-test-on-main-scheduled: - python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR -v -n auto --update_ref 1 -m create_ref_part2 --ref_encoder_path ./IVAS_cod --ref_decoder_path ./IVAS_dec - python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR/test_param_file.py -v -n auto --update_ref 1 -m create_ref --param_file scripts/config/self_test_evs.prm --ref_encoder_path ./IVAS_cod --ref_decoder_path ./IVAS_dec - bash ci/smoke_test.sh coverage - - python3 -m pytest -q -n auto tests/renderer/test_renderer_be_comparison.py + - python3 -m pytest -q -n auto tests/renderer/test_renderer.py --create_ref + - python3 -m pytest -q -n auto tests/renderer/test_renderer.py --create_cut - python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR/test_param_file.py -v -n auto --update_ref 1 -m create_ref --param_file scripts/config/self_test_ltv.prm --ref_encoder_path ./IVAS_cod --ref_decoder_path ./IVAS_dec - bash ci/ivas_voip_be_test.sh coverage - lcov -c -d obj -o coverage.info -- GitLab From 68b2d5fce8a6cf270347a461dd3041aada36a0e2 Mon Sep 17 00:00:00 2001 From: Archit Tamarapu Date: Fri, 27 Oct 2023 15:29:49 +0200 Subject: [PATCH 08/13] fix missing imports for split rendering tests --- tests/split_rendering/constants.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/tests/split_rendering/constants.py b/tests/split_rendering/constants.py index e73247cb91..e4eccba39b 100644 --- a/tests/split_rendering/constants.py +++ b/tests/split_rendering/constants.py @@ -32,6 +32,16 @@ from pathlib import Path + +from tests.renderer.constants import ( + CUSTOM_LS_TO_TEST, + FORMAT_TO_FILE_COMPARETEST, + FORMAT_TO_FILE_SMOKETEST, + FORMAT_TO_METADATA_FILES, + METADATA_SCENES_TO_TEST, +) + + """ Set up paths """ TESTS_DIR = Path(__file__).parent RENDER_CFG_DIR = TESTS_DIR.joinpath("renderer_configs").resolve() -- GitLab From 346a671ff13cd23174c2248e5274246c33ec4f5e Mon Sep 17 00:00:00 2001 From: Archit Tamarapu Date: Fri, 27 Oct 2023 15:43:23 +0200 Subject: [PATCH 09/13] [fix] failing PLC comparetest due to incorrect ref filename --- tests/split_rendering/utils.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/tests/split_rendering/utils.py b/tests/split_rendering/utils.py index 06f7e985cb..b07bc3a319 100644 --- a/tests/split_rendering/utils.py +++ b/tests/split_rendering/utils.py @@ -270,6 +270,10 @@ def run_external_split_rendering( if test_info.config.option.create_cut: # CUT creation mode will run a comparison with REF out_file_ref = str(OUTPUT_PATH_REF.joinpath(out_file_stem)) + if plc_error_pattern: + out_file_ref = out_file_ref.with_stem( + f"{out_file_ref.stem}_plc_{plc_error_pattern.stem}" + ) try: ref, ref_fs = readfile(out_file_ref) -- GitLab From fc0058b71494b2d0c80fb4994374a8aab2880c15 Mon Sep 17 00:00:00 2001 From: Archit Tamarapu Date: Fri, 27 Oct 2023 16:03:35 +0200 Subject: [PATCH 10/13] [fix] split rendering test PLC path issue again --- tests/split_rendering/utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/split_rendering/utils.py b/tests/split_rendering/utils.py index b07bc3a319..a0e126499c 100644 --- a/tests/split_rendering/utils.py +++ b/tests/split_rendering/utils.py @@ -269,7 +269,7 @@ def run_external_split_rendering( if test_info.config.option.create_cut: # CUT creation mode will run a comparison with REF - out_file_ref = str(OUTPUT_PATH_REF.joinpath(out_file_stem)) + out_file_ref = OUTPUT_PATH_REF.joinpath(out_file_stem) if plc_error_pattern: out_file_ref = out_file_ref.with_stem( f"{out_file_ref.stem}_plc_{plc_error_pattern.stem}" -- GitLab From ffc71938e46c9339a20c97f847c31d0142356389 Mon Sep 17 00:00:00 2001 From: knj Date: Mon, 30 Oct 2023 15:03:17 +0100 Subject: [PATCH 11/13] update renderer and split rendering jobs --- .gitlab-ci.yml | 24 +++++------------------- 1 file changed, 5 insertions(+), 19 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 72ad63fda2..5d1f033f0c 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -110,7 +110,7 @@ stages: - make -j - mv IVAS_cod ../IVAS_cod_test - mv IVAS_dec ../IVAS_dec_test - - mv IVAS_rend .. + - mv IVAS_rend ../IVAS_rend_test - cd .. - rm -rf build/* @@ -129,11 +129,13 @@ stages: - make -j - mv IVAS_cod ../IVAS_cod_ref - mv IVAS_dec ../IVAS_dec_ref + - mv IVAS_rend ../IVAS_rend_ref - cd .. # rename test binaries back - mv IVAS_cod_test IVAS_cod - mv IVAS_dec_test IVAS_dec + - mv IVAS_rend_test IVAS_rend .merge-request-comparison-check: &merge-request-comparison-check - if [ $zero_errors != 1 ]; then echo "Run errors encountered!"; exit $EXIT_CODE_FAIL; fi @@ -618,6 +620,7 @@ renderer-pytest-on-merge-request: - *print-common-info - *get-commits-behind-count - *check-commits-behind-count-in-compare-jobs + - *merge-request-comparison-setup-codec # some helper variables - "|| true" to prevent failures from grep not finding anything # write to temporary file as workaround for failures observed with piping echo @@ -625,32 +628,16 @@ renderer-pytest-on-merge-request: - non_be_flag=$(grep -c --ignore-case "\[rend\(erer\)*[ -]*non[ -]*be\]" tmp.txt) || true - ref_using_main=$(grep -c --ignore-case "\[ref[ -]*using[ -]*main\]" tmp.txt) || true - # store the current commit hash - - source_branch_commit_sha=$(git rev-parse HEAD) - - - *mr-fetch-target-branch - - *mr-get-target-commit - - git checkout $target_commit - - # build reference binaries - - make -j IVAS_rend - - mv IVAS_rend IVAS_rend_ref - ### If ref_using_main is not set, checkout the source branch to use scripts and input from there - if [ $ref_using_main == 0 ]; then git checkout $source_branch_commit_sha; fi + - exit_code=0 - python3 -m pytest -q --log-level ERROR -n auto -rA --html=report.html --self-contained-html --junit-xml=report-junit.xml tests/renderer/test_renderer.py --create_ref || exit_code=$? - # back to source branch - - git checkout $source_branch_commit_sha - - make clean - - make -j IVAS_rend - ### Run test using branch scripts and input - if [ $ref_using_main == 1 ]; then git checkout $source_branch_commit_sha; fi # run test - - exit_code=0 - python3 -m pytest -q --log-level ERROR -n auto -rA --html=report.html --self-contained-html --junit-xml=report-junit.xml tests/renderer/test_renderer_.py --create_cut || exit_code=$? - zero_errors=$(cat report-junit.xml | grep -c 'errors="0"') || true @@ -745,7 +732,6 @@ split-rendering-pytest-on-merge-request: - if [ $ref_using_main == 1 ]; then git checkout $source_branch_commit_sha; fi # run test - - exit_code=0 - python3 -m pytest -q --log-level ERROR -n auto -rA --html=report.html --self-contained-html --junit-xml=report-junit.xml tests/split_rendering/test_split_rendering.py --create_cut || exit_code=$? - zero_errors=$(cat report-junit.xml | grep -c 'errors="0"') || true -- GitLab From c8d6a857f826461fc7594b50833e48b4feaa85b8 Mon Sep 17 00:00:00 2001 From: Archit Tamarapu Date: Tue, 31 Oct 2023 09:33:51 +0100 Subject: [PATCH 12/13] [fix] crash when out_fmt is a Path + remove unused functions --- tests/renderer/utils.py | 2 +- tests/split_rendering/utils.py | 33 --------------------------------- 2 files changed, 1 insertion(+), 34 deletions(-) diff --git a/tests/renderer/utils.py b/tests/renderer/utils.py index a4fafe2ae0..21d9d89fbb 100644 --- a/tests/renderer/utils.py +++ b/tests/renderer/utils.py @@ -232,7 +232,7 @@ def run_renderer( check_BE(test_info, ref, ref_fs, cut, cut_fs) # compare metadata files in case of MASA prerendering - if "MASA" in out_fmt: + if "MASA" in str(out_fmt): meta_file_ref = out_file_ref + ".met" meta_file_cut = out_file + ".met" if not filecmp.cmp(meta_file_cut, meta_file_ref): diff --git a/tests/split_rendering/utils.py b/tests/split_rendering/utils.py index a0e126499c..975e232f70 100644 --- a/tests/split_rendering/utils.py +++ b/tests/split_rendering/utils.py @@ -287,36 +287,3 @@ def run_external_split_rendering( check_BE(test_info, ref, ref_fs, cut, cut_fs) return out_file - - -def compare_full_chain_split_args( - test_info, - **kwargs, -): - ref, ref_fs = run_full_chain_split_rendering( - output_path_base=OUTPUT_PATH_REF, - binary_suffix=BIN_SUFFIX_MERGETARGET, - is_comparetest=True, - **kwargs, - ) - cut, cut_fs = run_full_chain_split_rendering( - output_path_base=OUTPUT_PATH_CUT, - is_comparetest=True, - **kwargs, - ) - check_BE(test_info, ref, ref_fs, cut, cut_fs) - - -def compare_external_split_args(test_info, **kwargs): - ref, ref_fs = run_external_split_rendering( - output_path_base=OUTPUT_PATH_REF, - binary_suffix=BIN_SUFFIX_MERGETARGET, - is_comparetest=True, - **kwargs, - ) - cut, cut_fs = run_external_split_rendering( - output_path_base=OUTPUT_PATH_CUT, - is_comparetest=True, - **kwargs, - ) - check_BE(test_info, ref, ref_fs, cut, cut_fs) -- GitLab From 387fb807fd2ebd246c0bb351be13e0b0533f2a18 Mon Sep 17 00:00:00 2001 From: Archit Tamarapu Date: Tue, 31 Oct 2023 09:43:57 +0100 Subject: [PATCH 13/13] update ref creation command to use the correct constant for BIN_SUFFIX_MERGETARGET + formatting --- tests/renderer/utils.py | 2 +- tests/split_rendering/constants.py | 4 +--- tests/split_rendering/utils.py | 10 +++++----- 3 files changed, 7 insertions(+), 9 deletions(-) diff --git a/tests/renderer/utils.py b/tests/renderer/utils.py index 21d9d89fbb..2d8d31c421 100644 --- a/tests/renderer/utils.py +++ b/tests/renderer/utils.py @@ -177,7 +177,7 @@ def run_renderer( cmd[8] = str(out_fmt) if test_info.config.option.create_ref: - cmd[0] += "_ref" + cmd[0] += BIN_SUFFIX_MERGETARGET cmd[0] += binary_suffix if in_meta_files is not None: diff --git a/tests/split_rendering/constants.py b/tests/split_rendering/constants.py index e4eccba39b..499e8b8216 100644 --- a/tests/split_rendering/constants.py +++ b/tests/split_rendering/constants.py @@ -32,8 +32,8 @@ from pathlib import Path - from tests.renderer.constants import ( + BIN_SUFFIX_MERGETARGET, CUSTOM_LS_TO_TEST, FORMAT_TO_FILE_COMPARETEST, FORMAT_TO_FILE_SMOKETEST, @@ -41,7 +41,6 @@ from tests.renderer.constants import ( METADATA_SCENES_TO_TEST, ) - """ Set up paths """ TESTS_DIR = Path(__file__).parent RENDER_CFG_DIR = TESTS_DIR.joinpath("renderer_configs").resolve() @@ -55,7 +54,6 @@ CUSTOM_LAYOUT_DIR = SCRIPTS_DIR.joinpath("ls_layouts") HR_TRAJECTORY_DIR = SCRIPTS_DIR.joinpath("trajectories") TESTV_DIR = SCRIPTS_DIR.joinpath("testv") -BIN_SUFFIX_MERGETARGET = "_ref" """ Renderer configurations """ RENDERER_CONFIGS_DEFAULT_CODEC = [ diff --git a/tests/split_rendering/utils.py b/tests/split_rendering/utils.py index 975e232f70..02c9199910 100644 --- a/tests/split_rendering/utils.py +++ b/tests/split_rendering/utils.py @@ -108,7 +108,7 @@ def run_full_chain_split_rendering( cmd = SPLIT_PRE_COD_CMD[:] if test_info.config.option.create_ref: - cmd[0] += "_ref" + cmd[0] += BIN_SUFFIX_MERGETARGET cmd[0] += binary_suffix cmd[1] = bitrate # if in REF or CUT creation mode use the comparetestv @@ -132,7 +132,7 @@ def run_full_chain_split_rendering( cmd = SPLIT_PRE_DEC_CMD[:] if test_info.config.option.create_ref: - cmd[0] += "_ref" + cmd[0] += BIN_SUFFIX_MERGETARGET cmd[0] += binary_suffix cmd[2] = str(pre_trajectory) cmd[4] = str(render_config) @@ -149,7 +149,7 @@ def run_full_chain_split_rendering( cmd = SPLIT_POST_REND_CMD[:] if test_info.config.option.create_ref: - cmd[0] += "_ref" + cmd[0] += BIN_SUFFIX_MERGETARGET cmd[0] += binary_suffix cmd[4] = str(split_bitstream) cmd[6] = renderer_fmt @@ -225,7 +225,7 @@ def run_external_split_rendering( cmd = SPLIT_PRE_REND_CMD[:] if test_info.config.option.create_ref: - cmd[0] += "_ref" + cmd[0] += BIN_SUFFIX_MERGETARGET cmd[0] += binary_suffix cmd[4] = str(render_config) if is_comparetest: @@ -252,7 +252,7 @@ def run_external_split_rendering( cmd = SPLIT_POST_REND_CMD[:] if test_info.config.option.create_ref: - cmd[0] += "_ref" + cmd[0] += BIN_SUFFIX_MERGETARGET cmd[0] += binary_suffix cmd[4] = str(split_bitstream) cmd[6] = renderer_fmt -- GitLab