diff --git a/.gitignore b/.gitignore index a2f6a6b95f2848887c77031f45f10e0d72de0d35..4d9dbe047441277b95272f54f85bd3d2bb9f9d91 100644 --- a/.gitignore +++ b/.gitignore @@ -42,10 +42,9 @@ scripts/out/ scripts/self_test_summary.txt scripts/cppp/ binary/ -tests/renderer/cut -tests/renderer/ref -tests/dut -tests/ref +tests/**/[c|d]ut +tests/**/ref +tests/*/testv scripts/testv/*_cut*.pcm # default reference binary name IVAS_cod_ref diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 5d97bec1567f600abfaa8ffbc05e5bbd6daadfb7..4bb570992d8ea2aae5234c4df2b877b8955bd883 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -11,6 +11,7 @@ variables: EXIT_CODE_NON_BE: 123 EXIT_CODE_FAIL: 1 PROCESSING_SCRIPTS_BIN_DIR: "/test-bin" + TESTS_DIR_CODEC_BE_ON_MR: "tests/codec_be_on_mr_nonselection" default: interruptible: true # Make all jobs by default interruptible @@ -531,15 +532,15 @@ ivas-pytest-on-merge-request: # create short test vectors - python3 tests/create_short_testvectors.py # create references - - python3 -m pytest tests -v --update_ref 1 -m create_ref - - python3 -m pytest tests -v --update_ref 1 -m create_ref_part2 + - python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR -v --update_ref 1 -m create_ref + - python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR -v --update_ref 1 -m create_ref_part2 ### Run test using branch scripts and input - if [ $ref_using_main == 1 ]; then git checkout $source_branch_commit_sha; fi ### run pytest - exit_code=0 - - python3 -m pytest tests -v --html=report.html --self-contained-html --junit-xml=report-junit.xml || exit_code=$? + - python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR -v --html=report.html --self-contained-html --junit-xml=report-junit.xml || exit_code=$? - zero_errors=$(cat report-junit.xml | grep -c 'errors="0"') || true - *merge-request-comparison-check @@ -579,14 +580,14 @@ evs-pytest-on-merge-request: ### prepare pytest # create references - - python3 -m pytest tests/test_param_file.py -v --update_ref 1 -m create_ref --param_file scripts/config/self_test_evs.prm + - python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR/test_param_file.py -v --update_ref 1 -m create_ref --param_file scripts/config/self_test_evs.prm ### Run test using branch scripts and input - if [ $ref_using_main == 1 ]; then git checkout $source_branch_commit_sha; fi ### run pytest for EVS cases - exit_code=0 - - python3 -m pytest tests/test_param_file.py -v --param_file scripts/config/self_test_evs.prm --html=report.html --self-contained-html --junit-xml=report-junit-evs.xml || exit_code=$? + - python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR/test_param_file.py -v --param_file scripts/config/self_test_evs.prm --html=report.html --self-contained-html --junit-xml=report-junit-evs.xml || exit_code=$? - zero_errors=$(cat report-junit-evs.xml | grep -c 'errors="0"') || true - *merge-request-comparison-check @@ -840,15 +841,15 @@ codec-comparison-on-main-push: - mv IVAS_cod_test IVAS_cod - mv IVAS_dec_test IVAS_dec # create references - - python3 -m pytest tests -v --update_ref 1 -m create_ref - - python3 -m pytest tests -v --update_ref 1 -m create_ref_part2 + - python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR -v --update_ref 1 -m create_ref + - python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR -v --update_ref 1 -m create_ref_part2 ### re-checkout the latest commit here, if ref_using_main is set - if [ $ref_using_main -eq 1 ]; then git checkout $latest_commit;fi ### run pytest - exit_code=0 - - python3 -m pytest tests -v --html=report.html --self-contained-html --junit-xml=report-junit.xml || exit_code=$? + - python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR -v --html=report.html --self-contained-html --junit-xml=report-junit.xml || exit_code=$? - if [ $exit_code -eq 1 ] && [ $non_be_flag == 0 ]; then echo "pytest run had failures and non-BE flag not present"; exit $EXIT_CODE_FAIL; fi - zero_errors=$(cat report-junit.xml | grep -c 'errors="0"') || true - if [ $exit_code -eq 1 ] && [ $zero_errors == 1 ]; then echo "pytest run had failures, but no errors and non-BE flag present"; exit $EXIT_CODE_NON_BE; fi @@ -1125,9 +1126,9 @@ coverage-test-on-main-scheduled: - make GCOV=1 -j - cp IVAS_rend IVAS_rend_ref # Copy exec to be able to run renderer script - python3 tests/create_short_testvectors.py - - python3 -m pytest tests -v -n 0 --update_ref 1 -m create_ref --ref_encoder_path ./IVAS_cod --ref_decoder_path ./IVAS_dec - - python3 -m pytest tests -v -n 0 --update_ref 1 -m create_ref_part2 --ref_encoder_path ./IVAS_cod --ref_decoder_path ./IVAS_dec - - python3 -m pytest tests/test_param_file.py -v -n 0 --update_ref 1 -m create_ref --param_file scripts/config/self_test_evs.prm --ref_encoder_path ./IVAS_cod --ref_decoder_path ./IVAS_dec + - python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR -v -n 0 --update_ref 1 -m create_ref --ref_encoder_path ./IVAS_cod --ref_decoder_path ./IVAS_dec + - python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR -v -n 0 --update_ref 1 -m create_ref_part2 --ref_encoder_path ./IVAS_cod --ref_decoder_path ./IVAS_dec + - python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR/test_param_file.py -v -n 0 --update_ref 1 -m create_ref --param_file scripts/config/self_test_evs.prm --ref_encoder_path ./IVAS_cod --ref_decoder_path ./IVAS_dec - bash ci/smoke_test.sh coverage - python3 -m pytest -q -n auto tests/renderer/test_renderer_be_comparison.py - bash ci/ivas_voip_be_test.sh coverage diff --git a/tests/__init__.py b/tests/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/tests/codec_be_on_mr_nonselection/__init__.py b/tests/codec_be_on_mr_nonselection/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/tests/test_masa_enc_dec.py b/tests/codec_be_on_mr_nonselection/test_masa_enc_dec.py similarity index 99% rename from tests/test_masa_enc_dec.py rename to tests/codec_be_on_mr_nonselection/test_masa_enc_dec.py index bfa24c6d30d1a74c21ca7c4f0a5a1dd954da194d..134611ae046c82de39df2d9b56038e41fec06ba1 100644 --- a/tests/test_masa_enc_dec.py +++ b/tests/codec_be_on_mr_nonselection/test_masa_enc_dec.py @@ -41,8 +41,8 @@ import pytest from typing import Optional from filecmp import cmp -from cmp_pcm import cmp_pcm -from conftest import EncoderFrontend, DecoderFrontend +from tests.conftest import EncoderFrontend, DecoderFrontend +from tests.cmp_pcm import cmp_pcm # params #output_mode_list = ['MONO', 'STEREO', '5_1', '7_1', '5_1_2', '5_1_4', '7_1_4', 'FOA', 'HOA2', 'HOA3', 'BINAURAL', 'BINAURAL_ROOM', 'EXT'] diff --git a/tests/test_param_file.py b/tests/codec_be_on_mr_nonselection/test_param_file.py similarity index 98% rename from tests/test_param_file.py rename to tests/codec_be_on_mr_nonselection/test_param_file.py index 5a2c64f589a12cca67aefcceffd5cbb39e459cb5..02e00336f0b7495e6f2266b481079d5fa5f23e7a 100644 --- a/tests/test_param_file.py +++ b/tests/codec_be_on_mr_nonselection/test_param_file.py @@ -40,10 +40,10 @@ import platform import filecmp from subprocess import run import pytest -from cmp_pcm import cmp_pcm -from cut_pcm import cut_samples -from conftest import EncoderFrontend, DecoderFrontend -from testconfig import PARAM_FILE +from tests.cmp_pcm import cmp_pcm +from tests.cut_pcm import cut_samples +from tests.conftest import EncoderFrontend, DecoderFrontend +from tests.testconfig import PARAM_FILE VALID_DEC_OUTPUT_CONF = [ diff --git a/tests/test_sba_bs_dec_plc.py b/tests/codec_be_on_mr_nonselection/test_sba_bs_dec_plc.py similarity index 98% rename from tests/test_sba_bs_dec_plc.py rename to tests/codec_be_on_mr_nonselection/test_sba_bs_dec_plc.py index bead7cda93796a66ea8ba83d5ff487e9d47f7975..c095f2755856f0d5db7b29365b518fea957eed8c 100644 --- a/tests/test_sba_bs_dec_plc.py +++ b/tests/codec_be_on_mr_nonselection/test_sba_bs_dec_plc.py @@ -38,8 +38,8 @@ import os import errno import pytest -from cmp_pcm import cmp_pcm -from conftest import DecoderFrontend +from tests.cmp_pcm import cmp_pcm +from tests.conftest import DecoderFrontend # params tag_list = ['stvFOA'] diff --git a/tests/test_sba_bs_enc.py b/tests/codec_be_on_mr_nonselection/test_sba_bs_enc.py similarity index 99% rename from tests/test_sba_bs_enc.py rename to tests/codec_be_on_mr_nonselection/test_sba_bs_enc.py index 759e1cce4fe44fa63babaed35c91c2d28d90f3d6..3b05399784a3d7ca8a323a6b1d3f2ec05d712f34 100644 --- a/tests/test_sba_bs_enc.py +++ b/tests/codec_be_on_mr_nonselection/test_sba_bs_enc.py @@ -39,9 +39,9 @@ import os import errno import pytest -from cmp_pcm import cmp_pcm -from cut_pcm import cut_samples -from conftest import EncoderFrontend, DecoderFrontend +from tests.cmp_pcm import cmp_pcm +from tests.cut_pcm import cut_samples +from tests.conftest import EncoderFrontend, DecoderFrontend from cut_bs import cut_from_start # params diff --git a/tests/codec_be_on_mr_selection/__init__.py b/tests/codec_be_on_mr_selection/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..2302cec3879e687d1b401d1890d3e618e1ae4b76 --- /dev/null +++ b/tests/codec_be_on_mr_selection/__init__.py @@ -0,0 +1,179 @@ +__copyright__ = """ +(C) 2022-2023 IVAS codec Public Collaboration with portions copyright Dolby International AB, Ericsson AB, +Fraunhofer-Gesellschaft zur Foerderung der angewandten Forschung e.V., Huawei Technologies Co. LTD., +Koninklijke Philips N.V., Nippon Telegraph and Telephone Corporation, Nokia Technologies Oy, Orange, +Panasonic Holdings Corporation, Qualcomm Technologies, Inc., VoiceAge Corporation, and other +contributors to this repository. All Rights Reserved. + +This software is protected by copyright law and by international treaties. +The IVAS codec Public Collaboration consisting of Dolby International AB, Ericsson AB, +Fraunhofer-Gesellschaft zur Foerderung der angewandten Forschung e.V., Huawei Technologies Co. LTD., +Koninklijke Philips N.V., Nippon Telegraph and Telephone Corporation, Nokia Technologies Oy, Orange, +Panasonic Holdings Corporation, Qualcomm Technologies, Inc., VoiceAge Corporation, and other +contributors to this repository retain full ownership rights in their respective contributions in +the software. This notice grants no license of any kind, including but not limited to patent +license, nor is any license granted by implication, estoppel or otherwise. + +Contributors are required to enter into the IVAS codec Public Collaboration agreement before making +contributions. + +This software is provided "AS IS", without any express or implied warranties. The software is in the +development stage. It is intended exclusively for experts who have experience with such software and +solely for the purpose of inspection. All implied warranties of non-infringement, merchantability +and fitness for a particular purpose are hereby disclaimed and excluded. + +Any dispute, controversy or claim arising under or in relation to providing this software shall be +submitted to and settled by the final, binding jurisdiction of the courts of Munich, Germany in +accordance with the laws of the Federal Republic of Germany excluding its conflict of law rules and +the United Nations Convention on Contracts on the International Sales of Goods. +""" + +import tempfile +import pytest +import os +import filecmp +from pathlib import Path +import subprocess +from .constants import OUTPUT_MODES_AND_OPTIONS_FOR_EXPERIMENT, DTX_ON, FER_5PERC + +HERE = Path(__file__).parent +# set environment variables in CI job +# need to wrap in path again as env var is string +TESTV_PATH = Path(os.environ.get("SELECTION_BE_TESTV_DIR", HERE.joinpath("testv"))) +REF_PATH = Path(os.environ.get("SELECTION_BE_REF_DIR", HERE.joinpath("ref"))) +DUT_PATH = HERE.joinpath("dut") + +ISM_NUM_FOR_EXP = { + "P800-6": 1, + "P800-7": 2, + "BS1534-6a": 3, + "BS1534-6b": 4, +} +MASA_TESTS = ("P800-8", "P800-9", "BS1534-7a", "BS1534-7b") + + +def get_testvectors_for_exp_cat_and_testset( + experiment, category, testset, input_file_num +): + input_file_num_str = "" if input_file_num is None else f"-{input_file_num}" + category = "-" if category == "" else f"-{category}-" + fname = f"{experiment}{category}{testset}-input{input_file_num_str}.wav" + signal = TESTV_PATH.joinpath(fname).absolute() + + # will be an empty list if not ISM experiment + metadata = sorted( + [ + TESTV_PATH.joinpath(fname + f".{i}.csv") + for i in range(ISM_NUM_FOR_EXP.get(experiment, 0)) + ] + ) + if experiment in MASA_TESTS: + metadata.append(TESTV_PATH.joinpath(fname + ".met")) + + return signal, metadata + + +def get_out_bitstream_and_synthesis_name(input_signal, bitrate, dtx, fer): + suffix = [".192", ".wav"] + return [ + DUT_PATH.joinpath(f"{input_signal.stem}-{bitrate}bps-{dtx}-{fer}{s}") + for s in suffix + ] + + +def get_error_pattern_for_exp_cat_and_testset(experiment, category, testset): + ep_file = TESTV_PATH.joinpath(f"{experiment}-{category}-{testset}-ep.192") + return ep_file + + +def apply_error_pattern_on_bitstream( + in_bitstream: Path, error_pattern: Path, out_bitstream: Path +): + # temporary file only really needed if same name is given for in and out bs + with tempfile.TemporaryDirectory() as tmpdir: + if in_bitstream == out_bitstream: + in_bitstream = Path(tmpdir).joinpath(in_bitstream.name) + + cmd = ["eid-xor", "-vbr", "-fer", in_bitstream, error_pattern, out_bitstream] + subprocess.run(cmd) + + +def files_equal(dut_file, ref_file): + return filecmp.cmp(dut_file, ref_file) + + +def run_check( + experiment, + category, + testset, + bitrate, + dtx, + fer, + encoder_frontend, + decoder_frontend, + is_ref_creation, + input_file_num=None, +): + sampling_rate = 48 + output_mode, options = OUTPUT_MODES_AND_OPTIONS_FOR_EXPERIMENT[experiment] + + testv, metadata = get_testvectors_for_exp_cat_and_testset( + experiment, category, testset, input_file_num + ) + error_pattern = ( + get_error_pattern_for_exp_cat_and_testset(experiment, category, testset) + if fer == FER_5PERC + else None + ) + + if not testv.exists(): + pytest.fail(f"Testv file not found: {testv}") + if error_pattern is not None and not error_pattern.exists(): + pytest.fail(f"Error pattern not found: {error_pattern}") + for md in metadata: + if not md.exists(): + pytest.fail(f"MD file not found: {md}") + + dut_bitstream, dut_output = get_out_bitstream_and_synthesis_name( + testv, bitrate, dtx, fer + ) + + encoder_frontend.run( + bitrate, + sampling_rate, + testv, + dut_bitstream, + dtx_mode=True if dtx == DTX_ON else False, + add_option_list=options + [str(f) for f in metadata], + ) + + ref_bitstream = REF_PATH.joinpath(dut_bitstream.name) + if not is_ref_creation and not files_equal(dut_bitstream, ref_bitstream): + pytest.fail("Bitstream file differs from reference") + + dut_bitstream_to_decoder = dut_bitstream + if error_pattern is not None: + dut_bitstream_to_decoder = dut_bitstream.with_suffix( + ".err" + dut_bitstream.suffix + ) + apply_error_pattern_on_bitstream( + dut_bitstream, error_pattern, dut_bitstream_to_decoder + ) + + decoder_frontend.run( + output_mode, sampling_rate, dut_bitstream_to_decoder, dut_output + ) + + # NOTE: file comparison is done via filecmp which compares the whole file, including the header for wav files + # this should not be a problem as both the reference and the tdut output was generated by the codec, so + # diverging headers should also indicate a problem - still, keep in mind if something bogus happens + if not is_ref_creation: + ref_output = REF_PATH.joinpath(dut_output.name) + if not files_equal(dut_output, ref_output): + pytest.fail("Decoder output differs from reference") + for md in metadata: + md_suffix = "".join(md.suffixes) + dut_md = DUT_PATH.joinpath(dut_output.with_suffix(md_suffix).name) + ref_md = REF_PATH.joinpath(dut_output.with_suffix(md_suffix).name) + if not files_equal(dut_md, ref_md): + pytest.fail("Metadata file {md.name} differs from reference") diff --git a/tests/codec_be_on_mr_selection/constants.py b/tests/codec_be_on_mr_selection/constants.py new file mode 100644 index 0000000000000000000000000000000000000000..e0afebc5d68f6d833a6599bcf692ffc4150b1a35 --- /dev/null +++ b/tests/codec_be_on_mr_selection/constants.py @@ -0,0 +1,386 @@ +__copyright__ = """ +(C) 2022-2023 IVAS codec Public Collaboration with portions copyright Dolby International AB, Ericsson AB, +Fraunhofer-Gesellschaft zur Foerderung der angewandten Forschung e.V., Huawei Technologies Co. LTD., +Koninklijke Philips N.V., Nippon Telegraph and Telephone Corporation, Nokia Technologies Oy, Orange, +Panasonic Holdings Corporation, Qualcomm Technologies, Inc., VoiceAge Corporation, and other +contributors to this repository. All Rights Reserved. + +This software is protected by copyright law and by international treaties. +The IVAS codec Public Collaboration consisting of Dolby International AB, Ericsson AB, +Fraunhofer-Gesellschaft zur Foerderung der angewandten Forschung e.V., Huawei Technologies Co. LTD., +Koninklijke Philips N.V., Nippon Telegraph and Telephone Corporation, Nokia Technologies Oy, Orange, +Panasonic Holdings Corporation, Qualcomm Technologies, Inc., VoiceAge Corporation, and other +contributors to this repository retain full ownership rights in their respective contributions in +the software. This notice grants no license of any kind, including but not limited to patent +license, nor is any license granted by implication, estoppel or otherwise. + +Contributors are required to enter into the IVAS codec Public Collaboration agreement before making +contributions. + +This software is provided "AS IS", without any express or implied warranties. The software is in the +development stage. It is intended exclusively for experts who have experience with such software and +solely for the purpose of inspection. All implied warranties of non-infringement, merchantability +and fitness for a particular purpose are hereby disclaimed and excluded. + +Any dispute, controversy or claim arising under or in relation to providing this software shall be +submitted to and settled by the final, binding jurisdiction of the courts of Munich, Germany in +accordance with the laws of the Federal Republic of Germany excluding its conflict of law rules and +the United Nations Convention on Contracts on the International Sales of Goods. +""" + +from itertools import product +from typing import Union, List + + +DTX_ON = "DTXon" +DTX_OFF = "DTXoff" +FER_5PERC = "FER_5perc" +FER_0PERC = "FER_0perc" + + +class ExperimentParams: + """ + Class to bundle the parameters for the experiments. Main purpose is to make things more readable over + just using a lot of lists and bundling sanitiy checks and needed utilities together. + """ + + def __init__( + self, + name: str, + n_conditions: int, + bitrates: List[int], + testsets: List[str], + dtx: Union[List[str], str] = DTX_OFF, + fer: Union[List[str], str] = FER_0PERC, + ): + dtx = n_conditions * [dtx] if isinstance(dtx, str) else dtx + fer = n_conditions * [fer] if isinstance(fer, str) else fer + assert ( + len(bitrates) == n_conditions + and len(testsets) == 2 + and len(dtx) == n_conditions + and len(fer) == n_conditions + ) + + self.name = name + self.bitrates = bitrates + self.testsets = testsets + self.dtx = dtx + self.fer = fer + + def assemble_unified_params(self): + """ + Create the parameter combinations for the tested operating points. + Just passing them to pytest would give too many (and not tested) combinations, so do it here. + """ + params = zip(self.bitrates, self.dtx, self.fer) + return [(self.name, *p, t) for p, t in product(params, self.testsets)] + + +P800_CATEGORIES = [f"cat{i}" for i in range(1, 7)] +BS1534_MASA_HOA2_FILE_NUMS = [1, 3, 5, 7, 9, 11, 13, 15] +BS1534_MASA_FOA_FILE_NUMS = [2, 4, 6, 8, 10, 12, 14, 16] +BS1534_N_FILES = 16 + +P800_PARAMS = [ + ExperimentParams( + name="P800-1", + n_conditions=12, + bitrates=[ + 13200, + 16400, + 24400, + 32000, + 48000, + 13200, + 16400, + 24400, + 32000, + 48000, + 24400, + 13200, + ], + dtx=10 * [DTX_OFF] + 2 * [DTX_ON], + fer=5 * [FER_0PERC] + 5 * [FER_5PERC] + [FER_0PERC, FER_5PERC], + testsets=["a", "d"], + ), + ExperimentParams( + name="P800-2", + n_conditions=11, + bitrates=[ + 13200, + 16400, + 24400, + 32000, + 48000, + 64000, + 13200, + 16400, + 24400, + 32000, + 48000, + ], + dtx=6 * [DTX_OFF] + 5 * [DTX_ON], + testsets=["b", "d"], + ), + ExperimentParams( + name="P800-3", + n_conditions=13, + bitrates=[ + 13200, + 16400, + 24400, + 32000, + 48000, + 64000, + 13200, + 16400, + 24400, + 32000, + 48000, + 24400, + 13200, + ], + dtx=11 * [DTX_OFF] + 2 * [DTX_ON], + fer=6 * [FER_0PERC] + 5 * [FER_5PERC] + [FER_0PERC, FER_5PERC], + testsets=["a", "d"], + ), + ExperimentParams( + name="P800-4", + n_conditions=13, + bitrates=[ + 16400, + 24400, + 32000, + 48000, + 64000, + 80000, + 96000, + 24400, + 32000, + 48000, + 64000, + 80000, + 96000, + ], + fer=7 * [FER_0PERC] + 6 * [FER_5PERC], + testsets=["a", "c"], + ), + ExperimentParams( + name="P800-5", + n_conditions=13, + bitrates=[ + 16400, + 24400, + 32000, + 48000, + 64000, + 80000, + 96000, + 16400, + 24400, + 32000, + 48000, + 64000, + 80000, + ], + dtx=7 * [DTX_OFF] + 6 * [DTX_ON], + testsets=["a", "b"], + ), + ExperimentParams( + name="P800-6", + n_conditions=13, + bitrates=[ + 13200, + 16400, + 24400, + 32000, + 48000, + 64000, + 13200, + 16400, + 24400, + 32000, + 13200, + 16400, + 24400, + ], + dtx=10 * [DTX_OFF] + 3 * [DTX_ON], + fer=6 * [FER_0PERC] + 4 * [FER_5PERC] + 3 * [FER_0PERC], + testsets=["a", "c"], + ), + ExperimentParams( + name="P800-7", + n_conditions=13, + bitrates=[ + 16400, + 24400, + 32000, + 48000, + 64000, + 16400, + 24400, + 32000, + 48000, + 16400, + 24400, + 32000, + 48000, + ], + dtx=9 * [DTX_OFF] + 4 * [DTX_ON], + fer=5 * [FER_0PERC] + 4 * [FER_5PERC] + 4 * [FER_0PERC], + testsets=["a", "d"], + ), + ExperimentParams( + name="P800-8", + n_conditions=12, + bitrates=[ + 13200, + 16400, + 24400, + 32000, + 48000, + 64000, + 80000, + 13200, + 16400, + 24400, + 48000, + 64000, + ], + fer=7 * [FER_0PERC] + 5 * [FER_5PERC], + testsets=["a", "b"], + ), + ExperimentParams( + name="P800-9", + n_conditions=13, + bitrates=[ + 13200, + 16400, + 24400, + 32000, + 48000, + 64000, + 80000, + 13200, + 16400, + 24400, + 32000, + 48000, + 64000, + ], + dtx=7 * [DTX_OFF] + 6 * [DTX_ON], + testsets=["a", "d"], + ), +] + +BS1534_PARAMS = [ + ExperimentParams( + name="BS1534-1a", n_conditions=2, bitrates=[48000, 64000], testsets=["a", "d"] + ), + ExperimentParams( + name="BS1534-1b", n_conditions=2, bitrates=[96000, 128000], testsets=["b", "d"] + ), + ExperimentParams( + name="BS1534-2a", n_conditions=2, bitrates=[64000, 96000], testsets=["a", "b"] + ), + ExperimentParams( + name="BS1534-2b", n_conditions=2, bitrates=[128000, 160000], testsets=["a", "b"] + ), + ExperimentParams( + name="BS1534-3a", n_conditions=2, bitrates=[128000, 160000], testsets=["a", "b"] + ), + ExperimentParams( + name="BS1534-3b", n_conditions=2, bitrates=[384000, 512000], testsets=["a", "b"] + ), + ExperimentParams( + name="BS1534-4a", + n_conditions=3, + bitrates=[96000, 128000, 160000], + testsets=["a", "d"], + ), + ExperimentParams( + name="BS1534-4b", n_conditions=2, bitrates=[160000, 192000], testsets=["b", "d"] + ), + ExperimentParams( + name="BS1534-5a", n_conditions=2, bitrates=[192000, 256000], testsets=["a", "d"] + ), + ExperimentParams( + name="BS1534-5b", n_conditions=2, bitrates=[384000, 512000], testsets=["a", "b"] + ), + ExperimentParams( + name="BS1534-6a", + n_conditions=3, + bitrates=[48000, 64000, 96000], + testsets=["b", "d"], + ), + ExperimentParams( + name="BS1534-6b", + n_conditions=3, + bitrates=[96000, 128000, 256000], + testsets=["b", "d"], + ), +] + +BS1534_PARAMS_MASA = [ + ExperimentParams( + name="BS1534-7a", n_conditions=2, bitrates=[96000, 128000], testsets=["b", "d"] + ), + ExperimentParams( + name="BS1534-7b", n_conditions=2, bitrates=[192000, 256000], testsets=["b", "d"] + ), +] + +# These "_UNIFIED" parameter lists are the thigns that get passed to the pytest parametrization + +P800_PARAMS_UNIFIED = [ + i for sl in [p.assemble_unified_params() for p in P800_PARAMS] for i in sl +] +BS1534_PARAMS_UNIFIED = [ + i for sl in [p.assemble_unified_params() for p in BS1534_PARAMS] for i in sl +] + +# for the MASA BS1534 tests everything is complicated because of categories and different numbering in them, hence the extra code here +BS1534_MASA_PARAMS = [ + i for sl in [p.assemble_unified_params() for p in BS1534_PARAMS_MASA] for i in sl +] +BS1534_MASA_PARAMS = [x + (y,) for x, y in product(BS1534_MASA_PARAMS, ["FOA", "HOA2"])] +BS1534_MASA_PARAMS_UNIFIED = [ + x + (y,) + for x, y in product( + [p for p in BS1534_MASA_PARAMS if p[5] == "FOA"], + range(2, 1 + BS1534_N_FILES, 2), + ) +] + [ + x + (y,) + for x, y in product( + [p for p in BS1534_MASA_PARAMS if p[5] == "HOA2"], + range(1, 1 + BS1534_N_FILES, 2), + ) +] + +OUTPUT_MODES_AND_OPTIONS_FOR_EXPERIMENT = { + "P800-1": ("STEREO", ["-stereo"]), + "P800-2": ("STEREO", ["-stereo"]), + "P800-3": ("STEREO", ["-stereo"]), + "P800-4": ("FOA", ["-sba", "1"]), + "P800-5": ("FOA", ["-sba", "1"]), + "P800-6": ("EXT", ["-ism", "1"]), + "P800-7": ("EXT", ["-ism", "2"]), + "P800-8": ("EXT", ["-masa", "2"]), + "P800-9": ("EXT", ["-masa", "2"]), + "BS1534-1a": ("STEREO", ["-stereo"]), + "BS1534-1b": ("STEREO", ["-stereo"]), + "BS1534-2a": ("5_1", ["-mc", "5_1"]), + "BS1534-2b": ("5_1", ["-mc", "5_1"]), + "BS1534-3a": ("7_1_4", ["-mc", "7_1_4"]), + "BS1534-3b": ("7_1_4", ["-mc", "7_1_4"]), + "BS1534-4a": ("HOA3", ["-sba", "1"]), + "BS1534-4b": ("HOA3", ["-sba", "2"]), + "BS1534-5a": ("HOA3", ["-sba", "3"]), + "BS1534-5b": ("HOA3", ["-sba", "3"]), + "BS1534-6a": ("EXT", ["-ism", "3"]), + "BS1534-6b": ("EXT", ["-ism", "4"]), + "BS1534-7a": ("EXT", ["-masa", "2"]), + "BS1534-7b": ("EXT", ["-masa", "2"]), +} diff --git a/tests/codec_be_on_mr_selection/dut/.gitkeep b/tests/codec_be_on_mr_selection/dut/.gitkeep new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/tests/codec_be_on_mr_selection/ref/.gitkeep b/tests/codec_be_on_mr_selection/ref/.gitkeep new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/tests/codec_be_on_mr_selection/test_experiments.py b/tests/codec_be_on_mr_selection/test_experiments.py new file mode 100644 index 0000000000000000000000000000000000000000..a0a4aa780a757cf887e6b80e72a8a48eed00899d --- /dev/null +++ b/tests/codec_be_on_mr_selection/test_experiments.py @@ -0,0 +1,126 @@ +__copyright__ = """ +(C) 2022-2023 IVAS codec Public Collaboration with portions copyright Dolby International AB, Ericsson AB, +Fraunhofer-Gesellschaft zur Foerderung der angewandten Forschung e.V., Huawei Technologies Co. LTD., +Koninklijke Philips N.V., Nippon Telegraph and Telephone Corporation, Nokia Technologies Oy, Orange, +Panasonic Holdings Corporation, Qualcomm Technologies, Inc., VoiceAge Corporation, and other +contributors to this repository. All Rights Reserved. + +This software is protected by copyright law and by international treaties. +The IVAS codec Public Collaboration consisting of Dolby International AB, Ericsson AB, +Fraunhofer-Gesellschaft zur Foerderung der angewandten Forschung e.V., Huawei Technologies Co. LTD., +Koninklijke Philips N.V., Nippon Telegraph and Telephone Corporation, Nokia Technologies Oy, Orange, +Panasonic Holdings Corporation, Qualcomm Technologies, Inc., VoiceAge Corporation, and other +contributors to this repository retain full ownership rights in their respective contributions in +the software. This notice grants no license of any kind, including but not limited to patent +license, nor is any license granted by implication, estoppel or otherwise. + +Contributors are required to enter into the IVAS codec Public Collaboration agreement before making +contributions. + +This software is provided "AS IS", without any express or implied warranties. The software is in the +development stage. It is intended exclusively for experts who have experience with such software and +solely for the purpose of inspection. All implied warranties of non-infringement, merchantability +and fitness for a particular purpose are hereby disclaimed and excluded. + +Any dispute, controversy or claim arising under or in relation to providing this software shall be +submitted to and settled by the final, binding jurisdiction of the courts of Munich, Germany in +accordance with the laws of the Federal Republic of Germany excluding its conflict of law rules and +the United Nations Convention on Contracts on the International Sales of Goods. +""" + +import pytest +from . import run_check +from .constants import ( + P800_CATEGORIES, + BS1534_N_FILES, + P800_PARAMS_UNIFIED, + BS1534_PARAMS_UNIFIED, + BS1534_MASA_PARAMS_UNIFIED, +) + + +@pytest.mark.create_ref +@pytest.mark.parametrize("experiment,bitrate,dtx,fer,testset", P800_PARAMS_UNIFIED) +@pytest.mark.parametrize("category", P800_CATEGORIES) +def test_p800( + experiment, + bitrate, + dtx, + fer, + testset, + category, + dut_encoder_frontend, + dut_decoder_frontend, + update_ref, +): + run_check( + experiment, + category, + testset, + bitrate, + dtx, + fer, + dut_encoder_frontend, + dut_decoder_frontend, + update_ref == 1, + ) + + +@pytest.mark.create_ref +@pytest.mark.parametrize("experiment,bitrate,dtx,fer,testset", BS1534_PARAMS_UNIFIED) +@pytest.mark.parametrize("input_file_num", range(1, 1 + BS1534_N_FILES)) +def test_bs1534_no_masa( + experiment, + bitrate, + dtx, + fer, + testset, + input_file_num, + dut_encoder_frontend, + dut_decoder_frontend, + update_ref, +): + category = "" + run_check( + experiment, + category, + testset, + bitrate, + dtx, + fer, + dut_encoder_frontend, + dut_decoder_frontend, + update_ref == 1, + input_file_num=input_file_num, + ) + + +@pytest.mark.create_ref +@pytest.mark.parametrize( + "experiment,bitrate,dtx,fer,testset,category,input_file_num", + BS1534_MASA_PARAMS_UNIFIED, +) +def test_bs1534_masa( + experiment, + bitrate, + dtx, + fer, + testset, + category, + input_file_num, + dut_encoder_frontend, + dut_decoder_frontend, + update_ref, +): + run_check( + experiment, + category, + testset, + bitrate, + dtx, + fer, + dut_encoder_frontend, + dut_decoder_frontend, + update_ref == 1, + input_file_num=input_file_num, + ) diff --git a/tests/codec_be_on_mr_selection/testv/.gitkeep b/tests/codec_be_on_mr_selection/testv/.gitkeep new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/tests/conftest.py b/tests/conftest.py index d7a35a36e03d2c55e7a50aeb1a2e66c482830574..12b62caccd9d49cd003b204cfe8ce7e307674a24 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -41,7 +41,7 @@ from subprocess import run import textwrap from typing import Optional import os -import testconfig +from tests import testconfig import pytest logger = logging.getLogger(__name__) diff --git a/tests/prepare_pytests.py b/tests/prepare_pytests.py index 9074a6704ab501a485af902fa8d0d99dbf575bc8..9e50ea90d11462ca46c64c9c299c8911d63e7f39 100755 --- a/tests/prepare_pytests.py +++ b/tests/prepare_pytests.py @@ -98,12 +98,12 @@ def main(argv): else: base_cmd = ["python3", "-m", "pytest"] if args.param_file: - base_cmd += ["tests/test_param_file.py", "--param_file", args.param_file] + base_cmd += ["tests/codec_be_on_mr_nonselection/test_param_file.py", "--param_file", args.param_file] else: - base_cmd += ["tests"] + base_cmd += ["tests/codec_be_on_mr_nonselection"] base_cmd += [ "-n", - args.numprocesses, + "14", "--update_ref", "1", ] diff --git a/tests/run_pytests.py b/tests/run_pytests.py index d20de6ef720096eae11b1389d82c975c26470d3a..91ab8f27d531fa9e648360aec94339c5d2e1c7f7 100755 --- a/tests/run_pytests.py +++ b/tests/run_pytests.py @@ -93,9 +93,9 @@ def main(argv): else: cmd = ["python3", "-m", "pytest"] if args.param_file: - cmd += ["tests/test_param_file.py", "--param_file", args.param_file] + cmd += ["tests/codec_be_on_mr_nonselection/test_param_file.py", "--param_file", args.param_file] else: - cmd += ["tests"] + cmd += ["tests/codec_be_on_mr_nonselection"] cmd += ["-n", args.numprocesses] result = subprocess.run(cmd, check=False)