From e07bed0861026adc135c9cb441813cf893452c89 Mon Sep 17 00:00:00 2001 From: knj Date: Thu, 24 Aug 2023 18:28:21 +0200 Subject: [PATCH 001/106] first encoder testcase --- .../codec_be_to_accepted_release/__init__.py | 56 ++++++++ .../codec_be_to_accepted_release/constants.py | 118 +++++++++++++++++ .../test_encoder.py | 120 ++++++++++++++++++ 3 files changed, 294 insertions(+) create mode 100644 tests/codec_be_to_accepted_release/__init__.py create mode 100644 tests/codec_be_to_accepted_release/constants.py create mode 100644 tests/codec_be_to_accepted_release/test_encoder.py diff --git a/tests/codec_be_to_accepted_release/__init__.py b/tests/codec_be_to_accepted_release/__init__.py new file mode 100644 index 0000000000..fe81f4c45f --- /dev/null +++ b/tests/codec_be_to_accepted_release/__init__.py @@ -0,0 +1,56 @@ +__copyright__ = """ +(C) 2022-2023 IVAS codec Public Collaboration with portions copyright Dolby International AB, Ericsson AB, +Fraunhofer-Gesellschaft zur Foerderung der angewandten Forschung e.V., Huawei Technologies Co. LTD., +Koninklijke Philips N.V., Nippon Telegraph and Telephone Corporation, Nokia Technologies Oy, Orange, +Panasonic Holdings Corporation, Qualcomm Technologies, Inc., VoiceAge Corporation, and other +contributors to this repository. All Rights Reserved. + +This software is protected by copyright law and by international treaties. +The IVAS codec Public Collaboration consisting of Dolby International AB, Ericsson AB, +Fraunhofer-Gesellschaft zur Foerderung der angewandten Forschung e.V., Huawei Technologies Co. LTD., +Koninklijke Philips N.V., Nippon Telegraph and Telephone Corporation, Nokia Technologies Oy, Orange, +Panasonic Holdings Corporation, Qualcomm Technologies, Inc., VoiceAge Corporation, and other +contributors to this repository retain full ownership rights in their respective contributions in +the software. This notice grants no license of any kind, including but not limited to patent +license, nor is any license granted by implication, estoppel or otherwise. + +Contributors are required to enter into the IVAS codec Public Collaboration agreement before making +contributions. + +This software is provided "AS IS", without any express or implied warranties. The software is in the +development stage. It is intended exclusively for experts who have experience with such software and +solely for the purpose of inspection. All implied warranties of non-infringement, merchantability +and fitness for a particular purpose are hereby disclaimed and excluded. + +Any dispute, controversy or claim arising under or in relation to providing this software shall be +submitted to and settled by the final, binding jurisdiction of the courts of Munich, Germany in +accordance with the laws of the Federal Republic of Germany excluding its conflict of law rules and +the United Nations Convention on Contracts on the International Sales of Goods. +""" + +import os +from pathlib import Path +import filecmp +import subprocess +from ..testconfig import MD5_REF_DICT + +HERE = Path(__file__).parent +TESTV_PATH = Path(os.environ.get("BE_TESTV_DIR", HERE.joinpath("testv"))) +REF_PATH = Path(os.environ.get("BE_REF_DIR", HERE.joinpath("ref"))) +DUT_PATH = HERE.joinpath("dut") + + +def is_be_to_ref(dut_file): + """ + Check bitexactness either by comparing files directly or by comparing MD5 sums + """ + if MD5_REF_DICT == dict(): + ref_file = REF_PATH.joinpath(dut_file.name) + is_be = filecmp.cmp(dut_file, ref_file) + else: + md5_ref = MD5_REF_DICT[dut_file.name] + cmd = f"powershell.exe (Get-FileHash {str(dut_file)} -Algorithm MD5).Hash" + md5_dut = subprocess.check_output(cmd, shell=True).decode().splitlines()[-1] + is_be = md5_ref == md5_dut + + return is_be diff --git a/tests/codec_be_to_accepted_release/constants.py b/tests/codec_be_to_accepted_release/constants.py new file mode 100644 index 0000000000..a7df89cf26 --- /dev/null +++ b/tests/codec_be_to_accepted_release/constants.py @@ -0,0 +1,118 @@ +__copyright__ = """ +(C) 2022-2023 IVAS codec Public Collaboration with portions copyright Dolby International AB, Ericsson AB, +Fraunhofer-Gesellschaft zur Foerderung der angewandten Forschung e.V., Huawei Technologies Co. LTD., +Koninklijke Philips N.V., Nippon Telegraph and Telephone Corporation, Nokia Technologies Oy, Orange, +Panasonic Holdings Corporation, Qualcomm Technologies, Inc., VoiceAge Corporation, and other +contributors to this repository. All Rights Reserved. + +This software is protected by copyright law and by international treaties. +The IVAS codec Public Collaboration consisting of Dolby International AB, Ericsson AB, +Fraunhofer-Gesellschaft zur Foerderung der angewandten Forschung e.V., Huawei Technologies Co. LTD., +Koninklijke Philips N.V., Nippon Telegraph and Telephone Corporation, Nokia Technologies Oy, Orange, +Panasonic Holdings Corporation, Qualcomm Technologies, Inc., VoiceAge Corporation, and other +contributors to this repository retain full ownership rights in their respective contributions in +the software. This notice grants no license of any kind, including but not limited to patent +license, nor is any license granted by implication, estoppel or otherwise. + +Contributors are required to enter into the IVAS codec Public Collaboration agreement before making +contributions. + +This software is provided "AS IS", without any express or implied warranties. The software is in the +development stage. It is intended exclusively for experts who have experience with such software and +solely for the purpose of inspection. All implied warranties of non-infringement, merchantability +and fitness for a particular purpose are hereby disclaimed and excluded. + +Any dispute, controversy or claim arising under or in relation to providing this software shall be +submitted to and settled by the final, binding jurisdiction of the courts of Munich, Germany in +accordance with the laws of the Federal Republic of Germany excluding its conflict of law rules and +the United Nations Convention on Contracts on the International Sales of Goods. +""" + +DTX_ON = "DTXon" +DTX_OFF = "DTXoff" + +TESTVECTOR_FOR_INPUT_FORMAT = { + "STEREO": "ltv{fs}_STEREO.wav", + "MC_5_1": "ltv{fs}_MC51.wav", + "MC_5_1_2": "ltv{fs}_MC512.wav", + "MC_5_1_4": "ltv{fs}_MC514.wav", + "MC_7_1": "ltv{fs}_MC71.wav", + "MC_7_1_4": "ltv{fs}_MC714.wav", + "FOA": "ltv{fs}_FOA.wav", + "HOA2": "ltv{fs}_HOA2.wav", + "HOA3": "ltv{fs}_HOA3.wav", + "ISM1": "ltv{fs}_1ISM.wav", + "ISM2": "ltv{fs}_2ISM.wav", + "ISM3": "ltv{fs}_3ISM.wav", + "ISM4": "ltv{fs}_4ISM.wav", + # TODO: MASA, once created +} +METADATA_FOR_INPUT_FORMAT = { + "ISM1": ["ltvISM1.csv"], + "ISM2": ["ltvISM1.csv", "ltvISM2.csv"], + "ISM3": ["ltvISM1.csv", "ltvISM2.csv", "ltvISM3.csv"], + "ISM4": ["ltvISM1.csv", "ltvISM2.csv", "ltvISM3.csv", "ltvISM4.csv"], + # TODO: MASA, once created +} +CMDL_OPTIONS_FOR_INPUT_FORMAT = { + "STEREO": ["-stereo"], + "MC_5_1": ["-mc", "5_1"], + "MC_5_1_2": ["-mc", "5_1_2"], + "MC_5_1_4": ["-mc", "5_1_4"], + "MC_7_1": ["-mc", "7_1"], + "MC_7_1_4": ["-mc", "7_1_4"], + "FOA": ["-sba", "+1"], + "HOA2": ["-sba", "+2"], + "HOA3": ["-sba", "+3"], + "PlanarFOA": ["-sba", "-1"], + "PlanarHOA2": ["-sba", "-2"], + "PlanarHOA3": ["-sba", "-3"], + "ISM1": ["-ism", "1"], + "ISM2": ["-ism", "2"], + "ISM3": ["-ism", "3"], + "ISM4": ["-ism", "4"], +} + +### Test case parameters + +# Input format IDs +INPUT_FORMATS_MC = ["MC_5_1", "MC_5_1_2", "MC_5_1_4", "MC_7_1", "MC_7_1_4"] +INPUT_FORMATS_CHANNEL_BASED = ["STEREO"] + INPUT_FORMATS_MC +INPUT_FORMATS_SCENE_BASED = ["FOA", "HOA2", "HOA3"] +INPUT_FORMATS_OBJECT_BASED = ["ISM1", "ISM2", "ISM3", "ISM4"] +INPUT_FORMATS_MASA = [ + "MASA_1TC_1DIR", + "MASA_2TC_1DIR", + "MASA_1TC_2DIR", + "MASA_2TC_2DIR", +] +# TODO: add MASA back +INPUT_FORMATS_SINGLE = ( + INPUT_FORMATS_CHANNEL_BASED + INPUT_FORMATS_SCENE_BASED + INPUT_FORMATS_OBJECT_BASED +) # + INPUT_FORMATS_MASA +# TODO: combined formats + +# bitrates +BITRATES_ALL = [ + 13200, + 16400, + 24400, + 32000, + 48000, + 64000, + 80000, + 96000, + 128000, + 192000, + 256000, + 384000, + 512000, +] +BITRATES_STEREO = BITRATES_ALL[:10] +BITRATES_ISM1 = BITRATES_ALL[:9] +BITRATES_ISM2 = BITRATES_ALL[1:10] +BITRATES_ISM3 = BITRATES_ALL[2:11] +BITRATES_ISM4 = BITRATES_ALL[2:] + +# sampling rates +SAMPLING_RATES_ALL = [16, 32, 48] diff --git a/tests/codec_be_to_accepted_release/test_encoder.py b/tests/codec_be_to_accepted_release/test_encoder.py new file mode 100644 index 0000000000..76a584ea02 --- /dev/null +++ b/tests/codec_be_to_accepted_release/test_encoder.py @@ -0,0 +1,120 @@ +__copyright__ = """ +(C) 2022-2023 IVAS codec Public Collaboration with portions copyright Dolby International AB, Ericsson AB, +Fraunhofer-Gesellschaft zur Foerderung der angewandten Forschung e.V., Huawei Technologies Co. LTD., +Koninklijke Philips N.V., Nippon Telegraph and Telephone Corporation, Nokia Technologies Oy, Orange, +Panasonic Holdings Corporation, Qualcomm Technologies, Inc., VoiceAge Corporation, and other +contributors to this repository. All Rights Reserved. + +This software is protected by copyright law and by international treaties. +The IVAS codec Public Collaboration consisting of Dolby International AB, Ericsson AB, +Fraunhofer-Gesellschaft zur Foerderung der angewandten Forschung e.V., Huawei Technologies Co. LTD., +Koninklijke Philips N.V., Nippon Telegraph and Telephone Corporation, Nokia Technologies Oy, Orange, +Panasonic Holdings Corporation, Qualcomm Technologies, Inc., VoiceAge Corporation, and other +contributors to this repository retain full ownership rights in their respective contributions in +the software. This notice grants no license of any kind, including but not limited to patent +license, nor is any license granted by implication, estoppel or otherwise. + +Contributors are required to enter into the IVAS codec Public Collaboration agreement before making +contributions. + +This software is provided "AS IS", without any express or implied warranties. The software is in the +development stage. It is intended exclusively for experts who have experience with such software and +solely for the purpose of inspection. All implied warranties of non-infringement, merchantability +and fitness for a particular purpose are hereby disclaimed and excluded. + +Any dispute, controversy or claim arising under or in relation to providing this software shall be +submitted to and settled by the final, binding jurisdiction of the courts of Munich, Germany in +accordance with the laws of the Federal Republic of Germany excluding its conflict of law rules and +the United Nations Convention on Contracts on the International Sales of Goods. +""" + +import pytest +from itertools import product +from .constants import * +from . import TESTV_PATH, DUT_PATH, is_be_to_ref + + +def get_testv(input_format, sampling_rate): + testv = TESTVECTOR_FOR_INPUT_FORMAT[input_format].format(fs=sampling_rate) + return TESTV_PATH.joinpath(testv) + + +def get_md(input_format): + md_files = METADATA_FOR_INPUT_FORMAT.get(input_format, list()) + return [str(TESTV_PATH.joinpath(f)) for f in md_files] + + +def get_options(input_format): + return list(CMDL_OPTIONS_FOR_INPUT_FORMAT[input_format]) + + +def run_check( + input_format, bitrate, sampling_rate, dtx, encoder_frontend, is_ref_creation +): + testv = get_testv(input_format, sampling_rate) + options = get_options(input_format) + md_files = get_md(input_format) + options.extend(md_files) + + bitstream = DUT_PATH.joinpath( + f"{testv.stem}-{input_format}-{bitrate}kbps-{sampling_rate}kHz-{dtx}.192" + ) + + encoder_frontend.run( + bitrate, + sampling_rate, + str(testv), + str(bitstream), + dtx_mode=dtx == DTX_ON, + add_option_list=options, + ) + + if not is_ref_creation: + if not is_be_to_ref(bitstream): + pytest.fail(f"Bitstream file differs from reference") + + +STEREO_PARAMS = list( + product(["STEREO"], BITRATES_STEREO, SAMPLING_RATES_ALL, [False, True]) +) +MC_PARAMS = list(product(INPUT_FORMATS_MC, BITRATES_ALL, SAMPLING_RATES_ALL, [False])) +SBA_PARAMS = list( + ( + product( + INPUT_FORMATS_SCENE_BASED, + BITRATES_ALL[:7], + SAMPLING_RATES_ALL, + [False, True], + ) + ) +) + list( + (product(INPUT_FORMATS_SCENE_BASED, BITRATES_ALL[7:], SAMPLING_RATES_ALL, [False])) +) +ISM1_PARAMS = list(product(["ISM1"], BITRATES_ISM1, SAMPLING_RATES_ALL, [False, True])) +ISM2_PARAMS = list(product(["ISM2"], BITRATES_ISM2, SAMPLING_RATES_ALL, [False, True])) +ISM3_PARAMS = list(product(["ISM3"], BITRATES_ISM3, SAMPLING_RATES_ALL, [False, True])) +ISM4_PARAMS = list(product(["ISM4"], BITRATES_ISM4, SAMPLING_RATES_ALL, [False, True])) +# TODO: MASA +ENCODER_PARAMS = ( + STEREO_PARAMS + + MC_PARAMS + + SBA_PARAMS + + ISM1_PARAMS + + ISM2_PARAMS + + ISM3_PARAMS + + ISM4_PARAMS +) + + +@pytest.mark.parametrize("input_format,bitrate,sampling_rate,dtx", ENCODER_PARAMS) +def test_encoder( + input_format, + bitrate, + sampling_rate, + dtx, + dut_encoder_frontend, + update_ref, +): + run_check( + input_format, bitrate, sampling_rate, dtx, dut_encoder_frontend, update_ref == 1 + ) -- GitLab From c88d98419640917cb84f178429939d6d45638f69 Mon Sep 17 00:00:00 2001 From: knj Date: Fri, 25 Aug 2023 09:54:24 +0200 Subject: [PATCH 002/106] add ISM "special MD" testcases --- .../codec_be_to_accepted_release/constants.py | 11 + .../test_encoder.py | 194 +++++++++++++++--- 2 files changed, 181 insertions(+), 24 deletions(-) diff --git a/tests/codec_be_to_accepted_release/constants.py b/tests/codec_be_to_accepted_release/constants.py index a7df89cf26..8f2caf5478 100644 --- a/tests/codec_be_to_accepted_release/constants.py +++ b/tests/codec_be_to_accepted_release/constants.py @@ -31,6 +31,9 @@ the United Nations Convention on Contracts on the International Sales of Goods. DTX_ON = "DTXon" DTX_OFF = "DTXoff" +ISM_MD_EXTENDED = "MDextended" +ISM_MD_NULL = "MDnull" + TESTVECTOR_FOR_INPUT_FORMAT = { "STEREO": "ltv{fs}_STEREO.wav", "MC_5_1": "ltv{fs}_MC51.wav", @@ -45,6 +48,10 @@ TESTVECTOR_FOR_INPUT_FORMAT = { "ISM2": "ltv{fs}_2ISM.wav", "ISM3": "ltv{fs}_3ISM.wav", "ISM4": "ltv{fs}_4ISM.wav", + "ISM1": "ltv{fs}_1ISM.wav", + "ISM2": "ltv{fs}_2ISM.wav", + "ISM3": "ltv{fs}_3ISM.wav", + "ISM4": "ltv{fs}_4ISM.wav", # TODO: MASA, once created } METADATA_FOR_INPUT_FORMAT = { @@ -113,6 +120,10 @@ BITRATES_ISM1 = BITRATES_ALL[:9] BITRATES_ISM2 = BITRATES_ALL[1:10] BITRATES_ISM3 = BITRATES_ALL[2:11] BITRATES_ISM4 = BITRATES_ALL[2:] +BITRATES_ISM1_EXTENDED = [b for b in BITRATES_ISM1 if b > 64000] +BITRATES_ISM2_EXTENDED = [b for b in BITRATES_ISM2 if b > 64000] +BITRATES_ISM3_EXTENDED = [b for b in BITRATES_ISM3 if b > 64000] +BITRATES_ISM4_EXTENDED = [b for b in BITRATES_ISM4 if b > 64000] # sampling rates SAMPLING_RATES_ALL = [16, 32, 48] diff --git a/tests/codec_be_to_accepted_release/test_encoder.py b/tests/codec_be_to_accepted_release/test_encoder.py index 76a584ea02..a0c81ecb9c 100644 --- a/tests/codec_be_to_accepted_release/test_encoder.py +++ b/tests/codec_be_to_accepted_release/test_encoder.py @@ -34,66 +34,99 @@ from .constants import * from . import TESTV_PATH, DUT_PATH, is_be_to_ref +### --------------- Helper functions --------------- + + def get_testv(input_format, sampling_rate): testv = TESTVECTOR_FOR_INPUT_FORMAT[input_format].format(fs=sampling_rate) return TESTV_PATH.joinpath(testv) -def get_md(input_format): +def get_md(input_format, md_type=None): md_files = METADATA_FOR_INPUT_FORMAT.get(input_format, list()) - return [str(TESTV_PATH.joinpath(f)) for f in md_files] + if md_type == ISM_MD_NULL: + md_files = ["NULL" for f in md_files] + else: + md_files = [str(TESTV_PATH.joinpath(f)) for f in md_files] + return md_files -def get_options(input_format): - return list(CMDL_OPTIONS_FOR_INPUT_FORMAT[input_format]) +def get_options(input_format, md_type=None): + options = list(CMDL_OPTIONS_FOR_INPUT_FORMAT[input_format]) + if md_type == ISM_MD_EXTENDED: + assert input_format in INPUT_FORMATS_OBJECT_BASED + options[1] = f"+{options[1]}" + return options -def run_check( - input_format, bitrate, sampling_rate, dtx, encoder_frontend, is_ref_creation -): +def get_encoder_inputs(input_format, sampling_rate, md_type=None): testv = get_testv(input_format, sampling_rate) - options = get_options(input_format) - md_files = get_md(input_format) + options = get_options(input_format, md_type) + md_files = get_md(input_format, md_type) options.extend(md_files) + return testv, options - bitstream = DUT_PATH.joinpath( - f"{testv.stem}-{input_format}-{bitrate}kbps-{sampling_rate}kHz-{dtx}.192" - ) +def run_check( + dut_bitstream, + bitrate, + sampling_rate, + testv, + options, + encoder_frontend, + is_ref_creation, + dtx_mode=False, +): encoder_frontend.run( bitrate, sampling_rate, str(testv), - str(bitstream), - dtx_mode=dtx == DTX_ON, + str(dut_bitstream), + dtx_mode=dtx_mode, add_option_list=options, ) if not is_ref_creation: - if not is_be_to_ref(bitstream): + if not is_be_to_ref(dut_bitstream): pytest.fail(f"Bitstream file differs from reference") +### --------------- Actual testcase definitions --------------- + +# "Vanilla" encoder modes + STEREO_PARAMS = list( - product(["STEREO"], BITRATES_STEREO, SAMPLING_RATES_ALL, [False, True]) + product(["STEREO"], BITRATES_STEREO, SAMPLING_RATES_ALL, [DTX_OFF, DTX_ON]) ) -MC_PARAMS = list(product(INPUT_FORMATS_MC, BITRATES_ALL, SAMPLING_RATES_ALL, [False])) +MC_PARAMS = list(product(INPUT_FORMATS_MC, BITRATES_ALL, SAMPLING_RATES_ALL, [DTX_OFF])) SBA_PARAMS = list( ( product( INPUT_FORMATS_SCENE_BASED, BITRATES_ALL[:7], SAMPLING_RATES_ALL, - [False, True], + [DTX_OFF, DTX_ON], ) ) ) + list( - (product(INPUT_FORMATS_SCENE_BASED, BITRATES_ALL[7:], SAMPLING_RATES_ALL, [False])) + ( + product( + INPUT_FORMATS_SCENE_BASED, BITRATES_ALL[7:], SAMPLING_RATES_ALL, [DTX_OFF] + ) + ) +) +ISM1_PARAMS = list( + product(["ISM1"], BITRATES_ISM1, SAMPLING_RATES_ALL, [DTX_OFF, DTX_ON]) +) +ISM2_PARAMS = list( + product(["ISM2"], BITRATES_ISM2, SAMPLING_RATES_ALL, [DTX_OFF, DTX_ON]) +) +ISM3_PARAMS = list( + product(["ISM3"], BITRATES_ISM3, SAMPLING_RATES_ALL, [DTX_OFF, DTX_ON]) +) +ISM4_PARAMS = list( + product(["ISM4"], BITRATES_ISM4, SAMPLING_RATES_ALL, [DTX_OFF, DTX_ON]) ) -ISM1_PARAMS = list(product(["ISM1"], BITRATES_ISM1, SAMPLING_RATES_ALL, [False, True])) -ISM2_PARAMS = list(product(["ISM2"], BITRATES_ISM2, SAMPLING_RATES_ALL, [False, True])) -ISM3_PARAMS = list(product(["ISM3"], BITRATES_ISM3, SAMPLING_RATES_ALL, [False, True])) -ISM4_PARAMS = list(product(["ISM4"], BITRATES_ISM4, SAMPLING_RATES_ALL, [False, True])) # TODO: MASA ENCODER_PARAMS = ( STEREO_PARAMS @@ -115,6 +148,119 @@ def test_encoder( dut_encoder_frontend, update_ref, ): + testv, options = get_encoder_inputs(input_format, sampling_rate) + bitstream = DUT_PATH.joinpath( + f"{testv.stem}-{input_format}-{bitrate}kbps-{sampling_rate}kHz-{dtx}.192" + ) run_check( - input_format, bitrate, sampling_rate, dtx, dut_encoder_frontend, update_ref == 1 + bitstream, + bitrate, + sampling_rate, + testv, + options, + dut_encoder_frontend, + update_ref == 1, + dtx == DTX_ON, + ) + + +# ISM "special" metadata cases: extended metadata and NULL + +ISM1_EXTENDED_PARAMS = list( + product( + ["ISM1"], + BITRATES_ISM1_EXTENDED, + SAMPLING_RATES_ALL, + [DTX_OFF, DTX_ON], + [ISM_MD_EXTENDED], + ) +) +ISM2_EXTENDED_PARAMS = list( + product( + ["ISM2"], + BITRATES_ISM2_EXTENDED, + SAMPLING_RATES_ALL, + [DTX_OFF, DTX_ON], + [ISM_MD_EXTENDED], + ) +) +ISM3_EXTENDED_PARAMS = list( + product( + ["ISM3"], + BITRATES_ISM3_EXTENDED, + SAMPLING_RATES_ALL, + [DTX_OFF, DTX_ON], + [ISM_MD_EXTENDED], + ) +) +ISM4_EXTENDED_PARAMS = list( + product( + ["ISM4"], + BITRATES_ISM4_EXTENDED, + SAMPLING_RATES_ALL, + [DTX_OFF, DTX_ON], + [ISM_MD_EXTENDED], + ) +) +ISM1_NULL_PARAMS = list( + product( + ["ISM1"], BITRATES_ISM1, SAMPLING_RATES_ALL, [DTX_OFF, DTX_ON], [ISM_MD_NULL] + ) +) +ISM2_NULL_PARAMS = list( + product( + ["ISM2"], BITRATES_ISM2, SAMPLING_RATES_ALL, [DTX_OFF, DTX_ON], [ISM_MD_NULL] + ) +) +ISM3_NULL_PARAMS = list( + product( + ["ISM3"], BITRATES_ISM3, SAMPLING_RATES_ALL, [DTX_OFF, DTX_ON], [ISM_MD_NULL] + ) +) +ISM4_NULL_PARAMS = list( + product( + ["ISM4"], BITRATES_ISM4, SAMPLING_RATES_ALL, [DTX_OFF, DTX_ON], [ISM_MD_NULL] + ) +) +ISM_MD_SPECIAL_PARAMS = ( + ISM1_EXTENDED_PARAMS + + ISM2_EXTENDED_PARAMS + + ISM3_EXTENDED_PARAMS + + ISM4_EXTENDED_PARAMS + + ISM1_EXTENDED_PARAMS + + ISM2_EXTENDED_PARAMS + + ISM3_EXTENDED_PARAMS + + ISM4_EXTENDED_PARAMS +) + + +@pytest.mark.parametrize( + "input_format,bitrate,sampling_rate,dtx,md_type", ISM_MD_SPECIAL_PARAMS +) +def test_encoder_ism_MD_special( + input_format, + bitrate, + sampling_rate, + dtx, + md_type, + dut_encoder_frontend, + update_ref, +): + testv, options = get_encoder_inputs(input_format, sampling_rate, md_type) + if md_type == ISM_MD_EXTENDED: + input_format += "_ext_MD" + elif md_type == ISM_MD_NULL: + input_format += "_null_MD" + bitstream = DUT_PATH.joinpath( + f"{testv.stem}-{input_format}-{bitrate}kbps-{sampling_rate}kHz-{dtx}.192" + ) + run_check( + bitstream, + bitrate, + sampling_rate, + testv, + options, + dut_encoder_frontend, + update_ref == 1, + dtx == DTX_ON, ) -- GitLab From cfc05e94f59b33b69cb82b8402c2b279946cb906 Mon Sep 17 00:00:00 2001 From: knj Date: Fri, 25 Aug 2023 13:15:40 +0200 Subject: [PATCH 003/106] add test for FOA PCA --- .../test_encoder.py | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/tests/codec_be_to_accepted_release/test_encoder.py b/tests/codec_be_to_accepted_release/test_encoder.py index a0c81ecb9c..3ef664d240 100644 --- a/tests/codec_be_to_accepted_release/test_encoder.py +++ b/tests/codec_be_to_accepted_release/test_encoder.py @@ -264,3 +264,20 @@ def test_encoder_ism_MD_special( update_ref == 1, dtx == DTX_ON, ) + + +@pytest.mark.parametrize("sampling_rate", SAMPLING_RATES_ALL) +def test_encoder_foa_pca_at_256kbps( + sampling_rate, + dut_encoder_frontend, + update_ref +): + input_format = "FOA" + bitrate = 256000 + dtx = "DTXoff" + + testv, options = get_encoder_inputs(input_format, sampling_rate) + bitstream = DUT_PATH.joinpath( + f"{testv.stem}-{input_format + '_pca'}-{bitrate}kbps-{sampling_rate}kHz-{dtx}.192" + ) + run_check(bitstream, bitrate, sampling_rate, testv, options, dut_encoder_frontend, update_ref == 1) \ No newline at end of file -- GitLab From 95d11cc339cb7f6388f1bb55e85b138e155bc484 Mon Sep 17 00:00:00 2001 From: knj Date: Fri, 25 Aug 2023 14:45:40 +0200 Subject: [PATCH 004/106] restructure a bit --- .../codec_be_to_accepted_release/constants.py | 4 + .../test_encoder.py | 129 ++++++++---------- 2 files changed, 60 insertions(+), 73 deletions(-) diff --git a/tests/codec_be_to_accepted_release/constants.py b/tests/codec_be_to_accepted_release/constants.py index 8f2caf5478..5870e66b34 100644 --- a/tests/codec_be_to_accepted_release/constants.py +++ b/tests/codec_be_to_accepted_release/constants.py @@ -31,9 +31,13 @@ the United Nations Convention on Contracts on the International Sales of Goods. DTX_ON = "DTXon" DTX_OFF = "DTXoff" +ISM_MD_BASIC = "MDbasic" ISM_MD_EXTENDED = "MDextended" ISM_MD_NULL = "MDnull" +SBA_FOA_PCA_ON = "PCAon" +SBA_FOA_PCA_OFF = "PCAoff" + TESTVECTOR_FOR_INPUT_FORMAT = { "STEREO": "ltv{fs}_STEREO.wav", "MC_5_1": "ltv{fs}_MC51.wav", diff --git a/tests/codec_be_to_accepted_release/test_encoder.py b/tests/codec_be_to_accepted_release/test_encoder.py index 3ef664d240..344863e29c 100644 --- a/tests/codec_be_to_accepted_release/test_encoder.py +++ b/tests/codec_be_to_accepted_release/test_encoder.py @@ -93,54 +93,20 @@ def run_check( ### --------------- Actual testcase definitions --------------- -# "Vanilla" encoder modes +# channel-based modes STEREO_PARAMS = list( product(["STEREO"], BITRATES_STEREO, SAMPLING_RATES_ALL, [DTX_OFF, DTX_ON]) ) MC_PARAMS = list(product(INPUT_FORMATS_MC, BITRATES_ALL, SAMPLING_RATES_ALL, [DTX_OFF])) -SBA_PARAMS = list( - ( - product( - INPUT_FORMATS_SCENE_BASED, - BITRATES_ALL[:7], - SAMPLING_RATES_ALL, - [DTX_OFF, DTX_ON], - ) - ) -) + list( - ( - product( - INPUT_FORMATS_SCENE_BASED, BITRATES_ALL[7:], SAMPLING_RATES_ALL, [DTX_OFF] - ) - ) -) -ISM1_PARAMS = list( - product(["ISM1"], BITRATES_ISM1, SAMPLING_RATES_ALL, [DTX_OFF, DTX_ON]) -) -ISM2_PARAMS = list( - product(["ISM2"], BITRATES_ISM2, SAMPLING_RATES_ALL, [DTX_OFF, DTX_ON]) -) -ISM3_PARAMS = list( - product(["ISM3"], BITRATES_ISM3, SAMPLING_RATES_ALL, [DTX_OFF, DTX_ON]) -) -ISM4_PARAMS = list( - product(["ISM4"], BITRATES_ISM4, SAMPLING_RATES_ALL, [DTX_OFF, DTX_ON]) -) -# TODO: MASA -ENCODER_PARAMS = ( +ENCODER_CHANNEL_BASED_PARAMS = ( STEREO_PARAMS + MC_PARAMS - + SBA_PARAMS - + ISM1_PARAMS - + ISM2_PARAMS - + ISM3_PARAMS - + ISM4_PARAMS ) -@pytest.mark.parametrize("input_format,bitrate,sampling_rate,dtx", ENCODER_PARAMS) -def test_encoder( +@pytest.mark.parametrize("input_format,bitrate,sampling_rate,dtx", ENCODER_CHANNEL_BASED_PARAMS) +def test_encoder_channel_based_modes( input_format, bitrate, sampling_rate, @@ -164,8 +130,20 @@ def test_encoder( ) -# ISM "special" metadata cases: extended metadata and NULL +# ISM modes +ISM1_BASIC_NULL_PARAMS = list( + product(["ISM1"], BITRATES_ISM1, SAMPLING_RATES_ALL, [DTX_OFF, DTX_ON], [ISM_MD_BASIC, ISM_MD_NULL]) +) +ISM2_BASIC_NULL_PARAMS = list( + product(["ISM2"], BITRATES_ISM2, SAMPLING_RATES_ALL, [DTX_OFF, DTX_ON], [ISM_MD_BASIC, ISM_MD_NULL]) +) +ISM3_BASIC_NULL_PARAMS = list( + product(["ISM3"], BITRATES_ISM3, SAMPLING_RATES_ALL, [DTX_OFF, DTX_ON], [ISM_MD_BASIC, ISM_MD_NULL]) +) +ISM4_BASIC_NULL_PARAMS = list( + product(["ISM4"], BITRATES_ISM4, SAMPLING_RATES_ALL, [DTX_OFF, DTX_ON], [ISM_MD_BASIC, ISM_MD_NULL]) +) ISM1_EXTENDED_PARAMS = list( product( ["ISM1"], @@ -202,31 +180,11 @@ ISM4_EXTENDED_PARAMS = list( [ISM_MD_EXTENDED], ) ) -ISM1_NULL_PARAMS = list( - product( - ["ISM1"], BITRATES_ISM1, SAMPLING_RATES_ALL, [DTX_OFF, DTX_ON], [ISM_MD_NULL] - ) -) -ISM2_NULL_PARAMS = list( - product( - ["ISM2"], BITRATES_ISM2, SAMPLING_RATES_ALL, [DTX_OFF, DTX_ON], [ISM_MD_NULL] - ) -) -ISM3_NULL_PARAMS = list( - product( - ["ISM3"], BITRATES_ISM3, SAMPLING_RATES_ALL, [DTX_OFF, DTX_ON], [ISM_MD_NULL] - ) -) -ISM4_NULL_PARAMS = list( - product( - ["ISM4"], BITRATES_ISM4, SAMPLING_RATES_ALL, [DTX_OFF, DTX_ON], [ISM_MD_NULL] - ) -) -ISM_MD_SPECIAL_PARAMS = ( - ISM1_EXTENDED_PARAMS - + ISM2_EXTENDED_PARAMS - + ISM3_EXTENDED_PARAMS - + ISM4_EXTENDED_PARAMS +ISM_PARAMS = ( + ISM1_BASIC_NULL_PARAMS + + ISM2_BASIC_NULL_PARAMS + + ISM3_BASIC_NULL_PARAMS + + ISM4_BASIC_NULL_PARAMS + ISM1_EXTENDED_PARAMS + ISM2_EXTENDED_PARAMS + ISM3_EXTENDED_PARAMS @@ -235,9 +193,9 @@ ISM_MD_SPECIAL_PARAMS = ( @pytest.mark.parametrize( - "input_format,bitrate,sampling_rate,dtx,md_type", ISM_MD_SPECIAL_PARAMS + "input_format,bitrate,sampling_rate,dtx,md_type", ISM_PARAMS ) -def test_encoder_ism_MD_special( +def test_encoder_object_based_modes( input_format, bitrate, sampling_rate, @@ -266,18 +224,43 @@ def test_encoder_ism_MD_special( ) -@pytest.mark.parametrize("sampling_rate", SAMPLING_RATES_ALL) -def test_encoder_foa_pca_at_256kbps( +# SBA modes + +SBA_LOW_BITRATES_PARAMS = list( + ( + product( + INPUT_FORMATS_SCENE_BASED, + BITRATES_ALL[:7], + SAMPLING_RATES_ALL, + [DTX_OFF, DTX_ON], + [SBA_FOA_PCA_OFF], + ) + ) +) +SBA_HIGH_BITRATES_PARAMS = list( + ( + product( + INPUT_FORMATS_SCENE_BASED, BITRATES_ALL[7:], SAMPLING_RATES_ALL, [DTX_OFF], [SBA_FOA_PCA_OFF] + ) + ) +) +SBA_FOA_PCA_PARAMS = list(product(["FOA"], [256000], SAMPLING_RATES_ALL, [DTX_OFF], [SBA_FOA_PCA_ON])) +SBA_PARAMS = SBA_LOW_BITRATES_PARAMS + SBA_HIGH_BITRATES_PARAMS + SBA_FOA_PCA_PARAMS + +@pytest.mark.parametrize("input_format,bitrate,sampling_rate,dtx,pca", SBA_PARAMS) +def test_encoder_sba( + input_format, + bitrate, sampling_rate, + dtx, + pca, dut_encoder_frontend, update_ref ): - input_format = "FOA" - bitrate = 256000 - dtx = "DTXoff" - testv, options = get_encoder_inputs(input_format, sampling_rate) + if pca == SBA_FOA_PCA_ON: + options.extend(["-bypass", "2"]) bitstream = DUT_PATH.joinpath( - f"{testv.stem}-{input_format + '_pca'}-{bitrate}kbps-{sampling_rate}kHz-{dtx}.192" + f"{testv.stem}-{input_format}-{bitrate}kbps-{sampling_rate}kHz-{dtx}-{pca}.192" ) run_check(bitstream, bitrate, sampling_rate, testv, options, dut_encoder_frontend, update_ref == 1) \ No newline at end of file -- GitLab From fa8061304077ac67636e9edb5eefe78bb9e5c1c7 Mon Sep 17 00:00:00 2001 From: knj Date: Fri, 25 Aug 2023 15:17:25 +0200 Subject: [PATCH 005/106] add MASA encoder testcases --- .../codec_be_to_accepted_release/constants.py | 14 +++- .../test_encoder.py | 81 +++++++++++++------ 2 files changed, 69 insertions(+), 26 deletions(-) diff --git a/tests/codec_be_to_accepted_release/constants.py b/tests/codec_be_to_accepted_release/constants.py index 5870e66b34..511c2bdfa5 100644 --- a/tests/codec_be_to_accepted_release/constants.py +++ b/tests/codec_be_to_accepted_release/constants.py @@ -56,14 +56,20 @@ TESTVECTOR_FOR_INPUT_FORMAT = { "ISM2": "ltv{fs}_2ISM.wav", "ISM3": "ltv{fs}_3ISM.wav", "ISM4": "ltv{fs}_4ISM.wav", - # TODO: MASA, once created + "MASA_1TC_1DIR": "ltv{fs}_MASA_1TC_1DIR.wav", + "MASA_2TC_1DIR": "ltv{fs}_MASA_2TC_1DIR.wav", + "MASA_1TC_2DIR": "ltv{fs}_MASA_1TC_2DIR.wav", + "MASA_2TC_2DIR": "ltv{fs}_MASA_2TC_2DIR.wav", } METADATA_FOR_INPUT_FORMAT = { "ISM1": ["ltvISM1.csv"], "ISM2": ["ltvISM1.csv", "ltvISM2.csv"], "ISM3": ["ltvISM1.csv", "ltvISM2.csv", "ltvISM3.csv"], "ISM4": ["ltvISM1.csv", "ltvISM2.csv", "ltvISM3.csv", "ltvISM4.csv"], - # TODO: MASA, once created + "MASA_1TC_1DIR": ["ltv_MASA_1TC_1DIR.met"], + "MASA_2TC_1DIR": ["ltv_MASA_2TC_1DIR.met"], + "MASA_1TC_2DIR": ["ltv_MASA_1TC_2DIR.met"], + "MASA_2TC_2DIR": ["ltv_MASA_2TC_2DIR.met"], } CMDL_OPTIONS_FOR_INPUT_FORMAT = { "STEREO": ["-stereo"], @@ -82,6 +88,10 @@ CMDL_OPTIONS_FOR_INPUT_FORMAT = { "ISM2": ["-ism", "2"], "ISM3": ["-ism", "3"], "ISM4": ["-ism", "4"], + "MASA_1TC_1DIR": ["-masa", "1"], + "MASA_2TC_1DIR": ["-masa", "2"], + "MASA_1TC_2DIR": ["-masa", "1"], + "MASA_2TC_2DIR": ["-masa", "2"], } ### Test case parameters diff --git a/tests/codec_be_to_accepted_release/test_encoder.py b/tests/codec_be_to_accepted_release/test_encoder.py index 344863e29c..aebdd0527c 100644 --- a/tests/codec_be_to_accepted_release/test_encoder.py +++ b/tests/codec_be_to_accepted_release/test_encoder.py @@ -93,20 +93,22 @@ def run_check( ### --------------- Actual testcase definitions --------------- -# channel-based modes +# channel-based modes + MASA STEREO_PARAMS = list( product(["STEREO"], BITRATES_STEREO, SAMPLING_RATES_ALL, [DTX_OFF, DTX_ON]) ) MC_PARAMS = list(product(INPUT_FORMATS_MC, BITRATES_ALL, SAMPLING_RATES_ALL, [DTX_OFF])) -ENCODER_CHANNEL_BASED_PARAMS = ( - STEREO_PARAMS - + MC_PARAMS +MASA_PARAMS = list( + product(INPUT_FORMATS_MASA, BITRATES_ALL, SAMPLING_RATES_ALL, [DTX_OFF, DTX_ON]) ) +ENCODER_CHANNEL_BASED_AND_MASA_PARAMS = STEREO_PARAMS + MC_PARAMS + MASA_PARAMS -@pytest.mark.parametrize("input_format,bitrate,sampling_rate,dtx", ENCODER_CHANNEL_BASED_PARAMS) -def test_encoder_channel_based_modes( +@pytest.mark.parametrize( + "input_format,bitrate,sampling_rate,dtx", ENCODER_CHANNEL_BASED_AND_MASA_PARAMS +) +def test_encoder_channel_based_and_masa_modes( input_format, bitrate, sampling_rate, @@ -130,19 +132,43 @@ def test_encoder_channel_based_modes( ) -# ISM modes +# (single) modes with metadata: ISM and MASA ISM1_BASIC_NULL_PARAMS = list( - product(["ISM1"], BITRATES_ISM1, SAMPLING_RATES_ALL, [DTX_OFF, DTX_ON], [ISM_MD_BASIC, ISM_MD_NULL]) + product( + ["ISM1"], + BITRATES_ISM1, + SAMPLING_RATES_ALL, + [DTX_OFF, DTX_ON], + [ISM_MD_BASIC, ISM_MD_NULL], + ) ) ISM2_BASIC_NULL_PARAMS = list( - product(["ISM2"], BITRATES_ISM2, SAMPLING_RATES_ALL, [DTX_OFF, DTX_ON], [ISM_MD_BASIC, ISM_MD_NULL]) + product( + ["ISM2"], + BITRATES_ISM2, + SAMPLING_RATES_ALL, + [DTX_OFF, DTX_ON], + [ISM_MD_BASIC, ISM_MD_NULL], + ) ) ISM3_BASIC_NULL_PARAMS = list( - product(["ISM3"], BITRATES_ISM3, SAMPLING_RATES_ALL, [DTX_OFF, DTX_ON], [ISM_MD_BASIC, ISM_MD_NULL]) + product( + ["ISM3"], + BITRATES_ISM3, + SAMPLING_RATES_ALL, + [DTX_OFF, DTX_ON], + [ISM_MD_BASIC, ISM_MD_NULL], + ) ) ISM4_BASIC_NULL_PARAMS = list( - product(["ISM4"], BITRATES_ISM4, SAMPLING_RATES_ALL, [DTX_OFF, DTX_ON], [ISM_MD_BASIC, ISM_MD_NULL]) + product( + ["ISM4"], + BITRATES_ISM4, + SAMPLING_RATES_ALL, + [DTX_OFF, DTX_ON], + [ISM_MD_BASIC, ISM_MD_NULL], + ) ) ISM1_EXTENDED_PARAMS = list( product( @@ -192,9 +218,7 @@ ISM_PARAMS = ( ) -@pytest.mark.parametrize( - "input_format,bitrate,sampling_rate,dtx,md_type", ISM_PARAMS -) +@pytest.mark.parametrize("input_format,bitrate,sampling_rate,dtx,md_type", ISM_PARAMS) def test_encoder_object_based_modes( input_format, bitrate, @@ -240,22 +264,23 @@ SBA_LOW_BITRATES_PARAMS = list( SBA_HIGH_BITRATES_PARAMS = list( ( product( - INPUT_FORMATS_SCENE_BASED, BITRATES_ALL[7:], SAMPLING_RATES_ALL, [DTX_OFF], [SBA_FOA_PCA_OFF] + INPUT_FORMATS_SCENE_BASED, + BITRATES_ALL[7:], + SAMPLING_RATES_ALL, + [DTX_OFF], + [SBA_FOA_PCA_OFF], ) ) ) -SBA_FOA_PCA_PARAMS = list(product(["FOA"], [256000], SAMPLING_RATES_ALL, [DTX_OFF], [SBA_FOA_PCA_ON])) +SBA_FOA_PCA_PARAMS = list( + product(["FOA"], [256000], SAMPLING_RATES_ALL, [DTX_OFF], [SBA_FOA_PCA_ON]) +) SBA_PARAMS = SBA_LOW_BITRATES_PARAMS + SBA_HIGH_BITRATES_PARAMS + SBA_FOA_PCA_PARAMS + @pytest.mark.parametrize("input_format,bitrate,sampling_rate,dtx,pca", SBA_PARAMS) def test_encoder_sba( - input_format, - bitrate, - sampling_rate, - dtx, - pca, - dut_encoder_frontend, - update_ref + input_format, bitrate, sampling_rate, dtx, pca, dut_encoder_frontend, update_ref ): testv, options = get_encoder_inputs(input_format, sampling_rate) if pca == SBA_FOA_PCA_ON: @@ -263,4 +288,12 @@ def test_encoder_sba( bitstream = DUT_PATH.joinpath( f"{testv.stem}-{input_format}-{bitrate}kbps-{sampling_rate}kHz-{dtx}-{pca}.192" ) - run_check(bitstream, bitrate, sampling_rate, testv, options, dut_encoder_frontend, update_ref == 1) \ No newline at end of file + run_check( + bitstream, + bitrate, + sampling_rate, + testv, + options, + dut_encoder_frontend, + update_ref == 1, + ) -- GitLab From 9311766ae13f4be1226f2615b26a097b9bc6f2f6 Mon Sep 17 00:00:00 2001 From: knj Date: Fri, 25 Aug 2023 15:42:40 +0200 Subject: [PATCH 006/106] add StereoDmxEvs modes --- .../codec_be_to_accepted_release/constants.py | 1 + .../test_encoder.py | 31 ++++++++++++++++++- 2 files changed, 31 insertions(+), 1 deletion(-) diff --git a/tests/codec_be_to_accepted_release/constants.py b/tests/codec_be_to_accepted_release/constants.py index 511c2bdfa5..96484b5f05 100644 --- a/tests/codec_be_to_accepted_release/constants.py +++ b/tests/codec_be_to_accepted_release/constants.py @@ -138,6 +138,7 @@ BITRATES_ISM1_EXTENDED = [b for b in BITRATES_ISM1 if b > 64000] BITRATES_ISM2_EXTENDED = [b for b in BITRATES_ISM2 if b > 64000] BITRATES_ISM3_EXTENDED = [b for b in BITRATES_ISM3 if b > 64000] BITRATES_ISM4_EXTENDED = [b for b in BITRATES_ISM4 if b > 64000] +BITRATES_EVS = [5900, 7200, 8000, 9600] + BITRATES_ALL[:6] + BITRATES_ALL[7:9] # sampling rates SAMPLING_RATES_ALL = [16, 32, 48] diff --git a/tests/codec_be_to_accepted_release/test_encoder.py b/tests/codec_be_to_accepted_release/test_encoder.py index aebdd0527c..6acc0d6f68 100644 --- a/tests/codec_be_to_accepted_release/test_encoder.py +++ b/tests/codec_be_to_accepted_release/test_encoder.py @@ -132,8 +132,9 @@ def test_encoder_channel_based_and_masa_modes( ) -# (single) modes with metadata: ISM and MASA +# ISM modes with three "flavours" of metadata +# different ranges of bitrates allowed depending on number of objects, ext MD only for >64kbps ISM1_BASIC_NULL_PARAMS = list( product( ["ISM1"], @@ -250,6 +251,7 @@ def test_encoder_object_based_modes( # SBA modes +# dtx only allowed at low bitrates, PCA only at FOA 256kbps SBA_LOW_BITRATES_PARAMS = list( ( product( @@ -297,3 +299,30 @@ def test_encoder_sba( dut_encoder_frontend, update_ref == 1, ) + + +# All EVS bitrates, 5900 VBR always needs -dtx +STEREO_DMX_EVS_PARAMS = list( + product(BITRATES_EVS[1:], SAMPLING_RATES_ALL, [DTX_OFF, DTX_ON]) +) + list(product(BITRATES_EVS[:1], SAMPLING_RATES_ALL, [DTX_ON])) + + +@pytest.mark.parametrize("bitrate,sampling_rate,dtx", STEREO_DMX_EVS_PARAMS) +def test_encoder_stereo_dmx_evs( + bitrate, sampling_rate, dtx, dut_encoder_frontend, update_ref +): + testv, _ = get_encoder_inputs("STEREO", sampling_rate) + options = ["-stereo_dmx_evs"] + bitstream = DUT_PATH.joinpath( + f"{testv.stem}-{'StereoDmxEvs'}-{bitrate}kbps-{sampling_rate}kHz-{dtx}.192" + ) + run_check( + bitstream, + bitrate, + sampling_rate, + testv, + options, + dut_encoder_frontend, + update_ref == 1, + dtx == DTX_ON, + ) -- GitLab From b61c52fb1fe6e0b965818dcada6b74282e22d2ac Mon Sep 17 00:00:00 2001 From: knj Date: Fri, 25 Aug 2023 16:07:30 +0200 Subject: [PATCH 007/106] some refactoring for clarity and reuse in decoder tests --- .../codec_be_to_accepted_release/__init__.py | 7 ++++ .../test_encoder.py | 41 +++++++++---------- 2 files changed, 26 insertions(+), 22 deletions(-) diff --git a/tests/codec_be_to_accepted_release/__init__.py b/tests/codec_be_to_accepted_release/__init__.py index fe81f4c45f..68b209e01b 100644 --- a/tests/codec_be_to_accepted_release/__init__.py +++ b/tests/codec_be_to_accepted_release/__init__.py @@ -54,3 +54,10 @@ def is_be_to_ref(dut_file): is_be = md5_ref == md5_dut return is_be + + +def get_bitstream_path( + base_path, testv_name, encoder_format, bitrate, sampling_rate, dtx, suffix="" +): + bs_name = f"{testv_name}-{encoder_format}-{bitrate}kbps-{sampling_rate}kHz-{dtx}{suffix}.192" + return base_path.joinpath(bs_name) diff --git a/tests/codec_be_to_accepted_release/test_encoder.py b/tests/codec_be_to_accepted_release/test_encoder.py index 6acc0d6f68..8b049b3daf 100644 --- a/tests/codec_be_to_accepted_release/test_encoder.py +++ b/tests/codec_be_to_accepted_release/test_encoder.py @@ -31,13 +31,13 @@ the United Nations Convention on Contracts on the International Sales of Goods. import pytest from itertools import product from .constants import * -from . import TESTV_PATH, DUT_PATH, is_be_to_ref +from . import TESTV_PATH, DUT_PATH, is_be_to_ref, get_bitstream_path ### --------------- Helper functions --------------- -def get_testv(input_format, sampling_rate): +def get_testv_path(input_format, sampling_rate): testv = TESTVECTOR_FOR_INPUT_FORMAT[input_format].format(fs=sampling_rate) return TESTV_PATH.joinpath(testv) @@ -56,17 +56,11 @@ def get_options(input_format, md_type=None): if md_type == ISM_MD_EXTENDED: assert input_format in INPUT_FORMATS_OBJECT_BASED options[1] = f"+{options[1]}" + md_options = get_md(input_format, md_type) + options.extend(md_options) return options -def get_encoder_inputs(input_format, sampling_rate, md_type=None): - testv = get_testv(input_format, sampling_rate) - options = get_options(input_format, md_type) - md_files = get_md(input_format, md_type) - options.extend(md_files) - return testv, options - - def run_check( dut_bitstream, bitrate, @@ -116,9 +110,10 @@ def test_encoder_channel_based_and_masa_modes( dut_encoder_frontend, update_ref, ): - testv, options = get_encoder_inputs(input_format, sampling_rate) - bitstream = DUT_PATH.joinpath( - f"{testv.stem}-{input_format}-{bitrate}kbps-{sampling_rate}kHz-{dtx}.192" + testv = get_testv_path(input_format, sampling_rate) + options = get_options(input_format) + bitstream = get_bitstream_path( + DUT_PATH, testv.stem, input_format, bitrate, sampling_rate, dtx ) run_check( bitstream, @@ -229,13 +224,14 @@ def test_encoder_object_based_modes( dut_encoder_frontend, update_ref, ): - testv, options = get_encoder_inputs(input_format, sampling_rate, md_type) + testv = get_testv_path(input_format, sampling_rate) + options = get_options(input_format, md_type) if md_type == ISM_MD_EXTENDED: input_format += "_ext_MD" elif md_type == ISM_MD_NULL: input_format += "_null_MD" - bitstream = DUT_PATH.joinpath( - f"{testv.stem}-{input_format}-{bitrate}kbps-{sampling_rate}kHz-{dtx}.192" + bitstream = get_bitstream_path( + DUT_PATH, testv.stem, input_format, bitrate, sampling_rate, dtx ) run_check( bitstream, @@ -284,11 +280,12 @@ SBA_PARAMS = SBA_LOW_BITRATES_PARAMS + SBA_HIGH_BITRATES_PARAMS + SBA_FOA_PCA_PA def test_encoder_sba( input_format, bitrate, sampling_rate, dtx, pca, dut_encoder_frontend, update_ref ): - testv, options = get_encoder_inputs(input_format, sampling_rate) + testv = get_testv_path(input_format, sampling_rate) + options = get_options(input_format) if pca == SBA_FOA_PCA_ON: options.extend(["-bypass", "2"]) - bitstream = DUT_PATH.joinpath( - f"{testv.stem}-{input_format}-{bitrate}kbps-{sampling_rate}kHz-{dtx}-{pca}.192" + bitstream = get_bitstream_path( + DUT_PATH, testv.stem, input_format, bitrate, sampling_rate, dtx, "-pca" ) run_check( bitstream, @@ -311,10 +308,10 @@ STEREO_DMX_EVS_PARAMS = list( def test_encoder_stereo_dmx_evs( bitrate, sampling_rate, dtx, dut_encoder_frontend, update_ref ): - testv, _ = get_encoder_inputs("STEREO", sampling_rate) + testv = get_testv_path("STEREO", sampling_rate) options = ["-stereo_dmx_evs"] - bitstream = DUT_PATH.joinpath( - f"{testv.stem}-{'StereoDmxEvs'}-{bitrate}kbps-{sampling_rate}kHz-{dtx}.192" + bitstream = get_bitstream_path( + DUT_PATH, testv.stem, "StereoDmxEvs", bitrate, sampling_rate, dtx ) run_check( bitstream, -- GitLab From 71e6fb1cba9281df082aedf47b1ebcb039d40daf Mon Sep 17 00:00:00 2001 From: knj Date: Fri, 25 Aug 2023 17:55:10 +0200 Subject: [PATCH 008/106] first decoder testcases --- .../codec_be_to_accepted_release/__init__.py | 6 + .../codec_be_to_accepted_release/constants.py | 180 +++++++++++++++++- .../test_decoder.py | 92 +++++++++ .../test_encoder.py | 163 ++-------------- 4 files changed, 292 insertions(+), 149 deletions(-) create mode 100644 tests/codec_be_to_accepted_release/test_decoder.py diff --git a/tests/codec_be_to_accepted_release/__init__.py b/tests/codec_be_to_accepted_release/__init__.py index 68b209e01b..eedb33110c 100644 --- a/tests/codec_be_to_accepted_release/__init__.py +++ b/tests/codec_be_to_accepted_release/__init__.py @@ -33,6 +33,7 @@ from pathlib import Path import filecmp import subprocess from ..testconfig import MD5_REF_DICT +from .constants import TESTVECTOR_FOR_INPUT_FORMAT HERE = Path(__file__).parent TESTV_PATH = Path(os.environ.get("BE_TESTV_DIR", HERE.joinpath("testv"))) @@ -61,3 +62,8 @@ def get_bitstream_path( ): bs_name = f"{testv_name}-{encoder_format}-{bitrate}kbps-{sampling_rate}kHz-{dtx}{suffix}.192" return base_path.joinpath(bs_name) + + +def get_testv_path(input_format, sampling_rate): + testv = TESTVECTOR_FOR_INPUT_FORMAT[input_format].format(fs=sampling_rate) + return TESTV_PATH.joinpath(testv) diff --git a/tests/codec_be_to_accepted_release/constants.py b/tests/codec_be_to_accepted_release/constants.py index 96484b5f05..8de597871b 100644 --- a/tests/codec_be_to_accepted_release/constants.py +++ b/tests/codec_be_to_accepted_release/constants.py @@ -28,6 +28,13 @@ accordance with the laws of the Federal Republic of Germany excluding its confli the United Nations Convention on Contracts on the International Sales of Goods. """ +from itertools import product + + +def collapse_into_list_of_pairs(params): + return [p[0] + p[1:] for p in params] + + DTX_ON = "DTXon" DTX_OFF = "DTXoff" @@ -107,13 +114,11 @@ INPUT_FORMATS_MASA = [ "MASA_1TC_2DIR", "MASA_2TC_2DIR", ] -# TODO: add MASA back INPUT_FORMATS_SINGLE = ( INPUT_FORMATS_CHANNEL_BASED + INPUT_FORMATS_SCENE_BASED + INPUT_FORMATS_OBJECT_BASED ) # + INPUT_FORMATS_MASA # TODO: combined formats -# bitrates BITRATES_ALL = [ 13200, 16400, @@ -140,5 +145,174 @@ BITRATES_ISM3_EXTENDED = [b for b in BITRATES_ISM3 if b > 64000] BITRATES_ISM4_EXTENDED = [b for b in BITRATES_ISM4 if b > 64000] BITRATES_EVS = [5900, 7200, 8000, 9600] + BITRATES_ALL[:6] + BITRATES_ALL[7:9] -# sampling rates SAMPLING_RATES_ALL = [16, 32, 48] + +OUTPUT_FORMATS_ALL = [ + "MONO", + "STEREO", + "5_1", + "5_1_2", + "5_1_4", + "7_1", + "7_1_4", + "FOA", + "HOA2", + "HOA3", + "EXT", + "BINAURAL", + "BINAURAL_ROOM_IR", + "BINAURAL_ROOM_REVERB", +] +OUTPUT_FORMATS_BINAURAL = OUTPUT_FORMATS_ALL[-3:] + + +### -------------- encoder parameter lists -------------- + +STEREO_PARAMS = list( + product(["STEREO"], BITRATES_STEREO, SAMPLING_RATES_ALL, [DTX_OFF, DTX_ON]) +) +MC_PARAMS = list(product(INPUT_FORMATS_MC, BITRATES_ALL, SAMPLING_RATES_ALL, [DTX_OFF])) +MASA_PARAMS = list( + product(INPUT_FORMATS_MASA, BITRATES_ALL, SAMPLING_RATES_ALL, [DTX_OFF, DTX_ON]) +) +ENCODER_CHANNEL_BASED_AND_MASA_PARAMS = STEREO_PARAMS + MC_PARAMS + MASA_PARAMS + + +# ISM modes with three "flavours" of metadata +# different ranges of bitrates allowed depending on number of objects, ext MD only for >64kbps +ISM1_BASIC_NULL_PARAMS = list( + product( + ["ISM1"], + BITRATES_ISM1, + SAMPLING_RATES_ALL, + [DTX_OFF, DTX_ON], + [ISM_MD_BASIC, ISM_MD_NULL], + ) +) +ISM2_BASIC_NULL_PARAMS = list( + product( + ["ISM2"], + BITRATES_ISM2, + SAMPLING_RATES_ALL, + [DTX_OFF, DTX_ON], + [ISM_MD_BASIC, ISM_MD_NULL], + ) +) +ISM3_BASIC_NULL_PARAMS = list( + product( + ["ISM3"], + BITRATES_ISM3, + SAMPLING_RATES_ALL, + [DTX_OFF, DTX_ON], + [ISM_MD_BASIC, ISM_MD_NULL], + ) +) +ISM4_BASIC_NULL_PARAMS = list( + product( + ["ISM4"], + BITRATES_ISM4, + SAMPLING_RATES_ALL, + [DTX_OFF, DTX_ON], + [ISM_MD_BASIC, ISM_MD_NULL], + ) +) +ISM1_EXTENDED_PARAMS = list( + product( + ["ISM1"], + BITRATES_ISM1_EXTENDED, + SAMPLING_RATES_ALL, + [DTX_OFF, DTX_ON], + [ISM_MD_EXTENDED], + ) +) +ISM2_EXTENDED_PARAMS = list( + product( + ["ISM2"], + BITRATES_ISM2_EXTENDED, + SAMPLING_RATES_ALL, + [DTX_OFF, DTX_ON], + [ISM_MD_EXTENDED], + ) +) +ISM3_EXTENDED_PARAMS = list( + product( + ["ISM3"], + BITRATES_ISM3_EXTENDED, + SAMPLING_RATES_ALL, + [DTX_OFF, DTX_ON], + [ISM_MD_EXTENDED], + ) +) +ISM4_EXTENDED_PARAMS = list( + product( + ["ISM4"], + BITRATES_ISM4_EXTENDED, + SAMPLING_RATES_ALL, + [DTX_OFF, DTX_ON], + [ISM_MD_EXTENDED], + ) +) +ISM_PARAMS = ( + ISM1_BASIC_NULL_PARAMS + + ISM2_BASIC_NULL_PARAMS + + ISM3_BASIC_NULL_PARAMS + + ISM4_BASIC_NULL_PARAMS + + ISM1_EXTENDED_PARAMS + + ISM2_EXTENDED_PARAMS + + ISM3_EXTENDED_PARAMS + + ISM4_EXTENDED_PARAMS +) + + +# dtx only allowed at low bitrates, PCA only at FOA 256kbps +SBA_LOW_BITRATES_PARAMS = list( + ( + product( + INPUT_FORMATS_SCENE_BASED, + BITRATES_ALL[:7], + SAMPLING_RATES_ALL, + [DTX_OFF, DTX_ON], + [SBA_FOA_PCA_OFF], + ) + ) +) +SBA_HIGH_BITRATES_PARAMS = list( + ( + product( + INPUT_FORMATS_SCENE_BASED, + BITRATES_ALL[7:], + SAMPLING_RATES_ALL, + [DTX_OFF], + [SBA_FOA_PCA_OFF], + ) + ) +) +SBA_FOA_PCA_PARAMS = list( + product(["FOA"], [256000], SAMPLING_RATES_ALL, [DTX_OFF], [SBA_FOA_PCA_ON]) +) +SBA_PARAMS = SBA_LOW_BITRATES_PARAMS + SBA_HIGH_BITRATES_PARAMS + SBA_FOA_PCA_PARAMS + + +# All EVS bitrates, 5900 VBR always needs -dtx +STEREO_DMX_EVS_PARAMS = list( + product(BITRATES_EVS[1:], SAMPLING_RATES_ALL, [DTX_OFF, DTX_ON]) +) + list(product(BITRATES_EVS[:1], SAMPLING_RATES_ALL, [DTX_ON])) + + +### -------------- decoder parameter lists -------------- + +DECODER_CLEAN_CHANNEL_CHANNELBASED_AND_MASA_PARAMS = ( + collapse_into_list_of_pairs( + product(STEREO_PARAMS, OUTPUT_FORMATS_ALL[:2], SAMPLING_RATES_ALL) + ) + + collapse_into_list_of_pairs( + product( + MC_PARAMS, + OUTPUT_FORMATS_ALL[:10] + OUTPUT_FORMATS_BINAURAL, + SAMPLING_RATES_ALL, + ) + ) + + collapse_into_list_of_pairs( + product(MASA_PARAMS, OUTPUT_FORMATS_ALL, SAMPLING_RATES_ALL) + ) +) diff --git a/tests/codec_be_to_accepted_release/test_decoder.py b/tests/codec_be_to_accepted_release/test_decoder.py new file mode 100644 index 0000000000..775b786834 --- /dev/null +++ b/tests/codec_be_to_accepted_release/test_decoder.py @@ -0,0 +1,92 @@ +__copyright__ = """ +(C) 2022-2023 IVAS codec Public Collaboration with portions copyright Dolby International AB, Ericsson AB, +Fraunhofer-Gesellschaft zur Foerderung der angewandten Forschung e.V., Huawei Technologies Co. LTD., +Koninklijke Philips N.V., Nippon Telegraph and Telephone Corporation, Nokia Technologies Oy, Orange, +Panasonic Holdings Corporation, Qualcomm Technologies, Inc., VoiceAge Corporation, and other +contributors to this repository. All Rights Reserved. + +This software is protected by copyright law and by international treaties. +The IVAS codec Public Collaboration consisting of Dolby International AB, Ericsson AB, +Fraunhofer-Gesellschaft zur Foerderung der angewandten Forschung e.V., Huawei Technologies Co. LTD., +Koninklijke Philips N.V., Nippon Telegraph and Telephone Corporation, Nokia Technologies Oy, Orange, +Panasonic Holdings Corporation, Qualcomm Technologies, Inc., VoiceAge Corporation, and other +contributors to this repository retain full ownership rights in their respective contributions in +the software. This notice grants no license of any kind, including but not limited to patent +license, nor is any license granted by implication, estoppel or otherwise. + +Contributors are required to enter into the IVAS codec Public Collaboration agreement before making +contributions. + +This software is provided "AS IS", without any express or implied warranties. The software is in the +development stage. It is intended exclusively for experts who have experience with such software and +solely for the purpose of inspection. All implied warranties of non-infringement, merchantability +and fitness for a particular purpose are hereby disclaimed and excluded. + +Any dispute, controversy or claim arising under or in relation to providing this software shall be +submitted to and settled by the final, binding jurisdiction of the courts of Munich, Germany in +accordance with the laws of the Federal Republic of Germany excluding its conflict of law rules and +the United Nations Convention on Contracts on the International Sales of Goods. +""" + +import pytest +from . import is_be_to_ref, get_bitstream_path, get_testv_path, REF_PATH, DUT_PATH +from .constants import * + + +### --------------- Helper functions --------------- + + +def get_output_path(bitstream_path, output_format, output_sampling_rate): + output_name = ( + f"{bitstream_path.stem}.dec-{output_format}-{output_sampling_rate}kHz.wav" + ) + return DUT_PATH.joinpath(output_name) + + +def run_check( + ref_bitstream, + output_format, + sampling_rate, + dut_output, + decoder_frontend, + is_ref_creation, +): + decoder_frontend.run(output_format, sampling_rate, ref_bitstream, dut_output) + + if not is_ref_creation and not is_be_to_ref(dut_output): + pytest.fail(f"Decoder output differs from reference") + + +### --------------- Actual testcase definitions --------------- + +# clean channel decoding + + +@pytest.mark.parametrize( + "input_format,bitrate,input_sampling_rate,dtx,output_format,output_sampling_rate", + DECODER_CLEAN_CHANNEL_CHANNELBASED_AND_MASA_PARAMS, +) +def test_decoder_clean_channel( + input_format, + bitrate, + input_sampling_rate, + dtx, + output_format, + output_sampling_rate, + dut_decoder_frontend, + update_ref, +): + testv_name = get_testv_path(input_format, input_sampling_rate).stem + ref_bitstream = get_bitstream_path( + REF_PATH, testv_name, input_format, bitrate, input_sampling_rate, dtx + ) + dut_output = get_output_path(ref_bitstream, output_format, output_sampling_rate) + + run_check( + str(ref_bitstream), + output_format, + output_sampling_rate, + str(dut_output), + dut_decoder_frontend, + update_ref == 1, + ) diff --git a/tests/codec_be_to_accepted_release/test_encoder.py b/tests/codec_be_to_accepted_release/test_encoder.py index 8b049b3daf..13d2247250 100644 --- a/tests/codec_be_to_accepted_release/test_encoder.py +++ b/tests/codec_be_to_accepted_release/test_encoder.py @@ -29,19 +29,25 @@ the United Nations Convention on Contracts on the International Sales of Goods. """ import pytest -from itertools import product -from .constants import * -from . import TESTV_PATH, DUT_PATH, is_be_to_ref, get_bitstream_path +from .constants import ( + METADATA_FOR_INPUT_FORMAT, + ISM_MD_NULL, + CMDL_OPTIONS_FOR_INPUT_FORMAT, + ISM_MD_EXTENDED, + INPUT_FORMATS_OBJECT_BASED, + ENCODER_CHANNEL_BASED_AND_MASA_PARAMS, + DTX_ON, + ISM_PARAMS, + SBA_PARAMS, + SBA_FOA_PCA_ON, + STEREO_DMX_EVS_PARAMS, +) +from . import TESTV_PATH, DUT_PATH, is_be_to_ref, get_bitstream_path, get_testv_path ### --------------- Helper functions --------------- -def get_testv_path(input_format, sampling_rate): - testv = TESTVECTOR_FOR_INPUT_FORMAT[input_format].format(fs=sampling_rate) - return TESTV_PATH.joinpath(testv) - - def get_md(input_format, md_type=None): md_files = METADATA_FOR_INPUT_FORMAT.get(input_format, list()) if md_type == ISM_MD_NULL: @@ -80,25 +86,14 @@ def run_check( add_option_list=options, ) - if not is_ref_creation: - if not is_be_to_ref(dut_bitstream): - pytest.fail(f"Bitstream file differs from reference") + if not is_ref_creation and not is_be_to_ref(dut_bitstream): + pytest.fail(f"Bitstream file differs from reference") ### --------------- Actual testcase definitions --------------- -# channel-based modes + MASA - -STEREO_PARAMS = list( - product(["STEREO"], BITRATES_STEREO, SAMPLING_RATES_ALL, [DTX_OFF, DTX_ON]) -) -MC_PARAMS = list(product(INPUT_FORMATS_MC, BITRATES_ALL, SAMPLING_RATES_ALL, [DTX_OFF])) -MASA_PARAMS = list( - product(INPUT_FORMATS_MASA, BITRATES_ALL, SAMPLING_RATES_ALL, [DTX_OFF, DTX_ON]) -) -ENCODER_CHANNEL_BASED_AND_MASA_PARAMS = STEREO_PARAMS + MC_PARAMS + MASA_PARAMS - +# channel-based modes + MASA @pytest.mark.parametrize( "input_format,bitrate,sampling_rate,dtx", ENCODER_CHANNEL_BASED_AND_MASA_PARAMS ) @@ -127,93 +122,6 @@ def test_encoder_channel_based_and_masa_modes( ) -# ISM modes with three "flavours" of metadata - -# different ranges of bitrates allowed depending on number of objects, ext MD only for >64kbps -ISM1_BASIC_NULL_PARAMS = list( - product( - ["ISM1"], - BITRATES_ISM1, - SAMPLING_RATES_ALL, - [DTX_OFF, DTX_ON], - [ISM_MD_BASIC, ISM_MD_NULL], - ) -) -ISM2_BASIC_NULL_PARAMS = list( - product( - ["ISM2"], - BITRATES_ISM2, - SAMPLING_RATES_ALL, - [DTX_OFF, DTX_ON], - [ISM_MD_BASIC, ISM_MD_NULL], - ) -) -ISM3_BASIC_NULL_PARAMS = list( - product( - ["ISM3"], - BITRATES_ISM3, - SAMPLING_RATES_ALL, - [DTX_OFF, DTX_ON], - [ISM_MD_BASIC, ISM_MD_NULL], - ) -) -ISM4_BASIC_NULL_PARAMS = list( - product( - ["ISM4"], - BITRATES_ISM4, - SAMPLING_RATES_ALL, - [DTX_OFF, DTX_ON], - [ISM_MD_BASIC, ISM_MD_NULL], - ) -) -ISM1_EXTENDED_PARAMS = list( - product( - ["ISM1"], - BITRATES_ISM1_EXTENDED, - SAMPLING_RATES_ALL, - [DTX_OFF, DTX_ON], - [ISM_MD_EXTENDED], - ) -) -ISM2_EXTENDED_PARAMS = list( - product( - ["ISM2"], - BITRATES_ISM2_EXTENDED, - SAMPLING_RATES_ALL, - [DTX_OFF, DTX_ON], - [ISM_MD_EXTENDED], - ) -) -ISM3_EXTENDED_PARAMS = list( - product( - ["ISM3"], - BITRATES_ISM3_EXTENDED, - SAMPLING_RATES_ALL, - [DTX_OFF, DTX_ON], - [ISM_MD_EXTENDED], - ) -) -ISM4_EXTENDED_PARAMS = list( - product( - ["ISM4"], - BITRATES_ISM4_EXTENDED, - SAMPLING_RATES_ALL, - [DTX_OFF, DTX_ON], - [ISM_MD_EXTENDED], - ) -) -ISM_PARAMS = ( - ISM1_BASIC_NULL_PARAMS - + ISM2_BASIC_NULL_PARAMS - + ISM3_BASIC_NULL_PARAMS - + ISM4_BASIC_NULL_PARAMS - + ISM1_EXTENDED_PARAMS - + ISM2_EXTENDED_PARAMS - + ISM3_EXTENDED_PARAMS - + ISM4_EXTENDED_PARAMS -) - - @pytest.mark.parametrize("input_format,bitrate,sampling_rate,dtx,md_type", ISM_PARAMS) def test_encoder_object_based_modes( input_format, @@ -245,37 +153,6 @@ def test_encoder_object_based_modes( ) -# SBA modes - -# dtx only allowed at low bitrates, PCA only at FOA 256kbps -SBA_LOW_BITRATES_PARAMS = list( - ( - product( - INPUT_FORMATS_SCENE_BASED, - BITRATES_ALL[:7], - SAMPLING_RATES_ALL, - [DTX_OFF, DTX_ON], - [SBA_FOA_PCA_OFF], - ) - ) -) -SBA_HIGH_BITRATES_PARAMS = list( - ( - product( - INPUT_FORMATS_SCENE_BASED, - BITRATES_ALL[7:], - SAMPLING_RATES_ALL, - [DTX_OFF], - [SBA_FOA_PCA_OFF], - ) - ) -) -SBA_FOA_PCA_PARAMS = list( - product(["FOA"], [256000], SAMPLING_RATES_ALL, [DTX_OFF], [SBA_FOA_PCA_ON]) -) -SBA_PARAMS = SBA_LOW_BITRATES_PARAMS + SBA_HIGH_BITRATES_PARAMS + SBA_FOA_PCA_PARAMS - - @pytest.mark.parametrize("input_format,bitrate,sampling_rate,dtx,pca", SBA_PARAMS) def test_encoder_sba( input_format, bitrate, sampling_rate, dtx, pca, dut_encoder_frontend, update_ref @@ -298,12 +175,6 @@ def test_encoder_sba( ) -# All EVS bitrates, 5900 VBR always needs -dtx -STEREO_DMX_EVS_PARAMS = list( - product(BITRATES_EVS[1:], SAMPLING_RATES_ALL, [DTX_OFF, DTX_ON]) -) + list(product(BITRATES_EVS[:1], SAMPLING_RATES_ALL, [DTX_ON])) - - @pytest.mark.parametrize("bitrate,sampling_rate,dtx", STEREO_DMX_EVS_PARAMS) def test_encoder_stereo_dmx_evs( bitrate, sampling_rate, dtx, dut_encoder_frontend, update_ref -- GitLab From 896b9372c844416dc07c83ad0477f5056455426e Mon Sep 17 00:00:00 2001 From: knj Date: Mon, 28 Aug 2023 11:33:02 +0200 Subject: [PATCH 009/106] first version of decoder tests --- .../codec_be_to_accepted_release/constants.py | 4 +- .../test_decoder.py | 70 ++++++++++++++++++- .../test_encoder.py | 11 +-- 3 files changed, 79 insertions(+), 6 deletions(-) diff --git a/tests/codec_be_to_accepted_release/constants.py b/tests/codec_be_to_accepted_release/constants.py index 8de597871b..67d0705109 100644 --- a/tests/codec_be_to_accepted_release/constants.py +++ b/tests/codec_be_to_accepted_release/constants.py @@ -134,7 +134,7 @@ BITRATES_ALL = [ 384000, 512000, ] -BITRATES_STEREO = BITRATES_ALL[:10] +BITRATES_STEREO = BITRATES_ALL[:11] BITRATES_ISM1 = BITRATES_ALL[:9] BITRATES_ISM2 = BITRATES_ALL[1:10] BITRATES_ISM3 = BITRATES_ALL[2:11] @@ -316,3 +316,5 @@ DECODER_CLEAN_CHANNEL_CHANNELBASED_AND_MASA_PARAMS = ( product(MASA_PARAMS, OUTPUT_FORMATS_ALL, SAMPLING_RATES_ALL) ) ) +DECODER_CLEAN_CHANNEL_OBJECTBASED = collapse_into_list_of_pairs(product(ISM_PARAMS, OUTPUT_FORMATS_ALL, SAMPLING_RATES_ALL)) +DECODER_CLEAN_CHANNEL_SCENEBASED = collapse_into_list_of_pairs(product(SBA_PARAMS, OUTPUT_FORMATS_ALL, SAMPLING_RATES_ALL)) \ No newline at end of file diff --git a/tests/codec_be_to_accepted_release/test_decoder.py b/tests/codec_be_to_accepted_release/test_decoder.py index 775b786834..752002188c 100644 --- a/tests/codec_be_to_accepted_release/test_decoder.py +++ b/tests/codec_be_to_accepted_release/test_decoder.py @@ -66,7 +66,7 @@ def run_check( "input_format,bitrate,input_sampling_rate,dtx,output_format,output_sampling_rate", DECODER_CLEAN_CHANNEL_CHANNELBASED_AND_MASA_PARAMS, ) -def test_decoder_clean_channel( +def test_decoder_clean_channel_channelbased_and_masa( input_format, bitrate, input_sampling_rate, @@ -90,3 +90,71 @@ def test_decoder_clean_channel( dut_decoder_frontend, update_ref == 1, ) + + +@pytest.mark.parametrize( + "input_format,bitrate,input_sampling_rate,dtx,md_type,output_format,output_sampling_rate", DECODER_CLEAN_CHANNEL_OBJECTBASED +) +def test_decoder_clean_channel_objectbased( + input_format, + bitrate, + input_sampling_rate, + dtx, + md_type, + output_format, + output_sampling_rate, + dut_decoder_frontend, + update_ref +): + testv_name = get_testv_path(input_format, input_sampling_rate).stem + suffix = "_basic_MD" + if md_type == ISM_MD_EXTENDED: + suffix = "_ext_MD" + elif md_type == ISM_MD_NULL: + suffix = "_null_MD" + ref_bitstream = get_bitstream_path( + REF_PATH, testv_name, input_format, bitrate, input_sampling_rate, dtx, suffix=suffix + ) + dut_output = get_output_path(ref_bitstream, output_format, output_sampling_rate) + + run_check( + str(ref_bitstream), + output_format, + output_sampling_rate, + str(dut_output), + dut_decoder_frontend, + update_ref == 1, + ) + + +@pytest.mark.parametrize( + "input_format,bitrate,input_sampling_rate,dtx,pca,output_format,output_sampling_rate", DECODER_CLEAN_CHANNEL_SCENEBASED +) +def test_decoder_clean_channel_scenebased( + input_format, + bitrate, + input_sampling_rate, + dtx, + pca, + output_format, + output_sampling_rate, + dut_decoder_frontend, + update_ref +): + testv_name = get_testv_path(input_format, input_sampling_rate).stem + suffix = "" + if pca == SBA_FOA_PCA_ON: + suffix="-pca" + ref_bitstream = get_bitstream_path( + REF_PATH, testv_name, input_format, bitrate, input_sampling_rate, dtx, suffix=suffix + ) + dut_output = get_output_path(ref_bitstream, output_format, output_sampling_rate) + + run_check( + str(ref_bitstream), + output_format, + output_sampling_rate, + str(dut_output), + dut_decoder_frontend, + update_ref == 1, + ) diff --git a/tests/codec_be_to_accepted_release/test_encoder.py b/tests/codec_be_to_accepted_release/test_encoder.py index 13d2247250..940e5d91e4 100644 --- a/tests/codec_be_to_accepted_release/test_encoder.py +++ b/tests/codec_be_to_accepted_release/test_encoder.py @@ -134,12 +134,13 @@ def test_encoder_object_based_modes( ): testv = get_testv_path(input_format, sampling_rate) options = get_options(input_format, md_type) + suffix = "_basic_MD" if md_type == ISM_MD_EXTENDED: - input_format += "_ext_MD" + suffix = "_ext_MD" elif md_type == ISM_MD_NULL: - input_format += "_null_MD" + suffix = "_null_MD" bitstream = get_bitstream_path( - DUT_PATH, testv.stem, input_format, bitrate, sampling_rate, dtx + DUT_PATH, testv.stem, input_format, bitrate, sampling_rate, dtx, suffix=suffix ) run_check( bitstream, @@ -159,10 +160,12 @@ def test_encoder_sba( ): testv = get_testv_path(input_format, sampling_rate) options = get_options(input_format) + suffix = "" if pca == SBA_FOA_PCA_ON: options.extend(["-bypass", "2"]) + suffix="-pca" bitstream = get_bitstream_path( - DUT_PATH, testv.stem, input_format, bitrate, sampling_rate, dtx, "-pca" + DUT_PATH, testv.stem, input_format, bitrate, sampling_rate, dtx, suffix=suffix ) run_check( bitstream, -- GitLab From 949e07edca8dc20c1195a0171a08a610c67cd581 Mon Sep 17 00:00:00 2001 From: knj Date: Mon, 28 Aug 2023 13:02:07 +0200 Subject: [PATCH 010/106] use context manager for getting the bitstream path --- .../test_decoder.py | 89 +++++++++++-------- 1 file changed, 52 insertions(+), 37 deletions(-) diff --git a/tests/codec_be_to_accepted_release/test_decoder.py b/tests/codec_be_to_accepted_release/test_decoder.py index 752002188c..c3b55680ab 100644 --- a/tests/codec_be_to_accepted_release/test_decoder.py +++ b/tests/codec_be_to_accepted_release/test_decoder.py @@ -31,6 +31,10 @@ the United Nations Convention on Contracts on the International Sales of Goods. import pytest from . import is_be_to_ref, get_bitstream_path, get_testv_path, REF_PATH, DUT_PATH from .constants import * +from pathlib import Path +import subprocess +from contextlib import contextmanager +from tempfile import TemporaryDirectory ### --------------- Helper functions --------------- @@ -43,6 +47,23 @@ def get_output_path(bitstream_path, output_format, output_sampling_rate): return DUT_PATH.joinpath(output_name) +@contextmanager +def get_bitstream(testv_name, encoder_format, bitrate, sampling_rate, dtx, suffix="", processing=None): + """ + Utility to get either the stored reference bitstream or the processed version as a temporary file + """ + with TemporaryDirectory() as tmp_dir: + bitstream = get_bitstream_path(REF_PATH, testv_name, encoder_format, bitrate, sampling_rate, dtx, suffix) + if processing == "FER_15": + bitstream_out = Path(tmp_dir).joinpath(bitstream.stem + f".{processing}.192") + ep_path = REF_PATH.joinpath("ltv_ep_015.192") + cmd = ["eid-xor", "-fer", "-vbr", str(bitstream), str(ep_path), str(bitstream_out)] + subprocess.run(cmd) + bitstream = bitstream_out + + yield bitstream + + def run_check( ref_bitstream, output_format, @@ -77,19 +98,17 @@ def test_decoder_clean_channel_channelbased_and_masa( update_ref, ): testv_name = get_testv_path(input_format, input_sampling_rate).stem - ref_bitstream = get_bitstream_path( - REF_PATH, testv_name, input_format, bitrate, input_sampling_rate, dtx - ) - dut_output = get_output_path(ref_bitstream, output_format, output_sampling_rate) + with get_bitstream(testv_name, input_format, bitrate, input_sampling_rate, dtx) as ref_bitstream: + dut_output = get_output_path(ref_bitstream, output_format, output_sampling_rate) - run_check( - str(ref_bitstream), - output_format, - output_sampling_rate, - str(dut_output), - dut_decoder_frontend, - update_ref == 1, - ) + run_check( + str(ref_bitstream), + output_format, + output_sampling_rate, + str(dut_output), + dut_decoder_frontend, + update_ref == 1, + ) @pytest.mark.parametrize( @@ -112,19 +131,17 @@ def test_decoder_clean_channel_objectbased( suffix = "_ext_MD" elif md_type == ISM_MD_NULL: suffix = "_null_MD" - ref_bitstream = get_bitstream_path( - REF_PATH, testv_name, input_format, bitrate, input_sampling_rate, dtx, suffix=suffix - ) - dut_output = get_output_path(ref_bitstream, output_format, output_sampling_rate) + with get_bitstream(testv_name, input_format, bitrate, input_sampling_rate, dtx, suffix=suffix) as ref_bitstream: + dut_output = get_output_path(ref_bitstream, output_format, output_sampling_rate) - run_check( - str(ref_bitstream), - output_format, - output_sampling_rate, - str(dut_output), - dut_decoder_frontend, - update_ref == 1, - ) + run_check( + str(ref_bitstream), + output_format, + output_sampling_rate, + str(dut_output), + dut_decoder_frontend, + update_ref == 1, + ) @pytest.mark.parametrize( @@ -145,16 +162,14 @@ def test_decoder_clean_channel_scenebased( suffix = "" if pca == SBA_FOA_PCA_ON: suffix="-pca" - ref_bitstream = get_bitstream_path( - REF_PATH, testv_name, input_format, bitrate, input_sampling_rate, dtx, suffix=suffix - ) - dut_output = get_output_path(ref_bitstream, output_format, output_sampling_rate) - - run_check( - str(ref_bitstream), - output_format, - output_sampling_rate, - str(dut_output), - dut_decoder_frontend, - update_ref == 1, - ) + with get_bitstream(testv_name, input_format, bitrate, input_sampling_rate, dtx, suffix=suffix) as ref_bitstream: + dut_output = get_output_path(ref_bitstream, output_format, output_sampling_rate) + + run_check( + str(ref_bitstream), + output_format, + output_sampling_rate, + str(dut_output), + dut_decoder_frontend, + update_ref == 1, + ) -- GitLab From ff83c8c7612824c92bf5f69057bb8734f92867fb Mon Sep 17 00:00:00 2001 From: knj Date: Mon, 28 Aug 2023 13:28:03 +0200 Subject: [PATCH 011/106] add 15% fer testcases --- .../codec_be_to_accepted_release/constants.py | 13 ++++++--- .../test_decoder.py | 27 ++++++++++--------- 2 files changed, 23 insertions(+), 17 deletions(-) diff --git a/tests/codec_be_to_accepted_release/constants.py b/tests/codec_be_to_accepted_release/constants.py index 67d0705109..ec03c97e69 100644 --- a/tests/codec_be_to_accepted_release/constants.py +++ b/tests/codec_be_to_accepted_release/constants.py @@ -301,20 +301,25 @@ STEREO_DMX_EVS_PARAMS = list( ### -------------- decoder parameter lists -------------- +FER_0 = "FER_0perc" +FER_15 = "FER_15perc" +BITSTREAM_PROCESSING_FER = [FER_0, FER_15] + DECODER_CLEAN_CHANNEL_CHANNELBASED_AND_MASA_PARAMS = ( collapse_into_list_of_pairs( - product(STEREO_PARAMS, OUTPUT_FORMATS_ALL[:2], SAMPLING_RATES_ALL) + product(STEREO_PARAMS, OUTPUT_FORMATS_ALL[:2], SAMPLING_RATES_ALL, BITSTREAM_PROCESSING_FER) ) + collapse_into_list_of_pairs( product( MC_PARAMS, OUTPUT_FORMATS_ALL[:10] + OUTPUT_FORMATS_BINAURAL, SAMPLING_RATES_ALL, + BITSTREAM_PROCESSING_FER, ) ) + collapse_into_list_of_pairs( - product(MASA_PARAMS, OUTPUT_FORMATS_ALL, SAMPLING_RATES_ALL) + product(MASA_PARAMS, OUTPUT_FORMATS_ALL, SAMPLING_RATES_ALL, BITSTREAM_PROCESSING_FER) ) ) -DECODER_CLEAN_CHANNEL_OBJECTBASED = collapse_into_list_of_pairs(product(ISM_PARAMS, OUTPUT_FORMATS_ALL, SAMPLING_RATES_ALL)) -DECODER_CLEAN_CHANNEL_SCENEBASED = collapse_into_list_of_pairs(product(SBA_PARAMS, OUTPUT_FORMATS_ALL, SAMPLING_RATES_ALL)) \ No newline at end of file +DECODER_CLEAN_CHANNEL_OBJECTBASED = collapse_into_list_of_pairs(product(ISM_PARAMS, OUTPUT_FORMATS_ALL, SAMPLING_RATES_ALL, BITSTREAM_PROCESSING_FER)) +DECODER_CLEAN_CHANNEL_SCENEBASED = collapse_into_list_of_pairs(product(SBA_PARAMS, OUTPUT_FORMATS_ALL, SAMPLING_RATES_ALL, BITSTREAM_PROCESSING_FER)) \ No newline at end of file diff --git a/tests/codec_be_to_accepted_release/test_decoder.py b/tests/codec_be_to_accepted_release/test_decoder.py index c3b55680ab..6a374ad1e1 100644 --- a/tests/codec_be_to_accepted_release/test_decoder.py +++ b/tests/codec_be_to_accepted_release/test_decoder.py @@ -29,7 +29,7 @@ the United Nations Convention on Contracts on the International Sales of Goods. """ import pytest -from . import is_be_to_ref, get_bitstream_path, get_testv_path, REF_PATH, DUT_PATH +from . import is_be_to_ref, get_bitstream_path, get_testv_path, REF_PATH, DUT_PATH, TESTV_PATH from .constants import * from pathlib import Path import subprocess @@ -54,9 +54,9 @@ def get_bitstream(testv_name, encoder_format, bitrate, sampling_rate, dtx, suffi """ with TemporaryDirectory() as tmp_dir: bitstream = get_bitstream_path(REF_PATH, testv_name, encoder_format, bitrate, sampling_rate, dtx, suffix) - if processing == "FER_15": + if processing == FER_15: bitstream_out = Path(tmp_dir).joinpath(bitstream.stem + f".{processing}.192") - ep_path = REF_PATH.joinpath("ltv_ep_015.192") + ep_path = TESTV_PATH.joinpath("ltv_ep_015.192") cmd = ["eid-xor", "-fer", "-vbr", str(bitstream), str(ep_path), str(bitstream_out)] subprocess.run(cmd) bitstream = bitstream_out @@ -80,11 +80,9 @@ def run_check( ### --------------- Actual testcase definitions --------------- -# clean channel decoding - @pytest.mark.parametrize( - "input_format,bitrate,input_sampling_rate,dtx,output_format,output_sampling_rate", + "input_format,bitrate,input_sampling_rate,dtx,output_format,output_sampling_rate,bitstream_processing", DECODER_CLEAN_CHANNEL_CHANNELBASED_AND_MASA_PARAMS, ) def test_decoder_clean_channel_channelbased_and_masa( @@ -94,11 +92,12 @@ def test_decoder_clean_channel_channelbased_and_masa( dtx, output_format, output_sampling_rate, + bitstream_processing, dut_decoder_frontend, update_ref, ): testv_name = get_testv_path(input_format, input_sampling_rate).stem - with get_bitstream(testv_name, input_format, bitrate, input_sampling_rate, dtx) as ref_bitstream: + with get_bitstream(testv_name, input_format, bitrate, input_sampling_rate, dtx, processing=bitstream_processing) as ref_bitstream: dut_output = get_output_path(ref_bitstream, output_format, output_sampling_rate) run_check( @@ -112,9 +111,9 @@ def test_decoder_clean_channel_channelbased_and_masa( @pytest.mark.parametrize( - "input_format,bitrate,input_sampling_rate,dtx,md_type,output_format,output_sampling_rate", DECODER_CLEAN_CHANNEL_OBJECTBASED + "input_format,bitrate,input_sampling_rate,dtx,md_type,output_format,output_sampling_rate,bitstream_processing", DECODER_CLEAN_CHANNEL_OBJECTBASED ) -def test_decoder_clean_channel_objectbased( +def test_decoder_objectbased( input_format, bitrate, input_sampling_rate, @@ -122,6 +121,7 @@ def test_decoder_clean_channel_objectbased( md_type, output_format, output_sampling_rate, + bitstream_processing, dut_decoder_frontend, update_ref ): @@ -131,7 +131,7 @@ def test_decoder_clean_channel_objectbased( suffix = "_ext_MD" elif md_type == ISM_MD_NULL: suffix = "_null_MD" - with get_bitstream(testv_name, input_format, bitrate, input_sampling_rate, dtx, suffix=suffix) as ref_bitstream: + with get_bitstream(testv_name, input_format, bitrate, input_sampling_rate, dtx, suffix=suffix, processing=bitstream_processing) as ref_bitstream: dut_output = get_output_path(ref_bitstream, output_format, output_sampling_rate) run_check( @@ -145,9 +145,9 @@ def test_decoder_clean_channel_objectbased( @pytest.mark.parametrize( - "input_format,bitrate,input_sampling_rate,dtx,pca,output_format,output_sampling_rate", DECODER_CLEAN_CHANNEL_SCENEBASED + "input_format,bitrate,input_sampling_rate,dtx,pca,output_format,output_sampling_rate,bitstream_processing", DECODER_CLEAN_CHANNEL_SCENEBASED ) -def test_decoder_clean_channel_scenebased( +def test_decoder_scenebased( input_format, bitrate, input_sampling_rate, @@ -155,6 +155,7 @@ def test_decoder_clean_channel_scenebased( pca, output_format, output_sampling_rate, + bitstream_processing, dut_decoder_frontend, update_ref ): @@ -162,7 +163,7 @@ def test_decoder_clean_channel_scenebased( suffix = "" if pca == SBA_FOA_PCA_ON: suffix="-pca" - with get_bitstream(testv_name, input_format, bitrate, input_sampling_rate, dtx, suffix=suffix) as ref_bitstream: + with get_bitstream(testv_name, input_format, bitrate, input_sampling_rate, dtx, suffix=suffix, processing=bitstream_processing) as ref_bitstream: dut_output = get_output_path(ref_bitstream, output_format, output_sampling_rate) run_check( -- GitLab From 2482c999856ed41e98a4373b9c13382995ca5302 Mon Sep 17 00:00:00 2001 From: knj Date: Mon, 28 Aug 2023 16:18:33 +0200 Subject: [PATCH 012/106] add encoder tests for combined formats --- .../codec_be_to_accepted_release/constants.py | 81 +++++++++++-- .../test_decoder.py | 107 +++++++++++++----- .../test_encoder.py | 48 +++++++- 3 files changed, 196 insertions(+), 40 deletions(-) diff --git a/tests/codec_be_to_accepted_release/constants.py b/tests/codec_be_to_accepted_release/constants.py index ec03c97e69..003cb9ff9b 100644 --- a/tests/codec_be_to_accepted_release/constants.py +++ b/tests/codec_be_to_accepted_release/constants.py @@ -67,6 +67,34 @@ TESTVECTOR_FOR_INPUT_FORMAT = { "MASA_2TC_1DIR": "ltv{fs}_MASA_2TC_1DIR.wav", "MASA_1TC_2DIR": "ltv{fs}_MASA_1TC_2DIR.wav", "MASA_2TC_2DIR": "ltv{fs}_MASA_2TC_2DIR.wav", + "OMASA_ISM1_MASA_1TC_1DIR": "ltv{fs}_OMASA_1ISM_MASA_1TC_1DIR.wav", + "OMASA_ISM1_MASA_1TC_2DIR": "ltv{fs}_OMASA_1ISM_MASA_1TC_2DIR.wav", + "OMASA_ISM1_MASA_2TC_1DIR": "ltv{fs}_OMASA_1ISM_MASA_2TC_1DIR.wav", + "OMASA_ISM1_MASA_2TC_2DIR": "ltv{fs}_OMASA_1ISM_MASA_2TC_2DIR.wav", + "OMASA_ISM2_MASA_1TC_1DIR": "ltv{fs}_OMASA_2ISM_MASA_1TC_1DIR.wav", + "OMASA_ISM2_MASA_1TC_2DIR": "ltv{fs}_OMASA_2ISM_MASA_1TC_2DIR.wav", + "OMASA_ISM2_MASA_2TC_1DIR": "ltv{fs}_OMASA_2ISM_MASA_2TC_1DIR.wav", + "OMASA_ISM2_MASA_2TC_2DIR": "ltv{fs}_OMASA_2ISM_MASA_2TC_2DIR.wav", + "OMASA_ISM3_MASA_1TC_1DIR": "ltv{fs}_OMASA_3ISM_MASA_1TC_1DIR.wav", + "OMASA_ISM3_MASA_1TC_2DIR": "ltv{fs}_OMASA_3ISM_MASA_1TC_2DIR.wav", + "OMASA_ISM3_MASA_2TC_1DIR": "ltv{fs}_OMASA_3ISM_MASA_2TC_1DIR.wav", + "OMASA_ISM3_MASA_2TC_2DIR": "ltv{fs}_OMASA_3ISM_MASA_2TC_2DIR.wav", + "OMASA_ISM4_MASA_1TC_1DIR": "ltv{fs}_OMASA_4ISM_MASA_1TC_1DIR.wav", + "OMASA_ISM4_MASA_1TC_2DIR": "ltv{fs}_OMASA_4ISM_MASA_1TC_2DIR.wav", + "OMASA_ISM4_MASA_2TC_1DIR": "ltv{fs}_OMASA_4ISM_MASA_2TC_1DIR.wav", + "OMASA_ISM4_MASA_2TC_2DIR": "ltv{fs}_OMASA_4ISM_MASA_2TC_2DIR.wav", + "OSBA_ISM1_FOA": "ltv{fs}_OSBA_1ISM_FOA.wav", + "OSBA_ISM1_HOA2": "ltv{fs}_OSBA_1ISM_HOA2.wav", + "OSBA_ISM1_HOA3": "ltv{fs}_OSBA_1ISM_HOA3.wav", + "OSBA_ISM2_FOA": "ltv{fs}_OSBA_2ISM_FOA.wav", + "OSBA_ISM2_HOA2": "ltv{fs}_OSBA_2ISM_HOA2.wav", + "OSBA_ISM2_HOA3": "ltv{fs}_OSBA_2ISM_HOA3.wav", + "OSBA_ISM3_FOA": "ltv{fs}_OSBA_3ISM_FOA.wav", + "OSBA_ISM3_HOA2": "ltv{fs}_OSBA_3ISM_HOA2.wav", + "OSBA_ISM3_HOA3": "ltv{fs}_OSBA_3ISM_HOA3.wav", + "OSBA_ISM4_FOA": "ltv{fs}_OSBA_4ISM_FOA.wav", + "OSBA_ISM4_HOA2": "ltv{fs}_OSBA_4ISM_HOA2.wav", + "OSBA_ISM4_HOA3": "ltv{fs}_OSBA_4ISM_HOA3.wav", } METADATA_FOR_INPUT_FORMAT = { "ISM1": ["ltvISM1.csv"], @@ -99,6 +127,8 @@ CMDL_OPTIONS_FOR_INPUT_FORMAT = { "MASA_2TC_1DIR": ["-masa", "2"], "MASA_1TC_2DIR": ["-masa", "1"], "MASA_2TC_2DIR": ["-masa", "2"], + "OMASA": ["-ism_masa"], + "OSBA": ["-ism_sba"], } ### Test case parameters @@ -114,10 +144,6 @@ INPUT_FORMATS_MASA = [ "MASA_1TC_2DIR", "MASA_2TC_2DIR", ] -INPUT_FORMATS_SINGLE = ( - INPUT_FORMATS_CHANNEL_BASED + INPUT_FORMATS_SCENE_BASED + INPUT_FORMATS_OBJECT_BASED -) # + INPUT_FORMATS_MASA -# TODO: combined formats BITRATES_ALL = [ 13200, @@ -299,6 +325,27 @@ STEREO_DMX_EVS_PARAMS = list( ) + list(product(BITRATES_EVS[:1], SAMPLING_RATES_ALL, [DTX_ON])) +OSBA_PARAMS = list( + product( + INPUT_FORMATS_OBJECT_BASED, + INPUT_FORMATS_SCENE_BASED, + BITRATES_ALL, + SAMPLING_RATES_ALL, + [ISM_MD_BASIC], + ) +) +OMASA_PARAMS = list( + product( + INPUT_FORMATS_OBJECT_BASED, + INPUT_FORMATS_MASA, + BITRATES_ALL, + SAMPLING_RATES_ALL, + [ISM_MD_BASIC], + ) +) +COMBINED_FORMATS_PARAMS = OSBA_PARAMS + OMASA_PARAMS + + ### -------------- decoder parameter lists -------------- FER_0 = "FER_0perc" @@ -307,7 +354,12 @@ BITSTREAM_PROCESSING_FER = [FER_0, FER_15] DECODER_CLEAN_CHANNEL_CHANNELBASED_AND_MASA_PARAMS = ( collapse_into_list_of_pairs( - product(STEREO_PARAMS, OUTPUT_FORMATS_ALL[:2], SAMPLING_RATES_ALL, BITSTREAM_PROCESSING_FER) + product( + STEREO_PARAMS, + OUTPUT_FORMATS_ALL[:2], + SAMPLING_RATES_ALL, + BITSTREAM_PROCESSING_FER, + ) ) + collapse_into_list_of_pairs( product( @@ -318,8 +370,21 @@ DECODER_CLEAN_CHANNEL_CHANNELBASED_AND_MASA_PARAMS = ( ) ) + collapse_into_list_of_pairs( - product(MASA_PARAMS, OUTPUT_FORMATS_ALL, SAMPLING_RATES_ALL, BITSTREAM_PROCESSING_FER) + product( + MASA_PARAMS, + OUTPUT_FORMATS_ALL, + SAMPLING_RATES_ALL, + BITSTREAM_PROCESSING_FER, + ) + ) +) +DECODER_CLEAN_CHANNEL_OBJECTBASED = collapse_into_list_of_pairs( + product( + ISM_PARAMS, OUTPUT_FORMATS_ALL, SAMPLING_RATES_ALL, BITSTREAM_PROCESSING_FER + ) +) +DECODER_CLEAN_CHANNEL_SCENEBASED = collapse_into_list_of_pairs( + product( + SBA_PARAMS, OUTPUT_FORMATS_ALL, SAMPLING_RATES_ALL, BITSTREAM_PROCESSING_FER ) ) -DECODER_CLEAN_CHANNEL_OBJECTBASED = collapse_into_list_of_pairs(product(ISM_PARAMS, OUTPUT_FORMATS_ALL, SAMPLING_RATES_ALL, BITSTREAM_PROCESSING_FER)) -DECODER_CLEAN_CHANNEL_SCENEBASED = collapse_into_list_of_pairs(product(SBA_PARAMS, OUTPUT_FORMATS_ALL, SAMPLING_RATES_ALL, BITSTREAM_PROCESSING_FER)) \ No newline at end of file diff --git a/tests/codec_be_to_accepted_release/test_decoder.py b/tests/codec_be_to_accepted_release/test_decoder.py index 6a374ad1e1..05c1af0750 100644 --- a/tests/codec_be_to_accepted_release/test_decoder.py +++ b/tests/codec_be_to_accepted_release/test_decoder.py @@ -29,7 +29,14 @@ the United Nations Convention on Contracts on the International Sales of Goods. """ import pytest -from . import is_be_to_ref, get_bitstream_path, get_testv_path, REF_PATH, DUT_PATH, TESTV_PATH +from . import ( + is_be_to_ref, + get_bitstream_path, + get_testv_path, + REF_PATH, + DUT_PATH, + TESTV_PATH, +) from .constants import * from pathlib import Path import subprocess @@ -48,16 +55,29 @@ def get_output_path(bitstream_path, output_format, output_sampling_rate): @contextmanager -def get_bitstream(testv_name, encoder_format, bitrate, sampling_rate, dtx, suffix="", processing=None): +def get_bitstream( + testv_name, encoder_format, bitrate, sampling_rate, dtx, suffix="", processing=None +): """ Utility to get either the stored reference bitstream or the processed version as a temporary file """ with TemporaryDirectory() as tmp_dir: - bitstream = get_bitstream_path(REF_PATH, testv_name, encoder_format, bitrate, sampling_rate, dtx, suffix) + bitstream = get_bitstream_path( + REF_PATH, testv_name, encoder_format, bitrate, sampling_rate, dtx, suffix + ) if processing == FER_15: - bitstream_out = Path(tmp_dir).joinpath(bitstream.stem + f".{processing}.192") + bitstream_out = Path(tmp_dir).joinpath( + bitstream.stem + f".{processing}.192" + ) ep_path = TESTV_PATH.joinpath("ltv_ep_015.192") - cmd = ["eid-xor", "-fer", "-vbr", str(bitstream), str(ep_path), str(bitstream_out)] + cmd = [ + "eid-xor", + "-fer", + "-vbr", + str(bitstream), + str(ep_path), + str(bitstream_out), + ] subprocess.run(cmd) bitstream = bitstream_out @@ -97,7 +117,14 @@ def test_decoder_clean_channel_channelbased_and_masa( update_ref, ): testv_name = get_testv_path(input_format, input_sampling_rate).stem - with get_bitstream(testv_name, input_format, bitrate, input_sampling_rate, dtx, processing=bitstream_processing) as ref_bitstream: + with get_bitstream( + testv_name, + input_format, + bitrate, + input_sampling_rate, + dtx, + processing=bitstream_processing, + ) as ref_bitstream: dut_output = get_output_path(ref_bitstream, output_format, output_sampling_rate) run_check( @@ -111,19 +138,20 @@ def test_decoder_clean_channel_channelbased_and_masa( @pytest.mark.parametrize( - "input_format,bitrate,input_sampling_rate,dtx,md_type,output_format,output_sampling_rate,bitstream_processing", DECODER_CLEAN_CHANNEL_OBJECTBASED + "input_format,bitrate,input_sampling_rate,dtx,md_type,output_format,output_sampling_rate,bitstream_processing", + DECODER_CLEAN_CHANNEL_OBJECTBASED, ) def test_decoder_objectbased( - input_format, - bitrate, - input_sampling_rate, - dtx, - md_type, - output_format, - output_sampling_rate, - bitstream_processing, - dut_decoder_frontend, - update_ref + input_format, + bitrate, + input_sampling_rate, + dtx, + md_type, + output_format, + output_sampling_rate, + bitstream_processing, + dut_decoder_frontend, + update_ref, ): testv_name = get_testv_path(input_format, input_sampling_rate).stem suffix = "_basic_MD" @@ -131,7 +159,15 @@ def test_decoder_objectbased( suffix = "_ext_MD" elif md_type == ISM_MD_NULL: suffix = "_null_MD" - with get_bitstream(testv_name, input_format, bitrate, input_sampling_rate, dtx, suffix=suffix, processing=bitstream_processing) as ref_bitstream: + with get_bitstream( + testv_name, + input_format, + bitrate, + input_sampling_rate, + dtx, + suffix=suffix, + processing=bitstream_processing, + ) as ref_bitstream: dut_output = get_output_path(ref_bitstream, output_format, output_sampling_rate) run_check( @@ -145,25 +181,34 @@ def test_decoder_objectbased( @pytest.mark.parametrize( - "input_format,bitrate,input_sampling_rate,dtx,pca,output_format,output_sampling_rate,bitstream_processing", DECODER_CLEAN_CHANNEL_SCENEBASED + "input_format,bitrate,input_sampling_rate,dtx,pca,output_format,output_sampling_rate,bitstream_processing", + DECODER_CLEAN_CHANNEL_SCENEBASED, ) def test_decoder_scenebased( - input_format, - bitrate, - input_sampling_rate, - dtx, - pca, - output_format, - output_sampling_rate, - bitstream_processing, - dut_decoder_frontend, - update_ref + input_format, + bitrate, + input_sampling_rate, + dtx, + pca, + output_format, + output_sampling_rate, + bitstream_processing, + dut_decoder_frontend, + update_ref, ): testv_name = get_testv_path(input_format, input_sampling_rate).stem suffix = "" if pca == SBA_FOA_PCA_ON: - suffix="-pca" - with get_bitstream(testv_name, input_format, bitrate, input_sampling_rate, dtx, suffix=suffix, processing=bitstream_processing) as ref_bitstream: + suffix = "-pca" + with get_bitstream( + testv_name, + input_format, + bitrate, + input_sampling_rate, + dtx, + suffix=suffix, + processing=bitstream_processing, + ) as ref_bitstream: dut_output = get_output_path(ref_bitstream, output_format, output_sampling_rate) run_check( diff --git a/tests/codec_be_to_accepted_release/test_encoder.py b/tests/codec_be_to_accepted_release/test_encoder.py index 940e5d91e4..4ba9fdc857 100644 --- a/tests/codec_be_to_accepted_release/test_encoder.py +++ b/tests/codec_be_to_accepted_release/test_encoder.py @@ -37,10 +37,12 @@ from .constants import ( INPUT_FORMATS_OBJECT_BASED, ENCODER_CHANNEL_BASED_AND_MASA_PARAMS, DTX_ON, + DTX_OFF, ISM_PARAMS, SBA_PARAMS, SBA_FOA_PCA_ON, STEREO_DMX_EVS_PARAMS, + COMBINED_FORMATS_PARAMS, ) from . import TESTV_PATH, DUT_PATH, is_be_to_ref, get_bitstream_path, get_testv_path @@ -163,7 +165,7 @@ def test_encoder_sba( suffix = "" if pca == SBA_FOA_PCA_ON: options.extend(["-bypass", "2"]) - suffix="-pca" + suffix = "-pca" bitstream = get_bitstream_path( DUT_PATH, testv.stem, input_format, bitrate, sampling_rate, dtx, suffix=suffix ) @@ -197,3 +199,47 @@ def test_encoder_stereo_dmx_evs( update_ref == 1, dtx == DTX_ON, ) + + +@pytest.mark.parametrize( + "input_format_ism,input_format_other,bitrate,sampling_rate,md_type", + COMBINED_FORMATS_PARAMS, +) +def test_encoder_combined_formats( + input_format_ism, + input_format_other, + bitrate, + sampling_rate, + md_type, + dut_encoder_frontend, + update_ref, +): + input_format_combined = "OMASA" if "MASA" in input_format_other else "OSBA" + testv = get_testv_path( + f"{input_format_combined}_{input_format_ism}_{input_format_other}", + sampling_rate, + ) + options_ism = get_options(input_format_ism, md_type) + options_other = get_options(input_format_other) + options_combined = get_options(input_format_combined) + # glue together: combined mode option, ism num, SBA order/MASA TCs, ISM MD files, MASA MD files (slice results in empty list for OSBA) + options = ( + options_combined + + options_ism[1:2] + + options_other[1:2] + + options_ism[2:] + + options_other[2:] + ) + bitstream = get_bitstream_path( + DUT_PATH, testv.stem, input_format_combined, bitrate, sampling_rate, DTX_OFF + ) + run_check( + bitstream, + bitrate, + sampling_rate, + testv, + options, + dut_encoder_frontend, + update_ref == 1, + False, + ) -- GitLab From 802afa8e384b949a9c8b8ab34a52bb4e99bdd155 Mon Sep 17 00:00:00 2001 From: knj Date: Mon, 28 Aug 2023 17:01:02 +0200 Subject: [PATCH 013/106] add decoder tests for combined formats --- .../codec_be_to_accepted_release/constants.py | 2 + .../test_decoder.py | 38 +++++++++++++++++++ 2 files changed, 40 insertions(+) diff --git a/tests/codec_be_to_accepted_release/constants.py b/tests/codec_be_to_accepted_release/constants.py index 003cb9ff9b..626ef8639b 100644 --- a/tests/codec_be_to_accepted_release/constants.py +++ b/tests/codec_be_to_accepted_release/constants.py @@ -155,6 +155,7 @@ BITRATES_ALL = [ 80000, 96000, 128000, + 160000, 192000, 256000, 384000, @@ -388,3 +389,4 @@ DECODER_CLEAN_CHANNEL_SCENEBASED = collapse_into_list_of_pairs( SBA_PARAMS, OUTPUT_FORMATS_ALL, SAMPLING_RATES_ALL, BITSTREAM_PROCESSING_FER ) ) +DECODER_COMBINED_PARAMS = collapse_into_list_of_pairs(product(COMBINED_FORMATS_PARAMS, OUTPUT_FORMATS_ALL, SAMPLING_RATES_ALL, BITSTREAM_PROCESSING_FER)) diff --git a/tests/codec_be_to_accepted_release/test_decoder.py b/tests/codec_be_to_accepted_release/test_decoder.py index 05c1af0750..6420f99644 100644 --- a/tests/codec_be_to_accepted_release/test_decoder.py +++ b/tests/codec_be_to_accepted_release/test_decoder.py @@ -219,3 +219,41 @@ def test_decoder_scenebased( dut_decoder_frontend, update_ref == 1, ) + + +@pytest.mark.parametrize("input_format_ism,input_format_other,bitrate,input_sampling_rate,md_type, output_format, output_sampling_rate, bitstream_processing", DECODER_COMBINED_PARAMS) +def test_decoder_combined_formats( + input_format_ism, + input_format_other, + bitrate, + input_sampling_rate, + md_type, + output_format, + output_sampling_rate, + bitstream_processing, + dut_decoder_frontend, + update_ref, +): + input_format_combined = "OMASA" if "MASA" in input_format_other else "OSBA" + testv_name = get_testv_path( + f"{input_format_combined}_{input_format_ism}_{input_format_other}", + input_sampling_rate, + ).stem + with get_bitstream( + testv_name, + input_format_combined, + bitrate, + input_sampling_rate, + DTX_OFF, + processing=bitstream_processing, + ) as ref_bitstream: + dut_output = get_output_path(ref_bitstream, output_format, output_sampling_rate) + + run_check( + str(ref_bitstream), + output_format, + output_sampling_rate, + str(dut_output), + dut_decoder_frontend, + update_ref == 1, + ) -- GitLab From 0489851d16c186a6a6d648b04fe9a37a2d934b5a Mon Sep 17 00:00:00 2001 From: knj Date: Mon, 28 Aug 2023 17:07:02 +0200 Subject: [PATCH 014/106] add helper function to fix bitrate slicing --- tests/codec_be_to_accepted_release/constants.py | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/tests/codec_be_to_accepted_release/constants.py b/tests/codec_be_to_accepted_release/constants.py index 626ef8639b..4c468f0969 100644 --- a/tests/codec_be_to_accepted_release/constants.py +++ b/tests/codec_be_to_accepted_release/constants.py @@ -35,6 +35,13 @@ def collapse_into_list_of_pairs(params): return [p[0] + p[1:] for p in params] +def bitrates_between(lowest, highest): + """ + Helper function to reduce slicing errors due to mistakes and later changes + """ + return [b for b in BITRATES_ALL if b >= lowest and b <= highest] + + DTX_ON = "DTXon" DTX_OFF = "DTXoff" @@ -161,11 +168,11 @@ BITRATES_ALL = [ 384000, 512000, ] -BITRATES_STEREO = BITRATES_ALL[:11] -BITRATES_ISM1 = BITRATES_ALL[:9] -BITRATES_ISM2 = BITRATES_ALL[1:10] -BITRATES_ISM3 = BITRATES_ALL[2:11] -BITRATES_ISM4 = BITRATES_ALL[2:] +BITRATES_STEREO = bitrates_between(13200, 256000) +BITRATES_ISM1 = bitrates_between(13200, 128000) +BITRATES_ISM2 = bitrates_between(16400, 256000) +BITRATES_ISM3 = bitrates_between(24400, 384000) +BITRATES_ISM4 = bitrates_between(24400, 512000) BITRATES_ISM1_EXTENDED = [b for b in BITRATES_ISM1 if b > 64000] BITRATES_ISM2_EXTENDED = [b for b in BITRATES_ISM2 if b > 64000] BITRATES_ISM3_EXTENDED = [b for b in BITRATES_ISM3 if b > 64000] -- GitLab From a11c11812a3619e3cdca1b3cd922d8d15088a1e0 Mon Sep 17 00:00:00 2001 From: knj Date: Tue, 29 Aug 2023 10:20:12 +0200 Subject: [PATCH 015/106] rename testcases --- tests/codec_be_to_accepted_release/constants.py | 6 +++--- tests/codec_be_to_accepted_release/test_decoder.py | 8 ++++---- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/tests/codec_be_to_accepted_release/constants.py b/tests/codec_be_to_accepted_release/constants.py index 4c468f0969..436e6395f9 100644 --- a/tests/codec_be_to_accepted_release/constants.py +++ b/tests/codec_be_to_accepted_release/constants.py @@ -360,7 +360,7 @@ FER_0 = "FER_0perc" FER_15 = "FER_15perc" BITSTREAM_PROCESSING_FER = [FER_0, FER_15] -DECODER_CLEAN_CHANNEL_CHANNELBASED_AND_MASA_PARAMS = ( +DECODER_CHANNELBASED_AND_MASA_PARAMS = ( collapse_into_list_of_pairs( product( STEREO_PARAMS, @@ -386,12 +386,12 @@ DECODER_CLEAN_CHANNEL_CHANNELBASED_AND_MASA_PARAMS = ( ) ) ) -DECODER_CLEAN_CHANNEL_OBJECTBASED = collapse_into_list_of_pairs( +DECODER_OBJECTBASED = collapse_into_list_of_pairs( product( ISM_PARAMS, OUTPUT_FORMATS_ALL, SAMPLING_RATES_ALL, BITSTREAM_PROCESSING_FER ) ) -DECODER_CLEAN_CHANNEL_SCENEBASED = collapse_into_list_of_pairs( +DECODER_SCENEBASED = collapse_into_list_of_pairs( product( SBA_PARAMS, OUTPUT_FORMATS_ALL, SAMPLING_RATES_ALL, BITSTREAM_PROCESSING_FER ) diff --git a/tests/codec_be_to_accepted_release/test_decoder.py b/tests/codec_be_to_accepted_release/test_decoder.py index 6420f99644..4356541592 100644 --- a/tests/codec_be_to_accepted_release/test_decoder.py +++ b/tests/codec_be_to_accepted_release/test_decoder.py @@ -103,9 +103,9 @@ def run_check( @pytest.mark.parametrize( "input_format,bitrate,input_sampling_rate,dtx,output_format,output_sampling_rate,bitstream_processing", - DECODER_CLEAN_CHANNEL_CHANNELBASED_AND_MASA_PARAMS, + DECODER_CHANNELBASED_AND_MASA_PARAMS, ) -def test_decoder_clean_channel_channelbased_and_masa( +def test_decoder_channelbased_and_masa( input_format, bitrate, input_sampling_rate, @@ -139,7 +139,7 @@ def test_decoder_clean_channel_channelbased_and_masa( @pytest.mark.parametrize( "input_format,bitrate,input_sampling_rate,dtx,md_type,output_format,output_sampling_rate,bitstream_processing", - DECODER_CLEAN_CHANNEL_OBJECTBASED, + DECODER_OBJECTBASED, ) def test_decoder_objectbased( input_format, @@ -182,7 +182,7 @@ def test_decoder_objectbased( @pytest.mark.parametrize( "input_format,bitrate,input_sampling_rate,dtx,pca,output_format,output_sampling_rate,bitstream_processing", - DECODER_CLEAN_CHANNEL_SCENEBASED, + DECODER_SCENEBASED, ) def test_decoder_scenebased( input_format, -- GitLab From 34efa8aae9db6d1adc387060f5234fcdabe0a2c6 Mon Sep 17 00:00:00 2001 From: knj Date: Tue, 29 Aug 2023 13:55:46 +0200 Subject: [PATCH 016/106] add planar SBA modes --- .../codec_be_to_accepted_release/constants.py | 33 ++++++++++++++++--- .../test_decoder.py | 7 ++-- .../test_encoder.py | 12 +++++-- 3 files changed, 43 insertions(+), 9 deletions(-) diff --git a/tests/codec_be_to_accepted_release/constants.py b/tests/codec_be_to_accepted_release/constants.py index 436e6395f9..f928eeb00e 100644 --- a/tests/codec_be_to_accepted_release/constants.py +++ b/tests/codec_be_to_accepted_release/constants.py @@ -62,6 +62,9 @@ TESTVECTOR_FOR_INPUT_FORMAT = { "FOA": "ltv{fs}_FOA.wav", "HOA2": "ltv{fs}_HOA2.wav", "HOA3": "ltv{fs}_HOA3.wav", + "PlanarFOA": "ltv{fs}_FOA.wav", + "PlanarHOA2": "ltv{fs}_HOA2.wav", + "PlanarHOA3": "ltv{fs}_HOA3.wav", "ISM1": "ltv{fs}_1ISM.wav", "ISM2": "ltv{fs}_2ISM.wav", "ISM3": "ltv{fs}_3ISM.wav", @@ -143,7 +146,14 @@ CMDL_OPTIONS_FOR_INPUT_FORMAT = { # Input format IDs INPUT_FORMATS_MC = ["MC_5_1", "MC_5_1_2", "MC_5_1_4", "MC_7_1", "MC_7_1_4"] INPUT_FORMATS_CHANNEL_BASED = ["STEREO"] + INPUT_FORMATS_MC -INPUT_FORMATS_SCENE_BASED = ["FOA", "HOA2", "HOA3"] +INPUT_FORMATS_SCENE_BASED = [ + "FOA", + "HOA2", + "HOA3", + "PlanarFOA", + "PlanarHOA2", + "PlanarHOA3", +] INPUT_FORMATS_OBJECT_BASED = ["ISM1", "ISM2", "ISM3", "ISM4"] INPUT_FORMATS_MASA = [ "MASA_1TC_1DIR", @@ -322,7 +332,13 @@ SBA_HIGH_BITRATES_PARAMS = list( ) ) SBA_FOA_PCA_PARAMS = list( - product(["FOA"], [256000], SAMPLING_RATES_ALL, [DTX_OFF], [SBA_FOA_PCA_ON]) + product( + [f for f in INPUT_FORMATS_SCENE_BASED if "FOA" in f], + [256000], + SAMPLING_RATES_ALL, + [DTX_OFF], + [SBA_FOA_PCA_ON], + ) ) SBA_PARAMS = SBA_LOW_BITRATES_PARAMS + SBA_HIGH_BITRATES_PARAMS + SBA_FOA_PCA_PARAMS @@ -336,7 +352,9 @@ STEREO_DMX_EVS_PARAMS = list( OSBA_PARAMS = list( product( INPUT_FORMATS_OBJECT_BASED, - INPUT_FORMATS_SCENE_BASED, + [ + f for f in INPUT_FORMATS_SCENE_BASED if not f.startswith("Planar") + ], # TODO: enable planar once clarified BITRATES_ALL, SAMPLING_RATES_ALL, [ISM_MD_BASIC], @@ -396,4 +414,11 @@ DECODER_SCENEBASED = collapse_into_list_of_pairs( SBA_PARAMS, OUTPUT_FORMATS_ALL, SAMPLING_RATES_ALL, BITSTREAM_PROCESSING_FER ) ) -DECODER_COMBINED_PARAMS = collapse_into_list_of_pairs(product(COMBINED_FORMATS_PARAMS, OUTPUT_FORMATS_ALL, SAMPLING_RATES_ALL, BITSTREAM_PROCESSING_FER)) +DECODER_COMBINED_PARAMS = collapse_into_list_of_pairs( + product( + COMBINED_FORMATS_PARAMS, + OUTPUT_FORMATS_ALL, + SAMPLING_RATES_ALL, + BITSTREAM_PROCESSING_FER, + ) +) diff --git a/tests/codec_be_to_accepted_release/test_decoder.py b/tests/codec_be_to_accepted_release/test_decoder.py index 4356541592..4e498ee0d0 100644 --- a/tests/codec_be_to_accepted_release/test_decoder.py +++ b/tests/codec_be_to_accepted_release/test_decoder.py @@ -199,7 +199,7 @@ def test_decoder_scenebased( testv_name = get_testv_path(input_format, input_sampling_rate).stem suffix = "" if pca == SBA_FOA_PCA_ON: - suffix = "-pca" + suffix += "-pca" with get_bitstream( testv_name, input_format, @@ -221,7 +221,10 @@ def test_decoder_scenebased( ) -@pytest.mark.parametrize("input_format_ism,input_format_other,bitrate,input_sampling_rate,md_type, output_format, output_sampling_rate, bitstream_processing", DECODER_COMBINED_PARAMS) +@pytest.mark.parametrize( + "input_format_ism,input_format_other,bitrate,input_sampling_rate,md_type, output_format, output_sampling_rate, bitstream_processing", + DECODER_COMBINED_PARAMS, +) def test_decoder_combined_formats( input_format_ism, input_format_other, diff --git a/tests/codec_be_to_accepted_release/test_encoder.py b/tests/codec_be_to_accepted_release/test_encoder.py index 4ba9fdc857..63ffc4f1e0 100644 --- a/tests/codec_be_to_accepted_release/test_encoder.py +++ b/tests/codec_be_to_accepted_release/test_encoder.py @@ -157,15 +157,21 @@ def test_encoder_object_based_modes( @pytest.mark.parametrize("input_format,bitrate,sampling_rate,dtx,pca", SBA_PARAMS) -def test_encoder_sba( - input_format, bitrate, sampling_rate, dtx, pca, dut_encoder_frontend, update_ref +def test_encoder_scenebased( + input_format, + bitrate, + sampling_rate, + dtx, + pca, + dut_encoder_frontend, + update_ref, ): testv = get_testv_path(input_format, sampling_rate) options = get_options(input_format) suffix = "" if pca == SBA_FOA_PCA_ON: options.extend(["-bypass", "2"]) - suffix = "-pca" + suffix += "-pca" bitstream = get_bitstream_path( DUT_PATH, testv.stem, input_format, bitrate, sampling_rate, dtx, suffix=suffix ) -- GitLab From e9cdcc92ae65736b4f4a10887440e8ba4b812e4a Mon Sep 17 00:00:00 2001 From: knj Date: Wed, 30 Aug 2023 09:45:04 +0200 Subject: [PATCH 017/106] test different kinds of ISM MD with combined modes --- tests/codec_be_to_accepted_release/constants.py | 4 ++-- tests/codec_be_to_accepted_release/test_decoder.py | 10 ++++++++-- tests/codec_be_to_accepted_release/test_encoder.py | 7 ++++++- 3 files changed, 16 insertions(+), 5 deletions(-) diff --git a/tests/codec_be_to_accepted_release/constants.py b/tests/codec_be_to_accepted_release/constants.py index f928eeb00e..934c027805 100644 --- a/tests/codec_be_to_accepted_release/constants.py +++ b/tests/codec_be_to_accepted_release/constants.py @@ -357,7 +357,7 @@ OSBA_PARAMS = list( ], # TODO: enable planar once clarified BITRATES_ALL, SAMPLING_RATES_ALL, - [ISM_MD_BASIC], + [ISM_MD_BASIC, ISM_MD_NULL, ISM_MD_EXTENDED], ) ) OMASA_PARAMS = list( @@ -366,7 +366,7 @@ OMASA_PARAMS = list( INPUT_FORMATS_MASA, BITRATES_ALL, SAMPLING_RATES_ALL, - [ISM_MD_BASIC], + [ISM_MD_BASIC, ISM_MD_NULL, ISM_MD_EXTENDED], ) ) COMBINED_FORMATS_PARAMS = OSBA_PARAMS + OMASA_PARAMS diff --git a/tests/codec_be_to_accepted_release/test_decoder.py b/tests/codec_be_to_accepted_release/test_decoder.py index 4e498ee0d0..23a012a28e 100644 --- a/tests/codec_be_to_accepted_release/test_decoder.py +++ b/tests/codec_be_to_accepted_release/test_decoder.py @@ -222,7 +222,7 @@ def test_decoder_scenebased( @pytest.mark.parametrize( - "input_format_ism,input_format_other,bitrate,input_sampling_rate,md_type, output_format, output_sampling_rate, bitstream_processing", + "input_format_ism,input_format_other,bitrate,input_sampling_rate,md_type,output_format,output_sampling_rate,bitstream_processing", DECODER_COMBINED_PARAMS, ) def test_decoder_combined_formats( @@ -241,13 +241,19 @@ def test_decoder_combined_formats( testv_name = get_testv_path( f"{input_format_combined}_{input_format_ism}_{input_format_other}", input_sampling_rate, - ).stem + ).stem + suffix = "_basic_MD" + if md_type == ISM_MD_EXTENDED: + suffix = "_ext_MD" + elif md_type == ISM_MD_NULL: + suffix = "_null_MD" with get_bitstream( testv_name, input_format_combined, bitrate, input_sampling_rate, DTX_OFF, + suffix=suffix, processing=bitstream_processing, ) as ref_bitstream: dut_output = get_output_path(ref_bitstream, output_format, output_sampling_rate) diff --git a/tests/codec_be_to_accepted_release/test_encoder.py b/tests/codec_be_to_accepted_release/test_encoder.py index 63ffc4f1e0..769fb22d4e 100644 --- a/tests/codec_be_to_accepted_release/test_encoder.py +++ b/tests/codec_be_to_accepted_release/test_encoder.py @@ -236,8 +236,13 @@ def test_encoder_combined_formats( + options_ism[2:] + options_other[2:] ) + suffix = "_basic_MD" + if md_type == ISM_MD_EXTENDED: + suffix = "_ext_MD" + elif md_type == ISM_MD_NULL: + suffix = "_null_MD" bitstream = get_bitstream_path( - DUT_PATH, testv.stem, input_format_combined, bitrate, sampling_rate, DTX_OFF + DUT_PATH, testv.stem, input_format_combined, bitrate, sampling_rate, DTX_OFF, suffix=suffix ) run_check( bitstream, -- GitLab From c010fe50a8e5c5a826c4a12237d565ed3cc5e55f Mon Sep 17 00:00:00 2001 From: knj Date: Wed, 30 Aug 2023 12:01:59 +0200 Subject: [PATCH 018/106] add JBM testcases --- .../codec_be_to_accepted_release/constants.py | 23 +++---- .../test_decoder.py | 61 +++++++++++++------ .../test_encoder.py | 8 ++- 3 files changed, 61 insertions(+), 31 deletions(-) diff --git a/tests/codec_be_to_accepted_release/constants.py b/tests/codec_be_to_accepted_release/constants.py index 934c027805..e278e6ccf3 100644 --- a/tests/codec_be_to_accepted_release/constants.py +++ b/tests/codec_be_to_accepted_release/constants.py @@ -374,9 +374,10 @@ COMBINED_FORMATS_PARAMS = OSBA_PARAMS + OMASA_PARAMS ### -------------- decoder parameter lists -------------- -FER_0 = "FER_0perc" -FER_15 = "FER_15perc" -BITSTREAM_PROCESSING_FER = [FER_0, FER_15] +BS_PROC_NONE = "CLEAN_CHANNEL" +BS_PROC_FER_15 = "FER_15perc" +BS_PROC_JBM_DLY_PROF_5 = "JBM_dlyprofile5" +BITSTREAM_PROCESSING = [BS_PROC_NONE, BS_PROC_FER_15, BS_PROC_JBM_DLY_PROF_5] DECODER_CHANNELBASED_AND_MASA_PARAMS = ( collapse_into_list_of_pairs( @@ -384,7 +385,7 @@ DECODER_CHANNELBASED_AND_MASA_PARAMS = ( STEREO_PARAMS, OUTPUT_FORMATS_ALL[:2], SAMPLING_RATES_ALL, - BITSTREAM_PROCESSING_FER, + BITSTREAM_PROCESSING, ) ) + collapse_into_list_of_pairs( @@ -392,7 +393,7 @@ DECODER_CHANNELBASED_AND_MASA_PARAMS = ( MC_PARAMS, OUTPUT_FORMATS_ALL[:10] + OUTPUT_FORMATS_BINAURAL, SAMPLING_RATES_ALL, - BITSTREAM_PROCESSING_FER, + BITSTREAM_PROCESSING, ) ) + collapse_into_list_of_pairs( @@ -400,25 +401,21 @@ DECODER_CHANNELBASED_AND_MASA_PARAMS = ( MASA_PARAMS, OUTPUT_FORMATS_ALL, SAMPLING_RATES_ALL, - BITSTREAM_PROCESSING_FER, + BITSTREAM_PROCESSING, ) ) ) DECODER_OBJECTBASED = collapse_into_list_of_pairs( - product( - ISM_PARAMS, OUTPUT_FORMATS_ALL, SAMPLING_RATES_ALL, BITSTREAM_PROCESSING_FER - ) + product(ISM_PARAMS, OUTPUT_FORMATS_ALL, SAMPLING_RATES_ALL, BITSTREAM_PROCESSING) ) DECODER_SCENEBASED = collapse_into_list_of_pairs( - product( - SBA_PARAMS, OUTPUT_FORMATS_ALL, SAMPLING_RATES_ALL, BITSTREAM_PROCESSING_FER - ) + product(SBA_PARAMS, OUTPUT_FORMATS_ALL, SAMPLING_RATES_ALL, BITSTREAM_PROCESSING) ) DECODER_COMBINED_PARAMS = collapse_into_list_of_pairs( product( COMBINED_FORMATS_PARAMS, OUTPUT_FORMATS_ALL, SAMPLING_RATES_ALL, - BITSTREAM_PROCESSING_FER, + BITSTREAM_PROCESSING, ) ) diff --git a/tests/codec_be_to_accepted_release/test_decoder.py b/tests/codec_be_to_accepted_release/test_decoder.py index 23a012a28e..cb14debde5 100644 --- a/tests/codec_be_to_accepted_release/test_decoder.py +++ b/tests/codec_be_to_accepted_release/test_decoder.py @@ -36,6 +36,7 @@ from . import ( REF_PATH, DUT_PATH, TESTV_PATH, + HERE, ) from .constants import * from pathlib import Path @@ -55,22 +56,22 @@ def get_output_path(bitstream_path, output_format, output_sampling_rate): @contextmanager -def get_bitstream( - testv_name, encoder_format, bitrate, sampling_rate, dtx, suffix="", processing=None +def get_bitstream_and_options( + testv_name, encoder_format, bitrate, sampling_rate, dtx, processing, suffix="" ): """ Utility to get either the stored reference bitstream or the processed version as a temporary file """ + options = list() with TemporaryDirectory() as tmp_dir: bitstream = get_bitstream_path( REF_PATH, testv_name, encoder_format, bitrate, sampling_rate, dtx, suffix ) - if processing == FER_15: - bitstream_out = Path(tmp_dir).joinpath( - bitstream.stem + f".{processing}.192" - ) + bitstream_out = Path(tmp_dir).joinpath(bitstream.stem + f".{processing}.192") + if processing == BS_PROC_FER_15: ep_path = TESTV_PATH.joinpath("ltv_ep_015.192") cmd = [ + # TODO: adapt for windows "eid-xor", "-fer", "-vbr", @@ -80,8 +81,27 @@ def get_bitstream( ] subprocess.run(cmd) bitstream = bitstream_out + elif processing == BS_PROC_JBM_DLY_PROF_5: + dly_profile_path = HERE.joinpath( + "../../scripts/dly_error_profiles/dly_error_profile_5.dat" + ) + tracefile_path = Path(tmp_dir).joinpath(bitstream_out.with_suffix(".trace")) + # TODO: adapt for windows + # TODO: get number of frames per packet from error profile name + cmd = [ + "networkSimulator_g192", + str(dly_profile_path), + str(bitstream), + str(bitstream_out), + str(tracefile_path), + "2", + "0", + ] + subprocess.run(cmd) + bitstream = bitstream_out + options.append("-VOIP") - yield bitstream + yield bitstream, options def run_check( @@ -89,10 +109,13 @@ def run_check( output_format, sampling_rate, dut_output, + options, decoder_frontend, is_ref_creation, ): - decoder_frontend.run(output_format, sampling_rate, ref_bitstream, dut_output) + decoder_frontend.run( + output_format, sampling_rate, ref_bitstream, dut_output, add_option_list=options + ) if not is_ref_creation and not is_be_to_ref(dut_output): pytest.fail(f"Decoder output differs from reference") @@ -117,14 +140,14 @@ def test_decoder_channelbased_and_masa( update_ref, ): testv_name = get_testv_path(input_format, input_sampling_rate).stem - with get_bitstream( + with get_bitstream_and_options( testv_name, input_format, bitrate, input_sampling_rate, dtx, processing=bitstream_processing, - ) as ref_bitstream: + ) as (ref_bitstream, options): dut_output = get_output_path(ref_bitstream, output_format, output_sampling_rate) run_check( @@ -132,6 +155,7 @@ def test_decoder_channelbased_and_masa( output_format, output_sampling_rate, str(dut_output), + options, dut_decoder_frontend, update_ref == 1, ) @@ -159,7 +183,7 @@ def test_decoder_objectbased( suffix = "_ext_MD" elif md_type == ISM_MD_NULL: suffix = "_null_MD" - with get_bitstream( + with get_bitstream_and_options( testv_name, input_format, bitrate, @@ -167,7 +191,7 @@ def test_decoder_objectbased( dtx, suffix=suffix, processing=bitstream_processing, - ) as ref_bitstream: + ) as (ref_bitstream, options): dut_output = get_output_path(ref_bitstream, output_format, output_sampling_rate) run_check( @@ -177,6 +201,7 @@ def test_decoder_objectbased( str(dut_output), dut_decoder_frontend, update_ref == 1, + add_option_list=options ) @@ -200,7 +225,7 @@ def test_decoder_scenebased( suffix = "" if pca == SBA_FOA_PCA_ON: suffix += "-pca" - with get_bitstream( + with get_bitstream_and_options( testv_name, input_format, bitrate, @@ -208,7 +233,7 @@ def test_decoder_scenebased( dtx, suffix=suffix, processing=bitstream_processing, - ) as ref_bitstream: + ) as (ref_bitstream, options): dut_output = get_output_path(ref_bitstream, output_format, output_sampling_rate) run_check( @@ -218,6 +243,7 @@ def test_decoder_scenebased( str(dut_output), dut_decoder_frontend, update_ref == 1, + add_option_list=options ) @@ -241,13 +267,13 @@ def test_decoder_combined_formats( testv_name = get_testv_path( f"{input_format_combined}_{input_format_ism}_{input_format_other}", input_sampling_rate, - ).stem + ).stem suffix = "_basic_MD" if md_type == ISM_MD_EXTENDED: suffix = "_ext_MD" elif md_type == ISM_MD_NULL: suffix = "_null_MD" - with get_bitstream( + with get_bitstream_and_options( testv_name, input_format_combined, bitrate, @@ -255,7 +281,7 @@ def test_decoder_combined_formats( DTX_OFF, suffix=suffix, processing=bitstream_processing, - ) as ref_bitstream: + ) as (ref_bitstream, options): dut_output = get_output_path(ref_bitstream, output_format, output_sampling_rate) run_check( @@ -265,4 +291,5 @@ def test_decoder_combined_formats( str(dut_output), dut_decoder_frontend, update_ref == 1, + add_option_list=options ) diff --git a/tests/codec_be_to_accepted_release/test_encoder.py b/tests/codec_be_to_accepted_release/test_encoder.py index 769fb22d4e..5b3d32689d 100644 --- a/tests/codec_be_to_accepted_release/test_encoder.py +++ b/tests/codec_be_to_accepted_release/test_encoder.py @@ -242,7 +242,13 @@ def test_encoder_combined_formats( elif md_type == ISM_MD_NULL: suffix = "_null_MD" bitstream = get_bitstream_path( - DUT_PATH, testv.stem, input_format_combined, bitrate, sampling_rate, DTX_OFF, suffix=suffix + DUT_PATH, + testv.stem, + input_format_combined, + bitrate, + sampling_rate, + DTX_OFF, + suffix=suffix, ) run_check( bitstream, -- GitLab From 14b375eb1eceb37a195b5d4bd46adcf4c8c49863 Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Wed, 30 Aug 2023 15:21:28 +0200 Subject: [PATCH 019/106] add function for br switching file creation --- .../test_encoder.py | 45 +++++++++++++++++++ 1 file changed, 45 insertions(+) diff --git a/tests/codec_be_to_accepted_release/test_encoder.py b/tests/codec_be_to_accepted_release/test_encoder.py index 5b3d32689d..78ed719eab 100644 --- a/tests/codec_be_to_accepted_release/test_encoder.py +++ b/tests/codec_be_to_accepted_release/test_encoder.py @@ -29,6 +29,9 @@ the United Nations Convention on Contracts on the International Sales of Goods. """ import pytest +import numpy as np +from itertools import permutations +import random from .constants import ( METADATA_FOR_INPUT_FORMAT, ISM_MD_NULL, @@ -50,6 +53,48 @@ from . import TESTV_PATH, DUT_PATH, is_be_to_ref, get_bitstream_path, get_testv_ ### --------------- Helper functions --------------- +def create_br_switching_file(bitrates: np.ndarray, strategy: str, switch_time: int, length_frames: int, seed: int=None): + """ + Create bitrate switching pattern files on the fly and return path to it + + bitrates - array of bitrate values to include + strategy - how to create pattern: + "from_list" - use given array as is + "exhaustive" - generate array where every bitrate is preceeded and followed at least once by every other bitrate + "random" - randomly pick from bitrates + switch_time - number of frames before next switch + length_frames - for "random" strategy: length of pattern in frames + seed - for "exhaustive" and "random" strategies: inject seed for shuffling/choosing from array + """ + assert strategy in ["from_list", "exhaustive", "random"] + + seed_str = f"_{seed}" if strategy == "random" else "" + fname = f"br_sw_pattern-{strategy}{seed_str}-{switch_time}" + + # TODO: maybe calculate seed based on given parameter to automatically have reproducibility + if seed is not None: + random.seed(seed) + + if strategy == "from_list": + brs = bitrates.astype(np.int32) + elif strategy == "exhaustive": + # TODO: review, probably needs more work to be really exhaustive... + n = len(bitrates) - 1 + permuts = permutations(bitrates, 2) + split_permuts = [permuts[i * n: (i + 1) * n] for i in range(len(bitrates))] + for i in range(len(split_permuts)): + random.shuffle(split_permuts[i]) + split_permuts = np.asarray(split_permuts) + brs = np.concatenate([split_permuts[:, i] for i in range(n)]).astype(np.int32) + elif strategy == "random": + brs = np.asarray(random.choices(bitrates, k=length_frames)).astype(np.int32) + + out_path = DUT_PATH.joinpath(fname) + brs.astype.tofile(out_path) + + return out_path + + def get_md(input_format, md_type=None): md_files = METADATA_FOR_INPUT_FORMAT.get(input_format, list()) if md_type == ISM_MD_NULL: -- GitLab From 0babb6b94fd7bfed727f5b9f1df68af7d0b89474 Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Wed, 30 Aug 2023 16:31:18 +0200 Subject: [PATCH 020/106] add 1st version util functions for head rot and br switching --- .../test_decoder.py | 37 +++++++++++++++++-- .../test_encoder.py | 36 +++++++++++------- 2 files changed, 56 insertions(+), 17 deletions(-) diff --git a/tests/codec_be_to_accepted_release/test_decoder.py b/tests/codec_be_to_accepted_release/test_decoder.py index cb14debde5..f4e0799356 100644 --- a/tests/codec_be_to_accepted_release/test_decoder.py +++ b/tests/codec_be_to_accepted_release/test_decoder.py @@ -29,6 +29,7 @@ the United Nations Convention on Contracts on the International Sales of Goods. """ import pytest +import numpy as np from . import ( is_be_to_ref, get_bitstream_path, @@ -41,6 +42,7 @@ from . import ( from .constants import * from pathlib import Path import subprocess +import random from contextlib import contextmanager from tempfile import TemporaryDirectory @@ -48,6 +50,32 @@ from tempfile import TemporaryDirectory ### --------------- Helper functions --------------- +def create_head_rotation_file( + trajectory: np.ndarray, + strategy: str, + switch_time: int, + length_frames: int, + axis: str = None, +): + assert strategy in ["from_array", "rotate_axis", "constant"] + + length_str = f"-l{length_frames}" if length_frames > 0 else "" + fname = f"head_rotation_trajectory-{strategy}-{switch_time}{length_str}" + + if strategy == "from_array": + rots = trajectory + elif strategy == "rotate_axis": + raise NotImplementedError("only idea for now") + elif strategy == "constant": + assert len(trajectory) == 1 + rots = np.repeat(trajectory, length_frames) + + out_path = DUT_PATH.joinpath(fname) + rots.astype(np.int32).tofile(out_path) + + return out_path + + def get_output_path(bitstream_path, output_format, output_sampling_rate): output_name = ( f"{bitstream_path.stem}.dec-{output_format}-{output_sampling_rate}kHz.wav" @@ -61,6 +89,9 @@ def get_bitstream_and_options( ): """ Utility to get either the stored reference bitstream or the processed version as a temporary file + + The reason for implementing this as a context manager instead of a simple function is that this way + the temporary processed bitstream files are cleaned up automatically and do not exploce disk space even more... """ options = list() with TemporaryDirectory() as tmp_dir: @@ -201,7 +232,7 @@ def test_decoder_objectbased( str(dut_output), dut_decoder_frontend, update_ref == 1, - add_option_list=options + add_option_list=options, ) @@ -243,7 +274,7 @@ def test_decoder_scenebased( str(dut_output), dut_decoder_frontend, update_ref == 1, - add_option_list=options + add_option_list=options, ) @@ -291,5 +322,5 @@ def test_decoder_combined_formats( str(dut_output), dut_decoder_frontend, update_ref == 1, - add_option_list=options + add_option_list=options, ) diff --git a/tests/codec_be_to_accepted_release/test_encoder.py b/tests/codec_be_to_accepted_release/test_encoder.py index 78ed719eab..16c42a2b43 100644 --- a/tests/codec_be_to_accepted_release/test_encoder.py +++ b/tests/codec_be_to_accepted_release/test_encoder.py @@ -53,44 +53,52 @@ from . import TESTV_PATH, DUT_PATH, is_be_to_ref, get_bitstream_path, get_testv_ ### --------------- Helper functions --------------- -def create_br_switching_file(bitrates: np.ndarray, strategy: str, switch_time: int, length_frames: int, seed: int=None): +def create_br_switching_file( + bitrates: np.ndarray, + strategy: str, + switch_time: int, + length_frames: int = 0, + seed: int = None, +): """ Create bitrate switching pattern files on the fly and return path to it bitrates - array of bitrate values to include strategy - how to create pattern: - "from_list" - use given array as is + "from_array" - use given array as is "exhaustive" - generate array where every bitrate is preceeded and followed at least once by every other bitrate "random" - randomly pick from bitrates switch_time - number of frames before next switch length_frames - for "random" strategy: length of pattern in frames seed - for "exhaustive" and "random" strategies: inject seed for shuffling/choosing from array """ - assert strategy in ["from_list", "exhaustive", "random"] + assert strategy in ["from_array", "exhaustive", "random"] - seed_str = f"_{seed}" if strategy == "random" else "" - fname = f"br_sw_pattern-{strategy}{seed_str}-{switch_time}" + seed_str = f"_{seed}" if strategy == "random" or strategy == "exhaustive" else "" + length_str = f"-l{length_frames}" if length_frames > 0 else "" + fname = f"br_sw_pattern-{strategy}{seed_str}-{switch_time}{length_str}" # TODO: maybe calculate seed based on given parameter to automatically have reproducibility if seed is not None: random.seed(seed) - if strategy == "from_list": - brs = bitrates.astype(np.int32) - elif strategy == "exhaustive": + brs = np.asarray(bitrates) + + # NOTE: nothing to do for from_array + if strategy == "exhaustive": # TODO: review, probably needs more work to be really exhaustive... - n = len(bitrates) - 1 - permuts = permutations(bitrates, 2) - split_permuts = [permuts[i * n: (i + 1) * n] for i in range(len(bitrates))] + n = len(brs) - 1 + permuts = list(permutations(brs, 2)) + split_permuts = [permuts[i * n : (i + 1) * n] for i in range(len(brs))] for i in range(len(split_permuts)): random.shuffle(split_permuts[i]) split_permuts = np.asarray(split_permuts) - brs = np.concatenate([split_permuts[:, i] for i in range(n)]).astype(np.int32) + brs = np.concatenate([split_permuts[:, i] for i in range(n)]) elif strategy == "random": - brs = np.asarray(random.choices(bitrates, k=length_frames)).astype(np.int32) + brs = np.asarray(random.choices(brs, k=length_frames)) out_path = DUT_PATH.joinpath(fname) - brs.astype.tofile(out_path) + brs.astype(np.int32).tofile(out_path) return out_path -- GitLab From 43e4bdc1adf5e4c73658204e4d5703bc446a0a6e Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Wed, 30 Aug 2023 17:12:33 +0200 Subject: [PATCH 021/106] refactor encoder test organization --- .../encoder/__init__.py | 127 ++++++++++++++++++ .../test_encoder_constant_bitrate.py} | 109 +-------------- 2 files changed, 130 insertions(+), 106 deletions(-) create mode 100644 tests/codec_be_to_accepted_release/encoder/__init__.py rename tests/codec_be_to_accepted_release/{test_encoder.py => encoder/test_encoder_constant_bitrate.py} (65%) diff --git a/tests/codec_be_to_accepted_release/encoder/__init__.py b/tests/codec_be_to_accepted_release/encoder/__init__.py new file mode 100644 index 0000000000..a87562a381 --- /dev/null +++ b/tests/codec_be_to_accepted_release/encoder/__init__.py @@ -0,0 +1,127 @@ +__copyright__ = """ +(C) 2022-2023 IVAS codec Public Collaboration with portions copyright Dolby International AB, Ericsson AB, +Fraunhofer-Gesellschaft zur Foerderung der angewandten Forschung e.V., Huawei Technologies Co. LTD., +Koninklijke Philips N.V., Nippon Telegraph and Telephone Corporation, Nokia Technologies Oy, Orange, +Panasonic Holdings Corporation, Qualcomm Technologies, Inc., VoiceAge Corporation, and other +contributors to this repository. All Rights Reserved. + +This software is protected by copyright law and by international treaties. +The IVAS codec Public Collaboration consisting of Dolby International AB, Ericsson AB, +Fraunhofer-Gesellschaft zur Foerderung der angewandten Forschung e.V., Huawei Technologies Co. LTD., +Koninklijke Philips N.V., Nippon Telegraph and Telephone Corporation, Nokia Technologies Oy, Orange, +Panasonic Holdings Corporation, Qualcomm Technologies, Inc., VoiceAge Corporation, and other +contributors to this repository retain full ownership rights in their respective contributions in +the software. This notice grants no license of any kind, including but not limited to patent +license, nor is any license granted by implication, estoppel or otherwise. + +Contributors are required to enter into the IVAS codec Public Collaboration agreement before making +contributions. + +This software is provided "AS IS", without any express or implied warranties. The software is in the +development stage. It is intended exclusively for experts who have experience with such software and +solely for the purpose of inspection. All implied warranties of non-infringement, merchantability +and fitness for a particular purpose are hereby disclaimed and excluded. + +Any dispute, controversy or claim arising under or in relation to providing this software shall be +submitted to and settled by the final, binding jurisdiction of the courts of Munich, Germany in +accordance with the laws of the Federal Republic of Germany excluding its conflict of law rules and +the United Nations Convention on Contracts on the International Sales of Goods. +""" + +import numpy as np +import random +import pytest +from itertools import permutations +from .. import DUT_PATH, TESTV_PATH, is_be_to_ref +from ..constants import METADATA_FOR_INPUT_FORMAT, ISM_MD_NULL, CMDL_OPTIONS_FOR_INPUT_FORMAT, ISM_MD_EXTENDED, INPUT_FORMATS_OBJECT_BASED + +def create_br_switching_file( + bitrates: np.ndarray, + strategy: str, + switch_time: int, + length_frames: int = 0, + seed: int = None, +): + """ + Create bitrate switching pattern files on the fly and return path to it + + bitrates - array of bitrate values to include + strategy - how to create pattern: + "from_array" - use given array as is + "exhaustive" - generate array where every bitrate is preceeded and followed at least once by every other bitrate + "random" - randomly pick from bitrates + switch_time - number of frames before next switch + length_frames - for "random" strategy: length of pattern in frames + seed - for "exhaustive" and "random" strategies: inject seed for shuffling/choosing from array + """ + assert strategy in ["from_array", "exhaustive", "random"] + + seed_str = f"_{seed}" if strategy == "random" or strategy == "exhaustive" else "" + length_str = f"-l{length_frames}" if length_frames > 0 else "" + fname = f"br_sw_pattern-{strategy}{seed_str}-{switch_time}{length_str}" + + # TODO: maybe calculate seed based on given parameter to automatically have reproducibility + if seed is not None: + random.seed(seed) + + brs = np.asarray(bitrates) + + # NOTE: nothing to do for from_array + if strategy == "exhaustive": + # TODO: review, probably needs more work to be really exhaustive... + n = len(brs) - 1 + permuts = list(permutations(brs, 2)) + split_permuts = [permuts[i * n : (i + 1) * n] for i in range(len(brs))] + for i in range(len(split_permuts)): + random.shuffle(split_permuts[i]) + split_permuts = np.asarray(split_permuts) + brs = np.concatenate([split_permuts[:, i] for i in range(n)]) + elif strategy == "random": + brs = np.asarray(random.choices(brs, k=length_frames)) + + out_path = DUT_PATH.joinpath(fname) + brs.astype(np.int32).tofile(out_path) + + return out_path + + +def get_md(input_format, md_type=None): + md_files = METADATA_FOR_INPUT_FORMAT.get(input_format, list()) + if md_type == ISM_MD_NULL: + md_files = ["NULL" for f in md_files] + else: + md_files = [str(TESTV_PATH.joinpath(f)) for f in md_files] + return md_files + + +def get_options(input_format, md_type=None): + options = list(CMDL_OPTIONS_FOR_INPUT_FORMAT[input_format]) + if md_type == ISM_MD_EXTENDED: + assert input_format in INPUT_FORMATS_OBJECT_BASED + options[1] = f"+{options[1]}" + md_options = get_md(input_format, md_type) + options.extend(md_options) + return options + + +def run_check( + dut_bitstream, + bitrate, + sampling_rate, + testv, + options, + encoder_frontend, + is_ref_creation, + dtx_mode=False, +): + encoder_frontend.run( + bitrate, + sampling_rate, + str(testv), + str(dut_bitstream), + dtx_mode=dtx_mode, + add_option_list=options, + ) + + if not is_ref_creation and not is_be_to_ref(dut_bitstream): + pytest.fail(f"Bitstream file differs from reference") \ No newline at end of file diff --git a/tests/codec_be_to_accepted_release/test_encoder.py b/tests/codec_be_to_accepted_release/encoder/test_encoder_constant_bitrate.py similarity index 65% rename from tests/codec_be_to_accepted_release/test_encoder.py rename to tests/codec_be_to_accepted_release/encoder/test_encoder_constant_bitrate.py index 16c42a2b43..e3d6b0b2f3 100644 --- a/tests/codec_be_to_accepted_release/test_encoder.py +++ b/tests/codec_be_to_accepted_release/encoder/test_encoder_constant_bitrate.py @@ -29,15 +29,9 @@ the United Nations Convention on Contracts on the International Sales of Goods. """ import pytest -import numpy as np -from itertools import permutations -import random -from .constants import ( - METADATA_FOR_INPUT_FORMAT, +from ..constants import ( ISM_MD_NULL, - CMDL_OPTIONS_FOR_INPUT_FORMAT, ISM_MD_EXTENDED, - INPUT_FORMATS_OBJECT_BASED, ENCODER_CHANNEL_BASED_AND_MASA_PARAMS, DTX_ON, DTX_OFF, @@ -47,105 +41,8 @@ from .constants import ( STEREO_DMX_EVS_PARAMS, COMBINED_FORMATS_PARAMS, ) -from . import TESTV_PATH, DUT_PATH, is_be_to_ref, get_bitstream_path, get_testv_path - - -### --------------- Helper functions --------------- - - -def create_br_switching_file( - bitrates: np.ndarray, - strategy: str, - switch_time: int, - length_frames: int = 0, - seed: int = None, -): - """ - Create bitrate switching pattern files on the fly and return path to it - - bitrates - array of bitrate values to include - strategy - how to create pattern: - "from_array" - use given array as is - "exhaustive" - generate array where every bitrate is preceeded and followed at least once by every other bitrate - "random" - randomly pick from bitrates - switch_time - number of frames before next switch - length_frames - for "random" strategy: length of pattern in frames - seed - for "exhaustive" and "random" strategies: inject seed for shuffling/choosing from array - """ - assert strategy in ["from_array", "exhaustive", "random"] - - seed_str = f"_{seed}" if strategy == "random" or strategy == "exhaustive" else "" - length_str = f"-l{length_frames}" if length_frames > 0 else "" - fname = f"br_sw_pattern-{strategy}{seed_str}-{switch_time}{length_str}" - - # TODO: maybe calculate seed based on given parameter to automatically have reproducibility - if seed is not None: - random.seed(seed) - - brs = np.asarray(bitrates) - - # NOTE: nothing to do for from_array - if strategy == "exhaustive": - # TODO: review, probably needs more work to be really exhaustive... - n = len(brs) - 1 - permuts = list(permutations(brs, 2)) - split_permuts = [permuts[i * n : (i + 1) * n] for i in range(len(brs))] - for i in range(len(split_permuts)): - random.shuffle(split_permuts[i]) - split_permuts = np.asarray(split_permuts) - brs = np.concatenate([split_permuts[:, i] for i in range(n)]) - elif strategy == "random": - brs = np.asarray(random.choices(brs, k=length_frames)) - - out_path = DUT_PATH.joinpath(fname) - brs.astype(np.int32).tofile(out_path) - - return out_path - - -def get_md(input_format, md_type=None): - md_files = METADATA_FOR_INPUT_FORMAT.get(input_format, list()) - if md_type == ISM_MD_NULL: - md_files = ["NULL" for f in md_files] - else: - md_files = [str(TESTV_PATH.joinpath(f)) for f in md_files] - return md_files - - -def get_options(input_format, md_type=None): - options = list(CMDL_OPTIONS_FOR_INPUT_FORMAT[input_format]) - if md_type == ISM_MD_EXTENDED: - assert input_format in INPUT_FORMATS_OBJECT_BASED - options[1] = f"+{options[1]}" - md_options = get_md(input_format, md_type) - options.extend(md_options) - return options - - -def run_check( - dut_bitstream, - bitrate, - sampling_rate, - testv, - options, - encoder_frontend, - is_ref_creation, - dtx_mode=False, -): - encoder_frontend.run( - bitrate, - sampling_rate, - str(testv), - str(dut_bitstream), - dtx_mode=dtx_mode, - add_option_list=options, - ) - - if not is_ref_creation and not is_be_to_ref(dut_bitstream): - pytest.fail(f"Bitstream file differs from reference") - - -### --------------- Actual testcase definitions --------------- +from . import get_options, run_check +from .. import get_testv_path, get_bitstream_path, DUT_PATH # channel-based modes + MASA -- GitLab From 54159e54429c2d19c0b23269fb22b257857fb159 Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Wed, 30 Aug 2023 17:33:07 +0200 Subject: [PATCH 022/106] restructure decoder tests as well --- .../decoder/__init__.py | 140 ++++++++++++++++++ .../test_decoder_constant_bitrate.py} | 124 +--------------- 2 files changed, 143 insertions(+), 121 deletions(-) create mode 100644 tests/codec_be_to_accepted_release/decoder/__init__.py rename tests/codec_be_to_accepted_release/{test_decoder.py => decoder/test_decoder_constant_bitrate.py} (65%) diff --git a/tests/codec_be_to_accepted_release/decoder/__init__.py b/tests/codec_be_to_accepted_release/decoder/__init__.py new file mode 100644 index 0000000000..0ea0f14843 --- /dev/null +++ b/tests/codec_be_to_accepted_release/decoder/__init__.py @@ -0,0 +1,140 @@ +__copyright__ = """ +(C) 2022-2023 IVAS codec Public Collaboration with portions copyright Dolby International AB, Ericsson AB, +Fraunhofer-Gesellschaft zur Foerderung der angewandten Forschung e.V., Huawei Technologies Co. LTD., +Koninklijke Philips N.V., Nippon Telegraph and Telephone Corporation, Nokia Technologies Oy, Orange, +Panasonic Holdings Corporation, Qualcomm Technologies, Inc., VoiceAge Corporation, and other +contributors to this repository. All Rights Reserved. + +This software is protected by copyright law and by international treaties. +The IVAS codec Public Collaboration consisting of Dolby International AB, Ericsson AB, +Fraunhofer-Gesellschaft zur Foerderung der angewandten Forschung e.V., Huawei Technologies Co. LTD., +Koninklijke Philips N.V., Nippon Telegraph and Telephone Corporation, Nokia Technologies Oy, Orange, +Panasonic Holdings Corporation, Qualcomm Technologies, Inc., VoiceAge Corporation, and other +contributors to this repository retain full ownership rights in their respective contributions in +the software. This notice grants no license of any kind, including but not limited to patent +license, nor is any license granted by implication, estoppel or otherwise. + +Contributors are required to enter into the IVAS codec Public Collaboration agreement before making +contributions. + +This software is provided "AS IS", without any express or implied warranties. The software is in the +development stage. It is intended exclusively for experts who have experience with such software and +solely for the purpose of inspection. All implied warranties of non-infringement, merchantability +and fitness for a particular purpose are hereby disclaimed and excluded. + +Any dispute, controversy or claim arising under or in relation to providing this software shall be +submitted to and settled by the final, binding jurisdiction of the courts of Munich, Germany in +accordance with the laws of the Federal Republic of Germany excluding its conflict of law rules and +the United Nations Convention on Contracts on the International Sales of Goods. +""" + +import numpy as np +import pytest +from contextlib import contextmanager +from pathlib import Path +from tempfile import TemporaryDirectory +import subprocess +from .. import DUT_PATH, REF_PATH, TESTV_PATH, HERE, is_be_to_ref, get_bitstream_path +from ..constants import BS_PROC_FER_15, BS_PROC_JBM_DLY_PROF_5 + + +def create_head_rotation_file( + trajectory: np.ndarray, + strategy: str, + switch_time: int, + length_frames: int, + axis: str = None, +): + assert strategy in ["from_array", "rotate_axis", "constant"] + + length_str = f"-l{length_frames}" if length_frames > 0 else "" + fname = f"head_rotation_trajectory-{strategy}-{switch_time}{length_str}" + + if strategy == "from_array": + rots = trajectory + elif strategy == "rotate_axis": + raise NotImplementedError("only idea for now") + elif strategy == "constant": + assert len(trajectory) == 1 + rots = np.repeat(trajectory, length_frames) + + out_path = DUT_PATH.joinpath(fname) + rots.astype(np.int32).tofile(out_path) + + return out_path + + +def get_output_path(bitstream_path, output_format, output_sampling_rate): + output_name = ( + f"{bitstream_path.stem}.dec-{output_format}-{output_sampling_rate}kHz.wav" + ) + return DUT_PATH.joinpath(output_name) + + +@contextmanager +def get_bitstream_and_options( + testv_name, encoder_format, bitrate, sampling_rate, dtx, processing, suffix="" +): + """ + Utility to get either the stored reference bitstream or the processed version as a temporary file + + The reason for implementing this as a context manager instead of a simple function is that this way + the temporary processed bitstream files are cleaned up automatically and do not exploce disk space even more... + """ + options = list() + with TemporaryDirectory() as tmp_dir: + bitstream = get_bitstream_path( + REF_PATH, testv_name, encoder_format, bitrate, sampling_rate, dtx, suffix + ) + bitstream_out = Path(tmp_dir).joinpath(bitstream.stem + f".{processing}.192") + if processing == BS_PROC_FER_15: + ep_path = TESTV_PATH.joinpath("ltv_ep_015.192") + cmd = [ + # TODO: adapt for windows + "eid-xor", + "-fer", + "-vbr", + str(bitstream), + str(ep_path), + str(bitstream_out), + ] + subprocess.run(cmd) + bitstream = bitstream_out + elif processing == BS_PROC_JBM_DLY_PROF_5: + dly_profile_path = HERE.joinpath( + "../../scripts/dly_error_profiles/dly_error_profile_5.dat" + ) + tracefile_path = Path(tmp_dir).joinpath(bitstream_out.with_suffix(".trace")) + # TODO: adapt for windows + # TODO: get number of frames per packet from error profile name + cmd = [ + "networkSimulator_g192", + str(dly_profile_path), + str(bitstream), + str(bitstream_out), + str(tracefile_path), + "2", + "0", + ] + subprocess.run(cmd) + bitstream = bitstream_out + options.append("-VOIP") + + yield bitstream, options + + +def run_check( + ref_bitstream, + output_format, + sampling_rate, + dut_output, + options, + decoder_frontend, + is_ref_creation, +): + decoder_frontend.run( + output_format, sampling_rate, ref_bitstream, dut_output, add_option_list=options + ) + + if not is_ref_creation and not is_be_to_ref(dut_output): + pytest.fail(f"Decoder output differs from reference") diff --git a/tests/codec_be_to_accepted_release/test_decoder.py b/tests/codec_be_to_accepted_release/decoder/test_decoder_constant_bitrate.py similarity index 65% rename from tests/codec_be_to_accepted_release/test_decoder.py rename to tests/codec_be_to_accepted_release/decoder/test_decoder_constant_bitrate.py index f4e0799356..f836f923e1 100644 --- a/tests/codec_be_to_accepted_release/test_decoder.py +++ b/tests/codec_be_to_accepted_release/decoder/test_decoder_constant_bitrate.py @@ -29,127 +29,9 @@ the United Nations Convention on Contracts on the International Sales of Goods. """ import pytest -import numpy as np -from . import ( - is_be_to_ref, - get_bitstream_path, - get_testv_path, - REF_PATH, - DUT_PATH, - TESTV_PATH, - HERE, -) -from .constants import * -from pathlib import Path -import subprocess -import random -from contextlib import contextmanager -from tempfile import TemporaryDirectory - - -### --------------- Helper functions --------------- - - -def create_head_rotation_file( - trajectory: np.ndarray, - strategy: str, - switch_time: int, - length_frames: int, - axis: str = None, -): - assert strategy in ["from_array", "rotate_axis", "constant"] - - length_str = f"-l{length_frames}" if length_frames > 0 else "" - fname = f"head_rotation_trajectory-{strategy}-{switch_time}{length_str}" - - if strategy == "from_array": - rots = trajectory - elif strategy == "rotate_axis": - raise NotImplementedError("only idea for now") - elif strategy == "constant": - assert len(trajectory) == 1 - rots = np.repeat(trajectory, length_frames) - - out_path = DUT_PATH.joinpath(fname) - rots.astype(np.int32).tofile(out_path) - - return out_path - - -def get_output_path(bitstream_path, output_format, output_sampling_rate): - output_name = ( - f"{bitstream_path.stem}.dec-{output_format}-{output_sampling_rate}kHz.wav" - ) - return DUT_PATH.joinpath(output_name) - - -@contextmanager -def get_bitstream_and_options( - testv_name, encoder_format, bitrate, sampling_rate, dtx, processing, suffix="" -): - """ - Utility to get either the stored reference bitstream or the processed version as a temporary file - - The reason for implementing this as a context manager instead of a simple function is that this way - the temporary processed bitstream files are cleaned up automatically and do not exploce disk space even more... - """ - options = list() - with TemporaryDirectory() as tmp_dir: - bitstream = get_bitstream_path( - REF_PATH, testv_name, encoder_format, bitrate, sampling_rate, dtx, suffix - ) - bitstream_out = Path(tmp_dir).joinpath(bitstream.stem + f".{processing}.192") - if processing == BS_PROC_FER_15: - ep_path = TESTV_PATH.joinpath("ltv_ep_015.192") - cmd = [ - # TODO: adapt for windows - "eid-xor", - "-fer", - "-vbr", - str(bitstream), - str(ep_path), - str(bitstream_out), - ] - subprocess.run(cmd) - bitstream = bitstream_out - elif processing == BS_PROC_JBM_DLY_PROF_5: - dly_profile_path = HERE.joinpath( - "../../scripts/dly_error_profiles/dly_error_profile_5.dat" - ) - tracefile_path = Path(tmp_dir).joinpath(bitstream_out.with_suffix(".trace")) - # TODO: adapt for windows - # TODO: get number of frames per packet from error profile name - cmd = [ - "networkSimulator_g192", - str(dly_profile_path), - str(bitstream), - str(bitstream_out), - str(tracefile_path), - "2", - "0", - ] - subprocess.run(cmd) - bitstream = bitstream_out - options.append("-VOIP") - - yield bitstream, options - - -def run_check( - ref_bitstream, - output_format, - sampling_rate, - dut_output, - options, - decoder_frontend, - is_ref_creation, -): - decoder_frontend.run( - output_format, sampling_rate, ref_bitstream, dut_output, add_option_list=options - ) - - if not is_ref_creation and not is_be_to_ref(dut_output): - pytest.fail(f"Decoder output differs from reference") +from . import get_bitstream_and_options, get_output_path, run_check +from .. import get_testv_path +from ..constants import * ### --------------- Actual testcase definitions --------------- -- GitLab From c72efafb1b52265bdbb008992893317c46cc1253 Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Thu, 31 Aug 2023 10:57:44 +0200 Subject: [PATCH 023/106] factor out binaural testcases for faster test collection --- .../codec_be_to_accepted_release/constants.py | 55 ++++- ...test_decoder_constant_bitrate_binaural.py} | 11 +- ...st_decoder_constant_bitrate_no_binaural.py | 205 ++++++++++++++++++ 3 files changed, 255 insertions(+), 16 deletions(-) rename tests/codec_be_to_accepted_release/decoder/{test_decoder_constant_bitrate.py => test_decoder_constant_bitrate_binaural.py} (97%) create mode 100644 tests/codec_be_to_accepted_release/decoder/test_decoder_constant_bitrate_no_binaural.py diff --git a/tests/codec_be_to_accepted_release/constants.py b/tests/codec_be_to_accepted_release/constants.py index e278e6ccf3..7f8b383389 100644 --- a/tests/codec_be_to_accepted_release/constants.py +++ b/tests/codec_be_to_accepted_release/constants.py @@ -208,6 +208,7 @@ OUTPUT_FORMATS_ALL = [ "BINAURAL_ROOM_REVERB", ] OUTPUT_FORMATS_BINAURAL = OUTPUT_FORMATS_ALL[-3:] +OUTPUT_FORMATS_NON_BINAURAL = [f for f in OUTPUT_FORMATS_ALL if f not in OUTPUT_FORMATS_BINAURAL] ### -------------- encoder parameter lists -------------- @@ -379,7 +380,8 @@ BS_PROC_FER_15 = "FER_15perc" BS_PROC_JBM_DLY_PROF_5 = "JBM_dlyprofile5" BITSTREAM_PROCESSING = [BS_PROC_NONE, BS_PROC_FER_15, BS_PROC_JBM_DLY_PROF_5] -DECODER_CHANNELBASED_AND_MASA_PARAMS = ( +# parameters for const bitrate test with non-binaural output +DECODER_CONST_BR_NO_BINAURAL_CHANNELBASED_AND_MASA = ( collapse_into_list_of_pairs( product( STEREO_PARAMS, @@ -391,7 +393,7 @@ DECODER_CHANNELBASED_AND_MASA_PARAMS = ( + collapse_into_list_of_pairs( product( MC_PARAMS, - OUTPUT_FORMATS_ALL[:10] + OUTPUT_FORMATS_BINAURAL, + OUTPUT_FORMATS_ALL[:10], SAMPLING_RATES_ALL, BITSTREAM_PROCESSING, ) @@ -399,23 +401,58 @@ DECODER_CHANNELBASED_AND_MASA_PARAMS = ( + collapse_into_list_of_pairs( product( MASA_PARAMS, - OUTPUT_FORMATS_ALL, + OUTPUT_FORMATS_NON_BINAURAL, SAMPLING_RATES_ALL, BITSTREAM_PROCESSING, ) ) ) -DECODER_OBJECTBASED = collapse_into_list_of_pairs( - product(ISM_PARAMS, OUTPUT_FORMATS_ALL, SAMPLING_RATES_ALL, BITSTREAM_PROCESSING) +DECODER_CONST_BR_NO_BINAURAL_OBJECTBASED = collapse_into_list_of_pairs( + product(ISM_PARAMS, OUTPUT_FORMATS_NON_BINAURAL, SAMPLING_RATES_ALL, BITSTREAM_PROCESSING) ) -DECODER_SCENEBASED = collapse_into_list_of_pairs( - product(SBA_PARAMS, OUTPUT_FORMATS_ALL, SAMPLING_RATES_ALL, BITSTREAM_PROCESSING) +DECODER_CONST_BR_NO_BINAURAL_SCENEBASED = collapse_into_list_of_pairs( + product(SBA_PARAMS, OUTPUT_FORMATS_NON_BINAURAL, SAMPLING_RATES_ALL, BITSTREAM_PROCESSING) ) -DECODER_COMBINED_PARAMS = collapse_into_list_of_pairs( +DECODER_CONST_BR_NO_BINAURAL_COMBINED_PARAMS = collapse_into_list_of_pairs( product( COMBINED_FORMATS_PARAMS, - OUTPUT_FORMATS_ALL, + OUTPUT_FORMATS_NON_BINAURAL, SAMPLING_RATES_ALL, BITSTREAM_PROCESSING, ) ) + +# parameters for const bitrate testcases with binaural output +# TODO: add parameters for all the binaural output options +DECODER_CONST_BR_BINAURAL_CHANNELBASED_AND_MASA = ( + collapse_into_list_of_pairs( + product( + MC_PARAMS, + OUTPUT_FORMATS_BINAURAL, + SAMPLING_RATES_ALL, + BITSTREAM_PROCESSING, + ) + ) + + collapse_into_list_of_pairs( + product( + MASA_PARAMS, + OUTPUT_FORMATS_BINAURAL, + SAMPLING_RATES_ALL, + BITSTREAM_PROCESSING, + ) + ) +) +DECODER_CONST_BR_BINAURAL_OBJECTBASED = collapse_into_list_of_pairs( + product(ISM_PARAMS, OUTPUT_FORMATS_BINAURAL, SAMPLING_RATES_ALL, BITSTREAM_PROCESSING) +) +DECODER_CONST_BR_BINAURAL_SCENEBASED = collapse_into_list_of_pairs( + product(SBA_PARAMS, OUTPUT_FORMATS_BINAURAL, SAMPLING_RATES_ALL, BITSTREAM_PROCESSING) +) +DECODER_CONST_BR_BINAURAL_COMBINED_PARAMS = collapse_into_list_of_pairs( + product( + COMBINED_FORMATS_PARAMS, + OUTPUT_FORMATS_BINAURAL, + SAMPLING_RATES_ALL, + BITSTREAM_PROCESSING, + ) +) \ No newline at end of file diff --git a/tests/codec_be_to_accepted_release/decoder/test_decoder_constant_bitrate.py b/tests/codec_be_to_accepted_release/decoder/test_decoder_constant_bitrate_binaural.py similarity index 97% rename from tests/codec_be_to_accepted_release/decoder/test_decoder_constant_bitrate.py rename to tests/codec_be_to_accepted_release/decoder/test_decoder_constant_bitrate_binaural.py index f836f923e1..7050d54323 100644 --- a/tests/codec_be_to_accepted_release/decoder/test_decoder_constant_bitrate.py +++ b/tests/codec_be_to_accepted_release/decoder/test_decoder_constant_bitrate_binaural.py @@ -34,12 +34,9 @@ from .. import get_testv_path from ..constants import * -### --------------- Actual testcase definitions --------------- - - @pytest.mark.parametrize( "input_format,bitrate,input_sampling_rate,dtx,output_format,output_sampling_rate,bitstream_processing", - DECODER_CHANNELBASED_AND_MASA_PARAMS, + DECODER_CONST_BR_BINAURAL_CHANNELBASED_AND_MASA, ) def test_decoder_channelbased_and_masa( input_format, @@ -76,7 +73,7 @@ def test_decoder_channelbased_and_masa( @pytest.mark.parametrize( "input_format,bitrate,input_sampling_rate,dtx,md_type,output_format,output_sampling_rate,bitstream_processing", - DECODER_OBJECTBASED, + DECODER_CONST_BR_BINAURAL_OBJECTBASED, ) def test_decoder_objectbased( input_format, @@ -120,7 +117,7 @@ def test_decoder_objectbased( @pytest.mark.parametrize( "input_format,bitrate,input_sampling_rate,dtx,pca,output_format,output_sampling_rate,bitstream_processing", - DECODER_SCENEBASED, + DECODER_CONST_BR_BINAURAL_SCENEBASED, ) def test_decoder_scenebased( input_format, @@ -162,7 +159,7 @@ def test_decoder_scenebased( @pytest.mark.parametrize( "input_format_ism,input_format_other,bitrate,input_sampling_rate,md_type,output_format,output_sampling_rate,bitstream_processing", - DECODER_COMBINED_PARAMS, + DECODER_CONST_BR_BINAURAL_COMBINED_PARAMS, ) def test_decoder_combined_formats( input_format_ism, diff --git a/tests/codec_be_to_accepted_release/decoder/test_decoder_constant_bitrate_no_binaural.py b/tests/codec_be_to_accepted_release/decoder/test_decoder_constant_bitrate_no_binaural.py new file mode 100644 index 0000000000..96d9af09ea --- /dev/null +++ b/tests/codec_be_to_accepted_release/decoder/test_decoder_constant_bitrate_no_binaural.py @@ -0,0 +1,205 @@ +__copyright__ = """ +(C) 2022-2023 IVAS codec Public Collaboration with portions copyright Dolby International AB, Ericsson AB, +Fraunhofer-Gesellschaft zur Foerderung der angewandten Forschung e.V., Huawei Technologies Co. LTD., +Koninklijke Philips N.V., Nippon Telegraph and Telephone Corporation, Nokia Technologies Oy, Orange, +Panasonic Holdings Corporation, Qualcomm Technologies, Inc., VoiceAge Corporation, and other +contributors to this repository. All Rights Reserved. + +This software is protected by copyright law and by international treaties. +The IVAS codec Public Collaboration consisting of Dolby International AB, Ericsson AB, +Fraunhofer-Gesellschaft zur Foerderung der angewandten Forschung e.V., Huawei Technologies Co. LTD., +Koninklijke Philips N.V., Nippon Telegraph and Telephone Corporation, Nokia Technologies Oy, Orange, +Panasonic Holdings Corporation, Qualcomm Technologies, Inc., VoiceAge Corporation, and other +contributors to this repository retain full ownership rights in their respective contributions in +the software. This notice grants no license of any kind, including but not limited to patent +license, nor is any license granted by implication, estoppel or otherwise. + +Contributors are required to enter into the IVAS codec Public Collaboration agreement before making +contributions. + +This software is provided "AS IS", without any express or implied warranties. The software is in the +development stage. It is intended exclusively for experts who have experience with such software and +solely for the purpose of inspection. All implied warranties of non-infringement, merchantability +and fitness for a particular purpose are hereby disclaimed and excluded. + +Any dispute, controversy or claim arising under or in relation to providing this software shall be +submitted to and settled by the final, binding jurisdiction of the courts of Munich, Germany in +accordance with the laws of the Federal Republic of Germany excluding its conflict of law rules and +the United Nations Convention on Contracts on the International Sales of Goods. +""" + +import pytest +from . import get_bitstream_and_options, get_output_path, run_check +from .. import get_testv_path +from ..constants import * + + +@pytest.mark.parametrize( + "input_format,bitrate,input_sampling_rate,dtx,output_format,output_sampling_rate,bitstream_processing", + DECODER_CONST_BR_NO_BINAURAL_CHANNELBASED_AND_MASA, +) +def test_decoder_channelbased_and_masa( + input_format, + bitrate, + input_sampling_rate, + dtx, + output_format, + output_sampling_rate, + bitstream_processing, + dut_decoder_frontend, + update_ref, +): + testv_name = get_testv_path(input_format, input_sampling_rate).stem + with get_bitstream_and_options( + testv_name, + input_format, + bitrate, + input_sampling_rate, + dtx, + processing=bitstream_processing, + ) as (ref_bitstream, options): + dut_output = get_output_path(ref_bitstream, output_format, output_sampling_rate) + + run_check( + str(ref_bitstream), + output_format, + output_sampling_rate, + str(dut_output), + options, + dut_decoder_frontend, + update_ref == 1, + ) + + +@pytest.mark.parametrize( + "input_format,bitrate,input_sampling_rate,dtx,md_type,output_format,output_sampling_rate,bitstream_processing", + DECODER_CONST_BR_NO_BINAURAL_OBJECTBASED, +) +def test_decoder_objectbased( + input_format, + bitrate, + input_sampling_rate, + dtx, + md_type, + output_format, + output_sampling_rate, + bitstream_processing, + dut_decoder_frontend, + update_ref, +): + testv_name = get_testv_path(input_format, input_sampling_rate).stem + suffix = "_basic_MD" + if md_type == ISM_MD_EXTENDED: + suffix = "_ext_MD" + elif md_type == ISM_MD_NULL: + suffix = "_null_MD" + with get_bitstream_and_options( + testv_name, + input_format, + bitrate, + input_sampling_rate, + dtx, + suffix=suffix, + processing=bitstream_processing, + ) as (ref_bitstream, options): + dut_output = get_output_path(ref_bitstream, output_format, output_sampling_rate) + + run_check( + str(ref_bitstream), + output_format, + output_sampling_rate, + str(dut_output), + dut_decoder_frontend, + update_ref == 1, + add_option_list=options, + ) + + +@pytest.mark.parametrize( + "input_format,bitrate,input_sampling_rate,dtx,pca,output_format,output_sampling_rate,bitstream_processing", + DECODER_CONST_BR_NO_BINAURAL_SCENEBASED, +) +def test_decoder_scenebased( + input_format, + bitrate, + input_sampling_rate, + dtx, + pca, + output_format, + output_sampling_rate, + bitstream_processing, + dut_decoder_frontend, + update_ref, +): + testv_name = get_testv_path(input_format, input_sampling_rate).stem + suffix = "" + if pca == SBA_FOA_PCA_ON: + suffix += "-pca" + with get_bitstream_and_options( + testv_name, + input_format, + bitrate, + input_sampling_rate, + dtx, + suffix=suffix, + processing=bitstream_processing, + ) as (ref_bitstream, options): + dut_output = get_output_path(ref_bitstream, output_format, output_sampling_rate) + + run_check( + str(ref_bitstream), + output_format, + output_sampling_rate, + str(dut_output), + dut_decoder_frontend, + update_ref == 1, + add_option_list=options, + ) + + +@pytest.mark.parametrize( + "input_format_ism,input_format_other,bitrate,input_sampling_rate,md_type,output_format,output_sampling_rate,bitstream_processing", + DECODER_CONST_BR_NO_BINAURAL_COMBINED_PARAMS, +) +def test_decoder_combined_formats( + input_format_ism, + input_format_other, + bitrate, + input_sampling_rate, + md_type, + output_format, + output_sampling_rate, + bitstream_processing, + dut_decoder_frontend, + update_ref, +): + input_format_combined = "OMASA" if "MASA" in input_format_other else "OSBA" + testv_name = get_testv_path( + f"{input_format_combined}_{input_format_ism}_{input_format_other}", + input_sampling_rate, + ).stem + suffix = "_basic_MD" + if md_type == ISM_MD_EXTENDED: + suffix = "_ext_MD" + elif md_type == ISM_MD_NULL: + suffix = "_null_MD" + with get_bitstream_and_options( + testv_name, + input_format_combined, + bitrate, + input_sampling_rate, + DTX_OFF, + suffix=suffix, + processing=bitstream_processing, + ) as (ref_bitstream, options): + dut_output = get_output_path(ref_bitstream, output_format, output_sampling_rate) + + run_check( + str(ref_bitstream), + output_format, + output_sampling_rate, + str(dut_output), + dut_decoder_frontend, + update_ref == 1, + add_option_list=options, + ) -- GitLab From d23eb6d26e3222d18cbe7ef1e06f7fc9569953dc Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Thu, 31 Aug 2023 11:20:43 +0200 Subject: [PATCH 024/106] also add STEREO -> MC decoding testcases --- tests/codec_be_to_accepted_release/constants.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/codec_be_to_accepted_release/constants.py b/tests/codec_be_to_accepted_release/constants.py index 7f8b383389..d5d3087ecf 100644 --- a/tests/codec_be_to_accepted_release/constants.py +++ b/tests/codec_be_to_accepted_release/constants.py @@ -385,7 +385,7 @@ DECODER_CONST_BR_NO_BINAURAL_CHANNELBASED_AND_MASA = ( collapse_into_list_of_pairs( product( STEREO_PARAMS, - OUTPUT_FORMATS_ALL[:2], + OUTPUT_FORMATS_ALL[:7], SAMPLING_RATES_ALL, BITSTREAM_PROCESSING, ) -- GitLab From c8ab3f0b09526da16858e36fbcd09c8dbbe41c3f Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Thu, 31 Aug 2023 16:22:09 +0200 Subject: [PATCH 025/106] add testcases with external hrtf file --- .../codec_be_to_accepted_release/__init__.py | 9 +-- .../codec_be_to_accepted_release/constants.py | 73 +++++++++++++------ .../decoder/__init__.py | 35 +++++++-- .../test_decoder_constant_bitrate_binaural.py | 27 +++++-- ...st_decoder_constant_bitrate_no_binaural.py | 16 +++- .../encoder/__init__.py | 28 ++++--- .../encoder/test_encoder_bitrate_switching.py | 62 ++++++++++++++++ .../encoder/test_encoder_constant_bitrate.py | 10 +-- 8 files changed, 199 insertions(+), 61 deletions(-) create mode 100644 tests/codec_be_to_accepted_release/encoder/test_encoder_bitrate_switching.py diff --git a/tests/codec_be_to_accepted_release/__init__.py b/tests/codec_be_to_accepted_release/__init__.py index eedb33110c..224f51d340 100644 --- a/tests/codec_be_to_accepted_release/__init__.py +++ b/tests/codec_be_to_accepted_release/__init__.py @@ -28,17 +28,10 @@ accordance with the laws of the Federal Republic of Germany excluding its confli the United Nations Convention on Contracts on the International Sales of Goods. """ -import os -from pathlib import Path import filecmp import subprocess from ..testconfig import MD5_REF_DICT -from .constants import TESTVECTOR_FOR_INPUT_FORMAT - -HERE = Path(__file__).parent -TESTV_PATH = Path(os.environ.get("BE_TESTV_DIR", HERE.joinpath("testv"))) -REF_PATH = Path(os.environ.get("BE_REF_DIR", HERE.joinpath("ref"))) -DUT_PATH = HERE.joinpath("dut") +from .constants import TESTVECTOR_FOR_INPUT_FORMAT, TESTV_PATH, REF_PATH def is_be_to_ref(dut_file): diff --git a/tests/codec_be_to_accepted_release/constants.py b/tests/codec_be_to_accepted_release/constants.py index d5d3087ecf..e7ecaab5ad 100644 --- a/tests/codec_be_to_accepted_release/constants.py +++ b/tests/codec_be_to_accepted_release/constants.py @@ -29,6 +29,13 @@ the United Nations Convention on Contracts on the International Sales of Goods. """ from itertools import product +from pathlib import Path +import os + +HERE = Path(__file__).parent +TESTV_PATH = Path(os.environ.get("BE_TESTV_DIR", HERE.joinpath("testv"))) +REF_PATH = Path(os.environ.get("BE_REF_DIR", HERE.joinpath("ref"))) +DUT_PATH = HERE.joinpath("dut") def collapse_into_list_of_pairs(params): @@ -208,7 +215,9 @@ OUTPUT_FORMATS_ALL = [ "BINAURAL_ROOM_REVERB", ] OUTPUT_FORMATS_BINAURAL = OUTPUT_FORMATS_ALL[-3:] -OUTPUT_FORMATS_NON_BINAURAL = [f for f in OUTPUT_FORMATS_ALL if f not in OUTPUT_FORMATS_BINAURAL] +OUTPUT_FORMATS_NON_BINAURAL = [ + f for f in OUTPUT_FORMATS_ALL if f not in OUTPUT_FORMATS_BINAURAL +] ### -------------- encoder parameter lists -------------- @@ -408,10 +417,20 @@ DECODER_CONST_BR_NO_BINAURAL_CHANNELBASED_AND_MASA = ( ) ) DECODER_CONST_BR_NO_BINAURAL_OBJECTBASED = collapse_into_list_of_pairs( - product(ISM_PARAMS, OUTPUT_FORMATS_NON_BINAURAL, SAMPLING_RATES_ALL, BITSTREAM_PROCESSING) + product( + ISM_PARAMS, + OUTPUT_FORMATS_NON_BINAURAL, + SAMPLING_RATES_ALL, + BITSTREAM_PROCESSING, + ) ) DECODER_CONST_BR_NO_BINAURAL_SCENEBASED = collapse_into_list_of_pairs( - product(SBA_PARAMS, OUTPUT_FORMATS_NON_BINAURAL, SAMPLING_RATES_ALL, BITSTREAM_PROCESSING) + product( + SBA_PARAMS, + OUTPUT_FORMATS_NON_BINAURAL, + SAMPLING_RATES_ALL, + BITSTREAM_PROCESSING, + ) ) DECODER_CONST_BR_NO_BINAURAL_COMBINED_PARAMS = collapse_into_list_of_pairs( product( @@ -424,29 +443,41 @@ DECODER_CONST_BR_NO_BINAURAL_COMBINED_PARAMS = collapse_into_list_of_pairs( # parameters for const bitrate testcases with binaural output # TODO: add parameters for all the binaural output options -DECODER_CONST_BR_BINAURAL_CHANNELBASED_AND_MASA = ( - collapse_into_list_of_pairs( - product( - MC_PARAMS, - OUTPUT_FORMATS_BINAURAL, - SAMPLING_RATES_ALL, - BITSTREAM_PROCESSING, - ) + +BINAURAL_HRTF_NONE = "HRTFdefault" +BINAURAL_HRTF_EXT_REPO = "HRFText_repo" +HRTF_PATHS = { + BINAURAL_HRTF_EXT_REPO: HERE.joinpath( + "../../scripts/binauralRenderer_interface/binaural_renderers_hrtf_data/ivas_binaural_{fs}kHz.bin" ) - + collapse_into_list_of_pairs( - product( - MASA_PARAMS, - OUTPUT_FORMATS_BINAURAL, - SAMPLING_RATES_ALL, - BITSTREAM_PROCESSING, - ) + .resolve() + .absolute() +} + +DECODER_CONST_BR_BINAURAL_MC_AND_MASA = collapse_into_list_of_pairs( + product( + MC_PARAMS, + OUTPUT_FORMATS_BINAURAL, + SAMPLING_RATES_ALL, + BITSTREAM_PROCESSING, + ) +) + collapse_into_list_of_pairs( + product( + MASA_PARAMS, + OUTPUT_FORMATS_BINAURAL, + SAMPLING_RATES_ALL, + BITSTREAM_PROCESSING, ) ) DECODER_CONST_BR_BINAURAL_OBJECTBASED = collapse_into_list_of_pairs( - product(ISM_PARAMS, OUTPUT_FORMATS_BINAURAL, SAMPLING_RATES_ALL, BITSTREAM_PROCESSING) + product( + ISM_PARAMS, OUTPUT_FORMATS_BINAURAL, SAMPLING_RATES_ALL, BITSTREAM_PROCESSING + ) ) DECODER_CONST_BR_BINAURAL_SCENEBASED = collapse_into_list_of_pairs( - product(SBA_PARAMS, OUTPUT_FORMATS_BINAURAL, SAMPLING_RATES_ALL, BITSTREAM_PROCESSING) + product( + SBA_PARAMS, OUTPUT_FORMATS_BINAURAL, SAMPLING_RATES_ALL, BITSTREAM_PROCESSING + ) ) DECODER_CONST_BR_BINAURAL_COMBINED_PARAMS = collapse_into_list_of_pairs( product( @@ -455,4 +486,4 @@ DECODER_CONST_BR_BINAURAL_COMBINED_PARAMS = collapse_into_list_of_pairs( SAMPLING_RATES_ALL, BITSTREAM_PROCESSING, ) -) \ No newline at end of file +) diff --git a/tests/codec_be_to_accepted_release/decoder/__init__.py b/tests/codec_be_to_accepted_release/decoder/__init__.py index 0ea0f14843..5c92a7d3de 100644 --- a/tests/codec_be_to_accepted_release/decoder/__init__.py +++ b/tests/codec_be_to_accepted_release/decoder/__init__.py @@ -34,8 +34,17 @@ from contextlib import contextmanager from pathlib import Path from tempfile import TemporaryDirectory import subprocess -from .. import DUT_PATH, REF_PATH, TESTV_PATH, HERE, is_be_to_ref, get_bitstream_path -from ..constants import BS_PROC_FER_15, BS_PROC_JBM_DLY_PROF_5 +from .. import is_be_to_ref, get_bitstream_path +from ..constants import ( + DUT_PATH, + REF_PATH, + TESTV_PATH, + HERE, + BS_PROC_FER_15, + BS_PROC_JBM_DLY_PROF_5, + BINAURAL_HRTF_NONE, + HRTF_PATHS, +) def create_head_rotation_file( @@ -64,16 +73,25 @@ def create_head_rotation_file( return out_path -def get_output_path(bitstream_path, output_format, output_sampling_rate): - output_name = ( - f"{bitstream_path.stem}.dec-{output_format}-{output_sampling_rate}kHz.wav" - ) +def get_output_path( + bitstream_path, output_format, output_sampling_rate, bitstream_processing, suffix="" +): + if suffix != "": + suffix = f"-{suffix}" + output_name = f"{bitstream_path.stem}.dec-{output_format}-{output_sampling_rate}kHz-{bitstream_processing}{suffix}.wav" return DUT_PATH.joinpath(output_name) @contextmanager def get_bitstream_and_options( - testv_name, encoder_format, bitrate, sampling_rate, dtx, processing, suffix="" + testv_name, + encoder_format, + bitrate, + sampling_rate, + dtx, + processing, + hrtf=BINAURAL_HRTF_NONE, + suffix="", ): """ Utility to get either the stored reference bitstream or the processed version as a temporary file @@ -120,6 +138,9 @@ def get_bitstream_and_options( bitstream = bitstream_out options.append("-VOIP") + if hrtf != BINAURAL_HRTF_NONE: + options.extend(["-hrtf", str(HRTF_PATHS[hrtf]).format(fs=sampling_rate)]) + yield bitstream, options diff --git a/tests/codec_be_to_accepted_release/decoder/test_decoder_constant_bitrate_binaural.py b/tests/codec_be_to_accepted_release/decoder/test_decoder_constant_bitrate_binaural.py index 7050d54323..fcd8766d10 100644 --- a/tests/codec_be_to_accepted_release/decoder/test_decoder_constant_bitrate_binaural.py +++ b/tests/codec_be_to_accepted_release/decoder/test_decoder_constant_bitrate_binaural.py @@ -34,11 +34,12 @@ from .. import get_testv_path from ..constants import * +@pytest.mark.parametrize("hrtf", [BINAURAL_HRTF_NONE, BINAURAL_HRTF_EXT_REPO]) @pytest.mark.parametrize( "input_format,bitrate,input_sampling_rate,dtx,output_format,output_sampling_rate,bitstream_processing", - DECODER_CONST_BR_BINAURAL_CHANNELBASED_AND_MASA, + DECODER_CONST_BR_BINAURAL_MC_AND_MASA, ) -def test_decoder_channelbased_and_masa( +def test_decoder_mc_and_masa( input_format, bitrate, input_sampling_rate, @@ -46,6 +47,7 @@ def test_decoder_channelbased_and_masa( output_format, output_sampling_rate, bitstream_processing, + hrtf, dut_decoder_frontend, update_ref, ): @@ -56,9 +58,16 @@ def test_decoder_channelbased_and_masa( bitrate, input_sampling_rate, dtx, + hrtf=hrtf, processing=bitstream_processing, ) as (ref_bitstream, options): - dut_output = get_output_path(ref_bitstream, output_format, output_sampling_rate) + dut_output = get_output_path( + ref_bitstream, + output_format, + output_sampling_rate, + bitstream_processing, + suffix=hrtf, + ) run_check( str(ref_bitstream), @@ -102,7 +111,9 @@ def test_decoder_objectbased( suffix=suffix, processing=bitstream_processing, ) as (ref_bitstream, options): - dut_output = get_output_path(ref_bitstream, output_format, output_sampling_rate) + dut_output = get_output_path( + ref_bitstream, output_format, output_sampling_rate, bitstream_processing + ) run_check( str(ref_bitstream), @@ -144,7 +155,9 @@ def test_decoder_scenebased( suffix=suffix, processing=bitstream_processing, ) as (ref_bitstream, options): - dut_output = get_output_path(ref_bitstream, output_format, output_sampling_rate) + dut_output = get_output_path( + ref_bitstream, output_format, output_sampling_rate, bitstream_processing + ) run_check( str(ref_bitstream), @@ -192,7 +205,9 @@ def test_decoder_combined_formats( suffix=suffix, processing=bitstream_processing, ) as (ref_bitstream, options): - dut_output = get_output_path(ref_bitstream, output_format, output_sampling_rate) + dut_output = get_output_path( + ref_bitstream, output_format, output_sampling_rate, bitstream_processing + ) run_check( str(ref_bitstream), diff --git a/tests/codec_be_to_accepted_release/decoder/test_decoder_constant_bitrate_no_binaural.py b/tests/codec_be_to_accepted_release/decoder/test_decoder_constant_bitrate_no_binaural.py index 96d9af09ea..aa50c26a8d 100644 --- a/tests/codec_be_to_accepted_release/decoder/test_decoder_constant_bitrate_no_binaural.py +++ b/tests/codec_be_to_accepted_release/decoder/test_decoder_constant_bitrate_no_binaural.py @@ -58,7 +58,9 @@ def test_decoder_channelbased_and_masa( dtx, processing=bitstream_processing, ) as (ref_bitstream, options): - dut_output = get_output_path(ref_bitstream, output_format, output_sampling_rate) + dut_output = get_output_path( + ref_bitstream, output_format, output_sampling_rate, bitstream_processing + ) run_check( str(ref_bitstream), @@ -102,7 +104,9 @@ def test_decoder_objectbased( suffix=suffix, processing=bitstream_processing, ) as (ref_bitstream, options): - dut_output = get_output_path(ref_bitstream, output_format, output_sampling_rate) + dut_output = get_output_path( + ref_bitstream, output_format, output_sampling_rate, bitstream_processing + ) run_check( str(ref_bitstream), @@ -144,7 +148,9 @@ def test_decoder_scenebased( suffix=suffix, processing=bitstream_processing, ) as (ref_bitstream, options): - dut_output = get_output_path(ref_bitstream, output_format, output_sampling_rate) + dut_output = get_output_path( + ref_bitstream, output_format, output_sampling_rate, bitstream_processing + ) run_check( str(ref_bitstream), @@ -192,7 +198,9 @@ def test_decoder_combined_formats( suffix=suffix, processing=bitstream_processing, ) as (ref_bitstream, options): - dut_output = get_output_path(ref_bitstream, output_format, output_sampling_rate) + dut_output = get_output_path( + ref_bitstream, output_format, output_sampling_rate, bitstream_processing + ) run_check( str(ref_bitstream), diff --git a/tests/codec_be_to_accepted_release/encoder/__init__.py b/tests/codec_be_to_accepted_release/encoder/__init__.py index a87562a381..83e498f47d 100644 --- a/tests/codec_be_to_accepted_release/encoder/__init__.py +++ b/tests/codec_be_to_accepted_release/encoder/__init__.py @@ -33,12 +33,19 @@ import random import pytest from itertools import permutations from .. import DUT_PATH, TESTV_PATH, is_be_to_ref -from ..constants import METADATA_FOR_INPUT_FORMAT, ISM_MD_NULL, CMDL_OPTIONS_FOR_INPUT_FORMAT, ISM_MD_EXTENDED, INPUT_FORMATS_OBJECT_BASED +from ..constants import ( + METADATA_FOR_INPUT_FORMAT, + ISM_MD_NULL, + CMDL_OPTIONS_FOR_INPUT_FORMAT, + ISM_MD_EXTENDED, + INPUT_FORMATS_OBJECT_BASED, +) + def create_br_switching_file( bitrates: np.ndarray, strategy: str, - switch_time: int, + switch_time: int = 1, length_frames: int = 0, seed: int = None, ): @@ -64,23 +71,24 @@ def create_br_switching_file( if seed is not None: random.seed(seed) - brs = np.asarray(bitrates) - - # NOTE: nothing to do for from_array - if strategy == "exhaustive": + if strategy == "from_array": + brs = bitrates + elif strategy == "exhaustive": # TODO: review, probably needs more work to be really exhaustive... n = len(brs) - 1 - permuts = list(permutations(brs, 2)) + permuts = list(permutations(bitrates, 2)) split_permuts = [permuts[i * n : (i + 1) * n] for i in range(len(brs))] for i in range(len(split_permuts)): random.shuffle(split_permuts[i]) split_permuts = np.asarray(split_permuts) brs = np.concatenate([split_permuts[:, i] for i in range(n)]) elif strategy == "random": - brs = np.asarray(random.choices(brs, k=length_frames)) + brs = np.asarray(random.choices(bitrates, k=length_frames)) + + brs = np.repeat(brs, switch_time).astype(np.int32) out_path = DUT_PATH.joinpath(fname) - brs.astype(np.int32).tofile(out_path) + brs.tofile(out_path) return out_path @@ -124,4 +132,4 @@ def run_check( ) if not is_ref_creation and not is_be_to_ref(dut_bitstream): - pytest.fail(f"Bitstream file differs from reference") \ No newline at end of file + pytest.fail(f"Bitstream file differs from reference") diff --git a/tests/codec_be_to_accepted_release/encoder/test_encoder_bitrate_switching.py b/tests/codec_be_to_accepted_release/encoder/test_encoder_bitrate_switching.py new file mode 100644 index 0000000000..cd4b773297 --- /dev/null +++ b/tests/codec_be_to_accepted_release/encoder/test_encoder_bitrate_switching.py @@ -0,0 +1,62 @@ +__copyright__ = """ +(C) 2022-2023 IVAS codec Public Collaboration with portions copyright Dolby International AB, Ericsson AB, +Fraunhofer-Gesellschaft zur Foerderung der angewandten Forschung e.V., Huawei Technologies Co. LTD., +Koninklijke Philips N.V., Nippon Telegraph and Telephone Corporation, Nokia Technologies Oy, Orange, +Panasonic Holdings Corporation, Qualcomm Technologies, Inc., VoiceAge Corporation, and other +contributors to this repository. All Rights Reserved. + +This software is protected by copyright law and by international treaties. +The IVAS codec Public Collaboration consisting of Dolby International AB, Ericsson AB, +Fraunhofer-Gesellschaft zur Foerderung der angewandten Forschung e.V., Huawei Technologies Co. LTD., +Koninklijke Philips N.V., Nippon Telegraph and Telephone Corporation, Nokia Technologies Oy, Orange, +Panasonic Holdings Corporation, Qualcomm Technologies, Inc., VoiceAge Corporation, and other +contributors to this repository retain full ownership rights in their respective contributions in +the software. This notice grants no license of any kind, including but not limited to patent +license, nor is any license granted by implication, estoppel or otherwise. + +Contributors are required to enter into the IVAS codec Public Collaboration agreement before making +contributions. + +This software is provided "AS IS", without any express or implied warranties. The software is in the +development stage. It is intended exclusively for experts who have experience with such software and +solely for the purpose of inspection. All implied warranties of non-infringement, merchantability +and fitness for a particular purpose are hereby disclaimed and excluded. + +Any dispute, controversy or claim arising under or in relation to providing this software shall be +submitted to and settled by the final, binding jurisdiction of the courts of Munich, Germany in +accordance with the laws of the Federal Republic of Germany excluding its conflict of law rules and +the United Nations Convention on Contracts on the International Sales of Goods. +""" + +import pytest +from ..constants import * +from . import get_options, run_check, create_br_switching_file +from .. import get_testv_path, get_bitstream_path, DUT_PATH + + +@pytest.mark.parametrize( + "input_format,bitrate,sampling_rate,dtx", ENCODER_CHANNEL_BASED_AND_MASA_PARAMS +) +def test_encoder_br_switching_channel_based_and_masa_modes( + input_format, + bitrate, + sampling_rate, + dtx, + dut_encoder_frontend, + update_ref, +): + testv = get_testv_path(input_format, sampling_rate) + options = get_options(input_format) + bitstream = get_bitstream_path( + DUT_PATH, testv.stem, input_format, bitrate, sampling_rate, dtx + ) + run_check( + bitstream, + bitrate, + sampling_rate, + testv, + options, + dut_encoder_frontend, + update_ref == 1, + dtx == DTX_ON, + ) diff --git a/tests/codec_be_to_accepted_release/encoder/test_encoder_constant_bitrate.py b/tests/codec_be_to_accepted_release/encoder/test_encoder_constant_bitrate.py index e3d6b0b2f3..ceb73c9e3b 100644 --- a/tests/codec_be_to_accepted_release/encoder/test_encoder_constant_bitrate.py +++ b/tests/codec_be_to_accepted_release/encoder/test_encoder_constant_bitrate.py @@ -49,7 +49,7 @@ from .. import get_testv_path, get_bitstream_path, DUT_PATH @pytest.mark.parametrize( "input_format,bitrate,sampling_rate,dtx", ENCODER_CHANNEL_BASED_AND_MASA_PARAMS ) -def test_encoder_channel_based_and_masa_modes( +def test_encoder_const_br_channel_based_and_masa_modes( input_format, bitrate, sampling_rate, @@ -75,7 +75,7 @@ def test_encoder_channel_based_and_masa_modes( @pytest.mark.parametrize("input_format,bitrate,sampling_rate,dtx,md_type", ISM_PARAMS) -def test_encoder_object_based_modes( +def test_encoder_const_br_object_based_modes( input_format, bitrate, sampling_rate, @@ -107,7 +107,7 @@ def test_encoder_object_based_modes( @pytest.mark.parametrize("input_format,bitrate,sampling_rate,dtx,pca", SBA_PARAMS) -def test_encoder_scenebased( +def test_encoder_const_br_scenebased( input_format, bitrate, sampling_rate, @@ -137,7 +137,7 @@ def test_encoder_scenebased( @pytest.mark.parametrize("bitrate,sampling_rate,dtx", STEREO_DMX_EVS_PARAMS) -def test_encoder_stereo_dmx_evs( +def test_encoder_const_br_stereo_dmx_evs( bitrate, sampling_rate, dtx, dut_encoder_frontend, update_ref ): testv = get_testv_path("STEREO", sampling_rate) @@ -161,7 +161,7 @@ def test_encoder_stereo_dmx_evs( "input_format_ism,input_format_other,bitrate,sampling_rate,md_type", COMBINED_FORMATS_PARAMS, ) -def test_encoder_combined_formats( +def test_encoder_const_br_combined_formats( input_format_ism, input_format_other, bitrate, -- GitLab From fe32274f918e860ea2dc5eedc0355ad3c5c973db Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Fri, 1 Sep 2023 16:04:48 +0200 Subject: [PATCH 026/106] fix typo in arguments --- .../decoder/test_decoder_constant_bitrate_binaural.py | 6 +++--- .../decoder/test_decoder_constant_bitrate_no_binaural.py | 6 +++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/tests/codec_be_to_accepted_release/decoder/test_decoder_constant_bitrate_binaural.py b/tests/codec_be_to_accepted_release/decoder/test_decoder_constant_bitrate_binaural.py index fcd8766d10..73ae290858 100644 --- a/tests/codec_be_to_accepted_release/decoder/test_decoder_constant_bitrate_binaural.py +++ b/tests/codec_be_to_accepted_release/decoder/test_decoder_constant_bitrate_binaural.py @@ -122,7 +122,7 @@ def test_decoder_objectbased( str(dut_output), dut_decoder_frontend, update_ref == 1, - add_option_list=options, + options=options, ) @@ -166,7 +166,7 @@ def test_decoder_scenebased( str(dut_output), dut_decoder_frontend, update_ref == 1, - add_option_list=options, + options=options, ) @@ -216,5 +216,5 @@ def test_decoder_combined_formats( str(dut_output), dut_decoder_frontend, update_ref == 1, - add_option_list=options, + options=options, ) diff --git a/tests/codec_be_to_accepted_release/decoder/test_decoder_constant_bitrate_no_binaural.py b/tests/codec_be_to_accepted_release/decoder/test_decoder_constant_bitrate_no_binaural.py index aa50c26a8d..dca04d3f58 100644 --- a/tests/codec_be_to_accepted_release/decoder/test_decoder_constant_bitrate_no_binaural.py +++ b/tests/codec_be_to_accepted_release/decoder/test_decoder_constant_bitrate_no_binaural.py @@ -115,7 +115,7 @@ def test_decoder_objectbased( str(dut_output), dut_decoder_frontend, update_ref == 1, - add_option_list=options, + options=options, ) @@ -159,7 +159,7 @@ def test_decoder_scenebased( str(dut_output), dut_decoder_frontend, update_ref == 1, - add_option_list=options, + options=options, ) @@ -209,5 +209,5 @@ def test_decoder_combined_formats( str(dut_output), dut_decoder_frontend, update_ref == 1, - add_option_list=options, + options=options, ) -- GitLab From af44ec0ae8ca69b5659bde092a4bcb863e7afecc Mon Sep 17 00:00:00 2001 From: knj Date: Fri, 1 Sep 2023 16:19:40 +0200 Subject: [PATCH 027/106] fix argument passing again... --- .../decoder/test_decoder_constant_bitrate_binaural.py | 6 +++--- .../decoder/test_decoder_constant_bitrate_no_binaural.py | 6 +++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/tests/codec_be_to_accepted_release/decoder/test_decoder_constant_bitrate_binaural.py b/tests/codec_be_to_accepted_release/decoder/test_decoder_constant_bitrate_binaural.py index 73ae290858..c384dab6ec 100644 --- a/tests/codec_be_to_accepted_release/decoder/test_decoder_constant_bitrate_binaural.py +++ b/tests/codec_be_to_accepted_release/decoder/test_decoder_constant_bitrate_binaural.py @@ -120,9 +120,9 @@ def test_decoder_objectbased( output_format, output_sampling_rate, str(dut_output), + options, dut_decoder_frontend, update_ref == 1, - options=options, ) @@ -164,9 +164,9 @@ def test_decoder_scenebased( output_format, output_sampling_rate, str(dut_output), + options, dut_decoder_frontend, update_ref == 1, - options=options, ) @@ -214,7 +214,7 @@ def test_decoder_combined_formats( output_format, output_sampling_rate, str(dut_output), + options, dut_decoder_frontend, update_ref == 1, - options=options, ) diff --git a/tests/codec_be_to_accepted_release/decoder/test_decoder_constant_bitrate_no_binaural.py b/tests/codec_be_to_accepted_release/decoder/test_decoder_constant_bitrate_no_binaural.py index dca04d3f58..b5678f855f 100644 --- a/tests/codec_be_to_accepted_release/decoder/test_decoder_constant_bitrate_no_binaural.py +++ b/tests/codec_be_to_accepted_release/decoder/test_decoder_constant_bitrate_no_binaural.py @@ -113,9 +113,9 @@ def test_decoder_objectbased( output_format, output_sampling_rate, str(dut_output), + options, dut_decoder_frontend, update_ref == 1, - options=options, ) @@ -157,9 +157,9 @@ def test_decoder_scenebased( output_format, output_sampling_rate, str(dut_output), + options, dut_decoder_frontend, update_ref == 1, - options=options, ) @@ -207,7 +207,7 @@ def test_decoder_combined_formats( output_format, output_sampling_rate, str(dut_output), + options, dut_decoder_frontend, update_ref == 1, - options=options, ) -- GitLab From 7f5ba8b7984944128deca32719c24e25b34de172 Mon Sep 17 00:00:00 2001 From: kiene Date: Fri, 1 Sep 2023 16:33:48 +0200 Subject: [PATCH 028/106] fix some things in the encoder tests --- tests/codec_be_to_accepted_release/encoder/__init__.py | 4 +++- .../encoder/test_encoder_bitrate_switching.py | 3 ++- .../encoder/test_encoder_constant_bitrate.py | 3 ++- 3 files changed, 7 insertions(+), 3 deletions(-) diff --git a/tests/codec_be_to_accepted_release/encoder/__init__.py b/tests/codec_be_to_accepted_release/encoder/__init__.py index 83e498f47d..cf58a911f7 100644 --- a/tests/codec_be_to_accepted_release/encoder/__init__.py +++ b/tests/codec_be_to_accepted_release/encoder/__init__.py @@ -32,13 +32,15 @@ import numpy as np import random import pytest from itertools import permutations -from .. import DUT_PATH, TESTV_PATH, is_be_to_ref +from .. import is_be_to_ref from ..constants import ( METADATA_FOR_INPUT_FORMAT, ISM_MD_NULL, CMDL_OPTIONS_FOR_INPUT_FORMAT, ISM_MD_EXTENDED, INPUT_FORMATS_OBJECT_BASED, + DUT_PATH, + TESTV_PATH, ) diff --git a/tests/codec_be_to_accepted_release/encoder/test_encoder_bitrate_switching.py b/tests/codec_be_to_accepted_release/encoder/test_encoder_bitrate_switching.py index cd4b773297..5e6e66824e 100644 --- a/tests/codec_be_to_accepted_release/encoder/test_encoder_bitrate_switching.py +++ b/tests/codec_be_to_accepted_release/encoder/test_encoder_bitrate_switching.py @@ -31,9 +31,10 @@ the United Nations Convention on Contracts on the International Sales of Goods. import pytest from ..constants import * from . import get_options, run_check, create_br_switching_file -from .. import get_testv_path, get_bitstream_path, DUT_PATH +from .. import get_testv_path, get_bitstream_path +@pytest.mark.skip("not ready, comitted by accident") @pytest.mark.parametrize( "input_format,bitrate,sampling_rate,dtx", ENCODER_CHANNEL_BASED_AND_MASA_PARAMS ) diff --git a/tests/codec_be_to_accepted_release/encoder/test_encoder_constant_bitrate.py b/tests/codec_be_to_accepted_release/encoder/test_encoder_constant_bitrate.py index ceb73c9e3b..8a968d8f7c 100644 --- a/tests/codec_be_to_accepted_release/encoder/test_encoder_constant_bitrate.py +++ b/tests/codec_be_to_accepted_release/encoder/test_encoder_constant_bitrate.py @@ -40,9 +40,10 @@ from ..constants import ( SBA_FOA_PCA_ON, STEREO_DMX_EVS_PARAMS, COMBINED_FORMATS_PARAMS, + DUT_PATH, ) from . import get_options, run_check -from .. import get_testv_path, get_bitstream_path, DUT_PATH +from .. import get_testv_path, get_bitstream_path # channel-based modes + MASA -- GitLab From d7c3a846c0b775011c7eeb02350a55d8b0fcf0dc Mon Sep 17 00:00:00 2001 From: kiene Date: Mon, 4 Sep 2023 15:39:48 +0200 Subject: [PATCH 029/106] add exhaustive testcases for non-diegetic panning in decoder --- .../decoder/__init__.py | 4 + .../decoder/test_decoder_non_diegetic_pan.py | 148 ++++++++++++++++++ 2 files changed, 152 insertions(+) create mode 100644 tests/codec_be_to_accepted_release/decoder/test_decoder_non_diegetic_pan.py diff --git a/tests/codec_be_to_accepted_release/decoder/__init__.py b/tests/codec_be_to_accepted_release/decoder/__init__.py index 5c92a7d3de..5b037b54a6 100644 --- a/tests/codec_be_to_accepted_release/decoder/__init__.py +++ b/tests/codec_be_to_accepted_release/decoder/__init__.py @@ -92,6 +92,7 @@ def get_bitstream_and_options( processing, hrtf=BINAURAL_HRTF_NONE, suffix="", + non_diegetic_pan_value=None, ): """ Utility to get either the stored reference bitstream or the processed version as a temporary file @@ -141,6 +142,9 @@ def get_bitstream_and_options( if hrtf != BINAURAL_HRTF_NONE: options.extend(["-hrtf", str(HRTF_PATHS[hrtf]).format(fs=sampling_rate)]) + if non_diegetic_pan_value is not None: + options.extend(["-non_diegetic_pan", f"{non_diegetic_pan_value}"]) + yield bitstream, options diff --git a/tests/codec_be_to_accepted_release/decoder/test_decoder_non_diegetic_pan.py b/tests/codec_be_to_accepted_release/decoder/test_decoder_non_diegetic_pan.py new file mode 100644 index 0000000000..faa6260e04 --- /dev/null +++ b/tests/codec_be_to_accepted_release/decoder/test_decoder_non_diegetic_pan.py @@ -0,0 +1,148 @@ +__copyright__ = """ +(C) 2022-2023 IVAS codec Public Collaboration with portions copyright Dolby International AB, Ericsson AB, +Fraunhofer-Gesellschaft zur Foerderung der angewandten Forschung e.V., Huawei Technologies Co. LTD., +Koninklijke Philips N.V., Nippon Telegraph and Telephone Corporation, Nokia Technologies Oy, Orange, +Panasonic Holdings Corporation, Qualcomm Technologies, Inc., VoiceAge Corporation, and other +contributors to this repository. All Rights Reserved. + +This software is protected by copyright law and by international treaties. +The IVAS codec Public Collaboration consisting of Dolby International AB, Ericsson AB, +Fraunhofer-Gesellschaft zur Foerderung der angewandten Forschung e.V., Huawei Technologies Co. LTD., +Koninklijke Philips N.V., Nippon Telegraph and Telephone Corporation, Nokia Technologies Oy, Orange, +Panasonic Holdings Corporation, Qualcomm Technologies, Inc., VoiceAge Corporation, and other +contributors to this repository retain full ownership rights in their respective contributions in +the software. This notice grants no license of any kind, including but not limited to patent +license, nor is any license granted by implication, estoppel or otherwise. + +Contributors are required to enter into the IVAS codec Public Collaboration agreement before making +contributions. + +This software is provided "AS IS", without any express or implied warranties. The software is in the +development stage. It is intended exclusively for experts who have experience with such software and +solely for the purpose of inspection. All implied warranties of non-infringement, merchantability +and fitness for a particular purpose are hereby disclaimed and excluded. + +Any dispute, controversy or claim arising under or in relation to providing this software shall be +submitted to and settled by the final, binding jurisdiction of the courts of Munich, Germany in +accordance with the laws of the Federal Republic of Germany excluding its conflict of law rules and +the United Nations Convention on Contracts on the International Sales of Goods. +""" + +import pytest +from . import get_bitstream_and_options, get_output_path, run_check +from .. import get_testv_path +from ..constants import ( + STEREO_DMX_EVS_PARAMS, + ISM_MD_EXTENDED, + ISM_MD_NULL, + DECODER_CONST_BR_NO_BINAURAL_OBJECTBASED, + SAMPLING_RATES_ALL, + BITSTREAM_PROCESSING, +) + +# NOTE: this way, all possible inputs for the non-diegetic panning value are used for all input/output configs +# less exhaustive testing is probably enough here, one can e.g. only test one panning value for each i/o config +NON_DIEGETIC_PAN_VALUES = ["c", "center", "middle", "left", "l", "right", "r"] + list( + range(-90, 91) +) +ISM1_DECODER_PARAMS = [ + p + for p in DECODER_CONST_BR_NO_BINAURAL_OBJECTBASED + if p[0].startswith("ISM1") and p[1] == "STEREO" +] + + +@pytest.mark.parametrize("non_diegetic_panning_value", NON_DIEGETIC_PAN_VALUES) +@pytest.mark.parametrize( + "input_format,bitrate,input_sampling_rate,dtx,md_type,output_format,output_sampling_rate,bitstream_processing", + ISM1_DECODER_PARAMS, +) +def test_decoder_non_diegetic_pan_ism( + input_format, + bitrate, + input_sampling_rate, + dtx, + md_type, + output_format, + output_sampling_rate, + bitstream_processing, + non_diegetic_panning_value, + dut_decoder_frontend, + update_ref, +): + testv_name = get_testv_path(input_format, input_sampling_rate).stem + suffix = "_basic_MD" + if md_type == ISM_MD_EXTENDED: + suffix = "_ext_MD" + elif md_type == ISM_MD_NULL: + suffix = "_null_MD" + with get_bitstream_and_options( + testv_name, + input_format, + bitrate, + input_sampling_rate, + dtx, + bitstream_processing, + suffix=suffix, + ) as (ref_bitstream, options): + suffix += f"-non_diegetic_pan_{non_diegetic_panning_value}" + dut_output = get_output_path( + ref_bitstream, + output_format, + output_sampling_rate, + bitstream_processing, + suffix=suffix, + ) + run_check( + str(ref_bitstream), + output_format, + output_sampling_rate, + str(dut_output), + options, + dut_decoder_frontend, + update_ref == 1, + ) + + +# for the mono input, we use the Stereo DMX EVS outputs from the decoder so no explicit mono encoder run is needed +@pytest.mark.parametrize("non_diegetic_panning_value", NON_DIEGETIC_PAN_VALUES) +@pytest.mark.parametrize("bitrate,input_sampling_rate,dtx", STEREO_DMX_EVS_PARAMS) +@pytest.mark.parametrize("output_sampling_rate", SAMPLING_RATES_ALL) +@pytest.mark.parametrize("bitstream_processing", BITSTREAM_PROCESSING) +def test_decoder_non_diegetic_pan_mono( + bitrate, + input_sampling_rate, + dtx, + output_sampling_rate, + bitstream_processing, + non_diegetic_panning_value, + dut_decoder_frontend, + update_ref, +): + testv_name = get_testv_path("STEREO", input_sampling_rate).stem + with get_bitstream_and_options( + testv_name, + "StereoDmxEvs", + bitrate, + input_sampling_rate, + dtx, + bitstream_processing, + non_diegetic_pan_value=non_diegetic_panning_value, + ) as (ref_bitstream, options): + suffix = f"-non_diegetic_pan_{non_diegetic_panning_value}" + dut_output = get_output_path( + ref_bitstream, + "STEREO", + output_sampling_rate, + bitstream_processing, + suffix=suffix, + ) + run_check( + str(ref_bitstream), + "", + output_sampling_rate, + str(dut_output), + options, + dut_decoder_frontend, + update_ref == 1, + ) -- GitLab From 88429e48f9c61ebfa1b0aa712527794a1e13e478 Mon Sep 17 00:00:00 2001 From: kiene Date: Mon, 4 Sep 2023 16:46:14 +0200 Subject: [PATCH 030/106] some restructuring and make --update_ref write to ref dir --- .../decoder/__init__.py | 42 ++++++++++---- .../test_decoder_constant_bitrate_binaural.py | 39 ++++--------- ...st_decoder_constant_bitrate_no_binaural.py | 34 +++-------- .../decoder/test_decoder_non_diegetic_pan.py | 26 +++------ .../encoder/__init__.py | 27 ++++++--- .../encoder/test_encoder_constant_bitrate.py | 58 +++++++------------ 6 files changed, 99 insertions(+), 127 deletions(-) diff --git a/tests/codec_be_to_accepted_release/decoder/__init__.py b/tests/codec_be_to_accepted_release/decoder/__init__.py index 5b037b54a6..6cd4a63566 100644 --- a/tests/codec_be_to_accepted_release/decoder/__init__.py +++ b/tests/codec_be_to_accepted_release/decoder/__init__.py @@ -42,6 +42,7 @@ from ..constants import ( HERE, BS_PROC_FER_15, BS_PROC_JBM_DLY_PROF_5, + BS_PROC_NONE, BINAURAL_HRTF_NONE, HRTF_PATHS, ) @@ -74,12 +75,19 @@ def create_head_rotation_file( def get_output_path( - bitstream_path, output_format, output_sampling_rate, bitstream_processing, suffix="" + bitstream_path: Path, + output_format: str, + output_sampling_rate: int, + bitstream_processing: str, + is_ref_creation: bool, + suffix: str = "", ): if suffix != "": suffix = f"-{suffix}" output_name = f"{bitstream_path.stem}.dec-{output_format}-{output_sampling_rate}kHz-{bitstream_processing}{suffix}.wav" - return DUT_PATH.joinpath(output_name) + + output_folder = REF_PATH if is_ref_creation else DUT_PATH + return output_folder.joinpath(output_name) @contextmanager @@ -102,6 +110,7 @@ def get_bitstream_and_options( """ options = list() with TemporaryDirectory() as tmp_dir: + # TODO: this should be coming from TESTV_PATH - current setuponly for development bitstream = get_bitstream_path( REF_PATH, testv_name, encoder_format, bitrate, sampling_rate, dtx, suffix ) @@ -149,17 +158,30 @@ def get_bitstream_and_options( def run_check( - ref_bitstream, - output_format, - sampling_rate, - dut_output, - options, + ref_bitstream: Path, + output_format: str, + output_sampling_rate: int, + options: list, decoder_frontend, - is_ref_creation, + is_ref_creation: bool, + output_suffix="", + bitstream_processing=BS_PROC_NONE, ): + output_path = get_output_path( + ref_bitstream, + output_format, + output_sampling_rate, + bitstream_processing, + is_ref_creation, + suffix=output_suffix, + ) decoder_frontend.run( - output_format, sampling_rate, ref_bitstream, dut_output, add_option_list=options + output_format, + output_sampling_rate, + ref_bitstream, + output_path, + add_option_list=options, ) - if not is_ref_creation and not is_be_to_ref(dut_output): + if not is_ref_creation and not is_be_to_ref(output_path): pytest.fail(f"Decoder output differs from reference") diff --git a/tests/codec_be_to_accepted_release/decoder/test_decoder_constant_bitrate_binaural.py b/tests/codec_be_to_accepted_release/decoder/test_decoder_constant_bitrate_binaural.py index c384dab6ec..c32c0959d6 100644 --- a/tests/codec_be_to_accepted_release/decoder/test_decoder_constant_bitrate_binaural.py +++ b/tests/codec_be_to_accepted_release/decoder/test_decoder_constant_bitrate_binaural.py @@ -29,7 +29,7 @@ the United Nations Convention on Contracts on the International Sales of Goods. """ import pytest -from . import get_bitstream_and_options, get_output_path, run_check +from . import get_bitstream_and_options, run_check from .. import get_testv_path from ..constants import * @@ -61,22 +61,15 @@ def test_decoder_mc_and_masa( hrtf=hrtf, processing=bitstream_processing, ) as (ref_bitstream, options): - dut_output = get_output_path( - ref_bitstream, - output_format, - output_sampling_rate, - bitstream_processing, - suffix=hrtf, - ) - run_check( - str(ref_bitstream), + ref_bitstream, output_format, output_sampling_rate, - str(dut_output), options, dut_decoder_frontend, update_ref == 1, + output_suffix=hrtf, + bitstream_processing=bitstream_processing, ) @@ -111,18 +104,14 @@ def test_decoder_objectbased( suffix=suffix, processing=bitstream_processing, ) as (ref_bitstream, options): - dut_output = get_output_path( - ref_bitstream, output_format, output_sampling_rate, bitstream_processing - ) - run_check( - str(ref_bitstream), + ref_bitstream, output_format, output_sampling_rate, - str(dut_output), options, dut_decoder_frontend, update_ref == 1, + bitstream_processing=bitstream_processing, ) @@ -155,18 +144,14 @@ def test_decoder_scenebased( suffix=suffix, processing=bitstream_processing, ) as (ref_bitstream, options): - dut_output = get_output_path( - ref_bitstream, output_format, output_sampling_rate, bitstream_processing - ) - run_check( - str(ref_bitstream), + ref_bitstream, output_format, output_sampling_rate, - str(dut_output), options, dut_decoder_frontend, update_ref == 1, + bitstream_processing=bitstream_processing, ) @@ -205,16 +190,12 @@ def test_decoder_combined_formats( suffix=suffix, processing=bitstream_processing, ) as (ref_bitstream, options): - dut_output = get_output_path( - ref_bitstream, output_format, output_sampling_rate, bitstream_processing - ) - run_check( - str(ref_bitstream), + ref_bitstream, output_format, output_sampling_rate, - str(dut_output), options, dut_decoder_frontend, update_ref == 1, + bitstream_processing=bitstream_processing, ) diff --git a/tests/codec_be_to_accepted_release/decoder/test_decoder_constant_bitrate_no_binaural.py b/tests/codec_be_to_accepted_release/decoder/test_decoder_constant_bitrate_no_binaural.py index b5678f855f..6054ae8e69 100644 --- a/tests/codec_be_to_accepted_release/decoder/test_decoder_constant_bitrate_no_binaural.py +++ b/tests/codec_be_to_accepted_release/decoder/test_decoder_constant_bitrate_no_binaural.py @@ -29,7 +29,7 @@ the United Nations Convention on Contracts on the International Sales of Goods. """ import pytest -from . import get_bitstream_and_options, get_output_path, run_check +from . import get_bitstream_and_options, run_check from .. import get_testv_path from ..constants import * @@ -58,18 +58,14 @@ def test_decoder_channelbased_and_masa( dtx, processing=bitstream_processing, ) as (ref_bitstream, options): - dut_output = get_output_path( - ref_bitstream, output_format, output_sampling_rate, bitstream_processing - ) - run_check( - str(ref_bitstream), + ref_bitstream, output_format, output_sampling_rate, - str(dut_output), options, dut_decoder_frontend, update_ref == 1, + bitstream_processing=bitstream_processing, ) @@ -104,18 +100,14 @@ def test_decoder_objectbased( suffix=suffix, processing=bitstream_processing, ) as (ref_bitstream, options): - dut_output = get_output_path( - ref_bitstream, output_format, output_sampling_rate, bitstream_processing - ) - run_check( - str(ref_bitstream), + ref_bitstream, output_format, output_sampling_rate, - str(dut_output), options, dut_decoder_frontend, update_ref == 1, + bitstream_processing=bitstream_processing, ) @@ -148,18 +140,14 @@ def test_decoder_scenebased( suffix=suffix, processing=bitstream_processing, ) as (ref_bitstream, options): - dut_output = get_output_path( - ref_bitstream, output_format, output_sampling_rate, bitstream_processing - ) - run_check( - str(ref_bitstream), + ref_bitstream, output_format, output_sampling_rate, - str(dut_output), options, dut_decoder_frontend, update_ref == 1, + bitstream_processing=bitstream_processing, ) @@ -198,16 +186,12 @@ def test_decoder_combined_formats( suffix=suffix, processing=bitstream_processing, ) as (ref_bitstream, options): - dut_output = get_output_path( - ref_bitstream, output_format, output_sampling_rate, bitstream_processing - ) - run_check( - str(ref_bitstream), + ref_bitstream, output_format, output_sampling_rate, - str(dut_output), options, dut_decoder_frontend, update_ref == 1, + bitstream_processing=bitstream_processing, ) diff --git a/tests/codec_be_to_accepted_release/decoder/test_decoder_non_diegetic_pan.py b/tests/codec_be_to_accepted_release/decoder/test_decoder_non_diegetic_pan.py index faa6260e04..39880aa7f1 100644 --- a/tests/codec_be_to_accepted_release/decoder/test_decoder_non_diegetic_pan.py +++ b/tests/codec_be_to_accepted_release/decoder/test_decoder_non_diegetic_pan.py @@ -29,7 +29,7 @@ the United Nations Convention on Contracts on the International Sales of Goods. """ import pytest -from . import get_bitstream_and_options, get_output_path, run_check +from . import get_bitstream_and_options, run_check from .. import get_testv_path from ..constants import ( STEREO_DMX_EVS_PARAMS, @@ -86,21 +86,15 @@ def test_decoder_non_diegetic_pan_ism( suffix=suffix, ) as (ref_bitstream, options): suffix += f"-non_diegetic_pan_{non_diegetic_panning_value}" - dut_output = get_output_path( - ref_bitstream, - output_format, - output_sampling_rate, - bitstream_processing, - suffix=suffix, - ) run_check( - str(ref_bitstream), + ref_bitstream, output_format, output_sampling_rate, - str(dut_output), options, dut_decoder_frontend, update_ref == 1, + output_suffix=suffix, + bitstream_processing=bitstream_processing, ) @@ -130,19 +124,13 @@ def test_decoder_non_diegetic_pan_mono( non_diegetic_pan_value=non_diegetic_panning_value, ) as (ref_bitstream, options): suffix = f"-non_diegetic_pan_{non_diegetic_panning_value}" - dut_output = get_output_path( - ref_bitstream, - "STEREO", - output_sampling_rate, - bitstream_processing, - suffix=suffix, - ) run_check( - str(ref_bitstream), + ref_bitstream, "", output_sampling_rate, - str(dut_output), options, dut_decoder_frontend, update_ref == 1, + output_suffix=suffix, + bitstream_processing=bitstream_processing, ) diff --git a/tests/codec_be_to_accepted_release/encoder/__init__.py b/tests/codec_be_to_accepted_release/encoder/__init__.py index cf58a911f7..409c2d36aa 100644 --- a/tests/codec_be_to_accepted_release/encoder/__init__.py +++ b/tests/codec_be_to_accepted_release/encoder/__init__.py @@ -32,7 +32,7 @@ import numpy as np import random import pytest from itertools import permutations -from .. import is_be_to_ref +from .. import is_be_to_ref, get_bitstream_path from ..constants import ( METADATA_FOR_INPUT_FORMAT, ISM_MD_NULL, @@ -41,6 +41,8 @@ from ..constants import ( INPUT_FORMATS_OBJECT_BASED, DUT_PATH, TESTV_PATH, + REF_PATH, + DTX_ON, ) @@ -115,23 +117,34 @@ def get_options(input_format, md_type=None): def run_check( - dut_bitstream, + input_format, bitrate, sampling_rate, testv, options, encoder_frontend, is_ref_creation, - dtx_mode=False, + dtx, + bitstream_suffix="", ): + bs_path = REF_PATH if is_ref_creation else DUT_PATH + bitstream = get_bitstream_path( + bs_path, + testv.stem, + input_format, + bitrate, + sampling_rate, + dtx, + suffix=bitstream_suffix, + ) encoder_frontend.run( bitrate, sampling_rate, - str(testv), - str(dut_bitstream), - dtx_mode=dtx_mode, + testv, + bitstream, + dtx_mode=dtx == DTX_ON, add_option_list=options, ) - if not is_ref_creation and not is_be_to_ref(dut_bitstream): + if not is_ref_creation and not is_be_to_ref(bitstream): pytest.fail(f"Bitstream file differs from reference") diff --git a/tests/codec_be_to_accepted_release/encoder/test_encoder_constant_bitrate.py b/tests/codec_be_to_accepted_release/encoder/test_encoder_constant_bitrate.py index 8a968d8f7c..3a617bf4be 100644 --- a/tests/codec_be_to_accepted_release/encoder/test_encoder_constant_bitrate.py +++ b/tests/codec_be_to_accepted_release/encoder/test_encoder_constant_bitrate.py @@ -60,18 +60,15 @@ def test_encoder_const_br_channel_based_and_masa_modes( ): testv = get_testv_path(input_format, sampling_rate) options = get_options(input_format) - bitstream = get_bitstream_path( - DUT_PATH, testv.stem, input_format, bitrate, sampling_rate, dtx - ) run_check( - bitstream, + input_format, bitrate, sampling_rate, testv, options, dut_encoder_frontend, update_ref == 1, - dtx == DTX_ON, + dtx, ) @@ -87,23 +84,21 @@ def test_encoder_const_br_object_based_modes( ): testv = get_testv_path(input_format, sampling_rate) options = get_options(input_format, md_type) - suffix = "_basic_MD" + bitstream_suffix = "_basic_MD" if md_type == ISM_MD_EXTENDED: - suffix = "_ext_MD" + bitstream_suffix = "_ext_MD" elif md_type == ISM_MD_NULL: - suffix = "_null_MD" - bitstream = get_bitstream_path( - DUT_PATH, testv.stem, input_format, bitrate, sampling_rate, dtx, suffix=suffix - ) + bitstream_suffix = "_null_MD" run_check( - bitstream, + input_format, bitrate, sampling_rate, testv, options, dut_encoder_frontend, update_ref == 1, - dtx == DTX_ON, + dtx, + bitstream_suffix=bitstream_suffix, ) @@ -119,21 +114,20 @@ def test_encoder_const_br_scenebased( ): testv = get_testv_path(input_format, sampling_rate) options = get_options(input_format) - suffix = "" + bitstream_suffix = "" if pca == SBA_FOA_PCA_ON: options.extend(["-bypass", "2"]) - suffix += "-pca" - bitstream = get_bitstream_path( - DUT_PATH, testv.stem, input_format, bitrate, sampling_rate, dtx, suffix=suffix - ) + bitstream_suffix += "-pca" run_check( - bitstream, + input_format, bitrate, sampling_rate, testv, options, dut_encoder_frontend, update_ref == 1, + dtx, + bitstream_suffix=bitstream_suffix, ) @@ -143,18 +137,16 @@ def test_encoder_const_br_stereo_dmx_evs( ): testv = get_testv_path("STEREO", sampling_rate) options = ["-stereo_dmx_evs"] - bitstream = get_bitstream_path( - DUT_PATH, testv.stem, "StereoDmxEvs", bitrate, sampling_rate, dtx - ) + input_format = "StereoDmxEvs" run_check( - bitstream, + input_format, bitrate, sampling_rate, testv, options, dut_encoder_frontend, update_ref == 1, - dtx == DTX_ON, + dtx, ) @@ -187,22 +179,13 @@ def test_encoder_const_br_combined_formats( + options_ism[2:] + options_other[2:] ) - suffix = "_basic_MD" + bitstream_suffix = "_basic_MD" if md_type == ISM_MD_EXTENDED: - suffix = "_ext_MD" + bitstream_suffix = "_ext_MD" elif md_type == ISM_MD_NULL: - suffix = "_null_MD" - bitstream = get_bitstream_path( - DUT_PATH, - testv.stem, - input_format_combined, - bitrate, - sampling_rate, - DTX_OFF, - suffix=suffix, - ) + bitstream_suffix = "_null_MD" run_check( - bitstream, + input_format_combined, bitrate, sampling_rate, testv, @@ -210,4 +193,5 @@ def test_encoder_const_br_combined_formats( dut_encoder_frontend, update_ref == 1, False, + bitstream_suffix=bitstream_suffix, ) -- GitLab From 8cb49adbe3b342004b9d602b333b1c07a387c1ec Mon Sep 17 00:00:00 2001 From: kiene Date: Mon, 4 Sep 2023 16:56:15 +0200 Subject: [PATCH 031/106] no EXT output for SBA input --- tests/codec_be_to_accepted_release/constants.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/tests/codec_be_to_accepted_release/constants.py b/tests/codec_be_to_accepted_release/constants.py index e7ecaab5ad..589cee104e 100644 --- a/tests/codec_be_to_accepted_release/constants.py +++ b/tests/codec_be_to_accepted_release/constants.py @@ -215,9 +215,7 @@ OUTPUT_FORMATS_ALL = [ "BINAURAL_ROOM_REVERB", ] OUTPUT_FORMATS_BINAURAL = OUTPUT_FORMATS_ALL[-3:] -OUTPUT_FORMATS_NON_BINAURAL = [ - f for f in OUTPUT_FORMATS_ALL if f not in OUTPUT_FORMATS_BINAURAL -] +OUTPUT_FORMATS_NON_BINAURAL = OUTPUT_FORMATS_ALL[:11] ### -------------- encoder parameter lists -------------- @@ -427,7 +425,7 @@ DECODER_CONST_BR_NO_BINAURAL_OBJECTBASED = collapse_into_list_of_pairs( DECODER_CONST_BR_NO_BINAURAL_SCENEBASED = collapse_into_list_of_pairs( product( SBA_PARAMS, - OUTPUT_FORMATS_NON_BINAURAL, + OUTPUT_FORMATS_NON_BINAURAL[:10], # one less to exclude EXT SAMPLING_RATES_ALL, BITSTREAM_PROCESSING, ) -- GitLab From b97da935a06125c4bfb3fd343f87fa9102f38cf7 Mon Sep 17 00:00:00 2001 From: kiene Date: Mon, 4 Sep 2023 18:05:15 +0200 Subject: [PATCH 032/106] add head rotation testcases --- .../codec_be_to_accepted_release/constants.py | 54 ++++++++++++++++--- .../decoder/__init__.py | 7 +++ 2 files changed, 55 insertions(+), 6 deletions(-) diff --git a/tests/codec_be_to_accepted_release/constants.py b/tests/codec_be_to_accepted_release/constants.py index 589cee104e..882493bce7 100644 --- a/tests/codec_be_to_accepted_release/constants.py +++ b/tests/codec_be_to_accepted_release/constants.py @@ -49,6 +49,14 @@ def bitrates_between(lowest, highest): return [b for b in BITRATES_ALL if b >= lowest and b <= highest] +def get_file_from_repo(path): + """ + Helper for getting a file from the repo, e.g. from scripts/testv. The path needs to be given relative to the repository root! + """ + repo_root = HERE.joinpath("../..").resolve().absolute() + return str(repo_root.joinpath(path)) + + DTX_ON = "DTXon" DTX_OFF = "DTXoff" @@ -425,7 +433,7 @@ DECODER_CONST_BR_NO_BINAURAL_OBJECTBASED = collapse_into_list_of_pairs( DECODER_CONST_BR_NO_BINAURAL_SCENEBASED = collapse_into_list_of_pairs( product( SBA_PARAMS, - OUTPUT_FORMATS_NON_BINAURAL[:10], # one less to exclude EXT + OUTPUT_FORMATS_NON_BINAURAL[:10], # one less to exclude EXT SAMPLING_RATES_ALL, BITSTREAM_PROCESSING, ) @@ -440,18 +448,52 @@ DECODER_CONST_BR_NO_BINAURAL_COMBINED_PARAMS = collapse_into_list_of_pairs( ) # parameters for const bitrate testcases with binaural output -# TODO: add parameters for all the binaural output options BINAURAL_HRTF_NONE = "HRTFdefault" BINAURAL_HRTF_EXT_REPO = "HRFText_repo" HRTF_PATHS = { - BINAURAL_HRTF_EXT_REPO: HERE.joinpath( - "../../scripts/binauralRenderer_interface/binaural_renderers_hrtf_data/ivas_binaural_{fs}kHz.bin" + BINAURAL_HRTF_EXT_REPO: get_file_from_repo( + "scripts/binauralRenderer_interface/binaural_renderers_hrtf_data/ivas_binaural_{fs}kHz.bin" ) - .resolve() - .absolute() } +# TODO: all the following binaural rendering parameters are takenfrom selftest for now -> would need to create critical ones +OTR_NONE = "OTRnone" +OTR_REF = "OTRref" +OTR_AVG = "OTRavg" +OTR_REF_VEC = "OTRref_vec" +OTR_REF_VEC_LEV = "OTRref_vec_lev" +# TODO: exof +OTR_OPTIONS = { + OTR_NONE: list(), + OTR_REF: ["-otr", "ref", "-rf", get_file_from_repo("scripts/testv/headrot.csv")], + OTR_AVG: ["-otr", "avg"], + OTR_REF_VEC: [ + "-otr", + "ref_vec", + "-rvf", + get_file_from_repo("scripts/trajectories/full-circle-4s-Vector3.csv"), + ], + OTR_REF_VEC_LEV: [ + "-otr", + "ref_vec_lev", + "-rvf", + get_file_from_repo( + "scripts/trajectories/full-circle-with-up-and-down-4s-Vector3.csv" + ), + ], +} +HEAD_ROTATION_OPTIONS = { + OTR_NONE: ["-t", get_file_from_repo("scripts/testv/headrot.csv")], + OTR_REF: ["-t", get_file_from_repo("scripts/testv/headrot.csv")], + OTR_REF_VEC: ["-t", get_file_from_repo("scripts/trajectories/full-circle-4s.csv")], + OTR_REF_VEC_LEV: [ + "-t", + get_file_from_repo("scripts/trajectories/full-circle-with-up-and-down-4s.csv"), + ], +} +OTR_PARAMS = [OTR_NONE, OTR_REF, OTR_AVG, OTR_REF_VEC, OTR_REF_VEC_LEV] + DECODER_CONST_BR_BINAURAL_MC_AND_MASA = collapse_into_list_of_pairs( product( MC_PARAMS, diff --git a/tests/codec_be_to_accepted_release/decoder/__init__.py b/tests/codec_be_to_accepted_release/decoder/__init__.py index 6cd4a63566..93a02fe1bb 100644 --- a/tests/codec_be_to_accepted_release/decoder/__init__.py +++ b/tests/codec_be_to_accepted_release/decoder/__init__.py @@ -45,6 +45,8 @@ from ..constants import ( BS_PROC_NONE, BINAURAL_HRTF_NONE, HRTF_PATHS, + OTR_OPTIONS, + HEAD_ROTATION_OPTIONS, ) @@ -99,6 +101,7 @@ def get_bitstream_and_options( dtx, processing, hrtf=BINAURAL_HRTF_NONE, + otr_mode=None, suffix="", non_diegetic_pan_value=None, ): @@ -154,6 +157,10 @@ def get_bitstream_and_options( if non_diegetic_pan_value is not None: options.extend(["-non_diegetic_pan", f"{non_diegetic_pan_value}"]) + if otr_mode is not None: + options.extend(HEAD_ROTATION_OPTIONS.get(otr_mode, list())) + options.extend(OTR_OPTIONS[otr_mode]) + yield bitstream, options -- GitLab From 3f1437aa6adad1513dc458f54de73a3bc7b0170c Mon Sep 17 00:00:00 2001 From: Charles Kinuthia Date: Tue, 5 Sep 2023 09:39:20 +0200 Subject: [PATCH 033/106] test generating reference for test-be-to-release job - specify number of processes to use `NPROC` in config.toml - tag job with test-be-to-release-windows - temporarily skip exit when there are build warnings as there are warnings in main --- .gitlab-ci.yml | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 227472fe39..9dea77e825 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1059,13 +1059,20 @@ test-be-to-release: - MSBuild.exe .\Workspace_msvc\Workspace_msvc.sln /property:Configuration=Debug /p:Platform=win32 /m | tee -variable winoutdata - $winoutdata | Out-File $BUILD_OUTPUT -Encoding Utf8 - ("& python ci/check_for_warnings.py '$BUILD_OUTPUT'") | Invoke-Expression - - ("exit $LASTEXITCODE") | Invoke-Expression + # - If($LASTEXITCODE) {("exit $LASTEXITCODE") | Invoke-Expression} # path to release candidate refs defined in config.toml - - - echo "Placeholder for BE test to release script" - mkdir logs + # test generating references, fail after first error on decoder + - echo "generate references" + + - ("& python -m pytest .\tests\codec_be_to_accepted_release\encoder --update_ref 1 -n $NPROC | tee 'logs/test_encoder-update_ref-log.txt' ") | Invoke-Expression + - If($LASTEXITCODE) {("exit $LASTEXITCODE") | Invoke-Expression} + - ("& python -m pytest .\tests\codec_be_to_accepted_release\decoder\test_decoder_constant_bitrate_no_binaural.py --update_ref 1 -n -x $NPROC | tee 'logs/test_decoder_constant_bitrate_no_binaural-update_ref-log.txt' ") | Invoke-Expression + - If($LASTEXITCODE) {("exit $LASTEXITCODE") | Invoke-Expression} + + artifacts: name: "test-be-to-release--sha-$CI_COMMIT_SHORT_SHA--results" when: always -- GitLab From e9aeeb30e04abeef9b5c801b01827601b6ec88cf Mon Sep 17 00:00:00 2001 From: Charles Kinuthia Date: Tue, 5 Sep 2023 09:43:50 +0200 Subject: [PATCH 034/106] tag job with test-be-to-release-window --- .gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 9dea77e825..8d8545687e 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1048,7 +1048,7 @@ codec-comparison-on-main-push: test-be-to-release: stage: test tags: - - ivas-windows + - test-be-to-release-windows resource_group: ivas-be-to-release-test-resource timeout: "20 minutes" rules: -- GitLab From b5966a2c324e8a1da47ec7829b3971369938b189 Mon Sep 17 00:00:00 2001 From: kiene Date: Tue, 5 Sep 2023 09:50:42 +0200 Subject: [PATCH 035/106] fix encoder dtx parameter in encoder combined format testcases --- .../encoder/test_encoder_constant_bitrate.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/codec_be_to_accepted_release/encoder/test_encoder_constant_bitrate.py b/tests/codec_be_to_accepted_release/encoder/test_encoder_constant_bitrate.py index 3a617bf4be..6e8a2348f4 100644 --- a/tests/codec_be_to_accepted_release/encoder/test_encoder_constant_bitrate.py +++ b/tests/codec_be_to_accepted_release/encoder/test_encoder_constant_bitrate.py @@ -192,6 +192,6 @@ def test_encoder_const_br_combined_formats( options, dut_encoder_frontend, update_ref == 1, - False, + DTX_OFF, bitstream_suffix=bitstream_suffix, ) -- GitLab From ff490e1288610e08da82a0f3bce2e7eabc265b07 Mon Sep 17 00:00:00 2001 From: Charles Kinuthia Date: Tue, 5 Sep 2023 11:32:29 +0200 Subject: [PATCH 036/106] fix typo in ci job test-be-to-release --- .gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 8d8545687e..8aca742049 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1069,7 +1069,7 @@ test-be-to-release: - ("& python -m pytest .\tests\codec_be_to_accepted_release\encoder --update_ref 1 -n $NPROC | tee 'logs/test_encoder-update_ref-log.txt' ") | Invoke-Expression - If($LASTEXITCODE) {("exit $LASTEXITCODE") | Invoke-Expression} - - ("& python -m pytest .\tests\codec_be_to_accepted_release\decoder\test_decoder_constant_bitrate_no_binaural.py --update_ref 1 -n -x $NPROC | tee 'logs/test_decoder_constant_bitrate_no_binaural-update_ref-log.txt' ") | Invoke-Expression + - ("& python -m pytest .\tests\codec_be_to_accepted_release\decoder\test_decoder_constant_bitrate_no_binaural.py --update_ref 1 -n $NPROC -x | tee 'logs/test_decoder_constant_bitrate_no_binaural-update_ref-log.txt' ") | Invoke-Expression - If($LASTEXITCODE) {("exit $LASTEXITCODE") | Invoke-Expression} -- GitLab From ca4c7326e45f939f2879947d7fcdc75a827971a5 Mon Sep 17 00:00:00 2001 From: kiene Date: Tue, 5 Sep 2023 14:10:05 +0200 Subject: [PATCH 037/106] remove JBM testcases from combined modes for now --- tests/codec_be_to_accepted_release/constants.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/codec_be_to_accepted_release/constants.py b/tests/codec_be_to_accepted_release/constants.py index 882493bce7..327a59f62f 100644 --- a/tests/codec_be_to_accepted_release/constants.py +++ b/tests/codec_be_to_accepted_release/constants.py @@ -443,7 +443,7 @@ DECODER_CONST_BR_NO_BINAURAL_COMBINED_PARAMS = collapse_into_list_of_pairs( COMBINED_FORMATS_PARAMS, OUTPUT_FORMATS_NON_BINAURAL, SAMPLING_RATES_ALL, - BITSTREAM_PROCESSING, + BITSTREAM_PROCESSING[:2], # TODO: re-add JBM once fully functional ) ) -- GitLab From b2cd56a5ad39bf57e675df1dbab5d71183755b10 Mon Sep 17 00:00:00 2001 From: Charles Kinuthia Date: Tue, 5 Sep 2023 15:10:39 +0200 Subject: [PATCH 038/106] increase job timeout, test a subset of decoder test --- .gitlab-ci.yml | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 8aca742049..759182e3a6 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1050,7 +1050,7 @@ test-be-to-release: tags: - test-be-to-release-windows resource_group: ivas-be-to-release-test-resource - timeout: "20 minutes" + timeout: "5 hours" # to be reviewed rules: - if: $CI_PIPELINE_SOURCE == 'web' && $MANUAL_PIPELINE_TYPE == 'test-be-release' script: @@ -1064,12 +1064,21 @@ test-be-to-release: # path to release candidate refs defined in config.toml - mkdir logs - # test generating references, fail after first error on decoder + # test generating references - echo "generate references" - ("& python -m pytest .\tests\codec_be_to_accepted_release\encoder --update_ref 1 -n $NPROC | tee 'logs/test_encoder-update_ref-log.txt' ") | Invoke-Expression - If($LASTEXITCODE) {("exit $LASTEXITCODE") | Invoke-Expression} - - ("& python -m pytest .\tests\codec_be_to_accepted_release\decoder\test_decoder_constant_bitrate_no_binaural.py --update_ref 1 -n $NPROC -x | tee 'logs/test_decoder_constant_bitrate_no_binaural-update_ref-log.txt' ") | Invoke-Expression + # - ("& python -m pytest .\tests\codec_be_to_accepted_release\decoder\test_decoder_constant_bitrate_no_binaural.py --update_ref 1 -n $NPROC -x | tee 'logs/test_decoder_constant_bitrate_no_binaural-update_ref-log.txt' ") | Invoke-Expression + + # temporarily test cases that fail + + # test_decoder_scenebased + - ("& python -m pytest .\tests\codec_be_to_accepted_release\decoder\test_decoder_constant_bitrate_no_binaural.py::test_decoder_scenebased --update_ref 1 -n $NPROC | tee 'logs/test_decoder_constant_bitrate_no_binaural_test_decoder_scenebased-update_ref-log.txt' ") | Invoke-Expression + - If($LASTEXITCODE) {("exit $LASTEXITCODE") | Invoke-Expression} + + # test_decoder_combined_formats + - ("& python -m pytest .\tests\codec_be_to_accepted_release\decoder\test_decoder_constant_bitrate_no_binaural.py::test_decoder_combined_formats --update_ref 1 -n $NPROC | tee 'logs/test_decoder_constant_bitrate_no_binaural_test_decoder_combined_formats-update_ref-log.txt' ") | Invoke-Expression - If($LASTEXITCODE) {("exit $LASTEXITCODE") | Invoke-Expression} -- GitLab From 229977137b637ceeab4424dfbaf2e52bbbe9d6cf Mon Sep 17 00:00:00 2001 From: Charles Kinuthia Date: Tue, 5 Sep 2023 15:14:15 +0200 Subject: [PATCH 039/106] fix typo in encoder ref generation --- .../encoder/test_encoder_constant_bitrate.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/codec_be_to_accepted_release/encoder/test_encoder_constant_bitrate.py b/tests/codec_be_to_accepted_release/encoder/test_encoder_constant_bitrate.py index 3a617bf4be..6e8a2348f4 100644 --- a/tests/codec_be_to_accepted_release/encoder/test_encoder_constant_bitrate.py +++ b/tests/codec_be_to_accepted_release/encoder/test_encoder_constant_bitrate.py @@ -192,6 +192,6 @@ def test_encoder_const_br_combined_formats( options, dut_encoder_frontend, update_ref == 1, - False, + DTX_OFF, bitstream_suffix=bitstream_suffix, ) -- GitLab From 65d2ca4bd2cf64875f4224d7499c1b20a1a6c58f Mon Sep 17 00:00:00 2001 From: Charles Kinuthia Date: Tue, 5 Sep 2023 15:41:15 +0200 Subject: [PATCH 040/106] turn off early exit in ci-job when tests fails --- .gitlab-ci.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 759182e3a6..3cea46f42e 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1073,13 +1073,13 @@ test-be-to-release: # temporarily test cases that fail - # test_decoder_scenebased - - ("& python -m pytest .\tests\codec_be_to_accepted_release\decoder\test_decoder_constant_bitrate_no_binaural.py::test_decoder_scenebased --update_ref 1 -n $NPROC | tee 'logs/test_decoder_constant_bitrate_no_binaural_test_decoder_scenebased-update_ref-log.txt' ") | Invoke-Expression - - If($LASTEXITCODE) {("exit $LASTEXITCODE") | Invoke-Expression} - # test_decoder_combined_formats - ("& python -m pytest .\tests\codec_be_to_accepted_release\decoder\test_decoder_constant_bitrate_no_binaural.py::test_decoder_combined_formats --update_ref 1 -n $NPROC | tee 'logs/test_decoder_constant_bitrate_no_binaural_test_decoder_combined_formats-update_ref-log.txt' ") | Invoke-Expression - - If($LASTEXITCODE) {("exit $LASTEXITCODE") | Invoke-Expression} + # - If($LASTEXITCODE) {("exit $LASTEXITCODE") | Invoke-Expression} + + # test_decoder_scenebased + - ("& python -m pytest .\tests\codec_be_to_accepted_release\decoder\test_decoder_constant_bitrate_no_binaural.py::test_decoder_scenebased --update_ref 1 -n $NPROC | tee 'logs/test_decoder_constant_bitrate_no_binaural_test_decoder_scenebased-update_ref-log.txt' ") | Invoke-Expression + # - If($LASTEXITCODE) {("exit $LASTEXITCODE") | Invoke-Expression} artifacts: -- GitLab From 07ed524d67a6cd422a1f8d0acd86c4515928e760 Mon Sep 17 00:00:00 2001 From: kiene Date: Tue, 5 Sep 2023 15:55:06 +0200 Subject: [PATCH 041/106] add exof and head rot testcases --- .../codec_be_to_accepted_release/constants.py | 5 +- .../decoder/__init__.py | 7 + .../decoder/test_decoder_head_rotation.py | 227 ++++++++++++++++++ 3 files changed, 238 insertions(+), 1 deletion(-) create mode 100644 tests/codec_be_to_accepted_release/decoder/test_decoder_head_rotation.py diff --git a/tests/codec_be_to_accepted_release/constants.py b/tests/codec_be_to_accepted_release/constants.py index 327a59f62f..3c781cbf9a 100644 --- a/tests/codec_be_to_accepted_release/constants.py +++ b/tests/codec_be_to_accepted_release/constants.py @@ -463,7 +463,6 @@ OTR_REF = "OTRref" OTR_AVG = "OTRavg" OTR_REF_VEC = "OTRref_vec" OTR_REF_VEC_LEV = "OTRref_vec_lev" -# TODO: exof OTR_OPTIONS = { OTR_NONE: list(), OTR_REF: ["-otr", "ref", "-rf", get_file_from_repo("scripts/testv/headrot.csv")], @@ -493,6 +492,10 @@ HEAD_ROTATION_OPTIONS = { ], } OTR_PARAMS = [OTR_NONE, OTR_REF, OTR_AVG, OTR_REF_VEC, OTR_REF_VEC_LEV] +EXOF_OFF = "EXOF_off" +EXOF_ON = "EXOF_on" +EXOF_OPTIONS = ["-exof", get_file_from_repo("scripts/testv/headrot_case00_3000_q_combinedRotationTest.csv")] +EXOF_PARAMS = [EXOF_OFF, EXOF_ON] DECODER_CONST_BR_BINAURAL_MC_AND_MASA = collapse_into_list_of_pairs( product( diff --git a/tests/codec_be_to_accepted_release/decoder/__init__.py b/tests/codec_be_to_accepted_release/decoder/__init__.py index 93a02fe1bb..c6d04aee85 100644 --- a/tests/codec_be_to_accepted_release/decoder/__init__.py +++ b/tests/codec_be_to_accepted_release/decoder/__init__.py @@ -47,6 +47,9 @@ from ..constants import ( HRTF_PATHS, OTR_OPTIONS, HEAD_ROTATION_OPTIONS, + EXOF_OFF, + EXOF_ON, + EXOF_OPTIONS ) @@ -104,6 +107,7 @@ def get_bitstream_and_options( otr_mode=None, suffix="", non_diegetic_pan_value=None, + exof=EXOF_OFF, ): """ Utility to get either the stored reference bitstream or the processed version as a temporary file @@ -161,6 +165,9 @@ def get_bitstream_and_options( options.extend(HEAD_ROTATION_OPTIONS.get(otr_mode, list())) options.extend(OTR_OPTIONS[otr_mode]) + if exof == EXOF_ON: + options.extend(EXOF_OPTIONS) + yield bitstream, options diff --git a/tests/codec_be_to_accepted_release/decoder/test_decoder_head_rotation.py b/tests/codec_be_to_accepted_release/decoder/test_decoder_head_rotation.py new file mode 100644 index 0000000000..f7bbc0a903 --- /dev/null +++ b/tests/codec_be_to_accepted_release/decoder/test_decoder_head_rotation.py @@ -0,0 +1,227 @@ +__copyright__ = """ +(C) 2022-2023 IVAS codec Public Collaboration with portions copyright Dolby International AB, Ericsson AB, +Fraunhofer-Gesellschaft zur Foerderung der angewandten Forschung e.V., Huawei Technologies Co. LTD., +Koninklijke Philips N.V., Nippon Telegraph and Telephone Corporation, Nokia Technologies Oy, Orange, +Panasonic Holdings Corporation, Qualcomm Technologies, Inc., VoiceAge Corporation, and other +contributors to this repository. All Rights Reserved. + +This software is protected by copyright law and by international treaties. +The IVAS codec Public Collaboration consisting of Dolby International AB, Ericsson AB, +Fraunhofer-Gesellschaft zur Foerderung der angewandten Forschung e.V., Huawei Technologies Co. LTD., +Koninklijke Philips N.V., Nippon Telegraph and Telephone Corporation, Nokia Technologies Oy, Orange, +Panasonic Holdings Corporation, Qualcomm Technologies, Inc., VoiceAge Corporation, and other +contributors to this repository retain full ownership rights in their respective contributions in +the software. This notice grants no license of any kind, including but not limited to patent +license, nor is any license granted by implication, estoppel or otherwise. + +Contributors are required to enter into the IVAS codec Public Collaboration agreement before making +contributions. + +This software is provided "AS IS", without any express or implied warranties. The software is in the +development stage. It is intended exclusively for experts who have experience with such software and +solely for the purpose of inspection. All implied warranties of non-infringement, merchantability +and fitness for a particular purpose are hereby disclaimed and excluded. + +Any dispute, controversy or claim arising under or in relation to providing this software shall be +submitted to and settled by the final, binding jurisdiction of the courts of Munich, Germany in +accordance with the laws of the Federal Republic of Germany excluding its conflict of law rules and +the United Nations Convention on Contracts on the International Sales of Goods. +""" + +import pytest +from . import get_bitstream_and_options, run_check +from .. import get_testv_path +from ..constants import * + + +@pytest.mark.parametrize("exof", EXOF_PARAMS) +@pytest.mark.parametrize("otr_mode", OTR_PARAMS) +@pytest.mark.parametrize( + "input_format,bitrate,input_sampling_rate,dtx,output_format,output_sampling_rate,bitstream_processing", + DECODER_CONST_BR_BINAURAL_MC_AND_MASA, +) +def test_decoder_hr_mc_and_masa( + input_format, + bitrate, + input_sampling_rate, + dtx, + output_format, + output_sampling_rate, + bitstream_processing, + otr_mode, + exof, + dut_decoder_frontend, + update_ref, +): + testv_name = get_testv_path(input_format, input_sampling_rate).stem + with get_bitstream_and_options( + testv_name, + input_format, + bitrate, + input_sampling_rate, + dtx, + processing=bitstream_processing, + otr_mode=otr_mode, + exof=exof + ) as (ref_bitstream, options): + run_check( + ref_bitstream, + output_format, + output_sampling_rate, + options, + dut_decoder_frontend, + update_ref == 1, + bitstream_processing=bitstream_processing, + output_suffix=f"{otr_mode}-{exof}" + ) + + +# NOTE: md_type probably does not need to be exhaustively tested in combination with OTR +# NOTE: same for FER/JBM +@pytest.mark.parametrize("exof", EXOF_PARAMS) +@pytest.mark.parametrize("otr_mode", OTR_PARAMS) +@pytest.mark.parametrize( + "input_format,bitrate,input_sampling_rate,dtx,md_type,output_format,output_sampling_rate,bitstream_processing", + DECODER_CONST_BR_BINAURAL_OBJECTBASED, +) +def test_decoder_hr_objectbased( + input_format, + bitrate, + input_sampling_rate, + dtx, + md_type, + output_format, + output_sampling_rate, + bitstream_processing, + otr_mode, + exof, + dut_decoder_frontend, + update_ref, +): + testv_name = get_testv_path(input_format, input_sampling_rate).stem + suffix = "_basic_MD" + if md_type == ISM_MD_EXTENDED: + suffix = "_ext_MD" + elif md_type == ISM_MD_NULL: + suffix = "_null_MD" + with get_bitstream_and_options( + testv_name, + input_format, + bitrate, + input_sampling_rate, + dtx, + suffix=suffix, + processing=bitstream_processing, + otr_mode=otr_mode, + exof=exof + ) as (ref_bitstream, options): + run_check( + ref_bitstream, + output_format, + output_sampling_rate, + options, + dut_decoder_frontend, + update_ref == 1, + bitstream_processing=bitstream_processing, + output_suffix=f"{otr_mode}-{exof}" + ) + + +@pytest.mark.parametrize("exof", EXOF_PARAMS) +@pytest.mark.parametrize("otr_mode", OTR_PARAMS) +@pytest.mark.parametrize( + "input_format,bitrate,input_sampling_rate,dtx,pca,output_format,output_sampling_rate,bitstream_processing", + DECODER_CONST_BR_BINAURAL_SCENEBASED, +) +def test_decoder_hr_scenebased( + input_format, + bitrate, + input_sampling_rate, + dtx, + pca, + output_format, + output_sampling_rate, + bitstream_processing, + otr_mode, + exof, + dut_decoder_frontend, + update_ref, +): + testv_name = get_testv_path(input_format, input_sampling_rate).stem + suffix = "" + if pca == SBA_FOA_PCA_ON: + suffix += "-pca" + with get_bitstream_and_options( + testv_name, + input_format, + bitrate, + input_sampling_rate, + dtx, + suffix=suffix, + processing=bitstream_processing, + otr_mode=otr_mode, + exof=exof + ) as (ref_bitstream, options): + run_check( + ref_bitstream, + output_format, + output_sampling_rate, + options, + dut_decoder_frontend, + update_ref == 1, + bitstream_processing=bitstream_processing, + output_suffix=f"{otr_mode}-{exof}" + ) + + +@pytest.mark.parametrize("exof", EXOF_PARAMS) +@pytest.mark.parametrize("otr_mode", OTR_PARAMS) +@pytest.mark.parametrize( + "input_format_ism,input_format_other,bitrate,input_sampling_rate,md_type,output_format,output_sampling_rate,bitstream_processing", + DECODER_CONST_BR_BINAURAL_COMBINED_PARAMS, +) +def test_decoder_hr_combined_formats( + input_format_ism, + input_format_other, + bitrate, + input_sampling_rate, + md_type, + output_format, + output_sampling_rate, + bitstream_processing, + otr_mode, + exof, + dut_decoder_frontend, + update_ref, +): + input_format_combined = "OMASA" if "MASA" in input_format_other else "OSBA" + testv_name = get_testv_path( + f"{input_format_combined}_{input_format_ism}_{input_format_other}", + input_sampling_rate, + ).stem + suffix = "_basic_MD" + if md_type == ISM_MD_EXTENDED: + suffix = "_ext_MD" + elif md_type == ISM_MD_NULL: + suffix = "_null_MD" + with get_bitstream_and_options( + testv_name, + input_format_combined, + bitrate, + input_sampling_rate, + DTX_OFF, + suffix=suffix, + processing=bitstream_processing, + otr_mode=otr_mode, + exof=exof, + ) as (ref_bitstream, options): + run_check( + ref_bitstream, + output_format, + output_sampling_rate, + options, + dut_decoder_frontend, + update_ref == 1, + bitstream_processing=bitstream_processing, + output_suffix=f"{otr_mode}-{exof}" + ) -- GitLab From 4a7779a61f96bccb31e276ef21351ea9c77704c8 Mon Sep 17 00:00:00 2001 From: kiene Date: Wed, 6 Sep 2023 11:22:10 +0200 Subject: [PATCH 042/106] skip planar FOA JBM decoder tests --- .../decoder/test_decoder_constant_bitrate_binaural.py | 3 +++ .../decoder/test_decoder_constant_bitrate_no_binaural.py | 3 +++ 2 files changed, 6 insertions(+) diff --git a/tests/codec_be_to_accepted_release/decoder/test_decoder_constant_bitrate_binaural.py b/tests/codec_be_to_accepted_release/decoder/test_decoder_constant_bitrate_binaural.py index c32c0959d6..39fd7057b9 100644 --- a/tests/codec_be_to_accepted_release/decoder/test_decoder_constant_bitrate_binaural.py +++ b/tests/codec_be_to_accepted_release/decoder/test_decoder_constant_bitrate_binaural.py @@ -131,6 +131,9 @@ def test_decoder_scenebased( dut_decoder_frontend, update_ref, ): + # TODO: remove once fixed + if input_format == "PlanarFOA" and bitstream_processing == BS_PROC_JBM_DLY_PROF_5: + pytest.skip("PlanarFOA and JBM currently crashes") testv_name = get_testv_path(input_format, input_sampling_rate).stem suffix = "" if pca == SBA_FOA_PCA_ON: diff --git a/tests/codec_be_to_accepted_release/decoder/test_decoder_constant_bitrate_no_binaural.py b/tests/codec_be_to_accepted_release/decoder/test_decoder_constant_bitrate_no_binaural.py index 6054ae8e69..f9b8f05f81 100644 --- a/tests/codec_be_to_accepted_release/decoder/test_decoder_constant_bitrate_no_binaural.py +++ b/tests/codec_be_to_accepted_release/decoder/test_decoder_constant_bitrate_no_binaural.py @@ -127,6 +127,9 @@ def test_decoder_scenebased( dut_decoder_frontend, update_ref, ): + # TODO: remove once fixed + if input_format == "PlanarFOA" and bitstream_processing == BS_PROC_JBM_DLY_PROF_5: + pytest.skip("PlanarFOA and JBM currently crashes") testv_name = get_testv_path(input_format, input_sampling_rate).stem suffix = "" if pca == SBA_FOA_PCA_ON: -- GitLab From ac4cc815e22ffe35f5de9d0d2df4390341a246db Mon Sep 17 00:00:00 2001 From: Charles Kinuthia Date: Wed, 6 Sep 2023 11:55:46 +0200 Subject: [PATCH 043/106] temporarily skip encoder generation also run test_decoder_scenebased first --- .gitlab-ci.yml | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 317f9616e1..128056aed5 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1086,14 +1086,13 @@ test-be-to-release: # temporarily test cases that fail - # test_decoder_combined_formats - - ("& python -m pytest .\tests\codec_be_to_accepted_release\decoder\test_decoder_constant_bitrate_no_binaural.py::test_decoder_combined_formats --update_ref 1 -n $NPROC | tee 'logs/test_decoder_constant_bitrate_no_binaural_test_decoder_combined_formats-update_ref-log.txt' ") | Invoke-Expression - # - If($LASTEXITCODE) {("exit $LASTEXITCODE") | Invoke-Expression} - # test_decoder_scenebased - ("& python -m pytest .\tests\codec_be_to_accepted_release\decoder\test_decoder_constant_bitrate_no_binaural.py::test_decoder_scenebased --update_ref 1 -n $NPROC | tee 'logs/test_decoder_constant_bitrate_no_binaural_test_decoder_scenebased-update_ref-log.txt' ") | Invoke-Expression - # - If($LASTEXITCODE) {("exit $LASTEXITCODE") | Invoke-Expression} + - If($LASTEXITCODE) {("exit $LASTEXITCODE") | Invoke-Expression} + # test_decoder_combined_formats + - ("& python -m pytest .\tests\codec_be_to_accepted_release\decoder\test_decoder_constant_bitrate_no_binaural.py::test_decoder_combined_formats --update_ref 1 -n $NPROC | tee 'logs/test_decoder_constant_bitrate_no_binaural_test_decoder_combined_formats-update_ref-log.txt' ") | Invoke-Expression + - If($LASTEXITCODE) {("exit $LASTEXITCODE") | Invoke-Expression} artifacts: name: "test-be-to-release--sha-$CI_COMMIT_SHORT_SHA--results" -- GitLab From 4cc4fde4eddc42448f0c249e4a67060f4d8593e6 Mon Sep 17 00:00:00 2001 From: Charles Kinuthia Date: Wed, 6 Sep 2023 12:02:33 +0200 Subject: [PATCH 044/106] temporarily skip encoder generation also run test_decoder_scenebased first --- .gitlab-ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 128056aed5..0696c26520 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1080,8 +1080,8 @@ test-be-to-release: # test generating references - echo "generate references" - - ("& python -m pytest .\tests\codec_be_to_accepted_release\encoder --update_ref 1 -n $NPROC | tee 'logs/test_encoder-update_ref-log.txt' ") | Invoke-Expression - - If($LASTEXITCODE) {("exit $LASTEXITCODE") | Invoke-Expression} + # - ("& python -m pytest .\tests\codec_be_to_accepted_release\encoder --update_ref 1 -n $NPROC | tee 'logs/test_encoder-update_ref-log.txt' ") | Invoke-Expression + # - If($LASTEXITCODE) {("exit $LASTEXITCODE") | Invoke-Expression} # - ("& python -m pytest .\tests\codec_be_to_accepted_release\decoder\test_decoder_constant_bitrate_no_binaural.py --update_ref 1 -n $NPROC -x | tee 'logs/test_decoder_constant_bitrate_no_binaural-update_ref-log.txt' ") | Invoke-Expression # temporarily test cases that fail -- GitLab From 6e20f1594c0d0510961f524edc3379954ae3d402 Mon Sep 17 00:00:00 2001 From: kiene Date: Wed, 6 Sep 2023 12:11:35 +0200 Subject: [PATCH 045/106] add forgotten hrtf parametrization in binaural test cases --- .../decoder/test_decoder_constant_bitrate_binaural.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tests/codec_be_to_accepted_release/decoder/test_decoder_constant_bitrate_binaural.py b/tests/codec_be_to_accepted_release/decoder/test_decoder_constant_bitrate_binaural.py index 39fd7057b9..26cd6cbf29 100644 --- a/tests/codec_be_to_accepted_release/decoder/test_decoder_constant_bitrate_binaural.py +++ b/tests/codec_be_to_accepted_release/decoder/test_decoder_constant_bitrate_binaural.py @@ -73,6 +73,7 @@ def test_decoder_mc_and_masa( ) +@pytest.mark.parametrize("hrtf", [BINAURAL_HRTF_NONE, BINAURAL_HRTF_EXT_REPO]) @pytest.mark.parametrize( "input_format,bitrate,input_sampling_rate,dtx,md_type,output_format,output_sampling_rate,bitstream_processing", DECODER_CONST_BR_BINAURAL_OBJECTBASED, @@ -115,6 +116,7 @@ def test_decoder_objectbased( ) +@pytest.mark.parametrize("hrtf", [BINAURAL_HRTF_NONE, BINAURAL_HRTF_EXT_REPO]) @pytest.mark.parametrize( "input_format,bitrate,input_sampling_rate,dtx,pca,output_format,output_sampling_rate,bitstream_processing", DECODER_CONST_BR_BINAURAL_SCENEBASED, @@ -158,6 +160,7 @@ def test_decoder_scenebased( ) +@pytest.mark.parametrize("hrtf", [BINAURAL_HRTF_NONE, BINAURAL_HRTF_EXT_REPO]) @pytest.mark.parametrize( "input_format_ism,input_format_other,bitrate,input_sampling_rate,md_type,output_format,output_sampling_rate,bitstream_processing", DECODER_CONST_BR_BINAURAL_COMBINED_PARAMS, -- GitLab From 8267c30370750a47055155b638ba4a1e940c7b27 Mon Sep 17 00:00:00 2001 From: kiene Date: Wed, 6 Sep 2023 15:09:37 +0200 Subject: [PATCH 046/106] fix hrtf usage in binaural testcases --- .../decoder/test_decoder_constant_bitrate_binaural.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/tests/codec_be_to_accepted_release/decoder/test_decoder_constant_bitrate_binaural.py b/tests/codec_be_to_accepted_release/decoder/test_decoder_constant_bitrate_binaural.py index 26cd6cbf29..9f18c4abf1 100644 --- a/tests/codec_be_to_accepted_release/decoder/test_decoder_constant_bitrate_binaural.py +++ b/tests/codec_be_to_accepted_release/decoder/test_decoder_constant_bitrate_binaural.py @@ -87,6 +87,7 @@ def test_decoder_objectbased( output_format, output_sampling_rate, bitstream_processing, + hrtf, dut_decoder_frontend, update_ref, ): @@ -112,6 +113,7 @@ def test_decoder_objectbased( options, dut_decoder_frontend, update_ref == 1, + output_suffix=hrtf, bitstream_processing=bitstream_processing, ) @@ -130,6 +132,7 @@ def test_decoder_scenebased( output_format, output_sampling_rate, bitstream_processing, + hrtf, dut_decoder_frontend, update_ref, ): @@ -156,6 +159,7 @@ def test_decoder_scenebased( options, dut_decoder_frontend, update_ref == 1, + output_suffix=hrtf, bitstream_processing=bitstream_processing, ) @@ -174,6 +178,7 @@ def test_decoder_combined_formats( output_format, output_sampling_rate, bitstream_processing, + hrtf, dut_decoder_frontend, update_ref, ): @@ -203,5 +208,6 @@ def test_decoder_combined_formats( options, dut_decoder_frontend, update_ref == 1, + output_suffix=hrtf, bitstream_processing=bitstream_processing, ) -- GitLab From e599d8844824f5d41845138bca76420cdb96c9a3 Mon Sep 17 00:00:00 2001 From: kiene Date: Wed, 6 Sep 2023 15:13:38 +0200 Subject: [PATCH 047/106] add another forgotten part --- .../decoder/test_decoder_constant_bitrate_binaural.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tests/codec_be_to_accepted_release/decoder/test_decoder_constant_bitrate_binaural.py b/tests/codec_be_to_accepted_release/decoder/test_decoder_constant_bitrate_binaural.py index 9f18c4abf1..ff2cc0d53a 100644 --- a/tests/codec_be_to_accepted_release/decoder/test_decoder_constant_bitrate_binaural.py +++ b/tests/codec_be_to_accepted_release/decoder/test_decoder_constant_bitrate_binaural.py @@ -104,6 +104,7 @@ def test_decoder_objectbased( input_sampling_rate, dtx, suffix=suffix, + hrtf=hrtf, processing=bitstream_processing, ) as (ref_bitstream, options): run_check( @@ -150,6 +151,7 @@ def test_decoder_scenebased( input_sampling_rate, dtx, suffix=suffix, + hrtf=hrtf, processing=bitstream_processing, ) as (ref_bitstream, options): run_check( @@ -199,6 +201,7 @@ def test_decoder_combined_formats( input_sampling_rate, DTX_OFF, suffix=suffix, + hrtf=hrtf, processing=bitstream_processing, ) as (ref_bitstream, options): run_check( -- GitLab From 01b3a95476f8392ca4c1327a23d39cf1437c83a0 Mon Sep 17 00:00:00 2001 From: Charles Kinuthia Date: Wed, 6 Sep 2023 15:56:21 +0200 Subject: [PATCH 048/106] temporarily add 1s timeout to decoder test decoder tests to run: - decoder\test_decoder_constant_bitrate_no_binaural.py::test_decoder_combined_formats --- .gitlab-ci.yml | 10 +-- .../decoder/__init__.py | 1 + tests/conftest.py | 61 +++++++++++-------- 3 files changed, 43 insertions(+), 29 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 0696c26520..571f1626f3 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1080,18 +1080,18 @@ test-be-to-release: # test generating references - echo "generate references" - # - ("& python -m pytest .\tests\codec_be_to_accepted_release\encoder --update_ref 1 -n $NPROC | tee 'logs/test_encoder-update_ref-log.txt' ") | Invoke-Expression + # - ("& python -m pytest .\tests\codec_be_to_accepted_release\encoder --update_ref 1 -n $NPROC 2>&1 | tee 'logs/test_encoder-update_ref-log.txt' ") | Invoke-Expression # - If($LASTEXITCODE) {("exit $LASTEXITCODE") | Invoke-Expression} - # - ("& python -m pytest .\tests\codec_be_to_accepted_release\decoder\test_decoder_constant_bitrate_no_binaural.py --update_ref 1 -n $NPROC -x | tee 'logs/test_decoder_constant_bitrate_no_binaural-update_ref-log.txt' ") | Invoke-Expression + # - ("& python -m pytest .\tests\codec_be_to_accepted_release\decoder\test_decoder_constant_bitrate_no_binaural.py --update_ref 1 -n $NPROC -x 2>&1 | tee 'logs/test_decoder_constant_bitrate_no_binaural-update_ref-log.txt' ") | Invoke-Expression # temporarily test cases that fail # test_decoder_scenebased - - ("& python -m pytest .\tests\codec_be_to_accepted_release\decoder\test_decoder_constant_bitrate_no_binaural.py::test_decoder_scenebased --update_ref 1 -n $NPROC | tee 'logs/test_decoder_constant_bitrate_no_binaural_test_decoder_scenebased-update_ref-log.txt' ") | Invoke-Expression - - If($LASTEXITCODE) {("exit $LASTEXITCODE") | Invoke-Expression} + #- ("& python -m pytest .\tests\codec_be_to_accepted_release\decoder\test_decoder_constant_bitrate_no_binaural.py::test_decoder_scenebased --update_ref 1 -n $NPROC 2>&1 | tee 'logs/test_decoder_constant_bitrate_no_binaural_test_decoder_scenebased-update_ref-log.txt' ") | Invoke-Expression + #- If($LASTEXITCODE) {("exit $LASTEXITCODE") | Invoke-Expression} # test_decoder_combined_formats - - ("& python -m pytest .\tests\codec_be_to_accepted_release\decoder\test_decoder_constant_bitrate_no_binaural.py::test_decoder_combined_formats --update_ref 1 -n $NPROC | tee 'logs/test_decoder_constant_bitrate_no_binaural_test_decoder_combined_formats-update_ref-log.txt' ") | Invoke-Expression + - ("& python -m pytest .\tests\codec_be_to_accepted_release\decoder\test_decoder_constant_bitrate_no_binaural.py::test_decoder_combined_formats --update_ref 1 -n $NPROC 2>&1 | tee 'logs/test_decoder_constant_bitrate_no_binaural_test_decoder_combined_formats-update_ref-log.txt' ") | Invoke-Expression - If($LASTEXITCODE) {("exit $LASTEXITCODE") | Invoke-Expression} artifacts: diff --git a/tests/codec_be_to_accepted_release/decoder/__init__.py b/tests/codec_be_to_accepted_release/decoder/__init__.py index c6d04aee85..42d119f34b 100644 --- a/tests/codec_be_to_accepted_release/decoder/__init__.py +++ b/tests/codec_be_to_accepted_release/decoder/__init__.py @@ -195,6 +195,7 @@ def run_check( ref_bitstream, output_path, add_option_list=options, + timeout=1, # for testing, to be reviewed ) if not is_ref_creation and not is_be_to_ref(output_path): diff --git a/tests/conftest.py b/tests/conftest.py index ac1b37121a..2ce7b1833d 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -38,6 +38,7 @@ import logging from pathlib import Path import platform from subprocess import run +from subprocess import TimeoutExpired import textwrap from typing import Optional import os @@ -219,6 +220,7 @@ class EncoderFrontend: bypass_mode: Optional[int] = None, quiet_mode: Optional[bool] = True, add_option_list: Optional[list] = None, + timeout: Optional[int] = None, ) -> None: command = [self._path] @@ -252,18 +254,23 @@ class EncoderFrontend: cmd_str = textwrap.indent(" ".join(command), prefix="\t") log_dbg_msg(f"{self._type} encoder command:\n{cmd_str}") - result = run(command, capture_output=True, check=False) - self.returncode = result.returncode - self.stderr = result.stderr.decode("ascii") - self.stdout = result.stdout.decode("ascii") - if self.stdout: - stdout_str = textwrap.indent(self.stdout, prefix="\t") - log_dbg_msg(f"{self._type} encoder stdout:\n{stdout_str}") - if self.stderr: - stderr_str = textwrap.indent(self.stderr, prefix="\t") - log_dbg_msg(f"{self._type} encoder stderr:\n{stderr_str}") - if self.returncode: - pytest.fail(f"{self._type} encoder terminated with a non-0 return code: {self.returncode}") + try: + result = run(command, capture_output=True, check=False, timeout=timeout) + self.returncode = result.returncode + self.stderr = result.stderr.decode("ascii") + self.stdout = result.stdout.decode("ascii") + if self.stdout: + stdout_str = textwrap.indent(self.stdout, prefix="\t") + log_dbg_msg(f"{self._type} encoder stdout:\n{stdout_str}") + if self.stderr: + stderr_str = textwrap.indent(self.stderr, prefix="\t") + log_dbg_msg(f"{self._type} encoder stderr:\n{stderr_str}") + if self.returncode: + pytest.fail(f"{self._type} encoder terminated with a non-0 return code: {self.returncode}") + except TimeoutExpired as err: + log_dbg_msg(f"{self._type} encoder timed stdout:\n{err.stdout}") + log_dbg_msg(f"{self._type} encoder timed stderr:\n{err.stderr}") + pytest.fail(f"{self._type} encoder timed out") def _check_run(self): if self.returncode is not None: @@ -360,6 +367,7 @@ class DecoderFrontend: quiet_mode: Optional[bool] = True, plc_file: Optional[Path] = None, add_option_list: Optional[list] = None, + timeout: Optional[int] = None, ) -> None: command = [self._path] @@ -386,18 +394,23 @@ class DecoderFrontend: cmd_str = textwrap.indent(" ".join(command), prefix="\t") log_dbg_msg(f"{self._type} decoder command:\n{cmd_str}") - result = run(command, capture_output=True, check=False) - self.returncode = result.returncode - self.stderr = result.stderr.decode("ascii") - self.stdout = result.stdout.decode("ascii") - if self.stdout: - stdout_str = textwrap.indent(self.stdout, prefix="\t") - log_dbg_msg(f"{self._type} decoder stdout:\n{stdout_str}") - if self.stderr: - stderr_str = textwrap.indent(self.stderr, prefix="\t") - log_dbg_msg(f"{self._type} decoder stderr:\n{stderr_str}") - if self.returncode: - pytest.fail(f"{self._type} decoder terminated with a non-0 return code: {self.returncode}") + try: + result = run(command, capture_output=True, check=False, timeout=timeout) + self.returncode = result.returncode + self.stderr = result.stderr.decode("ascii") + self.stdout = result.stdout.decode("ascii") + if self.stdout: + stdout_str = textwrap.indent(self.stdout, prefix="\t") + log_dbg_msg(f"{self._type} decoder stdout:\n{stdout_str}") + if self.stderr: + stderr_str = textwrap.indent(self.stderr, prefix="\t") + log_dbg_msg(f"{self._type} decoder stderr:\n{stderr_str}") + if self.returncode: + pytest.fail(f"{self._type} decoder terminated with a non-0 return code: {self.returncode}") + except TimeoutExpired as err: + log_dbg_msg(f"{self._type} decoder timed stdout:\n{err.stdout}") + log_dbg_msg(f"{self._type} decoder timed stderr:\n{err.stderr}") + pytest.fail(f"{self._type} decoder timed out") def _check_run(self): if self.returncode is not None: -- GitLab From e0a18b1a60a7cb98276ad24c9508238bc13c20f0 Mon Sep 17 00:00:00 2001 From: kinuthia Date: Wed, 6 Sep 2023 19:29:24 +0000 Subject: [PATCH 049/106] set timeout=10s for testing with short test vectors --- tests/codec_be_to_accepted_release/decoder/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/codec_be_to_accepted_release/decoder/__init__.py b/tests/codec_be_to_accepted_release/decoder/__init__.py index 42d119f34b..0c52142804 100644 --- a/tests/codec_be_to_accepted_release/decoder/__init__.py +++ b/tests/codec_be_to_accepted_release/decoder/__init__.py @@ -195,7 +195,7 @@ def run_check( ref_bitstream, output_path, add_option_list=options, - timeout=1, # for testing, to be reviewed + timeout=10, # for testing with short testv, to be reviewed ) if not is_ref_creation and not is_be_to_ref(output_path): -- GitLab From 56c3dcab8cefe1c972223e24a9cf95e0f90d6af8 Mon Sep 17 00:00:00 2001 From: kiene Date: Thu, 7 Sep 2023 11:01:49 +0200 Subject: [PATCH 050/106] add directivity ISM rendering testcases --- .../codec_be_to_accepted_release/constants.py | 15 +- .../decoder/__init__.py | 22 ++- .../decoder/test_decoder_head_rotation.py | 14 +- .../decoder/test_decoder_render_config.py | 128 ++++++++++++++++++ 4 files changed, 169 insertions(+), 10 deletions(-) create mode 100644 tests/codec_be_to_accepted_release/decoder/test_decoder_render_config.py diff --git a/tests/codec_be_to_accepted_release/constants.py b/tests/codec_be_to_accepted_release/constants.py index 3c781cbf9a..9b51c1ea78 100644 --- a/tests/codec_be_to_accepted_release/constants.py +++ b/tests/codec_be_to_accepted_release/constants.py @@ -443,7 +443,7 @@ DECODER_CONST_BR_NO_BINAURAL_COMBINED_PARAMS = collapse_into_list_of_pairs( COMBINED_FORMATS_PARAMS, OUTPUT_FORMATS_NON_BINAURAL, SAMPLING_RATES_ALL, - BITSTREAM_PROCESSING[:2], # TODO: re-add JBM once fully functional + BITSTREAM_PROCESSING[:2], # TODO: re-add JBM once fully functional ) ) @@ -494,9 +494,20 @@ HEAD_ROTATION_OPTIONS = { OTR_PARAMS = [OTR_NONE, OTR_REF, OTR_AVG, OTR_REF_VEC, OTR_REF_VEC_LEV] EXOF_OFF = "EXOF_off" EXOF_ON = "EXOF_on" -EXOF_OPTIONS = ["-exof", get_file_from_repo("scripts/testv/headrot_case00_3000_q_combinedRotationTest.csv")] +EXOF_OPTIONS = [ + "-exof", + get_file_from_repo("scripts/testv/headrot_case00_3000_q_combinedRotationTest.csv"), +] EXOF_PARAMS = [EXOF_OFF, EXOF_ON] +RENDER_CONFIG_OPTIONS = [ + "-render_config", + str(TESTV_PATH.joinpath("render_config.cfg")), +] +DPID_OPTIONS = ["-dpid", "0", "1", "2", "3"] +# For testing only +DPID_OPTIONS = ["-dpid", "0", "0", "0", "0"] + DECODER_CONST_BR_BINAURAL_MC_AND_MASA = collapse_into_list_of_pairs( product( MC_PARAMS, diff --git a/tests/codec_be_to_accepted_release/decoder/__init__.py b/tests/codec_be_to_accepted_release/decoder/__init__.py index c6d04aee85..54c71b5d9b 100644 --- a/tests/codec_be_to_accepted_release/decoder/__init__.py +++ b/tests/codec_be_to_accepted_release/decoder/__init__.py @@ -49,7 +49,9 @@ from ..constants import ( HEAD_ROTATION_OPTIONS, EXOF_OFF, EXOF_ON, - EXOF_OPTIONS + EXOF_OPTIONS, + RENDER_CONFIG_OPTIONS, + DPID_OPTIONS, ) @@ -108,6 +110,8 @@ def get_bitstream_and_options( suffix="", non_diegetic_pan_value=None, exof=EXOF_OFF, + render_config=False, + dpid=False, ): """ Utility to get either the stored reference bitstream or the processed version as a temporary file @@ -168,6 +172,22 @@ def get_bitstream_and_options( if exof == EXOF_ON: options.extend(EXOF_OPTIONS) + if render_config: + options.extend(RENDER_CONFIG_OPTIONS) + + if dpid: + assert render_config + if encoder_format.startswith("ISM"): + n_ism = int(encoder_format[-1]) + elif encoder_format in ["OMASA", "OSBA"]: + # get number of isms from testv name + n_ism = int(testv_name.split("_")[2][0]) + else: + raise ValueError("dpid only for ISM and combined_formats") + + # +1 for the "-dpid" at index 0 + options.extend(DPID_OPTIONS[: n_ism + 1]) + yield bitstream, options diff --git a/tests/codec_be_to_accepted_release/decoder/test_decoder_head_rotation.py b/tests/codec_be_to_accepted_release/decoder/test_decoder_head_rotation.py index f7bbc0a903..0a53842c5d 100644 --- a/tests/codec_be_to_accepted_release/decoder/test_decoder_head_rotation.py +++ b/tests/codec_be_to_accepted_release/decoder/test_decoder_head_rotation.py @@ -62,7 +62,7 @@ def test_decoder_hr_mc_and_masa( dtx, processing=bitstream_processing, otr_mode=otr_mode, - exof=exof + exof=exof, ) as (ref_bitstream, options): run_check( ref_bitstream, @@ -72,7 +72,7 @@ def test_decoder_hr_mc_and_masa( dut_decoder_frontend, update_ref == 1, bitstream_processing=bitstream_processing, - output_suffix=f"{otr_mode}-{exof}" + output_suffix=f"{otr_mode}-{exof}", ) @@ -113,7 +113,7 @@ def test_decoder_hr_objectbased( suffix=suffix, processing=bitstream_processing, otr_mode=otr_mode, - exof=exof + exof=exof, ) as (ref_bitstream, options): run_check( ref_bitstream, @@ -123,7 +123,7 @@ def test_decoder_hr_objectbased( dut_decoder_frontend, update_ref == 1, bitstream_processing=bitstream_processing, - output_suffix=f"{otr_mode}-{exof}" + output_suffix=f"{otr_mode}-{exof}", ) @@ -160,7 +160,7 @@ def test_decoder_hr_scenebased( suffix=suffix, processing=bitstream_processing, otr_mode=otr_mode, - exof=exof + exof=exof, ) as (ref_bitstream, options): run_check( ref_bitstream, @@ -170,7 +170,7 @@ def test_decoder_hr_scenebased( dut_decoder_frontend, update_ref == 1, bitstream_processing=bitstream_processing, - output_suffix=f"{otr_mode}-{exof}" + output_suffix=f"{otr_mode}-{exof}", ) @@ -223,5 +223,5 @@ def test_decoder_hr_combined_formats( dut_decoder_frontend, update_ref == 1, bitstream_processing=bitstream_processing, - output_suffix=f"{otr_mode}-{exof}" + output_suffix=f"{otr_mode}-{exof}", ) diff --git a/tests/codec_be_to_accepted_release/decoder/test_decoder_render_config.py b/tests/codec_be_to_accepted_release/decoder/test_decoder_render_config.py new file mode 100644 index 0000000000..ba3f45b9ca --- /dev/null +++ b/tests/codec_be_to_accepted_release/decoder/test_decoder_render_config.py @@ -0,0 +1,128 @@ +__copyright__ = """ +(C) 2022-2023 IVAS codec Public Collaboration with portions copyright Dolby International AB, Ericsson AB, +Fraunhofer-Gesellschaft zur Foerderung der angewandten Forschung e.V., Huawei Technologies Co. LTD., +Koninklijke Philips N.V., Nippon Telegraph and Telephone Corporation, Nokia Technologies Oy, Orange, +Panasonic Holdings Corporation, Qualcomm Technologies, Inc., VoiceAge Corporation, and other +contributors to this repository. All Rights Reserved. + +This software is protected by copyright law and by international treaties. +The IVAS codec Public Collaboration consisting of Dolby International AB, Ericsson AB, +Fraunhofer-Gesellschaft zur Foerderung der angewandten Forschung e.V., Huawei Technologies Co. LTD., +Koninklijke Philips N.V., Nippon Telegraph and Telephone Corporation, Nokia Technologies Oy, Orange, +Panasonic Holdings Corporation, Qualcomm Technologies, Inc., VoiceAge Corporation, and other +contributors to this repository retain full ownership rights in their respective contributions in +the software. This notice grants no license of any kind, including but not limited to patent +license, nor is any license granted by implication, estoppel or otherwise. + +Contributors are required to enter into the IVAS codec Public Collaboration agreement before making +contributions. + +This software is provided "AS IS", without any express or implied warranties. The software is in the +development stage. It is intended exclusively for experts who have experience with such software and +solely for the purpose of inspection. All implied warranties of non-infringement, merchantability +and fitness for a particular purpose are hereby disclaimed and excluded. + +Any dispute, controversy or claim arising under or in relation to providing this software shall be +submitted to and settled by the final, binding jurisdiction of the courts of Munich, Germany in +accordance with the laws of the Federal Republic of Germany excluding its conflict of law rules and +the United Nations Convention on Contracts on the International Sales of Goods. +""" + +import pytest +from . import get_bitstream_and_options, run_check +from .. import get_testv_path +from ..constants import * + + +@pytest.mark.parametrize( + "input_format,bitrate,input_sampling_rate,dtx,md_type,output_format,output_sampling_rate,bitstream_processing", + DECODER_CONST_BR_BINAURAL_OBJECTBASED, +) +def test_decoder_objectbased_directivity( + input_format, + bitrate, + input_sampling_rate, + dtx, + md_type, + output_format, + output_sampling_rate, + bitstream_processing, + dut_decoder_frontend, + update_ref, +): + testv_name = get_testv_path(input_format, input_sampling_rate).stem + suffix = "_basic_MD" + if md_type == ISM_MD_EXTENDED: + suffix = "_ext_MD" + elif md_type == ISM_MD_NULL: + suffix = "_null_MD" + with get_bitstream_and_options( + testv_name, + input_format, + bitrate, + input_sampling_rate, + dtx, + suffix=suffix, + processing=bitstream_processing, + render_config=True, + dpid=True, + ) as (ref_bitstream, options): + run_check( + ref_bitstream, + output_format, + output_sampling_rate, + options, + dut_decoder_frontend, + update_ref == 1, + output_suffix="dpid", + bitstream_processing=bitstream_processing, + ) + + +@pytest.mark.parametrize( + "input_format_ism,input_format_other,bitrate,input_sampling_rate,md_type,output_format,output_sampling_rate,bitstream_processing", + DECODER_CONST_BR_BINAURAL_COMBINED_PARAMS, +) +def test_decoder_combined_formats( + input_format_ism, + input_format_other, + bitrate, + input_sampling_rate, + md_type, + output_format, + output_sampling_rate, + bitstream_processing, + dut_decoder_frontend, + update_ref, +): + input_format_combined = "OMASA" if "MASA" in input_format_other else "OSBA" + testv_name = get_testv_path( + f"{input_format_combined}_{input_format_ism}_{input_format_other}", + input_sampling_rate, + ).stem + suffix = "_basic_MD" + if md_type == ISM_MD_EXTENDED: + suffix = "_ext_MD" + elif md_type == ISM_MD_NULL: + suffix = "_null_MD" + with get_bitstream_and_options( + testv_name, + input_format_combined, + bitrate, + input_sampling_rate, + DTX_OFF, + suffix=suffix, + processing=bitstream_processing, + render_config=True, + dpid=True, + ) as (ref_bitstream, options): + run_check( + ref_bitstream, + output_format, + output_sampling_rate, + options, + dut_decoder_frontend, + update_ref == 1, + output_suffix="dpid", + bitstream_processing=bitstream_processing, + ) -- GitLab From 9ac0d797314ca7dd754b18cef7d7601161dbf760 Mon Sep 17 00:00:00 2001 From: kiene Date: Thu, 7 Sep 2023 15:11:08 +0200 Subject: [PATCH 051/106] add render config tests with AEs for ISM --- .../codec_be_to_accepted_release/constants.py | 18 +++++++++++++++--- .../decoder/__init__.py | 12 +++++++++--- .../decoder/test_decoder_render_config.py | 8 +++++--- 3 files changed, 29 insertions(+), 9 deletions(-) diff --git a/tests/codec_be_to_accepted_release/constants.py b/tests/codec_be_to_accepted_release/constants.py index 9b51c1ea78..cc4a8f646a 100644 --- a/tests/codec_be_to_accepted_release/constants.py +++ b/tests/codec_be_to_accepted_release/constants.py @@ -500,9 +500,21 @@ EXOF_OPTIONS = [ ] EXOF_PARAMS = [EXOF_OFF, EXOF_ON] -RENDER_CONFIG_OPTIONS = [ - "-render_config", - str(TESTV_PATH.joinpath("render_config.cfg")), +RENDER_CONFIG_ER_LOW_COMPLEXITY = "RC_low" +RENDER_CONFIG_ER_HIGH_COMPLEXITY = "RC_high" +RENDER_CONFIG_OPTIONS = { + RENDER_CONFIG_ER_LOW_COMPLEXITY: [ + "-render_config", + str(TESTV_PATH.joinpath("render_config1.cfg")), + ], + RENDER_CONFIG_ER_HIGH_COMPLEXITY: [ + "-render_config", + str(TESTV_PATH.joinpath("render_config2.cfg")), + ], +} +RENDER_CONFIG_PARAMS = [ + RENDER_CONFIG_ER_LOW_COMPLEXITY, + RENDER_CONFIG_ER_HIGH_COMPLEXITY, ] DPID_OPTIONS = ["-dpid", "0", "1", "2", "3"] # For testing only diff --git a/tests/codec_be_to_accepted_release/decoder/__init__.py b/tests/codec_be_to_accepted_release/decoder/__init__.py index 54c71b5d9b..af502a3035 100644 --- a/tests/codec_be_to_accepted_release/decoder/__init__.py +++ b/tests/codec_be_to_accepted_release/decoder/__init__.py @@ -110,8 +110,10 @@ def get_bitstream_and_options( suffix="", non_diegetic_pan_value=None, exof=EXOF_OFF, - render_config=False, + render_config=None, dpid=False, + # TODO: keep for later when it is possible to use one unified render config file + # aeid=None, ): """ Utility to get either the stored reference bitstream or the processed version as a temporary file @@ -172,8 +174,9 @@ def get_bitstream_and_options( if exof == EXOF_ON: options.extend(EXOF_OPTIONS) - if render_config: - options.extend(RENDER_CONFIG_OPTIONS) + if render_config is not None: + options.extend(RENDER_CONFIG_OPTIONS[render_config]) + options.extend(["-aeid", "0"]) if dpid: assert render_config @@ -188,6 +191,9 @@ def get_bitstream_and_options( # +1 for the "-dpid" at index 0 options.extend(DPID_OPTIONS[: n_ism + 1]) + # if aeid is not None: + # options.extend(["-aeid", str(aeid)]) + yield bitstream, options diff --git a/tests/codec_be_to_accepted_release/decoder/test_decoder_render_config.py b/tests/codec_be_to_accepted_release/decoder/test_decoder_render_config.py index ba3f45b9ca..d55fb7ec1b 100644 --- a/tests/codec_be_to_accepted_release/decoder/test_decoder_render_config.py +++ b/tests/codec_be_to_accepted_release/decoder/test_decoder_render_config.py @@ -34,11 +34,12 @@ from .. import get_testv_path from ..constants import * +@pytest.mark.parametrize("render_config", RENDER_CONFIG_PARAMS) @pytest.mark.parametrize( "input_format,bitrate,input_sampling_rate,dtx,md_type,output_format,output_sampling_rate,bitstream_processing", DECODER_CONST_BR_BINAURAL_OBJECTBASED, ) -def test_decoder_objectbased_directivity( +def test_decoder_objectbased_render_conf_directivity( input_format, bitrate, input_sampling_rate, @@ -47,6 +48,7 @@ def test_decoder_objectbased_directivity( output_format, output_sampling_rate, bitstream_processing, + render_config, dut_decoder_frontend, update_ref, ): @@ -64,8 +66,8 @@ def test_decoder_objectbased_directivity( dtx, suffix=suffix, processing=bitstream_processing, - render_config=True, dpid=True, + render_config=render_config, ) as (ref_bitstream, options): run_check( ref_bitstream, @@ -74,7 +76,7 @@ def test_decoder_objectbased_directivity( options, dut_decoder_frontend, update_ref == 1, - output_suffix="dpid", + output_suffix=f"dpid-{render_config}", bitstream_processing=bitstream_processing, ) -- GitLab From d627eecb4df92f7f4f24cd202d4eb850f6606a70 Mon Sep 17 00:00:00 2001 From: kiene Date: Thu, 7 Sep 2023 16:00:56 +0200 Subject: [PATCH 052/106] add render_config testcases for combined formats --- .../decoder/test_decoder_render_config.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/tests/codec_be_to_accepted_release/decoder/test_decoder_render_config.py b/tests/codec_be_to_accepted_release/decoder/test_decoder_render_config.py index d55fb7ec1b..1480feafbb 100644 --- a/tests/codec_be_to_accepted_release/decoder/test_decoder_render_config.py +++ b/tests/codec_be_to_accepted_release/decoder/test_decoder_render_config.py @@ -81,11 +81,12 @@ def test_decoder_objectbased_render_conf_directivity( ) +@pytest.mark.parametrize("render_config", RENDER_CONFIG_PARAMS) @pytest.mark.parametrize( "input_format_ism,input_format_other,bitrate,input_sampling_rate,md_type,output_format,output_sampling_rate,bitstream_processing", DECODER_CONST_BR_BINAURAL_COMBINED_PARAMS, ) -def test_decoder_combined_formats( +def test_decoder_combined_formats_render_conf_directivity( input_format_ism, input_format_other, bitrate, @@ -94,6 +95,7 @@ def test_decoder_combined_formats( output_format, output_sampling_rate, bitstream_processing, + render_config, dut_decoder_frontend, update_ref, ): @@ -115,7 +117,7 @@ def test_decoder_combined_formats( DTX_OFF, suffix=suffix, processing=bitstream_processing, - render_config=True, + render_config=render_config, dpid=True, ) as (ref_bitstream, options): run_check( -- GitLab From 19a64fffd3d492c111fabb9f826f1884f0ffe535 Mon Sep 17 00:00:00 2001 From: kiene Date: Thu, 7 Sep 2023 16:28:02 +0200 Subject: [PATCH 053/106] add render_conf testcase for remaining formats --- .../decoder/test_decoder_render_config.py | 100 +++++++++++++++++- 1 file changed, 99 insertions(+), 1 deletion(-) diff --git a/tests/codec_be_to_accepted_release/decoder/test_decoder_render_config.py b/tests/codec_be_to_accepted_release/decoder/test_decoder_render_config.py index 1480feafbb..e6b34698d8 100644 --- a/tests/codec_be_to_accepted_release/decoder/test_decoder_render_config.py +++ b/tests/codec_be_to_accepted_release/decoder/test_decoder_render_config.py @@ -127,6 +127,104 @@ def test_decoder_combined_formats_render_conf_directivity( options, dut_decoder_frontend, update_ref == 1, - output_suffix="dpid", + output_suffix=f"dpid-{render_config}", + bitstream_processing=bitstream_processing, + ) + + +# all non-ISM modes can not use directivity parts from the render config +# we only need the REVERB output config for the AE parts + + +@pytest.mark.parametrize("render_config", RENDER_CONFIG_PARAMS) +@pytest.mark.parametrize( + "input_format,bitrate,input_sampling_rate,dtx,output_format,output_sampling_rate,bitstream_processing", + [ + params + for params in DECODER_CONST_BR_BINAURAL_MC_AND_MASA + if params[4] == "BINAURAL_ROOM_REVERB" + ], +) +def test_decoder_mc_and_masa_render_conf( + input_format, + bitrate, + input_sampling_rate, + dtx, + output_format, + output_sampling_rate, + bitstream_processing, + render_config, + dut_decoder_frontend, + update_ref, +): + testv_name = get_testv_path(input_format, input_sampling_rate).stem + with get_bitstream_and_options( + testv_name, + input_format, + bitrate, + input_sampling_rate, + dtx, + render_config=render_config, + processing=bitstream_processing, + ) as (ref_bitstream, options): + run_check( + ref_bitstream, + output_format, + output_sampling_rate, + options, + dut_decoder_frontend, + update_ref == 1, + output_suffix=f"{render_config}", + bitstream_processing=bitstream_processing, + ) + + +@pytest.mark.parametrize("render_config", RENDER_CONFIG_PARAMS) +@pytest.mark.parametrize( + "input_format,bitrate,input_sampling_rate,dtx,pca,output_format,output_sampling_rate,bitstream_processing", + [ + params + for params in DECODER_CONST_BR_BINAURAL_SCENEBASED + if params[5] == "BINAURAL_ROOM_REVERB" + ], +) +def test_decoder_scenebased_render_conf( + input_format, + bitrate, + input_sampling_rate, + dtx, + pca, + output_format, + output_sampling_rate, + bitstream_processing, + render_config, + dut_decoder_frontend, + update_ref, +): + # TODO: remove once fixed + if input_format == "PlanarFOA" and bitstream_processing == BS_PROC_JBM_DLY_PROF_5: + pytest.skip("PlanarFOA and JBM currently crashes") + testv_name = get_testv_path(input_format, input_sampling_rate).stem + suffix = "" + if pca == SBA_FOA_PCA_ON: + suffix += "-pca" + with get_bitstream_and_options( + testv_name, + input_format, + bitrate, + input_sampling_rate, + dtx, + suffix=suffix, + render_config=render_config, + processing=bitstream_processing, + ) as (ref_bitstream, options): + run_check( + ref_bitstream, + output_format, + output_sampling_rate, + options, + dut_decoder_frontend, + update_ref == 1, + output_suffix=f"{render_config}", bitstream_processing=bitstream_processing, ) -- GitLab From 8503d8cceb89a8eacb97e381acb4c1afca4e5087 Mon Sep 17 00:00:00 2001 From: kiene Date: Thu, 7 Sep 2023 16:36:47 +0200 Subject: [PATCH 054/106] remove temporary DPID_OPTIONS overwrite, is fixed now --- tests/codec_be_to_accepted_release/constants.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/tests/codec_be_to_accepted_release/constants.py b/tests/codec_be_to_accepted_release/constants.py index cc4a8f646a..e9ee304a88 100644 --- a/tests/codec_be_to_accepted_release/constants.py +++ b/tests/codec_be_to_accepted_release/constants.py @@ -517,8 +517,6 @@ RENDER_CONFIG_PARAMS = [ RENDER_CONFIG_ER_HIGH_COMPLEXITY, ] DPID_OPTIONS = ["-dpid", "0", "1", "2", "3"] -# For testing only -DPID_OPTIONS = ["-dpid", "0", "0", "0", "0"] DECODER_CONST_BR_BINAURAL_MC_AND_MASA = collapse_into_list_of_pairs( product( -- GitLab From 2fe19f7a3a654903d531b6caad36112a0baab24c Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Fri, 8 Sep 2023 15:56:58 +0200 Subject: [PATCH 055/106] add option for passing dtx update rate --- tests/conftest.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/tests/conftest.py b/tests/conftest.py index ac1b37121a..6407db75af 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -218,6 +218,7 @@ class EncoderFrontend: max_band: Optional[str] = None, bypass_mode: Optional[int] = None, quiet_mode: Optional[bool] = True, + dtx_update_rate: Optional[int] = None, add_option_list: Optional[list] = None, ) -> None: command = [self._path] @@ -227,7 +228,12 @@ class EncoderFrontend: command.extend(["-sba", sba_order]) if dtx_mode: - command.extend(["-dtx"]) + dtx_options = ["-dtx"] + if dtx_update_rate is not None: + if dtx_update_rate != 0 and not (dtx_update_rate in range(3, 101)): + raise ValueError(f"Invalid dtx update rate of {dtx_update_rate} given") + dtx_options.append(str(dtx_update_rate)) + command.extend(dtx_options) if max_band is not None: command.extend(["-max_band", max_band]) -- GitLab From 0b5515d7b8a3305375dcbb0f3bfa06bdb1c49c28 Mon Sep 17 00:00:00 2001 From: knj Date: Thu, 14 Sep 2023 13:12:21 +0200 Subject: [PATCH 056/106] add -max_band parameter in encoder --- .../codec_be_to_accepted_release/__init__.py | 36 +++++++++++++++-- .../codec_be_to_accepted_release/constants.py | 31 ++++---------- .../encoder/__init__.py | 12 +++++- .../encoder/test_encoder_constant_bitrate.py | 40 ++++++++++++------- 4 files changed, 77 insertions(+), 42 deletions(-) diff --git a/tests/codec_be_to_accepted_release/__init__.py b/tests/codec_be_to_accepted_release/__init__.py index 224f51d340..e01382cefb 100644 --- a/tests/codec_be_to_accepted_release/__init__.py +++ b/tests/codec_be_to_accepted_release/__init__.py @@ -30,8 +30,15 @@ the United Nations Convention on Contracts on the International Sales of Goods. import filecmp import subprocess +from itertools import product from ..testconfig import MD5_REF_DICT -from .constants import TESTVECTOR_FOR_INPUT_FORMAT, TESTV_PATH, REF_PATH +from .constants import ( + TESTVECTOR_FOR_INPUT_FORMAT, + TESTV_PATH, + REF_PATH, + SAMPLING_RATES_ALL, + MAX_BAND_ALL, +) def is_be_to_ref(dut_file): @@ -51,12 +58,35 @@ def is_be_to_ref(dut_file): def get_bitstream_path( - base_path, testv_name, encoder_format, bitrate, sampling_rate, dtx, suffix="" + base_path, + testv_name, + encoder_format, + bitrate, + sampling_rate, + max_band, + dtx, + suffix="", ): - bs_name = f"{testv_name}-{encoder_format}-{bitrate}kbps-{sampling_rate}kHz-{dtx}{suffix}.192" + bs_name = f"{testv_name}-{encoder_format}-{bitrate}kbps-{sampling_rate}kHz-max_band_{max_band}-{dtx}{suffix}.192" return base_path.joinpath(bs_name) def get_testv_path(input_format, sampling_rate): testv = TESTVECTOR_FOR_INPUT_FORMAT[input_format].format(fs=sampling_rate) return TESTV_PATH.joinpath(testv) + + +def get_valid_fs_max_band_pairs(add_narrow_band: bool = False): + """ + Get all sensible sampling rate and max_band value combinations, i.e. pair each sampling rate + with all bw options which indicate the same or lower bandwidth (codec will "crop" if bw + given is higher than sampling rate). + + add_narrow_band: include NB in max_band values, this only applies for stereo_dmx_evs + """ + fs, mb = SAMPLING_RATES_ALL, list(MAX_BAND_ALL) + if add_narrow_band: + mb.insert(0, "NB") + + params = [(r, b) for r, b in product(fs, mb) if fs.index(r) >= mb.index(b)] + return params diff --git a/tests/codec_be_to_accepted_release/constants.py b/tests/codec_be_to_accepted_release/constants.py index e9ee304a88..53774a56c2 100644 --- a/tests/codec_be_to_accepted_release/constants.py +++ b/tests/codec_be_to_accepted_release/constants.py @@ -154,6 +154,7 @@ CMDL_OPTIONS_FOR_INPUT_FORMAT = { "MASA_2TC_2DIR": ["-masa", "2"], "OMASA": ["-ism_masa"], "OSBA": ["-ism_sba"], + "STEREO_DMX_EVS": ["-stereo_dmx_evs"], } ### Test case parameters @@ -205,6 +206,7 @@ BITRATES_ISM4_EXTENDED = [b for b in BITRATES_ISM4 if b > 64000] BITRATES_EVS = [5900, 7200, 8000, 9600] + BITRATES_ALL[:6] + BITRATES_ALL[7:9] SAMPLING_RATES_ALL = [16, 32, 48] +MAX_BAND_ALL = ["WB", "SWB", "FB"] OUTPUT_FORMATS_ALL = [ "MONO", @@ -228,13 +230,9 @@ OUTPUT_FORMATS_NON_BINAURAL = OUTPUT_FORMATS_ALL[:11] ### -------------- encoder parameter lists -------------- -STEREO_PARAMS = list( - product(["STEREO"], BITRATES_STEREO, SAMPLING_RATES_ALL, [DTX_OFF, DTX_ON]) -) -MC_PARAMS = list(product(INPUT_FORMATS_MC, BITRATES_ALL, SAMPLING_RATES_ALL, [DTX_OFF])) -MASA_PARAMS = list( - product(INPUT_FORMATS_MASA, BITRATES_ALL, SAMPLING_RATES_ALL, [DTX_OFF, DTX_ON]) -) +STEREO_PARAMS = list(product(["STEREO"], BITRATES_STEREO, [DTX_OFF, DTX_ON])) +MC_PARAMS = list(product(INPUT_FORMATS_MC, BITRATES_ALL, [DTX_OFF])) +MASA_PARAMS = list(product(INPUT_FORMATS_MASA, BITRATES_ALL, [DTX_OFF, DTX_ON])) ENCODER_CHANNEL_BASED_AND_MASA_PARAMS = STEREO_PARAMS + MC_PARAMS + MASA_PARAMS @@ -244,7 +242,6 @@ ISM1_BASIC_NULL_PARAMS = list( product( ["ISM1"], BITRATES_ISM1, - SAMPLING_RATES_ALL, [DTX_OFF, DTX_ON], [ISM_MD_BASIC, ISM_MD_NULL], ) @@ -253,7 +250,6 @@ ISM2_BASIC_NULL_PARAMS = list( product( ["ISM2"], BITRATES_ISM2, - SAMPLING_RATES_ALL, [DTX_OFF, DTX_ON], [ISM_MD_BASIC, ISM_MD_NULL], ) @@ -262,7 +258,6 @@ ISM3_BASIC_NULL_PARAMS = list( product( ["ISM3"], BITRATES_ISM3, - SAMPLING_RATES_ALL, [DTX_OFF, DTX_ON], [ISM_MD_BASIC, ISM_MD_NULL], ) @@ -271,7 +266,6 @@ ISM4_BASIC_NULL_PARAMS = list( product( ["ISM4"], BITRATES_ISM4, - SAMPLING_RATES_ALL, [DTX_OFF, DTX_ON], [ISM_MD_BASIC, ISM_MD_NULL], ) @@ -280,7 +274,6 @@ ISM1_EXTENDED_PARAMS = list( product( ["ISM1"], BITRATES_ISM1_EXTENDED, - SAMPLING_RATES_ALL, [DTX_OFF, DTX_ON], [ISM_MD_EXTENDED], ) @@ -289,7 +282,6 @@ ISM2_EXTENDED_PARAMS = list( product( ["ISM2"], BITRATES_ISM2_EXTENDED, - SAMPLING_RATES_ALL, [DTX_OFF, DTX_ON], [ISM_MD_EXTENDED], ) @@ -298,7 +290,6 @@ ISM3_EXTENDED_PARAMS = list( product( ["ISM3"], BITRATES_ISM3_EXTENDED, - SAMPLING_RATES_ALL, [DTX_OFF, DTX_ON], [ISM_MD_EXTENDED], ) @@ -307,7 +298,6 @@ ISM4_EXTENDED_PARAMS = list( product( ["ISM4"], BITRATES_ISM4_EXTENDED, - SAMPLING_RATES_ALL, [DTX_OFF, DTX_ON], [ISM_MD_EXTENDED], ) @@ -330,7 +320,6 @@ SBA_LOW_BITRATES_PARAMS = list( product( INPUT_FORMATS_SCENE_BASED, BITRATES_ALL[:7], - SAMPLING_RATES_ALL, [DTX_OFF, DTX_ON], [SBA_FOA_PCA_OFF], ) @@ -341,7 +330,6 @@ SBA_HIGH_BITRATES_PARAMS = list( product( INPUT_FORMATS_SCENE_BASED, BITRATES_ALL[7:], - SAMPLING_RATES_ALL, [DTX_OFF], [SBA_FOA_PCA_OFF], ) @@ -351,7 +339,6 @@ SBA_FOA_PCA_PARAMS = list( product( [f for f in INPUT_FORMATS_SCENE_BASED if "FOA" in f], [256000], - SAMPLING_RATES_ALL, [DTX_OFF], [SBA_FOA_PCA_ON], ) @@ -360,9 +347,9 @@ SBA_PARAMS = SBA_LOW_BITRATES_PARAMS + SBA_HIGH_BITRATES_PARAMS + SBA_FOA_PCA_PA # All EVS bitrates, 5900 VBR always needs -dtx -STEREO_DMX_EVS_PARAMS = list( - product(BITRATES_EVS[1:], SAMPLING_RATES_ALL, [DTX_OFF, DTX_ON]) -) + list(product(BITRATES_EVS[:1], SAMPLING_RATES_ALL, [DTX_ON])) +STEREO_DMX_EVS_PARAMS = list(product(BITRATES_EVS[1:], [DTX_OFF, DTX_ON])) + list( + product(BITRATES_EVS[:1], [DTX_ON]) +) OSBA_PARAMS = list( @@ -372,7 +359,6 @@ OSBA_PARAMS = list( f for f in INPUT_FORMATS_SCENE_BASED if not f.startswith("Planar") ], # TODO: enable planar once clarified BITRATES_ALL, - SAMPLING_RATES_ALL, [ISM_MD_BASIC, ISM_MD_NULL, ISM_MD_EXTENDED], ) ) @@ -381,7 +367,6 @@ OMASA_PARAMS = list( INPUT_FORMATS_OBJECT_BASED, INPUT_FORMATS_MASA, BITRATES_ALL, - SAMPLING_RATES_ALL, [ISM_MD_BASIC, ISM_MD_NULL, ISM_MD_EXTENDED], ) ) diff --git a/tests/codec_be_to_accepted_release/encoder/__init__.py b/tests/codec_be_to_accepted_release/encoder/__init__.py index 409c2d36aa..9bad65088c 100644 --- a/tests/codec_be_to_accepted_release/encoder/__init__.py +++ b/tests/codec_be_to_accepted_release/encoder/__init__.py @@ -106,13 +106,18 @@ def get_md(input_format, md_type=None): return md_files -def get_options(input_format, md_type=None): +def get_options(input_format, max_band=None, md_type=None): options = list(CMDL_OPTIONS_FOR_INPUT_FORMAT[input_format]) + if md_type == ISM_MD_EXTENDED: assert input_format in INPUT_FORMATS_OBJECT_BASED options[1] = f"+{options[1]}" md_options = get_md(input_format, md_type) options.extend(md_options) + + if max_band is not None: + options.extend(["-max_band", max_band]) + return options @@ -128,15 +133,20 @@ def run_check( bitstream_suffix="", ): bs_path = REF_PATH if is_ref_creation else DUT_PATH + + max_band = options[options.index("-max_band") + 1] + bitstream = get_bitstream_path( bs_path, testv.stem, input_format, bitrate, sampling_rate, + max_band, dtx, suffix=bitstream_suffix, ) + print(options) encoder_frontend.run( bitrate, sampling_rate, diff --git a/tests/codec_be_to_accepted_release/encoder/test_encoder_constant_bitrate.py b/tests/codec_be_to_accepted_release/encoder/test_encoder_constant_bitrate.py index 6e8a2348f4..6b1eb6e97e 100644 --- a/tests/codec_be_to_accepted_release/encoder/test_encoder_constant_bitrate.py +++ b/tests/codec_be_to_accepted_release/encoder/test_encoder_constant_bitrate.py @@ -40,26 +40,33 @@ from ..constants import ( SBA_FOA_PCA_ON, STEREO_DMX_EVS_PARAMS, COMBINED_FORMATS_PARAMS, - DUT_PATH, ) from . import get_options, run_check -from .. import get_testv_path, get_bitstream_path +from .. import get_testv_path, get_valid_fs_max_band_pairs + + +def pytest_generate_tests(metafunc): + if "max_band" in metafunc.fixturenames and "sampling_rate" in metafunc.fixturenames: + add_narrow_band = "stereo_dmx_evs" in metafunc.function.__name__ + params = get_valid_fs_max_band_pairs(add_narrow_band) + metafunc.parametrize("sampling_rate,max_band", params) # channel-based modes + MASA @pytest.mark.parametrize( - "input_format,bitrate,sampling_rate,dtx", ENCODER_CHANNEL_BASED_AND_MASA_PARAMS + "input_format,bitrate,dtx", ENCODER_CHANNEL_BASED_AND_MASA_PARAMS ) def test_encoder_const_br_channel_based_and_masa_modes( input_format, bitrate, sampling_rate, + max_band, dtx, dut_encoder_frontend, update_ref, ): testv = get_testv_path(input_format, sampling_rate) - options = get_options(input_format) + options = get_options(input_format, max_band) run_check( input_format, bitrate, @@ -72,18 +79,19 @@ def test_encoder_const_br_channel_based_and_masa_modes( ) -@pytest.mark.parametrize("input_format,bitrate,sampling_rate,dtx,md_type", ISM_PARAMS) +@pytest.mark.parametrize("input_format,bitrate,dtx,md_type", ISM_PARAMS) def test_encoder_const_br_object_based_modes( input_format, bitrate, sampling_rate, + max_band, dtx, md_type, dut_encoder_frontend, update_ref, ): testv = get_testv_path(input_format, sampling_rate) - options = get_options(input_format, md_type) + options = get_options(input_format, max_band, md_type=md_type) bitstream_suffix = "_basic_MD" if md_type == ISM_MD_EXTENDED: bitstream_suffix = "_ext_MD" @@ -102,18 +110,19 @@ def test_encoder_const_br_object_based_modes( ) -@pytest.mark.parametrize("input_format,bitrate,sampling_rate,dtx,pca", SBA_PARAMS) +@pytest.mark.parametrize("input_format,bitrate,dtx,pca", SBA_PARAMS) def test_encoder_const_br_scenebased( input_format, bitrate, sampling_rate, + max_band, dtx, pca, dut_encoder_frontend, update_ref, ): testv = get_testv_path(input_format, sampling_rate) - options = get_options(input_format) + options = get_options(input_format, max_band) bitstream_suffix = "" if pca == SBA_FOA_PCA_ON: options.extend(["-bypass", "2"]) @@ -131,13 +140,13 @@ def test_encoder_const_br_scenebased( ) -@pytest.mark.parametrize("bitrate,sampling_rate,dtx", STEREO_DMX_EVS_PARAMS) +@pytest.mark.parametrize("bitrate,dtx", STEREO_DMX_EVS_PARAMS) def test_encoder_const_br_stereo_dmx_evs( - bitrate, sampling_rate, dtx, dut_encoder_frontend, update_ref + bitrate, sampling_rate, max_band, dtx, dut_encoder_frontend, update_ref ): testv = get_testv_path("STEREO", sampling_rate) - options = ["-stereo_dmx_evs"] - input_format = "StereoDmxEvs" + input_format = "STEREO_DMX_EVS" + options = get_options(input_format, max_band) run_check( input_format, bitrate, @@ -151,7 +160,7 @@ def test_encoder_const_br_stereo_dmx_evs( @pytest.mark.parametrize( - "input_format_ism,input_format_other,bitrate,sampling_rate,md_type", + "input_format_ism,input_format_other,bitrate,md_type", COMBINED_FORMATS_PARAMS, ) def test_encoder_const_br_combined_formats( @@ -159,6 +168,7 @@ def test_encoder_const_br_combined_formats( input_format_other, bitrate, sampling_rate, + max_band, md_type, dut_encoder_frontend, update_ref, @@ -168,9 +178,9 @@ def test_encoder_const_br_combined_formats( f"{input_format_combined}_{input_format_ism}_{input_format_other}", sampling_rate, ) - options_ism = get_options(input_format_ism, md_type) + options_ism = get_options(input_format_ism, md_type=md_type) options_other = get_options(input_format_other) - options_combined = get_options(input_format_combined) + options_combined = get_options(input_format_combined, max_band) # glue together: combined mode option, ism num, SBA order/MASA TCs, ISM MD files, MASA MD files (slice results in empty list for OSBA) options = ( options_combined -- GitLab From 97cde16273688f1c84d214151b83475b8716134c Mon Sep 17 00:00:00 2001 From: kiene Date: Thu, 14 Sep 2023 13:51:38 +0200 Subject: [PATCH 057/106] fixes for max_band parametrization --- .../encoder/__init__.py | 13 ++++++++----- .../encoder/test_encoder_constant_bitrate.py | 7 +++---- 2 files changed, 11 insertions(+), 9 deletions(-) diff --git a/tests/codec_be_to_accepted_release/encoder/__init__.py b/tests/codec_be_to_accepted_release/encoder/__init__.py index 9bad65088c..fcdc539e3a 100644 --- a/tests/codec_be_to_accepted_release/encoder/__init__.py +++ b/tests/codec_be_to_accepted_release/encoder/__init__.py @@ -107,17 +107,20 @@ def get_md(input_format, md_type=None): def get_options(input_format, max_band=None, md_type=None): - options = list(CMDL_OPTIONS_FOR_INPUT_FORMAT[input_format]) + + options = list() + if max_band is not None: + options.extend(["-max_band", max_band]) + + options.extend(list(CMDL_OPTIONS_FOR_INPUT_FORMAT[input_format])) if md_type == ISM_MD_EXTENDED: assert input_format in INPUT_FORMATS_OBJECT_BASED - options[1] = f"+{options[1]}" + idx = options.index("-ism") + 1 + options[idx] = f"+{options[idx]}" md_options = get_md(input_format, md_type) options.extend(md_options) - if max_band is not None: - options.extend(["-max_band", max_band]) - return options diff --git a/tests/codec_be_to_accepted_release/encoder/test_encoder_constant_bitrate.py b/tests/codec_be_to_accepted_release/encoder/test_encoder_constant_bitrate.py index 6b1eb6e97e..697d1f24f1 100644 --- a/tests/codec_be_to_accepted_release/encoder/test_encoder_constant_bitrate.py +++ b/tests/codec_be_to_accepted_release/encoder/test_encoder_constant_bitrate.py @@ -46,10 +46,9 @@ from .. import get_testv_path, get_valid_fs_max_band_pairs def pytest_generate_tests(metafunc): - if "max_band" in metafunc.fixturenames and "sampling_rate" in metafunc.fixturenames: - add_narrow_band = "stereo_dmx_evs" in metafunc.function.__name__ - params = get_valid_fs_max_band_pairs(add_narrow_band) - metafunc.parametrize("sampling_rate,max_band", params) + # NOTE: this excludes narrow-band modes for stereo_dmx_evs as those are not really IVAS but EVS and implementation is complicated... + fs_mb_params = get_valid_fs_max_band_pairs() + metafunc.parametrize("sampling_rate,max_band", fs_mb_params) # channel-based modes + MASA -- GitLab From 3ff956801918e5582c72429805f2e1f6f2daff1f Mon Sep 17 00:00:00 2001 From: kiene Date: Thu, 14 Sep 2023 15:08:56 +0200 Subject: [PATCH 058/106] max_band changes also in decoder --- .../decoder/__init__.py | 10 +++++++- .../test_decoder_constant_bitrate_binaural.py | 24 +++++++++++++++---- ...st_decoder_constant_bitrate_no_binaural.py | 24 +++++++++++++++---- .../decoder/test_decoder_render_config.py | 24 +++++++++++++++---- 4 files changed, 66 insertions(+), 16 deletions(-) diff --git a/tests/codec_be_to_accepted_release/decoder/__init__.py b/tests/codec_be_to_accepted_release/decoder/__init__.py index af502a3035..367a3b787d 100644 --- a/tests/codec_be_to_accepted_release/decoder/__init__.py +++ b/tests/codec_be_to_accepted_release/decoder/__init__.py @@ -105,6 +105,7 @@ def get_bitstream_and_options( sampling_rate, dtx, processing, + max_band=None, hrtf=BINAURAL_HRTF_NONE, otr_mode=None, suffix="", @@ -125,7 +126,14 @@ def get_bitstream_and_options( with TemporaryDirectory() as tmp_dir: # TODO: this should be coming from TESTV_PATH - current setuponly for development bitstream = get_bitstream_path( - REF_PATH, testv_name, encoder_format, bitrate, sampling_rate, dtx, suffix + REF_PATH, + testv_name, + encoder_format, + bitrate, + sampling_rate, + max_band, + dtx, + suffix, ) bitstream_out = Path(tmp_dir).joinpath(bitstream.stem + f".{processing}.192") if processing == BS_PROC_FER_15: diff --git a/tests/codec_be_to_accepted_release/decoder/test_decoder_constant_bitrate_binaural.py b/tests/codec_be_to_accepted_release/decoder/test_decoder_constant_bitrate_binaural.py index ff2cc0d53a..768dac1b37 100644 --- a/tests/codec_be_to_accepted_release/decoder/test_decoder_constant_bitrate_binaural.py +++ b/tests/codec_be_to_accepted_release/decoder/test_decoder_constant_bitrate_binaural.py @@ -30,19 +30,26 @@ the United Nations Convention on Contracts on the International Sales of Goods. import pytest from . import get_bitstream_and_options, run_check -from .. import get_testv_path +from .. import get_testv_path, get_valid_fs_max_band_pairs from ..constants import * +def pytest_generate_tests(metafunc): + # NOTE: this excludes narrow-band modes for stereo_dmx_evs as those are not really IVAS but EVS and implementation is complicated... + fs_mb_params = get_valid_fs_max_band_pairs() + metafunc.parametrize("input_sampling_rate,max_band", fs_mb_params) + + @pytest.mark.parametrize("hrtf", [BINAURAL_HRTF_NONE, BINAURAL_HRTF_EXT_REPO]) @pytest.mark.parametrize( - "input_format,bitrate,input_sampling_rate,dtx,output_format,output_sampling_rate,bitstream_processing", + "input_format,bitrate,dtx,output_format,output_sampling_rate,bitstream_processing", DECODER_CONST_BR_BINAURAL_MC_AND_MASA, ) def test_decoder_mc_and_masa( input_format, bitrate, input_sampling_rate, + max_band, dtx, output_format, output_sampling_rate, @@ -60,6 +67,7 @@ def test_decoder_mc_and_masa( dtx, hrtf=hrtf, processing=bitstream_processing, + max_band=max_band, ) as (ref_bitstream, options): run_check( ref_bitstream, @@ -75,13 +83,14 @@ def test_decoder_mc_and_masa( @pytest.mark.parametrize("hrtf", [BINAURAL_HRTF_NONE, BINAURAL_HRTF_EXT_REPO]) @pytest.mark.parametrize( - "input_format,bitrate,input_sampling_rate,dtx,md_type,output_format,output_sampling_rate,bitstream_processing", + "input_format,bitrate,dtx,md_type,output_format,output_sampling_rate,bitstream_processing", DECODER_CONST_BR_BINAURAL_OBJECTBASED, ) def test_decoder_objectbased( input_format, bitrate, input_sampling_rate, + max_band, dtx, md_type, output_format, @@ -106,6 +115,7 @@ def test_decoder_objectbased( suffix=suffix, hrtf=hrtf, processing=bitstream_processing, + max_band=max_band, ) as (ref_bitstream, options): run_check( ref_bitstream, @@ -121,13 +131,14 @@ def test_decoder_objectbased( @pytest.mark.parametrize("hrtf", [BINAURAL_HRTF_NONE, BINAURAL_HRTF_EXT_REPO]) @pytest.mark.parametrize( - "input_format,bitrate,input_sampling_rate,dtx,pca,output_format,output_sampling_rate,bitstream_processing", + "input_format,bitrate,dtx,pca,output_format,output_sampling_rate,bitstream_processing", DECODER_CONST_BR_BINAURAL_SCENEBASED, ) def test_decoder_scenebased( input_format, bitrate, input_sampling_rate, + max_band, dtx, pca, output_format, @@ -153,6 +164,7 @@ def test_decoder_scenebased( suffix=suffix, hrtf=hrtf, processing=bitstream_processing, + max_band=max_band, ) as (ref_bitstream, options): run_check( ref_bitstream, @@ -168,7 +180,7 @@ def test_decoder_scenebased( @pytest.mark.parametrize("hrtf", [BINAURAL_HRTF_NONE, BINAURAL_HRTF_EXT_REPO]) @pytest.mark.parametrize( - "input_format_ism,input_format_other,bitrate,input_sampling_rate,md_type,output_format,output_sampling_rate,bitstream_processing", + "input_format_ism,input_format_other,bitrate,md_type,output_format,output_sampling_rate,bitstream_processing", DECODER_CONST_BR_BINAURAL_COMBINED_PARAMS, ) def test_decoder_combined_formats( @@ -176,6 +188,7 @@ def test_decoder_combined_formats( input_format_other, bitrate, input_sampling_rate, + max_band, md_type, output_format, output_sampling_rate, @@ -203,6 +216,7 @@ def test_decoder_combined_formats( suffix=suffix, hrtf=hrtf, processing=bitstream_processing, + max_band=max_band, ) as (ref_bitstream, options): run_check( ref_bitstream, diff --git a/tests/codec_be_to_accepted_release/decoder/test_decoder_constant_bitrate_no_binaural.py b/tests/codec_be_to_accepted_release/decoder/test_decoder_constant_bitrate_no_binaural.py index f9b8f05f81..0ac17f72b2 100644 --- a/tests/codec_be_to_accepted_release/decoder/test_decoder_constant_bitrate_no_binaural.py +++ b/tests/codec_be_to_accepted_release/decoder/test_decoder_constant_bitrate_no_binaural.py @@ -30,18 +30,25 @@ the United Nations Convention on Contracts on the International Sales of Goods. import pytest from . import get_bitstream_and_options, run_check -from .. import get_testv_path +from .. import get_testv_path, get_valid_fs_max_band_pairs from ..constants import * +def pytest_generate_tests(metafunc): + # NOTE: this excludes narrow-band modes for stereo_dmx_evs as those are not really IVAS but EVS and implementation is complicated... + fs_mb_params = get_valid_fs_max_band_pairs() + metafunc.parametrize("input_sampling_rate,max_band", fs_mb_params) + + @pytest.mark.parametrize( - "input_format,bitrate,input_sampling_rate,dtx,output_format,output_sampling_rate,bitstream_processing", + "input_format,bitrate,dtx,output_format,output_sampling_rate,bitstream_processing", DECODER_CONST_BR_NO_BINAURAL_CHANNELBASED_AND_MASA, ) def test_decoder_channelbased_and_masa( input_format, bitrate, input_sampling_rate, + max_band, dtx, output_format, output_sampling_rate, @@ -57,6 +64,7 @@ def test_decoder_channelbased_and_masa( input_sampling_rate, dtx, processing=bitstream_processing, + max_band=max_band, ) as (ref_bitstream, options): run_check( ref_bitstream, @@ -70,13 +78,14 @@ def test_decoder_channelbased_and_masa( @pytest.mark.parametrize( - "input_format,bitrate,input_sampling_rate,dtx,md_type,output_format,output_sampling_rate,bitstream_processing", + "input_format,bitrate,dtx,md_type,output_format,output_sampling_rate,bitstream_processing", DECODER_CONST_BR_NO_BINAURAL_OBJECTBASED, ) def test_decoder_objectbased( input_format, bitrate, input_sampling_rate, + max_band, dtx, md_type, output_format, @@ -99,6 +108,7 @@ def test_decoder_objectbased( dtx, suffix=suffix, processing=bitstream_processing, + max_band=max_band, ) as (ref_bitstream, options): run_check( ref_bitstream, @@ -112,13 +122,14 @@ def test_decoder_objectbased( @pytest.mark.parametrize( - "input_format,bitrate,input_sampling_rate,dtx,pca,output_format,output_sampling_rate,bitstream_processing", + "input_format,bitrate,dtx,pca,output_format,output_sampling_rate,bitstream_processing", DECODER_CONST_BR_NO_BINAURAL_SCENEBASED, ) def test_decoder_scenebased( input_format, bitrate, input_sampling_rate, + max_band, dtx, pca, output_format, @@ -142,6 +153,7 @@ def test_decoder_scenebased( dtx, suffix=suffix, processing=bitstream_processing, + max_band=max_band, ) as (ref_bitstream, options): run_check( ref_bitstream, @@ -155,7 +167,7 @@ def test_decoder_scenebased( @pytest.mark.parametrize( - "input_format_ism,input_format_other,bitrate,input_sampling_rate,md_type,output_format,output_sampling_rate,bitstream_processing", + "input_format_ism,input_format_other,bitrate,md_type,output_format,output_sampling_rate,bitstream_processing", DECODER_CONST_BR_NO_BINAURAL_COMBINED_PARAMS, ) def test_decoder_combined_formats( @@ -163,6 +175,7 @@ def test_decoder_combined_formats( input_format_other, bitrate, input_sampling_rate, + max_band, md_type, output_format, output_sampling_rate, @@ -188,6 +201,7 @@ def test_decoder_combined_formats( DTX_OFF, suffix=suffix, processing=bitstream_processing, + max_band=max_band, ) as (ref_bitstream, options): run_check( ref_bitstream, diff --git a/tests/codec_be_to_accepted_release/decoder/test_decoder_render_config.py b/tests/codec_be_to_accepted_release/decoder/test_decoder_render_config.py index e6b34698d8..50e872c28c 100644 --- a/tests/codec_be_to_accepted_release/decoder/test_decoder_render_config.py +++ b/tests/codec_be_to_accepted_release/decoder/test_decoder_render_config.py @@ -30,19 +30,26 @@ the United Nations Convention on Contracts on the International Sales of Goods. import pytest from . import get_bitstream_and_options, run_check -from .. import get_testv_path +from .. import get_testv_path, get_valid_fs_max_band_pairs from ..constants import * +def pytest_generate_tests(metafunc): + # NOTE: this excludes narrow-band modes for stereo_dmx_evs as those are not really IVAS but EVS and implementation is complicated... + fs_mb_params = get_valid_fs_max_band_pairs() + metafunc.parametrize("input_sampling_rate,max_band", fs_mb_params) + + @pytest.mark.parametrize("render_config", RENDER_CONFIG_PARAMS) @pytest.mark.parametrize( - "input_format,bitrate,input_sampling_rate,dtx,md_type,output_format,output_sampling_rate,bitstream_processing", + "input_format,bitrate,dtx,md_type,output_format,output_sampling_rate,bitstream_processing", DECODER_CONST_BR_BINAURAL_OBJECTBASED, ) def test_decoder_objectbased_render_conf_directivity( input_format, bitrate, input_sampling_rate, + max_band, dtx, md_type, output_format, @@ -68,6 +75,7 @@ def test_decoder_objectbased_render_conf_directivity( processing=bitstream_processing, dpid=True, render_config=render_config, + max_band=max_band, ) as (ref_bitstream, options): run_check( ref_bitstream, @@ -83,7 +91,7 @@ def test_decoder_objectbased_render_conf_directivity( @pytest.mark.parametrize("render_config", RENDER_CONFIG_PARAMS) @pytest.mark.parametrize( - "input_format_ism,input_format_other,bitrate,input_sampling_rate,md_type,output_format,output_sampling_rate,bitstream_processing", + "input_format_ism,input_format_other,bitrate,md_type,output_format,output_sampling_rate,bitstream_processing", DECODER_CONST_BR_BINAURAL_COMBINED_PARAMS, ) def test_decoder_combined_formats_render_conf_directivity( @@ -91,6 +99,7 @@ def test_decoder_combined_formats_render_conf_directivity( input_format_other, bitrate, input_sampling_rate, + max_band, md_type, output_format, output_sampling_rate, @@ -119,6 +128,7 @@ def test_decoder_combined_formats_render_conf_directivity( processing=bitstream_processing, render_config=render_config, dpid=True, + max_band=max_band, ) as (ref_bitstream, options): run_check( ref_bitstream, @@ -138,7 +148,7 @@ def test_decoder_combined_formats_render_conf_directivity( @pytest.mark.parametrize("render_config", RENDER_CONFIG_PARAMS) @pytest.mark.parametrize( - "input_format,bitrate,input_sampling_rate,dtx,output_format,output_sampling_rate,bitstream_processing", + "input_format,bitrate,dtx,output_format,output_sampling_rate,bitstream_processing", [ params for params in DECODER_CONST_BR_BINAURAL_MC_AND_MASA @@ -149,6 +159,7 @@ def test_decoder_mc_and_masa_render_conf( input_format, bitrate, input_sampling_rate, + max_band, dtx, output_format, output_sampling_rate, @@ -166,6 +177,7 @@ def test_decoder_mc_and_masa_render_conf( dtx, render_config=render_config, processing=bitstream_processing, + max_band=max_band, ) as (ref_bitstream, options): run_check( ref_bitstream, @@ -181,7 +193,7 @@ def test_decoder_mc_and_masa_render_conf( @pytest.mark.parametrize("render_config", RENDER_CONFIG_PARAMS) @pytest.mark.parametrize( - "input_format,bitrate,input_sampling_rate,dtx,pca,output_format,output_sampling_rate,bitstream_processing", + "input_format,bitrate,dtx,pca,output_format,output_sampling_rate,bitstream_processing", [ params for params in DECODER_CONST_BR_BINAURAL_SCENEBASED @@ -192,6 +204,7 @@ def test_decoder_scenebased_render_conf( input_format, bitrate, input_sampling_rate, + max_band, dtx, pca, output_format, @@ -217,6 +230,7 @@ def test_decoder_scenebased_render_conf( suffix=suffix, render_config=render_config, processing=bitstream_processing, + max_band=max_band, ) as (ref_bitstream, options): run_check( ref_bitstream, -- GitLab From 5bff4cd2152402b910d13ed52cb1ec4ea83406db Mon Sep 17 00:00:00 2001 From: kiene Date: Thu, 14 Sep 2023 15:39:15 +0200 Subject: [PATCH 059/106] use only pytest_generate_tests for encoder parametrization --- .../encoder/test_encoder_constant_bitrate.py | 32 +++++++++++++------ 1 file changed, 22 insertions(+), 10 deletions(-) diff --git a/tests/codec_be_to_accepted_release/encoder/test_encoder_constant_bitrate.py b/tests/codec_be_to_accepted_release/encoder/test_encoder_constant_bitrate.py index 697d1f24f1..6475e8912b 100644 --- a/tests/codec_be_to_accepted_release/encoder/test_encoder_constant_bitrate.py +++ b/tests/codec_be_to_accepted_release/encoder/test_encoder_constant_bitrate.py @@ -45,16 +45,35 @@ from . import get_options, run_check from .. import get_testv_path, get_valid_fs_max_band_pairs +TEST_PARAMS = { + "test_encoder_const_br_channel_based_and_masa_modes": ( + "input_format,bitrate,dtx", + ENCODER_CHANNEL_BASED_AND_MASA_PARAMS, + ), + "test_encoder_const_br_object_based_modes": ( + "input_format,bitrate,dtx,md_type", + ISM_PARAMS, + ), + "test_encoder_const_br_scenebased": ("input_format,bitrate,dtx,pca", SBA_PARAMS), + "test_encoder_const_br_stereo_dmx_evs": ("bitrate,dtx", STEREO_DMX_EVS_PARAMS), + "test_encoder_const_br_combined_formats": ( + "input_format_ism,input_format_other,bitrate,md_type", + COMBINED_FORMATS_PARAMS, + ), +} + + def pytest_generate_tests(metafunc): # NOTE: this excludes narrow-band modes for stereo_dmx_evs as those are not really IVAS but EVS and implementation is complicated... fs_mb_params = get_valid_fs_max_band_pairs() metafunc.parametrize("sampling_rate,max_band", fs_mb_params) + func_name = metafunc.function.__name__ + param_string, params = TEST_PARAMS[func_name] + metafunc.parametrize(param_string, params) + # channel-based modes + MASA -@pytest.mark.parametrize( - "input_format,bitrate,dtx", ENCODER_CHANNEL_BASED_AND_MASA_PARAMS -) def test_encoder_const_br_channel_based_and_masa_modes( input_format, bitrate, @@ -78,7 +97,6 @@ def test_encoder_const_br_channel_based_and_masa_modes( ) -@pytest.mark.parametrize("input_format,bitrate,dtx,md_type", ISM_PARAMS) def test_encoder_const_br_object_based_modes( input_format, bitrate, @@ -109,7 +127,6 @@ def test_encoder_const_br_object_based_modes( ) -@pytest.mark.parametrize("input_format,bitrate,dtx,pca", SBA_PARAMS) def test_encoder_const_br_scenebased( input_format, bitrate, @@ -139,7 +156,6 @@ def test_encoder_const_br_scenebased( ) -@pytest.mark.parametrize("bitrate,dtx", STEREO_DMX_EVS_PARAMS) def test_encoder_const_br_stereo_dmx_evs( bitrate, sampling_rate, max_band, dtx, dut_encoder_frontend, update_ref ): @@ -158,10 +174,6 @@ def test_encoder_const_br_stereo_dmx_evs( ) -@pytest.mark.parametrize( - "input_format_ism,input_format_other,bitrate,md_type", - COMBINED_FORMATS_PARAMS, -) def test_encoder_const_br_combined_formats( input_format_ism, input_format_other, -- GitLab From 8d8e6e0d4f958018b41b1728ac9287db746ed314 Mon Sep 17 00:00:00 2001 From: kiene Date: Thu, 14 Sep 2023 16:49:29 +0200 Subject: [PATCH 060/106] add first experiment with "-reduced" cmdl option random subsampling for now, do it smarter later --- .../encoder/test_encoder_constant_bitrate.py | 28 ++++++++++++++----- tests/conftest.py | 2 ++ 2 files changed, 23 insertions(+), 7 deletions(-) diff --git a/tests/codec_be_to_accepted_release/encoder/test_encoder_constant_bitrate.py b/tests/codec_be_to_accepted_release/encoder/test_encoder_constant_bitrate.py index 6475e8912b..a32925dfc1 100644 --- a/tests/codec_be_to_accepted_release/encoder/test_encoder_constant_bitrate.py +++ b/tests/codec_be_to_accepted_release/encoder/test_encoder_constant_bitrate.py @@ -28,7 +28,8 @@ accordance with the laws of the Federal Republic of Germany excluding its confli the United Nations Convention on Contracts on the International Sales of Goods. """ -import pytest +import itertools +import random from ..constants import ( ISM_MD_NULL, ISM_MD_EXTENDED, @@ -45,7 +46,7 @@ from . import get_options, run_check from .. import get_testv_path, get_valid_fs_max_band_pairs -TEST_PARAMS = { +TEST_PARAMS_EXHAUSTIVE = { "test_encoder_const_br_channel_based_and_masa_modes": ( "input_format,bitrate,dtx", ENCODER_CHANNEL_BASED_AND_MASA_PARAMS, @@ -62,15 +63,28 @@ TEST_PARAMS = { ), } +N_PARAMS = {t: n for t, n in zip(TEST_PARAMS_EXHAUSTIVE.keys(), [40] * 5)} + def pytest_generate_tests(metafunc): - # NOTE: this excludes narrow-band modes for stereo_dmx_evs as those are not really IVAS but EVS and implementation is complicated... - fs_mb_params = get_valid_fs_max_band_pairs() - metafunc.parametrize("sampling_rate,max_band", fs_mb_params) func_name = metafunc.function.__name__ - param_string, params = TEST_PARAMS[func_name] - metafunc.parametrize(param_string, params) + param_string_fs_mb = "sampling_rate,max_band" + fs_mb_params = get_valid_fs_max_band_pairs() + param_string, params = TEST_PARAMS_EXHAUSTIVE[func_name] + + if metafunc.config.getoption("reduced"): + # randomly choose lower number of parameters + random.seed("IVAAAAS") + param_string = ",".join([param_string, param_string_fs_mb]) + combined_params = [p[0] + p[1] for p in itertools.product(params, fs_mb_params)] + metafunc.parametrize( + param_string, random.sample(combined_params, N_PARAMS[func_name]) + ) + else: + # NOTE: this excludes narrow-band modes for stereo_dmx_evs as those are not really IVAS but EVS and implementation is complicated... + metafunc.parametrize(param_string_fs_mb, fs_mb_params) + metafunc.parametrize(param_string, params) # channel-based modes + MASA diff --git a/tests/conftest.py b/tests/conftest.py index 6407db75af..8b7505c837 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -147,6 +147,8 @@ def pytest_addoption(parser): help="Path to file with md5 sums for the reference signals of the selection-BE test" ) + parser.addoption("--reduced", action="store_true", help="Flag for running the allpairs-reduced version of the long BE test") + @pytest.fixture(scope="session", autouse=True) def update_ref(request): -- GitLab From f18d68197272cf10df5f881d02fa806b9506c1d7 Mon Sep 17 00:00:00 2001 From: knj Date: Tue, 19 Sep 2023 16:55:16 +0200 Subject: [PATCH 061/106] add -no_delay_cmp and rewrite get_options() --- .../decoder/__init__.py | 2 +- .../encoder/__init__.py | 44 ++++++++++++++----- .../encoder/test_encoder_constant_bitrate.py | 20 +++------ 3 files changed, 40 insertions(+), 26 deletions(-) diff --git a/tests/codec_be_to_accepted_release/decoder/__init__.py b/tests/codec_be_to_accepted_release/decoder/__init__.py index 367a3b787d..ff1bb8e82e 100644 --- a/tests/codec_be_to_accepted_release/decoder/__init__.py +++ b/tests/codec_be_to_accepted_release/decoder/__init__.py @@ -122,7 +122,7 @@ def get_bitstream_and_options( The reason for implementing this as a context manager instead of a simple function is that this way the temporary processed bitstream files are cleaned up automatically and do not exploce disk space even more... """ - options = list() + options = ["-no_delay_cmp"] with TemporaryDirectory() as tmp_dir: # TODO: this should be coming from TESTV_PATH - current setuponly for development bitstream = get_bitstream_path( diff --git a/tests/codec_be_to_accepted_release/encoder/__init__.py b/tests/codec_be_to_accepted_release/encoder/__init__.py index fcdc539e3a..1cc7ba287c 100644 --- a/tests/codec_be_to_accepted_release/encoder/__init__.py +++ b/tests/codec_be_to_accepted_release/encoder/__init__.py @@ -43,6 +43,8 @@ from ..constants import ( TESTV_PATH, REF_PATH, DTX_ON, + INPUT_FORMATS_SCENE_BASED, + INPUT_FORMATS_MASA, ) @@ -106,20 +108,42 @@ def get_md(input_format, md_type=None): return md_files -def get_options(input_format, max_band=None, md_type=None): +def get_options(input_format_1, input_format_2=None, max_band=None, md_type=None): + + print(input_format_1) + options = list(CMDL_OPTIONS_FOR_INPUT_FORMAT[input_format_1]) + print(options) + + if md_type == ISM_MD_EXTENDED: + assert input_format_1 in INPUT_FORMATS_OBJECT_BASED + options[1] = f"+{options[1]}" + + md_options = get_md(input_format_1, md_type) + + # this block is for the combined formats only + if input_format_2 is not None: + options_other = list(CMDL_OPTIONS_FOR_INPUT_FORMAT.get(input_format_2, list())) + + input_format_combined = None + if input_format_2 in INPUT_FORMATS_SCENE_BASED: + input_format_combined = "OSBA" + elif input_format_2 in INPUT_FORMATS_MASA: + input_format_combined = "OMASA" + + options_combined = list(CMDL_OPTIONS_FOR_INPUT_FORMAT.get(input_format_combined, list())) + + options = options_combined + options[1:2] + options_other[1:2] + md_options + options_other[2:] + + else: + options.extend(md_options) - options = list() if max_band is not None: - options.extend(["-max_band", max_band]) + options.insert(0, "-max_band") + options.insert(1, max_band) - options.extend(list(CMDL_OPTIONS_FOR_INPUT_FORMAT[input_format])) + options.insert(0, "-no_delay_cmp") - if md_type == ISM_MD_EXTENDED: - assert input_format in INPUT_FORMATS_OBJECT_BASED - idx = options.index("-ism") + 1 - options[idx] = f"+{options[idx]}" - md_options = get_md(input_format, md_type) - options.extend(md_options) + print(options) return options diff --git a/tests/codec_be_to_accepted_release/encoder/test_encoder_constant_bitrate.py b/tests/codec_be_to_accepted_release/encoder/test_encoder_constant_bitrate.py index a32925dfc1..97b1c0d679 100644 --- a/tests/codec_be_to_accepted_release/encoder/test_encoder_constant_bitrate.py +++ b/tests/codec_be_to_accepted_release/encoder/test_encoder_constant_bitrate.py @@ -98,7 +98,7 @@ def test_encoder_const_br_channel_based_and_masa_modes( update_ref, ): testv = get_testv_path(input_format, sampling_rate) - options = get_options(input_format, max_band) + options = get_options(input_format, max_band=max_band) run_check( input_format, bitrate, @@ -122,7 +122,7 @@ def test_encoder_const_br_object_based_modes( update_ref, ): testv = get_testv_path(input_format, sampling_rate) - options = get_options(input_format, max_band, md_type=md_type) + options = get_options(input_format, max_band=max_band, md_type=md_type) bitstream_suffix = "_basic_MD" if md_type == ISM_MD_EXTENDED: bitstream_suffix = "_ext_MD" @@ -152,7 +152,7 @@ def test_encoder_const_br_scenebased( update_ref, ): testv = get_testv_path(input_format, sampling_rate) - options = get_options(input_format, max_band) + options = get_options(input_format, max_band=max_band) bitstream_suffix = "" if pca == SBA_FOA_PCA_ON: options.extend(["-bypass", "2"]) @@ -175,7 +175,7 @@ def test_encoder_const_br_stereo_dmx_evs( ): testv = get_testv_path("STEREO", sampling_rate) input_format = "STEREO_DMX_EVS" - options = get_options(input_format, max_band) + options = get_options(input_format, max_band=max_band) run_check( input_format, bitrate, @@ -203,17 +203,7 @@ def test_encoder_const_br_combined_formats( f"{input_format_combined}_{input_format_ism}_{input_format_other}", sampling_rate, ) - options_ism = get_options(input_format_ism, md_type=md_type) - options_other = get_options(input_format_other) - options_combined = get_options(input_format_combined, max_band) - # glue together: combined mode option, ism num, SBA order/MASA TCs, ISM MD files, MASA MD files (slice results in empty list for OSBA) - options = ( - options_combined - + options_ism[1:2] - + options_other[1:2] - + options_ism[2:] - + options_other[2:] - ) + options = get_options(input_format_ism, input_format_2=input_format_other, max_band=max_band, md_type=md_type) bitstream_suffix = "_basic_MD" if md_type == ISM_MD_EXTENDED: bitstream_suffix = "_ext_MD" -- GitLab From 59c14ec6ddbad08f6567db222d6a4a2d45ff50f8 Mon Sep 17 00:00:00 2001 From: knj Date: Wed, 20 Sep 2023 10:28:11 +0200 Subject: [PATCH 062/106] cleanup unused import --- .../encoder/test_encoder_constant_bitrate.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/codec_be_to_accepted_release/encoder/test_encoder_constant_bitrate.py b/tests/codec_be_to_accepted_release/encoder/test_encoder_constant_bitrate.py index 97b1c0d679..25111dfa39 100644 --- a/tests/codec_be_to_accepted_release/encoder/test_encoder_constant_bitrate.py +++ b/tests/codec_be_to_accepted_release/encoder/test_encoder_constant_bitrate.py @@ -34,7 +34,6 @@ from ..constants import ( ISM_MD_NULL, ISM_MD_EXTENDED, ENCODER_CHANNEL_BASED_AND_MASA_PARAMS, - DTX_ON, DTX_OFF, ISM_PARAMS, SBA_PARAMS, -- GitLab From 2ac12e5679714e5da7dd07387652265c775e77b5 Mon Sep 17 00:00:00 2001 From: knj Date: Wed, 20 Sep 2023 11:17:42 +0200 Subject: [PATCH 063/106] fix missing metadata file path in OMASA encoder command line --- tests/codec_be_to_accepted_release/encoder/__init__.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/tests/codec_be_to_accepted_release/encoder/__init__.py b/tests/codec_be_to_accepted_release/encoder/__init__.py index 1cc7ba287c..83137cd7c2 100644 --- a/tests/codec_be_to_accepted_release/encoder/__init__.py +++ b/tests/codec_be_to_accepted_release/encoder/__init__.py @@ -132,7 +132,10 @@ def get_options(input_format_1, input_format_2=None, max_band=None, md_type=None options_combined = list(CMDL_OPTIONS_FOR_INPUT_FORMAT.get(input_format_combined, list())) - options = options_combined + options[1:2] + options_other[1:2] + md_options + options_other[2:] + # add MASA MD files for OMASA modes + md_options += get_md(input_format_2) + + options = options_combined + options[1:2] + options_other[1:2] + md_options else: options.extend(md_options) -- GitLab From 66eb03a62dbee4966bf99a8203a772aacc60ceeb Mon Sep 17 00:00:00 2001 From: Charles Kinuthia Date: Fri, 29 Sep 2023 18:14:45 +0200 Subject: [PATCH 064/106] Revert "set timeout=10s for testing with short test vectors" This reverts commit e0a18b1a60a7cb98276ad24c9508238bc13c20f0. --- tests/codec_be_to_accepted_release/decoder/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/codec_be_to_accepted_release/decoder/__init__.py b/tests/codec_be_to_accepted_release/decoder/__init__.py index 0c52142804..42d119f34b 100644 --- a/tests/codec_be_to_accepted_release/decoder/__init__.py +++ b/tests/codec_be_to_accepted_release/decoder/__init__.py @@ -195,7 +195,7 @@ def run_check( ref_bitstream, output_path, add_option_list=options, - timeout=10, # for testing with short testv, to be reviewed + timeout=1, # for testing, to be reviewed ) if not is_ref_creation and not is_be_to_ref(output_path): -- GitLab From 3a4838c330b15db366fda80d8a0f16ad6872fc09 Mon Sep 17 00:00:00 2001 From: Charles Kinuthia Date: Fri, 29 Sep 2023 18:16:32 +0200 Subject: [PATCH 065/106] Revert "temporarily add 1s timeout to decoder test" This reverts commit 01b3a95476f8392ca4c1327a23d39cf1437c83a0. --- .gitlab-ci.yml | 10 +-- .../decoder/__init__.py | 1 - tests/conftest.py | 61 ++++++++----------- 3 files changed, 29 insertions(+), 43 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 571f1626f3..0696c26520 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1080,18 +1080,18 @@ test-be-to-release: # test generating references - echo "generate references" - # - ("& python -m pytest .\tests\codec_be_to_accepted_release\encoder --update_ref 1 -n $NPROC 2>&1 | tee 'logs/test_encoder-update_ref-log.txt' ") | Invoke-Expression + # - ("& python -m pytest .\tests\codec_be_to_accepted_release\encoder --update_ref 1 -n $NPROC | tee 'logs/test_encoder-update_ref-log.txt' ") | Invoke-Expression # - If($LASTEXITCODE) {("exit $LASTEXITCODE") | Invoke-Expression} - # - ("& python -m pytest .\tests\codec_be_to_accepted_release\decoder\test_decoder_constant_bitrate_no_binaural.py --update_ref 1 -n $NPROC -x 2>&1 | tee 'logs/test_decoder_constant_bitrate_no_binaural-update_ref-log.txt' ") | Invoke-Expression + # - ("& python -m pytest .\tests\codec_be_to_accepted_release\decoder\test_decoder_constant_bitrate_no_binaural.py --update_ref 1 -n $NPROC -x | tee 'logs/test_decoder_constant_bitrate_no_binaural-update_ref-log.txt' ") | Invoke-Expression # temporarily test cases that fail # test_decoder_scenebased - #- ("& python -m pytest .\tests\codec_be_to_accepted_release\decoder\test_decoder_constant_bitrate_no_binaural.py::test_decoder_scenebased --update_ref 1 -n $NPROC 2>&1 | tee 'logs/test_decoder_constant_bitrate_no_binaural_test_decoder_scenebased-update_ref-log.txt' ") | Invoke-Expression - #- If($LASTEXITCODE) {("exit $LASTEXITCODE") | Invoke-Expression} + - ("& python -m pytest .\tests\codec_be_to_accepted_release\decoder\test_decoder_constant_bitrate_no_binaural.py::test_decoder_scenebased --update_ref 1 -n $NPROC | tee 'logs/test_decoder_constant_bitrate_no_binaural_test_decoder_scenebased-update_ref-log.txt' ") | Invoke-Expression + - If($LASTEXITCODE) {("exit $LASTEXITCODE") | Invoke-Expression} # test_decoder_combined_formats - - ("& python -m pytest .\tests\codec_be_to_accepted_release\decoder\test_decoder_constant_bitrate_no_binaural.py::test_decoder_combined_formats --update_ref 1 -n $NPROC 2>&1 | tee 'logs/test_decoder_constant_bitrate_no_binaural_test_decoder_combined_formats-update_ref-log.txt' ") | Invoke-Expression + - ("& python -m pytest .\tests\codec_be_to_accepted_release\decoder\test_decoder_constant_bitrate_no_binaural.py::test_decoder_combined_formats --update_ref 1 -n $NPROC | tee 'logs/test_decoder_constant_bitrate_no_binaural_test_decoder_combined_formats-update_ref-log.txt' ") | Invoke-Expression - If($LASTEXITCODE) {("exit $LASTEXITCODE") | Invoke-Expression} artifacts: diff --git a/tests/codec_be_to_accepted_release/decoder/__init__.py b/tests/codec_be_to_accepted_release/decoder/__init__.py index 42d119f34b..c6d04aee85 100644 --- a/tests/codec_be_to_accepted_release/decoder/__init__.py +++ b/tests/codec_be_to_accepted_release/decoder/__init__.py @@ -195,7 +195,6 @@ def run_check( ref_bitstream, output_path, add_option_list=options, - timeout=1, # for testing, to be reviewed ) if not is_ref_creation and not is_be_to_ref(output_path): diff --git a/tests/conftest.py b/tests/conftest.py index 2ce7b1833d..ac1b37121a 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -38,7 +38,6 @@ import logging from pathlib import Path import platform from subprocess import run -from subprocess import TimeoutExpired import textwrap from typing import Optional import os @@ -220,7 +219,6 @@ class EncoderFrontend: bypass_mode: Optional[int] = None, quiet_mode: Optional[bool] = True, add_option_list: Optional[list] = None, - timeout: Optional[int] = None, ) -> None: command = [self._path] @@ -254,23 +252,18 @@ class EncoderFrontend: cmd_str = textwrap.indent(" ".join(command), prefix="\t") log_dbg_msg(f"{self._type} encoder command:\n{cmd_str}") - try: - result = run(command, capture_output=True, check=False, timeout=timeout) - self.returncode = result.returncode - self.stderr = result.stderr.decode("ascii") - self.stdout = result.stdout.decode("ascii") - if self.stdout: - stdout_str = textwrap.indent(self.stdout, prefix="\t") - log_dbg_msg(f"{self._type} encoder stdout:\n{stdout_str}") - if self.stderr: - stderr_str = textwrap.indent(self.stderr, prefix="\t") - log_dbg_msg(f"{self._type} encoder stderr:\n{stderr_str}") - if self.returncode: - pytest.fail(f"{self._type} encoder terminated with a non-0 return code: {self.returncode}") - except TimeoutExpired as err: - log_dbg_msg(f"{self._type} encoder timed stdout:\n{err.stdout}") - log_dbg_msg(f"{self._type} encoder timed stderr:\n{err.stderr}") - pytest.fail(f"{self._type} encoder timed out") + result = run(command, capture_output=True, check=False) + self.returncode = result.returncode + self.stderr = result.stderr.decode("ascii") + self.stdout = result.stdout.decode("ascii") + if self.stdout: + stdout_str = textwrap.indent(self.stdout, prefix="\t") + log_dbg_msg(f"{self._type} encoder stdout:\n{stdout_str}") + if self.stderr: + stderr_str = textwrap.indent(self.stderr, prefix="\t") + log_dbg_msg(f"{self._type} encoder stderr:\n{stderr_str}") + if self.returncode: + pytest.fail(f"{self._type} encoder terminated with a non-0 return code: {self.returncode}") def _check_run(self): if self.returncode is not None: @@ -367,7 +360,6 @@ class DecoderFrontend: quiet_mode: Optional[bool] = True, plc_file: Optional[Path] = None, add_option_list: Optional[list] = None, - timeout: Optional[int] = None, ) -> None: command = [self._path] @@ -394,23 +386,18 @@ class DecoderFrontend: cmd_str = textwrap.indent(" ".join(command), prefix="\t") log_dbg_msg(f"{self._type} decoder command:\n{cmd_str}") - try: - result = run(command, capture_output=True, check=False, timeout=timeout) - self.returncode = result.returncode - self.stderr = result.stderr.decode("ascii") - self.stdout = result.stdout.decode("ascii") - if self.stdout: - stdout_str = textwrap.indent(self.stdout, prefix="\t") - log_dbg_msg(f"{self._type} decoder stdout:\n{stdout_str}") - if self.stderr: - stderr_str = textwrap.indent(self.stderr, prefix="\t") - log_dbg_msg(f"{self._type} decoder stderr:\n{stderr_str}") - if self.returncode: - pytest.fail(f"{self._type} decoder terminated with a non-0 return code: {self.returncode}") - except TimeoutExpired as err: - log_dbg_msg(f"{self._type} decoder timed stdout:\n{err.stdout}") - log_dbg_msg(f"{self._type} decoder timed stderr:\n{err.stderr}") - pytest.fail(f"{self._type} decoder timed out") + result = run(command, capture_output=True, check=False) + self.returncode = result.returncode + self.stderr = result.stderr.decode("ascii") + self.stdout = result.stdout.decode("ascii") + if self.stdout: + stdout_str = textwrap.indent(self.stdout, prefix="\t") + log_dbg_msg(f"{self._type} decoder stdout:\n{stdout_str}") + if self.stderr: + stderr_str = textwrap.indent(self.stderr, prefix="\t") + log_dbg_msg(f"{self._type} decoder stderr:\n{stderr_str}") + if self.returncode: + pytest.fail(f"{self._type} decoder terminated with a non-0 return code: {self.returncode}") def _check_run(self): if self.returncode is not None: -- GitLab From 3685822bbf0fe8ac09efc9f2533809990adcfbf7 Mon Sep 17 00:00:00 2001 From: Charles Kinuthia Date: Fri, 29 Sep 2023 18:43:20 +0200 Subject: [PATCH 066/106] test ref generation enc and dec - enc const_br only - dec const_br including binaural modes --- .gitlab-ci.yml | 20 ++++++++------------ 1 file changed, 8 insertions(+), 12 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 55658e1bd5..7e38b0782a 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1132,7 +1132,7 @@ test-be-to-release: tags: - test-be-to-release-windows resource_group: ivas-be-to-release-test-resource - timeout: "5 hours" # to be reviewed + timeout: "22 hours" # to be reviewed rules: - if: $CI_PIPELINE_SOURCE == 'web' && $MANUAL_PIPELINE_TYPE == 'test-be-release' script: @@ -1149,19 +1149,12 @@ test-be-to-release: # test generating references - echo "generate references" - # - ("& python -m pytest .\tests\codec_be_to_accepted_release\encoder --update_ref 1 -n $NPROC | tee 'logs/test_encoder-update_ref-log.txt' ") | Invoke-Expression + - ("& python -m pytest .\tests\codec_be_to_accepted_release\encoder --update_ref 1 --html=logs/report-enc-const_br.html --self-contained-html --junit-xml=logs/report-junit-enc-const_br.xml -n $NPROC ") | Invoke-Expression # - If($LASTEXITCODE) {("exit $LASTEXITCODE") | Invoke-Expression} - # - ("& python -m pytest .\tests\codec_be_to_accepted_release\decoder\test_decoder_constant_bitrate_no_binaural.py --update_ref 1 -n $NPROC -x | tee 'logs/test_decoder_constant_bitrate_no_binaural-update_ref-log.txt' ") | Invoke-Expression + - ("& python -m pytest .\tests\codec_be_to_accepted_release\decoder\test_decoder_constant_bitrate_no_binaural.py --update_ref 1 -n $NPROC --html=report-dec-const_br-no_binaural.html --self-contained-html --junit-xml=logs/report-junit-dec-const_br-no_binaural.xml ") | Invoke-Expression - # temporarily test cases that fail - - # test_decoder_scenebased - - ("& python -m pytest .\tests\codec_be_to_accepted_release\decoder\test_decoder_constant_bitrate_no_binaural.py::test_decoder_scenebased --update_ref 1 -n $NPROC | tee 'logs/test_decoder_constant_bitrate_no_binaural_test_decoder_scenebased-update_ref-log.txt' ") | Invoke-Expression - - If($LASTEXITCODE) {("exit $LASTEXITCODE") | Invoke-Expression} - - # test_decoder_combined_formats - - ("& python -m pytest .\tests\codec_be_to_accepted_release\decoder\test_decoder_constant_bitrate_no_binaural.py::test_decoder_combined_formats --update_ref 1 -n $NPROC | tee 'logs/test_decoder_constant_bitrate_no_binaural_test_decoder_combined_formats-update_ref-log.txt' ") | Invoke-Expression - - If($LASTEXITCODE) {("exit $LASTEXITCODE") | Invoke-Expression} + # decoder binaural + - ("& python -m pytest .\tests\codec_be_to_accepted_release\decoder\test_decoder_constant_bitrate_binaural.py --update_ref 1 -n $NPROC --html=report-dec-const_br-binaural.html --self-contained-html --junit-xml=logs/report-junit-dec-const_br-binaural.xml ") | Invoke-Expression artifacts: name: "test-be-to-release--sha-$CI_COMMIT_SHORT_SHA--results" @@ -1170,6 +1163,9 @@ test-be-to-release: paths: - logs/ expose_as: "test-be-to-release results" + reports: + junit: + - logs/report-junit*.xml test-long-self-test: -- GitLab From 3c5de335731c4db698add6e64660ffc07b216950 Mon Sep 17 00:00:00 2001 From: Charles Kinuthia Date: Fri, 29 Sep 2023 18:52:12 +0200 Subject: [PATCH 067/106] exclude EXT for combined formats --- tests/codec_be_to_accepted_release/constants.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/codec_be_to_accepted_release/constants.py b/tests/codec_be_to_accepted_release/constants.py index 53774a56c2..946ef64dab 100644 --- a/tests/codec_be_to_accepted_release/constants.py +++ b/tests/codec_be_to_accepted_release/constants.py @@ -426,7 +426,7 @@ DECODER_CONST_BR_NO_BINAURAL_SCENEBASED = collapse_into_list_of_pairs( DECODER_CONST_BR_NO_BINAURAL_COMBINED_PARAMS = collapse_into_list_of_pairs( product( COMBINED_FORMATS_PARAMS, - OUTPUT_FORMATS_NON_BINAURAL, + OUTPUT_FORMATS_NON_BINAURAL[:10], # one less to exclude EXT SAMPLING_RATES_ALL, BITSTREAM_PROCESSING[:2], # TODO: re-add JBM once fully functional ) -- GitLab From 0db764532cb2d01ac0359e3f75977ab3ae1d7a7e Mon Sep 17 00:00:00 2001 From: Charles Kinuthia Date: Fri, 29 Sep 2023 19:07:16 +0200 Subject: [PATCH 068/106] specify --testcase_timeout 40 --- .gitlab-ci.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 7e38b0782a..494579c5a8 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1149,12 +1149,12 @@ test-be-to-release: # test generating references - echo "generate references" - - ("& python -m pytest .\tests\codec_be_to_accepted_release\encoder --update_ref 1 --html=logs/report-enc-const_br.html --self-contained-html --junit-xml=logs/report-junit-enc-const_br.xml -n $NPROC ") | Invoke-Expression + - ("& python -m pytest .\tests\codec_be_to_accepted_release\encoder --update_ref 1 --html=logs/report-enc-const_br.html --self-contained-html --junit-xml=logs/report-junit-enc-const_br.xml --testcase_timeout 40") | Invoke-Expression # - If($LASTEXITCODE) {("exit $LASTEXITCODE") | Invoke-Expression} - - ("& python -m pytest .\tests\codec_be_to_accepted_release\decoder\test_decoder_constant_bitrate_no_binaural.py --update_ref 1 -n $NPROC --html=report-dec-const_br-no_binaural.html --self-contained-html --junit-xml=logs/report-junit-dec-const_br-no_binaural.xml ") | Invoke-Expression + - ("& python -m pytest .\tests\codec_be_to_accepted_release\decoder\test_decoder_constant_bitrate_no_binaural.py --update_ref 1 --html=report-dec-const_br-no_binaural.html --self-contained-html --junit-xml=logs/report-junit-dec-const_br-no_binaural.xml --testcase_timeout 40") | Invoke-Expression # decoder binaural - - ("& python -m pytest .\tests\codec_be_to_accepted_release\decoder\test_decoder_constant_bitrate_binaural.py --update_ref 1 -n $NPROC --html=report-dec-const_br-binaural.html --self-contained-html --junit-xml=logs/report-junit-dec-const_br-binaural.xml ") | Invoke-Expression + - ("& python -m pytest .\tests\codec_be_to_accepted_release\decoder\test_decoder_constant_bitrate_binaural.py --update_ref 1 --html=report-dec-const_br-binaural.html --self-contained-html --junit-xml=logs/report-junit-dec-const_br-binaural.xml --testcase_timeout 40") | Invoke-Expression artifacts: name: "test-be-to-release--sha-$CI_COMMIT_SHORT_SHA--results" -- GitLab From 311d2ebf7346efb352480dd6c9b4853f24c1a48c Mon Sep 17 00:00:00 2001 From: Charles Kinuthia Date: Wed, 4 Oct 2023 13:50:51 +0200 Subject: [PATCH 069/106] attempt fix to unsupported test cases involving hrtf binaries --- .../decoder/test_decoder_constant_bitrate_binaural.py | 8 ++++---- .../decoder/test_decoder_constant_bitrate_no_binaural.py | 8 ++++---- .../decoder/test_decoder_head_rotation.py | 8 ++++---- .../decoder/test_decoder_non_diegetic_pan.py | 4 ++-- .../decoder/test_decoder_render_config.py | 8 ++++---- 5 files changed, 18 insertions(+), 18 deletions(-) diff --git a/tests/codec_be_to_accepted_release/decoder/test_decoder_constant_bitrate_binaural.py b/tests/codec_be_to_accepted_release/decoder/test_decoder_constant_bitrate_binaural.py index 768dac1b37..91df6ea0f6 100644 --- a/tests/codec_be_to_accepted_release/decoder/test_decoder_constant_bitrate_binaural.py +++ b/tests/codec_be_to_accepted_release/decoder/test_decoder_constant_bitrate_binaural.py @@ -72,7 +72,7 @@ def test_decoder_mc_and_masa( run_check( ref_bitstream, output_format, - output_sampling_rate, + input_sampling_rate, options, dut_decoder_frontend, update_ref == 1, @@ -120,7 +120,7 @@ def test_decoder_objectbased( run_check( ref_bitstream, output_format, - output_sampling_rate, + input_sampling_rate, options, dut_decoder_frontend, update_ref == 1, @@ -169,7 +169,7 @@ def test_decoder_scenebased( run_check( ref_bitstream, output_format, - output_sampling_rate, + input_sampling_rate, options, dut_decoder_frontend, update_ref == 1, @@ -221,7 +221,7 @@ def test_decoder_combined_formats( run_check( ref_bitstream, output_format, - output_sampling_rate, + input_sampling_rate, options, dut_decoder_frontend, update_ref == 1, diff --git a/tests/codec_be_to_accepted_release/decoder/test_decoder_constant_bitrate_no_binaural.py b/tests/codec_be_to_accepted_release/decoder/test_decoder_constant_bitrate_no_binaural.py index 0ac17f72b2..86e411a095 100644 --- a/tests/codec_be_to_accepted_release/decoder/test_decoder_constant_bitrate_no_binaural.py +++ b/tests/codec_be_to_accepted_release/decoder/test_decoder_constant_bitrate_no_binaural.py @@ -69,7 +69,7 @@ def test_decoder_channelbased_and_masa( run_check( ref_bitstream, output_format, - output_sampling_rate, + input_sampling_rate, options, dut_decoder_frontend, update_ref == 1, @@ -113,7 +113,7 @@ def test_decoder_objectbased( run_check( ref_bitstream, output_format, - output_sampling_rate, + input_sampling_rate, options, dut_decoder_frontend, update_ref == 1, @@ -158,7 +158,7 @@ def test_decoder_scenebased( run_check( ref_bitstream, output_format, - output_sampling_rate, + input_sampling_rate, options, dut_decoder_frontend, update_ref == 1, @@ -206,7 +206,7 @@ def test_decoder_combined_formats( run_check( ref_bitstream, output_format, - output_sampling_rate, + input_sampling_rate, options, dut_decoder_frontend, update_ref == 1, diff --git a/tests/codec_be_to_accepted_release/decoder/test_decoder_head_rotation.py b/tests/codec_be_to_accepted_release/decoder/test_decoder_head_rotation.py index 0a53842c5d..4587460bbf 100644 --- a/tests/codec_be_to_accepted_release/decoder/test_decoder_head_rotation.py +++ b/tests/codec_be_to_accepted_release/decoder/test_decoder_head_rotation.py @@ -67,7 +67,7 @@ def test_decoder_hr_mc_and_masa( run_check( ref_bitstream, output_format, - output_sampling_rate, + input_sampling_rate, options, dut_decoder_frontend, update_ref == 1, @@ -118,7 +118,7 @@ def test_decoder_hr_objectbased( run_check( ref_bitstream, output_format, - output_sampling_rate, + input_sampling_rate, options, dut_decoder_frontend, update_ref == 1, @@ -165,7 +165,7 @@ def test_decoder_hr_scenebased( run_check( ref_bitstream, output_format, - output_sampling_rate, + input_sampling_rate, options, dut_decoder_frontend, update_ref == 1, @@ -218,7 +218,7 @@ def test_decoder_hr_combined_formats( run_check( ref_bitstream, output_format, - output_sampling_rate, + input_sampling_rate, options, dut_decoder_frontend, update_ref == 1, diff --git a/tests/codec_be_to_accepted_release/decoder/test_decoder_non_diegetic_pan.py b/tests/codec_be_to_accepted_release/decoder/test_decoder_non_diegetic_pan.py index 39880aa7f1..d5a9f6f0b6 100644 --- a/tests/codec_be_to_accepted_release/decoder/test_decoder_non_diegetic_pan.py +++ b/tests/codec_be_to_accepted_release/decoder/test_decoder_non_diegetic_pan.py @@ -89,7 +89,7 @@ def test_decoder_non_diegetic_pan_ism( run_check( ref_bitstream, output_format, - output_sampling_rate, + input_sampling_rate, options, dut_decoder_frontend, update_ref == 1, @@ -127,7 +127,7 @@ def test_decoder_non_diegetic_pan_mono( run_check( ref_bitstream, "", - output_sampling_rate, + input_sampling_rate, options, dut_decoder_frontend, update_ref == 1, diff --git a/tests/codec_be_to_accepted_release/decoder/test_decoder_render_config.py b/tests/codec_be_to_accepted_release/decoder/test_decoder_render_config.py index 50e872c28c..f21b4cf4f9 100644 --- a/tests/codec_be_to_accepted_release/decoder/test_decoder_render_config.py +++ b/tests/codec_be_to_accepted_release/decoder/test_decoder_render_config.py @@ -80,7 +80,7 @@ def test_decoder_objectbased_render_conf_directivity( run_check( ref_bitstream, output_format, - output_sampling_rate, + input_sampling_rate, options, dut_decoder_frontend, update_ref == 1, @@ -133,7 +133,7 @@ def test_decoder_combined_formats_render_conf_directivity( run_check( ref_bitstream, output_format, - output_sampling_rate, + input_sampling_rate, options, dut_decoder_frontend, update_ref == 1, @@ -182,7 +182,7 @@ def test_decoder_mc_and_masa_render_conf( run_check( ref_bitstream, output_format, - output_sampling_rate, + input_sampling_rate, options, dut_decoder_frontend, update_ref == 1, @@ -235,7 +235,7 @@ def test_decoder_scenebased_render_conf( run_check( ref_bitstream, output_format, - output_sampling_rate, + input_sampling_rate, options, dut_decoder_frontend, update_ref == 1, -- GitLab From 4aa3d339b647c3ad950e5641ba5e2798ec47d35f Mon Sep 17 00:00:00 2001 From: Charles Kinuthia Date: Wed, 4 Oct 2023 14:51:09 +0200 Subject: [PATCH 070/106] test ref generation for enc & dec binaural modes --- .gitlab-ci.yml | 19 +++++++++++++++---- 1 file changed, 15 insertions(+), 4 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 494579c5a8..61b8d725a3 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1137,6 +1137,7 @@ test-be-to-release: - if: $CI_PIPELINE_SOURCE == 'web' && $MANUAL_PIPELINE_TYPE == 'test-be-release' script: + - $winoutdata = $null - echo "$CI_COMMIT_BRANCH" - MSBuild.exe .\Workspace_msvc\Workspace_msvc.sln /property:Configuration=Debug /p:Platform=win32 /m | tee -variable winoutdata - $winoutdata | Out-File $BUILD_OUTPUT -Encoding Utf8 @@ -1146,15 +1147,25 @@ test-be-to-release: # path to release candidate refs defined in config.toml - mkdir logs + - mkdir tests\codec_be_to_accepted_release\ref + - mkdir tests\codec_be_to_accepted_release\dut + # test generating references - echo "generate references" - - ("& python -m pytest .\tests\codec_be_to_accepted_release\encoder --update_ref 1 --html=logs/report-enc-const_br.html --self-contained-html --junit-xml=logs/report-junit-enc-const_br.xml --testcase_timeout 40") | Invoke-Expression - # - If($LASTEXITCODE) {("exit $LASTEXITCODE") | Invoke-Expression} - - ("& python -m pytest .\tests\codec_be_to_accepted_release\decoder\test_decoder_constant_bitrate_no_binaural.py --update_ref 1 --html=report-dec-const_br-no_binaural.html --self-contained-html --junit-xml=logs/report-junit-dec-const_br-no_binaural.xml --testcase_timeout 40") | Invoke-Expression + - $zero_errors_enc_const_br=0 + - ("& python -m pytest .\tests\codec_be_to_accepted_release\encoder\test_encoder_constant_bitrate.py --update_ref 1 --html=logs/report-enc-const_br.html --self-contained-html --junit-xml=logs/report-junit-enc-const_br.xml --testcase_timeout 40" | tee out-null) | Invoke-Expression + - IF (!(Select-String logs/report-junit-enc-const_br.xml -Pattern 'errors="0"')){$zero_errors_enc_const_br=1; echo "Errors encountered"} + + # - ("& python -m pytest .\tests\codec_be_to_accepted_release\decoder\test_decoder_constant_bitrate_no_binaural.py --update_ref 1 --html=report-dec-const_br-no_binaural.html --self-contained-html --junit-xml=logs/report-junit-dec-const_br-no_binaural.xml --testcase_timeout 40 | tee dec-const_br-non_binaural.txt") | Invoke-Expression # decoder binaural - - ("& python -m pytest .\tests\codec_be_to_accepted_release\decoder\test_decoder_constant_bitrate_binaural.py --update_ref 1 --html=report-dec-const_br-binaural.html --self-contained-html --junit-xml=logs/report-junit-dec-const_br-binaural.xml --testcase_timeout 40") | Invoke-Expression + - $zero_errors_dec_const_br_binaural=0 + - ("& python -m pytest .\tests\codec_be_to_accepted_release\decoder\test_decoder_constant_bitrate_binaural.py --update_ref 1 --html=report-dec-const_br-binaural.html --self-contained-html --junit-xml=logs/report-junit-dec-const_br-binaural.xml --testcase_timeout 40" | tee dec-const_br-binaural.txt ) | Invoke-Expression + - IF (!(Select-String logs/report-junit-dec-const_br-binaural.xml -Pattern 'errors="0"')){$zero_errors_dec_const_br_binaural=1; echo "Errors encountered"} + + - IF($zero_errors_enc_const_br -nq 0 or $zero_errors_dec_const_br_binaural -nq 0 ) {echo "Errors encountered"; exit $EXIT_CODE_FAIL} + - exit 0 artifacts: name: "test-be-to-release--sha-$CI_COMMIT_SHORT_SHA--results" -- GitLab From dc5e7f4f1b606b50d6af415203011b66a030664b Mon Sep 17 00:00:00 2001 From: Charles Kinuthia Date: Wed, 4 Oct 2023 14:57:36 +0200 Subject: [PATCH 071/106] syntax fix comparison opperators --- .gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 61b8d725a3..e77dc1d9ac 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1164,7 +1164,7 @@ test-be-to-release: - ("& python -m pytest .\tests\codec_be_to_accepted_release\decoder\test_decoder_constant_bitrate_binaural.py --update_ref 1 --html=report-dec-const_br-binaural.html --self-contained-html --junit-xml=logs/report-junit-dec-const_br-binaural.xml --testcase_timeout 40" | tee dec-const_br-binaural.txt ) | Invoke-Expression - IF (!(Select-String logs/report-junit-dec-const_br-binaural.xml -Pattern 'errors="0"')){$zero_errors_dec_const_br_binaural=1; echo "Errors encountered"} - - IF($zero_errors_enc_const_br -nq 0 or $zero_errors_dec_const_br_binaural -nq 0 ) {echo "Errors encountered"; exit $EXIT_CODE_FAIL} + - IF($zero_errors_enc_const_br -ne 0 -or $zero_errors_dec_const_br_binaural -ne 0 ) {echo "Errors encountered"; exit $EXIT_CODE_FAIL} - exit 0 artifacts: -- GitLab From 81e1a01557a1d6617bc93482340bc4c92ceab924 Mon Sep 17 00:00:00 2001 From: knj Date: Wed, 4 Oct 2023 16:07:27 +0200 Subject: [PATCH 072/106] fix incorrect HRTF fs selection --- tests/codec_be_to_accepted_release/decoder/__init__.py | 8 +++++--- .../decoder/test_decoder_constant_bitrate_binaural.py | 4 ++++ 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/tests/codec_be_to_accepted_release/decoder/__init__.py b/tests/codec_be_to_accepted_release/decoder/__init__.py index ff1bb8e82e..c567cf6d85 100644 --- a/tests/codec_be_to_accepted_release/decoder/__init__.py +++ b/tests/codec_be_to_accepted_release/decoder/__init__.py @@ -102,7 +102,7 @@ def get_bitstream_and_options( testv_name, encoder_format, bitrate, - sampling_rate, + input_sampling_rate, dtx, processing, max_band=None, @@ -113,6 +113,7 @@ def get_bitstream_and_options( exof=EXOF_OFF, render_config=None, dpid=False, + output_sampling_rate=None # TODO: keep for later when it is possible to use one unified render config file # aeid=None, ): @@ -130,7 +131,7 @@ def get_bitstream_and_options( testv_name, encoder_format, bitrate, - sampling_rate, + input_sampling_rate, max_band, dtx, suffix, @@ -170,7 +171,8 @@ def get_bitstream_and_options( options.append("-VOIP") if hrtf != BINAURAL_HRTF_NONE: - options.extend(["-hrtf", str(HRTF_PATHS[hrtf]).format(fs=sampling_rate)]) + assert output_sampling_rate is not None + options.extend(["-hrtf", str(HRTF_PATHS[hrtf]).format(fs=output_sampling_rate)]) if non_diegetic_pan_value is not None: options.extend(["-non_diegetic_pan", f"{non_diegetic_pan_value}"]) diff --git a/tests/codec_be_to_accepted_release/decoder/test_decoder_constant_bitrate_binaural.py b/tests/codec_be_to_accepted_release/decoder/test_decoder_constant_bitrate_binaural.py index 768dac1b37..9bbebb8065 100644 --- a/tests/codec_be_to_accepted_release/decoder/test_decoder_constant_bitrate_binaural.py +++ b/tests/codec_be_to_accepted_release/decoder/test_decoder_constant_bitrate_binaural.py @@ -66,6 +66,7 @@ def test_decoder_mc_and_masa( input_sampling_rate, dtx, hrtf=hrtf, + output_sampling_rate=output_sampling_rate, processing=bitstream_processing, max_band=max_band, ) as (ref_bitstream, options): @@ -114,6 +115,7 @@ def test_decoder_objectbased( dtx, suffix=suffix, hrtf=hrtf, + output_sampling_rate=output_sampling_rate, processing=bitstream_processing, max_band=max_band, ) as (ref_bitstream, options): @@ -163,6 +165,7 @@ def test_decoder_scenebased( dtx, suffix=suffix, hrtf=hrtf, + output_sampling_rate=output_sampling_rate, processing=bitstream_processing, max_band=max_band, ) as (ref_bitstream, options): @@ -215,6 +218,7 @@ def test_decoder_combined_formats( DTX_OFF, suffix=suffix, hrtf=hrtf, + output_sampling_rate=output_sampling_rate, processing=bitstream_processing, max_band=max_band, ) as (ref_bitstream, options): -- GitLab From 75b60e46279ad34ad7f7ae4a0b9174cab7dc2a2f Mon Sep 17 00:00:00 2001 From: Charles Kinuthia Date: Wed, 4 Oct 2023 17:12:24 +0200 Subject: [PATCH 073/106] [CI] try suppressing pytest output for encoder ref generation --- .gitlab-ci.yml | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 115e801e15..5db495baf5 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1154,18 +1154,17 @@ test-be-to-release: - echo "generate references" - $zero_errors_enc_const_br=0 - - ("& python -m pytest .\tests\codec_be_to_accepted_release\encoder\test_encoder_constant_bitrate.py --update_ref 1 --html=logs/report-enc-const_br.html --self-contained-html --junit-xml=logs/report-junit-enc-const_br.xml --testcase_timeout 40" | tee out-null) | Invoke-Expression + - ("& python -m pytest .\tests\codec_be_to_accepted_release\encoder\test_encoder_constant_bitrate.py --update_ref 1 --html=logs/report-enc-const_br.html --self-contained-html --junit-xml=logs/report-junit-enc-const_br.xml --testcase_timeout 40 | out-null") | Invoke-Expression - IF (!(Select-String logs/report-junit-enc-const_br.xml -Pattern 'errors="0"')){$zero_errors_enc_const_br=1; echo "Errors encountered"} # - ("& python -m pytest .\tests\codec_be_to_accepted_release\decoder\test_decoder_constant_bitrate_no_binaural.py --update_ref 1 --html=report-dec-const_br-no_binaural.html --self-contained-html --junit-xml=logs/report-junit-dec-const_br-no_binaural.xml --testcase_timeout 40 | tee dec-const_br-non_binaural.txt") | Invoke-Expression # decoder binaural - $zero_errors_dec_const_br_binaural=0 - - ("& python -m pytest .\tests\codec_be_to_accepted_release\decoder\test_decoder_constant_bitrate_binaural.py --update_ref 1 --html=report-dec-const_br-binaural.html --self-contained-html --junit-xml=logs/report-junit-dec-const_br-binaural.xml --testcase_timeout 40" | tee dec-const_br-binaural.txt ) | Invoke-Expression + - ("& python -m pytest .\tests\codec_be_to_accepted_release\decoder\test_decoder_constant_bitrate_binaural.py --update_ref 1 --html=report-dec-const_br-binaural.html --self-contained-html --junit-xml=logs/report-junit-dec-const_br-binaural.xml --testcase_timeout 40 | tee dec-const_br-binaural.txt") | Invoke-Expression - IF (!(Select-String logs/report-junit-dec-const_br-binaural.xml -Pattern 'errors="0"')){$zero_errors_dec_const_br_binaural=1; echo "Errors encountered"} - IF($zero_errors_enc_const_br -ne 0 -or $zero_errors_dec_const_br_binaural -ne 0 ) {echo "Errors encountered"; exit $EXIT_CODE_FAIL} - - exit 0 artifacts: name: "test-be-to-release--sha-$CI_COMMIT_SHORT_SHA--results" -- GitLab From 11cf77cf7f24ba3232e51f22c6321ecea9c6b43e Mon Sep 17 00:00:00 2001 From: Charles Kinuthia Date: Wed, 4 Oct 2023 17:28:40 +0200 Subject: [PATCH 074/106] try limiting number of workers - $NPROC specified in config.toml --- .gitlab-ci.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 5db495baf5..b0a462d8f8 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1154,14 +1154,14 @@ test-be-to-release: - echo "generate references" - $zero_errors_enc_const_br=0 - - ("& python -m pytest .\tests\codec_be_to_accepted_release\encoder\test_encoder_constant_bitrate.py --update_ref 1 --html=logs/report-enc-const_br.html --self-contained-html --junit-xml=logs/report-junit-enc-const_br.xml --testcase_timeout 40 | out-null") | Invoke-Expression + - ("& python -m pytest .\tests\codec_be_to_accepted_release\encoder\test_encoder_constant_bitrate.py --update_ref 1 --html=logs/report-enc-const_br.html --self-contained-html --junit-xml=logs/report-junit-enc-const_br.xml --testcase_timeout 40 -n $NPROC | out-null") | Invoke-Expression - IF (!(Select-String logs/report-junit-enc-const_br.xml -Pattern 'errors="0"')){$zero_errors_enc_const_br=1; echo "Errors encountered"} - # - ("& python -m pytest .\tests\codec_be_to_accepted_release\decoder\test_decoder_constant_bitrate_no_binaural.py --update_ref 1 --html=report-dec-const_br-no_binaural.html --self-contained-html --junit-xml=logs/report-junit-dec-const_br-no_binaural.xml --testcase_timeout 40 | tee dec-const_br-non_binaural.txt") | Invoke-Expression + # - ("& python -m pytest .\tests\codec_be_to_accepted_release\decoder\test_decoder_constant_bitrate_no_binaural.py --update_ref 1 --html=report-dec-const_br-no_binaural.html --self-contained-html --junit-xml=logs/report-junit-dec-const_br-no_binaural.xml --testcase_timeout 40 -n $NPROC | tee dec-const_br-non_binaural.txt") | Invoke-Expression # decoder binaural - $zero_errors_dec_const_br_binaural=0 - - ("& python -m pytest .\tests\codec_be_to_accepted_release\decoder\test_decoder_constant_bitrate_binaural.py --update_ref 1 --html=report-dec-const_br-binaural.html --self-contained-html --junit-xml=logs/report-junit-dec-const_br-binaural.xml --testcase_timeout 40 | tee dec-const_br-binaural.txt") | Invoke-Expression + - ("& python -m pytest .\tests\codec_be_to_accepted_release\decoder\test_decoder_constant_bitrate_binaural.py --update_ref 1 --html=report-dec-const_br-binaural.html --self-contained-html --junit-xml=logs/report-junit-dec-const_br-binaural.xml --testcase_timeout 40 -n $NPROC | tee dec-const_br-binaural.txt") | Invoke-Expression - IF (!(Select-String logs/report-junit-dec-const_br-binaural.xml -Pattern 'errors="0"')){$zero_errors_dec_const_br_binaural=1; echo "Errors encountered"} - IF($zero_errors_enc_const_br -ne 0 -or $zero_errors_dec_const_br_binaural -ne 0 ) {echo "Errors encountered"; exit $EXIT_CODE_FAIL} -- GitLab From 657747d7cded1a6d242352062f410728fb4715e4 Mon Sep 17 00:00:00 2001 From: Charles Kinuthia Date: Thu, 5 Oct 2023 15:22:31 +0200 Subject: [PATCH 075/106] Revert "attempt fix to unsupported test cases involving hrtf binaries" This reverts commit 311d2ebf7346efb352480dd6c9b4853f24c1a48c. --- .../decoder/test_decoder_constant_bitrate_binaural.py | 8 ++++---- .../decoder/test_decoder_constant_bitrate_no_binaural.py | 8 ++++---- .../decoder/test_decoder_head_rotation.py | 8 ++++---- .../decoder/test_decoder_non_diegetic_pan.py | 4 ++-- .../decoder/test_decoder_render_config.py | 8 ++++---- 5 files changed, 18 insertions(+), 18 deletions(-) diff --git a/tests/codec_be_to_accepted_release/decoder/test_decoder_constant_bitrate_binaural.py b/tests/codec_be_to_accepted_release/decoder/test_decoder_constant_bitrate_binaural.py index 91df6ea0f6..768dac1b37 100644 --- a/tests/codec_be_to_accepted_release/decoder/test_decoder_constant_bitrate_binaural.py +++ b/tests/codec_be_to_accepted_release/decoder/test_decoder_constant_bitrate_binaural.py @@ -72,7 +72,7 @@ def test_decoder_mc_and_masa( run_check( ref_bitstream, output_format, - input_sampling_rate, + output_sampling_rate, options, dut_decoder_frontend, update_ref == 1, @@ -120,7 +120,7 @@ def test_decoder_objectbased( run_check( ref_bitstream, output_format, - input_sampling_rate, + output_sampling_rate, options, dut_decoder_frontend, update_ref == 1, @@ -169,7 +169,7 @@ def test_decoder_scenebased( run_check( ref_bitstream, output_format, - input_sampling_rate, + output_sampling_rate, options, dut_decoder_frontend, update_ref == 1, @@ -221,7 +221,7 @@ def test_decoder_combined_formats( run_check( ref_bitstream, output_format, - input_sampling_rate, + output_sampling_rate, options, dut_decoder_frontend, update_ref == 1, diff --git a/tests/codec_be_to_accepted_release/decoder/test_decoder_constant_bitrate_no_binaural.py b/tests/codec_be_to_accepted_release/decoder/test_decoder_constant_bitrate_no_binaural.py index 86e411a095..0ac17f72b2 100644 --- a/tests/codec_be_to_accepted_release/decoder/test_decoder_constant_bitrate_no_binaural.py +++ b/tests/codec_be_to_accepted_release/decoder/test_decoder_constant_bitrate_no_binaural.py @@ -69,7 +69,7 @@ def test_decoder_channelbased_and_masa( run_check( ref_bitstream, output_format, - input_sampling_rate, + output_sampling_rate, options, dut_decoder_frontend, update_ref == 1, @@ -113,7 +113,7 @@ def test_decoder_objectbased( run_check( ref_bitstream, output_format, - input_sampling_rate, + output_sampling_rate, options, dut_decoder_frontend, update_ref == 1, @@ -158,7 +158,7 @@ def test_decoder_scenebased( run_check( ref_bitstream, output_format, - input_sampling_rate, + output_sampling_rate, options, dut_decoder_frontend, update_ref == 1, @@ -206,7 +206,7 @@ def test_decoder_combined_formats( run_check( ref_bitstream, output_format, - input_sampling_rate, + output_sampling_rate, options, dut_decoder_frontend, update_ref == 1, diff --git a/tests/codec_be_to_accepted_release/decoder/test_decoder_head_rotation.py b/tests/codec_be_to_accepted_release/decoder/test_decoder_head_rotation.py index 4587460bbf..0a53842c5d 100644 --- a/tests/codec_be_to_accepted_release/decoder/test_decoder_head_rotation.py +++ b/tests/codec_be_to_accepted_release/decoder/test_decoder_head_rotation.py @@ -67,7 +67,7 @@ def test_decoder_hr_mc_and_masa( run_check( ref_bitstream, output_format, - input_sampling_rate, + output_sampling_rate, options, dut_decoder_frontend, update_ref == 1, @@ -118,7 +118,7 @@ def test_decoder_hr_objectbased( run_check( ref_bitstream, output_format, - input_sampling_rate, + output_sampling_rate, options, dut_decoder_frontend, update_ref == 1, @@ -165,7 +165,7 @@ def test_decoder_hr_scenebased( run_check( ref_bitstream, output_format, - input_sampling_rate, + output_sampling_rate, options, dut_decoder_frontend, update_ref == 1, @@ -218,7 +218,7 @@ def test_decoder_hr_combined_formats( run_check( ref_bitstream, output_format, - input_sampling_rate, + output_sampling_rate, options, dut_decoder_frontend, update_ref == 1, diff --git a/tests/codec_be_to_accepted_release/decoder/test_decoder_non_diegetic_pan.py b/tests/codec_be_to_accepted_release/decoder/test_decoder_non_diegetic_pan.py index d5a9f6f0b6..39880aa7f1 100644 --- a/tests/codec_be_to_accepted_release/decoder/test_decoder_non_diegetic_pan.py +++ b/tests/codec_be_to_accepted_release/decoder/test_decoder_non_diegetic_pan.py @@ -89,7 +89,7 @@ def test_decoder_non_diegetic_pan_ism( run_check( ref_bitstream, output_format, - input_sampling_rate, + output_sampling_rate, options, dut_decoder_frontend, update_ref == 1, @@ -127,7 +127,7 @@ def test_decoder_non_diegetic_pan_mono( run_check( ref_bitstream, "", - input_sampling_rate, + output_sampling_rate, options, dut_decoder_frontend, update_ref == 1, diff --git a/tests/codec_be_to_accepted_release/decoder/test_decoder_render_config.py b/tests/codec_be_to_accepted_release/decoder/test_decoder_render_config.py index f21b4cf4f9..50e872c28c 100644 --- a/tests/codec_be_to_accepted_release/decoder/test_decoder_render_config.py +++ b/tests/codec_be_to_accepted_release/decoder/test_decoder_render_config.py @@ -80,7 +80,7 @@ def test_decoder_objectbased_render_conf_directivity( run_check( ref_bitstream, output_format, - input_sampling_rate, + output_sampling_rate, options, dut_decoder_frontend, update_ref == 1, @@ -133,7 +133,7 @@ def test_decoder_combined_formats_render_conf_directivity( run_check( ref_bitstream, output_format, - input_sampling_rate, + output_sampling_rate, options, dut_decoder_frontend, update_ref == 1, @@ -182,7 +182,7 @@ def test_decoder_mc_and_masa_render_conf( run_check( ref_bitstream, output_format, - input_sampling_rate, + output_sampling_rate, options, dut_decoder_frontend, update_ref == 1, @@ -235,7 +235,7 @@ def test_decoder_scenebased_render_conf( run_check( ref_bitstream, output_format, - input_sampling_rate, + output_sampling_rate, options, dut_decoder_frontend, update_ref == 1, -- GitLab From 1cad243db2f153a5a0055afd782fbdab634d3e19 Mon Sep 17 00:00:00 2001 From: knj Date: Thu, 5 Oct 2023 16:01:17 +0200 Subject: [PATCH 076/106] add pytest_generate_tests hook to non diegetic testcase file --- .../decoder/test_decoder_non_diegetic_pan.py | 18 +++++++++++++++--- 1 file changed, 15 insertions(+), 3 deletions(-) diff --git a/tests/codec_be_to_accepted_release/decoder/test_decoder_non_diegetic_pan.py b/tests/codec_be_to_accepted_release/decoder/test_decoder_non_diegetic_pan.py index 39880aa7f1..2d0b91e275 100644 --- a/tests/codec_be_to_accepted_release/decoder/test_decoder_non_diegetic_pan.py +++ b/tests/codec_be_to_accepted_release/decoder/test_decoder_non_diegetic_pan.py @@ -30,7 +30,7 @@ the United Nations Convention on Contracts on the International Sales of Goods. import pytest from . import get_bitstream_and_options, run_check -from .. import get_testv_path +from .. import get_testv_path, get_valid_fs_max_band_pairs from ..constants import ( STEREO_DMX_EVS_PARAMS, ISM_MD_EXTENDED, @@ -38,8 +38,16 @@ from ..constants import ( DECODER_CONST_BR_NO_BINAURAL_OBJECTBASED, SAMPLING_RATES_ALL, BITSTREAM_PROCESSING, + SAMPLING_RATES_ALL, ) + +def pytest_generate_tests(metafunc): + # NOTE: this excludes narrow-band modes for stereo_dmx_evs as those are not really IVAS but EVS and implementation is complicated... + fs_mb_params = get_valid_fs_max_band_pairs() + metafunc.parametrize("input_sampling_rate,max_band", fs_mb_params) + + # NOTE: this way, all possible inputs for the non-diegetic panning value are used for all input/output configs # less exhaustive testing is probably enough here, one can e.g. only test one panning value for each i/o config NON_DIEGETIC_PAN_VALUES = ["c", "center", "middle", "left", "l", "right", "r"] + list( @@ -54,13 +62,14 @@ ISM1_DECODER_PARAMS = [ @pytest.mark.parametrize("non_diegetic_panning_value", NON_DIEGETIC_PAN_VALUES) @pytest.mark.parametrize( - "input_format,bitrate,input_sampling_rate,dtx,md_type,output_format,output_sampling_rate,bitstream_processing", + "input_format,bitrate,dtx,md_type,output_format,output_sampling_rate,bitstream_processing", ISM1_DECODER_PARAMS, ) def test_decoder_non_diegetic_pan_ism( input_format, bitrate, input_sampling_rate, + max_band, dtx, md_type, output_format, @@ -84,6 +93,7 @@ def test_decoder_non_diegetic_pan_ism( dtx, bitstream_processing, suffix=suffix, + max_band=max_band, ) as (ref_bitstream, options): suffix += f"-non_diegetic_pan_{non_diegetic_panning_value}" run_check( @@ -100,12 +110,13 @@ def test_decoder_non_diegetic_pan_ism( # for the mono input, we use the Stereo DMX EVS outputs from the decoder so no explicit mono encoder run is needed @pytest.mark.parametrize("non_diegetic_panning_value", NON_DIEGETIC_PAN_VALUES) -@pytest.mark.parametrize("bitrate,input_sampling_rate,dtx", STEREO_DMX_EVS_PARAMS) +@pytest.mark.parametrize("bitrate,dtx", STEREO_DMX_EVS_PARAMS) @pytest.mark.parametrize("output_sampling_rate", SAMPLING_RATES_ALL) @pytest.mark.parametrize("bitstream_processing", BITSTREAM_PROCESSING) def test_decoder_non_diegetic_pan_mono( bitrate, input_sampling_rate, + max_band, dtx, output_sampling_rate, bitstream_processing, @@ -122,6 +133,7 @@ def test_decoder_non_diegetic_pan_mono( dtx, bitstream_processing, non_diegetic_pan_value=non_diegetic_panning_value, + max_band=max_band, ) as (ref_bitstream, options): suffix = f"-non_diegetic_pan_{non_diegetic_panning_value}" run_check( -- GitLab From ff59658b8054b77cfc3c4e9898e45b5063bece52 Mon Sep 17 00:00:00 2001 From: knj Date: Thu, 5 Oct 2023 16:12:03 +0200 Subject: [PATCH 077/106] same for head rotation testcases --- .../decoder/test_decoder_head_rotation.py | 24 +++++++++++++++---- 1 file changed, 19 insertions(+), 5 deletions(-) diff --git a/tests/codec_be_to_accepted_release/decoder/test_decoder_head_rotation.py b/tests/codec_be_to_accepted_release/decoder/test_decoder_head_rotation.py index 0a53842c5d..0ce4f363a6 100644 --- a/tests/codec_be_to_accepted_release/decoder/test_decoder_head_rotation.py +++ b/tests/codec_be_to_accepted_release/decoder/test_decoder_head_rotation.py @@ -30,20 +30,27 @@ the United Nations Convention on Contracts on the International Sales of Goods. import pytest from . import get_bitstream_and_options, run_check -from .. import get_testv_path +from .. import get_testv_path, get_valid_fs_max_band_pairs from ..constants import * +def pytest_generate_tests(metafunc): + # NOTE: this excludes narrow-band modes for stereo_dmx_evs as those are not really IVAS but EVS and implementation is complicated... + fs_mb_params = get_valid_fs_max_band_pairs() + metafunc.parametrize("input_sampling_rate,max_band", fs_mb_params) + + @pytest.mark.parametrize("exof", EXOF_PARAMS) @pytest.mark.parametrize("otr_mode", OTR_PARAMS) @pytest.mark.parametrize( - "input_format,bitrate,input_sampling_rate,dtx,output_format,output_sampling_rate,bitstream_processing", + "input_format,bitrate,dtx,output_format,output_sampling_rate,bitstream_processing", DECODER_CONST_BR_BINAURAL_MC_AND_MASA, ) def test_decoder_hr_mc_and_masa( input_format, bitrate, input_sampling_rate, + max_band, dtx, output_format, output_sampling_rate, @@ -63,6 +70,7 @@ def test_decoder_hr_mc_and_masa( processing=bitstream_processing, otr_mode=otr_mode, exof=exof, + max_band=max_band, ) as (ref_bitstream, options): run_check( ref_bitstream, @@ -81,13 +89,14 @@ def test_decoder_hr_mc_and_masa( @pytest.mark.parametrize("exof", EXOF_PARAMS) @pytest.mark.parametrize("otr_mode", OTR_PARAMS) @pytest.mark.parametrize( - "input_format,bitrate,input_sampling_rate,dtx,md_type,output_format,output_sampling_rate,bitstream_processing", + "input_format,bitrate,dtx,md_type,output_format,output_sampling_rate,bitstream_processing", DECODER_CONST_BR_BINAURAL_OBJECTBASED, ) def test_decoder_hr_objectbased( input_format, bitrate, input_sampling_rate, + max_band, dtx, md_type, output_format, @@ -114,6 +123,7 @@ def test_decoder_hr_objectbased( processing=bitstream_processing, otr_mode=otr_mode, exof=exof, + max_band=max_band, ) as (ref_bitstream, options): run_check( ref_bitstream, @@ -130,13 +140,14 @@ def test_decoder_hr_objectbased( @pytest.mark.parametrize("exof", EXOF_PARAMS) @pytest.mark.parametrize("otr_mode", OTR_PARAMS) @pytest.mark.parametrize( - "input_format,bitrate,input_sampling_rate,dtx,pca,output_format,output_sampling_rate,bitstream_processing", + "input_format,bitrate,dtx,pca,output_format,output_sampling_rate,bitstream_processing", DECODER_CONST_BR_BINAURAL_SCENEBASED, ) def test_decoder_hr_scenebased( input_format, bitrate, input_sampling_rate, + max_band, dtx, pca, output_format, @@ -161,6 +172,7 @@ def test_decoder_hr_scenebased( processing=bitstream_processing, otr_mode=otr_mode, exof=exof, + max_band=max_band, ) as (ref_bitstream, options): run_check( ref_bitstream, @@ -177,7 +189,7 @@ def test_decoder_hr_scenebased( @pytest.mark.parametrize("exof", EXOF_PARAMS) @pytest.mark.parametrize("otr_mode", OTR_PARAMS) @pytest.mark.parametrize( - "input_format_ism,input_format_other,bitrate,input_sampling_rate,md_type,output_format,output_sampling_rate,bitstream_processing", + "input_format_ism,input_format_other,bitrate,md_type,output_format,output_sampling_rate,bitstream_processing", DECODER_CONST_BR_BINAURAL_COMBINED_PARAMS, ) def test_decoder_hr_combined_formats( @@ -185,6 +197,7 @@ def test_decoder_hr_combined_formats( input_format_other, bitrate, input_sampling_rate, + max_band, md_type, output_format, output_sampling_rate, @@ -214,6 +227,7 @@ def test_decoder_hr_combined_formats( processing=bitstream_processing, otr_mode=otr_mode, exof=exof, + max_band=max_band, ) as (ref_bitstream, options): run_check( ref_bitstream, -- GitLab From cea012f443b50084363890dc3473f7df8c52657a Mon Sep 17 00:00:00 2001 From: knj Date: Thu, 5 Oct 2023 16:15:51 +0200 Subject: [PATCH 078/106] no EXT for OSBA --- tests/codec_be_to_accepted_release/constants.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/tests/codec_be_to_accepted_release/constants.py b/tests/codec_be_to_accepted_release/constants.py index 53774a56c2..edb0e03e14 100644 --- a/tests/codec_be_to_accepted_release/constants.py +++ b/tests/codec_be_to_accepted_release/constants.py @@ -425,7 +425,13 @@ DECODER_CONST_BR_NO_BINAURAL_SCENEBASED = collapse_into_list_of_pairs( ) DECODER_CONST_BR_NO_BINAURAL_COMBINED_PARAMS = collapse_into_list_of_pairs( product( - COMBINED_FORMATS_PARAMS, + OSBA_PARAMS, + OUTPUT_FORMATS_NON_BINAURAL[:-1], # no EXT here + SAMPLING_RATES_ALL, + BITSTREAM_PROCESSING[:2], # TODO: re-add JBM once fully functional + ) + + product( + OMASA_PARAMS, OUTPUT_FORMATS_NON_BINAURAL, SAMPLING_RATES_ALL, BITSTREAM_PROCESSING[:2], # TODO: re-add JBM once fully functional -- GitLab From fa703876884f5d0fb73c483cbac77b01696664ae Mon Sep 17 00:00:00 2001 From: knj Date: Thu, 5 Oct 2023 16:16:17 +0200 Subject: [PATCH 079/106] JBM back for combined formats --- tests/codec_be_to_accepted_release/constants.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/codec_be_to_accepted_release/constants.py b/tests/codec_be_to_accepted_release/constants.py index edb0e03e14..74d535e614 100644 --- a/tests/codec_be_to_accepted_release/constants.py +++ b/tests/codec_be_to_accepted_release/constants.py @@ -428,13 +428,13 @@ DECODER_CONST_BR_NO_BINAURAL_COMBINED_PARAMS = collapse_into_list_of_pairs( OSBA_PARAMS, OUTPUT_FORMATS_NON_BINAURAL[:-1], # no EXT here SAMPLING_RATES_ALL, - BITSTREAM_PROCESSING[:2], # TODO: re-add JBM once fully functional + BITSTREAM_PROCESSING, ) + product( OMASA_PARAMS, OUTPUT_FORMATS_NON_BINAURAL, SAMPLING_RATES_ALL, - BITSTREAM_PROCESSING[:2], # TODO: re-add JBM once fully functional + BITSTREAM_PROCESSING, ) ) -- GitLab From 222824e3f6b8a13b2519ce1f7ff4c030a856c3c2 Mon Sep 17 00:00:00 2001 From: knj Date: Thu, 5 Oct 2023 16:16:37 +0200 Subject: [PATCH 080/106] run formatter --- tests/codec_be_to_accepted_release/constants.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/codec_be_to_accepted_release/constants.py b/tests/codec_be_to_accepted_release/constants.py index 74d535e614..d59003d8c4 100644 --- a/tests/codec_be_to_accepted_release/constants.py +++ b/tests/codec_be_to_accepted_release/constants.py @@ -426,11 +426,11 @@ DECODER_CONST_BR_NO_BINAURAL_SCENEBASED = collapse_into_list_of_pairs( DECODER_CONST_BR_NO_BINAURAL_COMBINED_PARAMS = collapse_into_list_of_pairs( product( OSBA_PARAMS, - OUTPUT_FORMATS_NON_BINAURAL[:-1], # no EXT here + OUTPUT_FORMATS_NON_BINAURAL[:-1], # no EXT here SAMPLING_RATES_ALL, BITSTREAM_PROCESSING, - ) + - product( + ) + + product( OMASA_PARAMS, OUTPUT_FORMATS_NON_BINAURAL, SAMPLING_RATES_ALL, -- GitLab From 9c217e50f0af12ba8c95f37a2f0bb30f7b9d4e0b Mon Sep 17 00:00:00 2001 From: Charles Kinuthia Date: Thu, 5 Oct 2023 17:50:57 +0200 Subject: [PATCH 081/106] Revert "exclude EXT for combined formats" This reverts commit 3c5de335731c4db698add6e64660ffc07b216950. --- tests/codec_be_to_accepted_release/constants.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/codec_be_to_accepted_release/constants.py b/tests/codec_be_to_accepted_release/constants.py index 946ef64dab..53774a56c2 100644 --- a/tests/codec_be_to_accepted_release/constants.py +++ b/tests/codec_be_to_accepted_release/constants.py @@ -426,7 +426,7 @@ DECODER_CONST_BR_NO_BINAURAL_SCENEBASED = collapse_into_list_of_pairs( DECODER_CONST_BR_NO_BINAURAL_COMBINED_PARAMS = collapse_into_list_of_pairs( product( COMBINED_FORMATS_PARAMS, - OUTPUT_FORMATS_NON_BINAURAL[:10], # one less to exclude EXT + OUTPUT_FORMATS_NON_BINAURAL, SAMPLING_RATES_ALL, BITSTREAM_PROCESSING[:2], # TODO: re-add JBM once fully functional ) -- GitLab From 8d9ed3bb180978e307c84448fa5305de7ee1cdf4 Mon Sep 17 00:00:00 2001 From: Charles Kinuthia Date: Thu, 5 Oct 2023 17:59:26 +0200 Subject: [PATCH 082/106] test ref generation: enc, dec subset - enc::constant brate - dec::{binaural,no_binaural} --- .gitlab-ci.yml | 21 +++++++++++++++++---- 1 file changed, 17 insertions(+), 4 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index b0a462d8f8..14103d994d 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1153,18 +1153,31 @@ test-be-to-release: # test generating references - echo "generate references" + # encoder, constant brate - $zero_errors_enc_const_br=0 - ("& python -m pytest .\tests\codec_be_to_accepted_release\encoder\test_encoder_constant_bitrate.py --update_ref 1 --html=logs/report-enc-const_br.html --self-contained-html --junit-xml=logs/report-junit-enc-const_br.xml --testcase_timeout 40 -n $NPROC | out-null") | Invoke-Expression - IF (!(Select-String logs/report-junit-enc-const_br.xml -Pattern 'errors="0"')){$zero_errors_enc_const_br=1; echo "Errors encountered"} - # - ("& python -m pytest .\tests\codec_be_to_accepted_release\decoder\test_decoder_constant_bitrate_no_binaural.py --update_ref 1 --html=report-dec-const_br-no_binaural.html --self-contained-html --junit-xml=logs/report-junit-dec-const_br-no_binaural.xml --testcase_timeout 40 -n $NPROC | tee dec-const_br-non_binaural.txt") | Invoke-Expression + - $tmp=((Get-content logs/report-junit-enc-const_br.xml) -split 'name="pytest" ')[1] + - echo ($tmp -split -split 'timestamp="')[0] - # decoder binaural + # decoder, no-binaural, constant brate + - $zero_errors_dec_const_br_no_binaural=0 + - ("& python -m pytest .\tests\codec_be_to_accepted_release\decoder\test_decoder_constant_bitrate_no_binaural.py --update_ref 1 --html=report-dec-const_br-no_binaural.html --self-contained-html --junit-xml=logs/report-junit-dec-const_br-no_binaural.xml --testcase_timeout 40 -n $NPROC | out-null) | Invoke-Expression + - IF (!(Select-String logs/report-junit-dec-const_br-no_binaural.xml -Pattern 'errors="0"')){$zero_errors_dec_const_br_no_binaural=1; echo "Errors encountered"} + + - $tmp=((Get-content logs/report-junit-dec-const_br-no_binaural.xml) -split 'name="pytest" ')[1] + - echo ($tmp -split -split 'timestamp="')[0] + + # decoder, binaural, constant brate - $zero_errors_dec_const_br_binaural=0 - - ("& python -m pytest .\tests\codec_be_to_accepted_release\decoder\test_decoder_constant_bitrate_binaural.py --update_ref 1 --html=report-dec-const_br-binaural.html --self-contained-html --junit-xml=logs/report-junit-dec-const_br-binaural.xml --testcase_timeout 40 -n $NPROC | tee dec-const_br-binaural.txt") | Invoke-Expression + - ("& python -m pytest .\tests\codec_be_to_accepted_release\decoder\test_decoder_constant_bitrate_binaural.py --update_ref 1 --html=report-dec-const_br-binaural.html --self-contained-html --junit-xml=logs/report-junit-dec-const_br-binaural.xml --testcase_timeout 40 -n $NPROC | out-null") | Invoke-Expression - IF (!(Select-String logs/report-junit-dec-const_br-binaural.xml -Pattern 'errors="0"')){$zero_errors_dec_const_br_binaural=1; echo "Errors encountered"} - - IF($zero_errors_enc_const_br -ne 0 -or $zero_errors_dec_const_br_binaural -ne 0 ) {echo "Errors encountered"; exit $EXIT_CODE_FAIL} + - $tmp=((Get-content logs/report-junit-dec-const_br-binaural.xml) -split 'name="pytest" ')[1] + - echo ($tmp -split -split 'timestamp="')[0] + + - IF($zero_errors_enc_const_br -ne 0 -or $zero_errors_dec_const_br_no_binaural -ne 0 -or $zero_errors_dec_const_br_binaural -ne 0 ) {echo "Errors encountered"; exit $EXIT_CODE_FAIL} artifacts: name: "test-be-to-release--sha-$CI_COMMIT_SHORT_SHA--results" -- GitLab From ad2b0192a4ea2ce19511c3d6b01522e6eba450f1 Mon Sep 17 00:00:00 2001 From: Charles Kinuthia Date: Thu, 5 Oct 2023 18:05:32 +0200 Subject: [PATCH 083/106] fix pwsh syntax error --- .gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 14103d994d..66c3b42b82 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1163,7 +1163,7 @@ test-be-to-release: # decoder, no-binaural, constant brate - $zero_errors_dec_const_br_no_binaural=0 - - ("& python -m pytest .\tests\codec_be_to_accepted_release\decoder\test_decoder_constant_bitrate_no_binaural.py --update_ref 1 --html=report-dec-const_br-no_binaural.html --self-contained-html --junit-xml=logs/report-junit-dec-const_br-no_binaural.xml --testcase_timeout 40 -n $NPROC | out-null) | Invoke-Expression + - ("& python -m pytest .\tests\codec_be_to_accepted_release\decoder\test_decoder_constant_bitrate_no_binaural.py --update_ref 1 --html=report-dec-const_br-no_binaural.html --self-contained-html --junit-xml=logs/report-junit-dec-const_br-no_binaural.xml --testcase_timeout 40 -n $NPROC | out-null") | Invoke-Expression - IF (!(Select-String logs/report-junit-dec-const_br-no_binaural.xml -Pattern 'errors="0"')){$zero_errors_dec_const_br_no_binaural=1; echo "Errors encountered"} - $tmp=((Get-content logs/report-junit-dec-const_br-no_binaural.xml) -split 'name="pytest" ')[1] -- GitLab From 1ca390dcca5736266733fd1c5b42468b558235b0 Mon Sep 17 00:00:00 2001 From: Charles Kinuthia Date: Thu, 5 Oct 2023 18:34:58 +0200 Subject: [PATCH 084/106] fix minor syntax error in pytest --- .../codec_be_to_accepted_release/constants.py | 26 +++++++++++-------- 1 file changed, 15 insertions(+), 11 deletions(-) diff --git a/tests/codec_be_to_accepted_release/constants.py b/tests/codec_be_to_accepted_release/constants.py index d59003d8c4..99cc8e85bd 100644 --- a/tests/codec_be_to_accepted_release/constants.py +++ b/tests/codec_be_to_accepted_release/constants.py @@ -423,18 +423,22 @@ DECODER_CONST_BR_NO_BINAURAL_SCENEBASED = collapse_into_list_of_pairs( BITSTREAM_PROCESSING, ) ) -DECODER_CONST_BR_NO_BINAURAL_COMBINED_PARAMS = collapse_into_list_of_pairs( - product( - OSBA_PARAMS, - OUTPUT_FORMATS_NON_BINAURAL[:-1], # no EXT here - SAMPLING_RATES_ALL, - BITSTREAM_PROCESSING, +DECODER_CONST_BR_NO_BINAURAL_COMBINED_PARAMS = ( + collapse_into_list_of_pairs( + product( + OSBA_PARAMS, + OUTPUT_FORMATS_NON_BINAURAL[:-1], # no EXT here + SAMPLING_RATES_ALL, + BITSTREAM_PROCESSING, + ) ) - + product( - OMASA_PARAMS, - OUTPUT_FORMATS_NON_BINAURAL, - SAMPLING_RATES_ALL, - BITSTREAM_PROCESSING, + + collapse_into_list_of_pairs( + product( + OMASA_PARAMS, + OUTPUT_FORMATS_NON_BINAURAL, + SAMPLING_RATES_ALL, + BITSTREAM_PROCESSING, + ) ) ) -- GitLab From 8b5f6c2c2afc9b4f96ce07308b0ab9304b12ed19 Mon Sep 17 00:00:00 2001 From: Charles Kinuthia Date: Fri, 6 Oct 2023 13:28:03 +0200 Subject: [PATCH 085/106] no EXT for OMASA --- .../codec_be_to_accepted_release/constants.py | 23 ++++++------------- 1 file changed, 7 insertions(+), 16 deletions(-) diff --git a/tests/codec_be_to_accepted_release/constants.py b/tests/codec_be_to_accepted_release/constants.py index 99cc8e85bd..a8b8508f5f 100644 --- a/tests/codec_be_to_accepted_release/constants.py +++ b/tests/codec_be_to_accepted_release/constants.py @@ -423,25 +423,16 @@ DECODER_CONST_BR_NO_BINAURAL_SCENEBASED = collapse_into_list_of_pairs( BITSTREAM_PROCESSING, ) ) -DECODER_CONST_BR_NO_BINAURAL_COMBINED_PARAMS = ( - collapse_into_list_of_pairs( - product( - OSBA_PARAMS, - OUTPUT_FORMATS_NON_BINAURAL[:-1], # no EXT here - SAMPLING_RATES_ALL, - BITSTREAM_PROCESSING, - ) - ) - + collapse_into_list_of_pairs( - product( - OMASA_PARAMS, - OUTPUT_FORMATS_NON_BINAURAL, - SAMPLING_RATES_ALL, - BITSTREAM_PROCESSING, - ) +DECODER_CONST_BR_NO_BINAURAL_COMBINED_PARAMS = collapse_into_list_of_pairs( + product( + COMBINED_FORMATS_PARAMS, + OUTPUT_FORMATS_NON_BINAURAL[:10], # one less to exclude EXT + SAMPLING_RATES_ALL, + BITSTREAM_PROCESSING[:2], # TODO: re-add JBM once fully functional ) ) + # parameters for const bitrate testcases with binaural output BINAURAL_HRTF_NONE = "HRTFdefault" -- GitLab From ea6586d435c4b149fc6fcc100e90e7f351153404 Mon Sep 17 00:00:00 2001 From: Charles Kinuthia Date: Fri, 6 Oct 2023 13:54:01 +0200 Subject: [PATCH 086/106] test ref generation for all tests - end, dec (including head rotation, render_conf) --- .gitlab-ci.yml | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 66c3b42b82..ac26909a5d 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1177,7 +1177,32 @@ test-be-to-release: - $tmp=((Get-content logs/report-junit-dec-const_br-binaural.xml) -split 'name="pytest" ')[1] - echo ($tmp -split -split 'timestamp="')[0] + # decoder, head_rot + - $zero_errors_dec_head_rot=0 + - ("& python -m pytest .\tests\codec_be_to_accepted_release\decoder\test_decoder_head_rotation.py --update_ref 1 --html=report-dec-head_rot.html --self-contained-html --junit-xml=logs/report-junit-dec-head_rot.xml --testcase_timeout 40 -n $NPROC | out-null") | Invoke-Expression + - IF (!(Select-String logs/report-junit-dec-head_rot.xml -Pattern 'errors="0"')){$zero_errors_dec_head_rot=1; echo "Errors encountered"} + + - $tmp=((Get-content logs/report-junit-dec-head_rot.xml) -split 'name="pytest" ')[1] + - echo ($tmp -split -split 'timestamp="')[0] + + # decoder, non-diegetic + - $zero_errors_dec_non_diegetic=0 + - ("& python -m pytest .\tests\codec_be_to_accepted_release\decoder\test_decoder_non_diegetic_pan.py --update_ref 1 --html=report-dec-non_diegetic.html --self-contained-html --junit-xml=logs/report-junit-dec-non_diegetic.xml --testcase_timeout 40 -n $NPROC | out-null") | Invoke-Expression + - IF (!(Select-String logs/report-junit-dec-non_diegetic.xml -Pattern 'errors="0"')){$zero_errors_dec_non_diegetic=1; echo "Errors encountered"} + + - $tmp=((Get-content logs/report-junit-dec-non_diegetic.xml) -split 'name="pytest" ')[1] + - echo ($tmp -split -split 'timestamp="')[0] + + # decoder, render_conf + - $zero_errors_dec_render_conf=0 + - ("& python -m pytest .\tests\codec_be_to_accepted_release\decoder\test_decoder_render_config.py --update_ref 1 --html=report-dec-render_conf.html --self-contained-html --junit-xml=logs/report-junit-dec-render_conf.xml --testcase_timeout 40 -n $NPROC | out-null") | Invoke-Expression + - IF (!(Select-String logs/report-junit-dec-render_conf.xml -Pattern 'errors="0"')){$zero_errors_dec_render_conf1; echo "Errors encountered"} + + - $tmp=((Get-content logs/report-junit-dec-render_conf.xml) -split 'name="pytest" ')[1] + - echo ($tmp -split -split 'timestamp="')[0] + - IF($zero_errors_enc_const_br -ne 0 -or $zero_errors_dec_const_br_no_binaural -ne 0 -or $zero_errors_dec_const_br_binaural -ne 0 ) {echo "Errors encountered"; exit $EXIT_CODE_FAIL} + - IF($zero_errors_dec_head_rot -ne 0 -or $zero_errors_dec_non_diegetic -ne 0 -or $zero_errors_dec_render_conf -ne 0 ) {echo "Errors encountered"; exit $EXIT_CODE_FAIL} artifacts: name: "test-be-to-release--sha-$CI_COMMIT_SHORT_SHA--results" -- GitLab From e0620ca3076f77c2d314a30cd047b9e3cefda249 Mon Sep 17 00:00:00 2001 From: Charles Kinuthia Date: Thu, 12 Oct 2023 17:30:43 +0200 Subject: [PATCH 087/106] test storing html logs --- .gitlab-ci.yml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index ac26909a5d..9b55ecae9e 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1160,6 +1160,7 @@ test-be-to-release: - $tmp=((Get-content logs/report-junit-enc-const_br.xml) -split 'name="pytest" ')[1] - echo ($tmp -split -split 'timestamp="')[0] + - cp logs/report-enc-const_br.html $BE_LOGS/report-enc-const_br--sha-$CI_COMMIT_SHORT_SHA.html # decoder, no-binaural, constant brate - $zero_errors_dec_const_br_no_binaural=0 @@ -1168,6 +1169,7 @@ test-be-to-release: - $tmp=((Get-content logs/report-junit-dec-const_br-no_binaural.xml) -split 'name="pytest" ')[1] - echo ($tmp -split -split 'timestamp="')[0] + - cp logs/report-dec-const_br-no_binaural.html $BE_LOGS/report-dec-const_br-no_binaural--sha-$CI_COMMIT_SHORT_SHA.html # decoder, binaural, constant brate - $zero_errors_dec_const_br_binaural=0 @@ -1176,6 +1178,7 @@ test-be-to-release: - $tmp=((Get-content logs/report-junit-dec-const_br-binaural.xml) -split 'name="pytest" ')[1] - echo ($tmp -split -split 'timestamp="')[0] + - cp logs/report-dec-const_br-binaural.html $BE_LOGS/report-dec-const_br-binaural--sha-$CI_COMMIT_SHORT_SHA.html # decoder, head_rot - $zero_errors_dec_head_rot=0 @@ -1184,6 +1187,7 @@ test-be-to-release: - $tmp=((Get-content logs/report-junit-dec-head_rot.xml) -split 'name="pytest" ')[1] - echo ($tmp -split -split 'timestamp="')[0] + - cp logs/report-dec-head_rot.html $BE_LOGS/report-dec-head_rot--sha-$CI_COMMIT_SHORT_SHA.html # decoder, non-diegetic - $zero_errors_dec_non_diegetic=0 @@ -1192,6 +1196,7 @@ test-be-to-release: - $tmp=((Get-content logs/report-junit-dec-non_diegetic.xml) -split 'name="pytest" ')[1] - echo ($tmp -split -split 'timestamp="')[0] + - cp logs/report-dec-non_diegetic.html $BE_LOGS/report-dec-non_diegetic--sha-$CI_COMMIT_SHORT_SHA.html # decoder, render_conf - $zero_errors_dec_render_conf=0 @@ -1200,6 +1205,7 @@ test-be-to-release: - $tmp=((Get-content logs/report-junit-dec-render_conf.xml) -split 'name="pytest" ')[1] - echo ($tmp -split -split 'timestamp="')[0] + - cp logs/report-dec-render_conf.html $BE_LOGS/report-dec-render_conf--sha-$CI_COMMIT_SHORT_SHA.html - IF($zero_errors_enc_const_br -ne 0 -or $zero_errors_dec_const_br_no_binaural -ne 0 -or $zero_errors_dec_const_br_binaural -ne 0 ) {echo "Errors encountered"; exit $EXIT_CODE_FAIL} - IF($zero_errors_dec_head_rot -ne 0 -or $zero_errors_dec_non_diegetic -ne 0 -or $zero_errors_dec_render_conf -ne 0 ) {echo "Errors encountered"; exit $EXIT_CODE_FAIL} -- GitLab From c83d0b46d7632419a9340d34dc0959608d306712 Mon Sep 17 00:00:00 2001 From: Charles Kinuthia Date: Fri, 13 Oct 2023 13:45:36 +0200 Subject: [PATCH 088/106] Fix typo in path --- .gitlab-ci.yml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 9b55ecae9e..cca2d2fd97 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1160,7 +1160,7 @@ test-be-to-release: - $tmp=((Get-content logs/report-junit-enc-const_br.xml) -split 'name="pytest" ')[1] - echo ($tmp -split -split 'timestamp="')[0] - - cp logs/report-enc-const_br.html $BE_LOGS/report-enc-const_br--sha-$CI_COMMIT_SHORT_SHA.html + - cp report-enc-const_br.html $BE_LOGS/report-enc-const_br--sha-$CI_COMMIT_SHORT_SHA.html # decoder, no-binaural, constant brate - $zero_errors_dec_const_br_no_binaural=0 @@ -1169,7 +1169,7 @@ test-be-to-release: - $tmp=((Get-content logs/report-junit-dec-const_br-no_binaural.xml) -split 'name="pytest" ')[1] - echo ($tmp -split -split 'timestamp="')[0] - - cp logs/report-dec-const_br-no_binaural.html $BE_LOGS/report-dec-const_br-no_binaural--sha-$CI_COMMIT_SHORT_SHA.html + - cp report-dec-const_br-no_binaural.html $BE_LOGS/report-dec-const_br-no_binaural--sha-$CI_COMMIT_SHORT_SHA.html # decoder, binaural, constant brate - $zero_errors_dec_const_br_binaural=0 @@ -1178,7 +1178,7 @@ test-be-to-release: - $tmp=((Get-content logs/report-junit-dec-const_br-binaural.xml) -split 'name="pytest" ')[1] - echo ($tmp -split -split 'timestamp="')[0] - - cp logs/report-dec-const_br-binaural.html $BE_LOGS/report-dec-const_br-binaural--sha-$CI_COMMIT_SHORT_SHA.html + - cp report-dec-const_br-binaural.html $BE_LOGS/report-dec-const_br-binaural--sha-$CI_COMMIT_SHORT_SHA.html # decoder, head_rot - $zero_errors_dec_head_rot=0 @@ -1187,7 +1187,7 @@ test-be-to-release: - $tmp=((Get-content logs/report-junit-dec-head_rot.xml) -split 'name="pytest" ')[1] - echo ($tmp -split -split 'timestamp="')[0] - - cp logs/report-dec-head_rot.html $BE_LOGS/report-dec-head_rot--sha-$CI_COMMIT_SHORT_SHA.html + - cp report-dec-head_rot.html $BE_LOGS/report-dec-head_rot--sha-$CI_COMMIT_SHORT_SHA.html # decoder, non-diegetic - $zero_errors_dec_non_diegetic=0 @@ -1196,7 +1196,7 @@ test-be-to-release: - $tmp=((Get-content logs/report-junit-dec-non_diegetic.xml) -split 'name="pytest" ')[1] - echo ($tmp -split -split 'timestamp="')[0] - - cp logs/report-dec-non_diegetic.html $BE_LOGS/report-dec-non_diegetic--sha-$CI_COMMIT_SHORT_SHA.html + - cp report-dec-non_diegetic.html $BE_LOGS/report-dec-non_diegetic--sha-$CI_COMMIT_SHORT_SHA.html # decoder, render_conf - $zero_errors_dec_render_conf=0 @@ -1205,7 +1205,7 @@ test-be-to-release: - $tmp=((Get-content logs/report-junit-dec-render_conf.xml) -split 'name="pytest" ')[1] - echo ($tmp -split -split 'timestamp="')[0] - - cp logs/report-dec-render_conf.html $BE_LOGS/report-dec-render_conf--sha-$CI_COMMIT_SHORT_SHA.html + - cp report-dec-render_conf.html $BE_LOGS/report-dec-render_conf--sha-$CI_COMMIT_SHORT_SHA.html - IF($zero_errors_enc_const_br -ne 0 -or $zero_errors_dec_const_br_no_binaural -ne 0 -or $zero_errors_dec_const_br_binaural -ne 0 ) {echo "Errors encountered"; exit $EXIT_CODE_FAIL} - IF($zero_errors_dec_head_rot -ne 0 -or $zero_errors_dec_non_diegetic -ne 0 -or $zero_errors_dec_render_conf -ne 0 ) {echo "Errors encountered"; exit $EXIT_CODE_FAIL} -- GitLab From c31f94fcfe351ae35258d7c4b5374c3fce7071b1 Mon Sep 17 00:00:00 2001 From: Charles Kinuthia Date: Fri, 13 Oct 2023 13:54:55 +0200 Subject: [PATCH 089/106] fix typo in path --- .gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index cca2d2fd97..415decc69e 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1155,7 +1155,7 @@ test-be-to-release: # encoder, constant brate - $zero_errors_enc_const_br=0 - - ("& python -m pytest .\tests\codec_be_to_accepted_release\encoder\test_encoder_constant_bitrate.py --update_ref 1 --html=logs/report-enc-const_br.html --self-contained-html --junit-xml=logs/report-junit-enc-const_br.xml --testcase_timeout 40 -n $NPROC | out-null") | Invoke-Expression + - ("& python -m pytest .\tests\codec_be_to_accepted_release\encoder\test_encoder_constant_bitrate.py --update_ref 1 --html=report-enc-const_br.html --self-contained-html --junit-xml=logs/report-junit-enc-const_br.xml --testcase_timeout 40 -n $NPROC | out-null") | Invoke-Expression - IF (!(Select-String logs/report-junit-enc-const_br.xml -Pattern 'errors="0"')){$zero_errors_enc_const_br=1; echo "Errors encountered"} - $tmp=((Get-content logs/report-junit-enc-const_br.xml) -split 'name="pytest" ')[1] -- GitLab From 525d76b8c813c42a47dd7c8ed9ac655d570d04ab Mon Sep 17 00:00:00 2001 From: Charles Kinuthia Date: Mon, 16 Oct 2023 11:31:44 +0200 Subject: [PATCH 090/106] test encoder be-pre-test --- .gitlab-ci.yml | 81 +++++++++++++------------------------------------- 1 file changed, 20 insertions(+), 61 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 415decc69e..2b3e8c430e 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1137,12 +1137,17 @@ test-be-to-release: - if: $CI_PIPELINE_SOURCE == 'web' && $MANUAL_PIPELINE_TYPE == 'test-be-release' script: - - $winoutdata = $null - - echo "$CI_COMMIT_BRANCH" - - MSBuild.exe .\Workspace_msvc\Workspace_msvc.sln /property:Configuration=Debug /p:Platform=win32 /m | tee -variable winoutdata - - $winoutdata | Out-File $BUILD_OUTPUT -Encoding Utf8 - - ("& python ci/check_for_warnings.py '$BUILD_OUTPUT'") | Invoke-Expression - # - If($LASTEXITCODE) {("exit $LASTEXITCODE") | Invoke-Expression} + - git checkout $REF_RELEASE_TAG + - echo "REF_RELEASE_TAG $REF_RELEASE_TAG" + - MSBuild.exe .\Workspace_msvc\Workspace_msvc.sln /property:Configuration=Debug /p:Platform=win32 /m + + - mv IVAS_cod.exe IVAS_cod_ref.exe + - mv IVAS_dec.exe IVAS_dec_ref.exe + - mv IVAS_rend.exe IVAS_rend_ref.exe + + - git checkout $DUT_RELEASE_TAG + - echo "DUT_RELEASE_TAG $DUT_RELEASE_TAG" + - MSBuild.exe .\Workspace_msvc\Workspace_msvc.sln /property:Configuration=Debug /p:Platform=win32 /m # path to release candidate refs defined in config.toml - mkdir logs @@ -1154,61 +1159,15 @@ test-be-to-release: - echo "generate references" # encoder, constant brate - - $zero_errors_enc_const_br=0 - - ("& python -m pytest .\tests\codec_be_to_accepted_release\encoder\test_encoder_constant_bitrate.py --update_ref 1 --html=report-enc-const_br.html --self-contained-html --junit-xml=logs/report-junit-enc-const_br.xml --testcase_timeout 40 -n $NPROC | out-null") | Invoke-Expression - - IF (!(Select-String logs/report-junit-enc-const_br.xml -Pattern 'errors="0"')){$zero_errors_enc_const_br=1; echo "Errors encountered"} - - - $tmp=((Get-content logs/report-junit-enc-const_br.xml) -split 'name="pytest" ')[1] - - echo ($tmp -split -split 'timestamp="')[0] - - cp report-enc-const_br.html $BE_LOGS/report-enc-const_br--sha-$CI_COMMIT_SHORT_SHA.html - - # decoder, no-binaural, constant brate - - $zero_errors_dec_const_br_no_binaural=0 - - ("& python -m pytest .\tests\codec_be_to_accepted_release\decoder\test_decoder_constant_bitrate_no_binaural.py --update_ref 1 --html=report-dec-const_br-no_binaural.html --self-contained-html --junit-xml=logs/report-junit-dec-const_br-no_binaural.xml --testcase_timeout 40 -n $NPROC | out-null") | Invoke-Expression - - IF (!(Select-String logs/report-junit-dec-const_br-no_binaural.xml -Pattern 'errors="0"')){$zero_errors_dec_const_br_no_binaural=1; echo "Errors encountered"} - - - $tmp=((Get-content logs/report-junit-dec-const_br-no_binaural.xml) -split 'name="pytest" ')[1] - - echo ($tmp -split -split 'timestamp="')[0] - - cp report-dec-const_br-no_binaural.html $BE_LOGS/report-dec-const_br-no_binaural--sha-$CI_COMMIT_SHORT_SHA.html - - # decoder, binaural, constant brate - - $zero_errors_dec_const_br_binaural=0 - - ("& python -m pytest .\tests\codec_be_to_accepted_release\decoder\test_decoder_constant_bitrate_binaural.py --update_ref 1 --html=report-dec-const_br-binaural.html --self-contained-html --junit-xml=logs/report-junit-dec-const_br-binaural.xml --testcase_timeout 40 -n $NPROC | out-null") | Invoke-Expression - - IF (!(Select-String logs/report-junit-dec-const_br-binaural.xml -Pattern 'errors="0"')){$zero_errors_dec_const_br_binaural=1; echo "Errors encountered"} - - - $tmp=((Get-content logs/report-junit-dec-const_br-binaural.xml) -split 'name="pytest" ')[1] - - echo ($tmp -split -split 'timestamp="')[0] - - cp report-dec-const_br-binaural.html $BE_LOGS/report-dec-const_br-binaural--sha-$CI_COMMIT_SHORT_SHA.html - - # decoder, head_rot - - $zero_errors_dec_head_rot=0 - - ("& python -m pytest .\tests\codec_be_to_accepted_release\decoder\test_decoder_head_rotation.py --update_ref 1 --html=report-dec-head_rot.html --self-contained-html --junit-xml=logs/report-junit-dec-head_rot.xml --testcase_timeout 40 -n $NPROC | out-null") | Invoke-Expression - - IF (!(Select-String logs/report-junit-dec-head_rot.xml -Pattern 'errors="0"')){$zero_errors_dec_head_rot=1; echo "Errors encountered"} - - - $tmp=((Get-content logs/report-junit-dec-head_rot.xml) -split 'name="pytest" ')[1] - - echo ($tmp -split -split 'timestamp="')[0] - - cp report-dec-head_rot.html $BE_LOGS/report-dec-head_rot--sha-$CI_COMMIT_SHORT_SHA.html - - # decoder, non-diegetic - - $zero_errors_dec_non_diegetic=0 - - ("& python -m pytest .\tests\codec_be_to_accepted_release\decoder\test_decoder_non_diegetic_pan.py --update_ref 1 --html=report-dec-non_diegetic.html --self-contained-html --junit-xml=logs/report-junit-dec-non_diegetic.xml --testcase_timeout 40 -n $NPROC | out-null") | Invoke-Expression - - IF (!(Select-String logs/report-junit-dec-non_diegetic.xml -Pattern 'errors="0"')){$zero_errors_dec_non_diegetic=1; echo "Errors encountered"} - - - $tmp=((Get-content logs/report-junit-dec-non_diegetic.xml) -split 'name="pytest" ')[1] - - echo ($tmp -split -split 'timestamp="')[0] - - cp report-dec-non_diegetic.html $BE_LOGS/report-dec-non_diegetic--sha-$CI_COMMIT_SHORT_SHA.html - - # decoder, render_conf - - $zero_errors_dec_render_conf=0 - - ("& python -m pytest .\tests\codec_be_to_accepted_release\decoder\test_decoder_render_config.py --update_ref 1 --html=report-dec-render_conf.html --self-contained-html --junit-xml=logs/report-junit-dec-render_conf.xml --testcase_timeout 40 -n $NPROC | out-null") | Invoke-Expression - - IF (!(Select-String logs/report-junit-dec-render_conf.xml -Pattern 'errors="0"')){$zero_errors_dec_render_conf1; echo "Errors encountered"} - - - $tmp=((Get-content logs/report-junit-dec-render_conf.xml) -split 'name="pytest" ')[1] - - echo ($tmp -split -split 'timestamp="')[0] - - cp report-dec-render_conf.html $BE_LOGS/report-dec-render_conf--sha-$CI_COMMIT_SHORT_SHA.html - - - IF($zero_errors_enc_const_br -ne 0 -or $zero_errors_dec_const_br_no_binaural -ne 0 -or $zero_errors_dec_const_br_binaural -ne 0 ) {echo "Errors encountered"; exit $EXIT_CODE_FAIL} - - IF($zero_errors_dec_head_rot -ne 0 -or $zero_errors_dec_non_diegetic -ne 0 -or $zero_errors_dec_render_conf -ne 0 ) {echo "Errors encountered"; exit $EXIT_CODE_FAIL} + - $zero_errors_enc_const_br_ref=0 + - ("& python -m pytest .\tests\codec_be_to_accepted_release\encoder\test_encoder_constant_bitrate.py --update_ref 1 --html=logs/report-enc-const_br-ref.html --self-contained-html --junit-xml=logs/report-junit-enc-const_br-ref.xml --ref_encoder_path IVAS_cod_ref.exe --ref_decoder_path IVAS_dec.exe --testcase_timeout 40 -n $NPROC | out-null") | Invoke-Expression + - IF (!(Select-String logs/report-junit-enc-const_br-ref.xml -Pattern 'errors="0"')){$zero_errors_enc_const_br_ref=1; echo "Errors encountered"} + + - $zero_errors_enc_const_br_dut=0 + - ("& python -m pytest .\tests\codec_be_to_accepted_release\encoder\test_encoder_constant_bitrate.py --html=logs/report-enc-const_br-dut.html --self-contained-html --junit-xml=logs/report-junit-enc-const_br-dut.xml --testcase_timeout 40 -n $NPROC | out-null") | Invoke-Expression + - IF (!(Select-String logs/report-junit-enc-const_br.xml -Pattern 'errors="0"')){$zero_errors_enc_const_br_dut=1; echo "Errors encountered"} + + - IF($zero_errors_enc_const_br_ref -ne 0 -or $zero_errors_enc_const_br_dut -ne 0 ) {echo "Errors encountered"; exit $EXIT_CODE_FAIL} artifacts: name: "test-be-to-release--sha-$CI_COMMIT_SHORT_SHA--results" -- GitLab From 57f59458ac000c9646b3bec56740b3a93ea8e031 Mon Sep 17 00:00:00 2001 From: Charles Kinuthia Date: Mon, 16 Oct 2023 11:39:04 +0200 Subject: [PATCH 091/106] fix typo --- .gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 2b3e8c430e..c6ec29240c 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1165,7 +1165,7 @@ test-be-to-release: - $zero_errors_enc_const_br_dut=0 - ("& python -m pytest .\tests\codec_be_to_accepted_release\encoder\test_encoder_constant_bitrate.py --html=logs/report-enc-const_br-dut.html --self-contained-html --junit-xml=logs/report-junit-enc-const_br-dut.xml --testcase_timeout 40 -n $NPROC | out-null") | Invoke-Expression - - IF (!(Select-String logs/report-junit-enc-const_br.xml -Pattern 'errors="0"')){$zero_errors_enc_const_br_dut=1; echo "Errors encountered"} + - IF (!(Select-String logs/report-junit-enc-const_br-dut.xml -Pattern 'errors="0"')){$zero_errors_enc_const_br_dut=1; echo "Errors encountered"} - IF($zero_errors_enc_const_br_ref -ne 0 -or $zero_errors_enc_const_br_dut -ne 0 ) {echo "Errors encountered"; exit $EXIT_CODE_FAIL} -- GitLab From b95a55be590dbefeac3dcfc19728d798c2e76371 Mon Sep 17 00:00:00 2001 From: Charles Kinuthia Date: Mon, 16 Oct 2023 13:07:48 +0200 Subject: [PATCH 092/106] fix typo --- .gitlab-ci.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index c6ec29240c..a7be4c58ae 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1137,7 +1137,7 @@ test-be-to-release: - if: $CI_PIPELINE_SOURCE == 'web' && $MANUAL_PIPELINE_TYPE == 'test-be-release' script: - - git checkout $REF_RELEASE_TAG + - ("& git checkout $REF_RELEASE_TAG") | Invoke-Expression - echo "REF_RELEASE_TAG $REF_RELEASE_TAG" - MSBuild.exe .\Workspace_msvc\Workspace_msvc.sln /property:Configuration=Debug /p:Platform=win32 /m @@ -1145,7 +1145,7 @@ test-be-to-release: - mv IVAS_dec.exe IVAS_dec_ref.exe - mv IVAS_rend.exe IVAS_rend_ref.exe - - git checkout $DUT_RELEASE_TAG + - ("& git checkout $DUT_RELEASE_TAG") | Invoke-Expression - echo "DUT_RELEASE_TAG $DUT_RELEASE_TAG" - MSBuild.exe .\Workspace_msvc\Workspace_msvc.sln /property:Configuration=Debug /p:Platform=win32 /m @@ -1160,7 +1160,7 @@ test-be-to-release: # encoder, constant brate - $zero_errors_enc_const_br_ref=0 - - ("& python -m pytest .\tests\codec_be_to_accepted_release\encoder\test_encoder_constant_bitrate.py --update_ref 1 --html=logs/report-enc-const_br-ref.html --self-contained-html --junit-xml=logs/report-junit-enc-const_br-ref.xml --ref_encoder_path IVAS_cod_ref.exe --ref_decoder_path IVAS_dec.exe --testcase_timeout 40 -n $NPROC | out-null") | Invoke-Expression + - ("& python -m pytest .\tests\codec_be_to_accepted_release\encoder\test_encoder_constant_bitrate.py --update_ref 1 --html=logs/report-enc-const_br-ref.html --self-contained-html --junit-xml=logs/report-junit-enc-const_br-ref.xml --ref_encoder_path IVAS_cod_ref.exe --ref_decoder_path IVAS_dec_ref.exe --testcase_timeout 40 -n $NPROC | out-null") | Invoke-Expression - IF (!(Select-String logs/report-junit-enc-const_br-ref.xml -Pattern 'errors="0"')){$zero_errors_enc_const_br_ref=1; echo "Errors encountered"} - $zero_errors_enc_const_br_dut=0 -- GitLab From 3b0d5e5616bb2bbf30d3ae71a1d2863093402aad Mon Sep 17 00:00:00 2001 From: Charles Kinuthia Date: Mon, 16 Oct 2023 13:21:13 +0200 Subject: [PATCH 093/106] fix, checkout branch with codec_be_to_accepted_release be-test script --- .gitlab-ci.yml | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index a7be4c58ae..d6e9eefb03 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1137,7 +1137,7 @@ test-be-to-release: - if: $CI_PIPELINE_SOURCE == 'web' && $MANUAL_PIPELINE_TYPE == 'test-be-release' script: - - ("& git checkout $REF_RELEASE_TAG") | Invoke-Expression + - git checkout $REF_RELEASE_TAG" - echo "REF_RELEASE_TAG $REF_RELEASE_TAG" - MSBuild.exe .\Workspace_msvc\Workspace_msvc.sln /property:Configuration=Debug /p:Platform=win32 /m @@ -1145,11 +1145,13 @@ test-be-to-release: - mv IVAS_dec.exe IVAS_dec_ref.exe - mv IVAS_rend.exe IVAS_rend_ref.exe - - ("& git checkout $DUT_RELEASE_TAG") | Invoke-Expression + - git checkout $DUT_RELEASE_TAG" - echo "DUT_RELEASE_TAG $DUT_RELEASE_TAG" - MSBuild.exe .\Workspace_msvc\Workspace_msvc.sln /property:Configuration=Debug /p:Platform=win32 /m - # path to release candidate refs defined in config.toml + # checkout current branch with codec_be_to_accepted_release be-test + - git checkout $CI_COMMIT_SHORT_SHA + - mkdir logs - mkdir tests\codec_be_to_accepted_release\ref @@ -1170,7 +1172,7 @@ test-be-to-release: - IF($zero_errors_enc_const_br_ref -ne 0 -or $zero_errors_enc_const_br_dut -ne 0 ) {echo "Errors encountered"; exit $EXIT_CODE_FAIL} artifacts: - name: "test-be-to-release--sha-$CI_COMMIT_SHORT_SHA--results" + name: "test-be-to-release--$REF_RELEASE_TAG-to-$DUT_RELEASE_TAG--results" when: always expire_in: 1 week paths: -- GitLab From 8ba0a0bb069ed82d1f3bf71f440bb73c91514496 Mon Sep 17 00:00:00 2001 From: Charles Kinuthia Date: Mon, 16 Oct 2023 13:24:20 +0200 Subject: [PATCH 094/106] syntax fix --- .gitlab-ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index d6e9eefb03..34dad097c8 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1137,7 +1137,7 @@ test-be-to-release: - if: $CI_PIPELINE_SOURCE == 'web' && $MANUAL_PIPELINE_TYPE == 'test-be-release' script: - - git checkout $REF_RELEASE_TAG" + - git checkout $REF_RELEASE_TAG - echo "REF_RELEASE_TAG $REF_RELEASE_TAG" - MSBuild.exe .\Workspace_msvc\Workspace_msvc.sln /property:Configuration=Debug /p:Platform=win32 /m @@ -1145,7 +1145,7 @@ test-be-to-release: - mv IVAS_dec.exe IVAS_dec_ref.exe - mv IVAS_rend.exe IVAS_rend_ref.exe - - git checkout $DUT_RELEASE_TAG" + - git checkout $DUT_RELEASE_TAG - echo "DUT_RELEASE_TAG $DUT_RELEASE_TAG" - MSBuild.exe .\Workspace_msvc\Workspace_msvc.sln /property:Configuration=Debug /p:Platform=win32 /m -- GitLab From fd303c0852d1e9447a16b54487bf562be7eec9f7 Mon Sep 17 00:00:00 2001 From: Charles Kinuthia Date: Tue, 17 Oct 2023 09:48:33 +0200 Subject: [PATCH 095/106] expose variables to gitlab interface and rename job and pipeline --- .gitlab-ci.yml | 39 ++++++++++++++++++++++++++++++++++++++- 1 file changed, 38 insertions(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 34dad097c8..f889d3170b 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -20,6 +20,11 @@ variables: - 'default' - 'test-be-release' - 'test-long-self-test' + - 'test-be-release-pre-test' + REF_RELEASE_TAG: + description: "Ref release tag" + DUT_RELEASE_TAG: + description: "Dut release tag" default: @@ -50,6 +55,9 @@ workflow: - if: $CI_PIPELINE_SOURCE == 'web' && $MANUAL_PIPELINE_TYPE == 'test-long-self-test' variables: IVAS_PIPELINE_NAME: 'Test long self-test against main pipeline: $CI_COMMIT_BRANCH' + - if: $CI_PIPELINE_SOURCE == 'web' && $MANUAL_PIPELINE_TYPE == 'test-be-release-pre-test' + variables: + IVAS_PIPELINE_NAME: 'Test BE to release pre-test pipeline: $REF_RELEASE_TAG vs $DUT_RELEASE_TAG' - if: $CI_PIPELINE_SOURCE == 'trigger' stages: @@ -1128,13 +1136,42 @@ codec-comparison-on-main-push: # --------------------------------------------------------------- test-be-to-release: + stage: test + tags: + - ivas-windows + resource_group: ivas-be-to-release-test-resource + timeout: "20 minutes" + rules: + - if: $CI_PIPELINE_SOURCE == 'web' && $MANUAL_PIPELINE_TYPE == 'test-be-release' + script: + + - echo "$CI_COMMIT_BRANCH" + - MSBuild.exe .\Workspace_msvc\Workspace_msvc.sln /property:Configuration=Debug /p:Platform=win32 /m | tee -variable winoutdata + - $winoutdata | Out-File $BUILD_OUTPUT -Encoding Utf8 + - ("& python ci/check_for_warnings.py '$BUILD_OUTPUT'") | Invoke-Expression + - ("exit $LASTEXITCODE") | Invoke-Expression + + # path to release candidate refs defined in config.toml + + - echo "Placeholder for BE test to release script" + - mkdir logs + + artifacts: + name: "test-be-to-release--sha-$CI_COMMIT_SHORT_SHA--results" + when: always + expire_in: 1 week + paths: + - logs/ + expose_as: "test-be-to-release results" + +test-be-release-pre-test: stage: test tags: - test-be-to-release-windows resource_group: ivas-be-to-release-test-resource timeout: "22 hours" # to be reviewed rules: - - if: $CI_PIPELINE_SOURCE == 'web' && $MANUAL_PIPELINE_TYPE == 'test-be-release' + - if: $CI_PIPELINE_SOURCE == 'web' && $MANUAL_PIPELINE_TYPE == 'test-be-to-release-pre-test' script: - git checkout $REF_RELEASE_TAG -- GitLab From 487508ce69dabda538c827b7b9c4c7558bddd013 Mon Sep 17 00:00:00 2001 From: Charles Kinuthia Date: Tue, 17 Oct 2023 09:58:03 +0200 Subject: [PATCH 096/106] skipped normal jobs for manual job --- .gitlab-ci.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index f889d3170b..0a982aedf0 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -188,6 +188,8 @@ stages: when: never - if: $MANUAL_PIPELINE_TYPE == 'test-long-self-test' # Skip all the normal jobs when testing manually against release codec when: never + - if: $MANUAL_PIPELINE_TYPE == 'test-be-release-pre-test' # Skip all the normal jobs when testing manually against release codec + when: never - when: on_success .rules-merge-request: -- GitLab From fdef95343300d94134c12ec97605daa8ccdd442e Mon Sep 17 00:00:00 2001 From: Charles Kinuthia Date: Tue, 17 Oct 2023 10:00:27 +0200 Subject: [PATCH 097/106] fix typo --- .gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 0a982aedf0..42b5c39729 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1173,7 +1173,7 @@ test-be-release-pre-test: resource_group: ivas-be-to-release-test-resource timeout: "22 hours" # to be reviewed rules: - - if: $CI_PIPELINE_SOURCE == 'web' && $MANUAL_PIPELINE_TYPE == 'test-be-to-release-pre-test' + - if: $CI_PIPELINE_SOURCE == 'web' && $MANUAL_PIPELINE_TYPE == 'test-be-release-pre-test' script: - git checkout $REF_RELEASE_TAG -- GitLab From dc1fa9b6c2431d0c8ee6a76f9c88098e721867bc Mon Sep 17 00:00:00 2001 From: Charles Kinuthia Date: Tue, 17 Oct 2023 14:13:09 +0200 Subject: [PATCH 098/106] remove '$NPROC' and instead use all available processes on machine also explicitly specify ref and dut paths --- .gitlab-ci.yml | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 42b5c39729..8f99ec1158 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1188,6 +1188,10 @@ test-be-release-pre-test: - echo "DUT_RELEASE_TAG $DUT_RELEASE_TAG" - MSBuild.exe .\Workspace_msvc\Workspace_msvc.sln /property:Configuration=Debug /p:Platform=win32 /m + - mv IVAS_cod.exe IVAS_cod_dut.exe + - mv IVAS_dec.exe IVAS_dec_dut.exe + - mv IVAS_rend.exe IVAS_rend_dut.exe + # checkout current branch with codec_be_to_accepted_release be-test - git checkout $CI_COMMIT_SHORT_SHA @@ -1201,11 +1205,11 @@ test-be-release-pre-test: # encoder, constant brate - $zero_errors_enc_const_br_ref=0 - - ("& python -m pytest .\tests\codec_be_to_accepted_release\encoder\test_encoder_constant_bitrate.py --update_ref 1 --html=logs/report-enc-const_br-ref.html --self-contained-html --junit-xml=logs/report-junit-enc-const_br-ref.xml --ref_encoder_path IVAS_cod_ref.exe --ref_decoder_path IVAS_dec_ref.exe --testcase_timeout 40 -n $NPROC | out-null") | Invoke-Expression + - ("& python -m pytest .\tests\codec_be_to_accepted_release\encoder\test_encoder_constant_bitrate.py --update_ref 1 --ref_encoder_path IVAS_cod_ref.exe --ref_decoder_path IVAS_dec_ref.exe --html=logs/report-enc-const_br-ref.html --self-contained-html --junit-xml=logs/report-junit-enc-const_br-ref.xml --testcase_timeout 40 | out-null") | Invoke-Expression - IF (!(Select-String logs/report-junit-enc-const_br-ref.xml -Pattern 'errors="0"')){$zero_errors_enc_const_br_ref=1; echo "Errors encountered"} - $zero_errors_enc_const_br_dut=0 - - ("& python -m pytest .\tests\codec_be_to_accepted_release\encoder\test_encoder_constant_bitrate.py --html=logs/report-enc-const_br-dut.html --self-contained-html --junit-xml=logs/report-junit-enc-const_br-dut.xml --testcase_timeout 40 -n $NPROC | out-null") | Invoke-Expression + - ("& python -m pytest .\tests\codec_be_to_accepted_release\encoder\test_encoder_constant_bitrate.py --dut_encoder_path IVAS_cod_dut.exe --dut_encoder_path IVAS_dec_dut.exe --html=logs/report-enc-const_br-dut.html --self-contained-html --junit-xml=logs/report-junit-enc-const_br-dut.xml --testcase_timeout 40 | out-null") | Invoke-Expression - IF (!(Select-String logs/report-junit-enc-const_br-dut.xml -Pattern 'errors="0"')){$zero_errors_enc_const_br_dut=1; echo "Errors encountered"} - IF($zero_errors_enc_const_br_ref -ne 0 -or $zero_errors_enc_const_br_dut -ne 0 ) {echo "Errors encountered"; exit $EXIT_CODE_FAIL} -- GitLab From a6823bdda8d9aa81be861ca099e8461f4dec5b27 Mon Sep 17 00:00:00 2001 From: Charles Kinuthia Date: Tue, 17 Oct 2023 14:26:06 +0200 Subject: [PATCH 099/106] fix typo --- .gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 8f99ec1158..a8196d830a 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1209,7 +1209,7 @@ test-be-release-pre-test: - IF (!(Select-String logs/report-junit-enc-const_br-ref.xml -Pattern 'errors="0"')){$zero_errors_enc_const_br_ref=1; echo "Errors encountered"} - $zero_errors_enc_const_br_dut=0 - - ("& python -m pytest .\tests\codec_be_to_accepted_release\encoder\test_encoder_constant_bitrate.py --dut_encoder_path IVAS_cod_dut.exe --dut_encoder_path IVAS_dec_dut.exe --html=logs/report-enc-const_br-dut.html --self-contained-html --junit-xml=logs/report-junit-enc-const_br-dut.xml --testcase_timeout 40 | out-null") | Invoke-Expression + - ("& python -m pytest .\tests\codec_be_to_accepted_release\encoder\test_encoder_constant_bitrate.py --dut_encoder_path IVAS_cod_dut.exe --dut_decoder_path IVAS_dec_dut.exe --html=logs/report-enc-const_br-dut.html --self-contained-html --junit-xml=logs/report-junit-enc-const_br-dut.xml --testcase_timeout 40 | out-null") | Invoke-Expression - IF (!(Select-String logs/report-junit-enc-const_br-dut.xml -Pattern 'errors="0"')){$zero_errors_enc_const_br_dut=1; echo "Errors encountered"} - IF($zero_errors_enc_const_br_ref -ne 0 -or $zero_errors_enc_const_br_dut -ne 0 ) {echo "Errors encountered"; exit $EXIT_CODE_FAIL} -- GitLab From d15ac09b2940ac8837f341322b1383b0d312faf9 Mon Sep 17 00:00:00 2001 From: Charles Kinuthia Date: Tue, 17 Oct 2023 14:46:33 +0200 Subject: [PATCH 100/106] specify ref and dut path --- .gitlab-ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index a8196d830a..0a2304b04c 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1205,11 +1205,11 @@ test-be-release-pre-test: # encoder, constant brate - $zero_errors_enc_const_br_ref=0 - - ("& python -m pytest .\tests\codec_be_to_accepted_release\encoder\test_encoder_constant_bitrate.py --update_ref 1 --ref_encoder_path IVAS_cod_ref.exe --ref_decoder_path IVAS_dec_ref.exe --html=logs/report-enc-const_br-ref.html --self-contained-html --junit-xml=logs/report-junit-enc-const_br-ref.xml --testcase_timeout 40 | out-null") | Invoke-Expression + - ("& python -m pytest .\tests\codec_be_to_accepted_release\encoder\test_encoder_constant_bitrate.py --update_ref 1 --ref_encoder_path IVAS_cod_ref.exe --ref_decoder_path IVAS_dec_ref.exe --dut_encoder_path IVAS_cod_dut.exe --dut_decoder_path IVAS_dec_dut.exe --html=logs/report-enc-const_br-ref.html --self-contained-html --junit-xml=logs/report-junit-enc-const_br-ref.xml --testcase_timeout 40 | out-null") | Invoke-Expression - IF (!(Select-String logs/report-junit-enc-const_br-ref.xml -Pattern 'errors="0"')){$zero_errors_enc_const_br_ref=1; echo "Errors encountered"} - $zero_errors_enc_const_br_dut=0 - - ("& python -m pytest .\tests\codec_be_to_accepted_release\encoder\test_encoder_constant_bitrate.py --dut_encoder_path IVAS_cod_dut.exe --dut_decoder_path IVAS_dec_dut.exe --html=logs/report-enc-const_br-dut.html --self-contained-html --junit-xml=logs/report-junit-enc-const_br-dut.xml --testcase_timeout 40 | out-null") | Invoke-Expression + - ("& python -m pytest .\tests\codec_be_to_accepted_release\encoder\test_encoder_constant_bitrate.py --ref_encoder_path IVAS_cod_ref.exe --ref_decoder_path IVAS_dec_ref.exe --dut_encoder_path IVAS_cod_dut.exe --dut_decoder_path IVAS_dec_dut.exe --html=logs/report-enc-const_br-dut.html --self-contained-html --junit-xml=logs/report-junit-enc-const_br-dut.xml --testcase_timeout 40 | out-null") | Invoke-Expression - IF (!(Select-String logs/report-junit-enc-const_br-dut.xml -Pattern 'errors="0"')){$zero_errors_enc_const_br_dut=1; echo "Errors encountered"} - IF($zero_errors_enc_const_br_ref -ne 0 -or $zero_errors_enc_const_br_dut -ne 0 ) {echo "Errors encountered"; exit $EXIT_CODE_FAIL} -- GitLab From 58a7cdd52a861b64dc24f357a76503565ebc4227 Mon Sep 17 00:00:00 2001 From: Charles Kinuthia Date: Wed, 18 Oct 2023 13:58:31 +0200 Subject: [PATCH 101/106] rename variables used in test-be-release-pre-test REF_RELEASE_TAG to REF_COMMIT DUT_RELEASE_TAG to DUT_COMMIT --- .gitlab-ci.yml | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 0a2304b04c..a66ad3589d 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -21,10 +21,10 @@ variables: - 'test-be-release' - 'test-long-self-test' - 'test-be-release-pre-test' - REF_RELEASE_TAG: - description: "Ref release tag" - DUT_RELEASE_TAG: - description: "Dut release tag" + REF_COMMIT: + description: "Ref commit id or ref release tag. To be used by Manual pipeline 'test-be-release-pre-test'" + DUT_COMMIT: + description: "Dut commit id or dut release tag. To be used by Manual pipeline 'test-be-release-pre-test'" default: @@ -57,7 +57,7 @@ workflow: IVAS_PIPELINE_NAME: 'Test long self-test against main pipeline: $CI_COMMIT_BRANCH' - if: $CI_PIPELINE_SOURCE == 'web' && $MANUAL_PIPELINE_TYPE == 'test-be-release-pre-test' variables: - IVAS_PIPELINE_NAME: 'Test BE to release pre-test pipeline: $REF_RELEASE_TAG vs $DUT_RELEASE_TAG' + IVAS_PIPELINE_NAME: 'Test BE to release pre-test pipeline: $REF_COMMIT vs $DUT_COMMIT' - if: $CI_PIPELINE_SOURCE == 'trigger' stages: @@ -1176,16 +1176,16 @@ test-be-release-pre-test: - if: $CI_PIPELINE_SOURCE == 'web' && $MANUAL_PIPELINE_TYPE == 'test-be-release-pre-test' script: - - git checkout $REF_RELEASE_TAG - - echo "REF_RELEASE_TAG $REF_RELEASE_TAG" + - git checkout $REF_COMMIT + - echo "REF_COMMIT $REF_COMMIT" - MSBuild.exe .\Workspace_msvc\Workspace_msvc.sln /property:Configuration=Debug /p:Platform=win32 /m - mv IVAS_cod.exe IVAS_cod_ref.exe - mv IVAS_dec.exe IVAS_dec_ref.exe - mv IVAS_rend.exe IVAS_rend_ref.exe - - git checkout $DUT_RELEASE_TAG - - echo "DUT_RELEASE_TAG $DUT_RELEASE_TAG" + - git checkout $DUT_COMMIT + - echo "DUT_COMMIT $DUT_COMMIT" - MSBuild.exe .\Workspace_msvc\Workspace_msvc.sln /property:Configuration=Debug /p:Platform=win32 /m - mv IVAS_cod.exe IVAS_cod_dut.exe @@ -1215,7 +1215,7 @@ test-be-release-pre-test: - IF($zero_errors_enc_const_br_ref -ne 0 -or $zero_errors_enc_const_br_dut -ne 0 ) {echo "Errors encountered"; exit $EXIT_CODE_FAIL} artifacts: - name: "test-be-to-release--$REF_RELEASE_TAG-to-$DUT_RELEASE_TAG--results" + name: "test-be-to-release--$REF_COMMIT-to-$DUT_COMMIT--results" when: always expire_in: 1 week paths: -- GitLab From 2c841bcd38146e0b29eccb9e614e11f93a29300c Mon Sep 17 00:00:00 2001 From: Charles Kinuthia Date: Wed, 18 Oct 2023 16:47:42 +0200 Subject: [PATCH 102/106] increase timeout & fail job if tests fail --- .gitlab-ci.yml | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index a66ad3589d..0dba4924b9 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1204,13 +1204,16 @@ test-be-release-pre-test: - echo "generate references" # encoder, constant brate + - testcase_timeout=240 # to be reviewed - $zero_errors_enc_const_br_ref=0 - - ("& python -m pytest .\tests\codec_be_to_accepted_release\encoder\test_encoder_constant_bitrate.py --update_ref 1 --ref_encoder_path IVAS_cod_ref.exe --ref_decoder_path IVAS_dec_ref.exe --dut_encoder_path IVAS_cod_dut.exe --dut_decoder_path IVAS_dec_dut.exe --html=logs/report-enc-const_br-ref.html --self-contained-html --junit-xml=logs/report-junit-enc-const_br-ref.xml --testcase_timeout 40 | out-null") | Invoke-Expression + - ("& python -m pytest .\tests\codec_be_to_accepted_release\encoder\test_encoder_constant_bitrate.py --update_ref 1 --ref_encoder_path IVAS_cod_ref.exe --ref_decoder_path IVAS_dec_ref.exe --dut_encoder_path IVAS_cod_dut.exe --dut_decoder_path IVAS_dec_dut.exe --html=logs/report-enc-const_br-ref.html --self-contained-html --junit-xml=logs/report-junit-enc-const_br-ref.xml --testcase_timeout $testcase_timeout | out-null") | Invoke-Expression - IF (!(Select-String logs/report-junit-enc-const_br-ref.xml -Pattern 'errors="0"')){$zero_errors_enc_const_br_ref=1; echo "Errors encountered"} + - IF (!(Select-String logs/report-junit-enc-const_br-ref.xml -Pattern 'failures="0"')){$zero_errors_enc_const_br_ref=1; echo "Failed tests encountered in ref generation"} - $zero_errors_enc_const_br_dut=0 - - ("& python -m pytest .\tests\codec_be_to_accepted_release\encoder\test_encoder_constant_bitrate.py --ref_encoder_path IVAS_cod_ref.exe --ref_decoder_path IVAS_dec_ref.exe --dut_encoder_path IVAS_cod_dut.exe --dut_decoder_path IVAS_dec_dut.exe --html=logs/report-enc-const_br-dut.html --self-contained-html --junit-xml=logs/report-junit-enc-const_br-dut.xml --testcase_timeout 40 | out-null") | Invoke-Expression + - ("& python -m pytest .\tests\codec_be_to_accepted_release\encoder\test_encoder_constant_bitrate.py --ref_encoder_path IVAS_cod_ref.exe --ref_decoder_path IVAS_dec_ref.exe --dut_encoder_path IVAS_cod_dut.exe --dut_decoder_path IVAS_dec_dut.exe --html=logs/report-enc-const_br-dut.html --self-contained-html --junit-xml=logs/report-junit-enc-const_br-dut.xml --testcase_timeout $testcase_timeout | out-null") | Invoke-Expression - IF (!(Select-String logs/report-junit-enc-const_br-dut.xml -Pattern 'errors="0"')){$zero_errors_enc_const_br_dut=1; echo "Errors encountered"} + - IF (!(Select-String logs/report-junit-enc-const_br-dut.xml -Pattern 'failures="0"')){$zero_errors_enc_const_br_dut=1; echo "Failed tests encountered"} - IF($zero_errors_enc_const_br_ref -ne 0 -or $zero_errors_enc_const_br_dut -ne 0 ) {echo "Errors encountered"; exit $EXIT_CODE_FAIL} -- GitLab From dff11a1e6f25634bc80c7dda8917f848cf989476 Mon Sep 17 00:00:00 2001 From: Charles Kinuthia Date: Wed, 18 Oct 2023 16:54:22 +0200 Subject: [PATCH 103/106] pwsh syntax fix --- .gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 0dba4924b9..a162ee0bb3 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1204,7 +1204,7 @@ test-be-release-pre-test: - echo "generate references" # encoder, constant brate - - testcase_timeout=240 # to be reviewed + - $testcase_timeout=240 # to be reviewed - $zero_errors_enc_const_br_ref=0 - ("& python -m pytest .\tests\codec_be_to_accepted_release\encoder\test_encoder_constant_bitrate.py --update_ref 1 --ref_encoder_path IVAS_cod_ref.exe --ref_decoder_path IVAS_dec_ref.exe --dut_encoder_path IVAS_cod_dut.exe --dut_decoder_path IVAS_dec_dut.exe --html=logs/report-enc-const_br-ref.html --self-contained-html --junit-xml=logs/report-junit-enc-const_br-ref.xml --testcase_timeout $testcase_timeout | out-null") | Invoke-Expression - IF (!(Select-String logs/report-junit-enc-const_br-ref.xml -Pattern 'errors="0"')){$zero_errors_enc_const_br_ref=1; echo "Errors encountered"} -- GitLab From f1e72e4fb8b0eb9366eb380d3cfaf0575b816b1b Mon Sep 17 00:00:00 2001 From: Charles Kinuthia Date: Thu, 19 Oct 2023 08:33:18 +0200 Subject: [PATCH 104/106] Use names from 'long-testv-on-runner' branch - masa metadata and masa file need to be reviewed. 1dir and 2dir currently mapped to the same file. --- .../codec_be_to_accepted_release/constants.py | 40 +++++++++---------- 1 file changed, 20 insertions(+), 20 deletions(-) diff --git a/tests/codec_be_to_accepted_release/constants.py b/tests/codec_be_to_accepted_release/constants.py index a8b8508f5f..10153f7dff 100644 --- a/tests/codec_be_to_accepted_release/constants.py +++ b/tests/codec_be_to_accepted_release/constants.py @@ -92,22 +92,22 @@ TESTVECTOR_FOR_INPUT_FORMAT = { "MASA_2TC_1DIR": "ltv{fs}_MASA_2TC_1DIR.wav", "MASA_1TC_2DIR": "ltv{fs}_MASA_1TC_2DIR.wav", "MASA_2TC_2DIR": "ltv{fs}_MASA_2TC_2DIR.wav", - "OMASA_ISM1_MASA_1TC_1DIR": "ltv{fs}_OMASA_1ISM_MASA_1TC_1DIR.wav", - "OMASA_ISM1_MASA_1TC_2DIR": "ltv{fs}_OMASA_1ISM_MASA_1TC_2DIR.wav", - "OMASA_ISM1_MASA_2TC_1DIR": "ltv{fs}_OMASA_1ISM_MASA_2TC_1DIR.wav", - "OMASA_ISM1_MASA_2TC_2DIR": "ltv{fs}_OMASA_1ISM_MASA_2TC_2DIR.wav", - "OMASA_ISM2_MASA_1TC_1DIR": "ltv{fs}_OMASA_2ISM_MASA_1TC_1DIR.wav", - "OMASA_ISM2_MASA_1TC_2DIR": "ltv{fs}_OMASA_2ISM_MASA_1TC_2DIR.wav", - "OMASA_ISM2_MASA_2TC_1DIR": "ltv{fs}_OMASA_2ISM_MASA_2TC_1DIR.wav", - "OMASA_ISM2_MASA_2TC_2DIR": "ltv{fs}_OMASA_2ISM_MASA_2TC_2DIR.wav", - "OMASA_ISM3_MASA_1TC_1DIR": "ltv{fs}_OMASA_3ISM_MASA_1TC_1DIR.wav", - "OMASA_ISM3_MASA_1TC_2DIR": "ltv{fs}_OMASA_3ISM_MASA_1TC_2DIR.wav", - "OMASA_ISM3_MASA_2TC_1DIR": "ltv{fs}_OMASA_3ISM_MASA_2TC_1DIR.wav", - "OMASA_ISM3_MASA_2TC_2DIR": "ltv{fs}_OMASA_3ISM_MASA_2TC_2DIR.wav", - "OMASA_ISM4_MASA_1TC_1DIR": "ltv{fs}_OMASA_4ISM_MASA_1TC_1DIR.wav", - "OMASA_ISM4_MASA_1TC_2DIR": "ltv{fs}_OMASA_4ISM_MASA_1TC_2DIR.wav", - "OMASA_ISM4_MASA_2TC_1DIR": "ltv{fs}_OMASA_4ISM_MASA_2TC_1DIR.wav", - "OMASA_ISM4_MASA_2TC_2DIR": "ltv{fs}_OMASA_4ISM_MASA_2TC_2DIR.wav", + "OMASA_ISM1_MASA_1TC_1DIR": "ltv{fs}_OMASA_1ISM_1TC.wav", # to be reviewed + "OMASA_ISM1_MASA_1TC_2DIR": "ltv{fs}_OMASA_1ISM_1TC.wav", + "OMASA_ISM1_MASA_2TC_1DIR": "ltv{fs}_OMASA_1ISM_2TC.wav", + "OMASA_ISM1_MASA_2TC_2DIR": "ltv{fs}_OMASA_1ISM_2TC.wav", + "OMASA_ISM2_MASA_1TC_1DIR": "ltv{fs}_OMASA_2ISM_1TC.wav", + "OMASA_ISM2_MASA_1TC_2DIR": "ltv{fs}_OMASA_2ISM_1TC.wav", + "OMASA_ISM2_MASA_2TC_1DIR": "ltv{fs}_OMASA_2ISM_2TC.wav", + "OMASA_ISM2_MASA_2TC_2DIR": "ltv{fs}_OMASA_2ISM_2TC.wav", + "OMASA_ISM3_MASA_1TC_1DIR": "ltv{fs}_OMASA_3ISM_1TC.wav", + "OMASA_ISM3_MASA_1TC_2DIR": "ltv{fs}_OMASA_3ISM_1TC.wav", + "OMASA_ISM3_MASA_2TC_1DIR": "ltv{fs}_OMASA_3ISM_2TC.wav", + "OMASA_ISM3_MASA_2TC_2DIR": "ltv{fs}_OMASA_3ISM_2TC.wav", + "OMASA_ISM4_MASA_1TC_1DIR": "ltv{fs}_OMASA_4ISM_1TC.wav", + "OMASA_ISM4_MASA_1TC_2DIR": "ltv{fs}_OMASA_4ISM_1TC.wav", + "OMASA_ISM4_MASA_2TC_1DIR": "ltv{fs}_OMASA_4ISM_2TC.wav", + "OMASA_ISM4_MASA_2TC_2DIR": "ltv{fs}_OMASA_4ISM_2TC.wav", "OSBA_ISM1_FOA": "ltv{fs}_OSBA_1ISM_FOA.wav", "OSBA_ISM1_HOA2": "ltv{fs}_OSBA_1ISM_HOA2.wav", "OSBA_ISM1_HOA3": "ltv{fs}_OSBA_1ISM_HOA3.wav", @@ -126,10 +126,10 @@ METADATA_FOR_INPUT_FORMAT = { "ISM2": ["ltvISM1.csv", "ltvISM2.csv"], "ISM3": ["ltvISM1.csv", "ltvISM2.csv", "ltvISM3.csv"], "ISM4": ["ltvISM1.csv", "ltvISM2.csv", "ltvISM3.csv", "ltvISM4.csv"], - "MASA_1TC_1DIR": ["ltv_MASA_1TC_1DIR.met"], - "MASA_2TC_1DIR": ["ltv_MASA_2TC_1DIR.met"], - "MASA_1TC_2DIR": ["ltv_MASA_1TC_2DIR.met"], - "MASA_2TC_2DIR": ["ltv_MASA_2TC_2DIR.met"], + "MASA_1TC_1DIR": ["ltv48_OMASA_1ISM_1TC.met"], # to be reviewed + "MASA_2TC_1DIR": ["ltv48_MASA2TC.met"], + "MASA_1TC_2DIR": ["ltv48_OMASA_2ISM_1TC.met"], + "MASA_2TC_2DIR": ["ltv48_MASA2TC.met"], } CMDL_OPTIONS_FOR_INPUT_FORMAT = { "STEREO": ["-stereo"], -- GitLab From d190a6f2a8916bc5b3f076058b4a996976d73402 Mon Sep 17 00:00:00 2001 From: Charles Kinuthia Date: Thu, 19 Oct 2023 13:07:25 +0200 Subject: [PATCH 105/106] update masa input filenames to those in 'long-testv-on-runner' --- tests/codec_be_to_accepted_release/constants.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/codec_be_to_accepted_release/constants.py b/tests/codec_be_to_accepted_release/constants.py index 10153f7dff..0f836040b7 100644 --- a/tests/codec_be_to_accepted_release/constants.py +++ b/tests/codec_be_to_accepted_release/constants.py @@ -88,10 +88,10 @@ TESTVECTOR_FOR_INPUT_FORMAT = { "ISM2": "ltv{fs}_2ISM.wav", "ISM3": "ltv{fs}_3ISM.wav", "ISM4": "ltv{fs}_4ISM.wav", - "MASA_1TC_1DIR": "ltv{fs}_MASA_1TC_1DIR.wav", - "MASA_2TC_1DIR": "ltv{fs}_MASA_2TC_1DIR.wav", - "MASA_1TC_2DIR": "ltv{fs}_MASA_1TC_2DIR.wav", - "MASA_2TC_2DIR": "ltv{fs}_MASA_2TC_2DIR.wav", + "MASA_1TC_1DIR": "ltv{fs}_MASA1TC.wav", + "MASA_2TC_1DIR": "ltv{fs}_MASA2TC.wav", + "MASA_1TC_2DIR": "ltv{fs}_MASA1TC.wav", + "MASA_2TC_2DIR": "ltv{fs}_MASA2TC.wav", "OMASA_ISM1_MASA_1TC_1DIR": "ltv{fs}_OMASA_1ISM_1TC.wav", # to be reviewed "OMASA_ISM1_MASA_1TC_2DIR": "ltv{fs}_OMASA_1ISM_1TC.wav", "OMASA_ISM1_MASA_2TC_1DIR": "ltv{fs}_OMASA_1ISM_2TC.wav", -- GitLab From 097ed75dee48a1b238b164e1f38dfed890fcaa7b Mon Sep 17 00:00:00 2001 From: Charles Kinuthia Date: Mon, 23 Oct 2023 11:29:37 +0200 Subject: [PATCH 106/106] use exit non-be exit code for non-be case --- .gitlab-ci.yml | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index a162ee0bb3..14eebac063 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1200,22 +1200,29 @@ test-be-release-pre-test: - mkdir tests\codec_be_to_accepted_release\ref - mkdir tests\codec_be_to_accepted_release\dut - # test generating references + # create references - echo "generate references" # encoder, constant brate - $testcase_timeout=240 # to be reviewed - $zero_errors_enc_const_br_ref=0 + - $zero_failures_ref=0 - ("& python -m pytest .\tests\codec_be_to_accepted_release\encoder\test_encoder_constant_bitrate.py --update_ref 1 --ref_encoder_path IVAS_cod_ref.exe --ref_decoder_path IVAS_dec_ref.exe --dut_encoder_path IVAS_cod_dut.exe --dut_decoder_path IVAS_dec_dut.exe --html=logs/report-enc-const_br-ref.html --self-contained-html --junit-xml=logs/report-junit-enc-const_br-ref.xml --testcase_timeout $testcase_timeout | out-null") | Invoke-Expression - - IF (!(Select-String logs/report-junit-enc-const_br-ref.xml -Pattern 'errors="0"')){$zero_errors_enc_const_br_ref=1; echo "Errors encountered"} - - IF (!(Select-String logs/report-junit-enc-const_br-ref.xml -Pattern 'failures="0"')){$zero_errors_enc_const_br_ref=1; echo "Failed tests encountered in ref generation"} + - IF (!(Select-String logs/report-junit-enc-const_br-ref.xml -Pattern 'errors="0"')){$zero_errors_enc_const_br_ref=1; echo "Errors encountered in ref generation"} + - IF (!(Select-String logs/report-junit-enc-const_br-ref.xml -Pattern 'failures="0"')){$zero_failures_ref=1; echo "Failed tests encountered in ref generation"} + + # test encoder dut + - echo "dut test encoder" - $zero_errors_enc_const_br_dut=0 + - $zero_failures_dut=0 - ("& python -m pytest .\tests\codec_be_to_accepted_release\encoder\test_encoder_constant_bitrate.py --ref_encoder_path IVAS_cod_ref.exe --ref_decoder_path IVAS_dec_ref.exe --dut_encoder_path IVAS_cod_dut.exe --dut_decoder_path IVAS_dec_dut.exe --html=logs/report-enc-const_br-dut.html --self-contained-html --junit-xml=logs/report-junit-enc-const_br-dut.xml --testcase_timeout $testcase_timeout | out-null") | Invoke-Expression - - IF (!(Select-String logs/report-junit-enc-const_br-dut.xml -Pattern 'errors="0"')){$zero_errors_enc_const_br_dut=1; echo "Errors encountered"} - - IF (!(Select-String logs/report-junit-enc-const_br-dut.xml -Pattern 'failures="0"')){$zero_errors_enc_const_br_dut=1; echo "Failed tests encountered"} + - IF (!(Select-String logs/report-junit-enc-const_br-dut.xml -Pattern 'errors="0"')){$zero_errors_enc_const_br_dut=1; echo "Errors encountered in dut"} + - IF (!(Select-String logs/report-junit-enc-const_br-dut.xml -Pattern 'failures="0"')){$zero_failures_dut=1; echo "Failed tests encountered in dut"} - IF($zero_errors_enc_const_br_ref -ne 0 -or $zero_errors_enc_const_br_dut -ne 0 ) {echo "Errors encountered"; exit $EXIT_CODE_FAIL} + - IF($zero_failures_ref -ne 0 ) {echo "Failed tests encountered in ref generation"; exit $EXIT_CODE_FAIL} + - IF($zero_failures_dut -ne 0 ) {echo "Non-bitexact cases encountered!"; exit $EXIT_CODE_NON_BE} artifacts: name: "test-be-to-release--$REF_COMMIT-to-$DUT_COMMIT--results" -- GitLab