From e601077c13339df3e3b4e3a3773152b855f0b22e Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Thu, 16 May 2024 13:55:49 +0200 Subject: [PATCH 1/7] add sanity check test for ext being be to transport format --- tests/conftest.py | 8 +-- tests/test_be_for_ext_outputs.py | 85 ++++++++++++++++++++++++++++++++ 2 files changed, 90 insertions(+), 3 deletions(-) create mode 100644 tests/test_be_for_ext_outputs.py diff --git a/tests/conftest.py b/tests/conftest.py index c975e409c0..6d93985243 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -312,6 +312,7 @@ class EncoderFrontend: pca: Optional[bool] = None, quiet_mode: Optional[bool] = True, add_option_list: Optional[list] = None, + run_dir: Optional[Path] = None, ) -> None: command = [self._path] @@ -347,7 +348,7 @@ class EncoderFrontend: try: result = run( - command, capture_output=True, check=False, timeout=self.timeout + command, capture_output=True, check=False, timeout=self.timeout, cwd=run_dir ) except TimeoutExpired: pytest.fail(f"{self._type} encoder run timed out after {self.timeout}s.") @@ -486,6 +487,7 @@ class DecoderFrontend: quiet_mode: Optional[bool] = True, plc_file: Optional[Path] = None, add_option_list: Optional[list] = None, + run_dir: Optional[Path] = None, ) -> None: command = [self._path] @@ -521,7 +523,7 @@ class DecoderFrontend: try: if not os.path.exists(str(input_bitstream_path) + eid_output_suffix): - result = run(eid_command, check=True) + result = run(eid_command, check=True, cwd=run_dir) except Exception: pytest.fail("eid-xor operation failed!") @@ -545,7 +547,7 @@ class DecoderFrontend: try: result = run( - command, capture_output=True, check=False, timeout=self.timeout + command, capture_output=True, check=False, timeout=self.timeout, cwd=run_dir ) except TimeoutExpired: pytest.fail(f"{self._type} decoder run timed out after {self.timeout}s.") diff --git a/tests/test_be_for_ext_outputs.py b/tests/test_be_for_ext_outputs.py new file mode 100644 index 0000000000..a2c7522dd1 --- /dev/null +++ b/tests/test_be_for_ext_outputs.py @@ -0,0 +1,85 @@ +import pytest +import pathlib +from filecmp import cmp +from tempfile import TemporaryDirectory +from .conftest import EncoderFrontend, DecoderFrontend + +HERE = pathlib.Path(__file__).parent.absolute() +TESTV_DIR = HERE.parent.joinpath("scripts/testv") + + +INPUT_FORMATS = [ + "stereo", + "5_1", + "5_1_2", + "5_1_4", + "7_1", + "7_1_4", + "FOA", + "HOA2", + "HOA3", +] + +INPUT_FILES = [ + "stvST48c.wav", + "stv51MC48c.wav", + "stv512MC48c.wav", + "stv514MC48c.wav", + "stv71MC48c.wav", + "stv714MC48c.wav", + "stvFOA48c.wav", + "stv2OA48c.wav", + "stv3OA48c.wav", + ] + +# TODO: add higher BRs +BITRATES = [13200, 16400, 24400, 32000, 48000, 64000, 80000, 96000, 128000, 160000, 192000, 256000] + + +def get_options(inp_format): + options = list() + if inp_format == "stereo": + options.append("-stereo") + elif "_" in inp_format: + # this indicates MC mode + options.extend(["-mc", inp_format]) + else: + try: + ambi_order = int(inp_format[-1]) + except ValueError: + ambi_order = 1 + options.extend(["-sba", str(ambi_order)]) + + return options + +@pytest.mark.parametrize("inp_format,inp_file", zip(INPUT_FORMATS, INPUT_FILES)) +@pytest.mark.parametrize("bitrate", BITRATES) +def test_be_for_ext_output( + inp_format, + inp_file, + bitrate, + dut_encoder_frontend: EncoderFrontend, + dut_decoder_frontend: DecoderFrontend, + ): + + with TemporaryDirectory() as tmp_dir: + tmp_dir = pathlib.Path(tmp_dir) + + # run encoder + bitstream_file = tmp_dir.joinpath(f"{inp_format}.192").absolute() + sampling_rate = 48 + input_file = TESTV_DIR.joinpath(inp_file) + options = get_options(inp_format) + dut_encoder_frontend.run(bitrate, sampling_rate, input_file, bitstream_file, add_option_list=options, run_dir=tmp_dir) + + # run decoder with "native" output format + output_native = tmp_dir.joinpath(f"{inp_format}-to-{inp_format}.wav").absolute() + dut_decoder_frontend.run(inp_format, sampling_rate, bitstream_file, output_native) + + # run decoder with "EXT" output + output_ext = tmp_dir.joinpath(f"{inp_format}-to-EXT.wav").absolute() + dut_decoder_frontend.run("EXT", sampling_rate, bitstream_file, output_ext) + + # check for BE + if not cmp(output_native, output_ext): + pytest.fail(f"{inp_format} -> {inp_format} and {inp_format} -> EXT not BE!") -- GitLab From 7789f4cca2348257678c5dea6b1f98e7e07fe1bd Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Thu, 16 May 2024 14:35:25 +0200 Subject: [PATCH 2/7] add higher bitrates for non-stereo format --- tests/test_be_for_ext_outputs.py | 41 ++++++++++++++++++-------------- 1 file changed, 23 insertions(+), 18 deletions(-) diff --git a/tests/test_be_for_ext_outputs.py b/tests/test_be_for_ext_outputs.py index a2c7522dd1..c4d2265cf9 100644 --- a/tests/test_be_for_ext_outputs.py +++ b/tests/test_be_for_ext_outputs.py @@ -1,5 +1,6 @@ import pytest import pathlib +import itertools from filecmp import cmp from tempfile import TemporaryDirectory from .conftest import EncoderFrontend, DecoderFrontend @@ -33,7 +34,7 @@ INPUT_FILES = [ ] # TODO: add higher BRs -BITRATES = [13200, 16400, 24400, 32000, 48000, 64000, 80000, 96000, 128000, 160000, 192000, 256000] +BITRATES = [13200, 16400, 24400, 32000, 48000, 64000, 80000, 96000, 128000, 160000, 192000, 256000, 384000, 512000] def get_options(inp_format): @@ -52,6 +53,7 @@ def get_options(inp_format): return options + @pytest.mark.parametrize("inp_format,inp_file", zip(INPUT_FORMATS, INPUT_FILES)) @pytest.mark.parametrize("bitrate", BITRATES) def test_be_for_ext_output( @@ -62,24 +64,27 @@ def test_be_for_ext_output( dut_decoder_frontend: DecoderFrontend, ): - with TemporaryDirectory() as tmp_dir: - tmp_dir = pathlib.Path(tmp_dir) + if inp_format == "stereo" and bitrate > 256000: + pytest.skip("Invalid bitrate for Stereo") + + with TemporaryDirectory() as tmp_dir: + tmp_dir = pathlib.Path(tmp_dir) - # run encoder - bitstream_file = tmp_dir.joinpath(f"{inp_format}.192").absolute() - sampling_rate = 48 - input_file = TESTV_DIR.joinpath(inp_file) - options = get_options(inp_format) - dut_encoder_frontend.run(bitrate, sampling_rate, input_file, bitstream_file, add_option_list=options, run_dir=tmp_dir) + # run encoder + bitstream_file = tmp_dir.joinpath(f"{inp_format}.192").absolute() + sampling_rate = 48 + input_file = TESTV_DIR.joinpath(inp_file) + options = get_options(inp_format) + dut_encoder_frontend.run(bitrate, sampling_rate, input_file, bitstream_file, add_option_list=options, run_dir=tmp_dir) - # run decoder with "native" output format - output_native = tmp_dir.joinpath(f"{inp_format}-to-{inp_format}.wav").absolute() - dut_decoder_frontend.run(inp_format, sampling_rate, bitstream_file, output_native) + # run decoder with "native" output format + output_native = tmp_dir.joinpath(f"{inp_format}-to-{inp_format}.wav").absolute() + dut_decoder_frontend.run(inp_format, sampling_rate, bitstream_file, output_native) - # run decoder with "EXT" output - output_ext = tmp_dir.joinpath(f"{inp_format}-to-EXT.wav").absolute() - dut_decoder_frontend.run("EXT", sampling_rate, bitstream_file, output_ext) + # run decoder with "EXT" output + output_ext = tmp_dir.joinpath(f"{inp_format}-to-EXT.wav").absolute() + dut_decoder_frontend.run("EXT", sampling_rate, bitstream_file, output_ext) - # check for BE - if not cmp(output_native, output_ext): - pytest.fail(f"{inp_format} -> {inp_format} and {inp_format} -> EXT not BE!") + # check for BE + if not cmp(output_native, output_ext): + pytest.fail(f"{inp_format} -> {inp_format} and {inp_format} -> EXT not BE!") -- GitLab From bfefdc77ff825f7a8abd51d44bcc31dddeb2cb35 Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Thu, 16 May 2024 14:38:15 +0200 Subject: [PATCH 3/7] use cut files --- tests/test_be_for_ext_outputs.py | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/tests/test_be_for_ext_outputs.py b/tests/test_be_for_ext_outputs.py index c4d2265cf9..cddd749adb 100644 --- a/tests/test_be_for_ext_outputs.py +++ b/tests/test_be_for_ext_outputs.py @@ -22,15 +22,15 @@ INPUT_FORMATS = [ ] INPUT_FILES = [ - "stvST48c.wav", - "stv51MC48c.wav", - "stv512MC48c.wav", - "stv514MC48c.wav", - "stv71MC48c.wav", - "stv714MC48c.wav", - "stvFOA48c.wav", - "stv2OA48c.wav", - "stv3OA48c.wav", + "stvST48c_cut.wav", + "stv51MC48c_cut.wav", + "stv512MC48c_cut.wav", + "stv514MC48c_cut.wav", + "stv71MC48c_cut.wav", + "stv714MC48c_cut.wav", + "stvFOA48c_cut.wav", + "stv2OA48c_cut.wav", + "stv3OA48c_cut.wav", ] # TODO: add higher BRs -- GitLab From ecc337dff70041eeb0af4b598da44e7575bda4b1 Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Thu, 16 May 2024 14:42:18 +0200 Subject: [PATCH 4/7] add stereo files to create_short_testvectors.py --- tests/create_short_testvectors.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/create_short_testvectors.py b/tests/create_short_testvectors.py index 87d05e001e..b90771cca1 100755 --- a/tests/create_short_testvectors.py +++ b/tests/create_short_testvectors.py @@ -50,6 +50,7 @@ from pyaudio3dtools import audiofile CUT_FROM = "0.0" GAIN = "1.0" FILE_IDS = [ + "stvST", "stv51MC", "stv71MC", "stv512MC", -- GitLab From f81e4ace4e605e7bca5ae0e62eda37c343679bc0 Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Thu, 16 May 2024 14:45:17 +0200 Subject: [PATCH 5/7] add job to CI config --- .gitlab-ci.yml | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 541aee3207..b0efa75256 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1188,6 +1188,28 @@ check-bitexactness-hrtf-rom-and-file: expose_as: "logs-hrtf-loading" expire_in: "5 days" +check-bitexactness-ext-and-transport-format: + extends: + - .test-job-linux + - .rules-merge-request + stage: test + needs: ["build-codec-linux-cmake"] + timeout: "5 minutes" + script: + - *print-common-info + - cmake . + - make -j + - python3 tests/create_short_testvectors.py --cut_len 1.0 + - python3 -m pytest tests/test_be_for_ext_outputs.py --html=report.html --junit-xml=report-junit.xml --self-contained-html + artifacts: + paths: + - report.html + - report-junit.xml + when: always + name: "$CI_JOB_NAME--$CI_MERGE_REQUEST_ID--sha-$CI_COMMIT_SHA--ext-sanity-check" + expose_as: "logs-ext-sanity-check" + expire_in: "5 days" + # --------------------------------------------------------------- # Test jobs for main branch -- GitLab From 32f693ad7f9ac1f95a2c6db5f7a385a6b1449b7a Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Wed, 22 May 2024 11:15:30 +0200 Subject: [PATCH 6/7] cleanup and more sane if clause --- tests/test_be_for_ext_outputs.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/tests/test_be_for_ext_outputs.py b/tests/test_be_for_ext_outputs.py index cddd749adb..d3a69d2978 100644 --- a/tests/test_be_for_ext_outputs.py +++ b/tests/test_be_for_ext_outputs.py @@ -1,6 +1,5 @@ import pytest import pathlib -import itertools from filecmp import cmp from tempfile import TemporaryDirectory from .conftest import EncoderFrontend, DecoderFrontend @@ -20,6 +19,8 @@ INPUT_FORMATS = [ "HOA2", "HOA3", ] +MC_FORMATS = INPUT_FORMATS[1:6] +SBA_FORMATS = INPUT_FORMATS[6:] INPUT_FILES = [ "stvST48c_cut.wav", @@ -33,7 +34,6 @@ INPUT_FILES = [ "stv3OA48c_cut.wav", ] -# TODO: add higher BRs BITRATES = [13200, 16400, 24400, 32000, 48000, 64000, 80000, 96000, 128000, 160000, 192000, 256000, 384000, 512000] @@ -41,15 +41,17 @@ def get_options(inp_format): options = list() if inp_format == "stereo": options.append("-stereo") - elif "_" in inp_format: + elif inp_format in MC_FORMATS: # this indicates MC mode options.extend(["-mc", inp_format]) - else: + elif inp_format in SBA_FORMATS: try: ambi_order = int(inp_format[-1]) except ValueError: ambi_order = 1 options.extend(["-sba", str(ambi_order)]) + else: + assert f"Can't handle input format {inp_format}" return options -- GitLab From 7828f576c6400a1193bbb66e04e97b6924d9bd4d Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Wed, 22 May 2024 11:16:09 +0200 Subject: [PATCH 7/7] format file --- tests/test_be_for_ext_outputs.py | 83 ++++++++++++++++++++------------ 1 file changed, 53 insertions(+), 30 deletions(-) diff --git a/tests/test_be_for_ext_outputs.py b/tests/test_be_for_ext_outputs.py index d3a69d2978..9be0966acd 100644 --- a/tests/test_be_for_ext_outputs.py +++ b/tests/test_be_for_ext_outputs.py @@ -9,32 +9,47 @@ TESTV_DIR = HERE.parent.joinpath("scripts/testv") INPUT_FORMATS = [ - "stereo", - "5_1", - "5_1_2", - "5_1_4", - "7_1", - "7_1_4", - "FOA", - "HOA2", - "HOA3", + "stereo", + "5_1", + "5_1_2", + "5_1_4", + "7_1", + "7_1_4", + "FOA", + "HOA2", + "HOA3", ] MC_FORMATS = INPUT_FORMATS[1:6] SBA_FORMATS = INPUT_FORMATS[6:] - + INPUT_FILES = [ - "stvST48c_cut.wav", - "stv51MC48c_cut.wav", - "stv512MC48c_cut.wav", - "stv514MC48c_cut.wav", - "stv71MC48c_cut.wav", - "stv714MC48c_cut.wav", - "stvFOA48c_cut.wav", - "stv2OA48c_cut.wav", - "stv3OA48c_cut.wav", - ] + "stvST48c_cut.wav", + "stv51MC48c_cut.wav", + "stv512MC48c_cut.wav", + "stv514MC48c_cut.wav", + "stv71MC48c_cut.wav", + "stv714MC48c_cut.wav", + "stvFOA48c_cut.wav", + "stv2OA48c_cut.wav", + "stv3OA48c_cut.wav", +] -BITRATES = [13200, 16400, 24400, 32000, 48000, 64000, 80000, 96000, 128000, 160000, 192000, 256000, 384000, 512000] +BITRATES = [ + 13200, + 16400, + 24400, + 32000, + 48000, + 64000, + 80000, + 96000, + 128000, + 160000, + 192000, + 256000, + 384000, + 512000, +] def get_options(inp_format): @@ -59,13 +74,12 @@ def get_options(inp_format): @pytest.mark.parametrize("inp_format,inp_file", zip(INPUT_FORMATS, INPUT_FILES)) @pytest.mark.parametrize("bitrate", BITRATES) def test_be_for_ext_output( - inp_format, - inp_file, - bitrate, - dut_encoder_frontend: EncoderFrontend, - dut_decoder_frontend: DecoderFrontend, - ): - + inp_format, + inp_file, + bitrate, + dut_encoder_frontend: EncoderFrontend, + dut_decoder_frontend: DecoderFrontend, +): if inp_format == "stereo" and bitrate > 256000: pytest.skip("Invalid bitrate for Stereo") @@ -77,11 +91,20 @@ def test_be_for_ext_output( sampling_rate = 48 input_file = TESTV_DIR.joinpath(inp_file) options = get_options(inp_format) - dut_encoder_frontend.run(bitrate, sampling_rate, input_file, bitstream_file, add_option_list=options, run_dir=tmp_dir) + dut_encoder_frontend.run( + bitrate, + sampling_rate, + input_file, + bitstream_file, + add_option_list=options, + run_dir=tmp_dir, + ) # run decoder with "native" output format output_native = tmp_dir.joinpath(f"{inp_format}-to-{inp_format}.wav").absolute() - dut_decoder_frontend.run(inp_format, sampling_rate, bitstream_file, output_native) + dut_decoder_frontend.run( + inp_format, sampling_rate, bitstream_file, output_native + ) # run decoder with "EXT" output output_ext = tmp_dir.joinpath(f"{inp_format}-to-EXT.wav").absolute() -- GitLab