From fb478d63580adead46a20bda91c2279a8f3a64b2 Mon Sep 17 00:00:00 2001 From: knj Date: Wed, 9 Nov 2022 11:23:35 +0100 Subject: [PATCH 1/3] add pytest cases for be comparison on mergerequest --- tests/renderer/constants.py | 2 + tests/renderer/test_renderer_be_comparison.py | 201 ++++++++++++++++++ tests/renderer/utils.py | 16 +- 3 files changed, 216 insertions(+), 3 deletions(-) create mode 100644 tests/renderer/test_renderer_be_comparison.py diff --git a/tests/renderer/constants.py b/tests/renderer/constants.py index 1cefbda25d..cb2d580ec3 100644 --- a/tests/renderer/constants.py +++ b/tests/renderer/constants.py @@ -40,6 +40,8 @@ CUSTOM_LAYOUT_DIR = SCRIPTS_DIR.joinpath("ls_layouts") HR_TRAJECTORY_DIR = SCRIPTS_DIR.joinpath("trajectories") TESTV_DIR = SCRIPTS_DIR.joinpath("testv") +BIN_SUFFIX_MERGETARGET = "_ref" + """ Encoder commandline template """ IVAS_COD_CMD = [ str(TESTS_DIR.parent.parent.joinpath("IVAS_cod")), diff --git a/tests/renderer/test_renderer_be_comparison.py b/tests/renderer/test_renderer_be_comparison.py new file mode 100644 index 0000000000..5d89136a89 --- /dev/null +++ b/tests/renderer/test_renderer_be_comparison.py @@ -0,0 +1,201 @@ +#!/usr/bin/env python3 + +""" + (C) 2022 Baseline Development Group with portions copyright Dolby International AB, Ericsson AB, + Fraunhofer-Gesellschaft zur Foerderung der angewandten Forschung e.V., Huawei Technologies Co. LTD., + Koninklijke Philips N.V., Nippon Telegraph and Telephone Corporation, Nokia Technologies OY, Orange, + Panasonic Corporation, Qualcomm Technologies, Inc., VoiceAge Corporation. All Rights Reserved. + + This software is protected by copyright law and by international treaties. + The Baseline Development Group consisting of Dolby International AB, Ericsson AB, + Fraunhofer-Gesellschaft zur Foerderung der angewandten Forschung e.V., Huawei Technologies Co. LTD., + Koninklijke Philips N.V., Nippon Telegraph and Telephone Corporation, Nokia Technologies OY, Orange, + Panasonic Corporation, Qualcomm Technologies, Inc., and VoiceAge Corporation retain full ownership + rights in their respective contributions in the software. No license of any kind, including but not + limited to patent license, of any foregoing parties is hereby granted by implication, estoppel or + otherwise. + + This software is provided "AS IS", without any express or implied warranties. The software is in the + development stage. It is intended exclusively for experts who have experience with such software and + solely for the purpose of inspection. All implied warranties of non-infringement, merchantability + and/or fitness for a particular purpose are hereby disclaimed and excluded. + + Any dispute, controversy or claim arising under or in relation to providing this software shall be + submitted to and settled by the final, binding jurisdiction of the courts of Munich, Germany in + accordance with the laws of the Federal Republic of Germany excluding its conflict of law rules and + the United Nations Convention on Contracts on the International Sales of Goods. +""" + + +import pytest +from .utils import * + + +""" Ambisonics """ + + +@pytest.mark.parametrize("out_fmt", OUTPUT_FORMATS) +@pytest.mark.parametrize("in_fmt", INPUT_FORMATS_AMBI) +def test_ambisonics(test_info, in_fmt, out_fmt): + compare_renderer_vs_mergetarget(test_info, in_fmt, out_fmt) + + +@pytest.mark.parametrize("out_fmt", OUTPUT_FORMATS_BINAURAL) +@pytest.mark.parametrize("in_fmt", INPUT_FORMATS_AMBI) +def test_ambisonics_binaural_static(test_info, in_fmt, out_fmt): + compare_renderer_vs_mergetarget(test_info, in_fmt, out_fmt) + + +@pytest.mark.parametrize("trj_file", HR_TRAJECTORIES_TO_TEST) +@pytest.mark.parametrize("out_fmt", OUTPUT_FORMATS_BINAURAL) +@pytest.mark.parametrize("in_fmt", INPUT_FORMATS_AMBI) +def test_ambisonics_binaural_headrotation(test_info, in_fmt, out_fmt, trj_file): + compare_renderer_vs_mergetarget( + test_info, + in_fmt, + out_fmt, + trj_file=HR_TRAJECTORY_DIR.joinpath(f"{trj_file}.csv"), + ) + + +""" Multichannel """ + + +@pytest.mark.parametrize("out_fmt", OUTPUT_FORMATS) +@pytest.mark.parametrize("in_fmt", INPUT_FORMATS_MC) +def test_multichannel(test_info, in_fmt, out_fmt): + compare_renderer_vs_mergetarget(test_info, in_fmt, out_fmt) + + +@pytest.mark.parametrize("out_fmt", OUTPUT_FORMATS_BINAURAL) +@pytest.mark.parametrize("in_fmt", INPUT_FORMATS_MC) +def test_multichannel_binaural_static(test_info, in_fmt, out_fmt): + if in_fmt in ["MONO", "STEREO"]: + pytest.skip("MONO or STEREO to Binaural rendering unsupported") + + compare_renderer_vs_mergetarget(test_info, in_fmt, out_fmt) + + +@pytest.mark.parametrize("trj_file", HR_TRAJECTORIES_TO_TEST) +@pytest.mark.parametrize("out_fmt", OUTPUT_FORMATS_BINAURAL) +@pytest.mark.parametrize("in_fmt", INPUT_FORMATS_MC) +def test_multichannel_binaural_headrotation(test_info, in_fmt, out_fmt, trj_file): + if in_fmt in ["MONO", "STEREO"]: + pytest.skip("MONO or STEREO to Binaural rendering unsupported") + + compare_renderer_vs_mergetarget( + test_info, + in_fmt, + out_fmt, + trj_file=HR_TRAJECTORY_DIR.joinpath(f"{trj_file}.csv"), + ) + + +""" ISM """ + + +@pytest.mark.parametrize("out_fmt", OUTPUT_FORMATS) +@pytest.mark.parametrize("in_fmt", INPUT_FORMATS_ISM) +def test_ism(test_info, in_fmt, out_fmt): + compare_renderer_vs_mergetarget( + test_info, in_fmt, out_fmt, in_meta_files=FORMAT_TO_METADATA_FILES[in_fmt] + ) + + +@pytest.mark.parametrize("out_fmt", OUTPUT_FORMATS_BINAURAL) +@pytest.mark.parametrize("in_fmt", INPUT_FORMATS_ISM) +def test_ism_binaural_static(test_info, in_fmt, out_fmt): + try: + in_meta_files = FORMAT_TO_METADATA_FILES[in_fmt] + except: + in_meta_files = None + + compare_renderer_vs_mergetarget( + test_info, in_fmt, out_fmt, in_meta_files=in_meta_files + ) + + +@pytest.mark.parametrize("trj_file", HR_TRAJECTORIES_TO_TEST) +@pytest.mark.parametrize("out_fmt", OUTPUT_FORMATS_BINAURAL) +@pytest.mark.parametrize("in_fmt", INPUT_FORMATS_ISM) +def test_ism_binaural_headrotation(test_info, in_fmt, out_fmt, trj_file): + try: + in_meta_files = FORMAT_TO_METADATA_FILES[in_fmt] + except: + in_meta_files = None + + compare_renderer_vs_mergetarget( + test_info, + in_fmt, + out_fmt, + trj_file=HR_TRAJECTORY_DIR.joinpath(f"{trj_file}.csv"), + in_meta_files=in_meta_files, + ) + + +""" MASA """ +# TODO: MASA inputs not supported yet + +""" Custom loudspeaker layouts """ + + +@pytest.mark.parametrize("out_fmt", OUTPUT_FORMATS) +@pytest.mark.parametrize("in_layout", CUSTOM_LS_TO_TEST) +def test_custom_ls_input(test_info, in_layout, out_fmt): + compare_renderer_vs_mergetarget( + test_info, CUSTOM_LAYOUT_DIR.joinpath(f"{in_layout}.txt"), out_fmt + ) + + +@pytest.mark.parametrize("out_fmt", CUSTOM_LS_TO_TEST) +@pytest.mark.parametrize("in_fmt", OUTPUT_FORMATS) +def test_custom_ls_output(test_info, in_fmt, out_fmt): + compare_renderer_vs_mergetarget( + test_info, in_fmt, CUSTOM_LAYOUT_DIR.joinpath(f"{out_fmt}.txt") + ) + + +@pytest.mark.parametrize("out_fmt", CUSTOM_LS_TO_TEST) +@pytest.mark.parametrize("in_fmt", CUSTOM_LS_TO_TEST) +def test_custom_ls_input_output(test_info, in_fmt, out_fmt): + compare_renderer_vs_mergetarget( + test_info, + CUSTOM_LAYOUT_DIR.joinpath(f"{in_fmt}.txt"), + CUSTOM_LAYOUT_DIR.joinpath(f"{out_fmt}.txt"), + ) + + +@pytest.mark.parametrize("out_fmt", OUTPUT_FORMATS_BINAURAL) +@pytest.mark.parametrize("in_layout", CUSTOM_LS_TO_TEST) +def test_custom_ls_input_binaural(test_info, in_layout, out_fmt): + compare_renderer_vs_mergetarget( + test_info, + CUSTOM_LAYOUT_DIR.joinpath(f"{in_layout}.txt"), + out_fmt, + ) + + +@pytest.mark.parametrize("trj_file", HR_TRAJECTORIES_TO_TEST) +@pytest.mark.parametrize("out_fmt", OUTPUT_FORMATS_BINAURAL) +@pytest.mark.parametrize("in_layout", CUSTOM_LS_TO_TEST) +def test_custom_ls_input_binaural_headrotation(test_info, in_layout, out_fmt, trj_file): + compare_renderer_vs_mergetarget( + test_info, + CUSTOM_LAYOUT_DIR.joinpath(f"{in_layout}.txt"), + out_fmt, + trj_file=HR_TRAJECTORY_DIR.joinpath(f"{trj_file}.csv"), + ) + + +""" Metadata / scene description input """ + + +@pytest.mark.parametrize("out_fmt", OUTPUT_FORMATS) +@pytest.mark.parametrize("in_fmt", METADATA_SCENES_TO_TEST) +def test_metadata(test_info, in_fmt, out_fmt): + compare_renderer_vs_mergetarget( + test_info, + "META", + out_fmt, + metadata_input=TEST_VECTOR_DIR.joinpath(f"{in_fmt}.txt"), + ) diff --git a/tests/renderer/utils.py b/tests/renderer/utils.py index 9811d73500..3fbf6521a6 100644 --- a/tests/renderer/utils.py +++ b/tests/renderer/utils.py @@ -151,6 +151,7 @@ def run_renderer( metadata_input: Optional[str] = None, in_meta_files: Optional[list] = None, trj_file: Optional[str] = None, + binary_suffix: str = "", ) -> Tuple[np.ndarray, int]: """CuT creation with standalone renderer""" if trj_file is not None: @@ -175,6 +176,9 @@ def run_renderer( out_file = str(OUTPUT_PATH_CUT.joinpath(f"{in_name}_to_{out_name}{trj_name}.wav")) + # append suffix to binary filename + RENDERER_CMD[0] += binary_suffix + cmd = RENDERER_CMD[:] cmd[2] = str(in_file) cmd[4] = str(in_fmt) @@ -259,9 +263,7 @@ def run_crend_unittest( in_file = FORMAT_TO_FILE[in_fmt] in_name = in_fmt - out_file = str( - OUTPUT_PATH_REF.joinpath(f"{in_name}_to_{out_name}{trj_name}.wav") - ) + out_file = str(OUTPUT_PATH_REF.joinpath(f"{in_name}_to_{out_name}{trj_name}.wav")) cmd = CREND_CMD[:] cmd[6] = FORMAT_TO_CREND_FORMAT[str(in_fmt)] @@ -384,6 +386,14 @@ def run_pyscripts( return pyaudio3dtools.audiofile.readfile(out_file) +def compare_renderer_vs_mergetarget(test_info, in_fmt, out_fmt, **kwargs): + ref, ref_fs = run_renderer(in_fmt, out_fmt, **kwargs) + cut, cut_fs = run_renderer( + in_fmt, out_fmt, binary_suffix=BIN_SUFFIX_MERGETARGET, **kwargs + ) + check_BE(test_info, ref, ref_fs, cut, cut_fs) + + def compare_renderer_vs_pyscripts(test_info, in_fmt, out_fmt, **kwargs): ref, ref_fs = run_pyscripts(in_fmt, out_fmt, **kwargs) cut, cut_fs = run_renderer(in_fmt, out_fmt, **kwargs) -- GitLab From f8a289fa3814cdd689dca3ab41427fd1becf833a Mon Sep 17 00:00:00 2001 From: knj Date: Wed, 9 Nov 2022 11:31:02 +0100 Subject: [PATCH 2/3] use new test for BE comparison --- .gitlab-ci.yml | 35 +++++++++++++++++++---------------- 1 file changed, 19 insertions(+), 16 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 0a3b6145d2..e7e2754712 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -408,46 +408,49 @@ external-renderer-cmake-msan-pytest: - .test-job-linux - .rules-merge-request needs: ["build-codec-linux-make"] - timeout: "12 minutes" + # TODO: set reasonable timeout, will most likely take less + timeout: "20 minutes" stage: compare script: - *print-common-info # some helper variables - "|| true" to prevent failures from grep not finding anything - non_be_flag=$(echo $CI_MERGE_REQUEST_TITLE | grep -c --ignore-case "\[rend(erer)*[ -]*non[ -]*be\]") || true - - ref_using_main=$(echo $CI_MERGE_REQUEST_TITLE | grep -c --ignore-case "\[ref[ -]*using[ -]*main\]") || true + # TODO: needs splitting the test between reference and cut generation + #- ref_using_main=$(echo $CI_MERGE_REQUEST_TITLE | grep -c --ignore-case "\[ref[ -]*using[ -]*main\]") || true # store the current commit hash - source_branch_commit_sha=$(git rev-parse HEAD) - # TODO: implement ref_using_main - - *mr-fetch-target-branch - *mr-get-target-commit - git checkout $target_commit - # run ext renderer test on target branch + # build reference binaries - make -j IVAS_rend - - make -j unittests - - make -j --directory scripts/td_object_renderer/object_renderer_standalone - - python3 -m pytest -q --log-level ERROR -n auto -rA --junit-xml=report-junit.xml tests/renderer/test_renderer.py - - mv tests/renderer/ref tests/renderer/ref_main + - mv IVAS_rend IVAS_rend_ref - # run ext renderer test on branch to be merged + # back to source branch - git checkout $source_branch_commit_sha - make clean - make -j IVAS_rend - - make -j unittests - - make -j --directory scripts/td_object_renderer/object_renderer_standalone - - python3 -m pytest -q --log-level ERROR -n auto -rA --junit-xml=report-junit.xml tests/renderer/test_renderer.py - - python scripts/batch_comp_audio.py ./tests/renderer/ref_main ./tests/renderer/ref | tee comparison_result.txt - # TODO: implmement actual comparison + + # run test + - exit_code=0 + - python3 -m pytest -q --log-level ERROR -n auto -rA --junit-xml=report-junit.xml tests/renderer/test_renderer_be_comparison.py || exit_code=$? + - zero_errors=$(cat report-junit.xml | grep -c 'errors="0"') || true + + - *merge-request-comparison-check + + allow_failure: + exit_codes: + - 123 artifacts: name: "mr-$CI_MERGE_REQUEST_IID--sha-$CI_COMMIT_SHORT_SHA--job-$CI_JOB_NAME--results" when: always paths: - report-junit.xml - expose_as: "external renderer pytest on merge request results" + expose_as: "pytest external renderer results" reports: junit: - report-junit.xml -- GitLab From 9a39ae657387dc85d5db467c786799e6189096ac Mon Sep 17 00:00:00 2001 From: knj Date: Wed, 9 Nov 2022 11:36:38 +0100 Subject: [PATCH 3/3] fix mixup of ref and cut --- tests/renderer/utils.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/renderer/utils.py b/tests/renderer/utils.py index 3fbf6521a6..928998ecdc 100644 --- a/tests/renderer/utils.py +++ b/tests/renderer/utils.py @@ -387,10 +387,10 @@ def run_pyscripts( def compare_renderer_vs_mergetarget(test_info, in_fmt, out_fmt, **kwargs): - ref, ref_fs = run_renderer(in_fmt, out_fmt, **kwargs) - cut, cut_fs = run_renderer( + ref, ref_fs = run_renderer( in_fmt, out_fmt, binary_suffix=BIN_SUFFIX_MERGETARGET, **kwargs ) + cut, cut_fs = run_renderer(in_fmt, out_fmt, **kwargs) check_BE(test_info, ref, ref_fs, cut, cut_fs) -- GitLab