diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 018b273ae6005bb04f87e523e9132d70c9de07bc..3fbaf04204e3b111aca6f8cacf8b7d2c5fe8fd47 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -31,6 +31,7 @@ variables: TESTCASE_TIMEOUT_LTV_SANITIZERS: 2400 BASOP_REFERENCE_BRANCH: "ivas-float-update" SCALE_FACTOR: "3.162" + PYTEST_ARGS: "" default: @@ -132,6 +133,15 @@ stages: - sed -i.bak -e "s/\/\*\ *\(#define\ *DEBUGGING\ *\)\*\//\1/g" lib_com/options.h - sed -i.bak -e "s/\/\/\ *\(#define\ *DEBUGGING\ *\)/\1/g" lib_com/options.h +.get-basop-float-reference: &get-basop-float-reference + - git clone -b $BASOP_REFERENCE_BRANCH https://forge.3gpp.org/rep/sa4/audio/ivas-basop.git --single-branch --depth 1 ivas-basop + - cd ivas-basop + - git status + - make -j + - cd - + - cp ivas-basop/IVAS_cod ./ + - cp ivas-basop/IVAS_dec ./ + .merge-request-comparison-setup-codec: &merge-request-comparison-setup-codec ### build test binaries, initial clean for paranoia reasons - *disable-debugging-macro @@ -279,10 +289,24 @@ stages: when: never - when: on_success -.rules-merge-request: +.rules-merge-request-to-main: + extends: .rules-basis + rules: + - if: $CI_PIPELINE_SOURCE == 'merge_request_event' && $CI_MERGE_REQUEST_TARGET_BRANCH_NAME == "main" + - if: $CI_PIPELINE_SOURCE == 'push' + when: never + +.rules-merge-request-to-basop-ci-branch: + extends: .rules-basis + rules: + - if: $CI_PIPELINE_SOURCE == 'merge_request_event' && $CI_MERGE_REQUEST_TARGET_BRANCH_NAME == "basop-ci-branch" + - if: $CI_PIPELINE_SOURCE == 'push' + when: never + +.rules-merge-request-to-basop-ci-branch-or-main: extends: .rules-basis rules: - - if: $CI_PIPELINE_SOURCE == 'merge_request_event' && $CI_MERGE_REQUEST_TARGET_BRANCH_NAME == "main" # only have MR pipelines for MRs to main + - if: $CI_PIPELINE_SOURCE == 'merge_request_event' && ($CI_MERGE_REQUEST_TARGET_BRANCH_NAME == "basop-ci-branch" || $CI_MERGE_REQUEST_TARGET_BRANCH_NAME == "main") - if: $CI_PIPELINE_SOURCE == 'push' when: never @@ -327,7 +351,7 @@ stages: stage: test extends: - .test-job-linux-needs-testv-dir - - .rules-merge-request + - .rules-merge-request-to-main artifacts: name: "mr-$CI_MERGE_REQUEST_IID--sha-$CI_COMMIT_SHORT_SHA--stage-$CI_JOB_STAGE--results" expire_in: 1 week @@ -396,7 +420,7 @@ uninterruptible: branch-is-up-to-date-with-main-pre: extends: - - .rules-merge-request + - .rules-merge-request-to-main stage: prevalidate needs: [] tags: @@ -408,7 +432,7 @@ branch-is-up-to-date-with-main-pre: check-self-test-names-pre: extends: - - .rules-merge-request + - .rules-merge-request-to-main stage: prevalidate needs: [] tags: @@ -418,7 +442,7 @@ check-self-test-names-pre: check-no-duplicates-in-self-tests: extends: - - .rules-merge-request + - .rules-merge-request-to-main stage: prevalidate needs: [] tags: @@ -428,7 +452,7 @@ check-no-duplicates-in-self-tests: branch-is-up-to-date-with-main-post: extends: - - .rules-merge-request + - .rules-merge-request-to-main stage: postvalidate tags: - ivas-linux @@ -440,24 +464,20 @@ branch-is-up-to-date-with-main-post: .basop-ci-branch-compat-template: extends: - .test-job-linux - rules: - - if: $CI_PIPELINE_SOURCE == 'merge_request_event' && $CI_MERGE_REQUEST_TARGET_BRANCH_NAME == "basop-ci-branch" - changes: - - tests/**/* - - scripts/**/* - - if: $CI_PIPELINE_SOURCE == 'push' - when: never + - .rules-merge-request-to-basop-ci-branch tags: - - ivas-linux-fast + - ivas-linux before_script: - - git clone -b $BASOP_REFERENCE_BRANCH https://forge.3gpp.org/rep/sa4/audio/ivas-basop.git --single-branch - - make -j -C ivas-basop - python3 ci/remove_unsupported_testcases.py $PARAM_FILE - python3 tests/create_short_testvectors.py + - python3 scripts/prepare_combined_format_inputs.py + - *update-ltv-repo - *copy-ltv-files-to-testv-dir - - python3 -m pytest tests/codec_be_on_mr_nonselection --param_file $PARAM_FILE --update_ref 1 --ref_encoder_path ivas-basop/IVAS_cod --ref_decoder_path ivas-basop/IVAS_dec --html=report.html --self-contained-html --junit-xml=report-junit.xml || true + - *get-basop-float-reference + + - python3 -m pytest tests/codec_be_on_mr_nonselection $PYTEST_ARGS --param_file $PARAM_FILE --update_ref 1 --ref_encoder_path IVAS_cod --ref_decoder_path IVAS_dec --html=report.html --self-contained-html --junit-xml=report-junit.xml --testcase_timeout=$TESTCASE_TIMEOUT || true - zero_errors=$(cat report-junit.xml | grep -c 'errors="0"') || true - if [ $zero_errors != 1 ]; then echo "Run errors encountered with $PARAM_FILE !"; exit $EXIT_CODE_FAIL; fi @@ -474,35 +494,54 @@ branch-is-up-to-date-with-main-post: junit: - report-junit.xml -check-compatibility-with-basop-reference-branch-stv: +basop-ref-compat-stv: extends: - .basop-ci-branch-compat-template variables: PARAM_FILE: scripts/config/self_test.prm + TESTCASE_TIMEOUT: $TESTCASE_TIMEOUT_STV_SANITIZERS script: - exit 0 -check-compatibility-with-basop-reference-branch-ltv: +basop-ref-compat-ltv: extends: - .basop-ci-branch-compat-template + tags: + - ivas-linux-fast variables: PARAM_FILE: scripts/config/self_test_ltv.prm + TESTCASE_TIMEOUT: $TESTCASE_TIMEOUT_LTV_SANITIZERS script: - exit 0 -check-compatibility-with-basop-reference-branch-encoder-stv: +basop-ref-compat-encoder-stv: extends: - .basop-ci-branch-compat-template variables: PARAM_FILE: scripts/config/self_test_basop_encoder.prm + TESTCASE_TIMEOUT: $TESTCASE_TIMEOUT_STV_SANITIZERS script: - exit 0 -check-compatibility-with-basop-reference-branch-encoder-ltv: +basop-ref-compat-encoder-ltv: extends: - .basop-ci-branch-compat-template + tags: + - ivas-linux-fast variables: PARAM_FILE: scripts/config/self_test_ltv_basop_encoder.prm + TESTCASE_TIMEOUT: $TESTCASE_TIMEOUT_LTV_SANITIZERS + script: + - exit 0 + +basop-ref-compat-encoder-dmx-comp: + extends: + - .basop-ci-branch-compat-template + variables: + PARAM_FILE: scripts/config/self_test_basop_encoder.prm + # USING PYTEST_ADDOPTS env var did not work for some reason when testing locally - maybe because this is a custom option + PYTEST_ARGS: "--compare_enc_dmx" + TESTCASE_TIMEOUT: $TESTCASE_TIMEOUT_STV_SANITIZERS script: - exit 0 @@ -579,7 +618,7 @@ build-codec-windows-msbuild: codec-smoke-test: extends: - .test-job-linux-needs-testv-dir - - .rules-merge-request + - .rules-merge-request-to-basop-ci-branch-or-main timeout: "20 minutes" tags: - ivas-linux-fast @@ -589,7 +628,17 @@ codec-smoke-test: - *print-common-info # LTV update needed as ltv ISM metadata files are used - *update-ltv-repo - - bash ci/smoke_test.sh + + # for MRs to basop-ci-branch, we want to test with the BASOP float reference build + # per default, the smoke test builds the current repo again with WMOPS activated to catch unbalanced instrumentation macros + - if [ "$CI_MERGE_REQUEST_TARGET_BRANCH_NAME" = "basop-ci-branch" ]; then + - python3 scripts/smoketest-basop-filter.py scripts/config/ivas_modes.json --inline + - *get-basop-float-reference + - bash ci/smoke_test.sh coverage + - else + - bash ci/smoke_test.sh + - fi + ### analyze for failures - if ! [ -s smoke_test_output.txt ] || ! [ -s smoke_test_output_jbm.txt ] || ! [ -s smoke_test_output_hrtf.txt ]; then echo "Error in smoke test"; exit 1; fi - ret_val=0 @@ -648,7 +697,7 @@ codec-usan: pytest-compare-20ms-and-5ms-rendering: extends: - .test-job-linux - - .rules-merge-request + - .rules-merge-request-to-main stage: test needs: ["build-codec-linux-cmake", "build-codec-linux-make", "build-codec-instrumented-linux", "build-codec-sanitizers-linux"] script: @@ -695,7 +744,7 @@ pytest-compare-20ms-and-5ms-rendering: renderer-smoke-test: extends: - .test-job-linux - - .rules-merge-request + - .rules-merge-request-to-main needs: ["build-codec-linux-make"] stage: test script: @@ -717,7 +766,7 @@ renderer-smoke-test: renderer-asan: extends: - .test-job-linux - - .rules-merge-request + - .rules-merge-request-to-main needs: ["build-codec-linux-cmake"] stage: test script: @@ -741,7 +790,7 @@ renderer-asan: renderer-msan: extends: - .test-job-linux - - .rules-merge-request + - .rules-merge-request-to-main needs: ["build-codec-linux-cmake"] stage: test script: @@ -765,7 +814,7 @@ renderer-msan: renderer-usan: extends: - .test-job-linux - - .rules-merge-request + - .rules-merge-request-to-main needs: ["build-codec-linux-cmake"] stage: test script: @@ -793,7 +842,7 @@ renderer-usan: renderer-pytest-on-merge-request: extends: - .test-job-linux-needs-testv-dir - - .rules-merge-request + - .rules-merge-request-to-main needs: ["build-codec-linux-make"] # TODO: set reasonable timeout, will most likely take less timeout: "20 minutes" @@ -845,7 +894,7 @@ renderer-pytest-on-merge-request: split-rendering-smoke-test: extends: - .test-job-linux - - .rules-merge-request + - .rules-merge-request-to-main needs: ["build-codec-linux-make"] stage: test script: @@ -866,7 +915,7 @@ split-rendering-smoke-test: lc3-wrapper-unit-test: extends: - .test-job-linux - - .rules-merge-request + - .rules-merge-request-to-main needs: ["build-codec-linux-cmake"] stage: test script: @@ -878,7 +927,7 @@ lc3-wrapper-unit-test: split-rendering-pytest-on-merge-request: extends: - .test-job-linux-needs-testv-dir - - .rules-merge-request + - .rules-merge-request-to-main needs: ["build-codec-linux-make"] # TODO: set reasonable timeout, will most likely take less timeout: "30 minutes" @@ -950,7 +999,7 @@ split-rendering-pytest-on-merge-request: ivas-pytest-on-merge-request: extends: - .test-job-linux-needs-testv-dir - - .rules-merge-request + - .rules-merge-request-to-main stage: compare needs: ["build-codec-linux-cmake", "codec-smoke-test"] timeout: "14 minutes" @@ -1004,7 +1053,7 @@ ivas-pytest-on-merge-request: ivas-interop-on-merge-request: extends: - .test-job-linux-needs-testv-dir - - .rules-merge-request + - .rules-merge-request-to-main stage: test needs: ["build-codec-linux-cmake"] timeout: "10 minutes" @@ -1053,7 +1102,7 @@ ivas-interop-on-merge-request: evs-pytest-on-merge-request: extends: - .test-job-linux - - .rules-merge-request + - .rules-merge-request-to-main stage: compare needs: ["build-codec-linux-cmake", "codec-smoke-test"] timeout: "10 minutes" @@ -1105,7 +1154,7 @@ evs-pytest-on-merge-request: voip-be-on-merge-request: extends: - .test-job-linux-needs-testv-dir - - .rules-merge-request + - .rules-merge-request-to-main stage: test needs: ["build-codec-linux-make"] timeout: "10 minutes" @@ -1118,7 +1167,7 @@ voip-be-on-merge-request: clang-format-check: extends: - .test-job-linux - - .rules-merge-request + - .rules-merge-request-to-main variables: ARTIFACT_BASE_NAME: "mr-$CI_MERGE_REQUEST_IID--sha-$CI_COMMIT_SHORT_SHA--formatting-fix" stage: prevalidate @@ -1165,7 +1214,7 @@ clang-format-check: check-first-frame-is-sid: extends: - .test-job-linux-needs-testv-dir - - .rules-merge-request + - .rules-merge-request-to-main tags: - ivas-linux stage: test @@ -1192,7 +1241,7 @@ check-first-frame-is-sid: .lc3plus-ensure-no-code-changes: extends: - .test-job-linux - - .rules-merge-request + - .rules-merge-request-to-main stage: postvalidate needs: [] timeout: "5 minutes" @@ -1207,7 +1256,7 @@ check-first-frame-is-sid: check-bitexactness-hrtf-rom-and-file: extends: - .test-job-linux - - .rules-merge-request + - .rules-merge-request-to-main stage: test needs: ["build-codec-linux-cmake"] timeout: "5 minutes" @@ -1229,7 +1278,7 @@ check-bitexactness-hrtf-rom-and-file: check-bitexactness-ext-and-transport-format: extends: - .test-job-linux - - .rules-merge-request + - .rules-merge-request-to-main stage: test needs: ["build-codec-linux-cmake"] timeout: "5 minutes" @@ -1256,7 +1305,7 @@ check-bitexactness-ext-and-transport-format: # check bitexactness to EVS windows binaries be-2-evs-windows: extends: - - .rules-merge-request + - .rules-merge-request-to-main tags: - ivas-windows stage: test diff --git a/ci/basop-pages/create_report_pages.py b/ci/basop-pages/create_report_pages.py index 2f01db7bdec36df2a5748b76b88035792254a091..41ee81af1bb51ff41f63f80e79f72d84c6f77d57 100644 --- a/ci/basop-pages/create_report_pages.py +++ b/ci/basop-pages/create_report_pages.py @@ -37,6 +37,7 @@ Comparing:

Summary page

+

Split comparison summary page



diff --git a/ci/basop-pages/create_summary_page.py b/ci/basop-pages/create_summary_page.py index 1c2b46480213f24913e7288aa2659a828f1ec1e6..32855fce251b418a24394210f77050827acdb6b7 100644 --- a/ci/basop-pages/create_summary_page.py +++ b/ci/basop-pages/create_summary_page.py @@ -4,11 +4,11 @@ from typing import List from create_report_pages import SUBPAGE_TMPL_CSS, FORMATS -title = { +TITLE_4_MEASURE = { "MLD": "Maximum MLD across channels", - "DIFF": "Maximim absolute difference across channels", - "SSNR": "Minimum SSNR across channels", - "ODG": "Minimum PEAQ ODG across channels", + "MAX_ABS_DIFF": "Maximum absolute difference across channels", + "MIN_SSNR": "Minimum SSNR across channels", + "MIN_ODG": "Minimum PEAQ ODG across channels", "DELTA_ODG": "PEAQ ODG using binauralized input and output", } @@ -16,9 +16,13 @@ SUMMARY_PAGE_TMPL_HTML = """

Summary for job {job_name}, ID: {id_current}

+
+ {images} """ +IMAGE_HTML_TMPL = "" +SUBHEADING_HTML_TMP = "

{subtitle}

\n" def create_summary_page( @@ -26,53 +30,50 @@ def create_summary_page( id_current: int, job_name: str, measures: List[str], + image_dir: str, ): - images = histogram_summary(job_name, measures) + html = "\n
\n".join( + [ + SUBHEADING_HTML_TMP.format(subtitle=TITLE_4_MEASURE[m]) + + " ".join( + [ + IMAGE_HTML_TMPL.format(measure=m, format=f, image_dir=image_dir) + for f in FORMATS + ] + ) + for m in measures + ] + ) new_summary_page = SUBPAGE_TMPL_CSS + SUMMARY_PAGE_TMPL_HTML.format( id_current=id_current, job_name=job_name, - images=images, + images=html, ) with open(html_out, "w") as f: f.write(new_summary_page) -def histogram_summary( - job_name: str, - measures: List[str], -): - images = "
" - for m in measures: - images += ( - f"

{title[m]}

\n" - + " ".join( - [f"" for x in FORMATS] - ) - + f'\n
summary_{m}.csv
\n\n' - ) - return images - - if __name__ == "__main__": parser = argparse.ArgumentParser() parser.add_argument("html_out") parser.add_argument("id_current", type=int) parser.add_argument("job_name") + parser.add_argument("image_dir") parser.add_argument( "--measures", nargs="+", - help=f"List of measures to include in summary. Allowed values: {' '.join(title.keys())}", - default=["MLD", "DIFF", "SSNR", "ODG"], + help=f"List of measures to include in summary. Allowed values: {' '.join(TITLE_4_MEASURE.keys())}", + # exclude DELTA_ODG here + default=list(TITLE_4_MEASURE.keys())[:-1], ) args = parser.parse_args() - if not all([m in title for m in args.measures]): - raise ValueError(f"Invalid list of measures: {args.measures}, expected one of {' '.join(title.keys())}") + if not all([m in TITLE_4_MEASURE for m in args.measures]): + raise ValueError( + f"Invalid list of measures: {args.measures}, expected one of {' '.join(TITLE_4_MEASURE.keys())}" + ) create_summary_page( - args.html_out, - args.id_current, - args.job_name, - args.measures, + args.html_out, args.id_current, args.job_name, args.measures, args.image_dir ) diff --git a/ci/get_float_ref_branch_name.sh b/ci/get_float_ref_branch_name.sh new file mode 100755 index 0000000000000000000000000000000000000000..fe160da02331b04249febb89971c90d729c198a8 --- /dev/null +++ b/ci/get_float_ref_branch_name.sh @@ -0,0 +1,65 @@ +#!/bin/bash + +# (C) 2022-2025 IVAS codec Public Collaboration with portions copyright Dolby International AB, Ericsson AB, +# Fraunhofer-Gesellschaft zur Foerderung der angewandten Forschung e.V., Huawei Technologies Co. LTD., +# Koninklijke Philips N.V., Nippon Telegraph and Telephone Corporation, Nokia Technologies Oy, Orange, +# Panasonic Holdings Corporation, Qualcomm Technologies, Inc., VoiceAge Corporation, and other +# contributors to this repository. All Rights Reserved. +# +# This software is protected by copyright law and by international treaties. +# The IVAS codec Public Collaboration consisting of Dolby International AB, Ericsson AB, +# Fraunhofer-Gesellschaft zur Foerderung der angewandten Forschung e.V., Huawei Technologies Co. LTD., +# Koninklijke Philips N.V., Nippon Telegraph and Telephone Corporation, Nokia Technologies Oy, Orange, +# Panasonic Holdings Corporation, Qualcomm Technologies, Inc., VoiceAge Corporation, and other +# contributors to this repository retain full ownership rights in their respective contributions in +# the software. This notice grants no license of any kind, including but not limited to patent +# license, nor is any license granted by implication, estoppel or otherwise. +# +# Contributors are required to enter into the IVAS codec Public Collaboration agreement before making +# contributions. +# +# This software is provided "AS IS", without any express or implied warranties. The software is in the +# development stage. It is intended exclusively for experts who have experience with such software and +# solely for the purpose of inspection. All implied warranties of non-infringement, merchantability +# and fitness for a particular purpose are hereby disclaimed and excluded. +# +# Any dispute, controversy or claim arising under or in relation to providing this software shall be +# submitted to and settled by the final, binding jurisdiction of the courts of Munich, Germany in +# accordance with the laws of the Federal Republic of Germany excluding its conflict of law rules and +# the United Nations Convention on Contracts on the International Sales of Goods. + +BASOP_PATTERN="^[0-9]+[_-]basop[_-].*" +MSG_DOES_NOT_MATCH="Your branch name %s does not match the template '_basop_', e.g. '123_basop_fix_this_one_bug-2'. +Sticking to this branch is needed for the testing system to match this branch with its float-reference counterpart. +Please rename your branch. You can easily do this by creating a new branch from this branch: + - git checkout -b +You then also need to create a new merge request and update the links in your issue. +See here for details on the porting work process: https://forge.3gpp.org/rep/sa4/audio/ivas-basop/-/wikis/Porting-MRs-from-floating-point-codec#workflow-for-porting-a-merge-request-from-floating-point-codec-to-basop-codec +" + +if [ $# -ne 1 ]; then + echo "Usage: $0 " + exit 1 +fi + +branchname="$1" + +if ! [[ "$branchname" =~ $BASOP_PATTERN ]]; then + printf "$MSG_DOES_NOT_MATCH" "$branchname" + exit 1 +fi + +# The float ref branch is just the same name, but with basop -> ref replacement +# Replace only the first occurrence, as "basop" may be present in the later description +# If the format is correct, then before "_basop", only numbers can occur +float_ref_branchname="${branchname/basop/ref}" +git_result=$(git branch -av) + +# If the branch does not exist, default to "float-pc" +if [[ "$git_result" =~ "$float_ref_branchname" ]]; then + branchname="${float_ref_branchname}" +else + branchname="float-pc" +fi + +echo "$branchname" diff --git a/ci/remove_unsupported_testcases.py b/ci/remove_unsupported_testcases.py index bee69670e54f82bb60d84399e580bdb581721cc1..870d513111e03c9fc3b59ae3d83d559705038194 100644 --- a/ci/remove_unsupported_testcases.py +++ b/ci/remove_unsupported_testcases.py @@ -32,7 +32,7 @@ from pathlib import Path import argparse # Enter tag of testcases to remove here WITHOUT the leading // -TESTCASES = [ +TESTCASES_MAIN = [ "OMASA 2Dir2TC 4ISM at br sw techs 13.2 to 512 kbps start 80 kbps, 48kHz in, 48kHz out, EXT out", "OSBA planar FOA 2ISM at 512 kbps, 48 kHz in, 48 kHz out, BINAURAL out", "4 ISM with extended metadata at 128 kbps, 48 kHz in, 48 kHz out, BINAURAL_ROOM_REVERB out, rendconf dir w id", @@ -66,10 +66,19 @@ TESTCASES = [ "MASA 1TC at 256kbps, 48kHz in, 48 kHz out, BINAURAL_ROOM_REVERB out custom configuration", "MASA 1TC at 256 kbps, 48kHz in, 48kHz out, BINAURAL_ROOM_REVERB out, HR custom configuration", "OMASA 2TC 4ISM at br sw techs 13.2 to 512 kbps start 80 kbps, 48kHz in, 48kHz out, EXT out", + "Multi-channel 7_1_4 bitrate switching, 48kHz in, 48kHz out, BINAURAL out, HR, JBM Prof 5", + "Multi-channel 7_1 bitrate switching, 48kHz in, 32kHz out, BINAURAL_ROOM_REVERB out, HR, JBM Prof 5", + "3 ISM with metadata bitrate switching from 48 kbps to 32 kbps, 48 kHz in, 32 kHz out, DTX, BINAURAL_ROOM_IR out, JBM Prof 5", + # object editing starts here "OMASA 2Dir2TC 4ISM at 80 kbps, 48kHz in, 48kHz out, BINAURAL out, default object editing, 1SEP-PARAM", "OMASA 2Dir2TC 4ISM at 256 kbps, 48kHz in, 48kHz out, BINAURAL out, object editing, JBM Prof 5, DISC", "OMASA 2Dir2TC 2ISM at 96 kbps, 48kHz in, 48kHz out, FOA out, object editing, JBM Prof 5, DISC", "OMASA 2Dir2TC 2ISM br sw techs 13.2 to 512 kbps start 48 kbps, 48kHz in, 48kHz out, BINAURAL out, object editing", + # the next four lines are basically duplicates of the last four, but without "Dir", because the ltv MASA cases do not have dedicated testvectors for numbers of directions + "OMASA 2TC 4ISM at 80 kbps, 48kHz in, 48kHz out, BINAURAL out, default object editing, 1SEP-PARAM", + "OMASA 2TC 4ISM at 256 kbps, 48kHz in, 48kHz out, BINAURAL out, object editing, JBM Prof 5, DISC", + "OMASA 2TC 2ISM at 96 kbps, 48kHz in, 48kHz out, FOA out, object editing, JBM Prof 5, DISC", + "OMASA 2TC 2ISM br sw techs 13.2 to 512 kbps start 48 kbps, 48kHz in, 48kHz out, BINAURAL out, object editing", "OSBA 3OA 4ISM at 256 kbps, 48kHz in, 48kHz out, BINAURAL out, object editing, DISC", "OSBA 2OA 3ISM at 128 kbps, 48kHz in, 48kHz out, FOA out, object editing, JBM Prof 5, DISC", "OSBA 2OA 3ISM at bitrate switching 13.2 to 512 kbps, 48kHz in, 48kHz out, BINAURAL out, object editing", @@ -79,12 +88,21 @@ TESTCASES = [ "3 ISM with metadata at 384 kbps, 48 kHz in, 48 kHz out, FOA out, object editing, JBM Prof 5, DISC", "4 ISM with metadata bitrate switching from 32 kbps to 48 kbps, 48 kHz in, 48 kHz out, BINAURAL_ROOM_IR out, object editing", "4 ISM with metadata bitrate switching from 48 kbps to 32 kbps, 48 kHz in, 48 kHz out, BINAURAL out, object editing, JBM Prof 5", - "OMASA 2TC 4ISM at 80 kbps, 48kHz in, 48kHz out, BINAURAL out, default object editing, 1SEP-PARAM", - "OMASA 2TC 4ISM at 256 kbps, 48kHz in, 48kHz out, BINAURAL out, object editing, JBM Prof 5, DISC", - "OMASA 2TC 2ISM at 96 kbps, 48kHz in, 48kHz out, FOA out, object editing, JBM Prof 5, DISC", - "OMASA 2TC 2ISM br sw techs 13.2 to 512 kbps start 48 kbps, 48kHz in, 48kHz out, BINAURAL out, object editing", ] - +TESTCASES_MAIN_PC = [ + "SBA at 128 kbps, 32kHZ in, 32kHz out, BINAURAL_ROOM_REVERB out, Config renderer, HR", + "SBA at 128 kbps, 32kHZ in, 16kHz out, BINAURAL_ROOM_REVERB out (Model from file), HR", + "Planar SBA at 128 kbps, 48kHZ in, 32kHz out, BINAURAL_ROOM_REVERB out (Model from file), Config renderer, HR", + "Multi-channel 7_1_4 at 160 kbps, 48kHz in, 48 kHz out, BINAURAL_ROOM_REVERB out default configuration", + "Multi-channel 5_1 at 80 kbps, 48kHz in, 32kHz out, BINAURAL_ROOM_REVERB out Config renderer, HR", + "OSBA planar 2OA 4ISM at 512 kbps, 48 kHz in, 48 kHz out, BINAURAL ROOM REVERB (Model from file) out", + "SBA 3OA 4ISM at 96 kbps, 48kHz in, 48kHz out, BINAURAL_ROOM_REVERB out custom configuration", + "Multi-channel 7_1_4 at 160 kbps, 48kHz in, 48kHz out, BINAURAL_ROOM_REVERB out Config recreation, HR", + "Multi-channel 5_1_2 at 64 kbps, 48kHz in, 48kHz out, BINAURAL_ROOM_REVERB out Config renderer, HR", + "OSBA 3OA 4ISM at 512 kbps, 48kHz in, 48kHz out, BINAURAL_ROOM_REVERB out", + "OSBA 3ISM 3OA at bitrate switching 13.2 to 512 kbps, 48kHz in, 32kHz out, BINAURAL ROOM REVERB out", + "Multi-channel 5_1 bitrate switching from 13.2 kbps to 512 kbps, 48kHz in, 16kHz out, BINAURAL_ROOM_REVERB out", +] def remove_testcases(cfg: Path, testcases: list): @@ -112,7 +130,12 @@ def remove_testcases(cfg: Path, testcases: list): if __name__ == "__main__": parser = argparse.ArgumentParser() parser.add_argument("cfg_files", nargs="+", type=Path) + parser.add_argument("--use-main-pc-set", action="store_true") args = parser.parse_args() + testcases = TESTCASES_MAIN + if args.use_main_pc_set: + testcases.extend(TESTCASES_MAIN_PC) + for f in args.cfg_files: - remove_testcases(f, TESTCASES) + remove_testcases(f, testcases) diff --git a/scripts/basop_check_for_changes_in_testcases.py b/scripts/basop_check_for_changes_in_testcases.py index 5552fcba8b394267d85319148a281baad9ffaee9..2b38e8a0d559e3b54161708d0c5becc41ea03a0f 100644 --- a/scripts/basop_check_for_changes_in_testcases.py +++ b/scripts/basop_check_for_changes_in_testcases.py @@ -35,6 +35,8 @@ import argparse import sys import os import pathlib +import re +import xml.etree.ElementTree as ET # set positive threshold for "lower is better" metrics, negative for "higher is better" @@ -45,11 +47,21 @@ COLS_2_THRESHOLDS = { "MIN_ODG": float(os.environ.get("CI_REGRESSION_THRESH_ODG", -0.05)), } -OUTFILE_CRASHES = "changes_crashes.csv" -OUTFILE_SCORES = "changes_{}.csv" +OUTFILE_CRASHES_FIXED = "improvements_crashes.csv" +OUTFILE_CRASHES_ADDED = "regressions_crashes.csv" +OUTFILE_SCORES_IMPROVEMENTS = "improvements_{}.csv" +OUTFILE_SCORES_REGRESSION = "regressions_{}.csv" + +PATTERN_ENC = r"... encoder command:\s*(.*)\s*... encoder stdout:" +PATTERN_DEC = r"... decoder command:\s*(.*)\s*... decoder stdout:" +PATTERN_EID = r"eid-xor command:\s*(.*)\s*" +PATTERN_NETSIM = r"netsim command:\s*(.*)\s*" + +PATTERNS = [PATTERN_ENC, PATTERN_DEC, PATTERN_EID, PATTERN_NETSIM] def main(args): + xml_report = args.xml_report df_curr = pd.read_csv(args.csv_current, sep=";") df_prev = pd.read_csv(args.csv_previous, sep=";") df_merged = pd.merge(df_curr, df_prev, on="testcase", suffixes=["-curr", "-prev"]) @@ -77,7 +89,6 @@ def main(args): df_crashes_introduced = df_merged[mask_crash_introduced][display_cols].reset_index( drop=True ) - df_crashes_introduced.to_csv(OUTFILE_CRASHES, sep=";") if sum(mask_crash_introduced) > 0: regressions_found = True @@ -85,15 +96,32 @@ def main(args): print(df_crashes_introduced) print() + if xml_report is not None: + cmdlines_crashes_introduced = get_command_lines_for_testcases( + df_crashes_introduced["testcase"], xml_report, args.inject_cwd + ) + df_crashes_introduced = pd.merge( + df_crashes_introduced, cmdlines_crashes_introduced, on="testcase" + ) + df_crashes_introduced.to_csv(OUTFILE_CRASHES_ADDED, sep=";") + if args.show_improvements and sum(mask_crash_fixed) > 0: df_crashes_fixed = df_merged[mask_crash_fixed][display_cols].reset_index( drop=True ) - df_crashes_fixed.to_csv(OUTFILE_CRASHES, mode="a", sep=";") print("---------------Testcases that fixed crashes---------------") print(df_crashes_fixed) print() + if xml_report is not None: + cmdlines_crashes_fixed = get_command_lines_for_testcases( + df_crashes_fixed["testcase"], xml_report, args.inject_cwd + ) + df_crashes_fixed = pd.merge( + df_crashes_fixed, cmdlines_crashes_fixed, on="testcase" + ) + df_crashes_fixed.to_csv(OUTFILE_CRASHES_FIXED, sep=";") + # remove columns with ERRORs in any of the csv files before comparing the numerical columns mask_no_errors = (df_merged[col_curr] != "ERROR") & (df_merged[col_prev] != "ERROR") df_merged = df_merged[mask_no_errors].reset_index(drop=True) @@ -113,9 +141,8 @@ def main(args): mask_better = (df_merged[col_diff] * fac) < -thresh display_cols = ["testcase", col_curr, col_prev, col_diff] - outfile = OUTFILE_SCORES.format(col.replace(" ", "_")) + outfile_regressions = OUTFILE_SCORES_REGRESSION.format(col.replace(" ", "_")) df_worse = df_merged[mask_worse][display_cols].reset_index(drop=True) - df_worse.to_csv(outfile, sep=";") if sum(mask_worse) > 0: regressions_found = True print( @@ -124,18 +151,110 @@ def main(args): print(df_worse) print() + if xml_report is not None: + cmdlines_worse = get_command_lines_for_testcases( + df_worse["testcase"], xml_report, args.inject_cwd + ) + df_worse = pd.merge(df_worse, cmdlines_worse, on="testcase") + df_worse.to_csv(outfile_regressions, sep=";") + if args.show_improvements and sum(mask_better) > 0: df_better = df_merged[mask_better][display_cols].reset_index(drop=True) - df_better.to_csv(outfile, mode="a", sep=";") print( f"---------------Testcases that got better wrt to {col}---------------" ) print(df_better) print() + if xml_report is not None: + cmdlines_better = get_command_lines_for_testcases( + df_better["testcase"], xml_report, args.inject_cwd + ) + df_better = pd.merge(df_better, cmdlines_better, on="testcase") + outfile_improvements = OUTFILE_SCORES_IMPROVEMENTS.format( + col.replace(" ", "_") + ) + df_better.to_csv(outfile_improvements, sep=";") + return int(regressions_found) +def get_command_lines_for_testcases( + testcases: pd.Series, xml_report: pathlib.Path, cwd: pathlib.Path +) -> pd.DataFrame: + testcase_elems = [ + e + for _, e in ET.iterparse(xml_report) + if e.tag == "testcase" and e.attrib["name"] in testcases.values + ] + + cmdlines = { + "testcase": [], + "enc_cmd": [], + "dec_cmd": [], + "eid-xor_cmd": [], + "netsim_cmd": [], + } + for elem in testcase_elems: + testcase_name = elem.attrib["name"] + enc_cmd = "" + dec_cmd = "" + eid_cmd = "" + netsim_cmd = "" + if (system_out := elem.find("system-out")) is not None: + ( + enc_cmd, + dec_cmd, + eid_cmd, + netsim_cmd, + ) = extract_cmdlines(system_out.text, cwd) + + cmdlines["testcase"].append(testcase_name) + cmdlines["enc_cmd"].append(enc_cmd) + cmdlines["dec_cmd"].append(dec_cmd) + cmdlines["eid-xor_cmd"].append(eid_cmd) + cmdlines["netsim_cmd"].append(netsim_cmd) + + return pd.DataFrame(cmdlines) + + +def extract_cmdlines(text: str, cwd: pathlib.Path) -> list[str]: + cmdlines = [] + for p in PATTERNS: + m = re.search(p, text) + if m is not None: + cmdline = postprocess_cmdline(m.group(1), cwd) + cmdlines.append(cmdline) + else: + cmdlines.append("") + + return cmdlines + + +def postprocess_cmdline(cmdline: str, cwd: pathlib.Path) -> str: + cmdline_split = cmdline.split() + cmdline_proc = [] + + # change absolute paths into relative ones + # remove the "quite" flag + # for output and bitstream files only keep the filename + for elem in cmdline_split: + if elem == "-q": + continue + elif (elem_as_path := pathlib.Path(elem)).is_absolute(): + if elem_as_path.suffix == ".192" or ( + elem_as_path.suffix == ".wav" + and cmdline_split.index(elem) == len(cmdline_split) - 1 + ): + cmdline_proc.append(elem_as_path.name) + else: + cmdline_proc.append(str(elem_as_path.relative_to(cwd))) + else: + cmdline_proc.append(elem) + + return " ".join(cmdline_proc) + + if __name__ == "__main__": parser = argparse.ArgumentParser() parser.add_argument("csv_current") @@ -147,6 +266,17 @@ if __name__ == "__main__": default=COLS_2_THRESHOLDS.keys(), ) parser.add_argument("--show_improvements", action="store_true") + parser.add_argument( + "--xml_report", + help="XMLxml_report report file from pytest run. Pass to add command lines to the output files.", + default=None, + ) + parser.add_argument( + "--inject_cwd", + help="Use this as cwd when pruning the long paths in the command lines. Debug option for testing.", + default=pathlib.Path(os.getcwd()).absolute(), + type=pathlib.Path, + ) args = parser.parse_args() sys.exit(main(args)) diff --git a/scripts/basop_create_ignorelist_for_ubsan.py b/scripts/basop_create_ignorelist_for_ubsan.py new file mode 100644 index 0000000000000000000000000000000000000000..d13c0d5f5b7054ed73e37fa1aa8339153bc77590 --- /dev/null +++ b/scripts/basop_create_ignorelist_for_ubsan.py @@ -0,0 +1,14 @@ +import pathlib + +here = pathlib.Path(".") + +all_files = here.glob("lib_*/*.[ch]") +ivas_files = here.glob("lib_*/ivas*.[ch]") +non_ivas_files = sorted(set(all_files) - set(ivas_files)) +basop_files = here.glob("lib_*/basop*.[ch]") + +ignorefiles = sorted(set(non_ivas_files) - set(basop_files)) + +with open("ubsan_ignorelist.txt", "w") as f: + for cfile in ignorefiles: + print(f"src:{cfile}", file=f) diff --git a/scripts/binauralRenderer_interface/README.md b/scripts/binauralRenderer_interface/README.md index d9e79ed24d6142de443a7c2200092a8c532f7ce4..4f8ceb9da54747444a59f22cfb057f68ac518d08 100644 --- a/scripts/binauralRenderer_interface/README.md +++ b/scripts/binauralRenderer_interface/README.md @@ -27,8 +27,8 @@ accordance with the laws of the Federal Republic of Germany excluding its conflict of law rules and the United Nations Convention on Contracts on the International Sales of Goods. --> -# Modeling tool for all IVAS binaural renderers -# Allows to convert SOFA file(s) to ROM tables files (*.c|h) or binaural binary file(s) +# Modeling tool for all IVAS binaural renderer +# Allows to convert SOFA file(s) to rom tables files (*.c|h) or binaural binary file(s) ## Requirements - MATLAB >= R2017b @@ -55,7 +55,7 @@ ``` - `generate_crend_ivas_tables` executable shall be in folder [`../../scripts/binauralRenderer_interface`](../../scripts/binauralRenderer_interface) - On Windows this executable requires DLL from MATLAB, you need to add to your path `*/matlab/version/bin/win64` - - For more details on `generate_crend_ivas_tables` see [`ivas_crend_sofa_to_rom_table_converter_readme.txt`](ivas_crend_sofa_to_rom_table_converter_readme.txt). + - For more details on `generate_crend_ivas_tables` see [`mixer_conv_sofa_to_rom_table_converter_readme.txt`](mixer_conv_sofa_to_rom_table_converter_readme.txt). - On Apple Silicon Mac with Intel MATLAB, it is necessary to add `SET(CMAKE_OSX_ARCHITECTURES "x86_64")` in the `CMakeLists.txt` near the top of the file. This forces the compilation to use Intel architecture instead ofo the native one. - if find_package do not find matlab, at the beginning of the cmakelists.txt file you can set the search path with the folowing line set(Matlab_ROOT_DIR matlab_path), for example on linux set(Matlab_ROOT_DIR ~/MATLAB/R2023b) on mac set(Matlab_ROOT_DIR /Applications/MATLAB_R2023b.app) @@ -109,4 +109,11 @@ See [`scripts/ThirdPartyLegalNotices`](../../scripts/ThirdPartyLegalNotices) for → Generated files are in folder [`binaural_renderers_hrtf_data`](binaural_renderers_hrtf_data) - Generated files : - `ivas_binaural_*kHz.bin`: file containing all tables values for all IVAS binaural renderers for one sample rate. + - `ivas_binaural_td_*kHz.bin`: file containing tables values for IVAS td binaural renderer for one sample rate. + - `ivas_binaural_reverb_*kHz.bin`: file containing tables values for IVAS td reverberation process for one sample rate. + - `ivas_binaural_fastconv_*kHz.bin`: file containing tables values for IVAS fastconv binaural renderers. Files for all sampling rate files are the same. + - `ivas_binaural_parambin_*kHz.bin`: file containing tables values for IVAS prametric binaural renderers. Files for all sampling rate files are the same. + - `ivas_binaural_td_*kHz.bin`: file containing tables values for IVAS td binaural renderers for one sample rate. + - `ivas_binaural_mixconv_hrir_*kHz.bin`: file containing tables values for IVAS mixer conv binaural renderer for HRIR for one sample rate. + - `ivas_binaural_mixconv_brir_*kHz.bin`: file containing tables values for IVAS mixer conv binaural renderer for BRIR for one sample rate. diff --git a/scripts/binauralRenderer_interface/Table_Format_Converter/generate_tables_from_rom_to_bin.c b/scripts/binauralRenderer_interface/Table_Format_Converter/generate_tables_from_rom_to_bin.c index 5e78c01b6db9cdfe8a8930b19fe88bb3a32b7811..41c8dbc16bcbd53e3bb0b45dcc0341bc30b93559 100644 --- a/scripts/binauralRenderer_interface/Table_Format_Converter/generate_tables_from_rom_to_bin.c +++ b/scripts/binauralRenderer_interface/Table_Format_Converter/generate_tables_from_rom_to_bin.c @@ -73,6 +73,21 @@ const HRTF_READER_RENDERER_TYPE rend_types[IVAS_NB_RENDERER_TYPE] = { HRTF_READER_RENDERER_BINAURAL_REVERB_ALL }; +#ifdef FIX_1226_FASTCONV_HRTF_LOADING_OPTIM +typedef enum +{ + BINAURAL_INPUT_AUDIO_CONFIG_INVALID, + BINAURAL_INPUT_AUDIO_CONFIG_COMBINED, /* 5_1, 5_1_2, 5_1_4, 7_1, 7_1_4 */ + BINAURAL_INPUT_AUDIO_CONFIG_HOA3, /* HOA3 */ + BINAURAL_INPUT_AUDIO_CONFIG_HOA2, /* HOA2 */ + BINAURAL_INPUT_AUDIO_CONFIG_FOA, /* FOA */ + BINAURAL_INPUT_AUDIO_CONFIG_UNDEFINED /* Not used */ + +} BINAURAL_INPUT_AUDIO_CONFIG; + +#endif + + const BINAURAL_INPUT_AUDIO_CONFIG input_cfgs[IVAS_NB_AUDIO_CONFIG] = { BINAURAL_INPUT_AUDIO_CONFIG_COMBINED, BINAURAL_INPUT_AUDIO_CONFIG_HOA3, diff --git a/scripts/binauralRenderer_interface/binaural_renderers_hrtf_data/ivas_binaural_16kHz.bin b/scripts/binauralRenderer_interface/binaural_renderers_hrtf_data/ivas_binaural_16kHz.bin index c52a50ae96fe29c7fad87ed54b631a51ad8f5715..9a988eb588c1be259b9c7ce01c3bbc872cf3b1e9 100644 --- a/scripts/binauralRenderer_interface/binaural_renderers_hrtf_data/ivas_binaural_16kHz.bin +++ b/scripts/binauralRenderer_interface/binaural_renderers_hrtf_data/ivas_binaural_16kHz.bin @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:3eba5d2368bb525e80b08252761277fefcdd9d1b0d5d61c2e43af6730a6f59c1 -size 998652 +oid sha256:a263c5498fc7ec0407c4f448e78e1f0aaf97163234ef44f2f79a1dfcb5adc37f +size 1968026 diff --git a/scripts/binauralRenderer_interface/binaural_renderers_hrtf_data/ivas_binaural_32kHz.bin b/scripts/binauralRenderer_interface/binaural_renderers_hrtf_data/ivas_binaural_32kHz.bin index 269aadead0bd33cda8d8cc9cb6961512418295fe..73f151217e89352c910668206ef00200577543d8 100644 --- a/scripts/binauralRenderer_interface/binaural_renderers_hrtf_data/ivas_binaural_32kHz.bin +++ b/scripts/binauralRenderer_interface/binaural_renderers_hrtf_data/ivas_binaural_32kHz.bin @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:ef8ac9d464c0cc82a7939a763330d367a6ae66efad8163c318b0bd51e7778192 -size 1228024 +oid sha256:8622b0f7969a3126f8ab147c10392916b5e9cbe8d8bd9c04798abd3836e36c76 +size 2431246 diff --git a/scripts/binauralRenderer_interface/binaural_renderers_hrtf_data/ivas_binaural_48kHz.bin b/scripts/binauralRenderer_interface/binaural_renderers_hrtf_data/ivas_binaural_48kHz.bin index 015b8019c28ccd1f03bb1d336742228d10431e37..d847e746131f589ddf50882fec0caf5b1da2724b 100644 --- a/scripts/binauralRenderer_interface/binaural_renderers_hrtf_data/ivas_binaural_48kHz.bin +++ b/scripts/binauralRenderer_interface/binaural_renderers_hrtf_data/ivas_binaural_48kHz.bin @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:cb43290dae7ecbc1e2879b25fb520d31c039fe2d4b7d074aed8927f3e9a4b638 -size 1349608 +oid sha256:30bcf48f0c851d4e676d5b92be7eb3032442f5106a1072ef8fc9b335d60d0fbf +size 2673694 diff --git a/scripts/binauralRenderer_interface/fastconv/write_fastconv_binary_data_old.m b/scripts/binauralRenderer_interface/fastconv/write_fastconv_binary_data_old.m new file mode 100644 index 0000000000000000000000000000000000000000..3371eef834d6c6940299ffe000abf76d7a480bac --- /dev/null +++ b/scripts/binauralRenderer_interface/fastconv/write_fastconv_binary_data_old.m @@ -0,0 +1,294 @@ +%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% +% +% (C) 2022-2024 IVAS codec Public Collaboration with portions copyright Dolby International AB, Ericsson AB, +% Fraunhofer-Gesellschaft zur Foerderung der angewandten Forschung e.V., Huawei Technologies Co. LTD., +% Koninklijke Philips N.V., Nippon Telegraph and Telephone Corporation, Nokia Technologies Oy, Orange, +% Panasonic Holdings Corporation, Qualcomm Technologies, Inc., VoiceAge Corporation, and other +% contributors to this repository. All Rights Reserved. +% +% This software is protected by copyright law and by international treaties. +% The IVAS codec Public Collaboration consisting of Dolby International AB, Ericsson AB, +% Fraunhofer-Gesellschaft zur Foerderung der angewandten Forschung e.V., Huawei Technologies Co. LTD., +% Koninklijke Philips N.V., Nippon Telegraph and Telephone Corporation, Nokia Technologies Oy, Orange, +% Panasonic Holdings Corporation, Qualcomm Technologies, Inc., VoiceAge Corporation, and other +% contributors to this repository retain full ownership rights in their respective contributions in +% the software. This notice grants no license of any kind, including but not limited to patent +% license, nor is any license granted by implication, estoppel or otherwise. +% +% Contributors are required to enter into the IVAS codec Public Collaboration agreement before making +% contributions. +% +% This software is provided "AS IS", without any express or implied warranties. The software is in the +% development stage. It is intended exclusively for experts who have experience with such software and +% solely for the purpose of inspection. All implied warranties of non-infringement, merchantability +% and fitness for a particular purpose are hereby disclaimed and excluded. +% +% Any dispute, controversy or claim arising under or in relation to providing this software shall be +% submitted to and settled by the final, binding jurisdiction of the courts of Munich, Germany in +% accordance with the laws of the Federal Republic of Germany excluding its conflict of law rules and +% the United Nations Convention on Contracts on the International Sales of Goods. +% +%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% +function write_fastconv_binary_data(ivas_path, bin_file, FastConv_SHD_IR_FOA, FastConv_SHD_IR_HOA2, FastConv_SHD_IR_HOA3, FastConv_SD_IR, FastConv_SD_BRIR) +% +% Writes HRIR & BRIR based data for FastConv binaural renderer into a binary file. +% +% write_fastconv_binary_data(rom_file, FastConv_SHD_IR_FOA, FastConv_SHD_IR_HOA2, FastConv_SHD_IR_HOA3, FastConv_SD_IR, FastConv_SD_BRIR) +% +% filename : string +% name of the file to be written +% +% +% Output file format: +% Header [Declaration of the HRTF] +% Renderer type (int32_t) : See "RENDERER_TYPE" +% Decoder output format (int32_t) : See "BINAURAL_INPUT_AUDIO_CONFIG" +% Sampling Frequency (int32_t) +% Raw data size (uint32_t) +% +% HRIRs +% latency_s => float32 +% BINAURAL_CONVBANDS => uint16_t +% num_channels => uint16_t +% BINAURAL_NTAPS => uint16_t +% leftHRIRReal => float32[BINAURAL_CONVBANDS][num_channels][num_taps] +% leftHRIRImag => float32[BINAURAL_CONVBANDS][num_channels][num_taps] +% rightHRIRReal => float32[BINAURAL_CONVBANDS][num_channels][num_taps] +% rightHRIRImag => float32[BINAURAL_CONVBANDS][num_channels][num_taps] +% +% BRIRs +% latency_s => float32 +% num_channels => uint16_t +% BINAURAL_NTAPS_MAX => uint16_t +% leftBRIRReal => float32[BINAURAL_CONVBANDS][num_channels][num_taps] +% leftBRIRImag => float32[BINAURAL_CONVBANDS][num_channels][num_taps] +% rightBRIRReal => float32[BINAURAL_CONVBANDS][num_channels][num_taps] +% rightBRIRImag => float32[BINAURAL_CONVBANDS][num_channels][num_taps] +% CLDFB_NO_CHANNELS_MAX => uint16_t +% fastConvReverberationTimes => float32[CLDFB_NO_CHANNELS_MAX] +% fastConvReverberationEneCorrections => float32[CLDFB_NO_CHANNELS_MAX] +% + +[f_id, err_msg] = fopen(bin_file, 'wb'); + +if f_id == -1 + error('Could not open file %s for writing. Error message:\n%s', filename, err_msg); +end + +%% File header +% We need to get the chunksize of all IRs to get total size +% SHD HRIRs +% FOA +IR = FastConv_SHD_IR_FOA; +[~, binaural_convbands, num_channels, binaural_ntaps] = size(IR.IR); + +header = get_ivas_binary_header(ivas_path,'HRTF_READER_RENDERER_BINAURAL_FASTCONV', ['BINAURAL_INPUT_AUDIO_CONFIG_' IR.order]); +header.chunksize = header.chunksize + 4; % latency_s +header.chunksize = header.chunksize + 2; % BINAURAL_CONVBANDS +header.chunksize = header.chunksize + 2; % num_channels +header.chunksize = header.chunksize + 2; % num_taps +header.chunksize = header.chunksize + 4 * (binaural_convbands * num_channels * binaural_ntaps * 4 ); % HRTF L/R Re/Im + +IR.header = header; +FastConv_SHD_IR_FOA = IR; + +% HOA2 +IR = FastConv_SHD_IR_HOA2; +[~, binaural_convbands, num_channels, binaural_ntaps] = size(IR.IR); + +header = get_ivas_binary_header(ivas_path, 'HRTF_READER_RENDERER_BINAURAL_FASTCONV', ['BINAURAL_INPUT_AUDIO_CONFIG_' IR.order]); +header.chunksize = header.chunksize + 4; % latency_s +header.chunksize = header.chunksize + 2; % BINAURAL_CONVBANDS +header.chunksize = header.chunksize + 2; % num_channels +header.chunksize = header.chunksize + 2; % num_taps +header.chunksize = header.chunksize + 4 * (binaural_convbands * num_channels * binaural_ntaps * 4 ); % HRTF L/R Re/Im + +IR.header = header; +FastConv_SHD_IR_HOA2 = IR; + +% HOA3 +IR = FastConv_SHD_IR_HOA3; +[~, binaural_convbands, num_channels, binaural_ntaps] = size(IR.IR); + +header = get_ivas_binary_header(ivas_path, 'HRTF_READER_RENDERER_BINAURAL_FASTCONV', ['BINAURAL_INPUT_AUDIO_CONFIG_' IR.order]); +header.chunksize = header.chunksize + 4; % latency_s +header.chunksize = header.chunksize + 2; % BINAURAL_CONVBANDS +header.chunksize = header.chunksize + 2; % num_channels +header.chunksize = header.chunksize + 2; % num_taps +header.chunksize = header.chunksize + 4 * (binaural_convbands * num_channels * binaural_ntaps * 4 ); % HRTF L/R Re/Im + +IR.header = header; +FastConv_SHD_IR_HOA3 = IR; + +% SD HRIRs +IR = FastConv_SD_IR; +[~, binaural_convbands, num_channels, binaural_ntaps] = size(IR.IR); + +header = get_ivas_binary_header(ivas_path, 'HRTF_READER_RENDERER_BINAURAL_FASTCONV', 'BINAURAL_INPUT_AUDIO_CONFIG_COMBINED'); +header.chunksize = header.chunksize + 4; % latency_s +header.chunksize = header.chunksize + 2; % BINAURAL_CONVBANDS +header.chunksize = header.chunksize + 2; % num_channels +header.chunksize = header.chunksize + 2; % num_taps +header.chunksize = header.chunksize + 4 * (binaural_convbands * num_channels * binaural_ntaps * 4 ); % HRTF L/R Re/Im + +IR.header = header; +FastConv_SD_IR = IR; + +% SD BRIRs +IR = FastConv_SD_BRIR; +[~, binaural_convbands, num_channels, ~] = size(IR.IR); +cldfb_no_channels_max = IR.rev_param.kAna; + +header = get_ivas_binary_header(ivas_path, 'HRTF_READER_RENDERER_BINAURAL_FASTCONV_ROOM', 'BINAURAL_INPUT_AUDIO_CONFIG_COMBINED'); +header.chunksize = header.chunksize + 4; % latency_s +header.chunksize = header.chunksize + 2; % BINAURAL_CONVBANDS +header.chunksize = header.chunksize + 2; % num_channels +header.chunksize = header.chunksize + 2; % num_taps +header.chunksize = header.chunksize + 4 * (binaural_convbands * num_channels * IR.rev_param.NFilter * 4 ); % HRTF L/R Re/Im +header.chunksize = header.chunksize + 2; % CLDFB_NO_CHANNELS_MAX +header.chunksize = header.chunksize + cldfb_no_channels_max * 4; % rt60 +header.chunksize = header.chunksize + cldfb_no_channels_max * 4; % nrgLr + +IR.header = header; +FastConv_SD_BRIR = IR; + +% calculate the size of all chunks +HRTFs = {FastConv_SHD_IR_FOA, FastConv_SHD_IR_HOA2, FastConv_SHD_IR_HOA3, FastConv_SD_IR, FastConv_SD_BRIR}; +hrtf_data_size = 0; +total_file_size = 0; +for i = 1:length(HRTFs) + hrtf_data_size = hrtf_data_size + HRTFs{i}.header.chunksize; + total_file_size = total_file_size + 4 * 4; % chunk header 4 (u)int32 values +end + +total_file_size = total_file_size + 8; % 'IVASHRTF' (char[8]) +total_file_size = total_file_size + 4; % file size (int32) +total_file_size = total_file_size + 2; % number of HRTFs in file (int16) +total_file_size = total_file_size + 4; % HRTF size (int32) +total_file_size = total_file_size + hrtf_data_size; % size of all HRTF data chunks + +fwrite(f_id, 'IVASHRTF', 'char'); % identifier +fwrite(f_id, total_file_size, 'int32'); % file size +fwrite(f_id, length(HRTFs), 'int16'); % number of HRTFs +fwrite(f_id, hrtf_data_size, 'int32'); % max data size (bytes to read after this header) + +%% HRIRs + +% SHD HRIRs +SHD_HRIRs = {FastConv_SHD_IR_FOA, FastConv_SHD_IR_HOA2, FastConv_SHD_IR_HOA3}; +for i = 1:length(SHD_HRIRs) + IR = SHD_HRIRs{i}; + [~, binaural_convbands, num_channels, binaural_ntaps] = size(IR.IR); + + % write header for this chunk + fwrite(f_id, IR.header.renderer_type, 'int32'); + fwrite(f_id, IR.header.in_fmt, 'int32'); + fwrite(f_id, IR.header.fs, 'int32'); + fwrite(f_id, IR.header.chunksize, 'uint32'); + + fwrite(f_id, IR.latency_s, 'float32'); + fwrite(f_id, binaural_convbands, 'uint16'); + fwrite(f_id, num_channels, 'uint16'); + fwrite(f_id, binaural_ntaps, 'uint16'); + + for band = 1:binaural_convbands + for ch = 1:num_channels + fwrite(f_id, real(squeeze(IR.IR(1, band, ch, :))), 'float32'); + end + end + for band = 1:binaural_convbands + for ch = 1:num_channels + fwrite(f_id, imag(squeeze(IR.IR(1, band, ch, :))), 'float32'); + end + end + for band = 1:binaural_convbands + for ch = 1:num_channels + fwrite(f_id, real(squeeze(IR.IR(2, band, ch, :))), 'float32'); + end + end + for band = 1:binaural_convbands + for ch = 1:num_channels + fwrite(f_id, imag(squeeze(IR.IR(2, band, ch, :))), 'float32'); + end + end +end + +% SD HRIRs +IR = FastConv_SD_IR; +[~, binaural_convbands, num_channels, binaural_ntaps] = size(IR.IR); + +% write header for this chunk +fwrite(f_id, IR.header.renderer_type, 'int32'); +fwrite(f_id, IR.header.in_fmt, 'int32'); +fwrite(f_id, IR.header.fs, 'int32'); +fwrite(f_id, IR.header.chunksize, 'uint32'); + +fwrite(f_id, IR.latency_s, 'float32'); +fwrite(f_id, binaural_convbands, 'uint16'); +fwrite(f_id, num_channels, 'uint16'); +fwrite(f_id, binaural_ntaps, 'uint16'); +for band = 1:binaural_convbands + for ch = 1:num_channels + fwrite(f_id, real(squeeze(IR.IR(1, band, ch, :))), 'float32'); + end +end +for band = 1:binaural_convbands + for ch = 1:num_channels + fwrite(f_id, imag(squeeze(IR.IR(1, band, ch, :))), 'float32'); + end +end +for band = 1:binaural_convbands + for ch = 1:num_channels + fwrite(f_id, real(squeeze(IR.IR(2, band, ch, :))), 'float32'); + end +end +for band = 1:binaural_convbands + for ch = 1:num_channels + fwrite(f_id, imag(squeeze(IR.IR(2, band, ch, :))), 'float32'); + end +end + + +% SD BRIRs +IR = FastConv_SD_BRIR; +[~, binaural_convbands, num_channels, ~] = size(IR.IR); +cldfb_no_channels_max = IR.rev_param.kAna; + +% write header for this chunk +fwrite(f_id, IR.header.renderer_type, 'int32'); +fwrite(f_id, IR.header.in_fmt, 'int32'); +fwrite(f_id, IR.header.fs, 'int32'); +fwrite(f_id, IR.header.chunksize, 'uint32'); + +fwrite(f_id, IR.rev_param.latency_s, 'float32'); +fwrite(f_id, binaural_convbands, 'uint16'); +fwrite(f_id, num_channels, 'uint16'); +fwrite(f_id, IR.rev_param.NFilter, 'uint16'); +for band = 1:binaural_convbands + for ch = 1:num_channels + fwrite(f_id, real(squeeze(IR.IR(1, band, ch, 1:IR.rev_param.NFilter))), 'float32' ); + end +end +for band = 1:binaural_convbands + for ch = 1:num_channels + fwrite(f_id, imag(squeeze(IR.IR(1, band, ch, 1:IR.rev_param.NFilter))), 'float32' ); + end +end +for band = 1:binaural_convbands + for ch = 1:num_channels + fwrite(f_id, real(squeeze(IR.IR(2, band, ch, 1:IR.rev_param.NFilter))), 'float32' ); + end +end +for band = 1:binaural_convbands + for ch = 1:num_channels + fwrite(f_id, imag(squeeze(IR.IR(2, band, ch, 1:IR.rev_param.NFilter))), 'float32' ); + end +end + +fwrite(f_id, cldfb_no_channels_max, 'uint16'); +fwrite(f_id, IR.rev_param.rt60, 'float32'); +fwrite(f_id, IR.rev_param.nrgLr, 'float32'); + +fclose(f_id); + +end \ No newline at end of file diff --git a/scripts/binauralRenderer_interface/ivas_crend_sofa_to_rom_table_converter_readme.txt b/scripts/binauralRenderer_interface/mixer_conv_sofa_to_rom_table_converter_readme.txt similarity index 100% rename from scripts/binauralRenderer_interface/ivas_crend_sofa_to_rom_table_converter_readme.txt rename to scripts/binauralRenderer_interface/mixer_conv_sofa_to_rom_table_converter_readme.txt diff --git a/scripts/binauralRenderer_interface/param_bin/write_parametric_binauralizer_binary_data.m b/scripts/binauralRenderer_interface/param_bin/write_parametric_binauralizer_binary_data.m new file mode 100644 index 0000000000000000000000000000000000000000..21c013120aa56957b1311a330feaba46aeca7ca6 --- /dev/null +++ b/scripts/binauralRenderer_interface/param_bin/write_parametric_binauralizer_binary_data.m @@ -0,0 +1,99 @@ +%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% +% +% (C) 2022-2024 IVAS codec Public Collaboration with portions copyright Dolby International AB, Ericsson AB, +% Fraunhofer-Gesellschaft zur Foerderung der angewandten Forschung e.V., Huawei Technologies Co. LTD., +% Koninklijke Philips N.V., Nippon Telegraph and Telephone Corporation, Nokia Technologies Oy, Orange, +% Panasonic Holdings Corporation, Qualcomm Technologies, Inc., VoiceAge Corporation, and other +% contributors to this repository. All Rights Reserved. +% +% This software is protected by copyright law and by international treaties. +% The IVAS codec Public Collaboration consisting of Dolby International AB, Ericsson AB, +% Fraunhofer-Gesellschaft zur Foerderung der angewandten Forschung e.V., Huawei Technologies Co. LTD., +% Koninklijke Philips N.V., Nippon Telegraph and Telephone Corporation, Nokia Technologies Oy, Orange, +% Panasonic Holdings Corporation, Qualcomm Technologies, Inc., VoiceAge Corporation, and other +% contributors to this repository retain full ownership rights in their respective contributions in +% the software. This notice grants no license of any kind, including but not limited to patent +% license, nor is any license granted by implication, estoppel or otherwise. +% +% Contributors are required to enter into the IVAS codec Public Collaboration agreement before making +% contributions. +% +% This software is provided "AS IS", without any express or implied warranties. The software is in the +% development stage. It is intended exclusively for experts who have experience with such software and +% solely for the purpose of inspection. All implied warranties of non-infringement, merchantability +% and fitness for a particular purpose are hereby disclaimed and excluded. +% +% Any dispute, controversy or claim arising under or in relation to providing this software shall be +% submitted to and settled by the final, binding jurisdiction of the courts of Munich, Germany in +% accordance with the laws of the Federal Republic of Germany excluding its conflict of law rules and +% the United Nations Convention on Contracts on the International Sales of Goods. +% +%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% + + +function write_parametric_binauralizer_binary_data(filename, SHhrtf, T60, late_enes, early_enes) +% +% Writes HRIR & BRIR based data for parametric binauralizer into a binary file. +% +% write_parametric_binauralizer_binary_data(filename, SHhrtf, T60, late_enes, early_enes) +% +% filename : string +% name of the file to be written +% SHhrtf : array of shape (2, 16, 60) i.e., (BINAURAL_CHANNELS, HRTF_SH_CHANNELS, HRTF_NUM_BINS), complex-valued +% HRTF coefficients +% T60 : array of shape (60, 1), i.e., (CLDFB_NO_CHANNELS_MAX, 1), double +% late_enes : array of shape (1, 60), i.e., (1, CLDFB_NO_CHANNELS_MAX), double +% early_enes : array of shape (1, 60), i.e., (1, CLDFB_NO_CHANNELS_MAX), double +% +% +% Output file format: +% HRTFs +% HRTF_SH_CHANNELS => uint16_t +% HRTF_NUM_BINS => uint16_t +% hrtfShCoeffsRe => float[BINAURAL_CHANNELS][HRTF_SH_CHANNELS][HRTF_NUM_BINS]; +% hrtfShCoeffsIm => float[BINAURAL_CHANNELS][HRTF_SH_CHANNELS][HRTF_NUM_BINS]; +% +% BRIR-based reverb +% CLDFB_NO_CHANNELS_MAX => uint16_t +% parametricReverberationTimes => float[CLDFB_NO_CHANNELS_MAX]; +% parametricReverberationEneCorrections => float[CLDFB_NO_CHANNELS_MAX]; +% parametricEarlyPartEneCorrection => float[CLDFB_NO_CHANNELS_MAX]; +% + +[f_id, err_msg] = fopen(filename, 'wb'); + +if f_id == -1 + error('Could not open file %s for writing. Error message:\n%s', filename, err_msg); +end + +% HRTFs +n_chnls_bin = 2; +hrtf_sh_channels = size(SHhrtf, 2); +hrtf_num_bins = size(SHhrtf, 3); + +fwrite(f_id, hrtf_sh_channels, 'uint16'); +fwrite(f_id, hrtf_num_bins, 'uint16'); + +% hrtfShCoeffsRe +for bin_chnl_idx = 1:n_chnls_bin + for hrtf_chnl_idx = 1:hrtf_sh_channels + fwrite(f_id, real(SHhrtf(bin_chnl_idx, hrtf_chnl_idx, :)), 'float32'); % HRTF_NUM_BINS elements + end +end + +% hrtfShCoeffsIm +for bin_chnl_idx = 1:n_chnls_bin + for hrtf_chnl_idx = 1:hrtf_sh_channels + fwrite(f_id, imag(SHhrtf(bin_chnl_idx, hrtf_chnl_idx, :)), 'float32'); % HRTF_NUM_BINS elements + end +end + +% BRIR-based reverb +cldfb_no_channels_max = size(T60, 1); + +fwrite(f_id, cldfb_no_channels_max, 'uint16'); +fwrite(f_id, T60, 'float32'); % parametricReverberationTimes +fwrite(f_id, late_enes, 'float32'); % parametricReverberationEneCorrections +fwrite(f_id, early_enes, 'float32'); % parametricEarlyPartEneCorrection + +fclose(f_id); diff --git a/scripts/config/self_test_ltv_basop_encoder.prm b/scripts/config/self_test_ltv_basop_encoder.prm index 66f96903d2c333b6ad72c51f4a74c414c3db7bc6..ca20519cb207d44bc5604e7f8f37dd67f6e90f2b 100644 --- a/scripts/config/self_test_ltv_basop_encoder.prm +++ b/scripts/config/self_test_ltv_basop_encoder.prm @@ -276,8 +276,8 @@ ../IVAS_dec HOA3 48 bit testv/ltv48_HOA3.wav_sw_48-48_HOA3.tst // SBA 3OA bitrate switching from 13.2 kbps to 128 kbps, 48kHz in, 48kHz out, DTX on, HOA3 out -../IVAS_cod -dtx -sba 3 ../scripts/switchPaths/sw_13k2_to_128k_10fr.bin 48 testv/ltv48_HOA3.wav bit -../IVAS_dec HOA3 48 bit testv/ltv48_HOA3.wav_sw_48-48_DTX_HOA3.tst +//../IVAS_cod -dtx -sba 3 ../scripts/switchPaths/sw_13k2_to_128k_10fr.bin 48 testv/ltv48_HOA3.wav bit +//../IVAS_dec HOA3 48 bit testv/ltv48_HOA3.wav_sw_48-48_DTX_HOA3.tst // SBA 3OA bitrate switching from 13.2 kbps to 128 kbps, 32kHz in, 32kHz out, DTX on, HOA3 out ../IVAS_cod -dtx -sba 3 ../scripts/switchPaths/sw_13k2_to_128k_10fr.bin 32 testv/ltv32_HOA3.wav bit @@ -320,35 +320,35 @@ ../IVAS_dec EXT 48 bit testv/ltvOSBA_4ISM_p3OA48c.wav_EXT_512000_48-48.tst // OSBA planar 2OA 4ISM at 512 kbps, 48 kHz in, 48 kHz out, EXT out -../IVAS_cod -ism_sba 4 -2 testv/ltv48_OSBA_1ISM_HOA3_ISM1.csv testv/ltv48_OSBA_2ISM_HOA3_ISM2.csv testv/ltv48_OSBA_3ISM_HOA3_ISM3.csv testv/ltv48_OSBA_4ISM_HOA3_ISM4.csv 512000 48 testv/ltv48_OSBA_4ISM_HOA2.wav bit +../IVAS_cod -ism_sba 4 -2 testv/ltv48_OSBA_1ISM_HOA2_ISM1.csv testv/ltv48_OSBA_2ISM_HOA2_ISM2.csv testv/ltv48_OSBA_3ISM_HOA2_ISM3.csv testv/ltv48_OSBA_4ISM_HOA2_ISM4.csv 512000 48 testv/ltv48_OSBA_4ISM_HOA2.wav bit ../IVAS_dec EXT 48 bit testv/ltvOSBA_4ISM_p3OA48c.wav_EXT_512000_48-48.tst // OSBA FOA 4ISM at 512 kbps, 48kHz in, 48kHz out, EXT out -../IVAS_cod -ism_sba 4 1 testv/ltv48_OSBA_1ISM_HOA3_ISM1.csv testv/ltv48_OSBA_2ISM_HOA3_ISM2.csv testv/ltv48_OSBA_3ISM_HOA3_ISM3.csv testv/ltv48_OSBA_4ISM_HOA3_ISM4.csv 512000 48 testv/ltv48_OSBA_4ISM_FOA.wav bit +../IVAS_cod -ism_sba 4 1 testv/ltv48_OSBA_1ISM_FOA_ISM1.csv testv/ltv48_OSBA_2ISM_FOA_ISM2.csv testv/ltv48_OSBA_3ISM_FOA_ISM3.csv testv/ltv48_OSBA_4ISM_FOA_ISM4.csv 512000 48 testv/ltv48_OSBA_4ISM_FOA.wav bit ../IVAS_dec EXT 48 bit testv/ltv48_OSBA_4ISM_FOA.wav_EXT_512000_48-48.tst // OSBA FOA 4ISM at 512 kbps, 32kHz in, 48kHz out, EXT out -../IVAS_cod -ism_sba 4 1 testv/ltv48_OSBA_1ISM_HOA3_ISM1.csv testv/ltv48_OSBA_2ISM_HOA3_ISM2.csv testv/ltv48_OSBA_3ISM_HOA3_ISM3.csv testv/ltv48_OSBA_4ISM_HOA3_ISM4.csv 512000 32 testv/ltv32_OSBA_4ISM_FOA.wav bit +../IVAS_cod -ism_sba 4 1 testv/ltv48_OSBA_1ISM_FOA_ISM1.csv testv/ltv48_OSBA_2ISM_FOA_ISM2.csv testv/ltv48_OSBA_3ISM_FOA_ISM3.csv testv/ltv48_OSBA_4ISM_FOA_ISM4.csv 512000 32 testv/ltv32_OSBA_4ISM_FOA.wav bit ../IVAS_dec EXT 48 bit testv/ltv32_OSBA_4ISM_FOA.wav_EXT_512000_32-48.tst // OSBA FOA 4ISM at 384 kbps, 32kHz in, 32kHz out, EXT out -../IVAS_cod -ism_sba 4 1 testv/ltv48_OSBA_1ISM_HOA3_ISM1.csv testv/ltv48_OSBA_2ISM_HOA3_ISM2.csv testv/ltv48_OSBA_3ISM_HOA3_ISM3.csv testv/ltv48_OSBA_4ISM_HOA3_ISM4.csv 384000 32 testv/ltv32_OSBA_4ISM_FOA.wav bit +../IVAS_cod -ism_sba 4 1 testv/ltv48_OSBA_1ISM_FOA_ISM1.csv testv/ltv48_OSBA_2ISM_FOA_ISM2.csv testv/ltv48_OSBA_3ISM_FOA_ISM3.csv testv/ltv48_OSBA_4ISM_FOA_ISM4.csv 384000 32 testv/ltv32_OSBA_4ISM_FOA.wav bit ../IVAS_dec EXT 32 bit testv/ltv32_OSBA_4ISM_FOA.wav_EXT_384000_32-32.tst // OSBA FOA 4ISM at 256 kbps, 48kHz in, 48kHz out, EXT out -../IVAS_cod -ism_sba 4 1 testv/ltv48_OSBA_1ISM_HOA3_ISM1.csv testv/ltv48_OSBA_2ISM_HOA3_ISM2.csv testv/ltv48_OSBA_3ISM_HOA3_ISM3.csv testv/ltv48_OSBA_4ISM_HOA3_ISM4.csv 256000 48 testv/ltv48_OSBA_4ISM_FOA.wav bit +../IVAS_cod -ism_sba 4 1 testv/ltv48_OSBA_1ISM_FOA_ISM1.csv testv/ltv48_OSBA_2ISM_FOA_ISM2.csv testv/ltv48_OSBA_3ISM_FOA_ISM3.csv testv/ltv48_OSBA_4ISM_FOA_ISM4.csv 256000 48 testv/ltv48_OSBA_4ISM_FOA.wav bit ../IVAS_dec EXT 48 bit testv/ltv48_OSBA_4ISM_FOA.wav_EXT_256000_48-48.tst // OSBA FOA 3ISM at 128 kbps, 48kHz in, 48kHz out, EXT out -../IVAS_cod -ism_sba 3 1 testv/ltv48_OSBA_1ISM_HOA3_ISM1.csv testv/ltv48_OSBA_2ISM_HOA3_ISM2.csv testv/ltv48_OSBA_3ISM_HOA3_ISM3.csv 128000 48 testv/ltv48_OSBA_3ISM_FOA.wav bit +../IVAS_cod -ism_sba 3 1 testv/ltv48_OSBA_1ISM_FOA_ISM1.csv testv/ltv48_OSBA_2ISM_FOA_ISM2.csv testv/ltv48_OSBA_3ISM_FOA_ISM3.csv 128000 48 testv/ltv48_OSBA_3ISM_FOA.wav bit ../IVAS_dec EXT 48 bit testv/ltv48_OSBA_3ISM_FOA.wav_EXT_128000_48-48.tst // OSBA FOA 1ISM at 48 kbps, 16kHz in, 16kHz out, EXT out -../IVAS_cod -ism_sba 1 1 testv/ltv48_OSBA_1ISM_HOA3_ISM1.csv 32000 16 testv/ltv16_OSBA_1ISM_FOA.wav bit +../IVAS_cod -ism_sba 1 1 testv/ltv48_OSBA_1ISM_FOA_ISM1.csv 32000 16 testv/ltv16_OSBA_1ISM_FOA.wav bit ../IVAS_dec EXT 16 bit testv/ltv16_OSBA_1ISM_FOA.wav_EXT_32000_16-16.tst // OSBA FOA 1ISM at 32 kbps, 48kHz in, 48kHz out, EXT out -../IVAS_cod -ism_sba 1 1 testv/ltv48_OSBA_1ISM_HOA3_ISM1.csv 32000 48 testv/ltv48_OSBA_1ISM_FOA.wav bit +../IVAS_cod -ism_sba 1 1 testv/ltv48_OSBA_1ISM_FOA_ISM1.csv 32000 48 testv/ltv48_OSBA_1ISM_FOA.wav bit ../IVAS_dec EXT 48 bit testv/ltv48_OSBA_1ISM_FOA.wav_EXT_32000_48-48.tst // OSBA 3OA 4ISM bitrate switching 16.4 to 512, 48kHz in, 48kHz out, EXT out @@ -405,32 +405,32 @@ ../IVAS_dec EXT 32 bit testv/ltv48_OSBA_3ISM_HOA3.wav_EXT_sw_48-32.tst // OSBA 3ISM 2OA at bitrate switching 13.2 to 512 kbps, 48kHz in, 32kHz out, EXT out -../IVAS_cod -ism_sba 3 2 testv/ltv48_OSBA_1ISM_HOA3_ISM1.csv testv/ltv48_OSBA_2ISM_HOA3_ISM2.csv testv/ltv48_OSBA_3ISM_HOA3_ISM3.csv ../scripts/switchPaths/sw_13k2_512k.bin 48 testv/ltv48_OSBA_3ISM_HOA2.wav bit +../IVAS_cod -ism_sba 3 2 testv/ltv48_OSBA_1ISM_HOA2_ISM1.csv testv/ltv48_OSBA_2ISM_HOA2_ISM2.csv testv/ltv48_OSBA_3ISM_HOA2_ISM3.csv ../scripts/switchPaths/sw_13k2_512k.bin 48 testv/ltv48_OSBA_3ISM_HOA2.wav bit ../IVAS_dec EXT 32 bit testv/ltv48_OSBA_3ISM_HOA2.wav_EXT_sw_48-32.tst // OSBA 2OA 4ISM at 384 kbps, 48kHz in, 48kHz out, EXT out -../IVAS_cod -ism_sba 4 2 testv/ltv48_OSBA_1ISM_HOA3_ISM1.csv testv/ltv48_OSBA_2ISM_HOA3_ISM2.csv testv/ltv48_OSBA_3ISM_HOA3_ISM3.csv testv/ltv48_OSBA_4ISM_HOA3_ISM4.csv 384000 48 testv/ltv48_OSBA_4ISM_HOA2.wav bit +../IVAS_cod -ism_sba 4 2 testv/ltv48_OSBA_1ISM_HOA2_ISM1.csv testv/ltv48_OSBA_2ISM_HOA2_ISM2.csv testv/ltv48_OSBA_3ISM_HOA2_ISM3.csv testv/ltv48_OSBA_4ISM_HOA2_ISM4.csv 384000 48 testv/ltv48_OSBA_4ISM_HOA2.wav bit ../IVAS_dec EXT 48 bit testv/ltv48_OSBA_4ISM_HOA2.wav_EXT_384000_48-48.tst // OSBA 2OA 3ISM at 96 kbps, 48kHz in, 48kHz out, EXT out -../IVAS_cod -ism_sba 3 2 testv/ltv48_OSBA_1ISM_HOA3_ISM1.csv testv/ltv48_OSBA_2ISM_HOA3_ISM2.csv testv/ltv48_OSBA_3ISM_HOA3_ISM3.csv 96000 48 testv/ltv48_OSBA_3ISM_HOA2.wav bit +../IVAS_cod -ism_sba 3 2 testv/ltv48_OSBA_1ISM_HOA2_ISM1.csv testv/ltv48_OSBA_2ISM_HOA2_ISM2.csv testv/ltv48_OSBA_3ISM_HOA2_ISM3.csv 96000 48 testv/ltv48_OSBA_3ISM_HOA2.wav bit ../IVAS_dec EXT 48 bit testv/ltv48_OSBA_3ISM_HOA2.wav_EXT_96000_48-48.tst // OSBA 2OA 3ISM at 384 kbps, 16kHz in, 16kHz out, EXT out -../IVAS_cod -ism_sba 3 2 testv/ltv48_OSBA_1ISM_HOA3_ISM1.csv testv/ltv48_OSBA_2ISM_HOA3_ISM2.csv testv/ltv48_OSBA_3ISM_HOA3_ISM3.csv 384000 16 testv/ltv16_OSBA_3ISM_HOA2.wav bit +../IVAS_cod -ism_sba 3 2 testv/ltv48_OSBA_1ISM_HOA2_ISM1.csv testv/ltv48_OSBA_2ISM_HOA2_ISM2.csv testv/ltv48_OSBA_3ISM_HOA2_ISM3.csv 384000 16 testv/ltv16_OSBA_3ISM_HOA2.wav bit ../IVAS_dec EXT 16 bit testv/ltv16_OSBA_3ISM_HOA2.wav_EXT56000_16-16.tst // OSBA 2OA 2ISM at 64 kbps, 32kHz in, 16kHz out, EXT out -../IVAS_cod -ism_sba 2 2 testv/ltv48_OSBA_1ISM_HOA3_ISM1.csv testv/ltv48_OSBA_2ISM_HOA3_ISM2.csv 48000 32 testv/ltv32_OSBA_2ISM_HOA2.wav bit +../IVAS_cod -ism_sba 2 2 testv/ltv48_OSBA_1ISM_HOA2_ISM1.csv testv/ltv48_OSBA_2ISM_HOA2_ISM2.csv 48000 32 testv/ltv32_OSBA_2ISM_HOA2.wav bit ../IVAS_dec EXT 16 bit testv/ltv32_OSBA_2ISM_HOA2.wav_EXT_64000_32-16.tst // OSBA 2OA 2ISM at 48 kbps, 48kHz in, 48kHz out, EXT out -../IVAS_cod -ism_sba 2 2 testv/ltv48_OSBA_1ISM_HOA3_ISM1.csv testv/ltv48_OSBA_2ISM_HOA3_ISM2.csv 48000 48 testv/ltv48_OSBA_2ISM_HOA2.wav bit +../IVAS_cod -ism_sba 2 2 testv/ltv48_OSBA_1ISM_HOA2_ISM1.csv testv/ltv48_OSBA_2ISM_HOA2_ISM2.csv 48000 48 testv/ltv48_OSBA_2ISM_HOA2.wav bit ../IVAS_dec EXT 48 bit testv/ltv48_OSBA_2ISM_HOA2.wav_EXT_48000_48-48.tst // OSBA 2OA 1ISM at 24.4 kbps, 48kHz in, 48kHz out, EXT out -../IVAS_cod -ism_sba 1 2 testv/ltv48_OSBA_1ISM_HOA3_ISM1.csv 24400 48 testv/ltv48_OSBA_1ISM_EXT.wav bit -../IVAS_dec EXT 48 bit testv/ltv48_OSBA_1ISM_EXT.wav_EXT_24400_48-48.tst +../IVAS_cod -ism_sba 1 2 testv/ltv48_OSBA_1ISM_HOA2_ISM1.csv 24400 48 testv/ltv48_OSBA_1ISM_HOA2.wav bit +../IVAS_dec EXT 48 bit testv/ltv48_OSBA_1ISM_HOA2.wav_EXT_24400_48-48.tst // OMASA 2TC 4ISM at br sw techs 13.2 to 512 kbps start 384 kbps, 48kHz in, 48kHz out, BINAURAL out ../IVAS_cod -ism_masa 4 2 testv/ltv48_OMASA_1ISM_2TC_ISM1.csv testv/ltv48_OMASA_2ISM_2TC_ISM2.csv testv/ltv48_OMASA_3ISM_2TC_ISM3.csv testv/ltv48_OMASA_4ISM_2TC_ISM4.csv testv/ltv48_OMASA_4ISM_2TC.met ../scripts/switchPaths/sw_13k2_512k_2fr_start_384k_omasatechs_4ism.bin 48 testv/ltv48_OMASA_4ISM_2TC.wav bit diff --git a/scripts/create_histogram_summary.py b/scripts/create_histogram_summary.py deleted file mode 100644 index af9a11de2ab48e10f897301da566b826ca06c0c7..0000000000000000000000000000000000000000 --- a/scripts/create_histogram_summary.py +++ /dev/null @@ -1,156 +0,0 @@ -#!/usr/bin/env python3 - -import argparse -import math -import numpy as np - -# These next three lines are added as a precaution in case the gitlab runner -# needs DISPLAY to render the plots, even if they are written to file. -import matplotlib - -matplotlib.use("Agg") -import matplotlib.pyplot as plt -import csv -import os -from parse_xml_report import IVAS_FORMATS, EVS_FORMATS, IVAS_CATEGORIES, EVS_CATEGORIES - -""" -Parses a CSV report and creates a summary report. -""" - - -# Main routine -if __name__ == "__main__": - parser = argparse.ArgumentParser( - description="Parses a CSV report and creates a summary report." - ) - parser.add_argument( - "csv_report", - type=str, - help="CSV report file of test cases, e.g. report.csv", - ) - parser.add_argument( - "csv_summary", type=str, help="Output CSV file, e.g. summary.csv" - ) - parser.add_argument( - "csv_image", - type=str, - nargs="?", - help="Summary image file, e.g. summary.png", - default=None, - ) - parser.add_argument( - "--measure", - type=str, - nargs=1, - help="Measure, any of: MLD, DIFF, SSNR, ODG, default: MLD", - default=["MLD"], - ) - parser.add_argument( - "--evs", - action="store_true", - help="Parse using EVS 26.444 formats", - default=False, - ) - parser.add_argument( - "--diff", - action="store_true", - help="Use limits for diff scores", - default=False, - ) - args = parser.parse_args() - csv_report = args.csv_report - csv_summary = args.csv_summary - csv_image = args.csv_image - measure = args.measure[0] - if args.evs: - FORMATS = EVS_FORMATS - CATEGORIES = EVS_CATEGORIES - else: - FORMATS = IVAS_FORMATS - CATEGORIES = IVAS_CATEGORIES - if args.diff: - limits_per_measure = { - "MLD": ("MLD", None), - "DIFF": ("MAXIMUM ABS DIFF", None), - "SSNR": ("MIN_SSNR", None), - "ODG": ("MIN_ODG", None), - "DELTA_ODG": ("DELTA_ODG", None), - } - else: - limits_per_measure = { - "MLD": ("MLD", [0, 1, 2, 3, 4, 5, 10, 20, math.inf]), - "DIFF": ("MAXIMUM ABS DIFF", [0, 16, 256, 1024, 2048, 4096, 8192, 16384, 32769]), - "SSNR": ("MIN_SSNR", [-math.inf, 0, 10, 20, 30, 40, 40, 50, 60, 100]), - "ODG": ("MIN_ODG", [-5, -2, -1, -0.5, -0.4, -0.3, -0.2, -0.1, 0, 0.1, 0.5]), - "DELTA_ODG": ("DELTA_ODG", [-5, -2, -1, -0.5, -0.4, -0.3, -0.2, -0.1, 0, 0.1, 0.5]), - } - (measure_label, limits) = limits_per_measure[measure] - - # Load CSV report - results_sorted = {} - with open(csv_report, "r") as fp: - reader = csv.reader(fp, delimiter=";") - header = next(reader) - keys = header[1:] - for row in reader: - testcase = row[0] - results_sorted[testcase] = {} - for k, val in zip(keys, row[1:]): - results_sorted[testcase][k] = val - - if limits is None: - vals = [float(x) for x in [m[measure_label] for m in results_sorted.values() if m[measure_label] != "None" and m[measure_label] != ""]] - start = min(vals) - f = 10 ** (2 - int(np.floor(np.log10(abs(start)))) - 1) - start = np.floor(start*f)/f - step = (max(vals) - start)/10 - f = 10 ** (2 - int(np.floor(np.log10(abs(step)))) - 1) - step = np.ceil(step*f)/f - limits = np.arange(start, 10*step, step) - - # Output CSV file - with open(csv_summary, "w") as fp: - limits_labels = [f"{a:g}" for a in limits] + ["","None"] # Put None cases in separate bin - headerline = f"Format;Category;" + ";".join(limits_labels) + "\n" - fp.write(headerline) - - for fmt in FORMATS: - fig, ax = plt.subplots() - bottom = np.zeros(len(limits_labels)) - for cat in CATEGORIES: - values = [ - x - for x in [ - m[measure_label] - for m in results_sorted.values() - if m["Format"] == fmt and m["Category"] == cat - ] - ] - # Create separate bin for None (errors) - val = [float(x) for x in values if x != "None" and x != ""] - none = [sum([1 for x in values if x == "None" or x == ""])] - hist, _ = np.histogram(val, limits) - data = np.array(list(hist) + [0] + none + [0]) - - # CSV output - line = f"{fmt};{cat};{'; '.join(map(str,data))}\n" - fp.write(line) - - # Matplotlib histogram - ax.bar(limits_labels, data, 1, align='edge', edgecolor='black', linewidth=0.5, label=cat, bottom=bottom) - bottom += data - - # Histogram layout - ax.set_title(fmt) - ax.legend(loc="best") - ax.set_xlabel(measure_label) - if "DIFF" in measure_label: - ax.set_xticks(range(len(limits_labels)), limits_labels, rotation=35) - ax.set_ylabel("Number of test cases") - - fig.set_figheight(4) - fig.set_figwidth(6) - if csv_image: - base, ext = os.path.splitext(csv_image) - plt.savefig(f"{base}_{fmt}{ext}") diff --git a/scripts/create_histograms.py b/scripts/create_histograms.py new file mode 100644 index 0000000000000000000000000000000000000000..b2a9f0ec1c810de755e05e95433e3897320fff40 --- /dev/null +++ b/scripts/create_histograms.py @@ -0,0 +1,200 @@ +#!/usr/bin/env python3 + +import argparse +import math +import pathlib +import sys +import pandas as pd +import numpy as np +from typing import List + + +# hack for avoiding missing DISPLAY variable in headless CI runners +import matplotlib + +matplotlib.use("Agg") +import matplotlib.pyplot as plt + + +BINS_FOR_MEASURES = { + "MLD": [0, 1, 2, 3, 4, 5, 10, 20, math.inf], + "MAX_ABS_DIFF": [0, 16, 256, 1024, 2048, 4096, 8192, 16384, 32769], + "MIN_SSNR": [-math.inf, 0, 10, 20, 30, 40, 40, 50, 60, 100, math.inf], + "MIN_ODG": [-5, -4, -3, -2, -1, -0.5, -0.4, -0.3, -0.2, -0.1, 0, 0.1, 0.5], + "DELTA_ODG": [-5, -4, -3, -2, -1, -0.5, -0.4, -0.3, -0.2, -0.1, 0, 0.1, 0.5], +} + +DEFAULT_MEASURES = ["MAX_ABS_DIFF", "MLD", "MIN_SSNR", "MIN_ODG"] + +### !!! Note: this is duplicated in tests/constatns.py. If you change this here, ALSO ADAPT IT THERE!!! +### (importing from there failed for unknown reasons in some jobs on some runners and I don't have time to properly investigate this...) +### below lines are the original solution, kept here for reference + +# HERE = pathlib.Path(__file__).parent +# ROOT_DIR = HERE.parent +# sys.path.append(str(ROOT_DIR)) +# from tests.constants import CAT_NORMAL, CAT_BITRATE_SWITCHING, CAT_DTX, CAT_JBM, CAT_PLC + +CAT_NORMAL = "normal operation" +CAT_DTX = "DTX" +CAT_PLC = "PLC" +CAT_BITRATE_SWITCHING = "bitrate switching" +CAT_JBM = "JBM" + +COLORS_FOR_CATEGORIES = { + CAT_DTX: "tab:blue", + CAT_PLC: "tab:orange", + CAT_NORMAL: "tab:green", + CAT_JBM: "tab:red", + CAT_BITRATE_SWITCHING: "tab:purple", +} + + +def get_bins_for_diff(data: pd.Series): + return np.round(np.linspace(data.min(), data.max(), num=10), decimals=2) + + +def create_histograms( + df: pd.DataFrame, + measures: List[str], + output_folder: pathlib.Path, + display_only: bool, + bins_for_measures=BINS_FOR_MEASURES, + prefix="", + write_out_histograms=False, +): + formats = df["format"].unique() + categories = df["category"].unique() + + if not display_only or write_out_histograms: + output_folder.mkdir(exist_ok=True, parents=True) + + for measure in measures: + measure_in_df = prefix + measure + bins = bins_for_measures.get(measure, get_bins_for_diff(df[measure_in_df])) + x = [f"{x}" for x in bins] + ["", "ERROR"] + + df_hist = pd.DataFrame(columns=["format", "category"] + x) + hist_row_count = 0 + + for fmt in formats: + fig, ax = plt.subplots() + ax.xaxis.set_major_formatter("{x:.1f}") + bottom = np.zeros(len(x)) + for cat in categories: + data_mask = np.logical_and(df["format"] == fmt, df["category"] == cat) + df_slice = df[data_mask] + error_mask = df_slice["result"] == "ERROR" + n_errors = np.sum(error_mask) + df_slice = df_slice[np.logical_not(error_mask)] + + counts, _ = np.histogram(df_slice[measure_in_df], bins) + + data = np.concatenate([counts, [0], [n_errors], [0]]) + ax.bar( + x, + data, + 1, + align="edge", + edgecolor="black", + linewidth=0.5, + label=cat, + bottom=bottom, + color=COLORS_FOR_CATEGORIES[cat], + ) + bottom += data + + hist_row = [fmt, cat] + list(counts) + [0] + [0, n_errors] + df_hist.loc[hist_row_count] = hist_row + hist_row_count += 1 + + # Histogram layout + ax.set_title(fmt) + ax.legend(loc="best") + ax.set_xlabel(measure) + if "DIFF" in measure or len(bins_for_measures) == 0: + ax.set_xticks(range(len(x)), x, rotation=35) + else: + ax.set_xticks(range(len(x)), x) + ax.set_ylabel("Number of test cases") + + fig.set_figheight(4) + fig.set_figwidth(6) + plt.tight_layout() + + if not display_only: + image_file = f"histogram_{measure}_{fmt}.png" + image_path = output_folder.joinpath(image_file) + plt.savefig(image_path) + plt.close(fig) + + if write_out_histograms: + df_hist.to_csv( + output_folder.joinpath(f"histogram_{measure}.csv"), index=False + ) + + if display_only: + plt.show() + + +if __name__ == "__main__": + parser = argparse.ArgumentParser( + description="Parses a csv file generated by parse_xml_report and creates histograms for the given measures." + ) + parser.add_argument( + "csv_report", + type=str, + help="CSV report file as generated by parse_xml_report.py", + ) + parser.add_argument( + "output_folder", type=pathlib.Path, help="Output folder for writing the " + ) + parser.add_argument( + "--display-only", + action="store_true", + help="Do not write the output files, but display the graphs instead.", + ) + parser.add_argument( + "--no-bins", + action="store_true", + help="""Do not use the hardcoded bins for creating the spectrograms. +Use this for visualising diff scores.""", + ) + allowed_measures = " ".join(BINS_FOR_MEASURES.keys()) + parser.add_argument( + "--measures", + nargs="+", + default=DEFAULT_MEASURES, + help=f"Measures to plot from the csv file. One of {allowed_measures}", + ) + parser.add_argument( + "--prefix", + default="", + help="Common suffix to use when collecting measures from the input csv file", + ) + parser.add_argument( + "--write-out-histograms", + action="store_true", + help="Write out the histogram values to csv", + ) + args = parser.parse_args() + df = pd.read_csv(args.csv_report) + + # filter out missing format/category values + mask_format_missing = df["format"].isna() + mask_category_missing = df["category"].isna() + df = df[~mask_format_missing | ~mask_category_missing] + + bins_for_measures = BINS_FOR_MEASURES + if args.no_bins: + bins_for_measures = {} + + create_histograms( + df, + args.measures, + args.output_folder, + args.display_only, + bins_for_measures, + args.prefix, + args.write_out_histograms, + ) diff --git a/scripts/diff_report.py b/scripts/diff_report.py index 5ab64e956fc65bedeefb39137ebc474a837dd860..0aefeaccdcb251eb1426c63d967dda23e5dec378 100644 --- a/scripts/diff_report.py +++ b/scripts/diff_report.py @@ -33,25 +33,29 @@ the United Nations Convention on Contracts on the International Sales of Goods. import pandas as pd import argparse import sys -import os -import pathlib COLUMNS_TO_COMPARE = [ "MLD", - "MAXIMUM ABS DIFF", + "MAX_ABS_DIFF", "MIN_SSNR", "MIN_ODG", ] + def main(args): - df_ref = pd.read_csv(args.csv_ref, sep=";") - df_test = pd.read_csv(args.csv_test, sep=";") + df_ref = pd.read_csv(args.csv_ref).sort_values( + by=["testcase", "format", "category"] + ) + df_test = pd.read_csv(args.csv_test).sort_values( + by=["testcase", "format", "category"] + ) for col in COLUMNS_TO_COMPARE: df_ref[col] = df_test[col] - df_ref[col] - df_ref.to_csv(args.csv_diff, index=False, sep=";") + df_ref.to_csv(args.csv_diff, index=False) return 0 + if __name__ == "__main__": parser = argparse.ArgumentParser() parser.add_argument("csv_ref") diff --git a/scripts/parse_xml_report.py b/scripts/parse_xml_report.py index 1f2157ca8f5b6dc30cdd53e9bf080b9a0e8afba3..948bda43086f1145b34babfbbefcf4182d2aaae6 100644 --- a/scripts/parse_xml_report.py +++ b/scripts/parse_xml_report.py @@ -1,186 +1,180 @@ #!/usr/bin/env python3 import argparse -import re -import math -import numpy as np +import pandas as pd from xml.etree import ElementTree +from collections import Counter +from typing import Optional +from enum import Enum + + +SPLIT_STRING = "_split" +WHOLE_STRING = "_whole" + + +class Result(str, Enum): + ERROR = "ERROR" + FAIL = "FAIL" + PASS = "PASS" + + +class TestcaseParser(dict): + def __init__(self, testcases: list): + super().__init__() + + for tc in testcases: + self.parse_testcase(tc) + + def parse_testcase(self, testcase): + """ + Get all properties + name for a testcase + """ + + filename = testcase.get( + "file", testcase.get("classname").replace(".", "/") + ".py" + ) + fulltestname = filename + "::" + testcase.get("name") + + result = get_result_from_testcase(testcase) + # for ERRORS, two testcases are recorded, one with FAIL and one with ERROR + # if we already have this testcase, do a sanity check and set result to ERROR + if fulltestname in self: + results = [self[fulltestname]["result"], result] + assert any(r == Result.ERROR for r in results) + self[fulltestname]["result"] = Result.ERROR + return + + ret = {} + ret["testcase"] = fulltestname + ret["result"] = result + properties = { + p.get("name"): p.get("value") for p in testcase.findall(".//property") + } + + ### handle split comparison results + split_props = {k: v for k, v in properties.items() if SPLIT_STRING in k} + whole_props = {k: v for k, v in properties.items() if WHOLE_STRING in k} + other_props = { + k: v + for k, v in properties.items() + if WHOLE_STRING not in k and SPLIT_STRING not in k + } + + if len(split_props) > 0 and len(whole_props) > 0: + measures_from_split = set( + [m.split(SPLIT_STRING)[0] for m in split_props.keys()] + ) + measures_from_whole = set( + [m.split(WHOLE_STRING)[0] for m in whole_props.keys()] + ) + assert measures_from_split == measures_from_whole + measures = measures_from_whole + + # collect existing split suffixes by evaluating one of the measures only + # get one measure from set and add it back immediately + m_tmp = measures.pop() + measures.add(m_tmp) + + splits = sorted( + [ + k.split(SPLIT_STRING)[-1] + for k in split_props.keys() + if k.startswith(m_tmp) + ] + ) + + # record each split under a separate key + # the dict per key has the same fulltestname and an additional key "split" + # this way, the resulting DataFrame in the end can be split by testnames + for s in splits: + split_key = f"{fulltestname} - {s}" + ret_split = {"testcase": fulltestname, "split": s} + for m in measures: + ret_split.update({m: split_props[m + SPLIT_STRING + f"{s}"]}) + + ret_split.update(other_props) + self[split_key] = ret_split + + # it can be the case that there are no splits defined in the pytest suite, e.g. for the renderer + # then, there are only "_whole" values recorded where we only need to remove the suffix + # this if also handles the split case - if there are splits, there was also a "_whole" comparison done + if len(whole_props) > 0: + properties = { + k.replace(WHOLE_STRING, ""): v for k, v in whole_props.items() + } + properties["split"] = "whole" + properties.update(other_props) + + ret.update(properties) + self[fulltestname] = ret + + def to_df(self) -> pd.DataFrame: + testcases = list(self.values()) + df = pd.DataFrame(testcases) + return df + + +def xml_to_dataframe(xml_report: str) -> pd.DataFrame: + tree = ElementTree.parse(xml_report) + root = tree.getroot() + + testcases = root[0].findall("testcase") + testcases = [tc for tc in testcases if tc.find("skipped") is None] + + testcase_parser = TestcaseParser(testcases) + testcase_df = testcase_parser.to_df() + + return testcase_df + + +def get_result_from_testcase(testcase: ElementTree.Element) -> str: + if testcase.find("failure") is not None: + testresult = Result.FAIL + elif testcase.find("error") is not None: + testresult = Result.ERROR + else: + testresult = Result.PASS + + return testresult + + +def main(xml_report: str, csv_file: str, split_csv_file: Optional[str]): + df = xml_to_dataframe(xml_report) + n_testcases = len(df) + count = Counter(df["result"]) + + if split_csv_file is not None: + mask_errors = df["result"] == Result.ERROR + mask_whole = df["split"] == "whole" + mask_single = mask_errors | mask_whole + df_split = df[~mask_single] + df_split.to_csv(split_csv_file, index=False) + + df = df[mask_single] + + df.to_csv(csv_file, index=False) + + print( + f"Parsed testsuite with {n_testcases} tests: {count[Result.PASS]} passes, {count[Result.FAIL]} failures and {count[Result.ERROR]} errors." + ) + -""" -Parse a junit report and create a summary report. -""" - -PROPERTIES = ["MLD", "MAXIMUM ABS DIFF", "MIN_SSNR", "MIN_ODG"] - -IVAS_FORMATS = { - "Stereo": r"stereo", - "ISM": r"ISM", - "Multichannel": r"Multi-channel|MC", - "MASA": r"(? {encoder_config_name}, skipping") + continue + else: + logger.info(f"Found EXT dec in {mode} -> {encoder_config_name}, removing") + + for encoder_config_name in tuple(modes["OSBA"].keys()): + if not "FOA" in encoder_config_name: + continue + + logger.info(f"Removing OSBA FOA encoder config {encoder_config_name}") + del modes["OSBA"][encoder_config_name] + + if args.inline: + with open(args.input.name, "w") as f: + json.dump(modes, f, indent=4) + else: + json.dump(modes, args.output, indent=4) + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/ubsan_basop.supp b/scripts/ubsan_basop.supp new file mode 100644 index 0000000000000000000000000000000000000000..bfd00f357312259871d5b67a802699393a92a559 --- /dev/null +++ b/scripts/ubsan_basop.supp @@ -0,0 +1,32 @@ +# From the scheduled sanitizer tests with self_test.py +implicit-integer-sign-change:ACcontextMapping_fx.c +implicit-integer-sign-change:bitstream_fx.c +implicit-integer-sign-change:enh64.c +implicit-signed-integer-truncation:enh1632.c +implicit-signed-integer-truncation:enhUL32.c +shift-base:enh64.c +shift-exponent:basop32.c +shift-exponent:enh64.c +bounds:ifft_rel.c +bounds:log2.c +bounds:tcx_ltp_fx.c +bounds:trans_direct_fx.c +bounds:trans_inv_fx.c +signed-integer-overflow:enh64.c +implicit-integer-sign-change:ACcontextMapping_enc_fx.c +implicit-integer-sign-change:bass_psfilter_enc_fx.c +implicit-integer-sign-change:cod4t64_fx.c +implicit-integer-sign-change:core_enc_ol_fx.c +implicit-integer-sign-change:enc_acelp_fx.c +implicit-integer-sign-change:enc_prm_fx.c +implicit-integer-sign-change:hq_classifier_enc_fx.c +implicit-integer-sign-change:inov_enc_fx.c +implicit-integer-sign-change:lead_indexing_fx.c +implicit-integer-sign-change:range_enc_fx.c +implicit-integer-sign-change:tcx_utils_enc_fx.c +bounds:enc_acelp_fx.c +bounds:enc_gain_fx.c +bounds:pit_enc_fx.c +bounds:pitch_ol2_fx.c +bounds:tcx_ltp_enc_fx.c +bounds:dec_acelp_fx.c diff --git a/tests/cmp_pcm.py b/tests/cmp_pcm.py index 127820525a6895c8dcb46daf236137710297466d..56dc5cd63c6f0a5428769379bcff82cd88499361 100755 --- a/tests/cmp_pcm.py +++ b/tests/cmp_pcm.py @@ -7,7 +7,7 @@ import tempfile import re import subprocess from pathlib import Path -from typing import Optional +from typing import Optional, List THIS_PATH = os.path.join(os.getcwd(), __file__) sys.path.append(os.path.join(os.path.dirname(THIS_PATH), "../scripts")) @@ -41,8 +41,9 @@ def cmp_pcm( ref_jbm_tf: Optional[Path] = None, cut_jbm_tf: Optional[Path] = None, quiet: Optional[bool] = False, - scalefac=1, -) -> tuple[int, str]: + scalefac: int = 1, + split_idx: np.ndarray = np.empty(0), +) -> tuple[List[int], List[str]]: """ Compare 2 PCM files for bitexactness """ @@ -67,13 +68,13 @@ def cmp_pcm( if fs1 != fs2: reason = "FAIL: Sampling rate differs." - return 1, reason + return [1], [reason] # In case number of channels do not match, fail already now. Could happen in case of # comparison to input with for a non-passthrough mode. if s1.shape[1] != s2.shape[1]: reason = "FAIL: Number of channels differ." - return 1, reason + return [1], [reason] handle_differing_lengths = "fail" if allow_differing_lengths: @@ -88,108 +89,115 @@ def cmp_pcm( if get_mld: reason += " - MLD: None" - return 1, reason - - # Apply scalefac if specified. Useful in case scaling has been applied on the input, and the inverse is scaling is supplied in scalefac. - if scalefac != 1: - s1 = np.round(s1*scalefac, 0) # Need rounding for max abs diff search - s2 = np.round(s2*scalefac, 0) - - cmp_result = pyaudio3dtools.audioarray.compare( - s1, - s2, - fs, - per_frame=False, - get_mld=get_mld, - get_ssnr=get_ssnr, - ssnr_thresh_low=-50, - ref_jbm_tf=ref_jbm_tf, - test_jbm_tf=cut_jbm_tf, - handle_differing_lengths=handle_differing_lengths, - ) + return [1], [reason] + + output_differs_parts = [] + reason_parts = [] + + for s1, s2 in zip(np.split(s1, split_idx), np.split(s2, split_idx)): + # Apply scalefac if specified. Useful in case scaling has been applied on the input, and the inverse is scaling is supplied in scalefac. + if scalefac != 1: + s1 = np.round(s1 * scalefac, 0) # Need rounding for max abs diff search + s2 = np.round(s2 * scalefac, 0) + + cmp_result = pyaudio3dtools.audioarray.compare( + s1, + s2, + fs, + per_frame=False, + get_mld=get_mld, + get_ssnr=get_ssnr, + ssnr_thresh_low=-50, + ref_jbm_tf=ref_jbm_tf, + test_jbm_tf=cut_jbm_tf, + handle_differing_lengths=handle_differing_lengths, + ) + + output_differs = 0 + reason = "SUCCESS: Files are bitexact" + + if not cmp_result["bitexact"] and cmp_result["max_abs_diff"] <= abs_tol: + reason = "SUCCESS: Maximum absolute diff below threshold" + elif not cmp_result["bitexact"]: + diff_msg = f"MAXIMUM ABS DIFF ==> {cmp_result['max_abs_diff']} at sample num {cmp_result['max_abs_diff_pos_sample']} (assuming {nchannels} channels)" + first_msg = f"First diff found at sample num {cmp_result['first_diff_pos_sample']} in channel {cmp_result['first_diff_pos_channel']}, frame {cmp_result['first_diff_pos_frame']} (assuming {nchannels} channels, {fs} sampling rate)" + print(diff_msg, file=output_target) + print(first_msg, file=output_target) - output_differs = 0 - reason = "SUCCESS: Files are bitexact" - - if not cmp_result["bitexact"] and cmp_result["max_abs_diff"] <= abs_tol: - reason = "SUCCESS: Maximum absolute diff below threshold" - elif not cmp_result["bitexact"]: - diff_msg = f"MAXIMUM ABS DIFF ==> {cmp_result['max_abs_diff']} at sample num {cmp_result['max_abs_diff_pos_sample']} (assuming {nchannels} channels)" - first_msg = f"First diff found at sample num {cmp_result['first_diff_pos_sample']} in channel {cmp_result['first_diff_pos_channel']}, frame {cmp_result['first_diff_pos_frame']} (assuming {nchannels} channels, {fs} sampling rate)" - print(diff_msg, file=output_target) - print(first_msg, file=output_target) - - reason = f"Non-BE - MAXIMUM ABS DIFF: {cmp_result['max_abs_diff']}" - output_differs = 1 - - if get_mld: - mld_msg = f"MLD: {cmp_result['MLD']}" - reason += " - " + mld_msg - print(mld_msg, file=output_target) - - if cmp_result["MLD"] <= mld_lim: - output_differs = 0 - reason += f" <= {mld_lim}" - else: - reason += f" > {mld_lim}" - - if get_ssnr: - reason += " - " - for i, s in enumerate(cmp_result["SSNR"], start=1): - msg = f"Channel {i} SSNR: {s}" - reason += msg + " - " - - if get_odg: - for n in range(nchannels): - pqeval_output = pqevalaudio_wrapper(s1[:, n], s2[:, n], fs) + reason = f"Non-BE - MAXIMUM ABS DIFF: {cmp_result['max_abs_diff']}" + output_differs = 1 + if get_mld: + mld_msg = f"MLD: {cmp_result['MLD']}" + reason += " - " + mld_msg + print(mld_msg, file=output_target) + + if cmp_result["MLD"] <= mld_lim: + output_differs = 0 + reason += f" <= {mld_lim}" + else: + reason += f" > {mld_lim}" + + if get_ssnr: + reason += " - " + for i, s in enumerate(cmp_result["SSNR"], start=1): + msg = f"Channel {i} SSNR: {s}" + reason += msg + " - " + + if get_odg: + for n in range(nchannels): + pqeval_output = pqevalaudio_wrapper(s1[:, n], s2[:, n], fs) + + match_odg = re.search(ODG_PATTERN_PQEVALAUDIO, pqeval_output) + odg = float(match_odg.groups()[0]) + msg = f"Channel {n} ODG: {odg}" + reason += " - " + msg + print(msg) + + if get_odg_bin: + odg_files = {} + for f in [odg_input, odg_test, odg_ref]: + # Load PEAQ test files and ensure 48 kHz sampling rate + s, fs = pyaudio3dtools.audiofile.readfile( + f, nchannels, fs, outdtype=np.int16 + ) + odg_files[f] = np.clip( + pyaudio3dtools.audioarray.resample(s.astype(float), fs, 48000), + -32768, + 32767, + ).astype(np.int16) + + pqeval_output = pqevalaudio_wrapper( + odg_files[odg_input], odg_files[odg_ref], 48000 + ) match_odg = re.search(ODG_PATTERN_PQEVALAUDIO, pqeval_output) - odg = float(match_odg.groups()[0]) - msg = f"Channel {n} ODG: {odg}" - reason += " - " + msg - print(msg) - - if get_odg_bin: - odg_files = {} - for f in [odg_input, odg_test, odg_ref]: - # Load PEAQ test files and ensure 48 kHz sampling rate - s, fs = pyaudio3dtools.audiofile.readfile( - f, nchannels, fs, outdtype=np.int16 + try: + odg_ref = float(match_odg.groups()[0]) + except AttributeError: + raise OdgParsingFailed("Could not get Odg for ref signal") + + pqeval_output = pqevalaudio_wrapper( + odg_files[odg_input], odg_files[odg_test], 48000 ) - odg_files[f] = np.clip( - pyaudio3dtools.audioarray.resample(s.astype(float), fs, 48000), - -32768, - 32767, - ).astype(np.int16) - - pqeval_output = pqevalaudio_wrapper( - odg_files[odg_input], odg_files[odg_ref], 48000 - ) - match_odg = re.search(ODG_PATTERN_PQEVALAUDIO, pqeval_output) - try: - odg_ref = float(match_odg.groups()[0]) - except AttributeError: - raise OdgParsingFailed("Could not get Odg for ref signal") - - pqeval_output = pqevalaudio_wrapper( - odg_files[odg_input], odg_files[odg_test], 48000 - ) - match_odg = re.search(ODG_PATTERN_PQEVALAUDIO, pqeval_output) - try: - odg_test = float(match_odg.groups()[0]) - except AttributeError: - raise OdgParsingFailed("Could not get Odg for test signal") + match_odg = re.search(ODG_PATTERN_PQEVALAUDIO, pqeval_output) + try: + odg_test = float(match_odg.groups()[0]) + except AttributeError: + raise OdgParsingFailed("Could not get Odg for test signal") + + odg = odg_test - odg_ref # Todo: store both rather than difference? - odg = odg_test - odg_ref # Todo: store both rather than difference? + msg = f"Delta-ODG: {odg}" + reason += " - " + msg + print(msg, file=output_target) - msg = f"Delta-ODG: {odg}" - reason += " - " + msg - print(msg, file=output_target) + if quiet: + output_target.close() - if quiet: - output_target.close() + output_differs_parts.append(output_differs) + reason_parts.append(reason) - return output_differs, reason + return output_differs_parts, reason_parts class OdgParsingFailed(Exception): @@ -262,12 +270,18 @@ if __name__ == "__main__": parser.add_argument("--get_odg", action="store_true") parser.add_argument("--get_ssnr", action="store_true") parser.add_argument("--allow_differing_lengths", action="store_true") - parser.add_argument("--scalefac", type=float, default=1, dest="scalefac", help="Scale factor to be applied before comparing the output. Useful when input scaling has been applied.") + parser.add_argument( + "--scalefac", + type=float, + default=1, + dest="scalefac", + help="Scale factor to be applied before comparing the output. Useful when input scaling has been applied.", + ) parser.add_argument("--quiet", action="store_true") args = vars(parser.parse_args()) args["nchannels"] = out_config_2_nchannels(args.pop("out_config")) result, msg = cmp_pcm(**args) - print(msg) - sys.exit(result) + print(msg[0]) + sys.exit(result[0]) diff --git a/tests/codec_be_on_mr_nonselection/test_param_file.py b/tests/codec_be_on_mr_nonselection/test_param_file.py index ac2d84d1e7af645500434e259bb2337f341c7f85..d2acacd39eacbb6513f0e01b5804f1ac6990c659 100644 --- a/tests/codec_be_on_mr_nonselection/test_param_file.py +++ b/tests/codec_be_on_mr_nonselection/test_param_file.py @@ -53,6 +53,8 @@ from tests.conftest import ( parse_properties, compare_dmx_signals, log_dbg_msg, + get_split_idx, + get_format_from_enc_opts, ) from tests.testconfig import PARAM_FILE from tests.constants import ( @@ -60,6 +62,11 @@ from tests.constants import ( MAX_ENC_STATS_DIFF, SCRIPTS_DIR, MAX_ENC_DIFF, + CAT_NORMAL, + CAT_DTX, + CAT_BITRATE_SWITCHING, + CAT_JBM, + CAT_PLC, ) from tests.renderer.utils import check_and_makedir, binauralize_input_and_output @@ -217,6 +224,7 @@ def test_param_file_tests( get_odg_bin, compare_to_input, compare_enc_dmx, + split_comparison, ): enc_opts, dec_opts, sim_opts, eid_opts = param_file_test_dict[test_tag] @@ -249,6 +257,7 @@ def test_param_file_tests( get_odg_bin, compare_to_input, compare_enc_dmx, + split_comparison, ) @@ -281,6 +290,7 @@ def run_test( get_odg_bin, compare_to_input, compare_enc_dmx, + split_comparison, ): # If compare_to_input is set, only run pass-through test cases if compare_to_input: @@ -294,6 +304,10 @@ def run_test( "All non-passthrough modes are skipped when --compare-to-input is set" ) + testcase_props = {} + testcase_props["format"] = get_format_from_enc_opts(enc_opts) + testcase_props["category"] = CAT_NORMAL + tag_str = convert_test_string_to_tag(test_tag) # evaluate encoder options @@ -313,6 +327,9 @@ def run_test( bitrate = enc_split.pop() in_sr = sampling_rate + if "-dtx" in enc_opts: + testcase_props["category"] = CAT_DTX + # bitrate can be a filename: change it to an absolute path if not bitrate.isdigit(): if compare_enc_dmx: @@ -320,11 +337,21 @@ def run_test( "Rate switching + --compare_enc_dmx currently skipped due to DEBUGGING code limitations with varying number of transport channels" ) bitrate = Path(bitrate[3:]).absolute() + testcase_props["category"] = CAT_BITRATE_SWITCHING testv_base = testv_file.split("/")[-1] if testv_base.endswith(".pcm"): testv_base = testv_base[:-4] + if sim_opts != "": + testcase_props["category"] = CAT_JBM + if eid_opts != "": + testcase_props["category"] = CAT_PLC + + if update_ref != 1: + for k, v in testcase_props.items(): + dut_encoder_frontend.record_property(k, v) + assert bitstream_file == "bit" # in the parameter file, only "bit" is used as bitstream file name # -> construct bitstream filename @@ -371,8 +398,8 @@ def run_test( # avoid double recording of the encoder diff if encoder_only: - props = parse_properties(cmp_result_msg, False, [MAX_ENC_DIFF]) - for k, v in props.items(): + result_props = parse_properties(cmp_result_msg, False, [MAX_ENC_DIFF]) + for k, v in result_props.items(): dut_encoder_frontend.record_property(k, v) if encoder_only: @@ -413,7 +440,6 @@ def run_test( ) # check for eid-xor command line - if eid_opts != "": eid_split = eid_opts.split() assert len(eid_split) >= 3, "eid-xor expects at least 3 parameters" @@ -588,7 +614,14 @@ def run_test( ref_file = ref_output_file fs = int(sampling_rate) * 1000 - output_differs, reason = cmp_pcm( + + ### run the comparison tools + split_idx = np.empty(0) + prop_suffix = [""] + + # 1. run comparison on whole files - this is done always, regardless of the presence of --split_comparison + + output_differs_parts, reason_parts = cmp_pcm( ref_file, dut_output_file, out_config_2_nchannels(output_config), @@ -606,13 +639,55 @@ def run_test( ref_jbm_tf=ref_tracefile_dec, cut_jbm_tf=dut_tracefile_dec, scalefac=test_info.config.option.scalefac, + split_idx=split_idx, ) - cmp_result_msg += reason + # 2. run comparison on split files if --split_comparison is given + # for JBM cases, comparison will fail because of length mismatch beetween split wav files and tracefiles + # -> skip split comparison for these cases + if split_comparison and not sim_opts: + split_idx = get_split_idx(str(Path(testv_file).stem), int(sampling_rate)) + + # this extra if takes care of cases where no splits are found, e.g. the "NOOP" case in the self_test_ltv prm file + # if this would not be there, then the comparison of the whole file would run twice + if len(split_idx) > 0: + output_differs_splits, reason_splits = cmp_pcm( + ref_file, + dut_output_file, + out_config_2_nchannels(output_config), + fs, + get_mld=get_mld, + mld_lim=get_mld_lim, + abs_tol=abs_tol, + allow_differing_lengths=allow_differing_lengths, + get_ssnr=get_ssnr, + get_odg=get_odg, + get_odg_bin=get_odg_bin, + odg_input=odg_input, + odg_test=odg_test, + odg_ref=odg_ref, + ref_jbm_tf=ref_tracefile_dec, + cut_jbm_tf=dut_tracefile_dec, + scalefac=test_info.config.option.scalefac, + split_idx=split_idx, + ) + output_differs_parts += output_differs_splits + reason_parts += reason_splits - props = parse_properties(cmp_result_msg, output_differs, props_to_record) - for k, v in props.items(): - dut_decoder_frontend.record_property(k, v) + # separate if to also record the whole-file comparison for JBM cases with "_whole" + if split_comparison: + prop_suffix = ["_whole"] + [ + f"_split{i:03d}" for i in range(1, len(split_idx) + 1) + ] + + for output_differs, reason, suffix in zip( + output_differs_parts, reason_parts, prop_suffix + ): + result_props = parse_properties( + reason, output_differs, props_to_record, suffix + ) + for k, v in result_props.items(): + dut_decoder_frontend.record_property(k, v) metadata_differs = False @@ -647,19 +722,22 @@ def run_test( if enc_test_result: pytest.fail("Too high difference in encoder statistics found.") + at_least_one_output_differs = any(output_differs_parts) + # TODO + reason = reason_parts[0] if tracefile_last_rtp_numbers_differ: pytest.fail( "Last RTP sequence num in tracefiles differ for JBM decoding - Not all frames were decoded in both ref and dut." ) elif get_mld and get_mld_lim > 0: - if output_differs: + if at_least_one_output_differs: pytest.fail(reason) else: - if output_differs or metadata_differs: + if at_least_one_output_differs or metadata_differs: msg = "Difference between ref and dut in " - if output_differs and metadata_differs: + if at_least_one_output_differs and metadata_differs: msg += f"output ({reason}) and metadata" - elif output_differs: + elif at_least_one_output_differs: msg += f"output only ({reason})" elif metadata_differs: msg += "metadata only" diff --git a/tests/codec_be_on_mr_nonselection/test_sba.py b/tests/codec_be_on_mr_nonselection/test_sba.py index f0e81e82d2dc71dcb689c066714de02437020f4e..6b1746f858a749059de4160906da9e3ddf97c7cf 100644 --- a/tests/codec_be_on_mr_nonselection/test_sba.py +++ b/tests/codec_be_on_mr_nonselection/test_sba.py @@ -35,6 +35,7 @@ __doc__ = """ import os import pytest +import numpy as np from cut_bs import cut_from_start from pathlib import Path @@ -44,9 +45,18 @@ from tests.conftest import ( EncoderFrontend, compare_dmx_signals, parse_properties, + get_split_idx, ) from ..cmp_stats_files import cmp_stats_files -from ..constants import TESTV_DIR, MAX_ENC_FILE_LENGTH_DIFF, MAX_ENC_STATS_DIFF +from ..constants import ( + CAT_BITRATE_SWITCHING, + CAT_DTX, + CAT_NORMAL, + CAT_PLC, + TESTV_DIR, + MAX_ENC_FILE_LENGTH_DIFF, + MAX_ENC_STATS_DIFF, +) from tests.testconfig import use_ltv from tests.renderer.utils import check_and_makedir, binauralize_input_and_output @@ -116,6 +126,7 @@ def test_pca_enc( get_enc_stats, compare_to_input, compare_enc_dmx, + split_comparison, ): pca = True bitrate = "256000" @@ -128,6 +139,7 @@ def test_pca_enc( cut_testv = True cut_gain = "1.0" plc_pattern = None + testcase_props = {"format": "SBA", "category": CAT_NORMAL} if "ltv" in tag: tag = f"ltv{sampling_rate}_FOA" @@ -140,6 +152,10 @@ def test_pca_enc( input_config = SBA_FORMAT[abs(int(sba_order))] + if update_ref != 1: + for k, v in testcase_props.items(): + dut_encoder_frontend.record_property(k, v) + if not decoder_only: sba_enc( dut_encoder_frontend, @@ -198,6 +214,7 @@ def test_pca_enc( get_odg=get_odg, get_odg_bin=get_odg_bin, compare_to_input=compare_to_input, + split_comparison=split_comparison, ) @@ -237,6 +254,7 @@ def test_sba_enc_system( get_enc_stats, compare_to_input, compare_enc_dmx, + split_comparison, ): plc_pattern = None pca = False @@ -246,6 +264,7 @@ def test_sba_enc_system( cut_gain = "1.0" plc_pattern = None cut_testv = True + testcase_props = {"format": "SBA", "category": CAT_NORMAL} if dtx == "1" and bitrate not in ["13200", "16400", "24400", "32000", "64000"]: # skip high bitrates for DTX until DTX issue is resolved @@ -283,6 +302,16 @@ def test_sba_enc_system( cut_gain = "1.0" input_config = SBA_FORMAT[abs(int(sba_order))] + if dtx: + testcase_props["category"] = CAT_DTX + try: + int(bitrate) + except ValueError: + testcase_props["category"] = CAT_BITRATE_SWITCHING + + if update_ref != 1: + for k, v in testcase_props.items(): + dut_encoder_frontend.record_property(k, v) if not decoder_only: ref_stats_file, dut_stats_file = sba_enc( @@ -338,6 +367,7 @@ def test_sba_enc_system( props = parse_properties( enc_test_result_msg, enc_test_result != 0, props_to_record ) + props.update(testcase_props) for k, v in props.items(): dut_encoder_frontend.record_property(k, v) @@ -377,6 +407,7 @@ def test_sba_enc_system( get_odg=get_odg, get_odg_bin=get_odg_bin, compare_to_input=compare_to_input, + split_comparison=split_comparison, ) @@ -408,6 +439,7 @@ def test_spar_hoa2_enc_system( get_enc_stats, compare_to_input, compare_enc_dmx, + split_comparison, ): sampling_rate = "48" pca = False @@ -420,6 +452,7 @@ def test_spar_hoa2_enc_system( cut_gain = "1.0" plc_pattern = None cut_testv = False + testcase_props = {"format": "SBA", "category": CAT_NORMAL} if "ltv" in tag: tag = f"ltv{sampling_rate}_HOA2" @@ -429,6 +462,9 @@ def test_spar_hoa2_enc_system( assert 0 input_config = SBA_FORMAT[abs(int(sba_order))] + if update_ref != 1: + for k, v in testcase_props.items(): + dut_encoder_frontend.record_property(k, v) if not decoder_only: ref_stats_file, dut_stats_file = sba_enc( @@ -484,6 +520,7 @@ def test_spar_hoa2_enc_system( props = parse_properties( enc_test_result_msg, enc_test_result != 0, props_to_record ) + props.update(testcase_props) for k, v in props.items(): dut_encoder_frontend.record_property(k, v) @@ -523,6 +560,7 @@ def test_spar_hoa2_enc_system( get_odg=get_odg, get_odg_bin=get_odg_bin, compare_to_input=compare_to_input, + split_comparison=split_comparison, ) @@ -554,6 +592,7 @@ def test_spar_hoa3_enc_system( get_enc_stats, compare_to_input, compare_enc_dmx, + split_comparison, ): sampling_rate = "48" pca = False @@ -566,6 +605,7 @@ def test_spar_hoa3_enc_system( cut_gain = "1.0" plc_pattern = None cut_testv = False + testcase_props = {"format": "SBA", "category": CAT_NORMAL} if "ltv" in tag: tag = f"ltv{sampling_rate}_HOA3" @@ -575,6 +615,9 @@ def test_spar_hoa3_enc_system( assert 0 input_config = SBA_FORMAT[abs(int(sba_order))] + if update_ref != 1: + for k, v in testcase_props.items(): + dut_encoder_frontend.record_property(k, v) if not decoder_only: ref_stats_file, dut_stats_file = sba_enc( @@ -624,6 +667,7 @@ def test_spar_hoa3_enc_system( props = parse_properties( enc_test_result_msg, enc_test_result != 0, props_to_record ) + props.update(testcase_props) for k, v in props.items(): dut_encoder_frontend.record_property(k, v) @@ -663,6 +707,7 @@ def test_spar_hoa3_enc_system( get_odg=get_odg, get_odg_bin=get_odg_bin, compare_to_input=compare_to_input, + split_comparison=split_comparison, ) @@ -698,6 +743,7 @@ def test_sba_enc_BWforce_system( get_enc_stats, compare_to_input, compare_enc_dmx, + split_comparison, ): sid = 0 plc_pattern = None @@ -709,6 +755,7 @@ def test_sba_enc_BWforce_system( cut_testv = False sampling_rate = sample_rate_bw_idx[0] max_bw = sample_rate_bw_idx[1] + testcase_props = {"format": "SBA", "category": CAT_NORMAL} if dtx == "1" and bitrate not in ["32000", "64000"]: # skip high bitrates for DTX until DTX issue is resolved @@ -729,6 +776,16 @@ def test_sba_enc_BWforce_system( assert 0 input_config = SBA_FORMAT[abs(int(sba_order))] + if dtx: + testcase_props["category"] = CAT_DTX + try: + int(bitrate) + except ValueError: + testcase_props["category"] = CAT_BITRATE_SWITCHING + + if update_ref != 1: + for k, v in testcase_props.items(): + dut_encoder_frontend.record_property(k, v) if not decoder_only: ref_stats_file, dut_stats_file = sba_enc( @@ -784,6 +841,7 @@ def test_sba_enc_BWforce_system( props = parse_properties( enc_test_result_msg, enc_test_result != 0, props_to_record ) + props.update(testcase_props) for k, v in props.items(): dut_encoder_frontend.record_property(k, v) @@ -823,6 +881,7 @@ def test_sba_enc_BWforce_system( get_odg=get_odg, get_odg_bin=get_odg_bin, compare_to_input=compare_to_input, + split_comparison=split_comparison, ) @@ -862,6 +921,7 @@ def test_sba_plc_system( get_enc_stats, compare_to_input, compare_enc_dmx, + split_comparison, ): sid = 0 pca = False @@ -869,6 +929,7 @@ def test_sba_plc_system( sba_order = "+1" cut_testv = True output_config = "FOA" + testcase_props = {"format": "SBA", "category": CAT_NORMAL} if dtx == "1" and bitrate not in ["13200", "16400", "24400", "32000", "64000"]: # skip high bitrates for DTX until DTX issue is resolved @@ -905,6 +966,18 @@ def test_sba_plc_system( cut_gain = "1.0" input_config = SBA_FORMAT[abs(int(sba_order))] + if dtx: + testcase_props["category"] = CAT_DTX + try: + int(bitrate) + except ValueError: + testcase_props["category"] = CAT_BITRATE_SWITCHING + if plc_pattern is not None: + testcase_props["category"] = CAT_PLC + + if update_ref != 1: + for k, v in testcase_props.items(): + dut_encoder_frontend.record_property(k, v) if not decoder_only: sba_enc( @@ -964,6 +1037,7 @@ def test_sba_plc_system( get_odg=get_odg, get_odg_bin=get_odg_bin, compare_to_input=compare_to_input, + split_comparison=split_comparison, ) @@ -1140,6 +1214,7 @@ def sba_dec( get_odg=False, get_odg_bin=False, compare_to_input=False, + split_comparison=False, ): dut_pkt_dir = f"{dut_base_path}/sba_bs/pkt" ref_pkt_dir = f"{reference_path}/sba_bs/pkt" @@ -1251,7 +1326,14 @@ def sba_dec( allow_differing_lengths = True sampling_rate_Hz = int(sampling_rate) * 1000 - cmp_result, reason = cmp_pcm( + + ### run the comparison tools + split_idx = np.empty(0) + prop_suffix = [""] + + # 1. run comparison on whole files - this is done always, regardless of the presence of --split_comparison + + output_differs_parts, reason_parts = cmp_pcm( ref_out_file, dut_out_file, output_config, @@ -1267,13 +1349,51 @@ def sba_dec( odg_test=odg_test, odg_ref=odg_ref, scalefac=test_info.config.option.scalefac, + split_idx=split_idx, ) - text_to_parse = reason - props = parse_properties(text_to_parse, cmp_result != 0, props_to_record) - for k, v in props.items(): - dut_decoder_frontend.record_property(k, v) + # 2. run comparison on split files if --split_comparison is given + # in the "cut until bitstream starts with SID" case, we can't do the split comp as the indices would not match anymore + if split_comparison and sid != 1: + input_file = f"{test_vector_path}/{tag}.wav" + split_idx = get_split_idx(str(Path(input_file).stem), int(sampling_rate)) + + # this extra if takes care of cases where no splits are found, e.g. the "NOOP" case in the self_test_ltv prm file + # if this would not be there, then the comparison of the whole file would run twice + if len(split_idx) > 0: + output_differs_splits, reason_splits = cmp_pcm( + ref_out_file, + dut_out_file, + output_config, + sampling_rate_Hz, + get_mld=get_mld, + mld_lim=get_mld_lim, + abs_tol=abs_tol, + allow_differing_lengths=allow_differing_lengths, + get_ssnr=get_ssnr, + get_odg=get_odg, + get_odg_bin=get_odg_bin, + odg_input=odg_input, + odg_test=odg_test, + odg_ref=odg_ref, + scalefac=test_info.config.option.scalefac, + split_idx=split_idx, + ) + output_differs_parts += output_differs_splits + reason_parts += reason_splits + + if split_comparison: + prop_suffix = ["_whole"] + [ + f"_split{i:03d}" for i in range(1, len(split_idx) + 1) + ] + + for output_differs, reason, suffix in zip( + output_differs_parts, reason_parts, prop_suffix + ): + props = parse_properties(reason, output_differs, props_to_record, suffix) + for k, v in props.items(): + dut_decoder_frontend.record_property(k, v) # report compare result - if cmp_result != 0: - pytest.fail(text_to_parse) + if output_differs_parts[0] != 0: + pytest.fail(reason_parts[0]) diff --git a/tests/conftest.py b/tests/conftest.py index 3b75cbc5570f91cd98259ef7a31fe7464e9b8d8c..5d3632a762c2818b21363114c83a964b5b5ece90 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -36,7 +36,6 @@ import logging import os import re import json -import shutil from tests import testconfig import pytest import platform @@ -46,16 +45,16 @@ from subprocess import TimeoutExpired, run from tempfile import NamedTemporaryFile from shutil import move import tempfile -from typing import Optional, Union +from typing import Optional, Union, List import numpy as np from .constants import ( - # MAX_ENC_DIFF_NAME_PATTERN, DMX_DIFF, DMX_MLD, DMX_SSNR, MAX_ENC_DIFF_PARAM_NAME, MLD_PATTERN, MAX_DIFF_PATTERN, + SPLIT_IDX, SSNR_PATTERN, ENC_AUX_FILES, ODG_PATTERN, @@ -306,6 +305,13 @@ def pytest_addoption(parser): default=1, ) + parser.addoption( + "--split-comparison", + action="store_true", + help="If given, split the output files by the provided indices", + ) + + @pytest.fixture(scope="session", autouse=True) def update_ref(request): """ @@ -365,6 +371,7 @@ def get_odg(request): """ return request.config.option.odg + @pytest.fixture(scope="session", autouse=True) def get_odg_bin(request): """ @@ -440,6 +447,11 @@ def test_info(request): pytest.fail(request.error) +@pytest.fixture(scope="session") +def split_comparison(request): + return request.config.option.split_comparison + + class EncoderFrontend: def __init__(self, path, enc_type, record_property, timeout=None) -> None: self._path = Path(path).absolute() @@ -1157,7 +1169,9 @@ def props_to_record( return props -def parse_properties(text_to_parse: str, output_differs: bool, props_to_record: list): +def parse_properties( + text_to_parse: str, output_differs: bool, props_to_record: list, suffix: str = "" +): """ Record the given properties in the report by parsing their values from the text. """ @@ -1167,7 +1181,7 @@ def parse_properties(text_to_parse: str, output_differs: bool, props_to_record: for prop in props_to_record: if prop == MLD or prop == DMX_MLD: mld = float(re.search(MLD_PATTERN, text_to_parse).groups(1)[0]) - props[prop] = mld + props[prop + suffix] = mld elif prop == MAX_ABS_DIFF or prop == DMX_DIFF: max_diff = 0 if output_differs: @@ -1175,33 +1189,35 @@ def parse_properties(text_to_parse: str, output_differs: bool, props_to_record: max_diff = match.groups(1)[0] else: raise MaxDiffPatternNotFound() - props[prop] = max_diff + props[prop + suffix] = max_diff elif prop == SSNR or prop == DMX_SSNR: ssnrs = re.findall(SSNR_PATTERN, text_to_parse) min_ssnr = min(ssnrs) min_ssnr_channel = ssnrs.index(min_ssnr) - prefix = "MIN" if prop == SSNR else "DMX" - props[f"{prefix}_SSNR"] = min_ssnr - props[f"{prefix}_SSNR_CHANNEL"] = min_ssnr_channel + propname = "MIN_SSNR" + if prop == DMX_SSNR: + propname = "DMX_MIN_SSNR" + props[propname + suffix] = min_ssnr + props[f"{propname}_CHANNEL" + suffix] = min_ssnr_channel elif prop == ODG: odgs = re.findall(ODG_PATTERN, text_to_parse) min_odg = min(odgs) min_odg_channel = odgs.index(min_odg) - props["MIN_ODG"] = min_odg - props["MIN_ODG_CHANNEL"] = min_odg_channel + props["MIN_ODG" + suffix] = min_odg + props["MIN_ODG_CHANNEL" + suffix] = min_odg_channel elif prop == MAX_ENC_DIFF: search_result = re.search(MAX_ENC_DIFF_PATTERN, text_to_parse) max_enc_diff_ratio = 0.0 max_enc_diff_param_name = "" if search_result: max_enc_diff_param_name, _, max_enc_diff_ratio = search_result.groups(0) - props[MAX_ENC_DIFF] = float(max_enc_diff_ratio) - props[MAX_ENC_DIFF_PARAM_NAME] = max_enc_diff_param_name + props[MAX_ENC_DIFF + suffix] = float(max_enc_diff_ratio) + props[MAX_ENC_DIFF_PARAM_NAME + suffix] = max_enc_diff_param_name elif prop == DELTA_ODG: delta_odg = re.search(DELTA_ODG_PATTERN, text_to_parse) if delta_odg: - props["DELTA_ODG"] = delta_odg.groups(1)[0] + props["DELTA_ODG" + suffix] = delta_odg.groups(1)[0] return props @@ -1217,26 +1233,86 @@ def compare_dmx_signals(ref_dmx_files, dut_dmx_files, fs) -> dict: ref_dmx_files, dmx_file_ref_tmp.name, out_nchans=nchannels, - in_fs=fs*1000, + in_fs=fs * 1000, ) pyaudio3dtools.audiofile.combinefiles( dut_dmx_files, dmx_file_dut_tmp.name, out_nchans=nchannels, - in_fs=fs*1000, + in_fs=fs * 1000, ) dmx_differs, reason = cmp_pcm( dmx_file_ref_tmp.name, dmx_file_dut_tmp.name, nchannels, - fs*1000, + fs * 1000, get_mld=True, get_ssnr=True, quiet=True, ) + dmx_differs = dmx_differs[0] + reason = reason[0] dmx_props = [DMX_DIFF, DMX_MLD, DMX_SSNR] prop_results = parse_properties(reason, dmx_differs, dmx_props) return prop_results + + +def get_split_idx(input_file: str, sampling_rate_khz: int) -> Optional[np.ndarray]: + """ + Return array for splitting the output file before doing the comparison. + + If no list of indices is available for the given input file, an empty array is returned. + """ + assert sampling_rate_khz in [16, 32, 48] + + input_file = input_file.lower() + if "omasa" in input_file: + format = "_".join(input_file.split("_")[1:-1]) + elif "osba" in input_file: + if "foa" in input_file: + format = "osba_foa" + else: + format = "osba_hoa" + else: + format = input_file.split("_")[-1].lower() + idx = SPLIT_IDX.get(format, np.empty(0)) + + # copy is important because we modify the array below for fs != 16 + # without copy, the constant would be modified and future split values would be wrong + idx = idx.copy() + + if len(idx) > 0 and sampling_rate_khz != 16: + idx *= sampling_rate_khz // 16 + + return idx + + +IVAS_ENC_FORMATS = { + "sba": "SBA", + "masa": "MASA", + "ism_sba": "OSBA", + "ism_masa": "OMASA", + "ism": "ISM", + "mc": "Multichannel", + "stereo_dmx_evs": "Stereo DMX EVS", + "stereo": "Stereo", +} +# NOTE: the blank at the end is important to prevent e.g. "-ism_masa" matching on "-ism" only +PATTERN_IVAS_ENC_FORMAT = re.compile(r"-(" + r"|".join(IVAS_ENC_FORMATS.keys()) + ") ") + + +def get_format_from_enc_opts(enc_opts: str) -> str: + """ + Parse the encoder format from the encoder options by searching for any of the + '-' arguments. If none of them is given, encoder will run in EVS mode. + """ + format = "Mono" + m = re.search(PATTERN_IVAS_ENC_FORMAT, enc_opts) + if m is not None: + enc_format_str = m.groups()[0] + format = IVAS_ENC_FORMATS[enc_format_str] + + return format diff --git a/tests/constants.py b/tests/constants.py index 9990db37cc690d2a52eb983df927740eacf5b46b..d70d1251943f6b675613f0210b6f1f8675bf4fea 100644 --- a/tests/constants.py +++ b/tests/constants.py @@ -8,17 +8,17 @@ TESTV_DIR = SCRIPTS_DIR.joinpath("testv") # Properties to record MLD = "MLD" -MAX_ABS_DIFF = "MAXIMUM ABS DIFF" +MAX_ABS_DIFF = "MAX_ABS_DIFF" SSNR = "SSNR" ODG = "ODG" -DELTA_ODG = "Delta-ODG" -MAX_ENC_DIFF = "MAXIMUM ENC DIFF" -MAX_ENC_DIFF_PARAM_NAME = "MAXIMUM ENC DIFF PARAM" +DELTA_ODG = "DELTA_ODG" +MAX_ENC_DIFF = "MAXIMUM_ENC_DIFF" +MAX_ENC_DIFF_PARAM_NAME = "MAXIMUM_ENC_DIFF_PARAM" ENC_CORE_OVL = "ENC_CORE_OVL" MAX_OVL = "MAX_OVL" MIN_OVL = "MIN_OVL" -DMX_DIFF = "DMX MAXIMUM ABS DIFF" -DMX_MLD = "DMX MLD" +DMX_DIFF = "DMX_MAX_ABS_DIFF" +DMX_MLD = "DMX_MLD" DMX_SSNR = "DMX_SSNR" # regex patterns for parsing the output from comparisons -> mainly for BASOP ci @@ -65,3 +65,403 @@ ENC_AUX_FILES = [ ["total_brate", np.float32, "fs/50"], ["vad_flag", np.int16, "fs/50"], ] + +### !!! Note: this is duplicated in scripts/create_histogram.py. If you change this here, ALSO ADAPT IT THERE!!! +### (importing from here failed for unknown reasons in some jobs on some runners and I don't have time to properly investigate this...) +CAT_NORMAL = "normal operation" +CAT_DTX = "DTX" +CAT_PLC = "PLC" +CAT_BITRATE_SWITCHING = "bitrate switching" +CAT_JBM = "JBM" + +# lists of indices for splitting of output files +# values are in SAMPLES for 16kHz (!). For higher rates, need to multiply +SPLIT_IDX_LTV_STEREO_16KHZ = np.asarray( + [ + 302096, + 519280, + 613072, + 690176, + 740992, + 749024, + 782096, + 950064, + 1045408, + 1141408, + 1237424, + 1333392, + 1397712, + 1472912, + 1755840, + 1781680, + 1789712, + 1957664, + 2158112, + 2294704, + 2454688, + 2525872, + 2573888, + 2645504, + 2746096, + 2842096, + 2970080, + 3219792, + ] +) +SPLIT_IDX_LTV_ISM1_16KHZ = np.asarray( + [ + 72000, + 164800, + 233600, + 422400, + 492800, + 592000, + 720000, + 792000, + 873600, + 947200, + 1001600, + 1060805, + 1120000, + 1284809, + 1356800, + 1448000, + 1513600, + 1731200, + 1828480, + 1910400, + 1966409, + 2150400, + 2235088, + 2299552, + 2460800, + ] +) +SPLIT_IDX_LTV_ISM234_16KHZ = np.asarray( + [ + 72000, + 164800, + 233600, + 432000, + 592000, + 720000, + 792000, + 873600, + 937600, + 1216000, + 1448000, + 1513600, + 1731200, + 1828480, + 1931200, + 2156800, + 2235088, + 2299552, + 2460800, + ] +) +SPLIT_IDX_LTV_FOA_16KHZ = np.asarray( + [ + 111840, + 176320, + 231200, + 327280, + 359280, + 404384, + 452800, + 465600, + 496272, + 565632, + 735712, + 846624, + 1007184, + 1166720, + 1316640, + 1391840, + 1699200, + 1716800, + 1812640, + 1908640, + 2036304, + 2196320, + 2364592, + 2524624, + ] +) +SPLIT_IDX_LTV_HOA3_16KHZ = np.asarray( + [ + 69328, + 169120, + 339200, + 450144, + 610128, + 685120, + 759600, + 845120, + 930144, + 1090144, + 1240128, + 1313600, + 1619200, + 1640000, + 1704416, + 1759360, + 1855360, + 1887360, + 1932480, + 1982400, + 1993600, + 2024640, + 2136128, + 2232400, + 2328480, + 2424640, + ] +) +SPLIT_IDX_LTV_MASA_1TC_16KHZ = np.asarray( + [ + 112000, + 192012, + 288000, + 376008, + 528000, + 659200, + 716803, + 822406, + 892800, + 1024002, + 1120000, + 1220789, + 1316831, + 1444183, + 1588641, + 1719739, + ] +) +SPLIT_IDX_LTV_MASA_2TC_16KHZ = np.asarray( + [ + 119984, + 255588, + 286417, + 480008, + 560002, + 625505, + 716938, + 944138, + 1037348, + 1076332, + 1131937, + 1295806, + 1479906, + 1616003, + 1743895, + 1792355, + 1920075, + 1968066, + 2112000, + 2207637, + 2317218, + 2436529, + 2613994, + ] +) +SPLIT_IDX_LTV_MC_16KHZ = np.asarray( + [ + 69328, + 169120, + 339200, + 450144, + 610128, + 685120, + 759600, + 845120, + 930144, + 1090144, + 1240128, + 1313600, + 1619200, + 1640000, + 1704416, + 1759360, + 1855360, + 1887360, + 1932480, + 1982400, + 1993600, + 2024640, + 2136128, + 2232400, + 2328480, + 2424640, + 2552400, + ] +) +SPLIT_IDX_LTV_OMASA_1ISM_16KHZ = np.asarray( + [ + 182561, + 250497, + 321192, + 398875, + 462969, + 589104, + 634642, + 704772, + 761550, + 841565, + 905594, + 1077445, + 1147515, + 1204145, + 1284198, + 1348213, + 1474397, + 1519986, + 1590029, + 1686404, + 1726821, + 1790877, + 1872012, + ] +) +SPLIT_IDX_LTV_OMASA_2ISM_16KHZ = np.asarray( + [ + 104025, + 188810, + 248000, + 600000, + 727048, + 832000, + 952951, + 970604, + 1037526, + 1104909, + 1233053, + 1295925, + 1361517, + 1414356, + 1476020, + 1549127, + 1605498, + 1727087, + 1876418, + ] +) +SPLIT_IDX_LTV_OMASA_3ISM_16KHZ = np.asarray( + [ + 155662, + 238144, + 340776, + 415986, + 575167, + 707032, + 832054, + 975997, + 1104017, + 1197434, + 1247993, + 1344330, + 1482134, + 1549575, + 1688766, + 1745715, + ] +) +SPLIT_IDX_LTV_OMASA_4ISM_16KHZ = np.asarray( + [ + 63985, + 192015, + 316768, + 391082, + 500584, + 597477, + 719995, + 838699, + 952531, + 1040041, + 1143121, + 1271008, + 1503986, + 1626407, + 1687966, + 1796807, + 1878304, + ] +) +SPLIT_IDX_LTV_OSBA_FOA_16KHZ = np.asarray( + [ + 110243, + 176353, + 231257, + 327228, + 406430, + 453959, + 465000, + 496215, + 565603, + 735726, + 846637, + 1166658, + 1316644, + 1391826, + 1697613, + 1716813, + 1812764, + 1908681, + 2036312, + 2193583, + 2361556, + ] +) +SPLIT_IDX_LTV_OSBA_HOA_16KHZ = np.asarray( + [ + 69324, + 169120, + 339217, + 450133, + 739726, + 930163, + 1090182, + 1240385, + 1316803, + 1617775, + 1640120, + 1704436, + 1759348, + 1854498, + 1887201, + 1932676, + 1983352, + 1991890, + 2024762, + 2136276, + 2231825, + 2328364, + 2424105, + ] +) + +SPLIT_IDX = { + "stereo": SPLIT_IDX_LTV_STEREO_16KHZ, + "mono": SPLIT_IDX_LTV_STEREO_16KHZ, + "1ism": SPLIT_IDX_LTV_ISM1_16KHZ, + "2ism": SPLIT_IDX_LTV_ISM234_16KHZ, + "3ism": SPLIT_IDX_LTV_ISM234_16KHZ, + "4ism": SPLIT_IDX_LTV_ISM234_16KHZ, + "foa": SPLIT_IDX_LTV_FOA_16KHZ, + "hoa2": SPLIT_IDX_LTV_HOA3_16KHZ, + "hoa3": SPLIT_IDX_LTV_HOA3_16KHZ, + "masa1tc": SPLIT_IDX_LTV_MASA_1TC_16KHZ, + "masa2tc": SPLIT_IDX_LTV_MASA_2TC_16KHZ, + "mc51": SPLIT_IDX_LTV_MC_16KHZ, + # there is always one signal that does something different... + "mc512": SPLIT_IDX_LTV_MC_16KHZ[:-1], + "mc514": SPLIT_IDX_LTV_MC_16KHZ, + "mc71": SPLIT_IDX_LTV_MC_16KHZ, + "mc714": SPLIT_IDX_LTV_MC_16KHZ, + "omasa_1ism": SPLIT_IDX_LTV_OMASA_1ISM_16KHZ, + "omasa_2ism": SPLIT_IDX_LTV_OMASA_2ISM_16KHZ, + "omasa_3ism": SPLIT_IDX_LTV_OMASA_3ISM_16KHZ, + "omasa_4ism": SPLIT_IDX_LTV_OMASA_4ISM_16KHZ, + "osba_foa": SPLIT_IDX_LTV_OSBA_FOA_16KHZ, + "osba_hoa": SPLIT_IDX_LTV_OSBA_HOA_16KHZ, +} diff --git a/tests/renderer/constants.py b/tests/renderer/constants.py index 39f57a6d892893f0d1405326205b0c646758ad5a..c61800a1a28829ef70bb9e7b7a0c6e1be3a4087e 100644 --- a/tests/renderer/constants.py +++ b/tests/renderer/constants.py @@ -1,33 +1,33 @@ #!/usr/bin/env python3 """ - (C) 2022-2025 IVAS codec Public Collaboration with portions copyright Dolby International AB, Ericsson AB, - Fraunhofer-Gesellschaft zur Foerderung der angewandten Forschung e.V., Huawei Technologies Co. LTD., - Koninklijke Philips N.V., Nippon Telegraph and Telephone Corporation, Nokia Technologies Oy, Orange, - Panasonic Holdings Corporation, Qualcomm Technologies, Inc., VoiceAge Corporation, and other - contributors to this repository. All Rights Reserved. +(C) 2022-2025 IVAS codec Public Collaboration with portions copyright Dolby International AB, Ericsson AB, +Fraunhofer-Gesellschaft zur Foerderung der angewandten Forschung e.V., Huawei Technologies Co. LTD., +Koninklijke Philips N.V., Nippon Telegraph and Telephone Corporation, Nokia Technologies Oy, Orange, +Panasonic Holdings Corporation, Qualcomm Technologies, Inc., VoiceAge Corporation, and other +contributors to this repository. All Rights Reserved. - This software is protected by copyright law and by international treaties. - The IVAS codec Public Collaboration consisting of Dolby International AB, Ericsson AB, - Fraunhofer-Gesellschaft zur Foerderung der angewandten Forschung e.V., Huawei Technologies Co. LTD., - Koninklijke Philips N.V., Nippon Telegraph and Telephone Corporation, Nokia Technologies Oy, Orange, - Panasonic Holdings Corporation, Qualcomm Technologies, Inc., VoiceAge Corporation, and other - contributors to this repository retain full ownership rights in their respective contributions in - the software. This notice grants no license of any kind, including but not limited to patent - license, nor is any license granted by implication, estoppel or otherwise. +This software is protected by copyright law and by international treaties. +The IVAS codec Public Collaboration consisting of Dolby International AB, Ericsson AB, +Fraunhofer-Gesellschaft zur Foerderung der angewandten Forschung e.V., Huawei Technologies Co. LTD., +Koninklijke Philips N.V., Nippon Telegraph and Telephone Corporation, Nokia Technologies Oy, Orange, +Panasonic Holdings Corporation, Qualcomm Technologies, Inc., VoiceAge Corporation, and other +contributors to this repository retain full ownership rights in their respective contributions in +the software. This notice grants no license of any kind, including but not limited to patent +license, nor is any license granted by implication, estoppel or otherwise. - Contributors are required to enter into the IVAS codec Public Collaboration agreement before making - contributions. +Contributors are required to enter into the IVAS codec Public Collaboration agreement before making +contributions. - This software is provided "AS IS", without any express or implied warranties. The software is in the - development stage. It is intended exclusively for experts who have experience with such software and - solely for the purpose of inspection. All implied warranties of non-infringement, merchantability - and fitness for a particular purpose are hereby disclaimed and excluded. +This software is provided "AS IS", without any express or implied warranties. The software is in the +development stage. It is intended exclusively for experts who have experience with such software and +solely for the purpose of inspection. All implied warranties of non-infringement, merchantability +and fitness for a particular purpose are hereby disclaimed and excluded. - Any dispute, controversy or claim arising under or in relation to providing this software shall be - submitted to and settled by the final, binding jurisdiction of the courts of Munich, Germany in - accordance with the laws of the Federal Republic of Germany excluding its conflict of law rules and - the United Nations Convention on Contracts on the International Sales of Goods. +Any dispute, controversy or claim arising under or in relation to providing this software shall be +submitted to and settled by the final, binding jurisdiction of the courts of Munich, Germany in +accordance with the laws of the Federal Republic of Germany excluding its conflict of law rules and +the United Nations Convention on Contracts on the International Sales of Goods. """ from pathlib import Path @@ -298,6 +298,77 @@ FORMAT_TO_METADATA_FILES = { ], } +FORMAT_TO_METADATA_FILES_LTV = { + "ISM1": [str(TESTV_DIR.joinpath("ltvISM1.csv"))], + "ISM2": [ + str(TESTV_DIR.joinpath("ltvISM1.csv")), + str(TESTV_DIR.joinpath("ltvISM2.csv")), + ], + "ISM3": [ + str(TESTV_DIR.joinpath("ltvISM1.csv")), + str(TESTV_DIR.joinpath("ltvISM2.csv")), + str(TESTV_DIR.joinpath("ltvISM3.csv")), + ], + "ISM4": [ + str(TESTV_DIR.joinpath("ltvISM1.csv")), + str(TESTV_DIR.joinpath("ltvISM2.csv")), + str(TESTV_DIR.joinpath("ltvISM3.csv")), + str(TESTV_DIR.joinpath("ltvISM4.csv")), + ], + "NDP_ISM4": [ + str(TESTV_DIR.joinpath("ltvISM1.csv")), + str(TESTV_DIR.joinpath("stvISM2_non-diegetic-pan.csv")), + str(TESTV_DIR.joinpath("ltvISM3.csv")), + str(TESTV_DIR.joinpath("ltvISM4.csv")), + ], + "MASA1": [str(TESTV_DIR.joinpath("ltv48_MASA1TC.met"))], + "MASA2": [str(TESTV_DIR.joinpath("ltv48_MASA2TC.met"))], + "OMASA_1_1": [ + str(TESTV_DIR.joinpath("ltvISM1.csv")), + str(TESTV_DIR.joinpath("ltv48_OMASA_1ISM_1TC.met")), + ], + "OMASA_1_2": [ + str(TESTV_DIR.joinpath("ltvISM1.csv")), + str(TESTV_DIR.joinpath("ltvISM2.csv")), + str(TESTV_DIR.joinpath("ltv48_OMASA_2ISM_1TC.met")), + ], + "OMASA_1_3": [ + str(TESTV_DIR.joinpath("ltvISM1.csv")), + str(TESTV_DIR.joinpath("ltvISM2.csv")), + str(TESTV_DIR.joinpath("ltvISM3.csv")), + str(TESTV_DIR.joinpath("ltv48_OMASA_3ISM_1TC.met")), + ], + "OMASA_1_4": [ + str(TESTV_DIR.joinpath("ltvISM1.csv")), + str(TESTV_DIR.joinpath("ltvISM2.csv")), + str(TESTV_DIR.joinpath("ltvISM3.csv")), + str(TESTV_DIR.joinpath("ltvISM4.csv")), + str(TESTV_DIR.joinpath("ltv48_OMASA_4ISM_1TC.met")), + ], + "OMASA_2_1": [ + str(TESTV_DIR.joinpath("ltvISM1.csv")), + str(TESTV_DIR.joinpath("ltv48_OMASA_1ISM_2TC.met")), + ], + "OMASA_2_2": [ + str(TESTV_DIR.joinpath("ltvISM1.csv")), + str(TESTV_DIR.joinpath("ltvISM2.csv")), + str(TESTV_DIR.joinpath("ltv48_OMASA_2ISM_2TC.met")), + ], + "OMASA_2_3": [ + str(TESTV_DIR.joinpath("ltvISM1.csv")), + str(TESTV_DIR.joinpath("ltvISM2.csv")), + str(TESTV_DIR.joinpath("ltvISM3.csv")), + str(TESTV_DIR.joinpath("ltv48_OMASA_3ISM_2TC.met")), + ], + "OMASA_2_4": [ + str(TESTV_DIR.joinpath("ltvISM1.csv")), + str(TESTV_DIR.joinpath("ltvISM2.csv")), + str(TESTV_DIR.joinpath("ltvISM3.csv")), + str(TESTV_DIR.joinpath("ltvISM4.csv")), + str(TESTV_DIR.joinpath("ltv48_OMASA_4ISM_2TC.met")), + ], +} + """ Input formats """ INPUT_FORMATS_AMBI = ["FOA", "HOA2", "HOA3"] @@ -347,4 +418,5 @@ PEAQ_SUPPORTED_FMT = [ "BINAURAL", "BINAURAL_ROOM_IR", "BINAURAL_ROOM_REVERB", -] \ No newline at end of file +] + diff --git a/tests/renderer/test_renderer.py b/tests/renderer/test_renderer.py index b5758d6f06edff56a2c3fbb64ec8be7e37314df9..60ec37a69c07daf64609076cf69b590250fee051 100644 --- a/tests/renderer/test_renderer.py +++ b/tests/renderer/test_renderer.py @@ -33,6 +33,7 @@ the United Nations Convention on Contracts on the International Sales of Goods. import pytest from .constants import ( + FORMAT_TO_METADATA_FILES_LTV, OUTPUT_FORMATS, INPUT_FORMATS_AMBI, FRAMING_TO_TEST, @@ -62,7 +63,6 @@ from ..conftest import props_to_record """ Ambisonics """ - @pytest.mark.parametrize("out_fmt", OUTPUT_FORMATS) @pytest.mark.parametrize("in_fmt", INPUT_FORMATS_AMBI) @pytest.mark.parametrize("frame_size", FRAMING_TO_TEST) @@ -78,6 +78,7 @@ def test_ambisonics( get_ssnr, get_odg, get_odg_bin, + split_comparison, ): run_renderer( record_property, @@ -92,10 +93,10 @@ def test_ambisonics( get_ssnr=get_ssnr, get_odg=get_odg, get_odg_bin=get_odg_bin, + split_comparison=split_comparison, ) - @pytest.mark.parametrize("out_fmt", OUTPUT_FORMATS_BINAURAL) @pytest.mark.parametrize("in_fmt", INPUT_FORMATS_AMBI) @pytest.mark.parametrize("frame_size", FRAMING_TO_TEST) @@ -111,6 +112,7 @@ def test_ambisonics_binaural_static( get_ssnr, get_odg, get_odg_bin, + split_comparison, ): run_renderer( record_property, @@ -125,10 +127,10 @@ def test_ambisonics_binaural_static( get_ssnr=get_ssnr, get_odg=get_odg, get_odg_bin=get_odg_bin, + split_comparison=split_comparison, ) - @pytest.mark.parametrize("trj_file", HR_TRAJECTORIES_TO_TEST) @pytest.mark.parametrize("out_fmt", OUTPUT_FORMATS_BINAURAL) @pytest.mark.parametrize("in_fmt", INPUT_FORMATS_AMBI) @@ -146,6 +148,7 @@ def test_ambisonics_binaural_headrotation( get_ssnr, get_odg, get_odg_bin, + split_comparison, ): run_renderer( record_property, @@ -161,11 +164,11 @@ def test_ambisonics_binaural_headrotation( get_ssnr=get_ssnr, get_odg=get_odg, get_odg_bin=get_odg_bin, + split_comparison=split_comparison, ) @pytest.mark.skip(reason="Not supported for BASOP code currently") - @pytest.mark.parametrize("out_fmt", OUTPUT_FORMATS_BINAURAL[2:]) @pytest.mark.parametrize("in_fmt", INPUT_FORMATS_AMBI) @pytest.mark.parametrize("frame_size", FRAMING_TO_TEST) @@ -183,6 +186,7 @@ def test_dynamic_acoustic_environment( get_odg, get_odg_bin, aeid, + split_comparison, ): rend_config_path = TEST_VECTOR_DIR.joinpath(f"rend_config_combined.cfg") rend_config_path.with_stem(f"rend_config") @@ -202,11 +206,11 @@ def test_dynamic_acoustic_environment( get_odg_bin=get_odg_bin, config_file=rend_config_path, aeid=aeid, + split_comparison=split_comparison, ) @pytest.mark.skip(reason="Not supported for BASOP code currently") - @pytest.mark.parametrize("out_fmt", OUTPUT_FORMATS_BINAURAL[2:]) @pytest.mark.parametrize("in_fmt", INPUT_FORMATS_AMBI) @pytest.mark.parametrize("frame_size", FRAMING_TO_TEST) @@ -222,6 +226,7 @@ def test_dynamic_acoustic_environment_file( get_ssnr, get_odg, get_odg_bin, + split_comparison, ): rend_config_path = TEST_VECTOR_DIR.joinpath(f"rend_config_combined.cfg") rend_config_path.with_stem(f"rend_config") @@ -243,13 +248,13 @@ def test_dynamic_acoustic_environment_file( get_odg_bin=get_odg_bin, config_file=rend_config_path, aeid=aeid, + split_comparison=split_comparison, ) """ Multichannel """ - @pytest.mark.parametrize("out_fmt", OUTPUT_FORMATS) @pytest.mark.parametrize("in_fmt", INPUT_FORMATS_MC) @pytest.mark.parametrize("frame_size", FRAMING_TO_TEST) @@ -265,6 +270,7 @@ def test_multichannel( get_ssnr, get_odg, get_odg_bin, + split_comparison, ): run_renderer( record_property, @@ -279,10 +285,10 @@ def test_multichannel( get_ssnr=get_ssnr, get_odg=get_odg, get_odg_bin=get_odg_bin, + split_comparison=split_comparison, ) - @pytest.mark.parametrize("out_fmt", OUTPUT_FORMATS_BINAURAL) @pytest.mark.parametrize("in_fmt", INPUT_FORMATS_MC) @pytest.mark.parametrize("frame_size", FRAMING_TO_TEST) @@ -298,6 +304,7 @@ def test_multichannel_binaural_static( get_ssnr, get_odg, get_odg_bin, + split_comparison, ): if in_fmt in ["MONO", "STEREO"]: pytest.skip("MONO or STEREO to Binaural rendering unsupported") @@ -315,10 +322,10 @@ def test_multichannel_binaural_static( get_ssnr=get_ssnr, get_odg=get_odg, get_odg_bin=get_odg_bin, + split_comparison=split_comparison, ) - @pytest.mark.parametrize("trj_file", HR_TRAJECTORIES_TO_TEST) @pytest.mark.parametrize("out_fmt", OUTPUT_FORMATS_BINAURAL) @pytest.mark.parametrize("in_fmt", INPUT_FORMATS_MC) @@ -336,6 +343,7 @@ def test_multichannel_binaural_headrotation( get_ssnr, get_odg, get_odg_bin, + split_comparison, ): if in_fmt in ["MONO", "STEREO"]: pytest.skip("MONO or STEREO to Binaural rendering unsupported") @@ -354,13 +362,13 @@ def test_multichannel_binaural_headrotation( get_ssnr=get_ssnr, get_odg=get_odg, get_odg_bin=get_odg_bin, + split_comparison=split_comparison, ) """ ISM """ - @pytest.mark.parametrize("out_fmt", OUTPUT_FORMATS) @pytest.mark.parametrize("in_fmt", INPUT_FORMATS_ISM) @pytest.mark.parametrize("frame_size", FRAMING_TO_TEST) @@ -376,14 +384,20 @@ def test_ism( get_ssnr, get_odg, get_odg_bin, + split_comparison, ): + md_files = ( + FORMAT_TO_METADATA_FILES_LTV[in_fmt] + if test_info.config.option.use_ltv + else FORMAT_TO_METADATA_FILES[in_fmt] + ) run_renderer( record_property, props_to_record, test_info, in_fmt, out_fmt, - in_meta_files=FORMAT_TO_METADATA_FILES[in_fmt], + in_meta_files=md_files, binary_suffix=EXE_SUFFIX, frame_size=frame_size, get_mld=get_mld, @@ -391,10 +405,10 @@ def test_ism( get_ssnr=get_ssnr, get_odg=get_odg, get_odg_bin=get_odg_bin, + split_comparison=split_comparison, ) - @pytest.mark.parametrize("out_fmt", OUTPUT_FORMATS_BINAURAL) @pytest.mark.parametrize("in_fmt", INPUT_FORMATS_ISM) @pytest.mark.parametrize("frame_size", FRAMING_TO_TEST) @@ -410,9 +424,14 @@ def test_ism_binaural_static( get_ssnr, get_odg, get_odg_bin, + split_comparison, ): try: - in_meta_files = FORMAT_TO_METADATA_FILES[in_fmt] + in_meta_files = ( + FORMAT_TO_METADATA_FILES_LTV[in_fmt] + if test_info.config.option.use_ltv + else FORMAT_TO_METADATA_FILES[in_fmt] + ) except KeyError: in_meta_files = None @@ -430,10 +449,10 @@ def test_ism_binaural_static( get_ssnr=get_ssnr, get_odg=get_odg, get_odg_bin=get_odg_bin, + split_comparison=split_comparison, ) - @pytest.mark.parametrize("trj_file", HR_TRAJECTORIES_TO_TEST) @pytest.mark.parametrize("out_fmt", OUTPUT_FORMATS_BINAURAL) @pytest.mark.parametrize("in_fmt", INPUT_FORMATS_ISM) @@ -451,9 +470,14 @@ def test_ism_binaural_headrotation( get_ssnr, get_odg, get_odg_bin, + split_comparison, ): try: - in_meta_files = FORMAT_TO_METADATA_FILES[in_fmt] + in_meta_files = ( + FORMAT_TO_METADATA_FILES_LTV[in_fmt] + if test_info.config.option.use_ltv + else FORMAT_TO_METADATA_FILES[in_fmt] + ) except KeyError: in_meta_files = None @@ -472,13 +496,13 @@ def test_ism_binaural_headrotation( get_ssnr=get_ssnr, get_odg=get_odg, get_odg_bin=get_odg_bin, + split_comparison=split_comparison, ) """ MASA """ - @pytest.mark.parametrize("out_fmt", OUTPUT_FORMATS) @pytest.mark.parametrize("in_fmt", INPUT_FORMATS_MASA) @pytest.mark.parametrize("frame_size", FRAMING_TO_TEST) @@ -494,14 +518,21 @@ def test_masa( get_ssnr, get_odg, get_odg_bin, + split_comparison, ): + md_files = ( + FORMAT_TO_METADATA_FILES_LTV[in_fmt] + if test_info.config.option.use_ltv + else FORMAT_TO_METADATA_FILES[in_fmt] + ) + run_renderer( record_property, props_to_record, test_info, in_fmt, out_fmt, - in_meta_files=FORMAT_TO_METADATA_FILES[in_fmt], + in_meta_files=md_files, binary_suffix=EXE_SUFFIX, frame_size=frame_size, get_mld=get_mld, @@ -509,10 +540,10 @@ def test_masa( get_ssnr=get_ssnr, get_odg=get_odg, get_odg_bin=get_odg_bin, + split_comparison=split_comparison, ) - @pytest.mark.parametrize("out_fmt", OUTPUT_FORMATS_BINAURAL) @pytest.mark.parametrize("in_fmt", INPUT_FORMATS_MASA) @pytest.mark.parametrize("frame_size", FRAMING_TO_TEST) @@ -528,17 +559,23 @@ def test_masa_binaural_static( get_ssnr, get_odg, get_odg_bin, + split_comparison, ): if out_fmt in ["BINAURAL_ROOM_IR", "BINAURAL_ROOM_REVERB"]: pytest.skip("Skipping binaural room outputs for MASA as unimplemented.") + md_files = ( + FORMAT_TO_METADATA_FILES_LTV[in_fmt] + if test_info.config.option.use_ltv + else FORMAT_TO_METADATA_FILES[in_fmt] + ) run_renderer( record_property, props_to_record, test_info, in_fmt, out_fmt, - in_meta_files=FORMAT_TO_METADATA_FILES[in_fmt], + in_meta_files=md_files, binary_suffix=EXE_SUFFIX, frame_size=frame_size, get_mld=get_mld, @@ -546,10 +583,10 @@ def test_masa_binaural_static( get_ssnr=get_ssnr, get_odg=get_odg, get_odg_bin=get_odg_bin, + split_comparison=split_comparison, ) - @pytest.mark.parametrize("trj_file", HR_TRAJECTORIES_TO_TEST) @pytest.mark.parametrize("out_fmt", OUTPUT_FORMATS_BINAURAL) @pytest.mark.parametrize("in_fmt", INPUT_FORMATS_MASA) @@ -567,10 +604,17 @@ def test_masa_binaural_headrotation( get_ssnr, get_odg, get_odg_bin, + split_comparison, ): if out_fmt in ["BINAURAL_ROOM_IR", "BINAURAL_ROOM_REVERB"]: pytest.skip("Skipping binaural room outputs for MASA as unimplemented.") + md_files = ( + FORMAT_TO_METADATA_FILES_LTV[in_fmt] + if test_info.config.option.use_ltv + else FORMAT_TO_METADATA_FILES[in_fmt] + ) + run_renderer( record_property, props_to_record, @@ -578,7 +622,7 @@ def test_masa_binaural_headrotation( in_fmt, out_fmt, trj_file=HR_TRAJECTORY_DIR.joinpath(f"{trj_file}.csv"), - in_meta_files=FORMAT_TO_METADATA_FILES[in_fmt], + in_meta_files=md_files, binary_suffix=EXE_SUFFIX, frame_size=frame_size, get_mld=get_mld, @@ -586,10 +630,10 @@ def test_masa_binaural_headrotation( get_ssnr=get_ssnr, get_odg=get_odg, get_odg_bin=get_odg_bin, + split_comparison=split_comparison, ) - @pytest.mark.parametrize("in_fmt", METADATA_SCENES_TO_TEST_MASA_PREREND) def test_masa_prerend( record_property, @@ -601,6 +645,7 @@ def test_masa_prerend( get_ssnr, get_odg, get_odg_bin, + split_comparison, ): run_renderer( record_property, @@ -615,13 +660,13 @@ def test_masa_prerend( get_ssnr=get_ssnr, get_odg=get_odg, get_odg_bin=get_odg_bin, + split_comparison=split_comparison, ) """ Custom loudspeaker layouts """ - @pytest.mark.parametrize("out_fmt", OUTPUT_FORMATS) @pytest.mark.parametrize("in_layout", CUSTOM_LS_TO_TEST) @pytest.mark.parametrize("frame_size", FRAMING_TO_TEST) @@ -637,6 +682,7 @@ def test_custom_ls_input( get_ssnr, get_odg, get_odg_bin, + split_comparison, ): run_renderer( record_property, @@ -651,10 +697,10 @@ def test_custom_ls_input( get_ssnr=get_ssnr, get_odg=get_odg, get_odg_bin=get_odg_bin, + split_comparison=split_comparison, ) - @pytest.mark.parametrize("out_fmt", CUSTOM_LS_TO_TEST) @pytest.mark.parametrize("in_fmt", OUTPUT_FORMATS) def test_custom_ls_output( @@ -668,6 +714,7 @@ def test_custom_ls_output( get_ssnr, get_odg, get_odg_bin, + split_comparison, ): run_renderer( record_property, @@ -681,10 +728,10 @@ def test_custom_ls_output( get_ssnr=get_ssnr, get_odg=get_odg, get_odg_bin=get_odg_bin, + split_comparison=split_comparison, ) - @pytest.mark.parametrize("out_fmt", CUSTOM_LS_TO_TEST) @pytest.mark.parametrize("in_fmt", CUSTOM_LS_TO_TEST) def test_custom_ls_input_output( @@ -698,6 +745,7 @@ def test_custom_ls_input_output( get_ssnr, get_odg, get_odg_bin, + split_comparison, ): run_renderer( record_property, @@ -711,10 +759,10 @@ def test_custom_ls_input_output( get_ssnr=get_ssnr, get_odg=get_odg, get_odg_bin=get_odg_bin, + split_comparison=split_comparison, ) - @pytest.mark.parametrize("out_fmt", OUTPUT_FORMATS_BINAURAL) @pytest.mark.parametrize("in_layout", CUSTOM_LS_TO_TEST) @pytest.mark.parametrize("frame_size", FRAMING_TO_TEST) @@ -730,6 +778,7 @@ def test_custom_ls_input_binaural( get_ssnr, get_odg, get_odg_bin, + split_comparison, ): run_renderer( record_property, @@ -744,10 +793,10 @@ def test_custom_ls_input_binaural( get_ssnr=get_ssnr, get_odg=get_odg, get_odg_bin=get_odg_bin, + split_comparison=split_comparison, ) - @pytest.mark.parametrize("trj_file", HR_TRAJECTORIES_TO_TEST) @pytest.mark.parametrize("out_fmt", OUTPUT_FORMATS_BINAURAL) @pytest.mark.parametrize("in_layout", CUSTOM_LS_TO_TEST) @@ -765,6 +814,7 @@ def test_custom_ls_input_binaural_headrotation( get_ssnr, get_odg, get_odg_bin, + split_comparison, ): run_renderer( record_property, @@ -780,13 +830,13 @@ def test_custom_ls_input_binaural_headrotation( get_ssnr=get_ssnr, get_odg=get_odg, get_odg_bin=get_odg_bin, + split_comparison=split_comparison, ) """ Metadata / scene description input """ - @pytest.mark.parametrize("out_fmt", OUTPUT_FORMATS) @pytest.mark.parametrize("in_fmt", METADATA_SCENES_TO_TEST) @pytest.mark.parametrize("frame_size", FRAMING_TO_TEST) @@ -802,6 +852,7 @@ def test_metadata( get_ssnr, get_odg, get_odg_bin, + split_comparison, ): run_renderer( record_property, @@ -817,13 +868,13 @@ def test_metadata( get_ssnr=get_ssnr, get_odg=get_odg, get_odg_bin=get_odg_bin, + split_comparison=split_comparison, ) """ non diegetic pan """ - @pytest.mark.parametrize("out_fmt", ["STEREO"]) @pytest.mark.parametrize("in_fmt", ["MONO"]) @pytest.mark.parametrize("non_diegetic_pan", ["0", "-30", "45", "90", "-90"]) @@ -839,6 +890,7 @@ def test_non_diegetic_pan_static( get_ssnr, get_odg, get_odg_bin, + split_comparison, ): run_renderer( record_property, @@ -853,10 +905,10 @@ def test_non_diegetic_pan_static( get_ssnr=get_ssnr, get_odg=get_odg, get_odg_bin=get_odg_bin, + split_comparison=split_comparison, ) - @pytest.mark.parametrize("out_fmt", ["STEREO"]) @pytest.mark.parametrize("in_fmt", ["ISM1"]) @pytest.mark.parametrize("non_diegetic_pan", ["0", "-30", "45", "90", "-90"]) @@ -872,6 +924,7 @@ def test_non_diegetic_pan_ism_static( get_ssnr, get_odg, get_odg_bin, + split_comparison, ): run_renderer( record_property, @@ -886,6 +939,7 @@ def test_non_diegetic_pan_ism_static( get_ssnr=get_ssnr, get_odg=get_odg, get_odg_bin=get_odg_bin, + split_comparison=split_comparison, ) @@ -909,9 +963,7 @@ def test_ambisonics_binaural_headrotation_refrotzero( in_fmt, out_fmt, trj_file, - get_mld, - get_mld_lim, - get_ssnr, + split_comparison, ): if test_info.config.option.create_ref or test_info.config.option.create_cut: pytest.skip("OTR tests only run for smoke test") @@ -932,6 +984,7 @@ def test_ambisonics_binaural_headrotation_refrotzero( "refrot_file": HR_TRAJECTORY_DIR.joinpath("const000.csv"), "frame_size": "5", }, + split_comparison=split_comparison, ) @@ -941,7 +994,12 @@ def test_ambisonics_binaural_headrotation_refrotzero( @pytest.mark.parametrize("out_fmt", OUTPUT_FORMATS_BINAURAL) @pytest.mark.parametrize("in_fmt", INPUT_FORMATS_AMBI) def test_ambisonics_binaural_headrotation_refrotequal( - record_property, props_to_record, test_info, in_fmt, out_fmt, get_mld, get_mld_lim + record_property, + props_to_record, + test_info, + in_fmt, + out_fmt, + split_comparison, ): if test_info.config.option.create_ref or test_info.config.option.create_cut: pytest.skip("OTR tests only run for smoke test") @@ -965,6 +1023,7 @@ def test_ambisonics_binaural_headrotation_refrotequal( ), "frame_size": "5", }, + split_comparison=split_comparison, ) @@ -982,8 +1041,7 @@ def test_ambisonics_binaural_headrotation_refveczero( in_fmt, out_fmt, trj_file, - get_mld, - get_mld_lim, + split_comparison, ): if test_info.config.option.create_ref or test_info.config.option.create_cut: pytest.skip("OTR tests only run for smoke test") @@ -1004,6 +1062,7 @@ def test_ambisonics_binaural_headrotation_refveczero( "refvec_file": HR_TRAJECTORY_DIR.joinpath("const000-Vector3.csv"), "frame_size": "5", }, + split_comparison=split_comparison, ) @@ -1014,7 +1073,12 @@ def test_ambisonics_binaural_headrotation_refveczero( @pytest.mark.parametrize("out_fmt", OUTPUT_FORMATS_BINAURAL) @pytest.mark.parametrize("in_fmt", INPUT_FORMATS_AMBI) def test_ambisonics_binaural_headrotation_refvecequal( - record_property, props_to_record, test_info, in_fmt, out_fmt, get_mld, get_mld_lim + record_property, + props_to_record, + test_info, + in_fmt, + out_fmt, + split_comparison, ): if test_info.config.option.create_ref or test_info.config.option.create_cut: pytest.skip("OTR tests only run for smoke test") @@ -1042,6 +1106,7 @@ def test_ambisonics_binaural_headrotation_refvecequal( ), "frame_size": "5", }, + split_comparison=split_comparison, ) @@ -1052,7 +1117,12 @@ def test_ambisonics_binaural_headrotation_refvecequal( @pytest.mark.parametrize("out_fmt", OUTPUT_FORMATS_BINAURAL) @pytest.mark.parametrize("in_fmt", INPUT_FORMATS_AMBI) def test_ambisonics_binaural_headrotation_refvec_rotating( - record_property, props_to_record, test_info, in_fmt, out_fmt, get_mld, get_mld_lim + record_property, + props_to_record, + test_info, + in_fmt, + out_fmt, + split_comparison, ): if test_info.config.option.create_ref or test_info.config.option.create_cut: pytest.skip("OTR tests only run for smoke test") @@ -1081,6 +1151,7 @@ def test_ambisonics_binaural_headrotation_refvec_rotating( ), "frame_size": "5", }, + split_comparison=split_comparison, ) @@ -1093,7 +1164,12 @@ def test_ambisonics_binaural_headrotation_refvec_rotating( @pytest.mark.parametrize("out_fmt", OUTPUT_FORMATS_BINAURAL) @pytest.mark.parametrize("in_fmt", INPUT_FORMATS_AMBI) def test_ambisonics_binaural_headrotation_refvec_rotating_fixed_pos_offset( - record_property, props_to_record, test_info, in_fmt, out_fmt, get_mld, get_mld_lim + record_property, + props_to_record, + test_info, + in_fmt, + out_fmt, + split_comparison, ): if test_info.config.option.create_ref or test_info.config.option.create_cut: pytest.skip("OTR tests only run for smoke test") @@ -1118,6 +1194,7 @@ def test_ambisonics_binaural_headrotation_refvec_rotating_fixed_pos_offset( ), "frame_size": "5", }, + split_comparison=split_comparison, ) @@ -1129,7 +1206,12 @@ def test_ambisonics_binaural_headrotation_refvec_rotating_fixed_pos_offset( @pytest.mark.parametrize("out_fmt", OUTPUT_FORMATS_BINAURAL) @pytest.mark.parametrize("in_fmt", INPUT_FORMATS_AMBI) def test_ambisonics_binaural_headrotation_refveclev_vs_refvec( - record_property, props_to_record, test_info, in_fmt, out_fmt, get_mld, get_mld_lim + record_property, + props_to_record, + test_info, + in_fmt, + out_fmt, + split_comparison, ): if test_info.config.option.create_ref or test_info.config.option.create_cut: pytest.skip("OTR tests only run for smoke test") @@ -1153,6 +1235,7 @@ def test_ambisonics_binaural_headrotation_refveclev_vs_refvec( "refvec_file": HR_TRAJECTORY_DIR.joinpath("full-circle-4s-Vector3.csv"), "frame_size": "5", }, + split_comparison=split_comparison, ) @@ -1163,7 +1246,12 @@ def test_ambisonics_binaural_headrotation_refveclev_vs_refvec( @pytest.mark.parametrize("out_fmt", OUTPUT_FORMATS_BINAURAL) @pytest.mark.parametrize("in_fmt", INPUT_FORMATS_MC) def test_multichannel_binaural_headrotation_refvec_rotating( - record_property, props_to_record, test_info, in_fmt, out_fmt, get_mld, get_mld_lim + record_property, + props_to_record, + test_info, + in_fmt, + out_fmt, + split_comparison, ): if test_info.config.option.create_ref or test_info.config.option.create_cut: pytest.skip("OTR tests only run for smoke test") @@ -1191,6 +1279,7 @@ def test_multichannel_binaural_headrotation_refvec_rotating( ), "frame_size": "5", }, + split_comparison=split_comparison, ) @@ -1201,13 +1290,22 @@ def test_multichannel_binaural_headrotation_refvec_rotating( @pytest.mark.parametrize("out_fmt", OUTPUT_FORMATS_BINAURAL) @pytest.mark.parametrize("in_fmt", INPUT_FORMATS_ISM) def test_ism_binaural_headrotation_refvec_rotating( - record_property, props_to_record, test_info, in_fmt, out_fmt, get_mld, get_mld_lim + record_property, + props_to_record, + test_info, + in_fmt, + out_fmt, + split_comparison, ): if test_info.config.option.create_ref or test_info.config.option.create_cut: pytest.skip("OTR tests only run for smoke test") try: - in_meta_files = FORMAT_TO_METADATA_FILES[in_fmt] + in_meta_files = ( + FORMAT_TO_METADATA_FILES_LTV[in_fmt] + if test_info.config.option.use_ltv + else FORMAT_TO_METADATA_FILES[in_fmt] + ) except KeyError: in_meta_files = None @@ -1233,4 +1331,5 @@ def test_ism_binaural_headrotation_refvec_rotating( "in_meta_files": in_meta_files, "frame_size": "5", }, + split_comparison=split_comparison, ) diff --git a/tests/renderer/utils.py b/tests/renderer/utils.py index 13b03edc0bb6288e7933575aa5dab5b49c34d469..0d4c2303d0b2d90c3a52a740966e15b4c32b5aa2 100644 --- a/tests/renderer/utils.py +++ b/tests/renderer/utils.py @@ -57,11 +57,12 @@ from .constants import ( BIN_SUFFIX_MERGETARGET, PEAQ_SUPPORTED_FMT, ) +from ..constants import CAT_NORMAL sys.path.append(SCRIPTS_DIR) from pyaudio3dtools.audiofile import readfile from ..cmp_pcm import cmp_pcm -from ..conftest import parse_properties +from ..conftest import parse_properties, get_split_idx def run_cmd(cmd, test_info, env=None): @@ -173,6 +174,7 @@ def run_renderer( out_file=None, sr=48, render_for_peaq=False, + split_comparison=False, ) -> str: # prepare arguments and filepaths if trj_file is not None: @@ -309,6 +311,15 @@ def run_renderer( env["UBSAN_OPTIONS"] + f",log_path=usan_log_{test_info.node.name}" ) + testcase_props = { + "format": "Renderer", + "category": CAT_NORMAL, + } + + if record_property is not None: + for k, v in testcase_props.items(): + record_property(k, v) + # run the renderer run_cmd(cmd, test_info, env) @@ -372,9 +383,13 @@ def run_renderer( else: odg_input = in_file - # see constants.py + ### run the comparison tools + split_idx = np.empty(0) + prop_suffix = [""] + + # 1. run comparison on whole files - this is done always, regardless of the presence of --split_comparison ref_fs = int(cmd[10]) * 1000 - output_differs, reason = cmp_pcm( + output_differs_parts, reason_parts = cmp_pcm( out_file_ref, out_file, out_fmt, @@ -389,14 +404,54 @@ def run_renderer( odg_test=odg_test, odg_ref=odg_ref, scalefac=test_info.config.option.scalefac, + split_idx=split_idx, ) - props = parse_properties(reason, output_differs, props_to_record) - for k, v in props.items(): - record_property(k, v) + # 2. run comparison on split files if --split_comparison is given + # for JBM cases, comparison will fail because of length mismatch beetween split wav files and tracefiles + # -> skip split comparison for these cases + if split_comparison: + split_idx = get_split_idx(str(Path(in_file).stem), ref_fs // 1000) + + # this extra if takes care of cases where no splits are found, e.g. the "NOOP" case in the self_test_ltv prm file + # if this would not be there, then the comparison of the whole file would run twice + if len(split_idx) > 0: + output_differs_splits, reason_splits = cmp_pcm( + out_file_ref, + out_file, + out_fmt, + ref_fs, + get_mld=get_mld, + mld_lim=get_mld_lim, + abs_tol=abs_tol, + get_ssnr=get_ssnr, + get_odg=get_odg, + get_odg_bin=get_odg_bin, + odg_input=odg_input, + odg_test=odg_test, + odg_ref=odg_ref, + scalefac=test_info.config.option.scalefac, + split_idx=split_idx, + ) + output_differs_parts += output_differs_splits + reason_parts += reason_splits + + if split_comparison: + prop_suffix = ["_whole"] + [ + f"_split{i:03d}" for i in range(1, len(split_idx) + 1) + ] + + for output_differs, reason, suffix in zip( + output_differs_parts, reason_parts, prop_suffix + ): + result_props = parse_properties( + reason, output_differs, props_to_record, suffix + ) + for k, v in result_props.items(): + record_property(k, v) - if output_differs: - pytest.fail(f"Output differs: ({reason})") + if output_differs_parts[0]: + pytest.fail(f"Output differs: ({reason_parts[0]})") # compare metadata files in case of MASA prerendering if "MASA" in str(out_fmt): @@ -416,6 +471,7 @@ def compare_renderer_args( out_fmt, ref_kwargs: Dict, cut_kwargs: Dict, + split_comparison=False, ): out_file_ref = run_renderer( record_property, @@ -424,6 +480,7 @@ def compare_renderer_args( in_fmt, out_fmt, **ref_kwargs, + split_comparison=split_comparison, ) ref, ref_fs = readfile(out_file_ref) out_file_cut = run_renderer( @@ -433,6 +490,7 @@ def compare_renderer_args( in_fmt, out_fmt, **cut_kwargs, + split_comparison=split_comparison, ) cut, cut_fs = readfile(out_file_cut) [diff_found, snr, gain_b, max_diff] = check_BE(test_info, ref, ref_fs, cut, cut_fs) diff --git a/tests/split_rendering/test_split_rendering.py b/tests/split_rendering/test_split_rendering.py index 4f1d0039ece6b6695d0d062ae8afe605ef626c2a..a881a7ad23f65902c620f7de9a1d39e6f9676832 100644 --- a/tests/split_rendering/test_split_rendering.py +++ b/tests/split_rendering/test_split_rendering.py @@ -44,12 +44,21 @@ from tests.split_rendering.utils import * @pytest.mark.parametrize("bitrate", IVAS_BITRATES_AMBI) @pytest.mark.parametrize("in_fmt", INPUT_FORMATS_AMBI_SPLIT_REND) def test_ambisonics_full_chain_split( + record_property, + props_to_record, + get_mld, + get_mld_lim, + get_ssnr, + get_odg, + get_odg_bin, test_info, in_fmt, bitrate, render_config, trajectory ): post_trajectory = HR_TRAJECTORY_DIR.joinpath(f"{trajectory}.csv") pre_trajectory = post_trajectory.with_stem(f"{post_trajectory.stem}_delayed") run_full_chain_split_rendering( + record_property, + props_to_record, test_info, in_fmt=in_fmt, bitrate=bitrate, @@ -57,6 +66,11 @@ def test_ambisonics_full_chain_split( binary_suffix=EXE_SUFFIX, pre_trajectory=pre_trajectory, post_trajectory=post_trajectory, + get_mld=get_mld, + mld_lim=get_mld_lim, + get_ssnr=get_ssnr, + get_odg=get_odg, + get_odg_bin=get_odg_bin, ) @@ -64,16 +78,31 @@ def test_ambisonics_full_chain_split( @pytest.mark.parametrize("trajectory", SPLIT_REND_HR_TRAJECTORIES_TO_TEST) @pytest.mark.parametrize("render_config", RENDERER_CONFIGS_TO_TEST_AMBI) @pytest.mark.parametrize("in_fmt", INPUT_FORMATS_AMBI_SPLIT_REND) -def test_ambisonics_external_split(test_info, in_fmt, render_config, trajectory): +def test_ambisonics_external_split( + record_property, + props_to_record, + get_mld, + get_mld_lim, + get_ssnr, + get_odg, + get_odg_bin, + test_info, in_fmt, render_config, trajectory): post_trajectory = HR_TRAJECTORY_DIR.joinpath(f"{trajectory}.csv") pre_trajectory = post_trajectory.with_stem(f"{post_trajectory.stem}_delayed") run_external_split_rendering( + record_property, + props_to_record, test_info, in_fmt=in_fmt, render_config=RENDER_CFG_DIR.joinpath(f"{render_config}.txt"), pre_trajectory=pre_trajectory, post_trajectory=post_trajectory, + get_mld=get_mld, + mld_lim=get_mld_lim, + get_ssnr=get_ssnr, + get_odg=get_odg, + get_odg_bin=get_odg_bin, ) @@ -86,12 +115,21 @@ def test_ambisonics_external_split(test_info, in_fmt, render_config, trajectory) @pytest.mark.parametrize("bitrate", IVAS_BITRATES_MC) @pytest.mark.parametrize("in_fmt", INPUT_FORMATS_MC_SPLIT_REND) def test_multichannel_full_chain_split( + record_property, + props_to_record, + get_mld, + get_mld_lim, + get_ssnr, + get_odg, + get_odg_bin, test_info, in_fmt, bitrate, render_config, trajectory ): post_trajectory = HR_TRAJECTORY_DIR.joinpath(f"{trajectory}.csv") pre_trajectory = post_trajectory.with_stem(f"{post_trajectory.stem}_delayed") run_full_chain_split_rendering( + record_property, + props_to_record, test_info, in_fmt=in_fmt, bitrate=bitrate, @@ -99,6 +137,11 @@ def test_multichannel_full_chain_split( binary_suffix=EXE_SUFFIX, pre_trajectory=pre_trajectory, post_trajectory=post_trajectory, + get_mld=get_mld, + mld_lim=get_mld_lim, + get_ssnr=get_ssnr, + get_odg=get_odg, + get_odg_bin=get_odg_bin, ) @@ -106,16 +149,31 @@ def test_multichannel_full_chain_split( @pytest.mark.parametrize("trajectory", SPLIT_REND_HR_TRAJECTORIES_TO_TEST) @pytest.mark.parametrize("render_config", RENDERER_CONFIGS_TO_TEST_MC) @pytest.mark.parametrize("in_fmt", INPUT_FORMATS_MC_SPLIT_REND) -def test_multichannel_external_split(test_info, in_fmt, render_config, trajectory): +def test_multichannel_external_split( + record_property, + props_to_record, + get_mld, + get_mld_lim, + get_ssnr, + get_odg, + get_odg_bin, +test_info, in_fmt, render_config, trajectory): post_trajectory = HR_TRAJECTORY_DIR.joinpath(f"{trajectory}.csv") pre_trajectory = post_trajectory.with_stem(f"{post_trajectory.stem}_delayed") run_external_split_rendering( + record_property, + props_to_record, test_info, in_fmt=in_fmt, render_config=RENDER_CFG_DIR.joinpath(f"{render_config}.txt"), pre_trajectory=pre_trajectory, post_trajectory=post_trajectory, + get_mld=get_mld, + mld_lim=get_mld_lim, + get_ssnr=get_ssnr, + get_odg=get_odg, + get_odg_bin=get_odg_bin, ) @@ -127,11 +185,21 @@ def test_multichannel_external_split(test_info, in_fmt, render_config, trajector @pytest.mark.parametrize("render_config", RENDERER_CONFIGS_TO_TEST_ISM) @pytest.mark.parametrize("bitrate", IVAS_BITRATES_ISM) @pytest.mark.parametrize("in_fmt", INPUT_FORMATS_ISM_SPLIT_REND) -def test_ism_full_chain_split(test_info, in_fmt, bitrate, render_config, trajectory): +def test_ism_full_chain_split( + record_property, + props_to_record, + get_mld, + get_mld_lim, + get_ssnr, + get_odg, + get_odg_bin, +test_info, in_fmt, bitrate, render_config, trajectory): post_trajectory = HR_TRAJECTORY_DIR.joinpath(f"{trajectory}.csv") pre_trajectory = post_trajectory.with_stem(f"{post_trajectory.stem}_delayed") run_full_chain_split_rendering( + record_property, + props_to_record, test_info, in_fmt=in_fmt, bitrate=bitrate, @@ -139,6 +207,11 @@ def test_ism_full_chain_split(test_info, in_fmt, bitrate, render_config, traject binary_suffix=EXE_SUFFIX, pre_trajectory=pre_trajectory, post_trajectory=post_trajectory, + get_mld=get_mld, + mld_lim=get_mld_lim, + get_ssnr=get_ssnr, + get_odg=get_odg, + get_odg_bin=get_odg_bin, ) @@ -146,16 +219,31 @@ def test_ism_full_chain_split(test_info, in_fmt, bitrate, render_config, traject @pytest.mark.parametrize("trajectory", SPLIT_REND_HR_TRAJECTORIES_TO_TEST) @pytest.mark.parametrize("render_config", RENDERER_CONFIGS_TO_TEST_ISM) @pytest.mark.parametrize("in_fmt", INPUT_FORMATS_ISM_SPLIT_REND) -def test_ism_external_split(test_info, in_fmt, render_config, trajectory): +def test_ism_external_split( + record_property, + props_to_record, + get_mld, + get_mld_lim, + get_ssnr, + get_odg, + get_odg_bin, +test_info, in_fmt, render_config, trajectory): post_trajectory = HR_TRAJECTORY_DIR.joinpath(f"{trajectory}.csv") pre_trajectory = post_trajectory.with_stem(f"{post_trajectory.stem}_delayed") run_external_split_rendering( + record_property, + props_to_record, test_info, in_fmt=in_fmt, render_config=RENDER_CFG_DIR.joinpath(f"{render_config}.txt"), pre_trajectory=pre_trajectory, post_trajectory=post_trajectory, + get_mld=get_mld, + mld_lim=get_mld_lim, + get_ssnr=get_ssnr, + get_odg=get_odg, + get_odg_bin=get_odg_bin, ) @@ -167,11 +255,21 @@ def test_ism_external_split(test_info, in_fmt, render_config, trajectory): @pytest.mark.parametrize("render_config", RENDERER_CONFIGS_TO_TEST_MASA) @pytest.mark.parametrize("bitrate", IVAS_BITRATES_MASA) @pytest.mark.parametrize("in_fmt", INPUT_FORMATS_MASA_SPLIT_REND) -def test_masa_full_chain_split(test_info, in_fmt, bitrate, render_config, trajectory): +def test_masa_full_chain_split( + record_property, + props_to_record, + get_mld, + get_mld_lim, + get_ssnr, + get_odg, + get_odg_bin, +test_info, in_fmt, bitrate, render_config, trajectory): post_trajectory = HR_TRAJECTORY_DIR.joinpath(f"{trajectory}.csv") pre_trajectory = post_trajectory.with_stem(f"{post_trajectory.stem}_delayed") run_full_chain_split_rendering( + record_property, + props_to_record, test_info, in_fmt=in_fmt, bitrate=bitrate, @@ -179,6 +277,11 @@ def test_masa_full_chain_split(test_info, in_fmt, bitrate, render_config, trajec binary_suffix=EXE_SUFFIX, pre_trajectory=pre_trajectory, post_trajectory=post_trajectory, + get_mld=get_mld, + mld_lim=get_mld_lim, + get_ssnr=get_ssnr, + get_odg=get_odg, + get_odg_bin=get_odg_bin, ) @@ -186,16 +289,31 @@ def test_masa_full_chain_split(test_info, in_fmt, bitrate, render_config, trajec @pytest.mark.parametrize("trajectory", SPLIT_REND_HR_TRAJECTORIES_TO_TEST) @pytest.mark.parametrize("render_config", RENDERER_CONFIGS_TO_TEST_MASA) @pytest.mark.parametrize("in_fmt", INPUT_FORMATS_MASA_SPLIT_REND) -def test_masa_external_split(test_info, in_fmt, render_config, trajectory): +def test_masa_external_split( + record_property, + props_to_record, + get_mld, + get_mld_lim, + get_ssnr, + get_odg, + get_odg_bin, +test_info, in_fmt, render_config, trajectory): post_trajectory = HR_TRAJECTORY_DIR.joinpath(f"{trajectory}.csv") pre_trajectory = post_trajectory.with_stem(f"{post_trajectory.stem}_delayed") run_external_split_rendering( + record_property, + props_to_record, test_info, in_fmt=in_fmt, render_config=RENDER_CFG_DIR.joinpath(f"{render_config}.txt"), pre_trajectory=pre_trajectory, post_trajectory=post_trajectory, + get_mld=get_mld, + mld_lim=get_mld_lim, + get_ssnr=get_ssnr, + get_odg=get_odg, + get_odg_bin=get_odg_bin, ) @@ -207,11 +325,21 @@ def test_masa_external_split(test_info, in_fmt, render_config, trajectory): @pytest.mark.parametrize("render_config", RENDERER_CONFIGS_TO_TEST_OMASA) @pytest.mark.parametrize("bitrate", IVAS_BITRATES_OMASA) @pytest.mark.parametrize("in_fmt", INPUT_FORMATS_OMASA_SPLIT_REND) -def test_omasa_full_chain_split(test_info, in_fmt, bitrate, render_config, trajectory): +def test_omasa_full_chain_split( + record_property, + props_to_record, + get_mld, + get_mld_lim, + get_ssnr, + get_odg, + get_odg_bin, +test_info, in_fmt, bitrate, render_config, trajectory): post_trajectory = HR_TRAJECTORY_DIR.joinpath(f"{trajectory}.csv") pre_trajectory = post_trajectory.with_stem(f"{post_trajectory.stem}_delayed") run_full_chain_split_rendering( + record_property, + props_to_record, test_info, in_fmt=in_fmt, bitrate=bitrate, @@ -219,6 +347,11 @@ def test_omasa_full_chain_split(test_info, in_fmt, bitrate, render_config, traje binary_suffix=EXE_SUFFIX, pre_trajectory=pre_trajectory, post_trajectory=post_trajectory, + get_mld=get_mld, + mld_lim=get_mld_lim, + get_ssnr=get_ssnr, + get_odg=get_odg, + get_odg_bin=get_odg_bin, ) @@ -230,11 +363,21 @@ def test_omasa_full_chain_split(test_info, in_fmt, bitrate, render_config, traje @pytest.mark.parametrize("render_config", RENDERER_CONFIGS_TO_TEST_OSBA) @pytest.mark.parametrize("bitrate", IVAS_BITRATES_OSBA) @pytest.mark.parametrize("in_fmt", INPUT_FORMATS_OSBA_SPLIT_REND) -def test_osba_full_chain_split(test_info, in_fmt, bitrate, render_config, trajectory): +def test_osba_full_chain_split( + record_property, + props_to_record, + get_mld, + get_mld_lim, + get_ssnr, + get_odg, + get_odg_bin, +test_info, in_fmt, bitrate, render_config, trajectory): post_trajectory = HR_TRAJECTORY_DIR.joinpath(f"{trajectory}.csv") pre_trajectory = post_trajectory.with_stem(f"{post_trajectory.stem}_delayed") run_full_chain_split_rendering( + record_property, + props_to_record, test_info, in_fmt=in_fmt, bitrate=bitrate, @@ -242,6 +385,11 @@ def test_osba_full_chain_split(test_info, in_fmt, bitrate, render_config, trajec binary_suffix=EXE_SUFFIX, pre_trajectory=pre_trajectory, post_trajectory=post_trajectory, + get_mld=get_mld, + mld_lim=get_mld_lim, + get_ssnr=get_ssnr, + get_odg=get_odg, + get_odg_bin=get_odg_bin, ) @@ -253,17 +401,32 @@ def test_osba_full_chain_split(test_info, in_fmt, bitrate, render_config, trajec @pytest.mark.parametrize("trajectory", SPLIT_REND_HR_TRAJECTORIES_TO_TEST) @pytest.mark.parametrize("render_config", RENDERER_CONFIGS_TO_TEST_PLC) @pytest.mark.parametrize("in_fmt", INPUT_FORMATS_AMBI_SPLIT_REND[-1:]) -def test_post_rend_plc(test_info, in_fmt, render_config, trajectory, error_pattern): +def test_post_rend_plc( + record_property, + props_to_record, + get_mld, + get_mld_lim, + get_ssnr, + get_odg, + get_odg_bin, +test_info, in_fmt, render_config, trajectory, error_pattern): post_trajectory = HR_TRAJECTORY_DIR.joinpath(f"{trajectory}.csv") pre_trajectory = post_trajectory.with_stem(f"{post_trajectory.stem}_delayed") run_external_split_rendering( + record_property, + props_to_record, test_info, in_fmt=in_fmt, render_config=RENDER_CFG_DIR.joinpath(f"{render_config}.txt"), pre_trajectory=pre_trajectory, post_trajectory=post_trajectory, plc_error_pattern=ERROR_PATTERNS_DIR.joinpath(f"{error_pattern}.ep"), + get_mld=get_mld, + mld_lim=get_mld_lim, + get_ssnr=get_ssnr, + get_odg=get_odg, + get_odg_bin=get_odg_bin, ) @@ -281,12 +444,22 @@ full_chain_split_pcm_params = [ @pytest.mark.parametrize("in_fmt,bitrate,render_config", full_chain_split_pcm_params) -def test_full_chain_split_pcm(test_info, in_fmt, bitrate, render_config): +def test_full_chain_split_pcm( + record_property, + props_to_record, + get_mld, + get_mld_lim, + get_ssnr, + get_odg, + get_odg_bin, +test_info, in_fmt, bitrate, render_config): trajectory = SPLIT_REND_HR_TRAJECTORIES_TO_TEST[0] post_trajectory = HR_TRAJECTORY_DIR.joinpath(f"{trajectory}.csv") pre_trajectory = post_trajectory.with_stem(f"{post_trajectory.stem}_delayed") run_full_chain_split_rendering( + record_property, + props_to_record, test_info, in_fmt=in_fmt, bitrate=bitrate, @@ -295,6 +468,11 @@ def test_full_chain_split_pcm(test_info, in_fmt, bitrate, render_config): pre_trajectory=pre_trajectory, post_trajectory=post_trajectory, renderer_fmt="BINAURAL_SPLIT_PCM", + get_mld=get_mld, + mld_lim=get_mld_lim, + get_ssnr=get_ssnr, + get_odg=get_odg, + get_odg_bin=get_odg_bin, ) @@ -307,18 +485,33 @@ external_split_pcm_params = [ @pytest.mark.parametrize("in_fmt,render_config", external_split_pcm_params) -def test_external_split_pcm(test_info, in_fmt, render_config): +def test_external_split_pcm( + record_property, + props_to_record, + get_mld, + get_mld_lim, + get_ssnr, + get_odg, + get_odg_bin, +test_info, in_fmt, render_config): trajectory = SPLIT_REND_HR_TRAJECTORIES_TO_TEST[0] post_trajectory = HR_TRAJECTORY_DIR.joinpath(f"{trajectory}.csv") pre_trajectory = post_trajectory.with_stem(f"{post_trajectory.stem}_delayed") run_external_split_rendering( + record_property, + props_to_record, test_info, in_fmt=in_fmt, render_config=RENDER_CFG_DIR.joinpath(f"{render_config}.txt"), pre_trajectory=pre_trajectory, post_trajectory=post_trajectory, renderer_fmt="BINAURAL_SPLIT_PCM", + get_mld=get_mld, + mld_lim=get_mld_lim, + get_ssnr=get_ssnr, + get_odg=get_odg, + get_odg_bin=get_odg_bin, ) @@ -327,11 +520,21 @@ def test_external_split_pcm(test_info, in_fmt, render_config): @pytest.mark.parametrize("in_fmt", ["5_1"]) @pytest.mark.parametrize("pre_rend_fr", SPLIT_RENDERER_PRE_FRAMINGS) @pytest.mark.parametrize("post_rend_fr", SPLIT_RENDERER_POST_FRAMINGS) -def test_framing_combinations_external_split(test_info, in_fmt, render_config, trajectory, post_rend_fr, pre_rend_fr): +def test_framing_combinations_external_split( + record_property, + props_to_record, + get_mld, + get_mld_lim, + get_ssnr, + get_odg, + get_odg_bin, +test_info, in_fmt, render_config, trajectory, post_rend_fr, pre_rend_fr): post_trajectory = HR_TRAJECTORY_DIR.joinpath(f"{trajectory}.csv") pre_trajectory = post_trajectory.with_stem(f"{post_trajectory.stem}_delayed") run_external_split_rendering( + record_property, + props_to_record, test_info, in_fmt=in_fmt, render_config=RENDER_FRAMING_CFG_DIR.joinpath(f"{render_config}.txt"), @@ -339,6 +542,11 @@ def test_framing_combinations_external_split(test_info, in_fmt, render_config, t post_trajectory=post_trajectory, post_rend_fr=post_rend_fr, pre_rend_fr=pre_rend_fr, + get_mld=get_mld, + mld_lim=get_mld_lim, + get_ssnr=get_ssnr, + get_odg=get_odg, + get_odg_bin=get_odg_bin, ) @pytest.mark.parametrize("trajectory", SPLIT_REND_HR_TRAJECTORIES_TO_TEST) @@ -347,12 +555,21 @@ def test_framing_combinations_external_split(test_info, in_fmt, render_config, t @pytest.mark.parametrize("pre_rend_fr", SPLIT_RENDERER_PRE_FRAMINGS) @pytest.mark.parametrize("post_rend_fr", SPLIT_RENDERER_POST_FRAMINGS) def test_framing_combinations_full_chain_split( + record_property, + props_to_record, + get_mld, + get_mld_lim, + get_ssnr, + get_odg, + get_odg_bin, test_info, in_fmt, render_config, trajectory, post_rend_fr, pre_rend_fr ): post_trajectory = HR_TRAJECTORY_DIR.joinpath(f"{trajectory}.csv") pre_trajectory = post_trajectory.with_stem(f"{post_trajectory.stem}_delayed") run_full_chain_split_rendering( + record_property, + props_to_record, test_info, in_fmt=in_fmt, render_config=RENDER_FRAMING_CFG_DIR.joinpath(f"{render_config}.txt"), @@ -362,4 +579,9 @@ def test_framing_combinations_full_chain_split( binary_suffix=EXE_SUFFIX, post_rend_fr=post_rend_fr, pre_rend_fr=pre_rend_fr, + get_mld=get_mld, + mld_lim=get_mld_lim, + get_ssnr=get_ssnr, + get_odg=get_odg, + get_odg_bin=get_odg_bin, ) diff --git a/tests/split_rendering/utils.py b/tests/split_rendering/utils.py index ed0d3495fc3d7a37327fadaaa3ca3b5c214d4d1c..53de170c917de2c62e7ae3d98dc08dd99352fe92 100644 --- a/tests/split_rendering/utils.py +++ b/tests/split_rendering/utils.py @@ -41,6 +41,8 @@ import pytest from tests.renderer.utils import check_BE, run_ivas_isar_enc_cmd, run_ivas_isar_dec_cmd, run_isar_post_rend_cmd, run_isar_ext_rend_cmd from tests.split_rendering.constants import * +from ..cmp_pcm import cmp_pcm +from ..conftest import parse_properties sys.path.append(SCRIPTS_DIR) from pyaudio3dtools.audiofile import readfile, writefile @@ -151,6 +153,8 @@ def truncate_signal( def run_full_chain_split_rendering( + record_property, + props_to_record, test_info, in_fmt: str, bitrate: str, @@ -161,6 +165,11 @@ def run_full_chain_split_rendering( binary_suffix: str = "", post_rend_fr: str = "20", pre_rend_fr: str = "20", + get_mld=False, + mld_lim=0, + get_ssnr=False, + get_odg=False, + get_odg_bin=False, ) -> str: """ Runs the full split rendering chain consisting of @@ -274,15 +283,41 @@ def run_full_chain_split_rendering( cut, cut_fs = readfile(out_file) - [diff_found, snr, gain_b, max_diff] = check_BE(test_info, ref, ref_fs, cut, cut_fs) - if diff_found : - pytest.fail( - f"CuT not BE to REF! SNR : {snr:3.2f} dB, Gain CuT: {gain_b:1.3f}, Max Diff = {int(max_diff)}" + if get_mld == False: + [diff_found, snr, gain_b, max_diff] = check_BE(test_info, ref, ref_fs, cut, cut_fs) + if diff_found : + pytest.fail( + f"CuT not BE to REF! SNR : {snr:3.2f} dB, Gain CuT: {gain_b:1.3f}, Max Diff = {int(max_diff)}" + ) + else: + # see constants.py + output_differs, reason = cmp_pcm( + out_file, + out_file_ref, + "BINAURAL", + ref_fs, + get_mld=get_mld, + mld_lim=mld_lim, + get_ssnr=get_ssnr, + get_odg=get_odg, + get_odg_bin=get_odg_bin, ) + + props = parse_properties(reason, output_differs, props_to_record) + for k, v in props.items(): + record_property(k, v) + + if output_differs: + pytest.fail(f"Output differs: ({reason})") + + + return out_file def run_external_split_rendering( + record_property, + props_to_record, test_info, in_fmt: str, render_config: Path, @@ -294,6 +329,11 @@ def run_external_split_rendering( is_comparetest: bool = False, post_rend_fr: str = "20", pre_rend_fr: str = "20", + get_mld=False, + mld_lim=0, + get_ssnr=False, + get_odg=False, + get_odg_bin=False, ) -> Tuple[np.ndarray, int]: """ Runs the exeternal split rendering chain consisting of @@ -393,9 +433,32 @@ def run_external_split_rendering( cut, cut_fs = readfile(out_file) - [diff_found, snr, gain_b, max_diff] = check_BE(test_info, ref, ref_fs, cut, cut_fs) - if diff_found : - pytest.fail( - f"CuT not BE to REF! SNR : {snr:3.2f} dB, Gain CuT: {gain_b:1.3f}, Max Diff = {int(max_diff)}" + if get_mld == False: + [diff_found, snr, gain_b, max_diff] = check_BE(test_info, ref, ref_fs, cut, cut_fs) + if diff_found : + pytest.fail( + f"CuT not BE to REF! SNR : {snr:3.2f} dB, Gain CuT: {gain_b:1.3f}, Max Diff = {int(max_diff)}" + ) + else: + # see constants.py + output_differs, reason = cmp_pcm( + out_file, + out_file_ref, + "BINAURAL", + ref_fs, + get_mld=get_mld, + mld_lim=mld_lim, + get_ssnr=get_ssnr, + get_odg=get_odg, + get_odg_bin=get_odg_bin, ) + + props = parse_properties(reason, output_differs, props_to_record) + for k, v in props.items(): + record_property(k, v) + + if output_differs: + pytest.fail(f"Output differs: ({reason})") + + return out_file diff --git a/tests/test_26444.py b/tests/test_26444.py index cdf065ef3e437fb740e61a34af4e065f8d4bc9e7..9420f15551033f7838b0df55d8744613496da509 100644 --- a/tests/test_26444.py +++ b/tests/test_26444.py @@ -40,11 +40,12 @@ import shutil from tests.cmp_pcm import cmp_pcm from tests.conftest import DecoderFrontend, EncoderFrontend, parse_properties +from tests.constants import CAT_BITRATE_SWITCHING, CAT_DTX, CAT_JBM, CAT_NORMAL, CAT_PLC test_dict = {} TEST_DIR = "evs_be_test" -scripts = [ +SCRIPTS = [ "Readme_AMRWB_IO_dec.txt", "Readme_AMRWB_IO_enc.txt", "Readme_EVS_dec.txt", @@ -52,7 +53,16 @@ scripts = [ "Readme_JBM_dec.txt", ] -for s in scripts: +FORMATS_4_SCRIPTS = dict( + zip( + [s.replace(".txt", "") for s in SCRIPTS], + ["AMRWBIO_dec", "AMRWBIO_enc", "EVS_dec", "EVS_enc", "EVS_JBM_dec"], + ) +) + +PATTERN_EVS_FORMAT = re.compile(r"(" + r"|".join(FORMATS_4_SCRIPTS.keys()) + ")") + +for s in SCRIPTS: with open(os.path.join(TEST_DIR, s), "r", encoding="UTF-8") as fp: tag = "" enc_opts = "" @@ -88,11 +98,33 @@ def test_evs_26444( abs_tol, get_ssnr, get_odg, + record_property, ): enc_opts, dec_opts, diff_opts = test_dict[test_tag] + testcase_props = {} + + # get format prop from test_tag + m = re.search(PATTERN_EVS_FORMAT, test_tag) + assert m is not None + testcase_props["format"] = FORMATS_4_SCRIPTS[m.groups()[0]] diff_opts = diff_opts.replace("./", TEST_DIR + "/") + category = CAT_NORMAL + if "JBM" in test_tag: + category = CAT_JBM + elif "br sw" in test_tag or "bitrate switching" in test_tag: + category = CAT_BITRATE_SWITCHING + elif "%" in test_tag: + category = CAT_PLC + elif "DTX" in test_tag: + category = CAT_DTX + + testcase_props["category"] = category + + for k, v in testcase_props.items(): + record_property(k, v) + if enc_opts: args = enc_opts.split()[1:] @@ -176,6 +208,8 @@ def test_evs_26444( get_ssnr=get_ssnr, get_odg=get_odg, ) + output_differs = output_differs[0] + reason = reason[0] props = parse_properties(reason, output_differs, props_to_record) for k, v in props.items():