diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index a35decca5d74adf44b8c6d2a58fcf18c02498fa6..e7a8c30475c9e583837c7f77b589da664fe70969 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -27,10 +27,17 @@ variables: - 'test-branch-vs-input-passthrough' GIT_CLEAN_FLAGS: -ffdxq - TESTCASE_TIMEOUT_STV_SANITIZERS: 180 + TESTCASE_TIMEOUT_STV_SANITIZERS: 240 TESTCASE_TIMEOUT_LTV_SANITIZERS: 2400 BASOP_REFERENCE_BRANCH: "ivas-float-update" SCALE_FACTOR: "3.162" + PYTEST_ARGS: "" + LONG_TEST_SUITE: "tests/codec_be_on_mr_nonselection tests/renderer --param_file scripts/config/self_test_ltv.prm --use_ltv" + LONG_TEST_SUITE_NO_RENDERER: "tests/codec_be_on_mr_nonselection --param_file scripts/config/self_test_ltv.prm --use_ltv" + SHORT_TEST_SUITE: "tests/codec_be_on_mr_nonselection" + SHORT_TEST_SUITE_ENCODER: "tests/codec_be_on_mr_nonselection/test_param_file.py --param_file scripts/config/self_test_basop_encoder.prm" + LONG_TEST_SUITE_ENCODER: "tests/codec_be_on_mr_nonselection/test_param_file.py --param_file scripts/config/self_test_ltv_basop_encoder.prm" + TEST_SUITE: "" default: @@ -132,6 +139,15 @@ stages: - sed -i.bak -e "s/\/\*\ *\(#define\ *DEBUGGING\ *\)\*\//\1/g" lib_com/options.h - sed -i.bak -e "s/\/\/\ *\(#define\ *DEBUGGING\ *\)/\1/g" lib_com/options.h +.get-basop-float-reference: &get-basop-float-reference + - git clone -b $BASOP_REFERENCE_BRANCH https://forge.3gpp.org/rep/sa4/audio/ivas-basop.git --single-branch --depth 1 ivas-basop + - cd ivas-basop + - git status + - make -j + - cd - + - cp ivas-basop/IVAS_cod ./ + - cp ivas-basop/IVAS_dec ./ + .merge-request-comparison-setup-codec: &merge-request-comparison-setup-codec ### build test binaries, initial clean for paranoia reasons - *disable-debugging-macro @@ -279,10 +295,24 @@ stages: when: never - when: on_success -.rules-merge-request: +.rules-merge-request-to-main: + extends: .rules-basis + rules: + - if: $CI_PIPELINE_SOURCE == 'merge_request_event' && $CI_MERGE_REQUEST_TARGET_BRANCH_NAME == "main" + - if: $CI_PIPELINE_SOURCE == 'push' + when: never + +.rules-merge-request-to-basop-ci-branch: + extends: .rules-basis + rules: + - if: $CI_PIPELINE_SOURCE == 'merge_request_event' && $CI_MERGE_REQUEST_TARGET_BRANCH_NAME == "basop-ci-branch" + - if: $CI_PIPELINE_SOURCE == 'push' + when: never + +.rules-merge-request-to-basop-ci-branch-or-main: extends: .rules-basis rules: - - if: $CI_PIPELINE_SOURCE == 'merge_request_event' && $CI_MERGE_REQUEST_TARGET_BRANCH_NAME == "main" # only have MR pipelines for MRs to main + - if: $CI_PIPELINE_SOURCE == 'merge_request_event' && ($CI_MERGE_REQUEST_TARGET_BRANCH_NAME == "basop-ci-branch" || $CI_MERGE_REQUEST_TARGET_BRANCH_NAME == "main") - if: $CI_PIPELINE_SOURCE == 'push' when: never @@ -327,7 +357,7 @@ stages: stage: test extends: - .test-job-linux-needs-testv-dir - - .rules-merge-request + - .rules-merge-request-to-main artifacts: name: "mr-$CI_MERGE_REQUEST_IID--sha-$CI_COMMIT_SHORT_SHA--stage-$CI_JOB_STAGE--results" expire_in: 1 week @@ -396,7 +426,7 @@ uninterruptible: branch-is-up-to-date-with-main-pre: extends: - - .rules-merge-request + - .rules-merge-request-to-main stage: prevalidate needs: [] tags: @@ -408,7 +438,7 @@ branch-is-up-to-date-with-main-pre: check-self-test-names-pre: extends: - - .rules-merge-request + - .rules-merge-request-to-main stage: prevalidate needs: [] tags: @@ -418,7 +448,7 @@ check-self-test-names-pre: check-no-duplicates-in-self-tests: extends: - - .rules-merge-request + - .rules-merge-request-to-main stage: prevalidate needs: [] tags: @@ -428,7 +458,7 @@ check-no-duplicates-in-self-tests: branch-is-up-to-date-with-main-post: extends: - - .rules-merge-request + - .rules-merge-request-to-main stage: postvalidate tags: - ivas-linux @@ -440,26 +470,22 @@ branch-is-up-to-date-with-main-post: .basop-ci-branch-compat-template: extends: - .test-job-linux - rules: - - if: $CI_PIPELINE_SOURCE == 'merge_request_event' && $CI_MERGE_REQUEST_TARGET_BRANCH_NAME == "basop-ci-branch" - changes: - - tests/**/* - - scripts/**/* - - if: $CI_PIPELINE_SOURCE == 'push' - when: never + - .rules-merge-request-to-basop-ci-branch tags: - - ivas-linux-fast + - ivas-linux before_script: - - git clone -b $BASOP_REFERENCE_BRANCH https://forge.3gpp.org/rep/sa4/audio/ivas-basop.git --single-branch - - make -j -C ivas-basop - - python3 ci/remove_unsupported_testcases.py $PARAM_FILE + - python3 ci/remove_unsupported_testcases.py scripts/config/self_test.prm scripts/config/self_test_ltv.prm - python3 tests/create_short_testvectors.py + - python3 scripts/prepare_combined_format_inputs.py + - *update-ltv-repo - *copy-ltv-files-to-testv-dir - - python3 -m pytest tests/codec_be_on_mr_nonselection --param_file $PARAM_FILE --update_ref 1 --ref_encoder_path ivas-basop/IVAS_cod --ref_decoder_path ivas-basop/IVAS_dec --html=report.html --self-contained-html --junit-xml=report-junit.xml || true + - *get-basop-float-reference + + - python3 -m pytest $TEST_SUITE $PYTEST_ARGS --update_ref 1 --ref_encoder_path IVAS_cod --ref_decoder_path IVAS_dec --html=report.html --self-contained-html --junit-xml=report-junit.xml --testcase_timeout=$TESTCASE_TIMEOUT || true - zero_errors=$(cat report-junit.xml | grep -c 'errors="0"') || true - - if [ $zero_errors != 1 ]; then echo "Run errors encountered with $PARAM_FILE !"; exit $EXIT_CODE_FAIL; fi + - if [ $zero_errors != 1 ]; then echo "Run errors encountered with TEST_SUITE=$TEST_SUITE !"; exit $EXIT_CODE_FAIL; fi artifacts: name: "$CI_JOB_NAME--sha-$CI_COMMIT_SHORT_SHA--results" @@ -474,35 +500,54 @@ branch-is-up-to-date-with-main-post: junit: - report-junit.xml -check-compatibility-with-basop-reference-branch-stv: +basop-ref-compat-stv: extends: - .basop-ci-branch-compat-template variables: - PARAM_FILE: scripts/config/self_test.prm + TEST_SUITE: "$SHORT_TEST_SUITE tests/renderer" + TESTCASE_TIMEOUT: $TESTCASE_TIMEOUT_STV_SANITIZERS script: - exit 0 -check-compatibility-with-basop-reference-branch-ltv: +basop-ref-compat-ltv: extends: - .basop-ci-branch-compat-template + tags: + - ivas-linux-fast variables: - PARAM_FILE: scripts/config/self_test_ltv.prm + TEST_SUITE: $LONG_TEST_SUITE + TESTCASE_TIMEOUT: $TESTCASE_TIMEOUT_LTV_SANITIZERS script: - exit 0 -check-compatibility-with-basop-reference-branch-encoder-stv: +basop-ref-compat-encoder-stv: extends: - .basop-ci-branch-compat-template variables: - PARAM_FILE: scripts/config/self_test_basop_encoder.prm + TEST_SUITE: $SHORT_TEST_SUITE_ENCODER + TESTCASE_TIMEOUT: $TESTCASE_TIMEOUT_STV_SANITIZERS script: - exit 0 -check-compatibility-with-basop-reference-branch-encoder-ltv: +basop-ref-compat-encoder-ltv: extends: - .basop-ci-branch-compat-template + tags: + - ivas-linux-fast variables: - PARAM_FILE: scripts/config/self_test_ltv_basop_encoder.prm + TEST_SUITE: $LONG_TEST_SUITE_ENCODER + TESTCASE_TIMEOUT: $TESTCASE_TIMEOUT_LTV_SANITIZERS + script: + - exit 0 + +basop-ref-compat-encoder-dmx-comp: + extends: + - .basop-ci-branch-compat-template + variables: + TEST_SUITE: $SHORT_TEST_SUITE_ENCODER + # USING PYTEST_ADDOPTS env var did not work for some reason when testing locally - maybe because this is a custom option + PYTEST_ARGS: "--compare_enc_dmx" + TESTCASE_TIMEOUT: $TESTCASE_TIMEOUT_STV_SANITIZERS script: - exit 0 @@ -579,7 +624,7 @@ build-codec-windows-msbuild: codec-smoke-test: extends: - .test-job-linux-needs-testv-dir - - .rules-merge-request + - .rules-merge-request-to-basop-ci-branch-or-main timeout: "20 minutes" tags: - ivas-linux-fast @@ -589,7 +634,17 @@ codec-smoke-test: - *print-common-info # LTV update needed as ltv ISM metadata files are used - *update-ltv-repo - - bash ci/smoke_test.sh + + # for MRs to basop-ci-branch, we want to test with the BASOP float reference build + # per default, the smoke test builds the current repo again with WMOPS activated to catch unbalanced instrumentation macros + - if [ "$CI_MERGE_REQUEST_TARGET_BRANCH_NAME" = "basop-ci-branch" ]; then + - python3 scripts/smoketest-basop-filter.py scripts/config/ivas_modes.json --inline + - *get-basop-float-reference + - bash ci/smoke_test.sh coverage + - else + - bash ci/smoke_test.sh + - fi + ### analyze for failures - if ! [ -s smoke_test_output.txt ] || ! [ -s smoke_test_output_jbm.txt ] || ! [ -s smoke_test_output_hrtf.txt ]; then echo "Error in smoke test"; exit 1; fi - ret_val=0 @@ -648,7 +703,7 @@ codec-usan: pytest-compare-20ms-and-5ms-rendering: extends: - .test-job-linux - - .rules-merge-request + - .rules-merge-request-to-main stage: test needs: ["build-codec-linux-cmake", "build-codec-linux-make", "build-codec-instrumented-linux", "build-codec-sanitizers-linux"] script: @@ -695,7 +750,7 @@ pytest-compare-20ms-and-5ms-rendering: renderer-smoke-test: extends: - .test-job-linux - - .rules-merge-request + - .rules-merge-request-to-main needs: ["build-codec-linux-make"] stage: test script: @@ -717,7 +772,7 @@ renderer-smoke-test: renderer-asan: extends: - .test-job-linux - - .rules-merge-request + - .rules-merge-request-to-main needs: ["build-codec-linux-cmake"] stage: test script: @@ -741,7 +796,7 @@ renderer-asan: renderer-msan: extends: - .test-job-linux - - .rules-merge-request + - .rules-merge-request-to-main needs: ["build-codec-linux-cmake"] stage: test script: @@ -765,7 +820,7 @@ renderer-msan: renderer-usan: extends: - .test-job-linux - - .rules-merge-request + - .rules-merge-request-to-main needs: ["build-codec-linux-cmake"] stage: test script: @@ -793,7 +848,7 @@ renderer-usan: renderer-pytest-on-merge-request: extends: - .test-job-linux-needs-testv-dir - - .rules-merge-request + - .rules-merge-request-to-main needs: ["build-codec-linux-make"] # TODO: set reasonable timeout, will most likely take less timeout: "20 minutes" @@ -845,7 +900,7 @@ renderer-pytest-on-merge-request: split-rendering-smoke-test: extends: - .test-job-linux - - .rules-merge-request + - .rules-merge-request-to-main needs: ["build-codec-linux-make"] stage: test script: @@ -866,7 +921,7 @@ split-rendering-smoke-test: lc3-wrapper-unit-test: extends: - .test-job-linux - - .rules-merge-request + - .rules-merge-request-to-main needs: ["build-codec-linux-cmake"] stage: test script: @@ -878,7 +933,7 @@ lc3-wrapper-unit-test: split-rendering-pytest-on-merge-request: extends: - .test-job-linux-needs-testv-dir - - .rules-merge-request + - .rules-merge-request-to-main needs: ["build-codec-linux-make"] # TODO: set reasonable timeout, will most likely take less timeout: "30 minutes" @@ -950,7 +1005,7 @@ split-rendering-pytest-on-merge-request: ivas-pytest-on-merge-request: extends: - .test-job-linux-needs-testv-dir - - .rules-merge-request + - .rules-merge-request-to-main stage: compare needs: ["build-codec-linux-cmake", "codec-smoke-test"] timeout: "14 minutes" @@ -1004,7 +1059,7 @@ ivas-pytest-on-merge-request: ivas-interop-on-merge-request: extends: - .test-job-linux-needs-testv-dir - - .rules-merge-request + - .rules-merge-request-to-main stage: test needs: ["build-codec-linux-cmake"] timeout: "10 minutes" @@ -1053,7 +1108,7 @@ ivas-interop-on-merge-request: evs-pytest-on-merge-request: extends: - .test-job-linux - - .rules-merge-request + - .rules-merge-request-to-main stage: compare needs: ["build-codec-linux-cmake", "codec-smoke-test"] timeout: "10 minutes" @@ -1105,7 +1160,7 @@ evs-pytest-on-merge-request: voip-be-on-merge-request: extends: - .test-job-linux-needs-testv-dir - - .rules-merge-request + - .rules-merge-request-to-main stage: test needs: ["build-codec-linux-make"] timeout: "10 minutes" @@ -1118,7 +1173,7 @@ voip-be-on-merge-request: clang-format-check: extends: - .test-job-linux - - .rules-merge-request + - .rules-merge-request-to-main variables: ARTIFACT_BASE_NAME: "mr-$CI_MERGE_REQUEST_IID--sha-$CI_COMMIT_SHORT_SHA--formatting-fix" stage: prevalidate @@ -1165,7 +1220,7 @@ clang-format-check: check-first-frame-is-sid: extends: - .test-job-linux-needs-testv-dir - - .rules-merge-request + - .rules-merge-request-to-main tags: - ivas-linux stage: test @@ -1192,7 +1247,7 @@ check-first-frame-is-sid: .lc3plus-ensure-no-code-changes: extends: - .test-job-linux - - .rules-merge-request + - .rules-merge-request-to-main stage: postvalidate needs: [] timeout: "5 minutes" @@ -1207,7 +1262,7 @@ check-first-frame-is-sid: check-bitexactness-hrtf-rom-and-file: extends: - .test-job-linux - - .rules-merge-request + - .rules-merge-request-to-main stage: test needs: ["build-codec-linux-cmake"] timeout: "5 minutes" @@ -1229,7 +1284,7 @@ check-bitexactness-hrtf-rom-and-file: check-bitexactness-ext-and-transport-format: extends: - .test-job-linux - - .rules-merge-request + - .rules-merge-request-to-main stage: test needs: ["build-codec-linux-cmake"] timeout: "5 minutes" @@ -1256,7 +1311,7 @@ check-bitexactness-ext-and-transport-format: # check bitexactness to EVS windows binaries be-2-evs-windows: extends: - - .rules-merge-request + - .rules-merge-request-to-main tags: - ivas-windows stage: test diff --git a/ci/get_float_ref_branch_name.sh b/ci/get_float_ref_branch_name.sh new file mode 100755 index 0000000000000000000000000000000000000000..fe160da02331b04249febb89971c90d729c198a8 --- /dev/null +++ b/ci/get_float_ref_branch_name.sh @@ -0,0 +1,65 @@ +#!/bin/bash + +# (C) 2022-2025 IVAS codec Public Collaboration with portions copyright Dolby International AB, Ericsson AB, +# Fraunhofer-Gesellschaft zur Foerderung der angewandten Forschung e.V., Huawei Technologies Co. LTD., +# Koninklijke Philips N.V., Nippon Telegraph and Telephone Corporation, Nokia Technologies Oy, Orange, +# Panasonic Holdings Corporation, Qualcomm Technologies, Inc., VoiceAge Corporation, and other +# contributors to this repository. All Rights Reserved. +# +# This software is protected by copyright law and by international treaties. +# The IVAS codec Public Collaboration consisting of Dolby International AB, Ericsson AB, +# Fraunhofer-Gesellschaft zur Foerderung der angewandten Forschung e.V., Huawei Technologies Co. LTD., +# Koninklijke Philips N.V., Nippon Telegraph and Telephone Corporation, Nokia Technologies Oy, Orange, +# Panasonic Holdings Corporation, Qualcomm Technologies, Inc., VoiceAge Corporation, and other +# contributors to this repository retain full ownership rights in their respective contributions in +# the software. This notice grants no license of any kind, including but not limited to patent +# license, nor is any license granted by implication, estoppel or otherwise. +# +# Contributors are required to enter into the IVAS codec Public Collaboration agreement before making +# contributions. +# +# This software is provided "AS IS", without any express or implied warranties. The software is in the +# development stage. It is intended exclusively for experts who have experience with such software and +# solely for the purpose of inspection. All implied warranties of non-infringement, merchantability +# and fitness for a particular purpose are hereby disclaimed and excluded. +# +# Any dispute, controversy or claim arising under or in relation to providing this software shall be +# submitted to and settled by the final, binding jurisdiction of the courts of Munich, Germany in +# accordance with the laws of the Federal Republic of Germany excluding its conflict of law rules and +# the United Nations Convention on Contracts on the International Sales of Goods. + +BASOP_PATTERN="^[0-9]+[_-]basop[_-].*" +MSG_DOES_NOT_MATCH="Your branch name %s does not match the template '_basop_', e.g. '123_basop_fix_this_one_bug-2'. +Sticking to this branch is needed for the testing system to match this branch with its float-reference counterpart. +Please rename your branch. You can easily do this by creating a new branch from this branch: + - git checkout -b +You then also need to create a new merge request and update the links in your issue. +See here for details on the porting work process: https://forge.3gpp.org/rep/sa4/audio/ivas-basop/-/wikis/Porting-MRs-from-floating-point-codec#workflow-for-porting-a-merge-request-from-floating-point-codec-to-basop-codec +" + +if [ $# -ne 1 ]; then + echo "Usage: $0 " + exit 1 +fi + +branchname="$1" + +if ! [[ "$branchname" =~ $BASOP_PATTERN ]]; then + printf "$MSG_DOES_NOT_MATCH" "$branchname" + exit 1 +fi + +# The float ref branch is just the same name, but with basop -> ref replacement +# Replace only the first occurrence, as "basop" may be present in the later description +# If the format is correct, then before "_basop", only numbers can occur +float_ref_branchname="${branchname/basop/ref}" +git_result=$(git branch -av) + +# If the branch does not exist, default to "float-pc" +if [[ "$git_result" =~ "$float_ref_branchname" ]]; then + branchname="${float_ref_branchname}" +else + branchname="float-pc" +fi + +echo "$branchname" diff --git a/ci/remove_unsupported_testcases.py b/ci/remove_unsupported_testcases.py index bee69670e54f82bb60d84399e580bdb581721cc1..231b823648016e76fb467deaf1e6649761f5cc0c 100644 --- a/ci/remove_unsupported_testcases.py +++ b/ci/remove_unsupported_testcases.py @@ -32,7 +32,7 @@ from pathlib import Path import argparse # Enter tag of testcases to remove here WITHOUT the leading // -TESTCASES = [ +TESTCASES_MAIN = [ "OMASA 2Dir2TC 4ISM at br sw techs 13.2 to 512 kbps start 80 kbps, 48kHz in, 48kHz out, EXT out", "OSBA planar FOA 2ISM at 512 kbps, 48 kHz in, 48 kHz out, BINAURAL out", "4 ISM with extended metadata at 128 kbps, 48 kHz in, 48 kHz out, BINAURAL_ROOM_REVERB out, rendconf dir w id", @@ -66,10 +66,19 @@ TESTCASES = [ "MASA 1TC at 256kbps, 48kHz in, 48 kHz out, BINAURAL_ROOM_REVERB out custom configuration", "MASA 1TC at 256 kbps, 48kHz in, 48kHz out, BINAURAL_ROOM_REVERB out, HR custom configuration", "OMASA 2TC 4ISM at br sw techs 13.2 to 512 kbps start 80 kbps, 48kHz in, 48kHz out, EXT out", + "Multi-channel 7_1_4 bitrate switching, 48kHz in, 48kHz out, BINAURAL out, HR, JBM Prof 5", + "Multi-channel 7_1 bitrate switching, 48kHz in, 32kHz out, BINAURAL_ROOM_REVERB out, HR, JBM Prof 5", + "3 ISM with metadata bitrate switching from 48 kbps to 32 kbps, 48 kHz in, 32 kHz out, DTX, BINAURAL_ROOM_IR out, JBM Prof 5", + # object editing starts here "OMASA 2Dir2TC 4ISM at 80 kbps, 48kHz in, 48kHz out, BINAURAL out, default object editing, 1SEP-PARAM", "OMASA 2Dir2TC 4ISM at 256 kbps, 48kHz in, 48kHz out, BINAURAL out, object editing, JBM Prof 5, DISC", "OMASA 2Dir2TC 2ISM at 96 kbps, 48kHz in, 48kHz out, FOA out, object editing, JBM Prof 5, DISC", "OMASA 2Dir2TC 2ISM br sw techs 13.2 to 512 kbps start 48 kbps, 48kHz in, 48kHz out, BINAURAL out, object editing", + # the next four lines are basically duplicates of the last four, but without "Dir", because the ltv MASA cases do not have dedicated testvectors for numbers of directions + "OMASA 2TC 4ISM at 80 kbps, 48kHz in, 48kHz out, BINAURAL out, default object editing, 1SEP-PARAM", + "OMASA 2TC 4ISM at 256 kbps, 48kHz in, 48kHz out, BINAURAL out, object editing, JBM Prof 5, DISC", + "OMASA 2TC 2ISM at 96 kbps, 48kHz in, 48kHz out, FOA out, object editing, JBM Prof 5, DISC", + "OMASA 2TC 2ISM br sw techs 13.2 to 512 kbps start 48 kbps, 48kHz in, 48kHz out, BINAURAL out, object editing", "OSBA 3OA 4ISM at 256 kbps, 48kHz in, 48kHz out, BINAURAL out, object editing, DISC", "OSBA 2OA 3ISM at 128 kbps, 48kHz in, 48kHz out, FOA out, object editing, JBM Prof 5, DISC", "OSBA 2OA 3ISM at bitrate switching 13.2 to 512 kbps, 48kHz in, 48kHz out, BINAURAL out, object editing", @@ -79,12 +88,22 @@ TESTCASES = [ "3 ISM with metadata at 384 kbps, 48 kHz in, 48 kHz out, FOA out, object editing, JBM Prof 5, DISC", "4 ISM with metadata bitrate switching from 32 kbps to 48 kbps, 48 kHz in, 48 kHz out, BINAURAL_ROOM_IR out, object editing", "4 ISM with metadata bitrate switching from 48 kbps to 32 kbps, 48 kHz in, 48 kHz out, BINAURAL out, object editing, JBM Prof 5", - "OMASA 2TC 4ISM at 80 kbps, 48kHz in, 48kHz out, BINAURAL out, default object editing, 1SEP-PARAM", - "OMASA 2TC 4ISM at 256 kbps, 48kHz in, 48kHz out, BINAURAL out, object editing, JBM Prof 5, DISC", - "OMASA 2TC 2ISM at 96 kbps, 48kHz in, 48kHz out, FOA out, object editing, JBM Prof 5, DISC", - "OMASA 2TC 2ISM br sw techs 13.2 to 512 kbps start 48 kbps, 48kHz in, 48kHz out, BINAURAL out, object editing", ] - +TESTCASES_MAIN_PC = [ + "SBA at 128 kbps, 32kHZ in, 32kHz out, BINAURAL_ROOM_REVERB out, Config renderer, HR", + "SBA at 128 kbps, 32kHZ in, 16kHz out, BINAURAL_ROOM_REVERB out (Model from file), HR", + "Planar SBA at 128 kbps, 48kHZ in, 32kHz out, BINAURAL_ROOM_REVERB out (Model from file), Config renderer, HR", + "Multi-channel 7_1_4 at 160 kbps, 48kHz in, 48 kHz out, BINAURAL_ROOM_REVERB out default configuration", + "Multi-channel 5_1 at 80 kbps, 48kHz in, 32kHz out, BINAURAL_ROOM_REVERB out Config renderer, HR", + "OSBA planar 2OA 4ISM at 512 kbps, 48 kHz in, 48 kHz out, BINAURAL ROOM REVERB (Model from file) out", + "SBA 3OA 4ISM at 96 kbps, 48kHz in, 48kHz out, BINAURAL_ROOM_REVERB out custom configuration", + "Multi-channel 7_1_4 at 160 kbps, 48kHz in, 48kHz out, BINAURAL_ROOM_REVERB out Config recreation, HR", + "Multi-channel 5_1_2 at 64 kbps, 48kHz in, 48kHz out, BINAURAL_ROOM_REVERB out Config renderer, HR", + "OSBA 3OA 4ISM at 512 kbps, 48kHz in, 48kHz out, BINAURAL_ROOM_REVERB out", + "OSBA 3ISM 3OA at bitrate switching 13.2 to 512 kbps, 48kHz in, 32kHz out, BINAURAL ROOM REVERB out", + "Multi-channel 5_1 bitrate switching from 13.2 kbps to 512 kbps, 48kHz in, 16kHz out, BINAURAL_ROOM_REVERB out", + "OSBA 2OA 2ISM bitrate switching, 32kHz in, 48kHz out, BINAURAL_ROOM_REVERB out, JBM Prof 5", +] def remove_testcases(cfg: Path, testcases: list): @@ -112,7 +131,12 @@ def remove_testcases(cfg: Path, testcases: list): if __name__ == "__main__": parser = argparse.ArgumentParser() parser.add_argument("cfg_files", nargs="+", type=Path) + parser.add_argument("--use-main-pc-set", action="store_true") args = parser.parse_args() + testcases = TESTCASES_MAIN + if args.use_main_pc_set: + testcases.extend(TESTCASES_MAIN_PC) + for f in args.cfg_files: - remove_testcases(f, TESTCASES) + remove_testcases(f, testcases) diff --git a/scripts/basop_check_for_changes_in_testcases.py b/scripts/basop_check_for_changes_in_testcases.py index 5552fcba8b394267d85319148a281baad9ffaee9..3a61c76bb459f986c1c4e39066b53c4d73860e28 100644 --- a/scripts/basop_check_for_changes_in_testcases.py +++ b/scripts/basop_check_for_changes_in_testcases.py @@ -35,23 +35,35 @@ import argparse import sys import os import pathlib +import re +import xml.etree.ElementTree as ET # set positive threshold for "lower is better" metrics, negative for "higher is better" COLS_2_THRESHOLDS = { "MLD": float(os.environ.get("CI_REGRESSION_THRESH_MLD", 0.1)), - "MAXIMUM ABS DIFF": float(os.environ.get("CI_REGRESSION_THRESH_MAX_ABS_DIFF", 5)), + "MAX_ABS_DIFF": float(os.environ.get("CI_REGRESSION_THRESH_MAX_ABS_DIFF", 5)), "MIN_SSNR": float(os.environ.get("CI_REGRESSION_THRESH_SSNR", -1)), "MIN_ODG": float(os.environ.get("CI_REGRESSION_THRESH_ODG", -0.05)), } -OUTFILE_CRASHES = "changes_crashes.csv" -OUTFILE_SCORES = "changes_{}.csv" +OUTFILE_CRASHES_FIXED = "improvements_crashes.csv" +OUTFILE_CRASHES_ADDED = "regressions_crashes.csv" +OUTFILE_SCORES_IMPROVEMENTS = "improvements_{}.csv" +OUTFILE_SCORES_REGRESSION = "regressions_{}.csv" + +PATTERN_ENC = r"... encoder command:\s*(.*)\s*... encoder stdout:" +PATTERN_DEC = r"... decoder command:\s*(.*)\s*... decoder stdout:" +PATTERN_EID = r"eid-xor command:\s*(.*)\s*" +PATTERN_NETSIM = r"netsim command:\s*(.*)\s*" + +PATTERNS = [PATTERN_ENC, PATTERN_DEC, PATTERN_EID, PATTERN_NETSIM] def main(args): - df_curr = pd.read_csv(args.csv_current, sep=";") - df_prev = pd.read_csv(args.csv_previous, sep=";") + xml_report = args.xml_report + df_curr = pd.read_csv(args.csv_current) + df_prev = pd.read_csv(args.csv_previous) df_merged = pd.merge(df_curr, df_prev, on="testcase", suffixes=["-curr", "-prev"]) # remove leading path from testcase names for better readability @@ -64,8 +76,8 @@ def main(args): regressions_found = False # check for newly introduced crashes - col_curr = "Result-curr" - col_prev = "Result-prev" + col_curr = "result-curr" + col_prev = "result-prev" mask_crash_introduced = (df_merged[col_curr] == "ERROR") & ( df_merged[col_prev] != "ERROR" ) @@ -77,7 +89,6 @@ def main(args): df_crashes_introduced = df_merged[mask_crash_introduced][display_cols].reset_index( drop=True ) - df_crashes_introduced.to_csv(OUTFILE_CRASHES, sep=";") if sum(mask_crash_introduced) > 0: regressions_found = True @@ -85,15 +96,32 @@ def main(args): print(df_crashes_introduced) print() + if xml_report is not None: + cmdlines_crashes_introduced = get_command_lines_for_testcases( + df_crashes_introduced["testcase"], xml_report, args.inject_cwd + ) + df_crashes_introduced = pd.merge( + df_crashes_introduced, cmdlines_crashes_introduced, on="testcase" + ) + df_crashes_introduced.to_csv(OUTFILE_CRASHES_ADDED, sep=";") + if args.show_improvements and sum(mask_crash_fixed) > 0: df_crashes_fixed = df_merged[mask_crash_fixed][display_cols].reset_index( drop=True ) - df_crashes_fixed.to_csv(OUTFILE_CRASHES, mode="a", sep=";") print("---------------Testcases that fixed crashes---------------") print(df_crashes_fixed) print() + if xml_report is not None: + cmdlines_crashes_fixed = get_command_lines_for_testcases( + df_crashes_fixed["testcase"], xml_report, args.inject_cwd + ) + df_crashes_fixed = pd.merge( + df_crashes_fixed, cmdlines_crashes_fixed, on="testcase" + ) + df_crashes_fixed.to_csv(OUTFILE_CRASHES_FIXED, sep=";") + # remove columns with ERRORs in any of the csv files before comparing the numerical columns mask_no_errors = (df_merged[col_curr] != "ERROR") & (df_merged[col_prev] != "ERROR") df_merged = df_merged[mask_no_errors].reset_index(drop=True) @@ -113,9 +141,8 @@ def main(args): mask_better = (df_merged[col_diff] * fac) < -thresh display_cols = ["testcase", col_curr, col_prev, col_diff] - outfile = OUTFILE_SCORES.format(col.replace(" ", "_")) + outfile_regressions = OUTFILE_SCORES_REGRESSION.format(col.replace(" ", "_")) df_worse = df_merged[mask_worse][display_cols].reset_index(drop=True) - df_worse.to_csv(outfile, sep=";") if sum(mask_worse) > 0: regressions_found = True print( @@ -124,18 +151,110 @@ def main(args): print(df_worse) print() + if xml_report is not None: + cmdlines_worse = get_command_lines_for_testcases( + df_worse["testcase"], xml_report, args.inject_cwd + ) + df_worse = pd.merge(df_worse, cmdlines_worse, on="testcase") + df_worse.to_csv(outfile_regressions, sep=";") + if args.show_improvements and sum(mask_better) > 0: df_better = df_merged[mask_better][display_cols].reset_index(drop=True) - df_better.to_csv(outfile, mode="a", sep=";") print( f"---------------Testcases that got better wrt to {col}---------------" ) print(df_better) print() + if xml_report is not None: + cmdlines_better = get_command_lines_for_testcases( + df_better["testcase"], xml_report, args.inject_cwd + ) + df_better = pd.merge(df_better, cmdlines_better, on="testcase") + outfile_improvements = OUTFILE_SCORES_IMPROVEMENTS.format( + col.replace(" ", "_") + ) + df_better.to_csv(outfile_improvements, sep=";") + return int(regressions_found) +def get_command_lines_for_testcases( + testcases: pd.Series, xml_report: pathlib.Path, cwd: pathlib.Path +) -> pd.DataFrame: + testcase_elems = [ + e + for _, e in ET.iterparse(xml_report) + if e.tag == "testcase" and e.attrib["name"] in testcases.values + ] + + cmdlines = { + "testcase": [], + "enc_cmd": [], + "dec_cmd": [], + "eid-xor_cmd": [], + "netsim_cmd": [], + } + for elem in testcase_elems: + testcase_name = elem.attrib["name"] + enc_cmd = "" + dec_cmd = "" + eid_cmd = "" + netsim_cmd = "" + if (system_out := elem.find("system-out")) is not None: + ( + enc_cmd, + dec_cmd, + eid_cmd, + netsim_cmd, + ) = extract_cmdlines(system_out.text, cwd) + + cmdlines["testcase"].append(testcase_name) + cmdlines["enc_cmd"].append(enc_cmd) + cmdlines["dec_cmd"].append(dec_cmd) + cmdlines["eid-xor_cmd"].append(eid_cmd) + cmdlines["netsim_cmd"].append(netsim_cmd) + + return pd.DataFrame(cmdlines) + + +def extract_cmdlines(text: str, cwd: pathlib.Path) -> list[str]: + cmdlines = [] + for p in PATTERNS: + m = re.search(p, text) + if m is not None: + cmdline = postprocess_cmdline(m.group(1), cwd) + cmdlines.append(cmdline) + else: + cmdlines.append("") + + return cmdlines + + +def postprocess_cmdline(cmdline: str, cwd: pathlib.Path) -> str: + cmdline_split = cmdline.split() + cmdline_proc = [] + + # change absolute paths into relative ones + # remove the "quite" flag + # for output and bitstream files only keep the filename + for elem in cmdline_split: + if elem == "-q": + continue + elif (elem_as_path := pathlib.Path(elem)).is_absolute(): + if elem_as_path.suffix == ".192" or ( + elem_as_path.suffix == ".wav" + and cmdline_split.index(elem) == len(cmdline_split) - 1 + ): + cmdline_proc.append(elem_as_path.name) + else: + cmdline_proc.append(str(elem_as_path.relative_to(cwd))) + else: + cmdline_proc.append(elem) + + return " ".join(cmdline_proc) + + if __name__ == "__main__": parser = argparse.ArgumentParser() parser.add_argument("csv_current") @@ -147,6 +266,17 @@ if __name__ == "__main__": default=COLS_2_THRESHOLDS.keys(), ) parser.add_argument("--show_improvements", action="store_true") + parser.add_argument( + "--xml_report", + help="XMLxml_report report file from pytest run. Pass to add command lines to the output files.", + default=None, + ) + parser.add_argument( + "--inject_cwd", + help="Use this as cwd when pruning the long paths in the command lines. Debug option for testing.", + default=pathlib.Path(os.getcwd()).absolute(), + type=pathlib.Path, + ) args = parser.parse_args() sys.exit(main(args)) diff --git a/scripts/basop_create_ignorelist_for_ubsan.py b/scripts/basop_create_ignorelist_for_ubsan.py new file mode 100644 index 0000000000000000000000000000000000000000..d13c0d5f5b7054ed73e37fa1aa8339153bc77590 --- /dev/null +++ b/scripts/basop_create_ignorelist_for_ubsan.py @@ -0,0 +1,14 @@ +import pathlib + +here = pathlib.Path(".") + +all_files = here.glob("lib_*/*.[ch]") +ivas_files = here.glob("lib_*/ivas*.[ch]") +non_ivas_files = sorted(set(all_files) - set(ivas_files)) +basop_files = here.glob("lib_*/basop*.[ch]") + +ignorefiles = sorted(set(non_ivas_files) - set(basop_files)) + +with open("ubsan_ignorelist.txt", "w") as f: + for cfile in ignorefiles: + print(f"src:{cfile}", file=f) diff --git a/scripts/config/self_test_ltv_basop_encoder.prm b/scripts/config/self_test_ltv_basop_encoder.prm index 66f96903d2c333b6ad72c51f4a74c414c3db7bc6..ca20519cb207d44bc5604e7f8f37dd67f6e90f2b 100644 --- a/scripts/config/self_test_ltv_basop_encoder.prm +++ b/scripts/config/self_test_ltv_basop_encoder.prm @@ -276,8 +276,8 @@ ../IVAS_dec HOA3 48 bit testv/ltv48_HOA3.wav_sw_48-48_HOA3.tst // SBA 3OA bitrate switching from 13.2 kbps to 128 kbps, 48kHz in, 48kHz out, DTX on, HOA3 out -../IVAS_cod -dtx -sba 3 ../scripts/switchPaths/sw_13k2_to_128k_10fr.bin 48 testv/ltv48_HOA3.wav bit -../IVAS_dec HOA3 48 bit testv/ltv48_HOA3.wav_sw_48-48_DTX_HOA3.tst +//../IVAS_cod -dtx -sba 3 ../scripts/switchPaths/sw_13k2_to_128k_10fr.bin 48 testv/ltv48_HOA3.wav bit +//../IVAS_dec HOA3 48 bit testv/ltv48_HOA3.wav_sw_48-48_DTX_HOA3.tst // SBA 3OA bitrate switching from 13.2 kbps to 128 kbps, 32kHz in, 32kHz out, DTX on, HOA3 out ../IVAS_cod -dtx -sba 3 ../scripts/switchPaths/sw_13k2_to_128k_10fr.bin 32 testv/ltv32_HOA3.wav bit @@ -320,35 +320,35 @@ ../IVAS_dec EXT 48 bit testv/ltvOSBA_4ISM_p3OA48c.wav_EXT_512000_48-48.tst // OSBA planar 2OA 4ISM at 512 kbps, 48 kHz in, 48 kHz out, EXT out -../IVAS_cod -ism_sba 4 -2 testv/ltv48_OSBA_1ISM_HOA3_ISM1.csv testv/ltv48_OSBA_2ISM_HOA3_ISM2.csv testv/ltv48_OSBA_3ISM_HOA3_ISM3.csv testv/ltv48_OSBA_4ISM_HOA3_ISM4.csv 512000 48 testv/ltv48_OSBA_4ISM_HOA2.wav bit +../IVAS_cod -ism_sba 4 -2 testv/ltv48_OSBA_1ISM_HOA2_ISM1.csv testv/ltv48_OSBA_2ISM_HOA2_ISM2.csv testv/ltv48_OSBA_3ISM_HOA2_ISM3.csv testv/ltv48_OSBA_4ISM_HOA2_ISM4.csv 512000 48 testv/ltv48_OSBA_4ISM_HOA2.wav bit ../IVAS_dec EXT 48 bit testv/ltvOSBA_4ISM_p3OA48c.wav_EXT_512000_48-48.tst // OSBA FOA 4ISM at 512 kbps, 48kHz in, 48kHz out, EXT out -../IVAS_cod -ism_sba 4 1 testv/ltv48_OSBA_1ISM_HOA3_ISM1.csv testv/ltv48_OSBA_2ISM_HOA3_ISM2.csv testv/ltv48_OSBA_3ISM_HOA3_ISM3.csv testv/ltv48_OSBA_4ISM_HOA3_ISM4.csv 512000 48 testv/ltv48_OSBA_4ISM_FOA.wav bit +../IVAS_cod -ism_sba 4 1 testv/ltv48_OSBA_1ISM_FOA_ISM1.csv testv/ltv48_OSBA_2ISM_FOA_ISM2.csv testv/ltv48_OSBA_3ISM_FOA_ISM3.csv testv/ltv48_OSBA_4ISM_FOA_ISM4.csv 512000 48 testv/ltv48_OSBA_4ISM_FOA.wav bit ../IVAS_dec EXT 48 bit testv/ltv48_OSBA_4ISM_FOA.wav_EXT_512000_48-48.tst // OSBA FOA 4ISM at 512 kbps, 32kHz in, 48kHz out, EXT out -../IVAS_cod -ism_sba 4 1 testv/ltv48_OSBA_1ISM_HOA3_ISM1.csv testv/ltv48_OSBA_2ISM_HOA3_ISM2.csv testv/ltv48_OSBA_3ISM_HOA3_ISM3.csv testv/ltv48_OSBA_4ISM_HOA3_ISM4.csv 512000 32 testv/ltv32_OSBA_4ISM_FOA.wav bit +../IVAS_cod -ism_sba 4 1 testv/ltv48_OSBA_1ISM_FOA_ISM1.csv testv/ltv48_OSBA_2ISM_FOA_ISM2.csv testv/ltv48_OSBA_3ISM_FOA_ISM3.csv testv/ltv48_OSBA_4ISM_FOA_ISM4.csv 512000 32 testv/ltv32_OSBA_4ISM_FOA.wav bit ../IVAS_dec EXT 48 bit testv/ltv32_OSBA_4ISM_FOA.wav_EXT_512000_32-48.tst // OSBA FOA 4ISM at 384 kbps, 32kHz in, 32kHz out, EXT out -../IVAS_cod -ism_sba 4 1 testv/ltv48_OSBA_1ISM_HOA3_ISM1.csv testv/ltv48_OSBA_2ISM_HOA3_ISM2.csv testv/ltv48_OSBA_3ISM_HOA3_ISM3.csv testv/ltv48_OSBA_4ISM_HOA3_ISM4.csv 384000 32 testv/ltv32_OSBA_4ISM_FOA.wav bit +../IVAS_cod -ism_sba 4 1 testv/ltv48_OSBA_1ISM_FOA_ISM1.csv testv/ltv48_OSBA_2ISM_FOA_ISM2.csv testv/ltv48_OSBA_3ISM_FOA_ISM3.csv testv/ltv48_OSBA_4ISM_FOA_ISM4.csv 384000 32 testv/ltv32_OSBA_4ISM_FOA.wav bit ../IVAS_dec EXT 32 bit testv/ltv32_OSBA_4ISM_FOA.wav_EXT_384000_32-32.tst // OSBA FOA 4ISM at 256 kbps, 48kHz in, 48kHz out, EXT out -../IVAS_cod -ism_sba 4 1 testv/ltv48_OSBA_1ISM_HOA3_ISM1.csv testv/ltv48_OSBA_2ISM_HOA3_ISM2.csv testv/ltv48_OSBA_3ISM_HOA3_ISM3.csv testv/ltv48_OSBA_4ISM_HOA3_ISM4.csv 256000 48 testv/ltv48_OSBA_4ISM_FOA.wav bit +../IVAS_cod -ism_sba 4 1 testv/ltv48_OSBA_1ISM_FOA_ISM1.csv testv/ltv48_OSBA_2ISM_FOA_ISM2.csv testv/ltv48_OSBA_3ISM_FOA_ISM3.csv testv/ltv48_OSBA_4ISM_FOA_ISM4.csv 256000 48 testv/ltv48_OSBA_4ISM_FOA.wav bit ../IVAS_dec EXT 48 bit testv/ltv48_OSBA_4ISM_FOA.wav_EXT_256000_48-48.tst // OSBA FOA 3ISM at 128 kbps, 48kHz in, 48kHz out, EXT out -../IVAS_cod -ism_sba 3 1 testv/ltv48_OSBA_1ISM_HOA3_ISM1.csv testv/ltv48_OSBA_2ISM_HOA3_ISM2.csv testv/ltv48_OSBA_3ISM_HOA3_ISM3.csv 128000 48 testv/ltv48_OSBA_3ISM_FOA.wav bit +../IVAS_cod -ism_sba 3 1 testv/ltv48_OSBA_1ISM_FOA_ISM1.csv testv/ltv48_OSBA_2ISM_FOA_ISM2.csv testv/ltv48_OSBA_3ISM_FOA_ISM3.csv 128000 48 testv/ltv48_OSBA_3ISM_FOA.wav bit ../IVAS_dec EXT 48 bit testv/ltv48_OSBA_3ISM_FOA.wav_EXT_128000_48-48.tst // OSBA FOA 1ISM at 48 kbps, 16kHz in, 16kHz out, EXT out -../IVAS_cod -ism_sba 1 1 testv/ltv48_OSBA_1ISM_HOA3_ISM1.csv 32000 16 testv/ltv16_OSBA_1ISM_FOA.wav bit +../IVAS_cod -ism_sba 1 1 testv/ltv48_OSBA_1ISM_FOA_ISM1.csv 32000 16 testv/ltv16_OSBA_1ISM_FOA.wav bit ../IVAS_dec EXT 16 bit testv/ltv16_OSBA_1ISM_FOA.wav_EXT_32000_16-16.tst // OSBA FOA 1ISM at 32 kbps, 48kHz in, 48kHz out, EXT out -../IVAS_cod -ism_sba 1 1 testv/ltv48_OSBA_1ISM_HOA3_ISM1.csv 32000 48 testv/ltv48_OSBA_1ISM_FOA.wav bit +../IVAS_cod -ism_sba 1 1 testv/ltv48_OSBA_1ISM_FOA_ISM1.csv 32000 48 testv/ltv48_OSBA_1ISM_FOA.wav bit ../IVAS_dec EXT 48 bit testv/ltv48_OSBA_1ISM_FOA.wav_EXT_32000_48-48.tst // OSBA 3OA 4ISM bitrate switching 16.4 to 512, 48kHz in, 48kHz out, EXT out @@ -405,32 +405,32 @@ ../IVAS_dec EXT 32 bit testv/ltv48_OSBA_3ISM_HOA3.wav_EXT_sw_48-32.tst // OSBA 3ISM 2OA at bitrate switching 13.2 to 512 kbps, 48kHz in, 32kHz out, EXT out -../IVAS_cod -ism_sba 3 2 testv/ltv48_OSBA_1ISM_HOA3_ISM1.csv testv/ltv48_OSBA_2ISM_HOA3_ISM2.csv testv/ltv48_OSBA_3ISM_HOA3_ISM3.csv ../scripts/switchPaths/sw_13k2_512k.bin 48 testv/ltv48_OSBA_3ISM_HOA2.wav bit +../IVAS_cod -ism_sba 3 2 testv/ltv48_OSBA_1ISM_HOA2_ISM1.csv testv/ltv48_OSBA_2ISM_HOA2_ISM2.csv testv/ltv48_OSBA_3ISM_HOA2_ISM3.csv ../scripts/switchPaths/sw_13k2_512k.bin 48 testv/ltv48_OSBA_3ISM_HOA2.wav bit ../IVAS_dec EXT 32 bit testv/ltv48_OSBA_3ISM_HOA2.wav_EXT_sw_48-32.tst // OSBA 2OA 4ISM at 384 kbps, 48kHz in, 48kHz out, EXT out -../IVAS_cod -ism_sba 4 2 testv/ltv48_OSBA_1ISM_HOA3_ISM1.csv testv/ltv48_OSBA_2ISM_HOA3_ISM2.csv testv/ltv48_OSBA_3ISM_HOA3_ISM3.csv testv/ltv48_OSBA_4ISM_HOA3_ISM4.csv 384000 48 testv/ltv48_OSBA_4ISM_HOA2.wav bit +../IVAS_cod -ism_sba 4 2 testv/ltv48_OSBA_1ISM_HOA2_ISM1.csv testv/ltv48_OSBA_2ISM_HOA2_ISM2.csv testv/ltv48_OSBA_3ISM_HOA2_ISM3.csv testv/ltv48_OSBA_4ISM_HOA2_ISM4.csv 384000 48 testv/ltv48_OSBA_4ISM_HOA2.wav bit ../IVAS_dec EXT 48 bit testv/ltv48_OSBA_4ISM_HOA2.wav_EXT_384000_48-48.tst // OSBA 2OA 3ISM at 96 kbps, 48kHz in, 48kHz out, EXT out -../IVAS_cod -ism_sba 3 2 testv/ltv48_OSBA_1ISM_HOA3_ISM1.csv testv/ltv48_OSBA_2ISM_HOA3_ISM2.csv testv/ltv48_OSBA_3ISM_HOA3_ISM3.csv 96000 48 testv/ltv48_OSBA_3ISM_HOA2.wav bit +../IVAS_cod -ism_sba 3 2 testv/ltv48_OSBA_1ISM_HOA2_ISM1.csv testv/ltv48_OSBA_2ISM_HOA2_ISM2.csv testv/ltv48_OSBA_3ISM_HOA2_ISM3.csv 96000 48 testv/ltv48_OSBA_3ISM_HOA2.wav bit ../IVAS_dec EXT 48 bit testv/ltv48_OSBA_3ISM_HOA2.wav_EXT_96000_48-48.tst // OSBA 2OA 3ISM at 384 kbps, 16kHz in, 16kHz out, EXT out -../IVAS_cod -ism_sba 3 2 testv/ltv48_OSBA_1ISM_HOA3_ISM1.csv testv/ltv48_OSBA_2ISM_HOA3_ISM2.csv testv/ltv48_OSBA_3ISM_HOA3_ISM3.csv 384000 16 testv/ltv16_OSBA_3ISM_HOA2.wav bit +../IVAS_cod -ism_sba 3 2 testv/ltv48_OSBA_1ISM_HOA2_ISM1.csv testv/ltv48_OSBA_2ISM_HOA2_ISM2.csv testv/ltv48_OSBA_3ISM_HOA2_ISM3.csv 384000 16 testv/ltv16_OSBA_3ISM_HOA2.wav bit ../IVAS_dec EXT 16 bit testv/ltv16_OSBA_3ISM_HOA2.wav_EXT56000_16-16.tst // OSBA 2OA 2ISM at 64 kbps, 32kHz in, 16kHz out, EXT out -../IVAS_cod -ism_sba 2 2 testv/ltv48_OSBA_1ISM_HOA3_ISM1.csv testv/ltv48_OSBA_2ISM_HOA3_ISM2.csv 48000 32 testv/ltv32_OSBA_2ISM_HOA2.wav bit +../IVAS_cod -ism_sba 2 2 testv/ltv48_OSBA_1ISM_HOA2_ISM1.csv testv/ltv48_OSBA_2ISM_HOA2_ISM2.csv 48000 32 testv/ltv32_OSBA_2ISM_HOA2.wav bit ../IVAS_dec EXT 16 bit testv/ltv32_OSBA_2ISM_HOA2.wav_EXT_64000_32-16.tst // OSBA 2OA 2ISM at 48 kbps, 48kHz in, 48kHz out, EXT out -../IVAS_cod -ism_sba 2 2 testv/ltv48_OSBA_1ISM_HOA3_ISM1.csv testv/ltv48_OSBA_2ISM_HOA3_ISM2.csv 48000 48 testv/ltv48_OSBA_2ISM_HOA2.wav bit +../IVAS_cod -ism_sba 2 2 testv/ltv48_OSBA_1ISM_HOA2_ISM1.csv testv/ltv48_OSBA_2ISM_HOA2_ISM2.csv 48000 48 testv/ltv48_OSBA_2ISM_HOA2.wav bit ../IVAS_dec EXT 48 bit testv/ltv48_OSBA_2ISM_HOA2.wav_EXT_48000_48-48.tst // OSBA 2OA 1ISM at 24.4 kbps, 48kHz in, 48kHz out, EXT out -../IVAS_cod -ism_sba 1 2 testv/ltv48_OSBA_1ISM_HOA3_ISM1.csv 24400 48 testv/ltv48_OSBA_1ISM_EXT.wav bit -../IVAS_dec EXT 48 bit testv/ltv48_OSBA_1ISM_EXT.wav_EXT_24400_48-48.tst +../IVAS_cod -ism_sba 1 2 testv/ltv48_OSBA_1ISM_HOA2_ISM1.csv 24400 48 testv/ltv48_OSBA_1ISM_HOA2.wav bit +../IVAS_dec EXT 48 bit testv/ltv48_OSBA_1ISM_HOA2.wav_EXT_24400_48-48.tst // OMASA 2TC 4ISM at br sw techs 13.2 to 512 kbps start 384 kbps, 48kHz in, 48kHz out, BINAURAL out ../IVAS_cod -ism_masa 4 2 testv/ltv48_OMASA_1ISM_2TC_ISM1.csv testv/ltv48_OMASA_2ISM_2TC_ISM2.csv testv/ltv48_OMASA_3ISM_2TC_ISM3.csv testv/ltv48_OMASA_4ISM_2TC_ISM4.csv testv/ltv48_OMASA_4ISM_2TC.met ../scripts/switchPaths/sw_13k2_512k_2fr_start_384k_omasatechs_4ism.bin 48 testv/ltv48_OMASA_4ISM_2TC.wav bit diff --git a/scripts/smoketest-basop-filter.py b/scripts/smoketest-basop-filter.py new file mode 100644 index 0000000000000000000000000000000000000000..faa170f7e0e511396fd2ae5b9b919916d303933e --- /dev/null +++ b/scripts/smoketest-basop-filter.py @@ -0,0 +1,53 @@ +#! /usr/bin/env python3 + +import argparse +import json +import logging +import sys + + +logging.basicConfig(level=logging.INFO, format='%(levelname)s: %(message)s') +logger = logging.getLogger() + + +def main(): + parser = argparse.ArgumentParser() + + parser.add_argument("input", help="input JSON", type=argparse.FileType('r')) + parser.add_argument("output", help="output JSON", type=argparse.FileType('w'), nargs="?") + parser.add_argument("--inline", "-i", help="patch input inline", action="store_true") + + args = parser.parse_args() + + if not args.inline and not args.output: + logging.error("You must specify an output file or use --inline") + return 1 + + modes = json.load(args.input) + + for mode in ("MC", "stereo", "OMASA"): + for encoder_config_name in modes[mode].keys(): + try: + del modes[mode][encoder_config_name]["dec"]["EXT"] + except KeyError: + logger.warning(f"No EXT dec in {mode} -> {encoder_config_name}, skipping") + continue + else: + logger.info(f"Found EXT dec in {mode} -> {encoder_config_name}, removing") + + for encoder_config_name in tuple(modes["OSBA"].keys()): + if not "FOA" in encoder_config_name: + continue + + logger.info(f"Removing OSBA FOA encoder config {encoder_config_name}") + del modes["OSBA"][encoder_config_name] + + if args.inline: + with open(args.input.name, "w") as f: + json.dump(modes, f, indent=4) + else: + json.dump(modes, args.output, indent=4) + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/ubsan_basop.supp b/scripts/ubsan_basop.supp new file mode 100644 index 0000000000000000000000000000000000000000..bfd00f357312259871d5b67a802699393a92a559 --- /dev/null +++ b/scripts/ubsan_basop.supp @@ -0,0 +1,32 @@ +# From the scheduled sanitizer tests with self_test.py +implicit-integer-sign-change:ACcontextMapping_fx.c +implicit-integer-sign-change:bitstream_fx.c +implicit-integer-sign-change:enh64.c +implicit-signed-integer-truncation:enh1632.c +implicit-signed-integer-truncation:enhUL32.c +shift-base:enh64.c +shift-exponent:basop32.c +shift-exponent:enh64.c +bounds:ifft_rel.c +bounds:log2.c +bounds:tcx_ltp_fx.c +bounds:trans_direct_fx.c +bounds:trans_inv_fx.c +signed-integer-overflow:enh64.c +implicit-integer-sign-change:ACcontextMapping_enc_fx.c +implicit-integer-sign-change:bass_psfilter_enc_fx.c +implicit-integer-sign-change:cod4t64_fx.c +implicit-integer-sign-change:core_enc_ol_fx.c +implicit-integer-sign-change:enc_acelp_fx.c +implicit-integer-sign-change:enc_prm_fx.c +implicit-integer-sign-change:hq_classifier_enc_fx.c +implicit-integer-sign-change:inov_enc_fx.c +implicit-integer-sign-change:lead_indexing_fx.c +implicit-integer-sign-change:range_enc_fx.c +implicit-integer-sign-change:tcx_utils_enc_fx.c +bounds:enc_acelp_fx.c +bounds:enc_gain_fx.c +bounds:pit_enc_fx.c +bounds:pitch_ol2_fx.c +bounds:tcx_ltp_enc_fx.c +bounds:dec_acelp_fx.c diff --git a/tests/split_rendering/test_split_rendering.py b/tests/split_rendering/test_split_rendering.py index 4f1d0039ece6b6695d0d062ae8afe605ef626c2a..a881a7ad23f65902c620f7de9a1d39e6f9676832 100644 --- a/tests/split_rendering/test_split_rendering.py +++ b/tests/split_rendering/test_split_rendering.py @@ -44,12 +44,21 @@ from tests.split_rendering.utils import * @pytest.mark.parametrize("bitrate", IVAS_BITRATES_AMBI) @pytest.mark.parametrize("in_fmt", INPUT_FORMATS_AMBI_SPLIT_REND) def test_ambisonics_full_chain_split( + record_property, + props_to_record, + get_mld, + get_mld_lim, + get_ssnr, + get_odg, + get_odg_bin, test_info, in_fmt, bitrate, render_config, trajectory ): post_trajectory = HR_TRAJECTORY_DIR.joinpath(f"{trajectory}.csv") pre_trajectory = post_trajectory.with_stem(f"{post_trajectory.stem}_delayed") run_full_chain_split_rendering( + record_property, + props_to_record, test_info, in_fmt=in_fmt, bitrate=bitrate, @@ -57,6 +66,11 @@ def test_ambisonics_full_chain_split( binary_suffix=EXE_SUFFIX, pre_trajectory=pre_trajectory, post_trajectory=post_trajectory, + get_mld=get_mld, + mld_lim=get_mld_lim, + get_ssnr=get_ssnr, + get_odg=get_odg, + get_odg_bin=get_odg_bin, ) @@ -64,16 +78,31 @@ def test_ambisonics_full_chain_split( @pytest.mark.parametrize("trajectory", SPLIT_REND_HR_TRAJECTORIES_TO_TEST) @pytest.mark.parametrize("render_config", RENDERER_CONFIGS_TO_TEST_AMBI) @pytest.mark.parametrize("in_fmt", INPUT_FORMATS_AMBI_SPLIT_REND) -def test_ambisonics_external_split(test_info, in_fmt, render_config, trajectory): +def test_ambisonics_external_split( + record_property, + props_to_record, + get_mld, + get_mld_lim, + get_ssnr, + get_odg, + get_odg_bin, + test_info, in_fmt, render_config, trajectory): post_trajectory = HR_TRAJECTORY_DIR.joinpath(f"{trajectory}.csv") pre_trajectory = post_trajectory.with_stem(f"{post_trajectory.stem}_delayed") run_external_split_rendering( + record_property, + props_to_record, test_info, in_fmt=in_fmt, render_config=RENDER_CFG_DIR.joinpath(f"{render_config}.txt"), pre_trajectory=pre_trajectory, post_trajectory=post_trajectory, + get_mld=get_mld, + mld_lim=get_mld_lim, + get_ssnr=get_ssnr, + get_odg=get_odg, + get_odg_bin=get_odg_bin, ) @@ -86,12 +115,21 @@ def test_ambisonics_external_split(test_info, in_fmt, render_config, trajectory) @pytest.mark.parametrize("bitrate", IVAS_BITRATES_MC) @pytest.mark.parametrize("in_fmt", INPUT_FORMATS_MC_SPLIT_REND) def test_multichannel_full_chain_split( + record_property, + props_to_record, + get_mld, + get_mld_lim, + get_ssnr, + get_odg, + get_odg_bin, test_info, in_fmt, bitrate, render_config, trajectory ): post_trajectory = HR_TRAJECTORY_DIR.joinpath(f"{trajectory}.csv") pre_trajectory = post_trajectory.with_stem(f"{post_trajectory.stem}_delayed") run_full_chain_split_rendering( + record_property, + props_to_record, test_info, in_fmt=in_fmt, bitrate=bitrate, @@ -99,6 +137,11 @@ def test_multichannel_full_chain_split( binary_suffix=EXE_SUFFIX, pre_trajectory=pre_trajectory, post_trajectory=post_trajectory, + get_mld=get_mld, + mld_lim=get_mld_lim, + get_ssnr=get_ssnr, + get_odg=get_odg, + get_odg_bin=get_odg_bin, ) @@ -106,16 +149,31 @@ def test_multichannel_full_chain_split( @pytest.mark.parametrize("trajectory", SPLIT_REND_HR_TRAJECTORIES_TO_TEST) @pytest.mark.parametrize("render_config", RENDERER_CONFIGS_TO_TEST_MC) @pytest.mark.parametrize("in_fmt", INPUT_FORMATS_MC_SPLIT_REND) -def test_multichannel_external_split(test_info, in_fmt, render_config, trajectory): +def test_multichannel_external_split( + record_property, + props_to_record, + get_mld, + get_mld_lim, + get_ssnr, + get_odg, + get_odg_bin, +test_info, in_fmt, render_config, trajectory): post_trajectory = HR_TRAJECTORY_DIR.joinpath(f"{trajectory}.csv") pre_trajectory = post_trajectory.with_stem(f"{post_trajectory.stem}_delayed") run_external_split_rendering( + record_property, + props_to_record, test_info, in_fmt=in_fmt, render_config=RENDER_CFG_DIR.joinpath(f"{render_config}.txt"), pre_trajectory=pre_trajectory, post_trajectory=post_trajectory, + get_mld=get_mld, + mld_lim=get_mld_lim, + get_ssnr=get_ssnr, + get_odg=get_odg, + get_odg_bin=get_odg_bin, ) @@ -127,11 +185,21 @@ def test_multichannel_external_split(test_info, in_fmt, render_config, trajector @pytest.mark.parametrize("render_config", RENDERER_CONFIGS_TO_TEST_ISM) @pytest.mark.parametrize("bitrate", IVAS_BITRATES_ISM) @pytest.mark.parametrize("in_fmt", INPUT_FORMATS_ISM_SPLIT_REND) -def test_ism_full_chain_split(test_info, in_fmt, bitrate, render_config, trajectory): +def test_ism_full_chain_split( + record_property, + props_to_record, + get_mld, + get_mld_lim, + get_ssnr, + get_odg, + get_odg_bin, +test_info, in_fmt, bitrate, render_config, trajectory): post_trajectory = HR_TRAJECTORY_DIR.joinpath(f"{trajectory}.csv") pre_trajectory = post_trajectory.with_stem(f"{post_trajectory.stem}_delayed") run_full_chain_split_rendering( + record_property, + props_to_record, test_info, in_fmt=in_fmt, bitrate=bitrate, @@ -139,6 +207,11 @@ def test_ism_full_chain_split(test_info, in_fmt, bitrate, render_config, traject binary_suffix=EXE_SUFFIX, pre_trajectory=pre_trajectory, post_trajectory=post_trajectory, + get_mld=get_mld, + mld_lim=get_mld_lim, + get_ssnr=get_ssnr, + get_odg=get_odg, + get_odg_bin=get_odg_bin, ) @@ -146,16 +219,31 @@ def test_ism_full_chain_split(test_info, in_fmt, bitrate, render_config, traject @pytest.mark.parametrize("trajectory", SPLIT_REND_HR_TRAJECTORIES_TO_TEST) @pytest.mark.parametrize("render_config", RENDERER_CONFIGS_TO_TEST_ISM) @pytest.mark.parametrize("in_fmt", INPUT_FORMATS_ISM_SPLIT_REND) -def test_ism_external_split(test_info, in_fmt, render_config, trajectory): +def test_ism_external_split( + record_property, + props_to_record, + get_mld, + get_mld_lim, + get_ssnr, + get_odg, + get_odg_bin, +test_info, in_fmt, render_config, trajectory): post_trajectory = HR_TRAJECTORY_DIR.joinpath(f"{trajectory}.csv") pre_trajectory = post_trajectory.with_stem(f"{post_trajectory.stem}_delayed") run_external_split_rendering( + record_property, + props_to_record, test_info, in_fmt=in_fmt, render_config=RENDER_CFG_DIR.joinpath(f"{render_config}.txt"), pre_trajectory=pre_trajectory, post_trajectory=post_trajectory, + get_mld=get_mld, + mld_lim=get_mld_lim, + get_ssnr=get_ssnr, + get_odg=get_odg, + get_odg_bin=get_odg_bin, ) @@ -167,11 +255,21 @@ def test_ism_external_split(test_info, in_fmt, render_config, trajectory): @pytest.mark.parametrize("render_config", RENDERER_CONFIGS_TO_TEST_MASA) @pytest.mark.parametrize("bitrate", IVAS_BITRATES_MASA) @pytest.mark.parametrize("in_fmt", INPUT_FORMATS_MASA_SPLIT_REND) -def test_masa_full_chain_split(test_info, in_fmt, bitrate, render_config, trajectory): +def test_masa_full_chain_split( + record_property, + props_to_record, + get_mld, + get_mld_lim, + get_ssnr, + get_odg, + get_odg_bin, +test_info, in_fmt, bitrate, render_config, trajectory): post_trajectory = HR_TRAJECTORY_DIR.joinpath(f"{trajectory}.csv") pre_trajectory = post_trajectory.with_stem(f"{post_trajectory.stem}_delayed") run_full_chain_split_rendering( + record_property, + props_to_record, test_info, in_fmt=in_fmt, bitrate=bitrate, @@ -179,6 +277,11 @@ def test_masa_full_chain_split(test_info, in_fmt, bitrate, render_config, trajec binary_suffix=EXE_SUFFIX, pre_trajectory=pre_trajectory, post_trajectory=post_trajectory, + get_mld=get_mld, + mld_lim=get_mld_lim, + get_ssnr=get_ssnr, + get_odg=get_odg, + get_odg_bin=get_odg_bin, ) @@ -186,16 +289,31 @@ def test_masa_full_chain_split(test_info, in_fmt, bitrate, render_config, trajec @pytest.mark.parametrize("trajectory", SPLIT_REND_HR_TRAJECTORIES_TO_TEST) @pytest.mark.parametrize("render_config", RENDERER_CONFIGS_TO_TEST_MASA) @pytest.mark.parametrize("in_fmt", INPUT_FORMATS_MASA_SPLIT_REND) -def test_masa_external_split(test_info, in_fmt, render_config, trajectory): +def test_masa_external_split( + record_property, + props_to_record, + get_mld, + get_mld_lim, + get_ssnr, + get_odg, + get_odg_bin, +test_info, in_fmt, render_config, trajectory): post_trajectory = HR_TRAJECTORY_DIR.joinpath(f"{trajectory}.csv") pre_trajectory = post_trajectory.with_stem(f"{post_trajectory.stem}_delayed") run_external_split_rendering( + record_property, + props_to_record, test_info, in_fmt=in_fmt, render_config=RENDER_CFG_DIR.joinpath(f"{render_config}.txt"), pre_trajectory=pre_trajectory, post_trajectory=post_trajectory, + get_mld=get_mld, + mld_lim=get_mld_lim, + get_ssnr=get_ssnr, + get_odg=get_odg, + get_odg_bin=get_odg_bin, ) @@ -207,11 +325,21 @@ def test_masa_external_split(test_info, in_fmt, render_config, trajectory): @pytest.mark.parametrize("render_config", RENDERER_CONFIGS_TO_TEST_OMASA) @pytest.mark.parametrize("bitrate", IVAS_BITRATES_OMASA) @pytest.mark.parametrize("in_fmt", INPUT_FORMATS_OMASA_SPLIT_REND) -def test_omasa_full_chain_split(test_info, in_fmt, bitrate, render_config, trajectory): +def test_omasa_full_chain_split( + record_property, + props_to_record, + get_mld, + get_mld_lim, + get_ssnr, + get_odg, + get_odg_bin, +test_info, in_fmt, bitrate, render_config, trajectory): post_trajectory = HR_TRAJECTORY_DIR.joinpath(f"{trajectory}.csv") pre_trajectory = post_trajectory.with_stem(f"{post_trajectory.stem}_delayed") run_full_chain_split_rendering( + record_property, + props_to_record, test_info, in_fmt=in_fmt, bitrate=bitrate, @@ -219,6 +347,11 @@ def test_omasa_full_chain_split(test_info, in_fmt, bitrate, render_config, traje binary_suffix=EXE_SUFFIX, pre_trajectory=pre_trajectory, post_trajectory=post_trajectory, + get_mld=get_mld, + mld_lim=get_mld_lim, + get_ssnr=get_ssnr, + get_odg=get_odg, + get_odg_bin=get_odg_bin, ) @@ -230,11 +363,21 @@ def test_omasa_full_chain_split(test_info, in_fmt, bitrate, render_config, traje @pytest.mark.parametrize("render_config", RENDERER_CONFIGS_TO_TEST_OSBA) @pytest.mark.parametrize("bitrate", IVAS_BITRATES_OSBA) @pytest.mark.parametrize("in_fmt", INPUT_FORMATS_OSBA_SPLIT_REND) -def test_osba_full_chain_split(test_info, in_fmt, bitrate, render_config, trajectory): +def test_osba_full_chain_split( + record_property, + props_to_record, + get_mld, + get_mld_lim, + get_ssnr, + get_odg, + get_odg_bin, +test_info, in_fmt, bitrate, render_config, trajectory): post_trajectory = HR_TRAJECTORY_DIR.joinpath(f"{trajectory}.csv") pre_trajectory = post_trajectory.with_stem(f"{post_trajectory.stem}_delayed") run_full_chain_split_rendering( + record_property, + props_to_record, test_info, in_fmt=in_fmt, bitrate=bitrate, @@ -242,6 +385,11 @@ def test_osba_full_chain_split(test_info, in_fmt, bitrate, render_config, trajec binary_suffix=EXE_SUFFIX, pre_trajectory=pre_trajectory, post_trajectory=post_trajectory, + get_mld=get_mld, + mld_lim=get_mld_lim, + get_ssnr=get_ssnr, + get_odg=get_odg, + get_odg_bin=get_odg_bin, ) @@ -253,17 +401,32 @@ def test_osba_full_chain_split(test_info, in_fmt, bitrate, render_config, trajec @pytest.mark.parametrize("trajectory", SPLIT_REND_HR_TRAJECTORIES_TO_TEST) @pytest.mark.parametrize("render_config", RENDERER_CONFIGS_TO_TEST_PLC) @pytest.mark.parametrize("in_fmt", INPUT_FORMATS_AMBI_SPLIT_REND[-1:]) -def test_post_rend_plc(test_info, in_fmt, render_config, trajectory, error_pattern): +def test_post_rend_plc( + record_property, + props_to_record, + get_mld, + get_mld_lim, + get_ssnr, + get_odg, + get_odg_bin, +test_info, in_fmt, render_config, trajectory, error_pattern): post_trajectory = HR_TRAJECTORY_DIR.joinpath(f"{trajectory}.csv") pre_trajectory = post_trajectory.with_stem(f"{post_trajectory.stem}_delayed") run_external_split_rendering( + record_property, + props_to_record, test_info, in_fmt=in_fmt, render_config=RENDER_CFG_DIR.joinpath(f"{render_config}.txt"), pre_trajectory=pre_trajectory, post_trajectory=post_trajectory, plc_error_pattern=ERROR_PATTERNS_DIR.joinpath(f"{error_pattern}.ep"), + get_mld=get_mld, + mld_lim=get_mld_lim, + get_ssnr=get_ssnr, + get_odg=get_odg, + get_odg_bin=get_odg_bin, ) @@ -281,12 +444,22 @@ full_chain_split_pcm_params = [ @pytest.mark.parametrize("in_fmt,bitrate,render_config", full_chain_split_pcm_params) -def test_full_chain_split_pcm(test_info, in_fmt, bitrate, render_config): +def test_full_chain_split_pcm( + record_property, + props_to_record, + get_mld, + get_mld_lim, + get_ssnr, + get_odg, + get_odg_bin, +test_info, in_fmt, bitrate, render_config): trajectory = SPLIT_REND_HR_TRAJECTORIES_TO_TEST[0] post_trajectory = HR_TRAJECTORY_DIR.joinpath(f"{trajectory}.csv") pre_trajectory = post_trajectory.with_stem(f"{post_trajectory.stem}_delayed") run_full_chain_split_rendering( + record_property, + props_to_record, test_info, in_fmt=in_fmt, bitrate=bitrate, @@ -295,6 +468,11 @@ def test_full_chain_split_pcm(test_info, in_fmt, bitrate, render_config): pre_trajectory=pre_trajectory, post_trajectory=post_trajectory, renderer_fmt="BINAURAL_SPLIT_PCM", + get_mld=get_mld, + mld_lim=get_mld_lim, + get_ssnr=get_ssnr, + get_odg=get_odg, + get_odg_bin=get_odg_bin, ) @@ -307,18 +485,33 @@ external_split_pcm_params = [ @pytest.mark.parametrize("in_fmt,render_config", external_split_pcm_params) -def test_external_split_pcm(test_info, in_fmt, render_config): +def test_external_split_pcm( + record_property, + props_to_record, + get_mld, + get_mld_lim, + get_ssnr, + get_odg, + get_odg_bin, +test_info, in_fmt, render_config): trajectory = SPLIT_REND_HR_TRAJECTORIES_TO_TEST[0] post_trajectory = HR_TRAJECTORY_DIR.joinpath(f"{trajectory}.csv") pre_trajectory = post_trajectory.with_stem(f"{post_trajectory.stem}_delayed") run_external_split_rendering( + record_property, + props_to_record, test_info, in_fmt=in_fmt, render_config=RENDER_CFG_DIR.joinpath(f"{render_config}.txt"), pre_trajectory=pre_trajectory, post_trajectory=post_trajectory, renderer_fmt="BINAURAL_SPLIT_PCM", + get_mld=get_mld, + mld_lim=get_mld_lim, + get_ssnr=get_ssnr, + get_odg=get_odg, + get_odg_bin=get_odg_bin, ) @@ -327,11 +520,21 @@ def test_external_split_pcm(test_info, in_fmt, render_config): @pytest.mark.parametrize("in_fmt", ["5_1"]) @pytest.mark.parametrize("pre_rend_fr", SPLIT_RENDERER_PRE_FRAMINGS) @pytest.mark.parametrize("post_rend_fr", SPLIT_RENDERER_POST_FRAMINGS) -def test_framing_combinations_external_split(test_info, in_fmt, render_config, trajectory, post_rend_fr, pre_rend_fr): +def test_framing_combinations_external_split( + record_property, + props_to_record, + get_mld, + get_mld_lim, + get_ssnr, + get_odg, + get_odg_bin, +test_info, in_fmt, render_config, trajectory, post_rend_fr, pre_rend_fr): post_trajectory = HR_TRAJECTORY_DIR.joinpath(f"{trajectory}.csv") pre_trajectory = post_trajectory.with_stem(f"{post_trajectory.stem}_delayed") run_external_split_rendering( + record_property, + props_to_record, test_info, in_fmt=in_fmt, render_config=RENDER_FRAMING_CFG_DIR.joinpath(f"{render_config}.txt"), @@ -339,6 +542,11 @@ def test_framing_combinations_external_split(test_info, in_fmt, render_config, t post_trajectory=post_trajectory, post_rend_fr=post_rend_fr, pre_rend_fr=pre_rend_fr, + get_mld=get_mld, + mld_lim=get_mld_lim, + get_ssnr=get_ssnr, + get_odg=get_odg, + get_odg_bin=get_odg_bin, ) @pytest.mark.parametrize("trajectory", SPLIT_REND_HR_TRAJECTORIES_TO_TEST) @@ -347,12 +555,21 @@ def test_framing_combinations_external_split(test_info, in_fmt, render_config, t @pytest.mark.parametrize("pre_rend_fr", SPLIT_RENDERER_PRE_FRAMINGS) @pytest.mark.parametrize("post_rend_fr", SPLIT_RENDERER_POST_FRAMINGS) def test_framing_combinations_full_chain_split( + record_property, + props_to_record, + get_mld, + get_mld_lim, + get_ssnr, + get_odg, + get_odg_bin, test_info, in_fmt, render_config, trajectory, post_rend_fr, pre_rend_fr ): post_trajectory = HR_TRAJECTORY_DIR.joinpath(f"{trajectory}.csv") pre_trajectory = post_trajectory.with_stem(f"{post_trajectory.stem}_delayed") run_full_chain_split_rendering( + record_property, + props_to_record, test_info, in_fmt=in_fmt, render_config=RENDER_FRAMING_CFG_DIR.joinpath(f"{render_config}.txt"), @@ -362,4 +579,9 @@ def test_framing_combinations_full_chain_split( binary_suffix=EXE_SUFFIX, post_rend_fr=post_rend_fr, pre_rend_fr=pre_rend_fr, + get_mld=get_mld, + mld_lim=get_mld_lim, + get_ssnr=get_ssnr, + get_odg=get_odg, + get_odg_bin=get_odg_bin, ) diff --git a/tests/split_rendering/utils.py b/tests/split_rendering/utils.py index ed0d3495fc3d7a37327fadaaa3ca3b5c214d4d1c..f9a1e86fc7c591c5020bb08eaa774035e3527caa 100644 --- a/tests/split_rendering/utils.py +++ b/tests/split_rendering/utils.py @@ -1,33 +1,33 @@ #!/usr/bin/env python3 """ - (C) 2022-2025 IVAS codec Public Collaboration with portions copyright Dolby International AB, Ericsson AB, - Fraunhofer-Gesellschaft zur Foerderung der angewandten Forschung e.V., Huawei Technologies Co. LTD., - Koninklijke Philips N.V., Nippon Telegraph and Telephone Corporation, Nokia Technologies Oy, Orange, - Panasonic Holdings Corporation, Qualcomm Technologies, Inc., VoiceAge Corporation, and other - contributors to this repository. All Rights Reserved. - - This software is protected by copyright law and by international treaties. - The IVAS codec Public Collaboration consisting of Dolby International AB, Ericsson AB, - Fraunhofer-Gesellschaft zur Foerderung der angewandten Forschung e.V., Huawei Technologies Co. LTD., - Koninklijke Philips N.V., Nippon Telegraph and Telephone Corporation, Nokia Technologies Oy, Orange, - Panasonic Holdings Corporation, Qualcomm Technologies, Inc., VoiceAge Corporation, and other - contributors to this repository retain full ownership rights in their respective contributions in - the software. This notice grants no license of any kind, including but not limited to patent - license, nor is any license granted by implication, estoppel or otherwise. - - Contributors are required to enter into the IVAS codec Public Collaboration agreement before making - contributions. - - This software is provided "AS IS", without any express or implied warranties. The software is in the - development stage. It is intended exclusively for experts who have experience with such software and - solely for the purpose of inspection. All implied warranties of non-infringement, merchantability - and fitness for a particular purpose are hereby disclaimed and excluded. - - Any dispute, controversy or claim arising under or in relation to providing this software shall be - submitted to and settled by the final, binding jurisdiction of the courts of Munich, Germany in - accordance with the laws of the Federal Republic of Germany excluding its conflict of law rules and - the United Nations Convention on Contracts on the International Sales of Goods. +(C) 2022-2025 IVAS codec Public Collaboration with portions copyright Dolby International AB, Ericsson AB, +Fraunhofer-Gesellschaft zur Foerderung der angewandten Forschung e.V., Huawei Technologies Co. LTD., +Koninklijke Philips N.V., Nippon Telegraph and Telephone Corporation, Nokia Technologies Oy, Orange, +Panasonic Holdings Corporation, Qualcomm Technologies, Inc., VoiceAge Corporation, and other +contributors to this repository. All Rights Reserved. + +This software is protected by copyright law and by international treaties. +The IVAS codec Public Collaboration consisting of Dolby International AB, Ericsson AB, +Fraunhofer-Gesellschaft zur Foerderung der angewandten Forschung e.V., Huawei Technologies Co. LTD., +Koninklijke Philips N.V., Nippon Telegraph and Telephone Corporation, Nokia Technologies Oy, Orange, +Panasonic Holdings Corporation, Qualcomm Technologies, Inc., VoiceAge Corporation, and other +contributors to this repository retain full ownership rights in their respective contributions in +the software. This notice grants no license of any kind, including but not limited to patent +license, nor is any license granted by implication, estoppel or otherwise. + +Contributors are required to enter into the IVAS codec Public Collaboration agreement before making +contributions. + +This software is provided "AS IS", without any express or implied warranties. The software is in the +development stage. It is intended exclusively for experts who have experience with such software and +solely for the purpose of inspection. All implied warranties of non-infringement, merchantability +and fitness for a particular purpose are hereby disclaimed and excluded. + +Any dispute, controversy or claim arising under or in relation to providing this software shall be +submitted to and settled by the final, binding jurisdiction of the courts of Munich, Germany in +accordance with the laws of the Federal Republic of Germany excluding its conflict of law rules and +the United Nations Convention on Contracts on the International Sales of Goods. """ import sys @@ -39,8 +39,16 @@ from typing import Tuple import numpy as np import pytest -from tests.renderer.utils import check_BE, run_ivas_isar_enc_cmd, run_ivas_isar_dec_cmd, run_isar_post_rend_cmd, run_isar_ext_rend_cmd +from tests.renderer.utils import ( + check_BE, + run_ivas_isar_enc_cmd, + run_ivas_isar_dec_cmd, + run_isar_post_rend_cmd, + run_isar_ext_rend_cmd, +) from tests.split_rendering.constants import * +from ..cmp_pcm import cmp_pcm +from ..conftest import parse_properties sys.path.append(SCRIPTS_DIR) from pyaudio3dtools.audiofile import readfile, writefile @@ -60,8 +68,10 @@ def lc3plus_used(test_info, in_fmt, render_config): or ("default" in render_config and "external_split" in test_info.node.name) ) + FRAME_SIZE_RE = re.compile(r"(\d+)ms") + def transport_codec_frame_size_ms(test_info, in_fmt, render_config, pre_rend_fr): explicit_frame_size_ms = FRAME_SIZE_RE.search(render_config) @@ -151,6 +161,8 @@ def truncate_signal( def run_full_chain_split_rendering( + record_property, + props_to_record, test_info, in_fmt: str, bitrate: str, @@ -161,25 +173,32 @@ def run_full_chain_split_rendering( binary_suffix: str = "", post_rend_fr: str = "20", pre_rend_fr: str = "20", + get_mld=False, + mld_lim=0, + get_ssnr=False, + get_odg=False, + get_odg_bin=False, ) -> str: """ Runs the full split rendering chain consisting of the IVAS encoder, decoder and split renderer """ - check_xfail(test_info, in_fmt, render_config.name, pre_rend_fr, post_rend_fr, bitrate) + check_xfail( + test_info, in_fmt, render_config.name, pre_rend_fr, post_rend_fr, bitrate + ) with TemporaryDirectory() as tmp_dir: tmp_dir = Path(tmp_dir) cut_in_file = tmp_dir.joinpath("cut_input.wav") - #ivas_bitstream = tmp_dir.joinpath("ivas.192") + # ivas_bitstream = tmp_dir.joinpath("ivas.192") ivas_bitstream_stem = f"{in_fmt}_{bitrate}bps_{renderer_fmt}_split_full_config_{render_config.stem}_prerfr_{pre_rend_fr}_postrfr_{post_rend_fr}_ivas.192" - #split_bitstream = tmp_dir.joinpath("split.bit") + # split_bitstream = tmp_dir.joinpath("split.bit") split_bitstream_stem = f"{in_fmt}_{bitrate}bps_{renderer_fmt}_split_full_config_{render_config.stem}_prerfr_{pre_rend_fr}_postrfr_{post_rend_fr}_split.bit" if renderer_fmt == "BINAURAL_SPLIT_PCM": - #split_md_file = tmp_dir.joinpath("split_md.bin") + # split_md_file = tmp_dir.joinpath("split_md.bin") split_md_file_stem = f"{in_fmt}_{bitrate}bps_{renderer_fmt}_split_full_config_{render_config.stem}_prerfr_{pre_rend_fr}_postrfr_{post_rend_fr}_split_md.bit" - + out_file_stem = f"{in_fmt}_{bitrate}bps_{renderer_fmt}_split_full_config_{render_config.stem}_prerfr_{pre_rend_fr}_postrfr_{post_rend_fr}_.wav" if test_info.config.option.create_ref: @@ -197,7 +216,11 @@ def run_full_chain_split_rendering( if in_fmt.upper().startswith("OSBA"): # use same MD as ISM in_meta_files = FORMAT_TO_METADATA_FILES[f"ISM{in_fmt[5]}"] - elif in_fmt.upper().startswith("ISM") or in_fmt.upper().startswith("MASA") or in_fmt.upper().startswith("OMASA"): + elif ( + in_fmt.upper().startswith("ISM") + or in_fmt.upper().startswith("MASA") + or in_fmt.upper().startswith("OMASA") + ): in_meta_files = FORMAT_TO_METADATA_FILES[in_fmt] else: in_meta_files = None @@ -247,7 +270,7 @@ def run_full_chain_split_rendering( # run split renderer cmd = SPLIT_POST_REND_CMD[:] - #if test_info.config.option.create_ref: + # if test_info.config.option.create_ref: # cmd[0] += BIN_SUFFIX_MERGETARGET cmd[0] += binary_suffix cmd[4] = str(split_bitstream) @@ -274,15 +297,44 @@ def run_full_chain_split_rendering( cut, cut_fs = readfile(out_file) - [diff_found, snr, gain_b, max_diff] = check_BE(test_info, ref, ref_fs, cut, cut_fs) - if diff_found : - pytest.fail( - f"CuT not BE to REF! SNR : {snr:3.2f} dB, Gain CuT: {gain_b:1.3f}, Max Diff = {int(max_diff)}" + if not get_mld: + [diff_found, snr, gain_b, max_diff] = check_BE( + test_info, ref, ref_fs, cut, cut_fs ) + if diff_found: + pytest.fail( + f"CuT not BE to REF! SNR : {snr:3.2f} dB, Gain CuT: {gain_b:1.3f}, Max Diff = {int(max_diff)}" + ) + else: + output_differs, reason = cmp_pcm( + out_file, + out_file_ref, + 2, # is always "BINAURAL" + ref_fs, + get_mld=get_mld, + mld_lim=mld_lim, + get_ssnr=get_ssnr, + get_odg=get_odg, + get_odg_bin=get_odg_bin, + ) + + # NOTE: split comparison not implemented for split rendering, always unpack values + reason = reason[0] + output_differs = output_differs[0] + + props = parse_properties(reason, output_differs, props_to_record) + for k, v in props.items(): + record_property(k, v) + + if output_differs: + pytest.fail(f"Output differs: ({reason})") + return out_file def run_external_split_rendering( + record_property, + props_to_record, test_info, in_fmt: str, render_config: Path, @@ -294,6 +346,11 @@ def run_external_split_rendering( is_comparetest: bool = False, post_rend_fr: str = "20", pre_rend_fr: str = "20", + get_mld=False, + mld_lim=0, + get_ssnr=False, + get_odg=False, + get_odg_bin=False, ) -> Tuple[np.ndarray, int]: """ Runs the exeternal split rendering chain consisting of @@ -359,7 +416,7 @@ def run_external_split_rendering( # run split renderer cmd = SPLIT_POST_REND_CMD[:] - #if test_info.config.option.create_ref: + # if test_info.config.option.create_ref: # cmd[0] += BIN_SUFFIX_MERGETARGET cmd[0] += binary_suffix cmd[4] = str(split_bitstream) @@ -393,9 +450,36 @@ def run_external_split_rendering( cut, cut_fs = readfile(out_file) - [diff_found, snr, gain_b, max_diff] = check_BE(test_info, ref, ref_fs, cut, cut_fs) - if diff_found : - pytest.fail( - f"CuT not BE to REF! SNR : {snr:3.2f} dB, Gain CuT: {gain_b:1.3f}, Max Diff = {int(max_diff)}" + if get_mld == False: + [diff_found, snr, gain_b, max_diff] = check_BE( + test_info, ref, ref_fs, cut, cut_fs + ) + if diff_found: + pytest.fail( + f"CuT not BE to REF! SNR : {snr:3.2f} dB, Gain CuT: {gain_b:1.3f}, Max Diff = {int(max_diff)}" + ) + else: + # see constants.py + output_differs, reason = cmp_pcm( + out_file, + out_file_ref, + 2, # is always "BINAURAL", + ref_fs, + get_mld=get_mld, + mld_lim=mld_lim, + get_ssnr=get_ssnr, + get_odg=get_odg, + get_odg_bin=get_odg_bin, ) + # NOTE: split comparison not implemented for split rendering, always unpack values + reason = reason[0] + output_differs = output_differs[0] + + props = parse_properties(reason, output_differs, props_to_record) + for k, v in props.items(): + record_property(k, v) + + if output_differs: + pytest.fail(f"Output differs: ({reason})") + return out_file