diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 8dc11d880e94998ba16917fb01b360d01921a6bb..995f3320138bef0f9eccc710c39cf30dd0e35426 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -412,7 +412,7 @@ codec-smoke-test: # LTV update needed as ltv ISM metadata files are used - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/update-ltv-repo.sh - # for MRs to basop-ci-branch, we want to test with the BASOP float reference build + # for MRs to basop-ci-branch-pc, we want to test with the BASOP float reference build # per default, the smoke test builds the current repo again with WMOPS activated to catch unbalanced instrumentation macros - if [ "$CI_MERGE_REQUEST_TARGET_BRANCH_NAME" = "basop-ci-branch" ]; then - python3 scripts/smoketest-basop-filter.py scripts/config/ivas_modes.json --inline diff --git a/ci/remove_unsupported_testcases.py b/ci/remove_unsupported_testcases.py index 6751f55e76f09afaf5180a076c0f31a385396be9..28dc94c18be82224c143a37abd6a1d36eca758ee 100644 --- a/ci/remove_unsupported_testcases.py +++ b/ci/remove_unsupported_testcases.py @@ -32,81 +32,41 @@ from pathlib import Path import argparse # Enter tag of testcases to remove here WITHOUT the leading // -TESTCASES_MAIN = [ - "OMASA 2Dir2TC 4ISM at br sw techs 13.2 to 512 kbps start 80 kbps, 48kHz in, 48kHz out, EXT out", - "OSBA planar FOA 2ISM at 512 kbps, 48 kHz in, 48 kHz out, BINAURAL out", - "4 ISM with extended metadata at 128 kbps, 48 kHz in, 48 kHz out, BINAURAL_ROOM_REVERB out, rendconf dir w id", - "OSBA planar FOA 1ISM at 256 kbps, 48 kHz in, 48 kHz out, BINAURAL out", - "Multi-channel 5_1 bitrate switching from 24.4 kbps to 256 kbps, 48kHz in, 48kHz out, BINAURAL out, FER at 10%, bandwidth switching", - "OSBA FOA 4ISM at bitrate switching 13.2 to 512 kbps, 48kHz in, 48kHz out, BINAURAL out, FER at 5%, bandwidth switching", - "OSBA FOA 2ISM at 64 kbps, 48kHz in, 48kHz out, HOA3 out, bandwidth switching", - "OMASA 2Dir2TC 4ISM at br sw techs 13.2 to 512 kbps start 80 kbps, 48kHz in, 48kHz out, EXT out", - "OSBA planar FOA 2ISM at 512 kbps, 48 kHz in, 48 kHz out, BINAURAL out", - "OSBA planar FOA 1ISM at 256 kbps, 48 kHz in, 48 kHz out, BINAURAL out", - "SBA 3OA at 128 kbps, 48kHz in 48kHz out, BINAURAL_ROOM_REVERB rendconf sel acoustic env", - "OSBA FOA 4ISM at br sw 13.2 to 512 kbps, 48kHz in, 16kHz out, BINAURAL out (Model from file), FER at 5%, bandwidth switching", +TESTCASES = [ + # self_test.prm "stereo bitrate switching from 13.2 kbps to 128 kbps, 48kHz in, 48kHz out, DTX on, EXT out", - "SBA FOA bitrate switching from 13.2 kbps to 192 kbps, 32kHz in, 32kHz out, DTX on, EXT out", - "SBA 2OA bitrate switching from 13.2 kbps to 128 kbps, 32kHz in, 32kHz out, DTX on, EXT out", - "SBA 3OA bitrate switching from 13.2 kbps to 128 kbps, 48kHz in, 48kHz out, DTX on, random FER at 5%, EXT out", - "Multi-channel 5_1 bitrate switching from 13.2 kbps to 512 kbps, 48kHz in, 48kHz out, EXT out", - "Multi-channel 5_1_2 bitrate switching from 24.4 kbps to 256 kbps, 48kHz in, 48kHz out, EXT out", + "Multi-channel 5_1 at 512 kbps, 48kHz in 48kHz out, BINAURAL_ROOM_REVERB out custom acoustic environment with a sequence (CREND)", "Multi-channel 5_1_4 bitrate switching from 13.2 kbps to 512 kbps, 48kHz in, 32kHz out, EXT out", + "Multi-channel 5_1 at 64 kbps, 48kHz in 48kHz out, BINAURAL_ROOM_REVERB out custom acoustic environment with a sequence (FastConv)", "Multi-channel 7_1 bitrate switching from 24.4 kbps to 256 kbps, 48kHz in, 16kHz out, EXT out", "Multi-channel 7_1_4 bitrate switching from 13.2 kbps to 512 kbps, 48kHz in, 48kHz out, EXT out", - "SBA planar 2OA bitrate switching from 13.2 kbps to 128 kbps, 32kHz in, 32kHz out, DTX on, EXT out", - "SBA planar FOA bitrate switching from 13.2 kbps to 512 kbps, 32kHz in, 32kHz out, EXT out", - "SBA 2OA bitrate switching from 13.2 kbps to 512 kbps, 32kHz in, 32kHz out, EXT out", - "SBA planar 3OA bitrate switching from 13.2 kbps to 512 kbps, 48kHz in, 48kHz out, random FER at 5%, EXT out", - "Multi-channel 5_1 at 512 kbps, 48kHz in 48kHz out, BINAURAL_ROOM_REVERB out custom acoustic environment with a sequence (CREND)", - "Multi-channel 5_1 at 64 kbps, 48kHz in 48kHz out, BINAURAL_ROOM_REVERB out custom acoustic environment with a sequence (FastConv)", "Multi-channel 5_1 at 32 kbps, 48kHz in 48kHz out, BINAURAL_ROOM_REVERB out custom acoustic environment with a sequence (ParamBin)", - "MASA 1dir 1TC at 256 kbps, 48kHz in, 48kHz out, BINAURAL_ROOM_REVERB out, HR custom configuration", - "MASA 1dir 1TC at 256kbps, 48kHz in, 48 kHz out, BINAURAL_ROOM_REVERB out custom configuration", - "MASA 1TC at 256kbps, 48kHz in, 48 kHz out, BINAURAL_ROOM_REVERB out custom configuration", - "MASA 1TC at 256 kbps, 48kHz in, 48kHz out, BINAURAL_ROOM_REVERB out, HR custom configuration", - "OMASA 2TC 4ISM at br sw techs 13.2 to 512 kbps start 80 kbps, 48kHz in, 48kHz out, EXT out", "Multi-channel 7_1_4 bitrate switching, 48kHz in, 48kHz out, BINAURAL out, HR, JBM Prof 5", + "Multi-channel 5_1_2 bitrate switching from 24.4 kbps to 256 kbps, 48kHz in, 48kHz out, EXT out", "Multi-channel 7_1 bitrate switching, 48kHz in, 32kHz out, BINAURAL_ROOM_REVERB out, HR, JBM Prof 5", - "3 ISM with metadata bitrate switching from 48 kbps to 32 kbps, 48 kHz in, 32 kHz out, DTX, BINAURAL_ROOM_IR out, JBM Prof 5", - # object editing starts here + "OMASA 2Dir2TC 4ISM at br sw techs 13.2 to 512 kbps start 80 kbps, 48kHz in, 48kHz out, EXT out", "OMASA 2Dir2TC 4ISM at 80 kbps, 48kHz in, 48kHz out, BINAURAL out, default object editing, 1SEP-PARAM", + "OMASA 2Dir2TC 2ISM br sw techs 13.2 to 512 kbps start 48 kbps, 48kHz in, 48kHz out, BINAURAL out, object editing", + "4 ISM with metadata at 32 kbps, 48 kHz in, 48 kHz out, BINAURAL out, object editing, PARAM_ISM", "OMASA 2Dir2TC 4ISM at 256 kbps, 48kHz in, 48kHz out, BINAURAL out, object editing, JBM Prof 5, DISC", + "4 ISM with metadata bitrate switching from 48 kbps to 32 kbps, 48 kHz in, 48 kHz out, BINAURAL out, object editing, JBM Prof 5", + "3 ISM with metadata at 384 kbps, 48 kHz in, 48 kHz out, FOA out, object editing, JBM Prof 5, DISC", + "4 ISM with metadata at 128 kbps, 48 kHz in, 48 kHz out, BINAURAL out, object editing, DISC, FER at 5%", + "3 ISM with metadata at 24.4 kbps, 48 kHz in, 48 kHz out, FOA out, object editing, JBM Prof 5, PARAM_ISM", + "4 ISM with metadata bitrate switching from 32 kbps to 48 kbps, 48 kHz in, 48 kHz out, BINAURAL_ROOM_IR out, object editing", "OMASA 2Dir2TC 2ISM at 96 kbps, 48kHz in, 48kHz out, FOA out, object editing, JBM Prof 5, DISC", - "OMASA 2Dir2TC 2ISM br sw techs 13.2 to 512 kbps start 48 kbps, 48kHz in, 48kHz out, BINAURAL out, object editing", - # the next four lines are basically duplicates of the last four, but without "Dir", because the ltv MASA cases do not have dedicated testvectors for numbers of directions - "OMASA 2TC 4ISM at 80 kbps, 48kHz in, 48kHz out, BINAURAL out, default object editing, 1SEP-PARAM", - "OMASA 2TC 4ISM at 256 kbps, 48kHz in, 48kHz out, BINAURAL out, object editing, JBM Prof 5, DISC", - "OMASA 2TC 2ISM at 96 kbps, 48kHz in, 48kHz out, FOA out, object editing, JBM Prof 5, DISC", - "OMASA 2TC 2ISM br sw techs 13.2 to 512 kbps start 48 kbps, 48kHz in, 48kHz out, BINAURAL out, object editing", "OSBA 3OA 4ISM at 256 kbps, 48kHz in, 48kHz out, BINAURAL out, object editing, DISC", "OSBA 2OA 3ISM at 128 kbps, 48kHz in, 48kHz out, FOA out, object editing, JBM Prof 5, DISC", "OSBA 2OA 3ISM at bitrate switching 13.2 to 512 kbps, 48kHz in, 48kHz out, BINAURAL out, object editing", - "4 ISM with metadata at 32 kbps, 48 kHz in, 48 kHz out, BINAURAL out, object editing, PARAM_ISM", - "4 ISM with metadata at 128 kbps, 48 kHz in, 48 kHz out, BINAURAL out, object editing, DISC", - "3 ISM with metadata at 24.4 kbps, 48 kHz in, 48 kHz out, FOA out, object editing, JBM Prof 5, PARAM_ISM", - "3 ISM with metadata at 384 kbps, 48 kHz in, 48 kHz out, FOA out, object editing, JBM Prof 5, DISC", - "4 ISM with metadata bitrate switching from 32 kbps to 48 kbps, 48 kHz in, 48 kHz out, BINAURAL_ROOM_IR out, object editing", - "4 ISM with metadata bitrate switching from 48 kbps to 32 kbps, 48 kHz in, 48 kHz out, BINAURAL out, object editing, JBM Prof 5", -] -TESTCASES_MAIN_PC = [ - "SBA at 128 kbps, 32kHZ in, 32kHz out, BINAURAL_ROOM_REVERB out, Config renderer, HR", - "SBA at 128 kbps, 32kHZ in, 16kHz out, BINAURAL_ROOM_REVERB out (Model from file), HR", - "Planar SBA at 128 kbps, 48kHZ in, 32kHz out, BINAURAL_ROOM_REVERB out (Model from file), Config renderer, HR", - "Multi-channel 7_1_4 at 160 kbps, 48kHz in, 48 kHz out, BINAURAL_ROOM_REVERB out default configuration", - "Multi-channel 5_1 at 80 kbps, 48kHz in, 32kHz out, BINAURAL_ROOM_REVERB out Config renderer, HR", - "OSBA planar 2OA 4ISM at 512 kbps, 48 kHz in, 48 kHz out, BINAURAL ROOM REVERB (Model from file) out", - "SBA 3OA 4ISM at 96 kbps, 48kHz in, 48kHz out, BINAURAL_ROOM_REVERB out custom configuration", - "Multi-channel 7_1_4 at 160 kbps, 48kHz in, 48kHz out, BINAURAL_ROOM_REVERB out Config recreation, HR", - "Multi-channel 5_1_2 at 64 kbps, 48kHz in, 48kHz out, BINAURAL_ROOM_REVERB out Config renderer, HR", - "OSBA 3OA 4ISM at 512 kbps, 48kHz in, 48kHz out, BINAURAL_ROOM_REVERB out", - "OSBA 3ISM 3OA at bitrate switching 13.2 to 512 kbps, 48kHz in, 32kHz out, BINAURAL ROOM REVERB out", - "Multi-channel 5_1 bitrate switching from 13.2 kbps to 512 kbps, 48kHz in, 16kHz out, BINAURAL_ROOM_REVERB out", - "OSBA 2OA 2ISM bitrate switching, 32kHz in, 48kHz out, BINAURAL_ROOM_REVERB out, JBM Prof 5", -] -TESTCASES_MAIN_PC_REENABLE = [ - "4 ISM with extended metadata at 128 kbps, 48 kHz in, 48 kHz out, BINAURAL_ROOM_REVERB out, rendconf dir w id", + # self_test_ltv.prm + "OMASA 2TC 4ISM at br sw techs 13.2 to 512 kbps start 80 kbps, 48kHz in, 48kHz out, EXT out", + "OMASA 2TC 2ISM at 96 kbps, 48kHz in, 48kHz out, FOA out, object editing, JBM Prof 5, DISC", + "OMASA 2TC 4ISM at 80 kbps, 48kHz in, 48kHz out, BINAURAL out, default object editing, 1SEP-PARAM", + "OMASA 2TC 4ISM at 256 kbps, 48kHz in, 48kHz out, BINAURAL out, object editing, JBM Prof 5, DISC", + "OMASA 2TC 2ISM br sw techs 13.2 to 512 kbps start 48 kbps, 48kHz in, 48kHz out, BINAURAL out, object editing", ] + + def remove_testcases(cfg: Path, testcases: list): """ Go through file line by line and copy all testcases except the given ones @@ -132,16 +92,9 @@ def remove_testcases(cfg: Path, testcases: list): if __name__ == "__main__": parser = argparse.ArgumentParser() parser.add_argument("cfg_files", nargs="+", type=Path) - parser.add_argument("--use-main-pc-set", action="store_true") args = parser.parse_args() - testcases = TESTCASES_MAIN - if args.use_main_pc_set: - # Add further unsupported test cases - testcases.extend(TESTCASES_MAIN_PC) - # Re-enable test cases that are now supported - for case in TESTCASES_MAIN_PC_REENABLE: - testcases.remove(case) + testcases = TESTCASES for f in args.cfg_files: remove_testcases(f, testcases) diff --git a/scripts/binauralRenderer_interface/binaural_renderers_hrtf_data/ivas_binaural_16kHz.bin b/scripts/binauralRenderer_interface/binaural_renderers_hrtf_data/ivas_binaural_16kHz.bin index 9a988eb588c1be259b9c7ce01c3bbc872cf3b1e9..ffb4662687d6d2fa0e9cab79dd8bed54a3065ea5 100644 --- a/scripts/binauralRenderer_interface/binaural_renderers_hrtf_data/ivas_binaural_16kHz.bin +++ b/scripts/binauralRenderer_interface/binaural_renderers_hrtf_data/ivas_binaural_16kHz.bin @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:a263c5498fc7ec0407c4f448e78e1f0aaf97163234ef44f2f79a1dfcb5adc37f -size 1968026 +oid sha256:600f05865baeeb01c6435725f87c6a91e6e75e481ffae2ec3d6a7c62aba5fc7c +size 1971974 diff --git a/scripts/binauralRenderer_interface/binaural_renderers_hrtf_data/ivas_binaural_32kHz.bin b/scripts/binauralRenderer_interface/binaural_renderers_hrtf_data/ivas_binaural_32kHz.bin index 73f151217e89352c910668206ef00200577543d8..97190b44f8bc0e595ef3d5e9e22e02893056ba0c 100644 --- a/scripts/binauralRenderer_interface/binaural_renderers_hrtf_data/ivas_binaural_32kHz.bin +++ b/scripts/binauralRenderer_interface/binaural_renderers_hrtf_data/ivas_binaural_32kHz.bin @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:8622b0f7969a3126f8ab147c10392916b5e9cbe8d8bd9c04798abd3836e36c76 -size 2431246 +oid sha256:cda90cc841793f0a1ed76b580e46b9989564b055b664ed6746ebfa0033a74f2b +size 2430778 diff --git a/scripts/binauralRenderer_interface/binaural_renderers_hrtf_data/ivas_binaural_48kHz.bin b/scripts/binauralRenderer_interface/binaural_renderers_hrtf_data/ivas_binaural_48kHz.bin index d847e746131f589ddf50882fec0caf5b1da2724b..e1f6981162b9f9f917b78e4f6b342e803dc8a8c4 100644 --- a/scripts/binauralRenderer_interface/binaural_renderers_hrtf_data/ivas_binaural_48kHz.bin +++ b/scripts/binauralRenderer_interface/binaural_renderers_hrtf_data/ivas_binaural_48kHz.bin @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:30bcf48f0c851d4e676d5b92be7eb3032442f5106a1072ef8fc9b335d60d0fbf -size 2673694 +oid sha256:c1654ead069a420b97af8755edbb1c358806aa3c5c07db3113789fe0942642de +size 2673946 diff --git a/scripts/parse_xml_report.py b/scripts/parse_xml_report.py index 948bda43086f1145b34babfbbefcf4182d2aaae6..f157731713794f542d467ad257e7a7bf5fae74f2 100644 --- a/scripts/parse_xml_report.py +++ b/scripts/parse_xml_report.py @@ -18,6 +18,10 @@ class Result(str, Enum): PASS = "PASS" +SORT_KEYS = ["MAX_ABS_DIFF", "MLD", "MIN_SSNR", "MIN_ODG"] +SORT_ASCENDING = [False, False, True, True] + + class TestcaseParser(dict): def __init__(self, testcases: list): super().__init__() @@ -111,6 +115,11 @@ class TestcaseParser(dict): def to_df(self) -> pd.DataFrame: testcases = list(self.values()) df = pd.DataFrame(testcases) + + for key in SORT_KEYS: + df[key] = pd.to_numeric(df[key], errors="coerce") + df = df.sort_values(by=SORT_KEYS, ascending=SORT_ASCENDING) + return df