From fd1cab575498efe0fc9a63bb499169c78b3cbc86 Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Fri, 13 Dec 2024 10:53:18 +0100 Subject: [PATCH 001/106] revert changes to jbm BE test coming in from main --- tests/test_be_for_jbm_neutral_dly_profile.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/tests/test_be_for_jbm_neutral_dly_profile.py b/tests/test_be_for_jbm_neutral_dly_profile.py index 61bb168a2f..743a4e0cfb 100644 --- a/tests/test_be_for_jbm_neutral_dly_profile.py +++ b/tests/test_be_for_jbm_neutral_dly_profile.py @@ -17,7 +17,7 @@ DTX_OFF = "DTX_OFF" # TODO: Clarify if this should also be tested with DTX, see #1122 TESTCASES = [ # stereo - ["stereo", 32000, "EXT", "DTX_OFF"], + ["stereo", 32000, "STEREO", "DTX_OFF"], ["stereo", 48000, "MONO", "DTX_OFF"], ["stereo", 16400, "5_1", "DTX_OFF"], ["stereo", 256000, "7_1_4", "DTX_OFF"], @@ -42,21 +42,21 @@ TESTCASES = [ # McMasa ["MC_5_1", 16400, "BINAURAL_ROOM_IR", "DTX_OFF"], ["MC_7_1_4", 80000, "mono", "DTX_OFF"], - ["MC_5_1_2", 24400, "EXT", "DTX_OFF"], + ["MC_5_1_2", 24400, "5_1_2", "DTX_OFF"], # paramMC ["MC_5_1_2", 48000, "BINAURAL", "DTX_OFF"], - ["MC_7_1", 80000, "EXT", "DTX_OFF"], + ["MC_7_1", 80000, "7_1", "DTX_OFF"], ["MC_7_1_4", 128000, "FOA", "DTX_OFF"], # paramUpmix ["MC_7_1_4", 160000, "stereo", "DTX_OFF"], # discrete MC ["MC_5_1_2", 512000, "BINAURAL_ROOM_REVERB", "DTX_OFF"], - ["MC_7_1", 128000, "EXT", "DTX_OFF"], + ["MC_7_1", 128000, "7_1", "DTX_OFF"], ["MC_7_1_4", 256000, "5_1", "DTX_OFF"], # SBA ["HOA3", 64000, "BINAURAL", "DTX_OFF"], ["FOA", 256000, "BINAURAL_ROOM_REVERB", "DTX_OFF"], - ["HOA2", 80000, "EXT", "DTX_OFF"], + ["HOA2", 80000, "HOA2", "DTX_OFF"], ["HOA3", 512000, "5_1_4", "DTX_OFF"], ["FOA", 80000, "stereo", "DTX_OFF"], # OMASA @@ -64,11 +64,11 @@ TESTCASES = [ ["OMASA_ISM2", 24400, "MONO", "DTX_OFF"], ["OMASA_ISM3", 80000, "7_1_4", "DTX_OFF"], ["OMASA_ISM4", 64000, "HOA3", "DTX_OFF"], - ["OMASA_ISM2", 32000, "EXT", "DTX_OFF"], + # ["OMASA_ISM2", 32000, "EXT", "DTX_OFF"], # OSBA ["OSBA_ISM2_HOA2", 64000, "BINAURAL_ROOM_IR", "DTX_OFF"], - ["OSBA_ISM4_FOA", 512000, "BINAURAL_ROOM_REVERB", "DTX_OFF"], - ["OSBA_ISM3_HOA3", 128000, "EXT", "DTX_OFF"], + # ["OSBA_ISM4_FOA", 512000, "BINAURAL_ROOM_REVERB", "DTX_OFF"], + # ["OSBA_ISM3_HOA3", 128000, "EXT", "DTX_OFF"], ["OSBA_ISM2_HOA3", 96000, "5_1", "DTX_OFF"], ["OSBA_ISM1_HOA2", 32000, "mono", "DTX_OFF"], ] -- GitLab From 0e2f17954e99be323c05d86b1b7de2ad025651a6 Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Fri, 13 Dec 2024 12:26:47 +0100 Subject: [PATCH 002/106] fix ltv basop encoder .prm file --- .../config/self_test_ltv_basop_encoder.prm | 32 +++++++++---------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/scripts/config/self_test_ltv_basop_encoder.prm b/scripts/config/self_test_ltv_basop_encoder.prm index 66f96903d2..22cec5b67e 100644 --- a/scripts/config/self_test_ltv_basop_encoder.prm +++ b/scripts/config/self_test_ltv_basop_encoder.prm @@ -320,35 +320,35 @@ ../IVAS_dec EXT 48 bit testv/ltvOSBA_4ISM_p3OA48c.wav_EXT_512000_48-48.tst // OSBA planar 2OA 4ISM at 512 kbps, 48 kHz in, 48 kHz out, EXT out -../IVAS_cod -ism_sba 4 -2 testv/ltv48_OSBA_1ISM_HOA3_ISM1.csv testv/ltv48_OSBA_2ISM_HOA3_ISM2.csv testv/ltv48_OSBA_3ISM_HOA3_ISM3.csv testv/ltv48_OSBA_4ISM_HOA3_ISM4.csv 512000 48 testv/ltv48_OSBA_4ISM_HOA2.wav bit +../IVAS_cod -ism_sba 4 -2 testv/ltv48_OSBA_1ISM_HOA2_ISM1.csv testv/ltv48_OSBA_2ISM_HOA2_ISM2.csv testv/ltv48_OSBA_3ISM_HOA2_ISM3.csv testv/ltv48_OSBA_4ISM_HOA2_ISM4.csv 512000 48 testv/ltv48_OSBA_4ISM_HOA2.wav bit ../IVAS_dec EXT 48 bit testv/ltvOSBA_4ISM_p3OA48c.wav_EXT_512000_48-48.tst // OSBA FOA 4ISM at 512 kbps, 48kHz in, 48kHz out, EXT out -../IVAS_cod -ism_sba 4 1 testv/ltv48_OSBA_1ISM_HOA3_ISM1.csv testv/ltv48_OSBA_2ISM_HOA3_ISM2.csv testv/ltv48_OSBA_3ISM_HOA3_ISM3.csv testv/ltv48_OSBA_4ISM_HOA3_ISM4.csv 512000 48 testv/ltv48_OSBA_4ISM_FOA.wav bit +../IVAS_cod -ism_sba 4 1 testv/ltv48_OSBA_1ISM_FOA_ISM1.csv testv/ltv48_OSBA_2ISM_FOA_ISM2.csv testv/ltv48_OSBA_3ISM_FOA_ISM3.csv testv/ltv48_OSBA_4ISM_FOA_ISM4.csv 512000 48 testv/ltv48_OSBA_4ISM_FOA.wav bit ../IVAS_dec EXT 48 bit testv/ltv48_OSBA_4ISM_FOA.wav_EXT_512000_48-48.tst // OSBA FOA 4ISM at 512 kbps, 32kHz in, 48kHz out, EXT out -../IVAS_cod -ism_sba 4 1 testv/ltv48_OSBA_1ISM_HOA3_ISM1.csv testv/ltv48_OSBA_2ISM_HOA3_ISM2.csv testv/ltv48_OSBA_3ISM_HOA3_ISM3.csv testv/ltv48_OSBA_4ISM_HOA3_ISM4.csv 512000 32 testv/ltv32_OSBA_4ISM_FOA.wav bit +../IVAS_cod -ism_sba 4 1 testv/ltv48_OSBA_1ISM_FOA_ISM1.csv testv/ltv48_OSBA_2ISM_FOA_ISM2.csv testv/ltv48_OSBA_3ISM_FOA_ISM3.csv testv/ltv48_OSBA_4ISM_FOA_ISM4.csv 512000 32 testv/ltv32_OSBA_4ISM_FOA.wav bit ../IVAS_dec EXT 48 bit testv/ltv32_OSBA_4ISM_FOA.wav_EXT_512000_32-48.tst // OSBA FOA 4ISM at 384 kbps, 32kHz in, 32kHz out, EXT out -../IVAS_cod -ism_sba 4 1 testv/ltv48_OSBA_1ISM_HOA3_ISM1.csv testv/ltv48_OSBA_2ISM_HOA3_ISM2.csv testv/ltv48_OSBA_3ISM_HOA3_ISM3.csv testv/ltv48_OSBA_4ISM_HOA3_ISM4.csv 384000 32 testv/ltv32_OSBA_4ISM_FOA.wav bit +../IVAS_cod -ism_sba 4 1 testv/ltv48_OSBA_1ISM_FOA_ISM1.csv testv/ltv48_OSBA_2ISM_FOA_ISM2.csv testv/ltv48_OSBA_3ISM_FOA_ISM3.csv testv/ltv48_OSBA_4ISM_FOA_ISM4.csv 384000 32 testv/ltv32_OSBA_4ISM_FOA.wav bit ../IVAS_dec EXT 32 bit testv/ltv32_OSBA_4ISM_FOA.wav_EXT_384000_32-32.tst // OSBA FOA 4ISM at 256 kbps, 48kHz in, 48kHz out, EXT out -../IVAS_cod -ism_sba 4 1 testv/ltv48_OSBA_1ISM_HOA3_ISM1.csv testv/ltv48_OSBA_2ISM_HOA3_ISM2.csv testv/ltv48_OSBA_3ISM_HOA3_ISM3.csv testv/ltv48_OSBA_4ISM_HOA3_ISM4.csv 256000 48 testv/ltv48_OSBA_4ISM_FOA.wav bit +../IVAS_cod -ism_sba 4 1 testv/ltv48_OSBA_1ISM_FOA_ISM1.csv testv/ltv48_OSBA_2ISM_FOA_ISM2.csv testv/ltv48_OSBA_3ISM_FOA_ISM3.csv testv/ltv48_OSBA_4ISM_FOA_ISM4.csv 256000 48 testv/ltv48_OSBA_4ISM_FOA.wav bit ../IVAS_dec EXT 48 bit testv/ltv48_OSBA_4ISM_FOA.wav_EXT_256000_48-48.tst // OSBA FOA 3ISM at 128 kbps, 48kHz in, 48kHz out, EXT out -../IVAS_cod -ism_sba 3 1 testv/ltv48_OSBA_1ISM_HOA3_ISM1.csv testv/ltv48_OSBA_2ISM_HOA3_ISM2.csv testv/ltv48_OSBA_3ISM_HOA3_ISM3.csv 128000 48 testv/ltv48_OSBA_3ISM_FOA.wav bit +../IVAS_cod -ism_sba 3 1 testv/ltv48_OSBA_1ISM_FOA_ISM1.csv testv/ltv48_OSBA_2ISM_FOA_ISM2.csv testv/ltv48_OSBA_3ISM_FOA_ISM3.csv 128000 48 testv/ltv48_OSBA_3ISM_FOA.wav bit ../IVAS_dec EXT 48 bit testv/ltv48_OSBA_3ISM_FOA.wav_EXT_128000_48-48.tst // OSBA FOA 1ISM at 48 kbps, 16kHz in, 16kHz out, EXT out -../IVAS_cod -ism_sba 1 1 testv/ltv48_OSBA_1ISM_HOA3_ISM1.csv 32000 16 testv/ltv16_OSBA_1ISM_FOA.wav bit +../IVAS_cod -ism_sba 1 1 testv/ltv48_OSBA_1ISM_FOA_ISM1.csv 32000 16 testv/ltv16_OSBA_1ISM_FOA.wav bit ../IVAS_dec EXT 16 bit testv/ltv16_OSBA_1ISM_FOA.wav_EXT_32000_16-16.tst // OSBA FOA 1ISM at 32 kbps, 48kHz in, 48kHz out, EXT out -../IVAS_cod -ism_sba 1 1 testv/ltv48_OSBA_1ISM_HOA3_ISM1.csv 32000 48 testv/ltv48_OSBA_1ISM_FOA.wav bit +../IVAS_cod -ism_sba 1 1 testv/ltv48_OSBA_1ISM_FOA_ISM1.csv 32000 48 testv/ltv48_OSBA_1ISM_FOA.wav bit ../IVAS_dec EXT 48 bit testv/ltv48_OSBA_1ISM_FOA.wav_EXT_32000_48-48.tst // OSBA 3OA 4ISM bitrate switching 16.4 to 512, 48kHz in, 48kHz out, EXT out @@ -405,32 +405,32 @@ ../IVAS_dec EXT 32 bit testv/ltv48_OSBA_3ISM_HOA3.wav_EXT_sw_48-32.tst // OSBA 3ISM 2OA at bitrate switching 13.2 to 512 kbps, 48kHz in, 32kHz out, EXT out -../IVAS_cod -ism_sba 3 2 testv/ltv48_OSBA_1ISM_HOA3_ISM1.csv testv/ltv48_OSBA_2ISM_HOA3_ISM2.csv testv/ltv48_OSBA_3ISM_HOA3_ISM3.csv ../scripts/switchPaths/sw_13k2_512k.bin 48 testv/ltv48_OSBA_3ISM_HOA2.wav bit +../IVAS_cod -ism_sba 3 2 testv/ltv48_OSBA_1ISM_HOA2_ISM1.csv testv/ltv48_OSBA_2ISM_HOA2_ISM2.csv testv/ltv48_OSBA_3ISM_HOA2_ISM3.csv ../scripts/switchPaths/sw_13k2_512k.bin 48 testv/ltv48_OSBA_3ISM_HOA2.wav bit ../IVAS_dec EXT 32 bit testv/ltv48_OSBA_3ISM_HOA2.wav_EXT_sw_48-32.tst // OSBA 2OA 4ISM at 384 kbps, 48kHz in, 48kHz out, EXT out -../IVAS_cod -ism_sba 4 2 testv/ltv48_OSBA_1ISM_HOA3_ISM1.csv testv/ltv48_OSBA_2ISM_HOA3_ISM2.csv testv/ltv48_OSBA_3ISM_HOA3_ISM3.csv testv/ltv48_OSBA_4ISM_HOA3_ISM4.csv 384000 48 testv/ltv48_OSBA_4ISM_HOA2.wav bit +../IVAS_cod -ism_sba 4 2 testv/ltv48_OSBA_1ISM_HOA2_ISM1.csv testv/ltv48_OSBA_2ISM_HOA2_ISM2.csv testv/ltv48_OSBA_3ISM_HOA2_ISM3.csv testv/ltv48_OSBA_4ISM_HOA2_ISM4.csv 384000 48 testv/ltv48_OSBA_4ISM_HOA2.wav bit ../IVAS_dec EXT 48 bit testv/ltv48_OSBA_4ISM_HOA2.wav_EXT_384000_48-48.tst // OSBA 2OA 3ISM at 96 kbps, 48kHz in, 48kHz out, EXT out -../IVAS_cod -ism_sba 3 2 testv/ltv48_OSBA_1ISM_HOA3_ISM1.csv testv/ltv48_OSBA_2ISM_HOA3_ISM2.csv testv/ltv48_OSBA_3ISM_HOA3_ISM3.csv 96000 48 testv/ltv48_OSBA_3ISM_HOA2.wav bit +../IVAS_cod -ism_sba 3 2 testv/ltv48_OSBA_1ISM_HOA2_ISM1.csv testv/ltv48_OSBA_2ISM_HOA2_ISM2.csv testv/ltv48_OSBA_3ISM_HOA2_ISM3.csv 96000 48 testv/ltv48_OSBA_3ISM_HOA2.wav bit ../IVAS_dec EXT 48 bit testv/ltv48_OSBA_3ISM_HOA2.wav_EXT_96000_48-48.tst // OSBA 2OA 3ISM at 384 kbps, 16kHz in, 16kHz out, EXT out -../IVAS_cod -ism_sba 3 2 testv/ltv48_OSBA_1ISM_HOA3_ISM1.csv testv/ltv48_OSBA_2ISM_HOA3_ISM2.csv testv/ltv48_OSBA_3ISM_HOA3_ISM3.csv 384000 16 testv/ltv16_OSBA_3ISM_HOA2.wav bit +../IVAS_cod -ism_sba 3 2 testv/ltv48_OSBA_1ISM_HOA2_ISM1.csv testv/ltv48_OSBA_2ISM_HOA2_ISM2.csv testv/ltv48_OSBA_3ISM_HOA2_ISM3.csv 384000 16 testv/ltv16_OSBA_3ISM_HOA2.wav bit ../IVAS_dec EXT 16 bit testv/ltv16_OSBA_3ISM_HOA2.wav_EXT56000_16-16.tst // OSBA 2OA 2ISM at 64 kbps, 32kHz in, 16kHz out, EXT out -../IVAS_cod -ism_sba 2 2 testv/ltv48_OSBA_1ISM_HOA3_ISM1.csv testv/ltv48_OSBA_2ISM_HOA3_ISM2.csv 48000 32 testv/ltv32_OSBA_2ISM_HOA2.wav bit +../IVAS_cod -ism_sba 2 2 testv/ltv48_OSBA_1ISM_HOA2_ISM1.csv testv/ltv48_OSBA_2ISM_HOA2_ISM2.csv 48000 32 testv/ltv32_OSBA_2ISM_HOA2.wav bit ../IVAS_dec EXT 16 bit testv/ltv32_OSBA_2ISM_HOA2.wav_EXT_64000_32-16.tst // OSBA 2OA 2ISM at 48 kbps, 48kHz in, 48kHz out, EXT out -../IVAS_cod -ism_sba 2 2 testv/ltv48_OSBA_1ISM_HOA3_ISM1.csv testv/ltv48_OSBA_2ISM_HOA3_ISM2.csv 48000 48 testv/ltv48_OSBA_2ISM_HOA2.wav bit +../IVAS_cod -ism_sba 2 2 testv/ltv48_OSBA_1ISM_HOA2_ISM1.csv testv/ltv48_OSBA_2ISM_HOA2_ISM2.csv 48000 48 testv/ltv48_OSBA_2ISM_HOA2.wav bit ../IVAS_dec EXT 48 bit testv/ltv48_OSBA_2ISM_HOA2.wav_EXT_48000_48-48.tst // OSBA 2OA 1ISM at 24.4 kbps, 48kHz in, 48kHz out, EXT out -../IVAS_cod -ism_sba 1 2 testv/ltv48_OSBA_1ISM_HOA3_ISM1.csv 24400 48 testv/ltv48_OSBA_1ISM_EXT.wav bit -../IVAS_dec EXT 48 bit testv/ltv48_OSBA_1ISM_EXT.wav_EXT_24400_48-48.tst +../IVAS_cod -ism_sba 1 2 testv/ltv48_OSBA_1ISM_HOA2_ISM1.csv 24400 48 testv/ltv48_OSBA_1ISM_HOA2.wav bit +../IVAS_dec EXT 48 bit testv/ltv48_OSBA_1ISM_HOA2.wav_EXT_24400_48-48.tst // OMASA 2TC 4ISM at br sw techs 13.2 to 512 kbps start 384 kbps, 48kHz in, 48kHz out, BINAURAL out ../IVAS_cod -ism_masa 4 2 testv/ltv48_OMASA_1ISM_2TC_ISM1.csv testv/ltv48_OMASA_2ISM_2TC_ISM2.csv testv/ltv48_OMASA_3ISM_2TC_ISM3.csv testv/ltv48_OMASA_4ISM_2TC_ISM4.csv testv/ltv48_OMASA_4ISM_2TC.met ../scripts/switchPaths/sw_13k2_512k_2fr_start_384k_omasatechs_4ism.bin 48 testv/ltv48_OMASA_4ISM_2TC.wav bit -- GitLab From 6e3204d94a77e84b0632b5cfac90d49622997ff2 Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Fri, 13 Dec 2024 14:23:34 +0100 Subject: [PATCH 003/106] comment out problematic testcase there is an issue with ivas-float-update that needs to be fixed first --- scripts/config/self_test_ltv_basop_encoder.prm | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/config/self_test_ltv_basop_encoder.prm b/scripts/config/self_test_ltv_basop_encoder.prm index 22cec5b67e..ca20519cb2 100644 --- a/scripts/config/self_test_ltv_basop_encoder.prm +++ b/scripts/config/self_test_ltv_basop_encoder.prm @@ -276,8 +276,8 @@ ../IVAS_dec HOA3 48 bit testv/ltv48_HOA3.wav_sw_48-48_HOA3.tst // SBA 3OA bitrate switching from 13.2 kbps to 128 kbps, 48kHz in, 48kHz out, DTX on, HOA3 out -../IVAS_cod -dtx -sba 3 ../scripts/switchPaths/sw_13k2_to_128k_10fr.bin 48 testv/ltv48_HOA3.wav bit -../IVAS_dec HOA3 48 bit testv/ltv48_HOA3.wav_sw_48-48_DTX_HOA3.tst +//../IVAS_cod -dtx -sba 3 ../scripts/switchPaths/sw_13k2_to_128k_10fr.bin 48 testv/ltv48_HOA3.wav bit +//../IVAS_dec HOA3 48 bit testv/ltv48_HOA3.wav_sw_48-48_DTX_HOA3.tst // SBA 3OA bitrate switching from 13.2 kbps to 128 kbps, 32kHz in, 32kHz out, DTX on, HOA3 out ../IVAS_cod -dtx -sba 3 ../scripts/switchPaths/sw_13k2_to_128k_10fr.bin 32 testv/ltv32_HOA3.wav bit -- GitLab From 65c6d89216307ec1728657a883c7a33f922d9027 Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Wed, 8 Jan 2025 16:49:41 +0100 Subject: [PATCH 004/106] add parsing of command lines into the check_for_changes script --- .../basop_check_for_changes_in_testcases.py | 87 ++++++++++++++++++- 1 file changed, 83 insertions(+), 4 deletions(-) diff --git a/scripts/basop_check_for_changes_in_testcases.py b/scripts/basop_check_for_changes_in_testcases.py index e4116cb76d..0942bbb537 100644 --- a/scripts/basop_check_for_changes_in_testcases.py +++ b/scripts/basop_check_for_changes_in_testcases.py @@ -35,6 +35,9 @@ import argparse import sys import os import pathlib +import re +import xml.etree.ElementTree as ET +from typing import Tuple # set positive threshold for "lower is better" metrics, negative for "higher is better" @@ -48,8 +51,12 @@ COLS_2_THRESHOLDS = { OUTFILE_CRASHES = "changes_crashes.csv" OUTFILE_SCORES = "changes_{}.csv" +PATTERN_ENC = r"... encoder command:\s*(.*)\s*... encoder stdout:" +PATTERN_DEC = r"... decoder command:\s*(.*)\s*... decoder stdout:" + def main(args): + xml_report = args.xml_report df_curr = pd.read_csv(args.csv_current, sep=";") df_prev = pd.read_csv(args.csv_previous, sep=";") df_merged = pd.merge(df_curr, df_prev, on="testcase", suffixes=["-curr", "-prev"]) @@ -77,7 +84,6 @@ def main(args): df_crashes_introduced = df_merged[mask_crash_introduced][display_cols].reset_index( drop=True ) - df_crashes_introduced.to_csv(OUTFILE_CRASHES, sep=";") if sum(mask_crash_introduced) > 0: regressions_found = True @@ -85,15 +91,32 @@ def main(args): print(df_crashes_introduced) print() + if xml_report is not None: + cmdlines_crashes_introduced = get_command_lines_for_testcases( + df_crashes_introduced["testcase"], xml_report + ) + df_crashes_introduced = pd.merge( + df_crashes_introduced, cmdlines_crashes_introduced, on="testcase" + ) + df_crashes_introduced.to_csv(OUTFILE_CRASHES, sep=";") + if args.show_improvements and sum(mask_crash_fixed) > 0: df_crashes_fixed = df_merged[mask_crash_fixed][display_cols].reset_index( drop=True ) - df_crashes_fixed.to_csv(OUTFILE_CRASHES, mode="a", sep=";") print("---------------Testcases that fixed crashes---------------") print(df_crashes_fixed) print() + if xml_report is not None: + cmdlines_crashes_fixed = get_command_lines_for_testcases( + df_crashes_fixed["testcase"], xml_report + ) + df_crashes_fixed = pd.merge( + df_crashes_fixed, cmdlines_crashes_fixed, on="testcase" + ) + df_crashes_fixed.to_csv(OUTFILE_CRASHES, mode="a", sep=";") + # remove columns with ERRORs in any of the csv files before comparing the numerical columns mask_no_errors = (df_merged[col_curr] != "ERROR") & (df_merged[col_prev] != "ERROR") df_merged = df_merged[mask_no_errors].reset_index(drop=True) @@ -115,7 +138,6 @@ def main(args): display_cols = ["testcase", col_curr, col_prev, col_diff] outfile = OUTFILE_SCORES.format(col.replace(" ", "_")) df_worse = df_merged[mask_worse][display_cols].reset_index(drop=True) - df_worse.to_csv(outfile, sep=";") if sum(mask_worse) > 0: regressions_found = True print( @@ -124,18 +146,70 @@ def main(args): print(df_worse) print() + if xml_report is not None: + cmdlines_worse = get_command_lines_for_testcases( + df_worse["testcase"], xml_report + ) + df_worse = pd.merge(df_worse, cmdlines_worse, on="testcase") + df_worse.to_csv(outfile, sep=";") + if args.show_improvements and sum(mask_better) > 0: df_better = df_merged[mask_better][display_cols].reset_index(drop=True) - df_better.to_csv(outfile, mode="a", sep=";") print( f"---------------Testcases that got better wrt to {col}---------------" ) print(df_better) print() + if xml_report is not None: + cmdlines_better = get_command_lines_for_testcases( + df_better["testcase"], xml_report + ) + df_better = pd.merge(df_better, cmdlines_better, on="testcase") + df_better.to_csv(outfile, mode="a", sep=";") + return int(regressions_found) +def get_command_lines_for_testcases( + testcases: pd.Series, xml_report: pathlib.Path +) -> pd.DataFrame: + testcase_elems = [ + e + for _, e in ET.iterparse(xml_report) + if e.tag == "testcase" and e.attrib["name"] in testcases.values + ] + + cmdlines = {"testcase": [], "enc_cmd": [], "dec_cmd": []} + for elem in testcase_elems: + testcase_name = elem.attrib["name"] + enc_cmd = "" + dec_cmd = "" + if (system_out := elem.find("system-out")) is not None: + enc_cmd, dec_cmd = extract_cmdlines(system_out.text) + + cmdlines["testcase"].append(testcase_name) + cmdlines["enc_cmd"].append(enc_cmd) + cmdlines["dec_cmd"].append(dec_cmd) + + return pd.DataFrame(cmdlines) + + +def extract_cmdlines(text: str) -> Tuple[str, str]: + enc_cmdline = "" + dec_cmdline = "" + + match_enc = re.search(PATTERN_ENC, text) + match_dec = re.search(PATTERN_DEC, text) + if match_enc is not None and match_dec is not None: + enc_cmdline = match_enc.group(1) + dec_cmdline = match_dec.group(1) + + # TODO: post-process paths + + return enc_cmdline, dec_cmdline + + if __name__ == "__main__": parser = argparse.ArgumentParser() parser.add_argument("csv_current") @@ -147,6 +221,11 @@ if __name__ == "__main__": default=COLS_2_THRESHOLDS.keys(), ) parser.add_argument("--show_improvements", action="store_true") + parser.add_argument( + "--xml_report", + help="XMLxml_report report file from pytest run. Pass to add command lines to the output files.", + default=None, + ) args = parser.parse_args() sys.exit(main(args)) -- GitLab From 36824cdea534df54cc4115ec214b8a03614ac7dc Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Wed, 8 Jan 2025 17:23:59 +0100 Subject: [PATCH 005/106] normalize paths in command lines --- .../basop_check_for_changes_in_testcases.py | 48 +++++++++++++++---- 1 file changed, 40 insertions(+), 8 deletions(-) diff --git a/scripts/basop_check_for_changes_in_testcases.py b/scripts/basop_check_for_changes_in_testcases.py index 0942bbb537..5d904fee2d 100644 --- a/scripts/basop_check_for_changes_in_testcases.py +++ b/scripts/basop_check_for_changes_in_testcases.py @@ -93,7 +93,7 @@ def main(args): if xml_report is not None: cmdlines_crashes_introduced = get_command_lines_for_testcases( - df_crashes_introduced["testcase"], xml_report + df_crashes_introduced["testcase"], xml_report, args.inject_cwd ) df_crashes_introduced = pd.merge( df_crashes_introduced, cmdlines_crashes_introduced, on="testcase" @@ -110,7 +110,7 @@ def main(args): if xml_report is not None: cmdlines_crashes_fixed = get_command_lines_for_testcases( - df_crashes_fixed["testcase"], xml_report + df_crashes_fixed["testcase"], xml_report, args.inject_cwd ) df_crashes_fixed = pd.merge( df_crashes_fixed, cmdlines_crashes_fixed, on="testcase" @@ -148,7 +148,7 @@ def main(args): if xml_report is not None: cmdlines_worse = get_command_lines_for_testcases( - df_worse["testcase"], xml_report + df_worse["testcase"], xml_report, args.inject_cwd ) df_worse = pd.merge(df_worse, cmdlines_worse, on="testcase") df_worse.to_csv(outfile, sep=";") @@ -163,7 +163,7 @@ def main(args): if xml_report is not None: cmdlines_better = get_command_lines_for_testcases( - df_better["testcase"], xml_report + df_better["testcase"], xml_report, args.inject_cwd ) df_better = pd.merge(df_better, cmdlines_better, on="testcase") df_better.to_csv(outfile, mode="a", sep=";") @@ -172,7 +172,7 @@ def main(args): def get_command_lines_for_testcases( - testcases: pd.Series, xml_report: pathlib.Path + testcases: pd.Series, xml_report: pathlib.Path, cwd: pathlib.Path ) -> pd.DataFrame: testcase_elems = [ e @@ -186,7 +186,7 @@ def get_command_lines_for_testcases( enc_cmd = "" dec_cmd = "" if (system_out := elem.find("system-out")) is not None: - enc_cmd, dec_cmd = extract_cmdlines(system_out.text) + enc_cmd, dec_cmd = extract_cmdlines(system_out.text, cwd) cmdlines["testcase"].append(testcase_name) cmdlines["enc_cmd"].append(enc_cmd) @@ -195,7 +195,7 @@ def get_command_lines_for_testcases( return pd.DataFrame(cmdlines) -def extract_cmdlines(text: str) -> Tuple[str, str]: +def extract_cmdlines(text: str, cwd: pathlib.Path) -> Tuple[str, str]: enc_cmdline = "" dec_cmdline = "" @@ -205,11 +205,37 @@ def extract_cmdlines(text: str) -> Tuple[str, str]: enc_cmdline = match_enc.group(1) dec_cmdline = match_dec.group(1) - # TODO: post-process paths + enc_cmdline = postprocess_cmdline(enc_cmdline, cwd) + dec_cmdline = postprocess_cmdline(dec_cmdline, cwd) return enc_cmdline, dec_cmdline +def postprocess_cmdline(cmdline: str, cwd: pathlib.Path) -> str: + cmdline_split = cmdline.split() + cmdline_proc = [] + + # change absolute paths into relative ones + # remove the "quite" flag + # for output and bitstream files only keep the filename + for elem in cmdline_split: + print(elem) + if elem == "-q": + continue + elif (elem_as_path := pathlib.Path(elem)).is_absolute(): + if elem_as_path.suffix == ".192" or ( + elem_as_path.suffix == ".wav" + and cmdline_split.index(elem) == len(cmdline_split) - 1 + ): + cmdline_proc.append(elem_as_path.name) + else: + cmdline_proc.append(str(elem_as_path.relative_to(cwd))) + else: + cmdline_proc.append(elem) + + return " ".join(cmdline_proc) + + if __name__ == "__main__": parser = argparse.ArgumentParser() parser.add_argument("csv_current") @@ -226,6 +252,12 @@ if __name__ == "__main__": help="XMLxml_report report file from pytest run. Pass to add command lines to the output files.", default=None, ) + parser.add_argument( + "--inject_cwd", + help="Use this as cwd when pruning the long paths in the command lines. Debug option for testing.", + default=pathlib.Path(__file__).parent.absolute(), + type=pathlib.Path, + ) args = parser.parse_args() sys.exit(main(args)) -- GitLab From 366f249569a6ed229e174a6090fe36a532399702 Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Thu, 9 Jan 2025 11:29:59 +0100 Subject: [PATCH 006/106] remove leftover print from debugging --- scripts/basop_check_for_changes_in_testcases.py | 1 - 1 file changed, 1 deletion(-) diff --git a/scripts/basop_check_for_changes_in_testcases.py b/scripts/basop_check_for_changes_in_testcases.py index 5d904fee2d..ed67aa279d 100644 --- a/scripts/basop_check_for_changes_in_testcases.py +++ b/scripts/basop_check_for_changes_in_testcases.py @@ -219,7 +219,6 @@ def postprocess_cmdline(cmdline: str, cwd: pathlib.Path) -> str: # remove the "quite" flag # for output and bitstream files only keep the filename for elem in cmdline_split: - print(elem) if elem == "-q": continue elif (elem_as_path := pathlib.Path(elem)).is_absolute(): -- GitLab From 0696cb4a9e3ca61cc7347f369fc69b4bae9f1f05 Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Mon, 13 Jan 2025 17:11:33 +0100 Subject: [PATCH 007/106] add capturing of eid-xor and network sim cmdlines --- .../basop_check_for_changes_in_testcases.py | 46 +++++++++++++------ 1 file changed, 31 insertions(+), 15 deletions(-) diff --git a/scripts/basop_check_for_changes_in_testcases.py b/scripts/basop_check_for_changes_in_testcases.py index ed67aa279d..c7c2e0f785 100644 --- a/scripts/basop_check_for_changes_in_testcases.py +++ b/scripts/basop_check_for_changes_in_testcases.py @@ -53,6 +53,10 @@ OUTFILE_SCORES = "changes_{}.csv" PATTERN_ENC = r"... encoder command:\s*(.*)\s*... encoder stdout:" PATTERN_DEC = r"... decoder command:\s*(.*)\s*... decoder stdout:" +PATTERN_EID = r"eid-xor command:\s*(.*)\s*" +PATTERN_NETSIM = r"netsim command:\s*(.*)\s*" + +PATTERNS = [PATTERN_ENC, PATTERN_DEC, PATTERN_EID, PATTERN_NETSIM] def main(args): @@ -180,35 +184,47 @@ def get_command_lines_for_testcases( if e.tag == "testcase" and e.attrib["name"] in testcases.values ] - cmdlines = {"testcase": [], "enc_cmd": [], "dec_cmd": []} + cmdlines = { + "testcase": [], + "enc_cmd": [], + "dec_cmd": [], + "eid-xor_cmd": [], + "netsim_cmd": [], + } for elem in testcase_elems: testcase_name = elem.attrib["name"] enc_cmd = "" dec_cmd = "" + eid_cmd = "" + netsim_cmd = "" if (system_out := elem.find("system-out")) is not None: - enc_cmd, dec_cmd = extract_cmdlines(system_out.text, cwd) + ( + enc_cmd, + dec_cmd, + eid_cmd, + netsim_cmd, + ) = extract_cmdlines(system_out.text, cwd) cmdlines["testcase"].append(testcase_name) cmdlines["enc_cmd"].append(enc_cmd) cmdlines["dec_cmd"].append(dec_cmd) + cmdlines["eid-xor_cmd"].append(eid_cmd) + cmdlines["netsim_cmd"].append(netsim_cmd) return pd.DataFrame(cmdlines) -def extract_cmdlines(text: str, cwd: pathlib.Path) -> Tuple[str, str]: - enc_cmdline = "" - dec_cmdline = "" - - match_enc = re.search(PATTERN_ENC, text) - match_dec = re.search(PATTERN_DEC, text) - if match_enc is not None and match_dec is not None: - enc_cmdline = match_enc.group(1) - dec_cmdline = match_dec.group(1) - - enc_cmdline = postprocess_cmdline(enc_cmdline, cwd) - dec_cmdline = postprocess_cmdline(dec_cmdline, cwd) +def extract_cmdlines(text: str, cwd: pathlib.Path) -> list[str]: + cmdlines = [] + for p in PATTERNS: + m = re.search(p, text) + if m is not None: + cmdline = postprocess_cmdline(m.group(1), cwd) + cmdlines.append(cmdline) + else: + cmdlines.append("") - return enc_cmdline, dec_cmdline + return cmdlines def postprocess_cmdline(cmdline: str, cwd: pathlib.Path) -> str: -- GitLab From d004971f25e2ad2fd9ad0731450a4da55c5d921d Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Tue, 21 Jan 2025 13:15:04 +0100 Subject: [PATCH 008/106] correct default value for cwd in basop_check_for_changes_in_testcases.py --- scripts/basop_check_for_changes_in_testcases.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/basop_check_for_changes_in_testcases.py b/scripts/basop_check_for_changes_in_testcases.py index c7c2e0f785..b47f0d6cc2 100644 --- a/scripts/basop_check_for_changes_in_testcases.py +++ b/scripts/basop_check_for_changes_in_testcases.py @@ -270,7 +270,7 @@ if __name__ == "__main__": parser.add_argument( "--inject_cwd", help="Use this as cwd when pruning the long paths in the command lines. Debug option for testing.", - default=pathlib.Path(__file__).parent.absolute(), + default=pathlib.Path(os.getcwd()).absolute(), type=pathlib.Path, ) -- GitLab From a5c4bb901bcb5f7a047746c31e910f571371d14e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fabian=20M=C3=BCller?= Date: Wed, 29 Jan 2025 12:12:01 +0100 Subject: [PATCH 009/106] Add script to filter out smoke test items broken in basop --- scripts/smoketest-basop-filter.py | 53 +++++++++++++++++++++++++++++++ 1 file changed, 53 insertions(+) create mode 100644 scripts/smoketest-basop-filter.py diff --git a/scripts/smoketest-basop-filter.py b/scripts/smoketest-basop-filter.py new file mode 100644 index 0000000000..faa170f7e0 --- /dev/null +++ b/scripts/smoketest-basop-filter.py @@ -0,0 +1,53 @@ +#! /usr/bin/env python3 + +import argparse +import json +import logging +import sys + + +logging.basicConfig(level=logging.INFO, format='%(levelname)s: %(message)s') +logger = logging.getLogger() + + +def main(): + parser = argparse.ArgumentParser() + + parser.add_argument("input", help="input JSON", type=argparse.FileType('r')) + parser.add_argument("output", help="output JSON", type=argparse.FileType('w'), nargs="?") + parser.add_argument("--inline", "-i", help="patch input inline", action="store_true") + + args = parser.parse_args() + + if not args.inline and not args.output: + logging.error("You must specify an output file or use --inline") + return 1 + + modes = json.load(args.input) + + for mode in ("MC", "stereo", "OMASA"): + for encoder_config_name in modes[mode].keys(): + try: + del modes[mode][encoder_config_name]["dec"]["EXT"] + except KeyError: + logger.warning(f"No EXT dec in {mode} -> {encoder_config_name}, skipping") + continue + else: + logger.info(f"Found EXT dec in {mode} -> {encoder_config_name}, removing") + + for encoder_config_name in tuple(modes["OSBA"].keys()): + if not "FOA" in encoder_config_name: + continue + + logger.info(f"Removing OSBA FOA encoder config {encoder_config_name}") + del modes["OSBA"][encoder_config_name] + + if args.inline: + with open(args.input.name, "w") as f: + json.dump(modes, f, indent=4) + else: + json.dump(modes, args.output, indent=4) + + +if __name__ == "__main__": + sys.exit(main()) -- GitLab From 79e6dbad08720b20202745888eb52be09e8ec4f0 Mon Sep 17 00:00:00 2001 From: Archit Tamarapu Date: Tue, 28 Jan 2025 11:23:09 +0100 Subject: [PATCH 010/106] [fix] missing files for conformance test --- .gitlab-ci.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 9d3a624b20..416b86f3fc 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1460,6 +1460,7 @@ ivas-conformance: - mkdir testvec/bin - cp -force -ErrorAction Ignore scripts/testv/* testvec/testv - cp -r -force -ErrorAction Ignore scripts/ls_layouts testvec + - cp -r -force -ErrorAction Ignore scripts/object_edit testvec - cp -r -force -ErrorAction Ignore scripts/switchPaths testvec - cp -r -force -ErrorAction Ignore scripts/trajectories testvec - cp -r -force -ErrorAction Ignore scripts/binauralRenderer_interface/binaural_renderers_hrtf_data testvec/binauralRenderer_interface @@ -1546,6 +1547,7 @@ ivas-conformance-linux: - mkdir testvec/bin - cp -r scripts/testv/* testvec/testv - cp -r scripts/ls_layouts testvec + - cp -r scripts/object_edit testvec - cp -r scripts/switchPaths testvec - cp -r scripts/trajectories testvec - cp -r scripts/binauralRenderer_interface/binaural_renderers_hrtf_data testvec/binauralRenderer_interface -- GitLab From 7ba0706d6efb311df72a22dcf31bd961d377417d Mon Sep 17 00:00:00 2001 From: Lauros Pajunen Date: Tue, 28 Jan 2025 15:04:31 +0200 Subject: [PATCH 011/106] Update search keyword for WMOPS total line in logs --- scripts/pyivastest/IvasModeAnalyzer.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/pyivastest/IvasModeAnalyzer.py b/scripts/pyivastest/IvasModeAnalyzer.py index 8d1dd2ddf0..91bf0b44ae 100644 --- a/scripts/pyivastest/IvasModeAnalyzer.py +++ b/scripts/pyivastest/IvasModeAnalyzer.py @@ -43,7 +43,7 @@ from pyivastest.IvasModeCollector import IvasModeCollector INSTRUMENTED_RESULTS = { "WMOPS": { - "keyword": "total", + "keyword": " total ", "number_format": "{:.5g}", "position": 2, "max_or_add": "add", -- GitLab From 426e2c174bbbb670d177b1c37c8d09c4680376c1 Mon Sep 17 00:00:00 2001 From: Vladimir Malenovsky Date: Wed, 29 Jan 2025 17:12:48 +0100 Subject: [PATCH 012/106] fix if() condition, prevent crash when CPE is NULL --- lib_enc/ivas_core_pre_proc_front.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/lib_enc/ivas_core_pre_proc_front.c b/lib_enc/ivas_core_pre_proc_front.c index 36da36c128..4d43fce471 100644 --- a/lib_enc/ivas_core_pre_proc_front.c +++ b/lib_enc/ivas_core_pre_proc_front.c @@ -222,8 +222,9 @@ ivas_error pre_proc_front_ivas( } #ifdef DEBUG_MODE_INFO - if ( !( hCPE->hStereoTD != NULL && n > 0 ) ) + if ( !( hCPE != NULL && hCPE->hStereoTD != NULL && n > 0 ) ) { + /* for TD stereo only write out first channel. The existence of a second channel can vary, this is just easier to handle */ int16_t tmp_dmx_in[L_FRAME48k]; mvr2s( signal_in - NS2SA( st->input_Fs, ACELP_LOOK_NS ), tmp_dmx_in, input_frame ); dbgwrite( tmp_dmx_in, sizeof( int16_t ), input_frame, 1, strcat( fname( debug_dir, "ivas_input_dmx", 0, n + 1, ENC ), ".pcm" ) ); -- GitLab From 283001f6bbd21dbd29dc785ef8e56f0d5d2953d6 Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Thu, 30 Jan 2025 13:14:03 +0100 Subject: [PATCH 013/106] update remove_unsupported_testcases.py --- ci/remove_unsupported_testcases.py | 20 +++++++++++++++++++- 1 file changed, 19 insertions(+), 1 deletion(-) diff --git a/ci/remove_unsupported_testcases.py b/ci/remove_unsupported_testcases.py index b316a417e5..a1995b6d04 100644 --- a/ci/remove_unsupported_testcases.py +++ b/ci/remove_unsupported_testcases.py @@ -66,10 +66,28 @@ TESTCASES = [ "MASA 1TC at 256kbps, 48kHz in, 48 kHz out, BINAURAL_ROOM_REVERB out custom configuration", "MASA 1TC at 256 kbps, 48kHz in, 48kHz out, BINAURAL_ROOM_REVERB out, HR custom configuration", "OMASA 2TC 4ISM at br sw techs 13.2 to 512 kbps start 80 kbps, 48kHz in, 48kHz out, EXT out", + # object editing starts here + "OMASA 2Dir2TC 4ISM at 80 kbps, 48kHz in, 48kHz out, BINAURAL out, default object editing, 1SEP-PARAM", + "OMASA 2Dir2TC 4ISM at 256 kbps, 48kHz in, 48kHz out, BINAURAL out, object editing, JBM Prof 5, DISC", + "OMASA 2Dir2TC 2ISM at 96 kbps, 48kHz in, 48kHz out, FOA out, object editing, JBM Prof 5, DISC", + "OMASA 2Dir2TC 2ISM br sw techs 13.2 to 512 kbps start 48 kbps, 48kHz in, 48kHz out, BINAURAL out, object editing", + # the next four lines are basically duplicates of the last four, but without "Dir", because the ltv MASA cases do not have dedicated testvectors for numbers of directions + "OMASA 2TC 4ISM at 80 kbps, 48kHz in, 48kHz out, BINAURAL out, default object editing, 1SEP-PARAM", + "OMASA 2TC 4ISM at 256 kbps, 48kHz in, 48kHz out, BINAURAL out, object editing, JBM Prof 5, DISC", + "OMASA 2TC 2ISM at 96 kbps, 48kHz in, 48kHz out, FOA out, object editing, JBM Prof 5, DISC", + "OMASA 2TC 2ISM br sw techs 13.2 to 512 kbps start 48 kbps, 48kHz in, 48kHz out, BINAURAL out, object editing", + "OSBA 3OA 4ISM at 256 kbps, 48kHz in, 48kHz out, BINAURAL out, object editing, DISC", + "OSBA 2OA 3ISM at 128 kbps, 48kHz in, 48kHz out, FOA out, object editing, JBM Prof 5, DISC", + "OSBA 2OA 3ISM at bitrate switching 13.2 to 512 kbps, 48kHz in, 48kHz out, BINAURAL out, object editing", + "4 ISM with metadata at 32 kbps, 48 kHz in, 48 kHz out, BINAURAL out, object editing, PARAM_ISM", + "4 ISM with metadata at 128 kbps, 48 kHz in, 48 kHz out, BINAURAL out, object editing, DISC", + "3 ISM with metadata at 24.4 kbps, 48 kHz in, 48 kHz out, FOA out, object editing, JBM Prof 5, PARAM_ISM", + "3 ISM with metadata at 384 kbps, 48 kHz in, 48 kHz out, FOA out, object editing, JBM Prof 5, DISC", + "4 ISM with metadata bitrate switching from 32 kbps to 48 kbps, 48 kHz in, 48 kHz out, BINAURAL_ROOM_IR out, object editing", + "4 ISM with metadata bitrate switching from 48 kbps to 32 kbps, 48 kHz in, 48 kHz out, BINAURAL out, object editing, JBM Prof 5", ] - def remove_testcases(cfg: Path, testcases: list): """ Go through file line by line and copy all testcases except the given ones -- GitLab From c05306e832004a095fae908910a3ed9ae0b77385 Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Fri, 31 Jan 2025 15:48:24 +0100 Subject: [PATCH 014/106] run smoke test with filtered config in MRs to basop-ci-branch --- .gitlab-ci.yml | 101 ++++++++++++++++++++++++++++++------------------- 1 file changed, 63 insertions(+), 38 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 416b86f3fc..6d0d8bf458 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -132,6 +132,12 @@ stages: - sed -i.bak -e "s/\/\*\ *\(#define\ *DEBUGGING\ *\)\*\//\1/g" lib_com/options.h - sed -i.bak -e "s/\/\/\ *\(#define\ *DEBUGGING\ *\)/\1/g" lib_com/options.h +.get-basop-float-reference: &get-basop-float-reference + - git clone -b $BASOP_REFERENCE_BRANCH https://forge.3gpp.org/rep/sa4/audio/ivas-basop.git --single-branch + - make -j -C ivas-basop + - cp ivas-basop/IVAS_cod ./ + - cp ivas-basop/IVAS_dec ./ + .merge-request-comparison-setup-codec: &merge-request-comparison-setup-codec ### build test binaries, initial clean for paranoia reasons - *disable-debugging-macro @@ -279,10 +285,24 @@ stages: when: never - when: on_success -.rules-merge-request: +.rules-merge-request-to-main: + extends: .rules-basis + rules: + - if: $CI_PIPELINE_SOURCE == 'merge_request_event' && $CI_MERGE_REQUEST_TARGET_BRANCH_NAME == "main" + - if: $CI_PIPELINE_SOURCE == 'push' + when: never + +.rules-merge-request-to-basop-ci-branch: + extends: .rules-basis + rules: + - if: $CI_PIPELINE_SOURCE == 'merge_request_event' && $CI_MERGE_REQUEST_TARGET_BRANCH_NAME == "basop-ci-branch" + - if: $CI_PIPELINE_SOURCE == 'push' + when: never + +.rules-merge-request-to-basop-ci-branch-or-main: extends: .rules-basis rules: - - if: $CI_PIPELINE_SOURCE == 'merge_request_event' && $CI_MERGE_REQUEST_TARGET_BRANCH_NAME == "main" # only have MR pipelines for MRs to main + - if: $CI_PIPELINE_SOURCE == 'merge_request_event' && ($CI_MERGE_REQUEST_TARGET_BRANCH_NAME == "basop-ci-branch" || $CI_MERGE_REQUEST_TARGET_BRANCH_NAME == "main") - if: $CI_PIPELINE_SOURCE == 'push' when: never @@ -327,7 +347,7 @@ stages: stage: test extends: - .test-job-linux-needs-testv-dir - - .rules-merge-request + - .rules-merge-request-to-main artifacts: name: "mr-$CI_MERGE_REQUEST_IID--sha-$CI_COMMIT_SHORT_SHA--stage-$CI_JOB_STAGE--results" expire_in: 1 week @@ -396,7 +416,7 @@ uninterruptible: branch-is-up-to-date-with-main-pre: extends: - - .rules-merge-request + - .rules-merge-request-to-main stage: prevalidate needs: [] tags: @@ -408,7 +428,7 @@ branch-is-up-to-date-with-main-pre: check-self-test-names-pre: extends: - - .rules-merge-request + - .rules-merge-request-to-main stage: prevalidate needs: [] tags: @@ -418,7 +438,7 @@ check-self-test-names-pre: check-no-duplicates-in-self-tests: extends: - - .rules-merge-request + - .rules-merge-request-to-main stage: prevalidate needs: [] tags: @@ -428,7 +448,7 @@ check-no-duplicates-in-self-tests: branch-is-up-to-date-with-main-post: extends: - - .rules-merge-request + - .rules-merge-request-to-main stage: postvalidate tags: - ivas-linux @@ -440,24 +460,19 @@ branch-is-up-to-date-with-main-post: .basop-ci-branch-compat-template: extends: - .test-job-linux - rules: - - if: $CI_PIPELINE_SOURCE == 'merge_request_event' && $CI_MERGE_REQUEST_TARGET_BRANCH_NAME == "basop-ci-branch" - changes: - - tests/**/* - - scripts/**/* - - if: $CI_PIPELINE_SOURCE == 'push' - when: never + - .rules-merge-request-to-basop-ci-branch tags: - ivas-linux-fast before_script: - - git clone -b $BASOP_REFERENCE_BRANCH https://forge.3gpp.org/rep/sa4/audio/ivas-basop.git --single-branch - - make -j -C ivas-basop - python3 ci/remove_unsupported_testcases.py $PARAM_FILE - python3 tests/create_short_testvectors.py + - *update-ltv-repo - *copy-ltv-files-to-testv-dir - - python3 -m pytest tests/codec_be_on_mr_nonselection --param_file $PARAM_FILE --update_ref 1 --ref_encoder_path ivas-basop/IVAS_cod --ref_decoder_path ivas-basop/IVAS_dec --html=report.html --self-contained-html --junit-xml=report-junit.xml || true + - *get-basop-float-reference + + - python3 -m pytest tests/codec_be_on_mr_nonselection --param_file $PARAM_FILE --update_ref 1 --ref_encoder_path IVAS_cod --ref_decoder_path IVAS_dec --html=report.html --self-contained-html --junit-xml=report-junit.xml || true - zero_errors=$(cat report-junit.xml | grep -c 'errors="0"') || true - if [ $zero_errors != 1 ]; then echo "Run errors encountered with $PARAM_FILE !"; exit $EXIT_CODE_FAIL; fi @@ -579,7 +594,7 @@ build-codec-windows-msbuild: codec-smoke-test: extends: - .test-job-linux-needs-testv-dir - - .rules-merge-request + - .rules-merge-request-to-basop-ci-branch-or-main timeout: "20 minutes" tags: - ivas-linux-fast @@ -589,7 +604,17 @@ codec-smoke-test: - *print-common-info # LTV update needed as ltv ISM metadata files are used - *update-ltv-repo - - bash ci/smoke_test.sh + + # for MRs to basop-ci-branch, we want to test with the BASOP float reference build + # per default, the smoke test builds the current repo again with WMOPS activated to catch unbalanced instrumentation macros + - if [ "$CI_MERGE_REQUEST_TARGET_BRANCH_NAME" = "basop-ci-branch" ]; then + - python3 scripts/smoketest-basop-filter.py scripts/config/ivas_modes.json --inline + - *get-basop-float-reference + - bash ci/smoke_test.sh coverage + - else + - bash ci/smoke_test.sh + - fi + ### analyze for failures - if ! [ -s smoke_test_output.txt ] || ! [ -s smoke_test_output_jbm.txt ] || ! [ -s smoke_test_output_hrtf.txt ]; then echo "Error in smoke test"; exit 1; fi - ret_val=0 @@ -648,7 +673,7 @@ codec-usan: pytest-compare-20ms-and-5ms-rendering: extends: - .test-job-linux - - .rules-merge-request + - .rules-merge-request-to-main stage: test needs: ["build-codec-linux-cmake", "build-codec-linux-make", "build-codec-instrumented-linux", "build-codec-sanitizers-linux"] script: @@ -695,7 +720,7 @@ pytest-compare-20ms-and-5ms-rendering: renderer-smoke-test: extends: - .test-job-linux - - .rules-merge-request + - .rules-merge-request-to-main needs: ["build-codec-linux-make"] stage: test script: @@ -717,7 +742,7 @@ renderer-smoke-test: renderer-asan: extends: - .test-job-linux - - .rules-merge-request + - .rules-merge-request-to-main needs: ["build-codec-linux-cmake"] stage: test script: @@ -741,7 +766,7 @@ renderer-asan: renderer-msan: extends: - .test-job-linux - - .rules-merge-request + - .rules-merge-request-to-main needs: ["build-codec-linux-cmake"] stage: test script: @@ -765,7 +790,7 @@ renderer-msan: renderer-usan: extends: - .test-job-linux - - .rules-merge-request + - .rules-merge-request-to-main needs: ["build-codec-linux-cmake"] stage: test script: @@ -793,7 +818,7 @@ renderer-usan: renderer-pytest-on-merge-request: extends: - .test-job-linux-needs-testv-dir - - .rules-merge-request + - .rules-merge-request-to-main needs: ["build-codec-linux-make"] # TODO: set reasonable timeout, will most likely take less timeout: "20 minutes" @@ -845,7 +870,7 @@ renderer-pytest-on-merge-request: split-rendering-smoke-test: extends: - .test-job-linux - - .rules-merge-request + - .rules-merge-request-to-main needs: ["build-codec-linux-make"] stage: test script: @@ -866,7 +891,7 @@ split-rendering-smoke-test: lc3-wrapper-unit-test: extends: - .test-job-linux - - .rules-merge-request + - .rules-merge-request-to-main needs: ["build-codec-linux-cmake"] stage: test script: @@ -878,7 +903,7 @@ lc3-wrapper-unit-test: split-rendering-pytest-on-merge-request: extends: - .test-job-linux-needs-testv-dir - - .rules-merge-request + - .rules-merge-request-to-main needs: ["build-codec-linux-make"] # TODO: set reasonable timeout, will most likely take less timeout: "30 minutes" @@ -950,7 +975,7 @@ split-rendering-pytest-on-merge-request: ivas-pytest-on-merge-request: extends: - .test-job-linux-needs-testv-dir - - .rules-merge-request + - .rules-merge-request-to-main stage: compare needs: ["build-codec-linux-cmake", "codec-smoke-test"] timeout: "14 minutes" @@ -1004,7 +1029,7 @@ ivas-pytest-on-merge-request: ivas-interop-on-merge-request: extends: - .test-job-linux-needs-testv-dir - - .rules-merge-request + - .rules-merge-request-to-main stage: test needs: ["build-codec-linux-cmake"] timeout: "10 minutes" @@ -1053,7 +1078,7 @@ ivas-interop-on-merge-request: evs-pytest-on-merge-request: extends: - .test-job-linux - - .rules-merge-request + - .rules-merge-request-to-main stage: compare needs: ["build-codec-linux-cmake", "codec-smoke-test"] timeout: "10 minutes" @@ -1105,7 +1130,7 @@ evs-pytest-on-merge-request: voip-be-on-merge-request: extends: - .test-job-linux-needs-testv-dir - - .rules-merge-request + - .rules-merge-request-to-main stage: test needs: ["build-codec-linux-make"] timeout: "10 minutes" @@ -1118,7 +1143,7 @@ voip-be-on-merge-request: clang-format-check: extends: - .test-job-linux - - .rules-merge-request + - .rules-merge-request-to-main variables: ARTIFACT_BASE_NAME: "mr-$CI_MERGE_REQUEST_IID--sha-$CI_COMMIT_SHORT_SHA--formatting-fix" stage: prevalidate @@ -1165,7 +1190,7 @@ clang-format-check: check-first-frame-is-sid: extends: - .test-job-linux-needs-testv-dir - - .rules-merge-request + - .rules-merge-request-to-main tags: - ivas-linux stage: test @@ -1192,7 +1217,7 @@ check-first-frame-is-sid: .lc3plus-ensure-no-code-changes: extends: - .test-job-linux - - .rules-merge-request + - .rules-merge-request-to-main stage: postvalidate needs: [] timeout: "5 minutes" @@ -1207,7 +1232,7 @@ check-first-frame-is-sid: check-bitexactness-hrtf-rom-and-file: extends: - .test-job-linux - - .rules-merge-request + - .rules-merge-request-to-main stage: test needs: ["build-codec-linux-cmake"] timeout: "5 minutes" @@ -1229,7 +1254,7 @@ check-bitexactness-hrtf-rom-and-file: check-bitexactness-ext-and-transport-format: extends: - .test-job-linux - - .rules-merge-request + - .rules-merge-request-to-main stage: test needs: ["build-codec-linux-cmake"] timeout: "5 minutes" @@ -1256,7 +1281,7 @@ check-bitexactness-ext-and-transport-format: # check bitexactness to EVS windows binaries be-2-evs-windows: extends: - - .rules-merge-request + - .rules-merge-request-to-main tags: - ivas-windows stage: test -- GitLab From d685256d153a14d65e1c1d2f031609207860bcbb Mon Sep 17 00:00:00 2001 From: Erik Norvell Date: Wed, 5 Feb 2025 19:26:01 +0100 Subject: [PATCH 015/106] Add explicit trigger of prepare_combined_format_inputs.py --- .gitlab-ci.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 6d0d8bf458..4f53b30fcd 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -466,6 +466,7 @@ branch-is-up-to-date-with-main-post: before_script: - python3 ci/remove_unsupported_testcases.py $PARAM_FILE - python3 tests/create_short_testvectors.py + - python3 scripts/prepare_combined_format_inputs.py - *update-ltv-repo - *copy-ltv-files-to-testv-dir -- GitLab From 9509479a99c904fb2cd68fd99214e66b8fc77350 Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Thu, 6 Feb 2025 15:45:38 +0100 Subject: [PATCH 016/106] add ubsan suppression file for basop --- scripts/ubsan_basop.supp | 35 +++++++++++++++++++++++++++++++++++ 1 file changed, 35 insertions(+) create mode 100644 scripts/ubsan_basop.supp diff --git a/scripts/ubsan_basop.supp b/scripts/ubsan_basop.supp new file mode 100644 index 0000000000..b836305c7f --- /dev/null +++ b/scripts/ubsan_basop.supp @@ -0,0 +1,35 @@ +# From the scheduled sanitizer tests with self_test.py +# lib_com +implicit-integer-sign-change:lib_com/ACcontextMapping_fx.c +implicit-integer-sign-change:lib_com/bitstream_fx.c +implicit-integer-sign-change:lib_com/enh64.c +implicit-signed-integer-truncation:lib_com/enh1632.c +implicit-signed-integer-truncation:lib_com/enhUL32.c +invalid-shift-base:lib_com/enh64.c +invalid-shift-exponent:lib_com/basop32.c +invalid-shift-exponent:lib_com/enh64.c +out-of-bounds-index:lib_com/ifft_rel.c +out-of-bounds-index:lib_com/log2.c +out-of-bounds-index:lib_com/tcx_ltp_fx.c +out-of-bounds-index:lib_com/trans_direct_fx.c +out-of-bounds-index:lib_com/trans_inv_fx.c +signed-integer-overflow:lib_com/enh64.c +# lib_enc +implicit-integer-sign-change:lib_enc/ACcontextMapping_enc_fx.c +implicit-integer-sign-change:lib_enc/bass_psfilter_enc_fx.c +implicit-integer-sign-change:lib_enc/cod4t64_fx.c +implicit-integer-sign-change:lib_enc/core_enc_ol_fx.c +implicit-integer-sign-change:lib_enc/enc_acelp_fx.c +implicit-integer-sign-change:lib_enc/enc_prm_fx.c +implicit-integer-sign-change:lib_enc/hq_classifier_enc_fx.c +implicit-integer-sign-change:lib_enc/inov_enc_fx.c +implicit-integer-sign-change:lib_enc/lead_indexing_fx.c +implicit-integer-sign-change:lib_enc/range_enc_fx.c +implicit-integer-sign-change:lib_enc/tcx_utils_enc_fx.c +out-of-bounds-index:lib_enc/enc_acelp_fx.c +out-of-bounds-index:lib_enc/enc_gain_fx.c +out-of-bounds-index:lib_enc/pit_enc_fx.c +out-of-bounds-index:lib_enc/pitch_ol2_fx.c +out-of-bounds-index:lib_enc/tcx_ltp_enc_fx.c +# lib_dec +out-of-bounds-index:lib_dec/dec_acelp_fx.c -- GitLab From eb249c2fa46027966f125cb6e8dcfaa29c7460a2 Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Fri, 7 Feb 2025 09:57:46 +0100 Subject: [PATCH 017/106] fix basop usan supp file --- scripts/ubsan_basop.supp | 65 +++++++++++++++++++--------------------- 1 file changed, 31 insertions(+), 34 deletions(-) diff --git a/scripts/ubsan_basop.supp b/scripts/ubsan_basop.supp index b836305c7f..bfd00f3573 100644 --- a/scripts/ubsan_basop.supp +++ b/scripts/ubsan_basop.supp @@ -1,35 +1,32 @@ # From the scheduled sanitizer tests with self_test.py -# lib_com -implicit-integer-sign-change:lib_com/ACcontextMapping_fx.c -implicit-integer-sign-change:lib_com/bitstream_fx.c -implicit-integer-sign-change:lib_com/enh64.c -implicit-signed-integer-truncation:lib_com/enh1632.c -implicit-signed-integer-truncation:lib_com/enhUL32.c -invalid-shift-base:lib_com/enh64.c -invalid-shift-exponent:lib_com/basop32.c -invalid-shift-exponent:lib_com/enh64.c -out-of-bounds-index:lib_com/ifft_rel.c -out-of-bounds-index:lib_com/log2.c -out-of-bounds-index:lib_com/tcx_ltp_fx.c -out-of-bounds-index:lib_com/trans_direct_fx.c -out-of-bounds-index:lib_com/trans_inv_fx.c -signed-integer-overflow:lib_com/enh64.c -# lib_enc -implicit-integer-sign-change:lib_enc/ACcontextMapping_enc_fx.c -implicit-integer-sign-change:lib_enc/bass_psfilter_enc_fx.c -implicit-integer-sign-change:lib_enc/cod4t64_fx.c -implicit-integer-sign-change:lib_enc/core_enc_ol_fx.c -implicit-integer-sign-change:lib_enc/enc_acelp_fx.c -implicit-integer-sign-change:lib_enc/enc_prm_fx.c -implicit-integer-sign-change:lib_enc/hq_classifier_enc_fx.c -implicit-integer-sign-change:lib_enc/inov_enc_fx.c -implicit-integer-sign-change:lib_enc/lead_indexing_fx.c -implicit-integer-sign-change:lib_enc/range_enc_fx.c -implicit-integer-sign-change:lib_enc/tcx_utils_enc_fx.c -out-of-bounds-index:lib_enc/enc_acelp_fx.c -out-of-bounds-index:lib_enc/enc_gain_fx.c -out-of-bounds-index:lib_enc/pit_enc_fx.c -out-of-bounds-index:lib_enc/pitch_ol2_fx.c -out-of-bounds-index:lib_enc/tcx_ltp_enc_fx.c -# lib_dec -out-of-bounds-index:lib_dec/dec_acelp_fx.c +implicit-integer-sign-change:ACcontextMapping_fx.c +implicit-integer-sign-change:bitstream_fx.c +implicit-integer-sign-change:enh64.c +implicit-signed-integer-truncation:enh1632.c +implicit-signed-integer-truncation:enhUL32.c +shift-base:enh64.c +shift-exponent:basop32.c +shift-exponent:enh64.c +bounds:ifft_rel.c +bounds:log2.c +bounds:tcx_ltp_fx.c +bounds:trans_direct_fx.c +bounds:trans_inv_fx.c +signed-integer-overflow:enh64.c +implicit-integer-sign-change:ACcontextMapping_enc_fx.c +implicit-integer-sign-change:bass_psfilter_enc_fx.c +implicit-integer-sign-change:cod4t64_fx.c +implicit-integer-sign-change:core_enc_ol_fx.c +implicit-integer-sign-change:enc_acelp_fx.c +implicit-integer-sign-change:enc_prm_fx.c +implicit-integer-sign-change:hq_classifier_enc_fx.c +implicit-integer-sign-change:inov_enc_fx.c +implicit-integer-sign-change:lead_indexing_fx.c +implicit-integer-sign-change:range_enc_fx.c +implicit-integer-sign-change:tcx_utils_enc_fx.c +bounds:enc_acelp_fx.c +bounds:enc_gain_fx.c +bounds:pit_enc_fx.c +bounds:pitch_ol2_fx.c +bounds:tcx_ltp_enc_fx.c +bounds:dec_acelp_fx.c -- GitLab From 3c1d1312a8fe571ec5693ea95978f1fb86540002 Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Mon, 10 Feb 2025 10:44:09 +0100 Subject: [PATCH 018/106] do not use bash strict mode for getWmops.sh could be fixed also by using "|| retval=$?" on the IvasBuildAndRunChecks call, but better stay save to not loose complexity graphs --- ci/complexity_measurements/getWmops.sh | 2 -- 1 file changed, 2 deletions(-) diff --git a/ci/complexity_measurements/getWmops.sh b/ci/complexity_measurements/getWmops.sh index 3f49630530..750c2c299b 100755 --- a/ci/complexity_measurements/getWmops.sh +++ b/ci/complexity_measurements/getWmops.sh @@ -28,8 +28,6 @@ # accordance with the laws of the Federal Republic of Germany excluding its conflict of law rules and # the United Nations Convention on Contracts on the International Sales of Goods. -set -euxo pipefail - function usage { echo "Usage: $0 \"ivas-format(s)\" \"output-format(s)\" \"mode{full(default)|mem_only}\" \"repo{float(default)|basop}\" [wmops_ref_logfile]" exit 1 -- GitLab From f06d238c077bcbaeabbe541813129146e1dbfb24 Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Mon, 10 Feb 2025 14:28:04 +0100 Subject: [PATCH 019/106] add new job that tests BASOP ref compatibility with --compare_enc_dmx --- .gitlab-ci.yml | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index f27836fc82..efaf5c7b1b 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -31,6 +31,7 @@ variables: TESTCASE_TIMEOUT_LTV_SANITIZERS: 2400 BASOP_REFERENCE_BRANCH: "ivas-float-update" SCALE_FACTOR: "3.162" + PYTEST_ARGS: "" default: @@ -473,7 +474,7 @@ branch-is-up-to-date-with-main-post: - *get-basop-float-reference - - python3 -m pytest tests/codec_be_on_mr_nonselection --param_file $PARAM_FILE --update_ref 1 --ref_encoder_path IVAS_cod --ref_decoder_path IVAS_dec --html=report.html --self-contained-html --junit-xml=report-junit.xml || true + - python3 -m pytest tests/codec_be_on_mr_nonselection $PYTEST_ARGS --param_file $PARAM_FILE --update_ref 1 --ref_encoder_path IVAS_cod --ref_decoder_path IVAS_dec --html=report.html --self-contained-html --junit-xml=report-junit.xml || true - zero_errors=$(cat report-junit.xml | grep -c 'errors="0"') || true - if [ $zero_errors != 1 ]; then echo "Run errors encountered with $PARAM_FILE !"; exit $EXIT_CODE_FAIL; fi @@ -522,6 +523,16 @@ check-compatibility-with-basop-reference-branch-encoder-ltv: script: - exit 0 +check-compatibility-with-basop-reference-branch-encoder-dmx-comparison: + extends: + - .basop-ci-branch-compat-template + variables: + PARAM_FILE: scripts/config/self_test_basop_encoder.prm + # USING PYTEST_ADDOPTS env var did not work for some reason when testing locally - maybe because this is a custom option + PYTEST_ARGS: "--compare_dmx_enc" + script: + - exit 0 + # --------------------------------------------------------------- # Build jobs # --------------------------------------------------------------- -- GitLab From 3bdb33b5b59743eefcb90b2244d30075974666ec Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Mon, 10 Feb 2025 14:29:52 +0100 Subject: [PATCH 020/106] shorten names for convenience in web pipeline view --- .gitlab-ci.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index efaf5c7b1b..3f0283ba1e 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -491,7 +491,7 @@ branch-is-up-to-date-with-main-post: junit: - report-junit.xml -check-compatibility-with-basop-reference-branch-stv: +basop-ref-compat-stv: extends: - .basop-ci-branch-compat-template variables: @@ -499,7 +499,7 @@ check-compatibility-with-basop-reference-branch-stv: script: - exit 0 -check-compatibility-with-basop-reference-branch-ltv: +basop-ref-compat-ltv: extends: - .basop-ci-branch-compat-template variables: @@ -507,7 +507,7 @@ check-compatibility-with-basop-reference-branch-ltv: script: - exit 0 -check-compatibility-with-basop-reference-branch-encoder-stv: +basop-ref-compat-encoder-stv: extends: - .basop-ci-branch-compat-template variables: @@ -515,7 +515,7 @@ check-compatibility-with-basop-reference-branch-encoder-stv: script: - exit 0 -check-compatibility-with-basop-reference-branch-encoder-ltv: +basop-ref-compat-encoder-ltv: extends: - .basop-ci-branch-compat-template variables: @@ -523,7 +523,7 @@ check-compatibility-with-basop-reference-branch-encoder-ltv: script: - exit 0 -check-compatibility-with-basop-reference-branch-encoder-dmx-comparison: +basop-ref-compat-encoder-dmx-comp: extends: - .basop-ci-branch-compat-template variables: -- GitLab From 4751b1ee7c1bd6aa7bda9e8ed32f7a1968a9743d Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Mon, 10 Feb 2025 14:47:15 +0100 Subject: [PATCH 021/106] correct pytest argument --- .gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 3f0283ba1e..36b589383f 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -529,7 +529,7 @@ basop-ref-compat-encoder-dmx-comp: variables: PARAM_FILE: scripts/config/self_test_basop_encoder.prm # USING PYTEST_ADDOPTS env var did not work for some reason when testing locally - maybe because this is a custom option - PYTEST_ARGS: "--compare_dmx_enc" + PYTEST_ARGS: "--compare_enc_dmx" script: - exit 0 -- GitLab From f4cae5563189a87b604cd09d93551484e86fe55d Mon Sep 17 00:00:00 2001 From: rtyag Date: Tue, 18 Feb 2025 18:44:08 +1100 Subject: [PATCH 022/106] instrumentation fix for split rendering --- scripts/prepare_instrumentation.sh | 24 +++++++++++++++++------- 1 file changed, 17 insertions(+), 7 deletions(-) diff --git a/scripts/prepare_instrumentation.sh b/scripts/prepare_instrumentation.sh index 4ed7098442..35bbfe4420 100755 --- a/scripts/prepare_instrumentation.sh +++ b/scripts/prepare_instrumentation.sh @@ -160,6 +160,12 @@ if [ $? -ne 0 ]; then exit -1 fi +# Floating point code has lib_lc3plus/fft folder while BASOP code does not +fftdir="" +if [ -d $targetdir/lib_lc3plus/fft ]; then + fftdir="$targetdir/lib_lc3plus/fft/*.c" +fi + # strip switches, to remove the macros (turn on extended globing to allow !(pattern*) matching) shopt -s extglob if coan_exists; then @@ -171,7 +177,9 @@ if coan_exists; then if [ "$PROJECT" = "FLOAT" ]; then coan source --replace --no-transients -E -K --file $ifdef_list $targetdir/lib_{com,dec,enc,isar,rend,util,debug}/!(wmc_auto*).[hc] coan source --replace --no-transients -E -K --file $ifdef_list $targetdir/lib_lc3plus/!(wmc_auto*).[hc] - coan source --replace --no-transients -E -K --file $ifdef_list $targetdir/lib_lc3plus/fft/!(wmc_auto*).[hc] + if [ -n "$fftdir" ];then + coan source --replace --no-transients -E -K --file $ifdef_list $targetdir/lib_lc3plus/fft/!(wmc_auto*).[hc] + fi else # same as first call from if, but without "isar" and "debug" to avoid coan warning coan source --replace --no-transients -E -K --file $ifdef_list $targetdir/lib_{com,dec,enc,rend,util}/!(wmc_auto*).[hc] @@ -185,18 +193,20 @@ shopt -u extglob find $targetdir -name "*.[ch]" -exec sed -i.bak -e "s/\(0x[0-9a-fA-F]*\)UL/\(\(unsigned long\)\1\)/" \{\} \; # run wmc_tool -"tools/$system/wmc_tool" $wmc_opt -m "$targetdir/apps/encoder.c" "$targetdir/lib_enc/*.c" "$targetdir/lib_com/*.c" >> /dev/null -"tools/$system/wmc_tool" $wmc_opt -m "$targetdir/apps/decoder.c" "$targetdir/lib_dec/*.c" "$targetdir/lib_rend/*.c" >> /dev/null +"tools/$system/wmc_tool" $wmc_opt -m "$targetdir/apps/encoder.c" "$targetdir/lib_enc/*.c" "$targetdir/lib_com/*.c" >> output.txt 2>&1 +"tools/$system/wmc_tool" $wmc_opt -m "$targetdir/apps/decoder.c" "$targetdir/lib_dec/*.c" "$targetdir/lib_rend/*.c" >> output.txt 2>&1 # ISAR post-renderer and lc3plus sources only need to be instrumented in float code if [ "$PROJECT" = "FLOAT" ]; then for bak_file in $targetdir/lib_rend/*.bak; do mv "$bak_file" "${bak_file%.*}"; done # restore fresh .c files to avoid time-consuming des-instrumentation of files by the WMC tool - "tools/$system/wmc_tool" $wmc_opt -m "$targetdir/apps/renderer.c" "$targetdir/lib_rend/*.c" "$targetdir/lib_lc3plus/*.c" "$targetdir/lib_lc3plus/fft/*.c" >> /dev/null + "tools/$system/wmc_tool" $wmc_opt -m "$targetdir/apps/renderer.c" "$targetdir/lib_rend/*.c" "$targetdir/lib_lc3plus/*.c" "$fftdir" >> output.txt 2>&1 for bak_file in $targetdir/lib_lc3plus/*.bak; do mv "$bak_file" "${bak_file%.*}"; done # restore fresh .c files to avoid time-consuming des-instrumentation of files by the WMC tool - for bak_file in $targetdir/lib_lc3plus/fft/*.bak; do mv "$bak_file" "${bak_file%.*}"; done # restore fresh .c files to avoid time-consuming des-instrumentation of files by the WMC tool - "tools/$system/wmc_tool" $wmc_opt -m "$targetdir/apps/isar_post_rend.c" "$targetdir/lib_isar/*.c" "$targetdir/lib_lc3plus/*.c" "$targetdir/lib_lc3plus/fft/*.c" >> /dev/null + if [ -n "$fftdir" ];then + for bak_file in $targetdir/lib_lc3plus/fft/*.bak; do mv "$bak_file" "${bak_file%.*}"; done # restore fresh .c files to avoid time-consuming des-instrumentation of files by the WMC tool + fi + "tools/$system/wmc_tool" $wmc_opt -m "$targetdir/apps/isar_post_rend.c" "$targetdir/lib_isar/*.c" "$targetdir/lib_lc3plus/*.c" "$fftdir" >> output.txt 2>&1 else for bak_file in $targetdir/lib_rend/*.bak; do mv "$bak_file" "${bak_file%.*}"; done # restore fresh .c files to avoid time-consuming des-instrumentation of files by the WMC tool - "tools/$system/wmc_tool" $wmc_opt -m "$targetdir/apps/renderer.c" "$targetdir/lib_rend/*.c" >> /dev/null + "tools/$system/wmc_tool" $wmc_opt -m "$targetdir/apps/renderer.c" "$targetdir/lib_rend/*.c" >> output.txt 2>&1 fi # automatically enable #define WMOPS in options.h -- GitLab From d964a0a0c68b1afbfaec3d23fad4d85f71745053 Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Thu, 20 Feb 2025 10:48:04 +0100 Subject: [PATCH 023/106] add script for creating ignorelist for ubsan ignore all non-ivas_* files in ubsan for now --- scripts/basop_create_ignorelist_for_ubsan.py | 9 +++++++++ 1 file changed, 9 insertions(+) create mode 100644 scripts/basop_create_ignorelist_for_ubsan.py diff --git a/scripts/basop_create_ignorelist_for_ubsan.py b/scripts/basop_create_ignorelist_for_ubsan.py new file mode 100644 index 0000000000..7369fc6c9f --- /dev/null +++ b/scripts/basop_create_ignorelist_for_ubsan.py @@ -0,0 +1,9 @@ +import pathlib + +all_c_files = pathlib.Path(".").glob("lib_*/*.c") +ivas_c_files = pathlib.Path(".").glob("lib_*/ivas*.c") +non_ivas_c_files = sorted(set(all_c_files) - set(ivas_c_files)) + +with open("ubsan_ignorelist.txt", "w") as f: + for cfile in non_ivas_c_files: + print(f"src:{cfile}", file=f) -- GitLab From 03cb598a2de0074840ba10ef2069d3267fd6247f Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Thu, 20 Feb 2025 14:23:45 +0100 Subject: [PATCH 024/106] also add header files to ignorelist e.g. enh40.h contains function code --- scripts/basop_create_ignorelist_for_ubsan.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/basop_create_ignorelist_for_ubsan.py b/scripts/basop_create_ignorelist_for_ubsan.py index 7369fc6c9f..e6dfdfa075 100644 --- a/scripts/basop_create_ignorelist_for_ubsan.py +++ b/scripts/basop_create_ignorelist_for_ubsan.py @@ -1,7 +1,7 @@ import pathlib -all_c_files = pathlib.Path(".").glob("lib_*/*.c") -ivas_c_files = pathlib.Path(".").glob("lib_*/ivas*.c") +all_c_files = pathlib.Path(".").glob("lib_*/*.[ch]") +ivas_c_files = pathlib.Path(".").glob("lib_*/ivas*.[ch]") non_ivas_c_files = sorted(set(all_c_files) - set(ivas_c_files)) with open("ubsan_ignorelist.txt", "w") as f: -- GitLab From 183c425143626d2602c27150ac4fb20926b4fe10 Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Mon, 24 Feb 2025 12:41:54 +0100 Subject: [PATCH 025/106] add script for getting the ref branch of a basop port branch --- ci/get_float_ref_branch_name.py | 59 +++++++++++++++++++++++++++++++++ 1 file changed, 59 insertions(+) create mode 100644 ci/get_float_ref_branch_name.py diff --git a/ci/get_float_ref_branch_name.py b/ci/get_float_ref_branch_name.py new file mode 100644 index 0000000000..b8da8979e7 --- /dev/null +++ b/ci/get_float_ref_branch_name.py @@ -0,0 +1,59 @@ +#!/usr/bin/env python3 +""" +(C) 2022-2025 IVAS codec Public Collaboration with portions copyright Dolby International AB, Ericsson AB, +Fraunhofer-Gesellschaft zur Foerderung der angewandten Forschung e.V., Huawei Technologies Co. LTD., +Koninklijke Philips N.V., Nippon Telegraph and Telephone Corporation, Nokia Technologies Oy, Orange, +Panasonic Holdings Corporation, Qualcomm Technologies, Inc., VoiceAge Corporation, and other +contributors to this repository. All Rights Reserved. + +This software is protected by copyright law and by international treaties. +The IVAS codec Public Collaboration consisting of Dolby International AB, Ericsson AB, +Fraunhofer-Gesellschaft zur Foerderung der angewandten Forschung e.V., Huawei Technologies Co. LTD., +Koninklijke Philips N.V., Nippon Telegraph and Telephone Corporation, Nokia Technologies Oy, Orange, +Panasonic Holdings Corporation, Qualcomm Technologies, Inc., VoiceAge Corporation, and other +contributors to this repository retain full ownership rights in their respective contributions in +the software. This notice grants no license of any kind, including but not limited to patent +license, nor is any license granted by implication, estoppel or otherwise. + +Contributors are required to enter into the IVAS codec Public Collaboration agreement before making +contributions. + +This software is provided "AS IS", without any express or implied warranties. The software is in the +development stage. It is intended exclusively for experts who have experience with such software and +solely for the purpose of inspection. All implied warranties of non-infringement, merchantability +and fitness for a particular purpose are hereby disclaimed and excluded. + +Any dispute, controversy or claim arising under or in relation to providing this software shall be +submitted to and settled by the final, binding jurisdiction of the courts of Munich, Germany in +accordance with the laws of the Federal Republic of Germany excluding its conflict of law rules and +the United Nations Convention on Contracts on the International Sales of Goods. +""" + +import re +import sys +import argparse + +BASOP_PATTERN = r"\d+[_-]basop[_-].*" + +MSG_DOES_NOT_MATCH = """Your branch name {branchname} does not match the template '_basop_', e.g. '123_basop_fix_this_one_bug-2'. +Sticking to this branch is needed for the testing system to match this branch with its float-reference counterpart. +Please rename your branch. You can easily do this by creating a new branch from this branch: + - git checkout -b +You then also need to create a new merge request and update the links in your issue. +See here for details on the porting work process: https://forge.3gpp.org/rep/sa4/audio/ivas-basop/-/wikis/Porting-MRs-from-floating-point-codec#workflow-for-porting-a-merge-request-from-floating-point-codec-to-basop-codec +""" + +parser = argparse.ArgumentParser() +parser.add_argument("branchname") +args = parser.parse_args() +branchname = args.branchname + +re_match = re.match(BASOP_PATTERN, branchname) +if re_match is None: + print(MSG_DOES_NOT_MATCH.format(branchname=branchname)) + sys.exit(1) + +# the float ref branch is just the same name, but with basop -> ref replacement +# replace only the first occurrence, as "basop" mit be present in the later description +# if the format is correct, then before "_basop", only numbers can occur +print(branchname.replace("basop", "ref", 1)) -- GitLab From 7247d074c01462138b878b1339b8ccdf39902a7d Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Tue, 25 Feb 2025 11:07:07 +0100 Subject: [PATCH 026/106] extend script + turn into bash script --- ci/get_float_ref_branch_name.py | 59 ------------------------------ ci/get_float_ref_branch_name.sh | 65 +++++++++++++++++++++++++++++++++ 2 files changed, 65 insertions(+), 59 deletions(-) delete mode 100644 ci/get_float_ref_branch_name.py create mode 100644 ci/get_float_ref_branch_name.sh diff --git a/ci/get_float_ref_branch_name.py b/ci/get_float_ref_branch_name.py deleted file mode 100644 index b8da8979e7..0000000000 --- a/ci/get_float_ref_branch_name.py +++ /dev/null @@ -1,59 +0,0 @@ -#!/usr/bin/env python3 -""" -(C) 2022-2025 IVAS codec Public Collaboration with portions copyright Dolby International AB, Ericsson AB, -Fraunhofer-Gesellschaft zur Foerderung der angewandten Forschung e.V., Huawei Technologies Co. LTD., -Koninklijke Philips N.V., Nippon Telegraph and Telephone Corporation, Nokia Technologies Oy, Orange, -Panasonic Holdings Corporation, Qualcomm Technologies, Inc., VoiceAge Corporation, and other -contributors to this repository. All Rights Reserved. - -This software is protected by copyright law and by international treaties. -The IVAS codec Public Collaboration consisting of Dolby International AB, Ericsson AB, -Fraunhofer-Gesellschaft zur Foerderung der angewandten Forschung e.V., Huawei Technologies Co. LTD., -Koninklijke Philips N.V., Nippon Telegraph and Telephone Corporation, Nokia Technologies Oy, Orange, -Panasonic Holdings Corporation, Qualcomm Technologies, Inc., VoiceAge Corporation, and other -contributors to this repository retain full ownership rights in their respective contributions in -the software. This notice grants no license of any kind, including but not limited to patent -license, nor is any license granted by implication, estoppel or otherwise. - -Contributors are required to enter into the IVAS codec Public Collaboration agreement before making -contributions. - -This software is provided "AS IS", without any express or implied warranties. The software is in the -development stage. It is intended exclusively for experts who have experience with such software and -solely for the purpose of inspection. All implied warranties of non-infringement, merchantability -and fitness for a particular purpose are hereby disclaimed and excluded. - -Any dispute, controversy or claim arising under or in relation to providing this software shall be -submitted to and settled by the final, binding jurisdiction of the courts of Munich, Germany in -accordance with the laws of the Federal Republic of Germany excluding its conflict of law rules and -the United Nations Convention on Contracts on the International Sales of Goods. -""" - -import re -import sys -import argparse - -BASOP_PATTERN = r"\d+[_-]basop[_-].*" - -MSG_DOES_NOT_MATCH = """Your branch name {branchname} does not match the template '_basop_', e.g. '123_basop_fix_this_one_bug-2'. -Sticking to this branch is needed for the testing system to match this branch with its float-reference counterpart. -Please rename your branch. You can easily do this by creating a new branch from this branch: - - git checkout -b -You then also need to create a new merge request and update the links in your issue. -See here for details on the porting work process: https://forge.3gpp.org/rep/sa4/audio/ivas-basop/-/wikis/Porting-MRs-from-floating-point-codec#workflow-for-porting-a-merge-request-from-floating-point-codec-to-basop-codec -""" - -parser = argparse.ArgumentParser() -parser.add_argument("branchname") -args = parser.parse_args() -branchname = args.branchname - -re_match = re.match(BASOP_PATTERN, branchname) -if re_match is None: - print(MSG_DOES_NOT_MATCH.format(branchname=branchname)) - sys.exit(1) - -# the float ref branch is just the same name, but with basop -> ref replacement -# replace only the first occurrence, as "basop" mit be present in the later description -# if the format is correct, then before "_basop", only numbers can occur -print(branchname.replace("basop", "ref", 1)) diff --git a/ci/get_float_ref_branch_name.sh b/ci/get_float_ref_branch_name.sh new file mode 100644 index 0000000000..0d791b6f5c --- /dev/null +++ b/ci/get_float_ref_branch_name.sh @@ -0,0 +1,65 @@ +#!/bin/bash + +# (C) 2022-2025 IVAS codec Public Collaboration with portions copyright Dolby International AB, Ericsson AB, +# Fraunhofer-Gesellschaft zur Foerderung der angewandten Forschung e.V., Huawei Technologies Co. LTD., +# Koninklijke Philips N.V., Nippon Telegraph and Telephone Corporation, Nokia Technologies Oy, Orange, +# Panasonic Holdings Corporation, Qualcomm Technologies, Inc., VoiceAge Corporation, and other +# contributors to this repository. All Rights Reserved. +# +# This software is protected by copyright law and by international treaties. +# The IVAS codec Public Collaboration consisting of Dolby International AB, Ericsson AB, +# Fraunhofer-Gesellschaft zur Foerderung der angewandten Forschung e.V., Huawei Technologies Co. LTD., +# Koninklijke Philips N.V., Nippon Telegraph and Telephone Corporation, Nokia Technologies Oy, Orange, +# Panasonic Holdings Corporation, Qualcomm Technologies, Inc., VoiceAge Corporation, and other +# contributors to this repository retain full ownership rights in their respective contributions in +# the software. This notice grants no license of any kind, including but not limited to patent +# license, nor is any license granted by implication, estoppel or otherwise. +# +# Contributors are required to enter into the IVAS codec Public Collaboration agreement before making +# contributions. +# +# This software is provided "AS IS", without any express or implied warranties. The software is in the +# development stage. It is intended exclusively for experts who have experience with such software and +# solely for the purpose of inspection. All implied warranties of non-infringement, merchantability +# and fitness for a particular purpose are hereby disclaimed and excluded. +# +# Any dispute, controversy or claim arising under or in relation to providing this software shall be +# submitted to and settled by the final, binding jurisdiction of the courts of Munich, Germany in +# accordance with the laws of the Federal Republic of Germany excluding its conflict of law rules and +# the United Nations Convention on Contracts on the International Sales of Goods. + +BASOP_PATTERN="^[0-9]+[_-]basop[_-].*" +MSG_DOES_NOT_MATCH="Your branch name %s does not match the template '_basop_', e.g. '123_basop_fix_this_one_bug-2'. +Sticking to this branch is needed for the testing system to match this branch with its float-reference counterpart. +Please rename your branch. You can easily do this by creating a new branch from this branch: + - git checkout -b +You then also need to create a new merge request and update the links in your issue. +See here for details on the porting work process: https://forge.3gpp.org/rep/sa4/audio/ivas-basop/-/wikis/Porting-MRs-from-floating-point-codec#workflow-for-porting-a-merge-request-from-floating-point-codec-to-basop-codec +" + +if [ $# -ne 1 ]; then + echo "Usage: $0 " + exit 1 +fi + +branchname="$1" + +if ! [[ "$branchname" =~ $BASOP_PATTERN ]]; then + printf "$MSG_DOES_NOT_MATCH" "$branchname" + exit 1 +fi + +float_ref_branchname="${branchname/basop/ref}" +git_result=$(git branch --list "$float_ref_branchname") + +# If the branch does not exist, default to "float-pc" +if [[ -z "$git_result" ]]; then + branchname="float-pc" +else + # The float ref branch is just the same name, but with basop -> ref replacement + # Replace only the first occurrence, as "basop" may be present in the later description + # If the format is correct, then before "_basop", only numbers can occur + branchname="${float_ref_branchname}" +fi + +echo "$branchname" -- GitLab From d89d577d131ba88c8bb0c2adc88a9fac643f5cfb Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Tue, 25 Feb 2025 12:57:57 +0100 Subject: [PATCH 027/106] make script executable --- ci/get_float_ref_branch_name.sh | 0 1 file changed, 0 insertions(+), 0 deletions(-) mode change 100644 => 100755 ci/get_float_ref_branch_name.sh diff --git a/ci/get_float_ref_branch_name.sh b/ci/get_float_ref_branch_name.sh old mode 100644 new mode 100755 -- GitLab From b99a2e5f599bb210acc4f00766e6ded59be87721 Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Fri, 28 Feb 2025 10:23:58 +0100 Subject: [PATCH 028/106] report regressions and improvements in separate files --- .../basop_check_for_changes_in_testcases.py | 20 +++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/scripts/basop_check_for_changes_in_testcases.py b/scripts/basop_check_for_changes_in_testcases.py index b47f0d6cc2..1ac3dfca18 100644 --- a/scripts/basop_check_for_changes_in_testcases.py +++ b/scripts/basop_check_for_changes_in_testcases.py @@ -37,7 +37,6 @@ import os import pathlib import re import xml.etree.ElementTree as ET -from typing import Tuple # set positive threshold for "lower is better" metrics, negative for "higher is better" @@ -48,8 +47,10 @@ COLS_2_THRESHOLDS = { "MIN_ODG": float(os.environ.get("CI_REGRESSION_THRESH_ODG", -0.05)), } -OUTFILE_CRASHES = "changes_crashes.csv" -OUTFILE_SCORES = "changes_{}.csv" +OUTFILE_CRASHES_FIXED = "improvements_crashes.csv" +OUTFILE_CRASHES_ADDED = "regressions_crashes.csv" +OUTFILE_SCORES_IMPROVEMENTS = "improvements_{}.csv" +OUTFILE_SCORES_REGRESSION = "regressions_{}.csv" PATTERN_ENC = r"... encoder command:\s*(.*)\s*... encoder stdout:" PATTERN_DEC = r"... decoder command:\s*(.*)\s*... decoder stdout:" @@ -102,7 +103,7 @@ def main(args): df_crashes_introduced = pd.merge( df_crashes_introduced, cmdlines_crashes_introduced, on="testcase" ) - df_crashes_introduced.to_csv(OUTFILE_CRASHES, sep=";") + df_crashes_introduced.to_csv(OUTFILE_CRASHES_ADDED, sep=";") if args.show_improvements and sum(mask_crash_fixed) > 0: df_crashes_fixed = df_merged[mask_crash_fixed][display_cols].reset_index( @@ -119,7 +120,7 @@ def main(args): df_crashes_fixed = pd.merge( df_crashes_fixed, cmdlines_crashes_fixed, on="testcase" ) - df_crashes_fixed.to_csv(OUTFILE_CRASHES, mode="a", sep=";") + df_crashes_fixed.to_csv(OUTFILE_CRASHES_FIXED, mode="a", sep=";") # remove columns with ERRORs in any of the csv files before comparing the numerical columns mask_no_errors = (df_merged[col_curr] != "ERROR") & (df_merged[col_prev] != "ERROR") @@ -140,7 +141,7 @@ def main(args): mask_better = (df_merged[col_diff] * fac) < -thresh display_cols = ["testcase", col_curr, col_prev, col_diff] - outfile = OUTFILE_SCORES.format(col.replace(" ", "_")) + outfile_regressions = OUTFILE_SCORES_REGRESSION.format(col.replace(" ", "_")) df_worse = df_merged[mask_worse][display_cols].reset_index(drop=True) if sum(mask_worse) > 0: regressions_found = True @@ -155,7 +156,7 @@ def main(args): df_worse["testcase"], xml_report, args.inject_cwd ) df_worse = pd.merge(df_worse, cmdlines_worse, on="testcase") - df_worse.to_csv(outfile, sep=";") + df_worse.to_csv(outfile_regressions, sep=";") if args.show_improvements and sum(mask_better) > 0: df_better = df_merged[mask_better][display_cols].reset_index(drop=True) @@ -170,7 +171,10 @@ def main(args): df_better["testcase"], xml_report, args.inject_cwd ) df_better = pd.merge(df_better, cmdlines_better, on="testcase") - df_better.to_csv(outfile, mode="a", sep=";") + outfile_improvements = OUTFILE_SCORES_IMPROVEMENTS.format( + col.replace(" ", "_") + ) + df_better.to_csv(outfile_improvements, mode="a", sep=";") return int(regressions_found) -- GitLab From de027e8b9a58935378d96c61edcbafd9cf495aea Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Fri, 28 Feb 2025 10:37:28 +0100 Subject: [PATCH 029/106] remove now obsolete append mode --- scripts/basop_check_for_changes_in_testcases.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/basop_check_for_changes_in_testcases.py b/scripts/basop_check_for_changes_in_testcases.py index 1ac3dfca18..315afd0a88 100644 --- a/scripts/basop_check_for_changes_in_testcases.py +++ b/scripts/basop_check_for_changes_in_testcases.py @@ -120,7 +120,7 @@ def main(args): df_crashes_fixed = pd.merge( df_crashes_fixed, cmdlines_crashes_fixed, on="testcase" ) - df_crashes_fixed.to_csv(OUTFILE_CRASHES_FIXED, mode="a", sep=";") + df_crashes_fixed.to_csv(OUTFILE_CRASHES_FIXED, sep=";") # remove columns with ERRORs in any of the csv files before comparing the numerical columns mask_no_errors = (df_merged[col_curr] != "ERROR") & (df_merged[col_prev] != "ERROR") @@ -174,7 +174,7 @@ def main(args): outfile_improvements = OUTFILE_SCORES_IMPROVEMENTS.format( col.replace(" ", "_") ) - df_better.to_csv(outfile_improvements, mode="a", sep=";") + df_better.to_csv(outfile_improvements, sep=";") return int(regressions_found) -- GitLab From 3220d72fe9b814a97dab30ff48ae6800d0c467e7 Mon Sep 17 00:00:00 2001 From: rtyag Date: Mon, 3 Mar 2025 15:05:24 +1100 Subject: [PATCH 030/106] add MLD, ODG and SSNR to split rendering CI --- tests/split_rendering/test_split_rendering.py | 246 +++++++++++++++++- tests/split_rendering/utils.py | 79 +++++- 2 files changed, 305 insertions(+), 20 deletions(-) diff --git a/tests/split_rendering/test_split_rendering.py b/tests/split_rendering/test_split_rendering.py index 24360208ab..7403669251 100644 --- a/tests/split_rendering/test_split_rendering.py +++ b/tests/split_rendering/test_split_rendering.py @@ -44,12 +44,21 @@ from tests.split_rendering.utils import * @pytest.mark.parametrize("bitrate", IVAS_BITRATES_AMBI) @pytest.mark.parametrize("in_fmt", INPUT_FORMATS_AMBI_SPLIT_REND) def test_ambisonics_full_chain_split( + record_property, + props_to_record, + get_mld, + get_mld_lim, + get_ssnr, + get_odg, + get_odg_bin, test_info, in_fmt, bitrate, render_config, trajectory ): post_trajectory = HR_TRAJECTORY_DIR.joinpath(f"{trajectory}.csv") pre_trajectory = post_trajectory.with_stem(f"{post_trajectory.stem}_delayed") run_full_chain_split_rendering( + record_property, + props_to_record, test_info, in_fmt=in_fmt, bitrate=bitrate, @@ -57,6 +66,11 @@ def test_ambisonics_full_chain_split( binary_suffix=EXE_SUFFIX, pre_trajectory=pre_trajectory, post_trajectory=post_trajectory, + get_mld=get_mld, + mld_lim=get_mld_lim, + get_ssnr=get_ssnr, + get_odg=get_odg, + get_odg_bin=get_odg_bin, ) @@ -64,16 +78,31 @@ def test_ambisonics_full_chain_split( @pytest.mark.parametrize("trajectory", SPLIT_REND_HR_TRAJECTORIES_TO_TEST) @pytest.mark.parametrize("render_config", RENDERER_CONFIGS_TO_TEST_AMBI) @pytest.mark.parametrize("in_fmt", INPUT_FORMATS_AMBI_SPLIT_REND) -def test_ambisonics_external_split(test_info, in_fmt, render_config, trajectory): +def test_ambisonics_external_split( + record_property, + props_to_record, + get_mld, + get_mld_lim, + get_ssnr, + get_odg, + get_odg_bin, + test_info, in_fmt, render_config, trajectory): post_trajectory = HR_TRAJECTORY_DIR.joinpath(f"{trajectory}.csv") pre_trajectory = post_trajectory.with_stem(f"{post_trajectory.stem}_delayed") run_external_split_rendering( + record_property, + props_to_record, test_info, in_fmt=in_fmt, render_config=RENDER_CFG_DIR.joinpath(f"{render_config}.txt"), pre_trajectory=pre_trajectory, post_trajectory=post_trajectory, + get_mld=get_mld, + mld_lim=get_mld_lim, + get_ssnr=get_ssnr, + get_odg=get_odg, + get_odg_bin=get_odg_bin, ) @@ -86,12 +115,21 @@ def test_ambisonics_external_split(test_info, in_fmt, render_config, trajectory) @pytest.mark.parametrize("bitrate", IVAS_BITRATES_MC) @pytest.mark.parametrize("in_fmt", INPUT_FORMATS_MC_SPLIT_REND) def test_multichannel_full_chain_split( + record_property, + props_to_record, + get_mld, + get_mld_lim, + get_ssnr, + get_odg, + get_odg_bin, test_info, in_fmt, bitrate, render_config, trajectory ): post_trajectory = HR_TRAJECTORY_DIR.joinpath(f"{trajectory}.csv") pre_trajectory = post_trajectory.with_stem(f"{post_trajectory.stem}_delayed") run_full_chain_split_rendering( + record_property, + props_to_record, test_info, in_fmt=in_fmt, bitrate=bitrate, @@ -99,6 +137,11 @@ def test_multichannel_full_chain_split( binary_suffix=EXE_SUFFIX, pre_trajectory=pre_trajectory, post_trajectory=post_trajectory, + get_mld=get_mld, + mld_lim=get_mld_lim, + get_ssnr=get_ssnr, + get_odg=get_odg, + get_odg_bin=get_odg_bin, ) @@ -106,16 +149,31 @@ def test_multichannel_full_chain_split( @pytest.mark.parametrize("trajectory", SPLIT_REND_HR_TRAJECTORIES_TO_TEST) @pytest.mark.parametrize("render_config", RENDERER_CONFIGS_TO_TEST_MC) @pytest.mark.parametrize("in_fmt", INPUT_FORMATS_MC_SPLIT_REND) -def test_multichannel_external_split(test_info, in_fmt, render_config, trajectory): +def test_multichannel_external_split( + record_property, + props_to_record, + get_mld, + get_mld_lim, + get_ssnr, + get_odg, + get_odg_bin, +test_info, in_fmt, render_config, trajectory): post_trajectory = HR_TRAJECTORY_DIR.joinpath(f"{trajectory}.csv") pre_trajectory = post_trajectory.with_stem(f"{post_trajectory.stem}_delayed") run_external_split_rendering( + record_property, + props_to_record, test_info, in_fmt=in_fmt, render_config=RENDER_CFG_DIR.joinpath(f"{render_config}.txt"), pre_trajectory=pre_trajectory, post_trajectory=post_trajectory, + get_mld=get_mld, + mld_lim=get_mld_lim, + get_ssnr=get_ssnr, + get_odg=get_odg, + get_odg_bin=get_odg_bin, ) @@ -127,11 +185,21 @@ def test_multichannel_external_split(test_info, in_fmt, render_config, trajector @pytest.mark.parametrize("render_config", RENDERER_CONFIGS_TO_TEST_ISM) @pytest.mark.parametrize("bitrate", IVAS_BITRATES_ISM) @pytest.mark.parametrize("in_fmt", INPUT_FORMATS_ISM_SPLIT_REND) -def test_ism_full_chain_split(test_info, in_fmt, bitrate, render_config, trajectory): +def test_ism_full_chain_split( + record_property, + props_to_record, + get_mld, + get_mld_lim, + get_ssnr, + get_odg, + get_odg_bin, +test_info, in_fmt, bitrate, render_config, trajectory): post_trajectory = HR_TRAJECTORY_DIR.joinpath(f"{trajectory}.csv") pre_trajectory = post_trajectory.with_stem(f"{post_trajectory.stem}_delayed") run_full_chain_split_rendering( + record_property, + props_to_record, test_info, in_fmt=in_fmt, bitrate=bitrate, @@ -139,6 +207,11 @@ def test_ism_full_chain_split(test_info, in_fmt, bitrate, render_config, traject binary_suffix=EXE_SUFFIX, pre_trajectory=pre_trajectory, post_trajectory=post_trajectory, + get_mld=get_mld, + mld_lim=get_mld_lim, + get_ssnr=get_ssnr, + get_odg=get_odg, + get_odg_bin=get_odg_bin, ) @@ -146,16 +219,31 @@ def test_ism_full_chain_split(test_info, in_fmt, bitrate, render_config, traject @pytest.mark.parametrize("trajectory", SPLIT_REND_HR_TRAJECTORIES_TO_TEST) @pytest.mark.parametrize("render_config", RENDERER_CONFIGS_TO_TEST_ISM) @pytest.mark.parametrize("in_fmt", INPUT_FORMATS_ISM_SPLIT_REND) -def test_ism_external_split(test_info, in_fmt, render_config, trajectory): +def test_ism_external_split( + record_property, + props_to_record, + get_mld, + get_mld_lim, + get_ssnr, + get_odg, + get_odg_bin, +test_info, in_fmt, render_config, trajectory): post_trajectory = HR_TRAJECTORY_DIR.joinpath(f"{trajectory}.csv") pre_trajectory = post_trajectory.with_stem(f"{post_trajectory.stem}_delayed") run_external_split_rendering( + record_property, + props_to_record, test_info, in_fmt=in_fmt, render_config=RENDER_CFG_DIR.joinpath(f"{render_config}.txt"), pre_trajectory=pre_trajectory, post_trajectory=post_trajectory, + get_mld=get_mld, + mld_lim=get_mld_lim, + get_ssnr=get_ssnr, + get_odg=get_odg, + get_odg_bin=get_odg_bin, ) @@ -167,11 +255,21 @@ def test_ism_external_split(test_info, in_fmt, render_config, trajectory): @pytest.mark.parametrize("render_config", RENDERER_CONFIGS_TO_TEST_MASA) @pytest.mark.parametrize("bitrate", IVAS_BITRATES_MASA) @pytest.mark.parametrize("in_fmt", INPUT_FORMATS_MASA_SPLIT_REND) -def test_masa_full_chain_split(test_info, in_fmt, bitrate, render_config, trajectory): +def test_masa_full_chain_split( + record_property, + props_to_record, + get_mld, + get_mld_lim, + get_ssnr, + get_odg, + get_odg_bin, +test_info, in_fmt, bitrate, render_config, trajectory): post_trajectory = HR_TRAJECTORY_DIR.joinpath(f"{trajectory}.csv") pre_trajectory = post_trajectory.with_stem(f"{post_trajectory.stem}_delayed") run_full_chain_split_rendering( + record_property, + props_to_record, test_info, in_fmt=in_fmt, bitrate=bitrate, @@ -179,6 +277,11 @@ def test_masa_full_chain_split(test_info, in_fmt, bitrate, render_config, trajec binary_suffix=EXE_SUFFIX, pre_trajectory=pre_trajectory, post_trajectory=post_trajectory, + get_mld=get_mld, + mld_lim=get_mld_lim, + get_ssnr=get_ssnr, + get_odg=get_odg, + get_odg_bin=get_odg_bin, ) @@ -186,16 +289,31 @@ def test_masa_full_chain_split(test_info, in_fmt, bitrate, render_config, trajec @pytest.mark.parametrize("trajectory", SPLIT_REND_HR_TRAJECTORIES_TO_TEST) @pytest.mark.parametrize("render_config", RENDERER_CONFIGS_TO_TEST_MASA) @pytest.mark.parametrize("in_fmt", INPUT_FORMATS_MASA_SPLIT_REND) -def test_masa_external_split(test_info, in_fmt, render_config, trajectory): +def test_masa_external_split( + record_property, + props_to_record, + get_mld, + get_mld_lim, + get_ssnr, + get_odg, + get_odg_bin, +test_info, in_fmt, render_config, trajectory): post_trajectory = HR_TRAJECTORY_DIR.joinpath(f"{trajectory}.csv") pre_trajectory = post_trajectory.with_stem(f"{post_trajectory.stem}_delayed") run_external_split_rendering( + record_property, + props_to_record, test_info, in_fmt=in_fmt, render_config=RENDER_CFG_DIR.joinpath(f"{render_config}.txt"), pre_trajectory=pre_trajectory, post_trajectory=post_trajectory, + get_mld=get_mld, + mld_lim=get_mld_lim, + get_ssnr=get_ssnr, + get_odg=get_odg, + get_odg_bin=get_odg_bin, ) @@ -207,11 +325,21 @@ def test_masa_external_split(test_info, in_fmt, render_config, trajectory): @pytest.mark.parametrize("render_config", RENDERER_CONFIGS_TO_TEST_OMASA) @pytest.mark.parametrize("bitrate", IVAS_BITRATES_OMASA) @pytest.mark.parametrize("in_fmt", INPUT_FORMATS_OMASA_SPLIT_REND) -def test_omasa_full_chain_split(test_info, in_fmt, bitrate, render_config, trajectory): +def test_omasa_full_chain_split( + record_property, + props_to_record, + get_mld, + get_mld_lim, + get_ssnr, + get_odg, + get_odg_bin, +test_info, in_fmt, bitrate, render_config, trajectory): post_trajectory = HR_TRAJECTORY_DIR.joinpath(f"{trajectory}.csv") pre_trajectory = post_trajectory.with_stem(f"{post_trajectory.stem}_delayed") run_full_chain_split_rendering( + record_property, + props_to_record, test_info, in_fmt=in_fmt, bitrate=bitrate, @@ -219,6 +347,11 @@ def test_omasa_full_chain_split(test_info, in_fmt, bitrate, render_config, traje binary_suffix=EXE_SUFFIX, pre_trajectory=pre_trajectory, post_trajectory=post_trajectory, + get_mld=get_mld, + mld_lim=get_mld_lim, + get_ssnr=get_ssnr, + get_odg=get_odg, + get_odg_bin=get_odg_bin, ) @@ -230,11 +363,21 @@ def test_omasa_full_chain_split(test_info, in_fmt, bitrate, render_config, traje @pytest.mark.parametrize("render_config", RENDERER_CONFIGS_TO_TEST_OSBA) @pytest.mark.parametrize("bitrate", IVAS_BITRATES_OSBA) @pytest.mark.parametrize("in_fmt", INPUT_FORMATS_OSBA_SPLIT_REND) -def test_osba_full_chain_split(test_info, in_fmt, bitrate, render_config, trajectory): +def test_osba_full_chain_split( + record_property, + props_to_record, + get_mld, + get_mld_lim, + get_ssnr, + get_odg, + get_odg_bin, +test_info, in_fmt, bitrate, render_config, trajectory): post_trajectory = HR_TRAJECTORY_DIR.joinpath(f"{trajectory}.csv") pre_trajectory = post_trajectory.with_stem(f"{post_trajectory.stem}_delayed") run_full_chain_split_rendering( + record_property, + props_to_record, test_info, in_fmt=in_fmt, bitrate=bitrate, @@ -242,6 +385,11 @@ def test_osba_full_chain_split(test_info, in_fmt, bitrate, render_config, trajec binary_suffix=EXE_SUFFIX, pre_trajectory=pre_trajectory, post_trajectory=post_trajectory, + get_mld=get_mld, + mld_lim=get_mld_lim, + get_ssnr=get_ssnr, + get_odg=get_odg, + get_odg_bin=get_odg_bin, ) @@ -253,17 +401,32 @@ def test_osba_full_chain_split(test_info, in_fmt, bitrate, render_config, trajec @pytest.mark.parametrize("trajectory", SPLIT_REND_HR_TRAJECTORIES_TO_TEST) @pytest.mark.parametrize("render_config", RENDERER_CONFIGS_TO_TEST_PLC) @pytest.mark.parametrize("in_fmt", INPUT_FORMATS_AMBI_SPLIT_REND[-1:]) -def test_post_rend_plc(test_info, in_fmt, render_config, trajectory, error_pattern): +def test_post_rend_plc( + record_property, + props_to_record, + get_mld, + get_mld_lim, + get_ssnr, + get_odg, + get_odg_bin, +test_info, in_fmt, render_config, trajectory, error_pattern): post_trajectory = HR_TRAJECTORY_DIR.joinpath(f"{trajectory}.csv") pre_trajectory = post_trajectory.with_stem(f"{post_trajectory.stem}_delayed") run_external_split_rendering( + record_property, + props_to_record, test_info, in_fmt=in_fmt, render_config=RENDER_CFG_DIR.joinpath(f"{render_config}.txt"), pre_trajectory=pre_trajectory, post_trajectory=post_trajectory, plc_error_pattern=ERROR_PATTERNS_DIR.joinpath(f"{error_pattern}.ep"), + get_mld=get_mld, + mld_lim=get_mld_lim, + get_ssnr=get_ssnr, + get_odg=get_odg, + get_odg_bin=get_odg_bin, ) @@ -281,12 +444,22 @@ full_chain_split_pcm_params = [ @pytest.mark.parametrize("in_fmt,bitrate,render_config", full_chain_split_pcm_params) -def test_full_chain_split_pcm(test_info, in_fmt, bitrate, render_config): +def test_full_chain_split_pcm( + record_property, + props_to_record, + get_mld, + get_mld_lim, + get_ssnr, + get_odg, + get_odg_bin, +test_info, in_fmt, bitrate, render_config): trajectory = SPLIT_REND_HR_TRAJECTORIES_TO_TEST[0] post_trajectory = HR_TRAJECTORY_DIR.joinpath(f"{trajectory}.csv") pre_trajectory = post_trajectory.with_stem(f"{post_trajectory.stem}_delayed") run_full_chain_split_rendering( + record_property, + props_to_record, test_info, in_fmt=in_fmt, bitrate=bitrate, @@ -295,6 +468,11 @@ def test_full_chain_split_pcm(test_info, in_fmt, bitrate, render_config): pre_trajectory=pre_trajectory, post_trajectory=post_trajectory, renderer_fmt="BINAURAL_SPLIT_PCM", + get_mld=get_mld, + mld_lim=get_mld_lim, + get_ssnr=get_ssnr, + get_odg=get_odg, + get_odg_bin=get_odg_bin, ) @@ -307,18 +485,33 @@ external_split_pcm_params = [ @pytest.mark.parametrize("in_fmt,render_config", external_split_pcm_params) -def test_external_split_pcm(test_info, in_fmt, render_config): +def test_external_split_pcm( + record_property, + props_to_record, + get_mld, + get_mld_lim, + get_ssnr, + get_odg, + get_odg_bin, +test_info, in_fmt, render_config): trajectory = SPLIT_REND_HR_TRAJECTORIES_TO_TEST[0] post_trajectory = HR_TRAJECTORY_DIR.joinpath(f"{trajectory}.csv") pre_trajectory = post_trajectory.with_stem(f"{post_trajectory.stem}_delayed") run_external_split_rendering( + record_property, + props_to_record, test_info, in_fmt=in_fmt, render_config=RENDER_CFG_DIR.joinpath(f"{render_config}.txt"), pre_trajectory=pre_trajectory, post_trajectory=post_trajectory, renderer_fmt="BINAURAL_SPLIT_PCM", + get_mld=get_mld, + mld_lim=get_mld_lim, + get_ssnr=get_ssnr, + get_odg=get_odg, + get_odg_bin=get_odg_bin, ) @@ -327,11 +520,21 @@ def test_external_split_pcm(test_info, in_fmt, render_config): @pytest.mark.parametrize("in_fmt", ["5_1"]) @pytest.mark.parametrize("pre_rend_fr", SPLIT_RENDERER_PRE_FRAMINGS) @pytest.mark.parametrize("post_rend_fr", SPLIT_RENDERER_POST_FRAMINGS) -def test_framing_combinations_external_split(test_info, in_fmt, render_config, trajectory, post_rend_fr, pre_rend_fr): +def test_framing_combinations_external_split( + record_property, + props_to_record, + get_mld, + get_mld_lim, + get_ssnr, + get_odg, + get_odg_bin, +test_info, in_fmt, render_config, trajectory, post_rend_fr, pre_rend_fr): post_trajectory = HR_TRAJECTORY_DIR.joinpath(f"{trajectory}.csv") pre_trajectory = post_trajectory.with_stem(f"{post_trajectory.stem}_delayed") run_external_split_rendering( + record_property, + props_to_record, test_info, in_fmt=in_fmt, render_config=RENDER_FRAMING_CFG_DIR.joinpath(f"{render_config}.txt"), @@ -339,6 +542,11 @@ def test_framing_combinations_external_split(test_info, in_fmt, render_config, t post_trajectory=post_trajectory, post_rend_fr=post_rend_fr, pre_rend_fr=pre_rend_fr, + get_mld=get_mld, + mld_lim=get_mld_lim, + get_ssnr=get_ssnr, + get_odg=get_odg, + get_odg_bin=get_odg_bin, ) @pytest.mark.parametrize("trajectory", SPLIT_REND_HR_TRAJECTORIES_TO_TEST) @@ -347,12 +555,21 @@ def test_framing_combinations_external_split(test_info, in_fmt, render_config, t @pytest.mark.parametrize("pre_rend_fr", SPLIT_RENDERER_PRE_FRAMINGS) @pytest.mark.parametrize("post_rend_fr", SPLIT_RENDERER_POST_FRAMINGS) def test_framing_combinations_full_chain_split( + record_property, + props_to_record, + get_mld, + get_mld_lim, + get_ssnr, + get_odg, + get_odg_bin, test_info, in_fmt, render_config, trajectory, post_rend_fr, pre_rend_fr ): post_trajectory = HR_TRAJECTORY_DIR.joinpath(f"{trajectory}.csv") pre_trajectory = post_trajectory.with_stem(f"{post_trajectory.stem}_delayed") run_full_chain_split_rendering( + record_property, + props_to_record, test_info, in_fmt=in_fmt, render_config=RENDER_FRAMING_CFG_DIR.joinpath(f"{render_config}.txt"), @@ -362,4 +579,9 @@ def test_framing_combinations_full_chain_split( binary_suffix=EXE_SUFFIX, post_rend_fr=post_rend_fr, pre_rend_fr=pre_rend_fr, + get_mld=get_mld, + mld_lim=get_mld_lim, + get_ssnr=get_ssnr, + get_odg=get_odg, + get_odg_bin=get_odg_bin, ) diff --git a/tests/split_rendering/utils.py b/tests/split_rendering/utils.py index 504929b29a..b8aa6737ea 100644 --- a/tests/split_rendering/utils.py +++ b/tests/split_rendering/utils.py @@ -41,6 +41,8 @@ import pytest from tests.renderer.utils import check_BE, run_ivas_isar_enc_cmd, run_ivas_isar_dec_cmd, run_isar_post_rend_cmd, run_isar_ext_rend_cmd from tests.split_rendering.constants import * +from ..cmp_pcm import cmp_pcm +from ..conftest import parse_properties sys.path.append(SCRIPTS_DIR) from pyaudio3dtools.audiofile import readfile, writefile @@ -151,6 +153,8 @@ def truncate_signal( def run_full_chain_split_rendering( + record_property, + props_to_record, test_info, in_fmt: str, bitrate: str, @@ -161,6 +165,11 @@ def run_full_chain_split_rendering( binary_suffix: str = "", post_rend_fr: str = "20", pre_rend_fr: str = "20", + get_mld=False, + mld_lim=0, + get_ssnr=False, + get_odg=False, + get_odg_bin=False, ) -> str: """ Runs the full split rendering chain consisting of @@ -274,15 +283,41 @@ def run_full_chain_split_rendering( cut, cut_fs = readfile(out_file) - [diff_found, snr, gain_b, max_diff] = check_BE(test_info, ref, ref_fs, cut, cut_fs) - if diff_found : - pytest.fail( - f"CuT not BE to REF! SNR : {snr:3.2f} dB, Gain CuT: {gain_b:1.3f}, Max Diff = {int(max_diff)}" + if get_mld == False: + [diff_found, snr, gain_b, max_diff] = check_BE(test_info, ref, ref_fs, cut, cut_fs) + if diff_found : + pytest.fail( + f"CuT not BE to REF! SNR : {snr:3.2f} dB, Gain CuT: {gain_b:1.3f}, Max Diff = {int(max_diff)}" + ) + else: + # see constants.py + output_differs, reason = cmp_pcm( + out_file, + out_file_ref, + "BINAURAL", + ref_fs, + get_mld=get_mld, + mld_lim=mld_lim, + get_ssnr=get_ssnr, + get_odg=get_odg, + get_odg_bin=get_odg_bin, ) + + props = parse_properties(reason, output_differs, props_to_record) + for k, v in props.items(): + record_property(k, v) + + if output_differs: + pytest.fail(f"Output differs: ({reason})") + + + return out_file def run_external_split_rendering( + record_property, + props_to_record, test_info, in_fmt: str, render_config: Path, @@ -294,6 +329,11 @@ def run_external_split_rendering( is_comparetest: bool = False, post_rend_fr: str = "20", pre_rend_fr: str = "20", + get_mld=False, + mld_lim=0, + get_ssnr=False, + get_odg=False, + get_odg_bin=False, ) -> Tuple[np.ndarray, int]: """ Runs the exeternal split rendering chain consisting of @@ -393,9 +433,32 @@ def run_external_split_rendering( cut, cut_fs = readfile(out_file) - [diff_found, snr, gain_b, max_diff] = check_BE(test_info, ref, ref_fs, cut, cut_fs) - if diff_found : - pytest.fail( - f"CuT not BE to REF! SNR : {snr:3.2f} dB, Gain CuT: {gain_b:1.3f}, Max Diff = {int(max_diff)}" + if get_mld == False: + [diff_found, snr, gain_b, max_diff] = check_BE(test_info, ref, ref_fs, cut, cut_fs) + if diff_found : + pytest.fail( + f"CuT not BE to REF! SNR : {snr:3.2f} dB, Gain CuT: {gain_b:1.3f}, Max Diff = {int(max_diff)}" + ) + else: + # see constants.py + output_differs, reason = cmp_pcm( + out_file, + out_file_ref, + "BINAURAL", + ref_fs, + get_mld=get_mld, + mld_lim=mld_lim, + get_ssnr=get_ssnr, + get_odg=get_odg, + get_odg_bin=get_odg_bin, ) + + props = parse_properties(reason, output_differs, props_to_record) + for k, v in props.items(): + record_property(k, v) + + if output_differs: + pytest.fail(f"Output differs: ({reason})") + + return out_file -- GitLab From 55754c39f179f7feb9bb03367d9af01d30303cf5 Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Wed, 12 Mar 2025 08:44:10 +0100 Subject: [PATCH 031/106] remove basop files from usan ignorelist --- scripts/basop_create_ignorelist_for_ubsan.py | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/scripts/basop_create_ignorelist_for_ubsan.py b/scripts/basop_create_ignorelist_for_ubsan.py index e6dfdfa075..d13c0d5f5b 100644 --- a/scripts/basop_create_ignorelist_for_ubsan.py +++ b/scripts/basop_create_ignorelist_for_ubsan.py @@ -1,9 +1,14 @@ import pathlib -all_c_files = pathlib.Path(".").glob("lib_*/*.[ch]") -ivas_c_files = pathlib.Path(".").glob("lib_*/ivas*.[ch]") -non_ivas_c_files = sorted(set(all_c_files) - set(ivas_c_files)) +here = pathlib.Path(".") + +all_files = here.glob("lib_*/*.[ch]") +ivas_files = here.glob("lib_*/ivas*.[ch]") +non_ivas_files = sorted(set(all_files) - set(ivas_files)) +basop_files = here.glob("lib_*/basop*.[ch]") + +ignorefiles = sorted(set(non_ivas_files) - set(basop_files)) with open("ubsan_ignorelist.txt", "w") as f: - for cfile in non_ivas_c_files: + for cfile in ignorefiles: print(f"src:{cfile}", file=f) -- GitLab From f6d9e5f64a2fb864b70fe0a053f35a0d79f6e996 Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Thu, 13 Mar 2025 18:34:18 +0100 Subject: [PATCH 032/106] add first infrastructure for splitting still functionally equivalent to before --- tests/cmp_pcm.py | 214 ++++++++++-------- .../test_param_file.py | 29 ++- tests/conftest.py | 31 ++- tests/constants.py | 37 +++ 4 files changed, 196 insertions(+), 115 deletions(-) diff --git a/tests/cmp_pcm.py b/tests/cmp_pcm.py index 127820525a..e254f1d59a 100755 --- a/tests/cmp_pcm.py +++ b/tests/cmp_pcm.py @@ -7,7 +7,7 @@ import tempfile import re import subprocess from pathlib import Path -from typing import Optional +from typing import Optional, List THIS_PATH = os.path.join(os.getcwd(), __file__) sys.path.append(os.path.join(os.path.dirname(THIS_PATH), "../scripts")) @@ -41,8 +41,9 @@ def cmp_pcm( ref_jbm_tf: Optional[Path] = None, cut_jbm_tf: Optional[Path] = None, quiet: Optional[bool] = False, - scalefac=1, -) -> tuple[int, str]: + scalefac: int = 1, + split_idx: np.ndarray = np.empty(0), +) -> tuple[List[int], List[str]]: """ Compare 2 PCM files for bitexactness """ @@ -67,13 +68,13 @@ def cmp_pcm( if fs1 != fs2: reason = "FAIL: Sampling rate differs." - return 1, reason + return [1], [reason] # In case number of channels do not match, fail already now. Could happen in case of # comparison to input with for a non-passthrough mode. if s1.shape[1] != s2.shape[1]: reason = "FAIL: Number of channels differ." - return 1, reason + return [1], [reason] handle_differing_lengths = "fail" if allow_differing_lengths: @@ -88,108 +89,115 @@ def cmp_pcm( if get_mld: reason += " - MLD: None" - return 1, reason - - # Apply scalefac if specified. Useful in case scaling has been applied on the input, and the inverse is scaling is supplied in scalefac. - if scalefac != 1: - s1 = np.round(s1*scalefac, 0) # Need rounding for max abs diff search - s2 = np.round(s2*scalefac, 0) - - cmp_result = pyaudio3dtools.audioarray.compare( - s1, - s2, - fs, - per_frame=False, - get_mld=get_mld, - get_ssnr=get_ssnr, - ssnr_thresh_low=-50, - ref_jbm_tf=ref_jbm_tf, - test_jbm_tf=cut_jbm_tf, - handle_differing_lengths=handle_differing_lengths, - ) + return [1], [reason] + + output_differs_parts = [] + reason_parts = [] + + for s1, s2 in zip(np.split(s1, split_idx), np.split(s2, split_idx)): + # Apply scalefac if specified. Useful in case scaling has been applied on the input, and the inverse is scaling is supplied in scalefac. + if scalefac != 1: + s1 = np.round(s1 * scalefac, 0) # Need rounding for max abs diff search + s2 = np.round(s2 * scalefac, 0) + + cmp_result = pyaudio3dtools.audioarray.compare( + s1, + s2, + fs, + per_frame=False, + get_mld=get_mld, + get_ssnr=get_ssnr, + ssnr_thresh_low=-50, + ref_jbm_tf=ref_jbm_tf, + test_jbm_tf=cut_jbm_tf, + handle_differing_lengths=handle_differing_lengths, + ) + + output_differs = 0 + reason = "SUCCESS: Files are bitexact" + + if not cmp_result["bitexact"] and cmp_result["max_abs_diff"] <= abs_tol: + reason = "SUCCESS: Maximum absolute diff below threshold" + elif not cmp_result["bitexact"]: + diff_msg = f"MAXIMUM ABS DIFF ==> {cmp_result['max_abs_diff']} at sample num {cmp_result['max_abs_diff_pos_sample']} (assuming {nchannels} channels)" + first_msg = f"First diff found at sample num {cmp_result['first_diff_pos_sample']} in channel {cmp_result['first_diff_pos_channel']}, frame {cmp_result['first_diff_pos_frame']} (assuming {nchannels} channels, {fs} sampling rate)" + print(diff_msg, file=output_target) + print(first_msg, file=output_target) - output_differs = 0 - reason = "SUCCESS: Files are bitexact" - - if not cmp_result["bitexact"] and cmp_result["max_abs_diff"] <= abs_tol: - reason = "SUCCESS: Maximum absolute diff below threshold" - elif not cmp_result["bitexact"]: - diff_msg = f"MAXIMUM ABS DIFF ==> {cmp_result['max_abs_diff']} at sample num {cmp_result['max_abs_diff_pos_sample']} (assuming {nchannels} channels)" - first_msg = f"First diff found at sample num {cmp_result['first_diff_pos_sample']} in channel {cmp_result['first_diff_pos_channel']}, frame {cmp_result['first_diff_pos_frame']} (assuming {nchannels} channels, {fs} sampling rate)" - print(diff_msg, file=output_target) - print(first_msg, file=output_target) - - reason = f"Non-BE - MAXIMUM ABS DIFF: {cmp_result['max_abs_diff']}" - output_differs = 1 - - if get_mld: - mld_msg = f"MLD: {cmp_result['MLD']}" - reason += " - " + mld_msg - print(mld_msg, file=output_target) - - if cmp_result["MLD"] <= mld_lim: - output_differs = 0 - reason += f" <= {mld_lim}" - else: - reason += f" > {mld_lim}" - - if get_ssnr: - reason += " - " - for i, s in enumerate(cmp_result["SSNR"], start=1): - msg = f"Channel {i} SSNR: {s}" - reason += msg + " - " - - if get_odg: - for n in range(nchannels): - pqeval_output = pqevalaudio_wrapper(s1[:, n], s2[:, n], fs) + reason = f"Non-BE - MAXIMUM ABS DIFF: {cmp_result['max_abs_diff']}" + output_differs = 1 + if get_mld: + mld_msg = f"MLD: {cmp_result['MLD']}" + reason += " - " + mld_msg + print(mld_msg, file=output_target) + + if cmp_result["MLD"] <= mld_lim: + output_differs = 0 + reason += f" <= {mld_lim}" + else: + reason += f" > {mld_lim}" + + if get_ssnr: + reason += " - " + for i, s in enumerate(cmp_result["SSNR"], start=1): + msg = f"Channel {i} SSNR: {s}" + reason += msg + " - " + + if get_odg: + for n in range(nchannels): + pqeval_output = pqevalaudio_wrapper(s1[:, n], s2[:, n], fs) + + match_odg = re.search(ODG_PATTERN_PQEVALAUDIO, pqeval_output) + odg = float(match_odg.groups()[0]) + msg = f"Channel {n} ODG: {odg}" + reason += " - " + msg + print(msg) + + if get_odg_bin: + odg_files = {} + for f in [odg_input, odg_test, odg_ref]: + # Load PEAQ test files and ensure 48 kHz sampling rate + s, fs = pyaudio3dtools.audiofile.readfile( + f, nchannels, fs, outdtype=np.int16 + ) + odg_files[f] = np.clip( + pyaudio3dtools.audioarray.resample(s.astype(float), fs, 48000), + -32768, + 32767, + ).astype(np.int16) + + pqeval_output = pqevalaudio_wrapper( + odg_files[odg_input], odg_files[odg_ref], 48000 + ) match_odg = re.search(ODG_PATTERN_PQEVALAUDIO, pqeval_output) - odg = float(match_odg.groups()[0]) - msg = f"Channel {n} ODG: {odg}" - reason += " - " + msg - print(msg) - - if get_odg_bin: - odg_files = {} - for f in [odg_input, odg_test, odg_ref]: - # Load PEAQ test files and ensure 48 kHz sampling rate - s, fs = pyaudio3dtools.audiofile.readfile( - f, nchannels, fs, outdtype=np.int16 + try: + odg_ref = float(match_odg.groups()[0]) + except AttributeError: + raise OdgParsingFailed("Could not get Odg for ref signal") + + pqeval_output = pqevalaudio_wrapper( + odg_files[odg_input], odg_files[odg_test], 48000 ) - odg_files[f] = np.clip( - pyaudio3dtools.audioarray.resample(s.astype(float), fs, 48000), - -32768, - 32767, - ).astype(np.int16) - - pqeval_output = pqevalaudio_wrapper( - odg_files[odg_input], odg_files[odg_ref], 48000 - ) - match_odg = re.search(ODG_PATTERN_PQEVALAUDIO, pqeval_output) - try: - odg_ref = float(match_odg.groups()[0]) - except AttributeError: - raise OdgParsingFailed("Could not get Odg for ref signal") - - pqeval_output = pqevalaudio_wrapper( - odg_files[odg_input], odg_files[odg_test], 48000 - ) - match_odg = re.search(ODG_PATTERN_PQEVALAUDIO, pqeval_output) - try: - odg_test = float(match_odg.groups()[0]) - except AttributeError: - raise OdgParsingFailed("Could not get Odg for test signal") + match_odg = re.search(ODG_PATTERN_PQEVALAUDIO, pqeval_output) + try: + odg_test = float(match_odg.groups()[0]) + except AttributeError: + raise OdgParsingFailed("Could not get Odg for test signal") + + odg = odg_test - odg_ref # Todo: store both rather than difference? - odg = odg_test - odg_ref # Todo: store both rather than difference? + msg = f"Delta-ODG: {odg}" + reason += " - " + msg + print(msg, file=output_target) - msg = f"Delta-ODG: {odg}" - reason += " - " + msg - print(msg, file=output_target) + if quiet: + output_target.close() - if quiet: - output_target.close() + output_differs_parts.append(output_differs) + reason_parts.append(reason) - return output_differs, reason + return output_differs_parts, reason_parts class OdgParsingFailed(Exception): @@ -262,7 +270,13 @@ if __name__ == "__main__": parser.add_argument("--get_odg", action="store_true") parser.add_argument("--get_ssnr", action="store_true") parser.add_argument("--allow_differing_lengths", action="store_true") - parser.add_argument("--scalefac", type=float, default=1, dest="scalefac", help="Scale factor to be applied before comparing the output. Useful when input scaling has been applied.") + parser.add_argument( + "--scalefac", + type=float, + default=1, + dest="scalefac", + help="Scale factor to be applied before comparing the output. Useful when input scaling has been applied.", + ) parser.add_argument("--quiet", action="store_true") args = vars(parser.parse_args()) diff --git a/tests/codec_be_on_mr_nonselection/test_param_file.py b/tests/codec_be_on_mr_nonselection/test_param_file.py index ac2d84d1e7..495c583cc3 100644 --- a/tests/codec_be_on_mr_nonselection/test_param_file.py +++ b/tests/codec_be_on_mr_nonselection/test_param_file.py @@ -53,6 +53,7 @@ from tests.conftest import ( parse_properties, compare_dmx_signals, log_dbg_msg, + get_split_idx, ) from tests.testconfig import PARAM_FILE from tests.constants import ( @@ -588,7 +589,13 @@ def run_test( ref_file = ref_output_file fs = int(sampling_rate) * 1000 - output_differs, reason = cmp_pcm( + + # split_idx = np.empty(0) + # # TODO:test on --split argument given + # + # split_idx = get_split_idx(str(Path(testv_file).stem), int(sampling_rate)) + + output_differs_parts, reason_parts = cmp_pcm( ref_file, dut_output_file, out_config_2_nchannels(output_config), @@ -608,11 +615,10 @@ def run_test( scalefac=test_info.config.option.scalefac, ) - cmp_result_msg += reason - - props = parse_properties(cmp_result_msg, output_differs, props_to_record) - for k, v in props.items(): - dut_decoder_frontend.record_property(k, v) + for output_differs, reason in zip(output_differs_parts, reason_parts): + props = parse_properties(reason, output_differs, props_to_record) + for k, v in props.items(): + dut_decoder_frontend.record_property(k, v) metadata_differs = False @@ -647,19 +653,22 @@ def run_test( if enc_test_result: pytest.fail("Too high difference in encoder statistics found.") + at_least_one_output_differs = any(output_differs_parts) + # TODO + reason = reason_parts[0] if tracefile_last_rtp_numbers_differ: pytest.fail( "Last RTP sequence num in tracefiles differ for JBM decoding - Not all frames were decoded in both ref and dut." ) elif get_mld and get_mld_lim > 0: - if output_differs: + if at_least_one_output_differs: pytest.fail(reason) else: - if output_differs or metadata_differs: + if at_least_one_output_differs or metadata_differs: msg = "Difference between ref and dut in " - if output_differs and metadata_differs: + if at_least_one_output_differs and metadata_differs: msg += f"output ({reason}) and metadata" - elif output_differs: + elif at_least_one_output_differs: msg += f"output only ({reason})" elif metadata_differs: msg += "metadata only" diff --git a/tests/conftest.py b/tests/conftest.py index 3b75cbc557..a5eab3ae09 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -36,7 +36,6 @@ import logging import os import re import json -import shutil from tests import testconfig import pytest import platform @@ -49,13 +48,13 @@ import tempfile from typing import Optional, Union import numpy as np from .constants import ( - # MAX_ENC_DIFF_NAME_PATTERN, DMX_DIFF, DMX_MLD, DMX_SSNR, MAX_ENC_DIFF_PARAM_NAME, MLD_PATTERN, MAX_DIFF_PATTERN, + SPLIT_IDX, SSNR_PATTERN, ENC_AUX_FILES, ODG_PATTERN, @@ -306,6 +305,13 @@ def pytest_addoption(parser): default=1, ) + parser.addoption( + "--split_comparison", + action="store_true", + help="If given, split the output files by the provided indices", + ) + + @pytest.fixture(scope="session", autouse=True) def update_ref(request): """ @@ -365,6 +371,7 @@ def get_odg(request): """ return request.config.option.odg + @pytest.fixture(scope="session", autouse=True) def get_odg_bin(request): """ @@ -1217,20 +1224,20 @@ def compare_dmx_signals(ref_dmx_files, dut_dmx_files, fs) -> dict: ref_dmx_files, dmx_file_ref_tmp.name, out_nchans=nchannels, - in_fs=fs*1000, + in_fs=fs * 1000, ) pyaudio3dtools.audiofile.combinefiles( dut_dmx_files, dmx_file_dut_tmp.name, out_nchans=nchannels, - in_fs=fs*1000, + in_fs=fs * 1000, ) dmx_differs, reason = cmp_pcm( dmx_file_ref_tmp.name, dmx_file_dut_tmp.name, nchannels, - fs*1000, + fs * 1000, get_mld=True, get_ssnr=True, quiet=True, @@ -1240,3 +1247,17 @@ def compare_dmx_signals(ref_dmx_files, dut_dmx_files, fs) -> dict: prop_results = parse_properties(reason, dmx_differs, dmx_props) return prop_results + + +def get_split_idx(input_file: str, sampling_rate_khz: int) -> Optional[np.ndarray]: + """ + Return array for splitting the output file before doing the comparison. + + If no list of indices is available for the given input file, an empty array is returned. + """ + idx = SPLIT_IDX.get(input_file, np.empty(0)) + + if len(idx) > 0 and sampling_rate_khz != 16: + idx *= sampling_rate_khz // 16 + + return idx diff --git a/tests/constants.py b/tests/constants.py index 9990db37cc..13f46b8553 100644 --- a/tests/constants.py +++ b/tests/constants.py @@ -65,3 +65,40 @@ ENC_AUX_FILES = [ ["total_brate", np.float32, "fs/50"], ["vad_flag", np.int16, "fs/50"], ] + +# lists of indices for splitting of output files +# values are in SAMPLES for 16kHz (!). For higher rates, need to multiply +SPLIT_IDX_LTV_STEREO_16KHZ = np.asarray( + [ + 302096, + 519280, + 613072, + 690176, + 740992, + 749024, + 782096, + 950064, + 1045408, + 1141408, + 1237424, + 1333392, + 1397712, + 1472912, + 1755840, + 1781680, + 1789712, + 1957664, + 2158112, + 2294704, + 2454688, + 2525872, + 2573888, + 2645504, + 2746096, + 2842096, + 2970080, + 3219792, + ] +) + +SPLIT_IDX = {"ltv16_stereo": SPLIT_IDX_LTV_STEREO_16KHZ} -- GitLab From ddaa327a0961b8893ec078ad49944e84617df66f Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Fri, 14 Mar 2025 09:42:03 +0100 Subject: [PATCH 033/106] activate split comparison via the cmdl switch recording of the results needs to be added still --- tests/codec_be_on_mr_nonselection/test_param_file.py | 12 ++++++++---- tests/conftest.py | 9 +++++++-- 2 files changed, 15 insertions(+), 6 deletions(-) diff --git a/tests/codec_be_on_mr_nonselection/test_param_file.py b/tests/codec_be_on_mr_nonselection/test_param_file.py index 495c583cc3..f39b22adf2 100644 --- a/tests/codec_be_on_mr_nonselection/test_param_file.py +++ b/tests/codec_be_on_mr_nonselection/test_param_file.py @@ -218,6 +218,7 @@ def test_param_file_tests( get_odg_bin, compare_to_input, compare_enc_dmx, + split_comparison, ): enc_opts, dec_opts, sim_opts, eid_opts = param_file_test_dict[test_tag] @@ -250,6 +251,7 @@ def test_param_file_tests( get_odg_bin, compare_to_input, compare_enc_dmx, + split_comparison, ) @@ -282,6 +284,7 @@ def run_test( get_odg_bin, compare_to_input, compare_enc_dmx, + split_comparison, ): # If compare_to_input is set, only run pass-through test cases if compare_to_input: @@ -590,10 +593,10 @@ def run_test( fs = int(sampling_rate) * 1000 - # split_idx = np.empty(0) - # # TODO:test on --split argument given - # - # split_idx = get_split_idx(str(Path(testv_file).stem), int(sampling_rate)) + pytest.set_trace() + split_idx = np.empty(0) + if split_comparison: + split_idx = get_split_idx(str(Path(testv_file).stem), int(sampling_rate)) output_differs_parts, reason_parts = cmp_pcm( ref_file, @@ -613,6 +616,7 @@ def run_test( ref_jbm_tf=ref_tracefile_dec, cut_jbm_tf=dut_tracefile_dec, scalefac=test_info.config.option.scalefac, + split_idx=split_idx, ) for output_differs, reason in zip(output_differs_parts, reason_parts): diff --git a/tests/conftest.py b/tests/conftest.py index a5eab3ae09..c5c74a61c6 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -306,7 +306,7 @@ def pytest_addoption(parser): ) parser.addoption( - "--split_comparison", + "--split-comparison", action="store_true", help="If given, split the output files by the provided indices", ) @@ -447,6 +447,11 @@ def test_info(request): pytest.fail(request.error) +@pytest.fixture(scope="session") +def split_comparison(request): + return request.config.option.split_comparison + + class EncoderFrontend: def __init__(self, path, enc_type, record_property, timeout=None) -> None: self._path = Path(path).absolute() @@ -1255,7 +1260,7 @@ def get_split_idx(input_file: str, sampling_rate_khz: int) -> Optional[np.ndarra If no list of indices is available for the given input file, an empty array is returned. """ - idx = SPLIT_IDX.get(input_file, np.empty(0)) + idx = SPLIT_IDX.get(input_file.lower(), np.empty(0)) if len(idx) > 0 and sampling_rate_khz != 16: idx *= sampling_rate_khz // 16 -- GitLab From 075846353fb0f26c9235d9bd34bba2a0c8ca5acd Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Fri, 14 Mar 2025 09:59:14 +0100 Subject: [PATCH 034/106] record measurements for individual splits --- .../test_param_file.py | 11 +++++++--- tests/conftest.py | 22 ++++++++++--------- 2 files changed, 20 insertions(+), 13 deletions(-) diff --git a/tests/codec_be_on_mr_nonselection/test_param_file.py b/tests/codec_be_on_mr_nonselection/test_param_file.py index f39b22adf2..c9854185ac 100644 --- a/tests/codec_be_on_mr_nonselection/test_param_file.py +++ b/tests/codec_be_on_mr_nonselection/test_param_file.py @@ -593,7 +593,6 @@ def run_test( fs = int(sampling_rate) * 1000 - pytest.set_trace() split_idx = np.empty(0) if split_comparison: split_idx = get_split_idx(str(Path(testv_file).stem), int(sampling_rate)) @@ -619,8 +618,14 @@ def run_test( split_idx=split_idx, ) - for output_differs, reason in zip(output_differs_parts, reason_parts): - props = parse_properties(reason, output_differs, props_to_record) + prop_suffix = [""] + if len(split_idx) > 0: + prop_suffix = [f"_split{i}" for i in range(1, len(split_idx) + 1)] + + for output_differs, reason, suffix in zip( + output_differs_parts, reason_parts, prop_suffix + ): + props = parse_properties(reason, output_differs, props_to_record, suffix) for k, v in props.items(): dut_decoder_frontend.record_property(k, v) diff --git a/tests/conftest.py b/tests/conftest.py index c5c74a61c6..d3717db395 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1169,7 +1169,9 @@ def props_to_record( return props -def parse_properties(text_to_parse: str, output_differs: bool, props_to_record: list): +def parse_properties( + text_to_parse: str, output_differs: bool, props_to_record: list, suffix: str = "" +): """ Record the given properties in the report by parsing their values from the text. """ @@ -1179,7 +1181,7 @@ def parse_properties(text_to_parse: str, output_differs: bool, props_to_record: for prop in props_to_record: if prop == MLD or prop == DMX_MLD: mld = float(re.search(MLD_PATTERN, text_to_parse).groups(1)[0]) - props[prop] = mld + props[prop + suffix] = mld elif prop == MAX_ABS_DIFF or prop == DMX_DIFF: max_diff = 0 if output_differs: @@ -1187,33 +1189,33 @@ def parse_properties(text_to_parse: str, output_differs: bool, props_to_record: max_diff = match.groups(1)[0] else: raise MaxDiffPatternNotFound() - props[prop] = max_diff + props[prop + suffix] = max_diff elif prop == SSNR or prop == DMX_SSNR: ssnrs = re.findall(SSNR_PATTERN, text_to_parse) min_ssnr = min(ssnrs) min_ssnr_channel = ssnrs.index(min_ssnr) prefix = "MIN" if prop == SSNR else "DMX" - props[f"{prefix}_SSNR"] = min_ssnr - props[f"{prefix}_SSNR_CHANNEL"] = min_ssnr_channel + props[f"{prefix}_SSNR" + suffix] = min_ssnr + props[f"{prefix}_SSNR_CHANNEL" + suffix] = min_ssnr_channel elif prop == ODG: odgs = re.findall(ODG_PATTERN, text_to_parse) min_odg = min(odgs) min_odg_channel = odgs.index(min_odg) - props["MIN_ODG"] = min_odg - props["MIN_ODG_CHANNEL"] = min_odg_channel + props["MIN_ODG" + suffix] = min_odg + props["MIN_ODG_CHANNEL" + suffix] = min_odg_channel elif prop == MAX_ENC_DIFF: search_result = re.search(MAX_ENC_DIFF_PATTERN, text_to_parse) max_enc_diff_ratio = 0.0 max_enc_diff_param_name = "" if search_result: max_enc_diff_param_name, _, max_enc_diff_ratio = search_result.groups(0) - props[MAX_ENC_DIFF] = float(max_enc_diff_ratio) - props[MAX_ENC_DIFF_PARAM_NAME] = max_enc_diff_param_name + props[MAX_ENC_DIFF + suffix] = float(max_enc_diff_ratio) + props[MAX_ENC_DIFF_PARAM_NAME + suffix] = max_enc_diff_param_name elif prop == DELTA_ODG: delta_odg = re.search(DELTA_ODG_PATTERN, text_to_parse) if delta_odg: - props["DELTA_ODG"] = delta_odg.groups(1)[0] + props["DELTA_ODG" + suffix] = delta_odg.groups(1)[0] return props -- GitLab From 024f2a722f937b86133f82829bad8b5c783f4cfe Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Fri, 14 Mar 2025 10:40:23 +0100 Subject: [PATCH 035/106] stereo-only config file for testing the split comparison --- ...self_test_ltv_basop_encoder_split_test.prm | 150 ++++++++++++++++++ 1 file changed, 150 insertions(+) create mode 100644 scripts/config/self_test_ltv_basop_encoder_split_test.prm diff --git a/scripts/config/self_test_ltv_basop_encoder_split_test.prm b/scripts/config/self_test_ltv_basop_encoder_split_test.prm new file mode 100644 index 0000000000..724b8f67bc --- /dev/null +++ b/scripts/config/self_test_ltv_basop_encoder_split_test.prm @@ -0,0 +1,150 @@ +// Self-test parameter file +// +// - each test must have a tag (unique name) which must be entered as a comment (you can use // /* or rem comment) +// - the following line must be the encoder command line +// - the following line must be the decoder command line +// - if the name of the output file are exactly the same as +// the name of the test vector located in ./testv directory, these files will be compared for bit-exactness +// (the easiest way how to achieve this is to use the name of the test vector itself, as shown below) + + +// stereo bitrate switching from 13.2 kbps to 128 kbps, 32kHz in, 32kHz out +../IVAS_cod -stereo ../scripts/switchPaths/sw_13k2_to_128k_10fr.bin 32 testv/ltv32_STEREO.wav bit +../IVAS_dec STEREO 32 bit testv/ltv32_STEREO.wav_stereo_sw_32-32.tst + +// stereo at 96 kbps, 32kHz in, 32kHz out +../IVAS_cod -stereo 96000 32 testv/ltv32_STEREO.wav bit +../IVAS_dec STEREO 32 bit testv/ltv32_STEREO.wav_stereo_96000_32-32.tst + +// stereo at 96 kbps, 16kHz in, 16kHz out +../IVAS_cod -stereo 96000 16 testv/ltv16_STEREO.wav bit +../IVAS_dec STEREO 16 bit testv/ltv16_STEREO.wav_stereo_96000_16-16.tst + +// stereo at 64 kbps, 32kHz in, 32kHz out, bandwidth switching +../IVAS_cod -max_band testv/ivas_bws_20fr_start_FB.txt -stereo 64000 32 testv/ltv32_STEREO.wav bit +../IVAS_dec STEREO 32 bit testv/ltv32_STEREO.wav_stereo_64000_32-32.tst + +// stereo at 64 kbps, 16kHz in, 16kHz out +../IVAS_cod -stereo 64000 16 testv/ltv16_STEREO.wav bit +../IVAS_dec STEREO 16 bit testv/ltv16_STEREO.wav_stereo_64000_16-16.tst + +// stereo at 48 kbps, 48 kHz in, 48 kHz out, DTX on, bandwidth switching +../IVAS_cod -max_band testv/ivas_bws_20fr_start_SWB.txt -stereo -dtx 48000 48 testv/ltv48_STEREO.wav bit +../IVAS_dec STEREO 48 bit testv/ltv48_STEREO.wav_stereo_48000_48-48_DTX.tst + +// stereo at 48 kbps, 32kHz in, 32kHz out +../IVAS_cod -stereo 48000 32 testv/ltv32_STEREO.wav bit +../IVAS_dec STEREO 32 bit testv/ltv32_STEREO.wav_stereo_48000_32-32.tst + +// stereo at 48 kbps, 32 kHz in, 32 kHz out, DTX on +../IVAS_cod -stereo -dtx 48000 32 testv/ltv32_STEREO.wav bit +../IVAS_dec STEREO 32 bit testv/ltv32_STEREO.wav_stereo_48000_32-32_DTX.tst + +// stereo at 48 kbps, 16kHz in, 16kHz out +../IVAS_cod -stereo 48000 16 testv/ltv16_STEREO.wav bit +../IVAS_dec STEREO 16 bit testv/ltv16_STEREO.wav_stereo_48000_16-16.tst + +// stereo at 48 kbps, 16 kHz in, 16 kHz out, DTX on +../IVAS_cod -stereo -dtx 48000 16 testv/ltv16_STEREO.wav bit +../IVAS_dec STEREO 16 bit testv/ltv16_STEREO.wav_stereo_48000_16-16_DTX.tst + +// stereo at 32 kbps, 48kHz in, 48kHz out, bandwidth switching +../IVAS_cod -stereo -max_band testv/bwidth_cntl.txt 32000 48 testv/ltv48_STEREO.wav bit +../IVAS_dec STEREO 48 bit testv/ltv48_STEREO.wav_stereo_32000_48-48_bandwidth_sw.tst + +// stereo at 32 kbps, 48kHz in, 48kHz out, DTX on, bandwidth switching +../IVAS_cod -max_band testv/ivas_bws_20fr_start_WB.txt -stereo -dtx 32000 48 testv/ltv48_STEREO.wav bit +../IVAS_dec STEREO 48 bit testv/ltv48_STEREO.wav_stereo_32000_48-48_DTX.tst + +// stereo at 32 kbps, 32kHz in, 48kHz out, STEREO out +../IVAS_cod -stereo 32000 32 testv/ltv32_STEREO.wav bit +../IVAS_dec STEREO 48 bit testv/ltv32_STEREO.wav_stereo_32000_32-48_STEREO.tst + +// stereo at 32 kbps, 32kHz in, 32kHz out, DTX on, STEREO out +../IVAS_cod -stereo -dtx 32000 32 testv/ltv32_STEREO.wav bit +../IVAS_dec STEREO 32 bit testv/ltv32_STEREO.wav_stereo_32000_32-32_DTX_STEREO.tst + +// stereo at 32 kbps, 16kHz in, 16kHz out, DTX on +../IVAS_cod -dtx -stereo -dtx 32000 16 testv/ltv16_STEREO.wav bit +../IVAS_dec STEREO 16 bit testv/ltv16_STEREO.wav_stereo_32000_16-16_DTX.tst + +// stereo at 32 kbps, 16kHz in, 16kHz out +../IVAS_cod -stereo 32000 16 testv/ltv16_STEREO.wav bit +../IVAS_dec STEREO 16 bit testv/ltv16_STEREO.wav_stereo_32000_16-16.tst + +// stereo at 24.4 kbps, 48kHz in, 48kHz out, DTX on +../IVAS_cod -stereo -dtx 24400 48 testv/ltv48_STEREO.wav bit +../IVAS_dec STEREO 48 bit testv/ltv48_STEREO.wav_24400_48_48_DTX.txt + +// stereo at 24.4 kbps, 32kHz in, 32kHz out, DTX on +../IVAS_cod -stereo -dtx 24400 32 testv/ltv32_STEREO.wav bit +../IVAS_dec STEREO 32 bit testv/ltv32_STEREO.wav_stereo_24400_32-32_DTX.tst + +// stereo at 24.4 kbps, 32kHz in, 32kHz out +../IVAS_cod -stereo 24400 32 testv/ltv32_STEREO.wav bit +../IVAS_dec STEREO 32 bit testv/ltv32_STEREO.wav_stereo_24400_32-32.tst + +// stereo at 24.4 kbps, 16kHz in, 16kHz out, DTX on, STEREO out +../IVAS_cod -stereo -dtx 24400 16 testv/ltv16_STEREO.wav bit +../IVAS_dec STEREO 16 bit testv/ltv16_STEREO.wav_stereo_24400_16-16_DTX_STEREO.tst + +// stereo at 24.4 kbps, 16kHz in, 16kHz out +../IVAS_cod -stereo 24400 16 testv/ltv16_STEREO.wav bit +../IVAS_dec STEREO 16 bit testv/ltv16_STEREO.wav_stereo_24400_16-16.tst + +// stereo at 16.4 kbps, 32kHz in, 32kHz out +../IVAS_cod -stereo 16400 32 testv/ltv32_STEREO.wav bit +../IVAS_dec STEREO 32 bit testv/ltv32_STEREO.wav_stereo_16400_32-32.tst + +// stereo at 16.4 kbps, 32kHz in, 16kHz out, DTX on +../IVAS_cod -stereo -dtx 16400 32 testv/ltv32_STEREO.wav bit +../IVAS_dec STEREO 16 bit testv/ltv32_STEREO.wav_stereo_16400_32-16_DTX.tst + +// stereo at 16.4 kbps, 16kHz in, 16kHz out, DTX on +../IVAS_cod -stereo -dtx 16400 16 testv/ltv16_STEREO.wav bit +../IVAS_dec STEREO 16 bit testv/ltv16_STEREO.wav_stereo_16400_16-16_DTX.tst + +// stereo at 16.4 kbps, 16kHz in, 16kHz out +../IVAS_cod -stereo 16400 16 testv/ltv16_STEREO.wav bit +../IVAS_dec STEREO 16 bit testv/ltv16_STEREO.wav_stereo_16400_16-16.tst + +// stereo at 13.2 kbps, 48kHz in, 48kHz out +../IVAS_cod -stereo 13200 48 testv/ltv48_STEREO.wav bit +../IVAS_dec STEREO 48 bit testv/ltv48_STEREO.wav_stereo_13200_48-48.tst + +// stereo at 13.2 kbps, 32kHz in, 32kHz out, DTX on +../IVAS_cod -stereo -dtx 13200 32 testv/ltv32_STEREO.wav bit +../IVAS_dec STEREO 32 bit testv/ltv32_STEREO.wav_stereo_13200_32-32_DTX.tst + +// stereo at 13.2 kbps, 32kHz in, 32kHz out +../IVAS_cod -stereo 13200 32 testv/ltv32_STEREO.wav bit +../IVAS_dec STEREO 32 bit testv/ltv32_STEREO.wav_stereo_13200_32-32.tst + +// stereo at 13.2 kbps, 16kHz in, 16kHz out, DTX on +../IVAS_cod -stereo -dtx 13200 16 testv/ltv16_STEREO.wav bit +../IVAS_dec STEREO 16 bit testv/ltv16_STEREO.wav_stereo_13200_16-16_DTX.tst + +// stereo at 13.2 kbps, 16kHz in, 16kHz out +../IVAS_cod -stereo 13200 16 testv/ltv16_STEREO.wav bit +../IVAS_dec STEREO 16 bit testv/ltv16_STEREO.wav_stereo_13200_16-16.tst + +// stereo at 128 kbps, 48kHz in, 48kHz out, STEREO out +../IVAS_cod -stereo 128000 48 testv/ltv48_STEREO.wav bit +../IVAS_dec STEREO 48 bit testv/ltv48_STEREO.wav_stereo_128000_48-48_STEREO.tst + +// stereo at 128 kbps, 32kHz in, 32kHz out +../IVAS_cod -stereo 128000 32 testv/ltv32_STEREO.wav bit +../IVAS_dec STEREO 32 bit testv/ltv32_STEREO.wav_stereo_128000_32-32.tst + +// stereo at 128 kbps, 16kHz in, 16kHz out +../IVAS_cod -stereo 128000 16 testv/ltv16_STEREO.wav bit +../IVAS_dec STEREO 16 bit testv/ltv16_STEREO.wav_stereo_128000_16-16.tst + +// Stereo downmix to bit-exact EVS at 24400 kbps, 48kHz in, 48kHz out +../IVAS_cod -stereo_dmx_evs 24400 48 testv/ltv48_STEREO.wav bit +../IVAS_dec 48 bit testv/ltv48_STEREO.wav_StereoDmxEVS_24400_48-48.tst + +// Stereo downmix to bit-exact EVS at 13200 kbps, 32kHz in, 32kHz out +../IVAS_cod -stereo_dmx_evs 13200 32 testv/ltv32_STEREO.wav bit +../IVAS_dec 32 bit testv/ltv32_STEREO.wav_StereoDmxEVS_13200_32-32.tst + -- GitLab From 5bf37d9176b713a9d2002674e00c57122ee56797 Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Fri, 14 Mar 2025 11:12:09 +0100 Subject: [PATCH 036/106] use format as key for split indices --- tests/conftest.py | 3 ++- tests/constants.py | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/tests/conftest.py b/tests/conftest.py index d3717db395..238c50432f 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1262,7 +1262,8 @@ def get_split_idx(input_file: str, sampling_rate_khz: int) -> Optional[np.ndarra If no list of indices is available for the given input file, an empty array is returned. """ - idx = SPLIT_IDX.get(input_file.lower(), np.empty(0)) + format = input_file.replace(".wav", "").split("_")[-1].lower() + idx = SPLIT_IDX.get(format, np.empty(0)) if len(idx) > 0 and sampling_rate_khz != 16: idx *= sampling_rate_khz // 16 diff --git a/tests/constants.py b/tests/constants.py index 13f46b8553..155d9bdf1d 100644 --- a/tests/constants.py +++ b/tests/constants.py @@ -101,4 +101,4 @@ SPLIT_IDX_LTV_STEREO_16KHZ = np.asarray( ] ) -SPLIT_IDX = {"ltv16_stereo": SPLIT_IDX_LTV_STEREO_16KHZ} +SPLIT_IDX = {"stereo": SPLIT_IDX_LTV_STEREO_16KHZ} -- GitLab From f29a79fc86a3135cd8d1fb389f81c439cb92eb50 Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Fri, 14 Mar 2025 14:11:33 +0100 Subject: [PATCH 037/106] make git check for branch on remote as well in CI container, sometimes the reference branch was not locally available and fetching did not bring it in scope... --- ci/get_float_ref_branch_name.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/get_float_ref_branch_name.sh b/ci/get_float_ref_branch_name.sh index 0d791b6f5c..6c183bfb48 100755 --- a/ci/get_float_ref_branch_name.sh +++ b/ci/get_float_ref_branch_name.sh @@ -50,7 +50,7 @@ if ! [[ "$branchname" =~ $BASOP_PATTERN ]]; then fi float_ref_branchname="${branchname/basop/ref}" -git_result=$(git branch --list "$float_ref_branchname") +git_result=$(git branch -av --list "$float_ref_branchname") # If the branch does not exist, default to "float-pc" if [[ -z "$git_result" ]]; then -- GitLab From 2ded29b229cb4db42b1ca2cfd33178541867a5b8 Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Fri, 14 Mar 2025 14:53:17 +0100 Subject: [PATCH 038/106] change check for ref branch existing --- ci/get_float_ref_branch_name.sh | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/ci/get_float_ref_branch_name.sh b/ci/get_float_ref_branch_name.sh index 6c183bfb48..fe160da023 100755 --- a/ci/get_float_ref_branch_name.sh +++ b/ci/get_float_ref_branch_name.sh @@ -49,17 +49,17 @@ if ! [[ "$branchname" =~ $BASOP_PATTERN ]]; then exit 1 fi +# The float ref branch is just the same name, but with basop -> ref replacement +# Replace only the first occurrence, as "basop" may be present in the later description +# If the format is correct, then before "_basop", only numbers can occur float_ref_branchname="${branchname/basop/ref}" -git_result=$(git branch -av --list "$float_ref_branchname") +git_result=$(git branch -av) # If the branch does not exist, default to "float-pc" -if [[ -z "$git_result" ]]; then - branchname="float-pc" -else - # The float ref branch is just the same name, but with basop -> ref replacement - # Replace only the first occurrence, as "basop" may be present in the later description - # If the format is correct, then before "_basop", only numbers can occur +if [[ "$git_result" =~ "$float_ref_branchname" ]]; then branchname="${float_ref_branchname}" +else + branchname="float-pc" fi echo "$branchname" -- GitLab From 16adf89dcb611fd5ae555f588151a325a9c5e1c3 Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Mon, 17 Mar 2025 17:38:24 +0100 Subject: [PATCH 039/106] add split markers for ism files --- tests/constants.py | 60 +++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 59 insertions(+), 1 deletion(-) diff --git a/tests/constants.py b/tests/constants.py index 155d9bdf1d..6413b5f539 100644 --- a/tests/constants.py +++ b/tests/constants.py @@ -100,5 +100,63 @@ SPLIT_IDX_LTV_STEREO_16KHZ = np.asarray( 3219792, ] ) +SPLIT_IDX_LTV_ISM1_16KHZ = np.asarray( + [ + 72000, + 164800, + 233600, + 422400, + 492800, + 592000, + 720000, + 792000, + 873600, + 947200, + 1001600, + 1060805, + 1120000, + 1284809, + 1356800, + 1448000, + 1513600, + 1731200, + 1828480, + 1910400, + 1966409, + 2150400, + 2235088, + 2299552, + 2460800, + ] +) +SPLIT_IDX_LTV_ISM234_16KHZ = np.asarray( + [ + 72000, + 164800, + 233600, + 432000, + 592000, + 720000, + 792000, + 873600, + 937600, + 1216000, + 1448000, + 1513600, + 1731200, + 1828480, + 1931200, + 2156800, + 2235088, + 2299552, + 2460800, + ] +) -SPLIT_IDX = {"stereo": SPLIT_IDX_LTV_STEREO_16KHZ} +SPLIT_IDX = { + "stereo": SPLIT_IDX_LTV_STEREO_16KHZ, + "1ism": SPLIT_IDX_LTV_ISM1_16KHZ, + "2ism": SPLIT_IDX_LTV_ISM234_16KHZ, + "3ism": SPLIT_IDX_LTV_ISM234_16KHZ, + "4ism": SPLIT_IDX_LTV_ISM234_16KHZ, +} -- GitLab From 842de937e300d349580185a5593dc32059f8b9e9 Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Mon, 17 Mar 2025 17:40:30 +0100 Subject: [PATCH 040/106] add markers for mono --- tests/constants.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/constants.py b/tests/constants.py index 6413b5f539..39955e377f 100644 --- a/tests/constants.py +++ b/tests/constants.py @@ -155,6 +155,7 @@ SPLIT_IDX_LTV_ISM234_16KHZ = np.asarray( SPLIT_IDX = { "stereo": SPLIT_IDX_LTV_STEREO_16KHZ, + "mono": SPLIT_IDX_LTV_STEREO_16KHZ, "1ism": SPLIT_IDX_LTV_ISM1_16KHZ, "2ism": SPLIT_IDX_LTV_ISM234_16KHZ, "3ism": SPLIT_IDX_LTV_ISM234_16KHZ, -- GitLab From d74b128392f978031965dccef889a40f89df8e34 Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Wed, 19 Mar 2025 09:38:55 +0100 Subject: [PATCH 041/106] add extra set of testcases to remove for main/float-pc branches --- ci/remove_unsupported_testcases.py | 23 +++++++++++++++++++++-- 1 file changed, 21 insertions(+), 2 deletions(-) diff --git a/ci/remove_unsupported_testcases.py b/ci/remove_unsupported_testcases.py index a4a0496809..e5e184fa83 100644 --- a/ci/remove_unsupported_testcases.py +++ b/ci/remove_unsupported_testcases.py @@ -32,7 +32,7 @@ from pathlib import Path import argparse # Enter tag of testcases to remove here WITHOUT the leading // -TESTCASES = [ +TESTCASES_MAIN = [ "OMASA 2Dir2TC 4ISM at br sw techs 13.2 to 512 kbps start 80 kbps, 48kHz in, 48kHz out, EXT out", "OSBA planar FOA 2ISM at 512 kbps, 48 kHz in, 48 kHz out, BINAURAL out", "4 ISM with extended metadata at 128 kbps, 48 kHz in, 48 kHz out, BINAURAL_ROOM_REVERB out, rendconf dir w id", @@ -86,6 +86,20 @@ TESTCASES = [ "4 ISM with metadata bitrate switching from 32 kbps to 48 kbps, 48 kHz in, 48 kHz out, BINAURAL_ROOM_IR out, object editing", "4 ISM with metadata bitrate switching from 48 kbps to 32 kbps, 48 kHz in, 48 kHz out, BINAURAL out, object editing, JBM Prof 5", ] +TESTCASES_MAIN_PC = [ + "SBA at 128 kbps, 32kHZ in, 32kHz out, BINAURAL_ROOM_REVERB out, Config renderer, HR", + "SBA at 128 kbps, 32kHZ in, 16kHz out, BINAURAL_ROOM_REVERB out (Model from file), HR", + "Planar SBA at 128 kbps, 48kHZ in, 32kHz out, BINAURAL_ROOM_REVERB out (Model from file), Config renderer, HR", + "Multi-channel 7_1_4 at 160 kbps, 48kHz in, 48 kHz out, BINAURAL_ROOM_REVERB out default configuration", + "Multi-channel 5_1 at 80 kbps, 48kHz in, 32kHz out, BINAURAL_ROOM_REVERB out Config renderer, HR", + "OSBA planar 2OA 4ISM at 512 kbps, 48 kHz in, 48 kHz out, BINAURAL ROOM REVERB (Model from file) out", + "SBA 3OA 4ISM at 96 kbps, 48kHz in, 48kHz out, BINAURAL_ROOM_REVERB out custom configuration", + "Multi-channel 7_1_4 at 160 kbps, 48kHz in, 48kHz out, BINAURAL_ROOM_REVERB out Config recreation, HR", + "Multi-channel 5_1_2 at 64 kbps, 48kHz in, 48kHz out, BINAURAL_ROOM_REVERB out Config renderer, HR", + "OSBA 3OA 4ISM at 512 kbps, 48kHz in, 48kHz out, BINAURAL_ROOM_REVERB out", + "OSBA 3ISM 3OA at bitrate switching 13.2 to 512 kbps, 48kHz in, 32kHz out, BINAURAL ROOM REVERB out", + "Multi-channel 5_1 bitrate switching from 13.2 kbps to 512 kbps, 48kHz in, 16kHz out, BINAURAL_ROOM_REVERB out", +] def remove_testcases(cfg: Path, testcases: list): @@ -113,7 +127,12 @@ def remove_testcases(cfg: Path, testcases: list): if __name__ == "__main__": parser = argparse.ArgumentParser() parser.add_argument("cfg_files", nargs="+", type=Path) + parser.add_argument("--use-main-pc-set", action="store_true") args = parser.parse_args() + testcases = TESTCASES_MAIN + if args.use_main_pc_set: + testcases.extend(TESTCASES_MAIN_PC) + for f in args.cfg_files: - remove_testcases(f, TESTCASES) + remove_testcases(f, testcases) -- GitLab From 47ebf75a6ee5f60b1380e600dcb5b40773b9ee08 Mon Sep 17 00:00:00 2001 From: Erik Norvell Date: Wed, 19 Mar 2025 17:41:07 +0100 Subject: [PATCH 042/106] Add handling of split measurements in parse_xml_report.py --- scripts/parse_xml_report.py | 31 ++++++++++++++++++++++++++----- 1 file changed, 26 insertions(+), 5 deletions(-) diff --git a/scripts/parse_xml_report.py b/scripts/parse_xml_report.py index 1f2157ca8f..fc253bb9d6 100644 --- a/scripts/parse_xml_report.py +++ b/scripts/parse_xml_report.py @@ -136,7 +136,17 @@ if __name__ == "__main__": else: testresult = "PASS" - properties_values = [str(properties_found.get(p)) for p in PROPERTIES] + # Extract number of splits, if any + splits = [p.split('_')[-1] for p in properties_found if 'split' in p] + splits = list(dict.fromkeys(splits)) # Remove duplicates, keeping order + + if splits: + properties_values = {} + for s in splits: + properties_suffixed = [p + '_' + s for p in PROPERTIES] + properties_values[s] = [str(properties_found.get(p)) for p in properties_suffixed] + else: + properties_values = [str(properties_found.get(p)) for p in PROPERTIES] # Identify format and category (mode of operation) # For the format, favor the earliest match in the test case name @@ -158,9 +168,16 @@ if __name__ == "__main__": # For ERROR cases, both a FAIL and an ERROR result is generated. # Here, a FAIL would be overwritten with an ERROR result since it has the same name. - results[fmt][cat][fulltestname] = {"Result": testresult} - for propertyname, propertyvalue in zip(PROPERTIES, properties_values): - results[fmt][cat][fulltestname][propertyname] = propertyvalue + if splits: + for s in splits: + fulltestname_split = f"{fulltestname}-{s}" + results[fmt][cat][fulltestname_split] = {"Result": testresult} + for propertyname, propertyvalue in zip(PROPERTIES, properties_values[s]): + results[fmt][cat][fulltestname_split][propertyname] = propertyvalue + else: + results[fmt][cat][fulltestname] = {"Result": testresult} + for propertyname, propertyvalue in zip(PROPERTIES, properties_values): + results[fmt][cat][fulltestname][propertyname] = propertyvalue header = ["testcase", "Format", "Category", "Result"] + PROPERTIES @@ -172,7 +189,11 @@ if __name__ == "__main__": for cat in CATEGORIES: results[fmt][cat] = dict(sorted(results[fmt][cat].items())) for test in results[fmt][cat]: - count[results[fmt][cat][test]["Result"]] += 1 + if splits: + if "split1" in test: + count[results[fmt][cat][test]["Result"]] += 1 + else: + count[results[fmt][cat][test]["Result"]] += 1 line = ( ";".join( [test, fmt, cat] + list(results[fmt][cat][test].values()) -- GitLab From fa52449220b84a181135ccd1a2172ab43f0446a2 Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Tue, 18 Mar 2025 16:45:55 +0100 Subject: [PATCH 043/106] add split marker for MC, MASA, SBA --- tests/constants.py | 147 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 147 insertions(+) diff --git a/tests/constants.py b/tests/constants.py index 39955e377f..939236b08f 100644 --- a/tests/constants.py +++ b/tests/constants.py @@ -152,6 +152,142 @@ SPLIT_IDX_LTV_ISM234_16KHZ = np.asarray( 2460800, ] ) +SPLIT_IDX_LTV_FOA_16KHZ = np.asarray( + [ + 111840, + 176320, + 231200, + 327280, + 359280, + 404384, + 452800, + 465600, + 496272, + 565632, + 735712, + 846624, + 1007184, + 1166720, + 1316640, + 1391840, + 1699200, + 1716800, + 1812640, + 1908640, + 2036304, + 2196320, + 2364592, + 2524624, + ] +) +SPLIT_IDX_LTV_HOA3_16KHZ = np.asarray( + [ + 69328, + 169120, + 339200, + 450144, + 610128, + 685120, + 759600, + 845120, + 930144, + 1090144, + 1240128, + 1313600, + 1619200, + 1640000, + 1704416, + 1759360, + 1855360, + 1887360, + 1932480, + 1982400, + 1993600, + 2024640, + 2136128, + 2232400, + 2328480, + 2424640, + ] +) +SPLIT_IDX_LTV_MASA_1TC_16KHZ = np.asarray( + [ + 112000, + 192012, + 288000, + 376008, + 528000, + 659200, + 716803, + 822406, + 892800, + 1024002, + 1120000, + 1220789, + 1316831, + 1444183, + 1588641, + 1719739, + ] +) +SPLIT_IDX_LTV_MASA_2TC_16KHZ = np.asarray( + [ + 119984, + 255588, + 286417, + 480008, + 560002, + 625505, + 716938, + 944138, + 1037348, + 1076332, + 1131937, + 1295806, + 1479906, + 1616003, + 1743895, + 1792355, + 1920075, + 1968066, + 2112000, + 2207637, + 2317218, + 2436529, + 2613994, + ] +) +SPLIT_IDX_LTV_MC_16KHZ = np.asarray( + [ + 69328, + 169120, + 339200, + 450144, + 610128, + 685120, + 759600, + 845120, + 930144, + 1090144, + 1240128, + 1313600, + 1619200, + 1640000, + 1704416, + 1759360, + 1855360, + 1887360, + 1932480, + 1982400, + 1993600, + 2024640, + 2136128, + 2232400, + 2328480, + 2424640, + 2552400, + ] +) SPLIT_IDX = { "stereo": SPLIT_IDX_LTV_STEREO_16KHZ, @@ -160,4 +296,15 @@ SPLIT_IDX = { "2ism": SPLIT_IDX_LTV_ISM234_16KHZ, "3ism": SPLIT_IDX_LTV_ISM234_16KHZ, "4ism": SPLIT_IDX_LTV_ISM234_16KHZ, + "foa": SPLIT_IDX_LTV_FOA_16KHZ, + "hoa2": SPLIT_IDX_LTV_HOA3_16KHZ, + "hoa3": SPLIT_IDX_LTV_HOA3_16KHZ, + "masa1tc": SPLIT_IDX_LTV_MASA_1TC_16KHZ, + "masa2tc": SPLIT_IDX_LTV_MASA_2TC_16KHZ, + "mc51": SPLIT_IDX_LTV_MC_16KHZ, + # there is always one signal that does something different... + "mc512": SPLIT_IDX_LTV_MC_16KHZ[:-1], + "mc514": SPLIT_IDX_LTV_MC_16KHZ, + "mc71": SPLIT_IDX_LTV_MC_16KHZ, + "mc714": SPLIT_IDX_LTV_MC_16KHZ, } -- GitLab From 2d481a897e16acebd36ad1a277ed63f867197e4e Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Tue, 18 Mar 2025 17:34:45 +0100 Subject: [PATCH 044/106] split markers for OMASA --- tests/conftest.py | 6 ++- tests/constants.py | 95 ++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 100 insertions(+), 1 deletion(-) diff --git a/tests/conftest.py b/tests/conftest.py index 238c50432f..0ff2df3479 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1262,7 +1262,11 @@ def get_split_idx(input_file: str, sampling_rate_khz: int) -> Optional[np.ndarra If no list of indices is available for the given input file, an empty array is returned. """ - format = input_file.replace(".wav", "").split("_")[-1].lower() + input_file = input_file.lower() + if "omasa" in input_file: + format = "_".join(input_file.replace(".wav", "").split("_")[:-1]) + else: + format = input_file.replace(".wav", "").split("_")[-1].lower() idx = SPLIT_IDX.get(format, np.empty(0)) if len(idx) > 0 and sampling_rate_khz != 16: diff --git a/tests/constants.py b/tests/constants.py index 939236b08f..296c3e7556 100644 --- a/tests/constants.py +++ b/tests/constants.py @@ -288,6 +288,97 @@ SPLIT_IDX_LTV_MC_16KHZ = np.asarray( 2552400, ] ) +SPLIT_IDX_LTV_OMASA_1ISM_16KHZ = np.asarray( + [ + 182561, + 250497, + 321192, + 398875, + 462969, + 589104, + 634642, + 704772, + 761550, + 841565, + 905594, + 1077445, + 1147515, + 1204145, + 1284198, + 1348213, + 1474397, + 1519986, + 1590029, + 1686404, + 1726821, + 1790877, + 1872012, + ] +) +SPLIT_IDX_LTV_OMASA_2ISM_16KHZ = np.asarray( + [ + 104025, + 188810, + 248000, + 600000, + 727048, + 832000, + 952951, + 970604, + 1037526, + 1104909, + 1233053, + 1295925, + 1361517, + 1414356, + 1476020, + 1549127, + 1605498, + 1727087, + 1876418, + ] +) +SPLIT_IDX_LTV_OMASA_3ISM_16KHZ = np.asarray( + [ + 155662, + 238144, + 340776, + 415986, + 575167, + 707032, + 832054, + 975997, + 1104017, + 1197434, + 1247993, + 1344330, + 1482134, + 1549575, + 1688766, + 1745715, + ] +) +SPLIT_IDX_LTV_OMASA_4ISM_16KHZ = np.asarray( + [ + 63985, + 192015, + 316768, + 391082, + 500584, + 597477, + 719995, + 838699, + 952531, + 1040041, + 1143121, + 1271008, + 1503986, + 1626407, + 1687966, + 1796807, + 1878304, + ] +) SPLIT_IDX = { "stereo": SPLIT_IDX_LTV_STEREO_16KHZ, @@ -307,4 +398,8 @@ SPLIT_IDX = { "mc514": SPLIT_IDX_LTV_MC_16KHZ, "mc71": SPLIT_IDX_LTV_MC_16KHZ, "mc714": SPLIT_IDX_LTV_MC_16KHZ, + "omasa_1ism": SPLIT_IDX_LTV_OMASA_1ISM_16KHZ, + "omasa_2ism": SPLIT_IDX_LTV_OMASA_2ISM_16KHZ, + "omasa_3ism": SPLIT_IDX_LTV_OMASA_3ISM_16KHZ, + "omasa_4ism": SPLIT_IDX_LTV_OMASA_4ISM_16KHZ, } -- GitLab From c74b0e7575459b4feabc04bc61d787afd6c1daba Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Thu, 20 Mar 2025 10:03:54 +0100 Subject: [PATCH 045/106] add osba split markers --- tests/conftest.py | 5 +++++ tests/constants.py | 54 ++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 59 insertions(+) diff --git a/tests/conftest.py b/tests/conftest.py index 0ff2df3479..a6e3e7596c 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1265,6 +1265,11 @@ def get_split_idx(input_file: str, sampling_rate_khz: int) -> Optional[np.ndarra input_file = input_file.lower() if "omasa" in input_file: format = "_".join(input_file.replace(".wav", "").split("_")[:-1]) + elif "osba" in input_file: + if "foa" in input_file: + format = "osba_foa" + else: + format = "osba_hoa" else: format = input_file.replace(".wav", "").split("_")[-1].lower() idx = SPLIT_IDX.get(format, np.empty(0)) diff --git a/tests/constants.py b/tests/constants.py index 296c3e7556..caa2c67247 100644 --- a/tests/constants.py +++ b/tests/constants.py @@ -379,6 +379,58 @@ SPLIT_IDX_LTV_OMASA_4ISM_16KHZ = np.asarray( 1878304, ] ) +SPLIT_IDX_LTV_OSBA_FOA_16KHZ = np.asarray( + [ + 110243, + 176353, + 231257, + 327228, + 406430, + 453959, + 465000, + 496215, + 565603, + 735726, + 846637, + 1166658, + 1316644, + 1391826, + 1697613, + 1716813, + 1812764, + 1908681, + 2036312, + 2193583, + 2361556, + ] +) +SPLIT_IDX_LTV_OSBA_HOA_16KHZ = np.asarray( + [ + 69324, + 169120, + 339217, + 450133, + 739726, + 930163, + 1090182, + 1240385, + 1316803, + 1617775, + 1640120, + 1704436, + 1759348, + 1854498, + 1887201, + 1932676, + 1983352, + 1991890, + 2024762, + 2136276, + 2231825, + 2328364, + 2424105, + ] +) SPLIT_IDX = { "stereo": SPLIT_IDX_LTV_STEREO_16KHZ, @@ -402,4 +454,6 @@ SPLIT_IDX = { "omasa_2ism": SPLIT_IDX_LTV_OMASA_2ISM_16KHZ, "omasa_3ism": SPLIT_IDX_LTV_OMASA_3ISM_16KHZ, "omasa_4ism": SPLIT_IDX_LTV_OMASA_4ISM_16KHZ, + "osba_foa": SPLIT_IDX_LTV_OSBA_FOA_16KHZ, + "osba_hoa": SPLIT_IDX_LTV_OSBA_HOA_16KHZ, } -- GitLab From 9344fdf23bc2a41a255a53eaf32eb4fab8b41239 Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Thu, 20 Mar 2025 11:08:31 +0100 Subject: [PATCH 046/106] run formatter on parse_xml_report.py --- scripts/parse_xml_report.py | 28 ++++++++++++++++------------ 1 file changed, 16 insertions(+), 12 deletions(-) diff --git a/scripts/parse_xml_report.py b/scripts/parse_xml_report.py index fc253bb9d6..6308dbe71f 100644 --- a/scripts/parse_xml_report.py +++ b/scripts/parse_xml_report.py @@ -2,8 +2,6 @@ import argparse import re -import math -import numpy as np from xml.etree import ElementTree """ @@ -81,7 +79,7 @@ if __name__ == "__main__": "--skip_formats", action="store_true", help="Parse without formats and categories. Suitable for general tests which do not match the IVAS categories.", - ) + ) args = parser.parse_args() xml_report = args.xml_report csv_file = args.csv_file @@ -94,7 +92,7 @@ if __name__ == "__main__": FORMATS = IVAS_FORMATS CATEGORIES = IVAS_CATEGORIES if args.clipping: - PROPERTIES += ["ENC_CORE_OVL","MAX_OVL","MIN_OVL"] + PROPERTIES += ["ENC_CORE_OVL", "MAX_OVL", "MIN_OVL"] if args.delta_odg: PROPERTIES += ["DELTA_ODG"] if args.skip_formats: @@ -137,14 +135,16 @@ if __name__ == "__main__": testresult = "PASS" # Extract number of splits, if any - splits = [p.split('_')[-1] for p in properties_found if 'split' in p] - splits = list(dict.fromkeys(splits)) # Remove duplicates, keeping order + splits = [p.split("_")[-1] for p in properties_found if "split" in p] + splits = list(dict.fromkeys(splits)) # Remove duplicates, keeping order if splits: properties_values = {} for s in splits: - properties_suffixed = [p + '_' + s for p in PROPERTIES] - properties_values[s] = [str(properties_found.get(p)) for p in properties_suffixed] + properties_suffixed = [p + "_" + s for p in PROPERTIES] + properties_values[s] = [ + str(properties_found.get(p)) for p in properties_suffixed + ] else: properties_values = [str(properties_found.get(p)) for p in PROPERTIES] @@ -172,8 +172,12 @@ if __name__ == "__main__": for s in splits: fulltestname_split = f"{fulltestname}-{s}" results[fmt][cat][fulltestname_split] = {"Result": testresult} - for propertyname, propertyvalue in zip(PROPERTIES, properties_values[s]): - results[fmt][cat][fulltestname_split][propertyname] = propertyvalue + for propertyname, propertyvalue in zip( + PROPERTIES, properties_values[s] + ): + results[fmt][cat][fulltestname_split][propertyname] = ( + propertyvalue + ) else: results[fmt][cat][fulltestname] = {"Result": testresult} for propertyname, propertyvalue in zip(PROPERTIES, properties_values): @@ -191,8 +195,8 @@ if __name__ == "__main__": for test in results[fmt][cat]: if splits: if "split1" in test: - count[results[fmt][cat][test]["Result"]] += 1 - else: + count[results[fmt][cat][test]["Result"]] += 1 + else: count[results[fmt][cat][test]["Result"]] += 1 line = ( ";".join( -- GitLab From 40ec8adf9cddc459554f193a18021197d913f99f Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Thu, 20 Mar 2025 11:19:17 +0100 Subject: [PATCH 047/106] remove prm file for testing --- ...self_test_ltv_basop_encoder_split_test.prm | 150 ------------------ 1 file changed, 150 deletions(-) delete mode 100644 scripts/config/self_test_ltv_basop_encoder_split_test.prm diff --git a/scripts/config/self_test_ltv_basop_encoder_split_test.prm b/scripts/config/self_test_ltv_basop_encoder_split_test.prm deleted file mode 100644 index 724b8f67bc..0000000000 --- a/scripts/config/self_test_ltv_basop_encoder_split_test.prm +++ /dev/null @@ -1,150 +0,0 @@ -// Self-test parameter file -// -// - each test must have a tag (unique name) which must be entered as a comment (you can use // /* or rem comment) -// - the following line must be the encoder command line -// - the following line must be the decoder command line -// - if the name of the output file are exactly the same as -// the name of the test vector located in ./testv directory, these files will be compared for bit-exactness -// (the easiest way how to achieve this is to use the name of the test vector itself, as shown below) - - -// stereo bitrate switching from 13.2 kbps to 128 kbps, 32kHz in, 32kHz out -../IVAS_cod -stereo ../scripts/switchPaths/sw_13k2_to_128k_10fr.bin 32 testv/ltv32_STEREO.wav bit -../IVAS_dec STEREO 32 bit testv/ltv32_STEREO.wav_stereo_sw_32-32.tst - -// stereo at 96 kbps, 32kHz in, 32kHz out -../IVAS_cod -stereo 96000 32 testv/ltv32_STEREO.wav bit -../IVAS_dec STEREO 32 bit testv/ltv32_STEREO.wav_stereo_96000_32-32.tst - -// stereo at 96 kbps, 16kHz in, 16kHz out -../IVAS_cod -stereo 96000 16 testv/ltv16_STEREO.wav bit -../IVAS_dec STEREO 16 bit testv/ltv16_STEREO.wav_stereo_96000_16-16.tst - -// stereo at 64 kbps, 32kHz in, 32kHz out, bandwidth switching -../IVAS_cod -max_band testv/ivas_bws_20fr_start_FB.txt -stereo 64000 32 testv/ltv32_STEREO.wav bit -../IVAS_dec STEREO 32 bit testv/ltv32_STEREO.wav_stereo_64000_32-32.tst - -// stereo at 64 kbps, 16kHz in, 16kHz out -../IVAS_cod -stereo 64000 16 testv/ltv16_STEREO.wav bit -../IVAS_dec STEREO 16 bit testv/ltv16_STEREO.wav_stereo_64000_16-16.tst - -// stereo at 48 kbps, 48 kHz in, 48 kHz out, DTX on, bandwidth switching -../IVAS_cod -max_band testv/ivas_bws_20fr_start_SWB.txt -stereo -dtx 48000 48 testv/ltv48_STEREO.wav bit -../IVAS_dec STEREO 48 bit testv/ltv48_STEREO.wav_stereo_48000_48-48_DTX.tst - -// stereo at 48 kbps, 32kHz in, 32kHz out -../IVAS_cod -stereo 48000 32 testv/ltv32_STEREO.wav bit -../IVAS_dec STEREO 32 bit testv/ltv32_STEREO.wav_stereo_48000_32-32.tst - -// stereo at 48 kbps, 32 kHz in, 32 kHz out, DTX on -../IVAS_cod -stereo -dtx 48000 32 testv/ltv32_STEREO.wav bit -../IVAS_dec STEREO 32 bit testv/ltv32_STEREO.wav_stereo_48000_32-32_DTX.tst - -// stereo at 48 kbps, 16kHz in, 16kHz out -../IVAS_cod -stereo 48000 16 testv/ltv16_STEREO.wav bit -../IVAS_dec STEREO 16 bit testv/ltv16_STEREO.wav_stereo_48000_16-16.tst - -// stereo at 48 kbps, 16 kHz in, 16 kHz out, DTX on -../IVAS_cod -stereo -dtx 48000 16 testv/ltv16_STEREO.wav bit -../IVAS_dec STEREO 16 bit testv/ltv16_STEREO.wav_stereo_48000_16-16_DTX.tst - -// stereo at 32 kbps, 48kHz in, 48kHz out, bandwidth switching -../IVAS_cod -stereo -max_band testv/bwidth_cntl.txt 32000 48 testv/ltv48_STEREO.wav bit -../IVAS_dec STEREO 48 bit testv/ltv48_STEREO.wav_stereo_32000_48-48_bandwidth_sw.tst - -// stereo at 32 kbps, 48kHz in, 48kHz out, DTX on, bandwidth switching -../IVAS_cod -max_band testv/ivas_bws_20fr_start_WB.txt -stereo -dtx 32000 48 testv/ltv48_STEREO.wav bit -../IVAS_dec STEREO 48 bit testv/ltv48_STEREO.wav_stereo_32000_48-48_DTX.tst - -// stereo at 32 kbps, 32kHz in, 48kHz out, STEREO out -../IVAS_cod -stereo 32000 32 testv/ltv32_STEREO.wav bit -../IVAS_dec STEREO 48 bit testv/ltv32_STEREO.wav_stereo_32000_32-48_STEREO.tst - -// stereo at 32 kbps, 32kHz in, 32kHz out, DTX on, STEREO out -../IVAS_cod -stereo -dtx 32000 32 testv/ltv32_STEREO.wav bit -../IVAS_dec STEREO 32 bit testv/ltv32_STEREO.wav_stereo_32000_32-32_DTX_STEREO.tst - -// stereo at 32 kbps, 16kHz in, 16kHz out, DTX on -../IVAS_cod -dtx -stereo -dtx 32000 16 testv/ltv16_STEREO.wav bit -../IVAS_dec STEREO 16 bit testv/ltv16_STEREO.wav_stereo_32000_16-16_DTX.tst - -// stereo at 32 kbps, 16kHz in, 16kHz out -../IVAS_cod -stereo 32000 16 testv/ltv16_STEREO.wav bit -../IVAS_dec STEREO 16 bit testv/ltv16_STEREO.wav_stereo_32000_16-16.tst - -// stereo at 24.4 kbps, 48kHz in, 48kHz out, DTX on -../IVAS_cod -stereo -dtx 24400 48 testv/ltv48_STEREO.wav bit -../IVAS_dec STEREO 48 bit testv/ltv48_STEREO.wav_24400_48_48_DTX.txt - -// stereo at 24.4 kbps, 32kHz in, 32kHz out, DTX on -../IVAS_cod -stereo -dtx 24400 32 testv/ltv32_STEREO.wav bit -../IVAS_dec STEREO 32 bit testv/ltv32_STEREO.wav_stereo_24400_32-32_DTX.tst - -// stereo at 24.4 kbps, 32kHz in, 32kHz out -../IVAS_cod -stereo 24400 32 testv/ltv32_STEREO.wav bit -../IVAS_dec STEREO 32 bit testv/ltv32_STEREO.wav_stereo_24400_32-32.tst - -// stereo at 24.4 kbps, 16kHz in, 16kHz out, DTX on, STEREO out -../IVAS_cod -stereo -dtx 24400 16 testv/ltv16_STEREO.wav bit -../IVAS_dec STEREO 16 bit testv/ltv16_STEREO.wav_stereo_24400_16-16_DTX_STEREO.tst - -// stereo at 24.4 kbps, 16kHz in, 16kHz out -../IVAS_cod -stereo 24400 16 testv/ltv16_STEREO.wav bit -../IVAS_dec STEREO 16 bit testv/ltv16_STEREO.wav_stereo_24400_16-16.tst - -// stereo at 16.4 kbps, 32kHz in, 32kHz out -../IVAS_cod -stereo 16400 32 testv/ltv32_STEREO.wav bit -../IVAS_dec STEREO 32 bit testv/ltv32_STEREO.wav_stereo_16400_32-32.tst - -// stereo at 16.4 kbps, 32kHz in, 16kHz out, DTX on -../IVAS_cod -stereo -dtx 16400 32 testv/ltv32_STEREO.wav bit -../IVAS_dec STEREO 16 bit testv/ltv32_STEREO.wav_stereo_16400_32-16_DTX.tst - -// stereo at 16.4 kbps, 16kHz in, 16kHz out, DTX on -../IVAS_cod -stereo -dtx 16400 16 testv/ltv16_STEREO.wav bit -../IVAS_dec STEREO 16 bit testv/ltv16_STEREO.wav_stereo_16400_16-16_DTX.tst - -// stereo at 16.4 kbps, 16kHz in, 16kHz out -../IVAS_cod -stereo 16400 16 testv/ltv16_STEREO.wav bit -../IVAS_dec STEREO 16 bit testv/ltv16_STEREO.wav_stereo_16400_16-16.tst - -// stereo at 13.2 kbps, 48kHz in, 48kHz out -../IVAS_cod -stereo 13200 48 testv/ltv48_STEREO.wav bit -../IVAS_dec STEREO 48 bit testv/ltv48_STEREO.wav_stereo_13200_48-48.tst - -// stereo at 13.2 kbps, 32kHz in, 32kHz out, DTX on -../IVAS_cod -stereo -dtx 13200 32 testv/ltv32_STEREO.wav bit -../IVAS_dec STEREO 32 bit testv/ltv32_STEREO.wav_stereo_13200_32-32_DTX.tst - -// stereo at 13.2 kbps, 32kHz in, 32kHz out -../IVAS_cod -stereo 13200 32 testv/ltv32_STEREO.wav bit -../IVAS_dec STEREO 32 bit testv/ltv32_STEREO.wav_stereo_13200_32-32.tst - -// stereo at 13.2 kbps, 16kHz in, 16kHz out, DTX on -../IVAS_cod -stereo -dtx 13200 16 testv/ltv16_STEREO.wav bit -../IVAS_dec STEREO 16 bit testv/ltv16_STEREO.wav_stereo_13200_16-16_DTX.tst - -// stereo at 13.2 kbps, 16kHz in, 16kHz out -../IVAS_cod -stereo 13200 16 testv/ltv16_STEREO.wav bit -../IVAS_dec STEREO 16 bit testv/ltv16_STEREO.wav_stereo_13200_16-16.tst - -// stereo at 128 kbps, 48kHz in, 48kHz out, STEREO out -../IVAS_cod -stereo 128000 48 testv/ltv48_STEREO.wav bit -../IVAS_dec STEREO 48 bit testv/ltv48_STEREO.wav_stereo_128000_48-48_STEREO.tst - -// stereo at 128 kbps, 32kHz in, 32kHz out -../IVAS_cod -stereo 128000 32 testv/ltv32_STEREO.wav bit -../IVAS_dec STEREO 32 bit testv/ltv32_STEREO.wav_stereo_128000_32-32.tst - -// stereo at 128 kbps, 16kHz in, 16kHz out -../IVAS_cod -stereo 128000 16 testv/ltv16_STEREO.wav bit -../IVAS_dec STEREO 16 bit testv/ltv16_STEREO.wav_stereo_128000_16-16.tst - -// Stereo downmix to bit-exact EVS at 24400 kbps, 48kHz in, 48kHz out -../IVAS_cod -stereo_dmx_evs 24400 48 testv/ltv48_STEREO.wav bit -../IVAS_dec 48 bit testv/ltv48_STEREO.wav_StereoDmxEVS_24400_48-48.tst - -// Stereo downmix to bit-exact EVS at 13200 kbps, 32kHz in, 32kHz out -../IVAS_cod -stereo_dmx_evs 13200 32 testv/ltv32_STEREO.wav bit -../IVAS_dec 32 bit testv/ltv32_STEREO.wav_StereoDmxEVS_13200_32-32.tst - -- GitLab From e9bb84c3dc60f72d94f0eaba0962e5cc3152ea99 Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Thu, 20 Mar 2025 12:51:34 +0100 Subject: [PATCH 048/106] add split comp for test_sba testcases also fix running without --split_comparison --- tests/codec_be_on_mr_nonselection/test_sba.py | 41 +++++++++++++++---- 1 file changed, 34 insertions(+), 7 deletions(-) diff --git a/tests/codec_be_on_mr_nonselection/test_sba.py b/tests/codec_be_on_mr_nonselection/test_sba.py index ce6d073942..d97cf9fa90 100644 --- a/tests/codec_be_on_mr_nonselection/test_sba.py +++ b/tests/codec_be_on_mr_nonselection/test_sba.py @@ -35,6 +35,7 @@ __doc__ = """ import os import pytest +import numpy as np from cut_bs import cut_from_start from pathlib import Path @@ -44,6 +45,7 @@ from tests.conftest import ( EncoderFrontend, compare_dmx_signals, parse_properties, + get_split_idx, ) from ..cmp_stats_files import cmp_stats_files from ..constants import TESTV_DIR, MAX_ENC_FILE_LENGTH_DIFF, MAX_ENC_STATS_DIFF @@ -116,6 +118,7 @@ def test_pca_enc( get_enc_stats, compare_to_input, compare_enc_dmx, + split_comparison, ): pca = True bitrate = "256000" @@ -198,6 +201,7 @@ def test_pca_enc( get_odg=get_odg, get_odg_bin=get_odg_bin, compare_to_input=compare_to_input, + split_comparison=split_comparison, ) @@ -237,6 +241,7 @@ def test_sba_enc_system( get_enc_stats, compare_to_input, compare_enc_dmx, + split_comparison, ): plc_pattern = None pca = False @@ -377,6 +382,7 @@ def test_sba_enc_system( get_odg=get_odg, get_odg_bin=get_odg_bin, compare_to_input=compare_to_input, + split_comparison=split_comparison, ) @@ -408,6 +414,7 @@ def test_spar_hoa2_enc_system( get_enc_stats, compare_to_input, compare_enc_dmx, + split_comparison, ): sampling_rate = "48" pca = False @@ -523,6 +530,7 @@ def test_spar_hoa2_enc_system( get_odg=get_odg, get_odg_bin=get_odg_bin, compare_to_input=compare_to_input, + split_comparison=split_comparison, ) @@ -554,6 +562,7 @@ def test_spar_hoa3_enc_system( get_enc_stats, compare_to_input, compare_enc_dmx, + split_comparison, ): sampling_rate = "48" pca = False @@ -663,6 +672,7 @@ def test_spar_hoa3_enc_system( get_odg=get_odg, get_odg_bin=get_odg_bin, compare_to_input=compare_to_input, + split_comparison=split_comparison, ) @@ -698,6 +708,7 @@ def test_sba_enc_BWforce_system( get_enc_stats, compare_to_input, compare_enc_dmx, + split_comparison, ): sid = 0 plc_pattern = None @@ -823,6 +834,7 @@ def test_sba_enc_BWforce_system( get_odg=get_odg, get_odg_bin=get_odg_bin, compare_to_input=compare_to_input, + split_comparison=split_comparison, ) @@ -862,6 +874,7 @@ def test_sba_plc_system( get_enc_stats, compare_to_input, compare_enc_dmx, + split_comparison, ): sid = 0 pca = False @@ -964,6 +977,7 @@ def test_sba_plc_system( get_odg=get_odg, get_odg_bin=get_odg_bin, compare_to_input=compare_to_input, + split_comparison=split_comparison, ) @@ -1140,6 +1154,7 @@ def sba_dec( get_odg=False, get_odg_bin=False, compare_to_input=False, + split_comparison=False, ): dut_pkt_dir = f"{dut_base_path}/sba_bs/pkt" ref_pkt_dir = f"{reference_path}/sba_bs/pkt" @@ -1251,7 +1266,13 @@ def sba_dec( allow_differing_lengths = True sampling_rate_Hz = int(sampling_rate) * 1000 - cmp_result, reason = cmp_pcm( + + split_idx = np.empty(0) + if split_comparison: + input_file = f"{test_vector_path}/{tag}.wav" + split_idx = get_split_idx(str(Path(input_file).stem), int(sampling_rate)) + + output_differs_parts, reason_parts = cmp_pcm( dut_out_file, ref_out_file, output_config, @@ -1267,13 +1288,19 @@ def sba_dec( odg_test=odg_test, odg_ref=odg_ref, scalefac=test_info.config.option.scalefac, + split_idx=split_idx, ) - text_to_parse = reason - props = parse_properties(text_to_parse, cmp_result != 0, props_to_record) - for k, v in props.items(): - dut_decoder_frontend.record_property(k, v) + prop_suffix = [""] + if len(split_idx) > 0: + prop_suffix = [f"_split{i}" for i in range(1, len(split_idx) + 1)] + for output_differs, reason, suffix in zip( + output_differs_parts, reason_parts, prop_suffix + ): + props = parse_properties(reason, output_differs, props_to_record, suffix) + for k, v in props.items(): + dut_decoder_frontend.record_property(k, v) # report compare result - if cmp_result != 0: - pytest.fail(text_to_parse) + if output_differs_parts[0] != 0: + pytest.fail(reason_parts[0]) -- GitLab From ef9c6f989a53a88029dc77dcdca056a5873482b0 Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Thu, 20 Mar 2025 13:57:05 +0100 Subject: [PATCH 049/106] add comparison of whole file along with splits --- .../test_param_file.py | 39 +++++++++++++++--- tests/codec_be_on_mr_nonselection/test_sba.py | 40 ++++++++++++++++--- 2 files changed, 68 insertions(+), 11 deletions(-) diff --git a/tests/codec_be_on_mr_nonselection/test_param_file.py b/tests/codec_be_on_mr_nonselection/test_param_file.py index c9854185ac..a9642d3ba0 100644 --- a/tests/codec_be_on_mr_nonselection/test_param_file.py +++ b/tests/codec_be_on_mr_nonselection/test_param_file.py @@ -593,9 +593,11 @@ def run_test( fs = int(sampling_rate) * 1000 + ### run the comparison tools split_idx = np.empty(0) - if split_comparison: - split_idx = get_split_idx(str(Path(testv_file).stem), int(sampling_rate)) + prop_suffix = [""] + + # 1. run comparison on whole files - this is done always, regardless of the presence of --split_comparison output_differs_parts, reason_parts = cmp_pcm( ref_file, @@ -618,9 +620,36 @@ def run_test( split_idx=split_idx, ) - prop_suffix = [""] - if len(split_idx) > 0: - prop_suffix = [f"_split{i}" for i in range(1, len(split_idx) + 1)] + # 2. run comparison on split files if --split_comparison is given + if split_comparison: + split_idx = get_split_idx(str(Path(testv_file).stem), int(sampling_rate)) + + output_differs_splits, reason_splits = cmp_pcm( + ref_file, + dut_output_file, + out_config_2_nchannels(output_config), + fs, + get_mld=get_mld, + mld_lim=get_mld_lim, + abs_tol=abs_tol, + allow_differing_lengths=allow_differing_lengths, + get_ssnr=get_ssnr, + get_odg=get_odg, + get_odg_bin=get_odg_bin, + odg_input=odg_input, + odg_test=odg_test, + odg_ref=odg_ref, + ref_jbm_tf=ref_tracefile_dec, + cut_jbm_tf=dut_tracefile_dec, + scalefac=test_info.config.option.scalefac, + split_idx=split_idx, + ) + output_differs_parts += output_differs_splits + reason_parts += reason_splits + + prop_suffix = ["_whole"] + [ + f"_split{i}" for i in range(1, len(split_idx) + 1) + ] for output_differs, reason, suffix in zip( output_differs_parts, reason_parts, prop_suffix diff --git a/tests/codec_be_on_mr_nonselection/test_sba.py b/tests/codec_be_on_mr_nonselection/test_sba.py index d97cf9fa90..65c68ae6a6 100644 --- a/tests/codec_be_on_mr_nonselection/test_sba.py +++ b/tests/codec_be_on_mr_nonselection/test_sba.py @@ -1267,10 +1267,11 @@ def sba_dec( sampling_rate_Hz = int(sampling_rate) * 1000 + ### run the comparison tools split_idx = np.empty(0) - if split_comparison: - input_file = f"{test_vector_path}/{tag}.wav" - split_idx = get_split_idx(str(Path(input_file).stem), int(sampling_rate)) + prop_suffix = [""] + + # 1. run comparison on whole files - this is done always, regardless of the presence of --split_comparison output_differs_parts, reason_parts = cmp_pcm( dut_out_file, @@ -1291,9 +1292,36 @@ def sba_dec( split_idx=split_idx, ) - prop_suffix = [""] - if len(split_idx) > 0: - prop_suffix = [f"_split{i}" for i in range(1, len(split_idx) + 1)] + # 2. run comparison on split files if --split_comparison is given + if split_comparison: + input_file = f"{test_vector_path}/{tag}.wav" + split_idx = get_split_idx(str(Path(input_file).stem), int(sampling_rate)) + + output_differs_splits, reason_splits = cmp_pcm( + dut_out_file, + ref_out_file, + output_config, + sampling_rate_Hz, + get_mld=get_mld, + mld_lim=get_mld_lim, + abs_tol=abs_tol, + allow_differing_lengths=allow_differing_lengths, + get_ssnr=get_ssnr, + get_odg=get_odg, + get_odg_bin=get_odg_bin, + odg_input=odg_input, + odg_test=odg_test, + odg_ref=odg_ref, + scalefac=test_info.config.option.scalefac, + split_idx=split_idx, + ) + output_differs_parts += output_differs_splits + reason_parts += reason_splits + + prop_suffix = ["_whole"] + [ + f"_split{i}" for i in range(1, len(split_idx) + 1) + ] + for output_differs, reason, suffix in zip( output_differs_parts, reason_parts, prop_suffix ): -- GitLab From 33bb409c158445c41c27a31a19e8c317d0729ae3 Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Thu, 20 Mar 2025 14:48:01 +0100 Subject: [PATCH 050/106] fix other usages of cmp_pcm --- tests/cmp_pcm.py | 4 +- tests/conftest.py | 2 + tests/renderer/utils.py | 81 ++++++++++++++++++++++------------------- tests/test_26444.py | 2 + 4 files changed, 50 insertions(+), 39 deletions(-) diff --git a/tests/cmp_pcm.py b/tests/cmp_pcm.py index e254f1d59a..56dc5cd63c 100755 --- a/tests/cmp_pcm.py +++ b/tests/cmp_pcm.py @@ -283,5 +283,5 @@ if __name__ == "__main__": args["nchannels"] = out_config_2_nchannels(args.pop("out_config")) result, msg = cmp_pcm(**args) - print(msg) - sys.exit(result) + print(msg[0]) + sys.exit(result[0]) diff --git a/tests/conftest.py b/tests/conftest.py index a6e3e7596c..1d5295d599 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1249,6 +1249,8 @@ def compare_dmx_signals(ref_dmx_files, dut_dmx_files, fs) -> dict: get_ssnr=True, quiet=True, ) + dmx_differs = dmx_differs[0] + reason = reason[0] dmx_props = [DMX_DIFF, DMX_MLD, DMX_SSNR] prop_results = parse_properties(reason, dmx_differs, dmx_props) diff --git a/tests/renderer/utils.py b/tests/renderer/utils.py index 2be90224a6..d2246abf40 100644 --- a/tests/renderer/utils.py +++ b/tests/renderer/utils.py @@ -325,37 +325,37 @@ def run_renderer( odg_ref = out_file_ref[0:-4] + ".BINAURAL.wav" if out_fmt not in PEAQ_SUPPORTED_FMT: - if in_fmt in PEAQ_SUPPORTED_FMT: - new_fmt = in_fmt # MONO or STEREO + if in_fmt in PEAQ_SUPPORTED_FMT: + new_fmt = in_fmt # MONO or STEREO + else: + # If input is META which contains stereo, new_fmt needs to be STEREO. + if in_fmt == "META": + with open(in_file, "r") as scene: + if "STEREO" in scene.read(): + new_fmt = "STEREO" + else: + new_fmt = "BINAURAL" else: - # If input is META which contains stereo, new_fmt needs to be STEREO. - if in_fmt == "META": - with open(in_file,"r") as scene: - if "STEREO" in scene.read(): - new_fmt = "STEREO" - else: - new_fmt = "BINAURAL" - else: - new_fmt = "BINAURAL" - - # Render test to PEAQ supported format (MONO, STEREO or BINAURAL) - cmd2 = RENDERER_CMD[:] - cmd2[2] = str(out_file) # in_file - cmd2[4] = str(out_fmt) # in_fmt - cmd2[6] = odg_test # out_file - cmd2[8] = new_fmt # out_fmt - cmd2[10] = str(sr) - cmd2[0] += BIN_SUFFIX_MERGETARGET # Use IVAS_rend_ref for re-rendering - cmd2[0] += binary_suffix - if "MASA" in str(out_fmt): - cmd2.extend(["-im", out_file + ".met"]) - run_cmd(cmd2, test_info, env) - - # Render ref to BINAURAL with same settings as test - cmd2[2] = str(out_file_ref) # in_file - cmd2[6] = odg_ref # out_file - run_cmd(cmd2, test_info, env) - out_fmt_bin = new_fmt + new_fmt = "BINAURAL" + + # Render test to PEAQ supported format (MONO, STEREO or BINAURAL) + cmd2 = RENDERER_CMD[:] + cmd2[2] = str(out_file) # in_file + cmd2[4] = str(out_fmt) # in_fmt + cmd2[6] = odg_test # out_file + cmd2[8] = new_fmt # out_fmt + cmd2[10] = str(sr) + cmd2[0] += BIN_SUFFIX_MERGETARGET # Use IVAS_rend_ref for re-rendering + cmd2[0] += binary_suffix + if "MASA" in str(out_fmt): + cmd2.extend(["-im", out_file + ".met"]) + run_cmd(cmd2, test_info, env) + + # Render ref to BINAURAL with same settings as test + cmd2[2] = str(out_file_ref) # in_file + cmd2[6] = odg_ref # out_file + run_cmd(cmd2, test_info, env) + out_fmt_bin = new_fmt else: out_fmt_bin = out_fmt odg_test = out_file @@ -365,8 +365,8 @@ def run_renderer( # Render input to match out_fmt_bin using same config as input, but with IVAS_rend_ref cmd[0] += BIN_SUFFIX_MERGETARGET cmd[0] += binary_suffix - cmd[6] = odg_input # out_file - cmd[8] = out_fmt_bin # out_fmt + cmd[6] = odg_input # out_file + cmd[8] = out_fmt_bin # out_fmt run_cmd(cmd, test_info, env) else: odg_input = in_file @@ -389,6 +389,8 @@ def run_renderer( odg_ref=odg_ref, scalefac=test_info.config.option.scalefac, ) + output_differs = output_differs[0] + reason = reason[0] props = parse_properties(reason, output_differs, props_to_record) for k, v in props.items(): @@ -455,7 +457,6 @@ def binauralize_input_and_output( in_sr, out_sr, ): - # Use current folder as location for temporary directory, since scene description does not handle spaces in path with tempfile.TemporaryDirectory(dir=".") as tmp_dir: tmp_dir = Path(tmp_dir) @@ -464,7 +465,7 @@ def binauralize_input_and_output( scene_in = str(tmp_dir.joinpath("scene_in.txt")) # File names for binauralized signals, if needed - ref_input_file_binaural = ref_output_file[0:-4] + ".INPUT.BINAURAL.wav" + ref_input_file_binaural = ref_output_file[0:-4] + ".INPUT.BINAURAL.wav" dut_output_file_binaural = dut_output_file[0:-4] + ".BINAURAL.wav" ref_output_file_binaural = ref_output_file[0:-4] + ".BINAURAL.wav" @@ -492,7 +493,7 @@ def binauralize_input_and_output( ",".join(line.split(",")[:2]) + "\n" ) # Keep only first two elements: azim, elev else: - md_out_file = "NULL" # Cannot truncate NULL, just insert it without modification + md_out_file = "NULL" # Cannot truncate NULL, just insert it without modification truncated_meta_files.append(md_out_file) in_meta_files = truncated_meta_files @@ -522,7 +523,9 @@ def binauralize_input_and_output( if "OSBA" in output_config or "OMASA" in output_config: if "OSBA" in output_config: - output_config = output_config[:-1] + '3' # Temporary fix to handle than IVAS_dec produces HOA3 for all OSBA configs. Needs to be removed when this fix is ported to BASOP. + output_config = ( + output_config[:-1] + "3" + ) # Temporary fix to handle than IVAS_dec produces HOA3 for all OSBA configs. Needs to be removed when this fix is ported to BASOP. scene_description_file( output_config, scene_dut, n_obj, dut_output_file, out_meta_files ) @@ -656,7 +659,11 @@ def binauralize_input_and_output( ) else: ref_input_file_binaural = input_file - return (ref_input_file_binaural, dut_output_file_binaural, ref_output_file_binaural) + return ( + ref_input_file_binaural, + dut_output_file_binaural, + ref_output_file_binaural, + ) def findstr(exp, s, one_element=True): diff --git a/tests/test_26444.py b/tests/test_26444.py index 054242c4c9..a1579dfec9 100644 --- a/tests/test_26444.py +++ b/tests/test_26444.py @@ -176,6 +176,8 @@ def test_evs_26444( get_ssnr=get_ssnr, get_odg=get_odg, ) + output_differs = output_differs[0] + reason = reason[0] props = parse_properties(reason, output_differs, props_to_record) for k, v in props.items(): -- GitLab From 5920a4cdee685a4255d32944ef7d5b12cfd6c93a Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Thu, 20 Mar 2025 16:57:08 +0100 Subject: [PATCH 051/106] add padding zeros to split index --- tests/codec_be_on_mr_nonselection/test_param_file.py | 2 +- tests/codec_be_on_mr_nonselection/test_sba.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/codec_be_on_mr_nonselection/test_param_file.py b/tests/codec_be_on_mr_nonselection/test_param_file.py index a9642d3ba0..3c9095db79 100644 --- a/tests/codec_be_on_mr_nonselection/test_param_file.py +++ b/tests/codec_be_on_mr_nonselection/test_param_file.py @@ -648,7 +648,7 @@ def run_test( reason_parts += reason_splits prop_suffix = ["_whole"] + [ - f"_split{i}" for i in range(1, len(split_idx) + 1) + f"_split{i:03d}" for i in range(1, len(split_idx) + 1) ] for output_differs, reason, suffix in zip( diff --git a/tests/codec_be_on_mr_nonselection/test_sba.py b/tests/codec_be_on_mr_nonselection/test_sba.py index 65c68ae6a6..76f42471a1 100644 --- a/tests/codec_be_on_mr_nonselection/test_sba.py +++ b/tests/codec_be_on_mr_nonselection/test_sba.py @@ -1319,7 +1319,7 @@ def sba_dec( reason_parts += reason_splits prop_suffix = ["_whole"] + [ - f"_split{i}" for i in range(1, len(split_idx) + 1) + f"_split{i:03d}" for i in range(1, len(split_idx) + 1) ] for output_differs, reason, suffix in zip( -- GitLab From 6be165b846d8675798835b3b43914e0e5fd304df Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Thu, 20 Mar 2025 17:17:59 +0100 Subject: [PATCH 052/106] filter out skipped cases b4 loop for one less indent --- scripts/parse_xml_report.py | 138 ++++++++++++++++++------------------ 1 file changed, 69 insertions(+), 69 deletions(-) diff --git a/scripts/parse_xml_report.py b/scripts/parse_xml_report.py index 6308dbe71f..f8577a7560 100644 --- a/scripts/parse_xml_report.py +++ b/scripts/parse_xml_report.py @@ -112,76 +112,76 @@ if __name__ == "__main__": results[fmt][cat] = {} count = {"PASS": 0, "FAIL": 0, "ERROR": 0} + # filter out skipped testcases + testcases = [tc for tc in testcases if tc.find(".//skipped") is not None] + for testcase in testcases: - if testcase.find(".//skipped") is None: - if testcase.get("file") is None: - fulltestname = ( - testcase.get("classname").replace(".", "/") - + ".py::" - + testcase.get("name") - ) - else: - fulltestname = testcase.get("file") + "::" + testcase.get("name") - - properties_found = { - p.get("name"): p.get("value") for p in testcase.findall(".//property") - } - - if testcase.find("failure") is not None: - testresult = "FAIL" - elif testcase.find("error") is not None: - testresult = "ERROR" - else: - testresult = "PASS" - - # Extract number of splits, if any - splits = [p.split("_")[-1] for p in properties_found if "split" in p] - splits = list(dict.fromkeys(splits)) # Remove duplicates, keeping order - - if splits: - properties_values = {} - for s in splits: - properties_suffixed = [p + "_" + s for p in PROPERTIES] - properties_values[s] = [ - str(properties_found.get(p)) for p in properties_suffixed - ] - else: - properties_values = [str(properties_found.get(p)) for p in PROPERTIES] - - # Identify format and category (mode of operation) - # For the format, favor the earliest match in the test case name - fmt = min( - [ - (f, re.search(FORMATS[f], fulltestname, re.IGNORECASE).end()) - for f in FORMATS - if re.search(FORMATS[f], fulltestname, re.IGNORECASE) - ], - key=lambda x: x[1], - )[0] - - # Note that only one category is selected, even though several may match, e.g. bitrate switching + JBM. Here the last match is picked. - cat = [ - c - for c in CATEGORIES - if re.search(CATEGORIES[c], fulltestname, re.IGNORECASE) - ][-1] - - # For ERROR cases, both a FAIL and an ERROR result is generated. - # Here, a FAIL would be overwritten with an ERROR result since it has the same name. - if splits: - for s in splits: - fulltestname_split = f"{fulltestname}-{s}" - results[fmt][cat][fulltestname_split] = {"Result": testresult} - for propertyname, propertyvalue in zip( - PROPERTIES, properties_values[s] - ): - results[fmt][cat][fulltestname_split][propertyname] = ( - propertyvalue - ) - else: - results[fmt][cat][fulltestname] = {"Result": testresult} - for propertyname, propertyvalue in zip(PROPERTIES, properties_values): - results[fmt][cat][fulltestname][propertyname] = propertyvalue + if testcase.get("file") is None: + fulltestname = ( + testcase.get("classname").replace(".", "/") + + ".py::" + + testcase.get("name") + ) + else: + fulltestname = testcase.get("file") + "::" + testcase.get("name") + + properties_found = { + p.get("name"): p.get("value") for p in testcase.findall(".//property") + } + + if testcase.find("failure") is not None: + testresult = "FAIL" + elif testcase.find("error") is not None: + testresult = "ERROR" + else: + testresult = "PASS" + + # Extract number of splits, if any + splits = [p.split("_")[-1] for p in properties_found if "split" in p] + splits = list(dict.fromkeys(splits)) # Remove duplicates, keeping order + + if splits: + properties_values = {} + for s in splits: + properties_suffixed = [p + "_" + s for p in PROPERTIES] + properties_values[s] = [ + str(properties_found.get(p)) for p in properties_suffixed + ] + else: + properties_values = [str(properties_found.get(p)) for p in PROPERTIES] + + # Identify format and category (mode of operation) + # For the format, favor the earliest match in the test case name + fmt = min( + [ + (f, re.search(FORMATS[f], fulltestname, re.IGNORECASE).end()) + for f in FORMATS + if re.search(FORMATS[f], fulltestname, re.IGNORECASE) + ], + key=lambda x: x[1], + )[0] + + # Note that only one category is selected, even though several may match, e.g. bitrate switching + JBM. Here the last match is picked. + cat = [ + c + for c in CATEGORIES + if re.search(CATEGORIES[c], fulltestname, re.IGNORECASE) + ][-1] + + # For ERROR cases, both a FAIL and an ERROR result is generated. + # Here, a FAIL would be overwritten with an ERROR result since it has the same name. + if splits: + for s in splits: + fulltestname_split = f"{fulltestname}-{s}" + results[fmt][cat][fulltestname_split] = {"Result": testresult} + for propertyname, propertyvalue in zip( + PROPERTIES, properties_values[s] + ): + results[fmt][cat][fulltestname_split][propertyname] = propertyvalue + else: + results[fmt][cat][fulltestname] = {"Result": testresult} + for propertyname, propertyvalue in zip(PROPERTIES, properties_values): + results[fmt][cat][fulltestname][propertyname] = propertyvalue header = ["testcase", "Format", "Category", "Result"] + PROPERTIES -- GitLab From ecc40ef3287528cf7b1fff8dc5bb7a55af0352f9 Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Thu, 20 Mar 2025 17:37:57 +0100 Subject: [PATCH 053/106] move some code into functions for better readability --- scripts/parse_xml_report.py | 69 +++++++++++++++++++++---------------- 1 file changed, 40 insertions(+), 29 deletions(-) diff --git a/scripts/parse_xml_report.py b/scripts/parse_xml_report.py index f8577a7560..d4bd7a2cab 100644 --- a/scripts/parse_xml_report.py +++ b/scripts/parse_xml_report.py @@ -49,6 +49,38 @@ EVS_CATEGORIES = { NO_CATEGORIES = {"N/A": r".*"} + +def get_format_from_fulltestname(fulltestname: str) -> str: + # For the format, favor the earliest match in the test case name + fmt = min( + [ + (f, re.search(FORMATS[f], fulltestname, re.IGNORECASE).end()) + for f in FORMATS + if re.search(FORMATS[f], fulltestname, re.IGNORECASE) is not None + ], + key=lambda x: x[1], + )[0] + return fmt + + +def get_category_from_fulltestname(fulltestname: str) -> str: + cat = [ + c for c in CATEGORIES if re.search(CATEGORIES[c], fulltestname, re.IGNORECASE) + ][-1] + return cat + + +def get_testresult(testcase: ElementTree.Element) -> str: + if testcase.find("failure") is not None: + testresult = "FAIL" + elif testcase.find("error") is not None: + testresult = "ERROR" + else: + testresult = "PASS" + + return testresult + + # Main routine if __name__ == "__main__": parser = argparse.ArgumentParser( @@ -116,26 +148,15 @@ if __name__ == "__main__": testcases = [tc for tc in testcases if tc.find(".//skipped") is not None] for testcase in testcases: - if testcase.get("file") is None: - fulltestname = ( - testcase.get("classname").replace(".", "/") - + ".py::" - + testcase.get("name") - ) - else: - fulltestname = testcase.get("file") + "::" + testcase.get("name") + filename = testcase.get( + "file", testcase.get("classname").replace(".", "/") + ".py" + ) + fulltestname = filename + "::" + testcase.get("name") properties_found = { p.get("name"): p.get("value") for p in testcase.findall(".//property") } - if testcase.find("failure") is not None: - testresult = "FAIL" - elif testcase.find("error") is not None: - testresult = "ERROR" - else: - testresult = "PASS" - # Extract number of splits, if any splits = [p.split("_")[-1] for p in properties_found if "split" in p] splits = list(dict.fromkeys(splits)) # Remove duplicates, keeping order @@ -152,21 +173,11 @@ if __name__ == "__main__": # Identify format and category (mode of operation) # For the format, favor the earliest match in the test case name - fmt = min( - [ - (f, re.search(FORMATS[f], fulltestname, re.IGNORECASE).end()) - for f in FORMATS - if re.search(FORMATS[f], fulltestname, re.IGNORECASE) - ], - key=lambda x: x[1], - )[0] - + fmt = get_format_from_fulltestname(fulltestname) # Note that only one category is selected, even though several may match, e.g. bitrate switching + JBM. Here the last match is picked. - cat = [ - c - for c in CATEGORIES - if re.search(CATEGORIES[c], fulltestname, re.IGNORECASE) - ][-1] + cat = get_category_from_fulltestname(fulltestname) + + testresult = get_testresult(testcase) # For ERROR cases, both a FAIL and an ERROR result is generated. # Here, a FAIL would be overwritten with an ERROR result since it has the same name. -- GitLab From d79f81cf7e29f627b27cdbf79b8455ebc3ee9455 Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Thu, 20 Mar 2025 18:18:36 +0100 Subject: [PATCH 054/106] unify split and non-split case and handle "whole" suffix --- scripts/parse_xml_report.py | 54 +++++++++++++++---------------------- 1 file changed, 21 insertions(+), 33 deletions(-) diff --git a/scripts/parse_xml_report.py b/scripts/parse_xml_report.py index d4bd7a2cab..313d2682a3 100644 --- a/scripts/parse_xml_report.py +++ b/scripts/parse_xml_report.py @@ -145,7 +145,7 @@ if __name__ == "__main__": count = {"PASS": 0, "FAIL": 0, "ERROR": 0} # filter out skipped testcases - testcases = [tc for tc in testcases if tc.find(".//skipped") is not None] + testcases = [tc for tc in testcases if tc.find(".//skipped") is None] for testcase in testcases: filename = testcase.get( @@ -153,24 +153,16 @@ if __name__ == "__main__": ) fulltestname = filename + "::" + testcase.get("name") + # only include the properties listed above + # we need to find all occurences with any suffixes to also handle the split-comparison + # runs correctly properties_found = { - p.get("name"): p.get("value") for p in testcase.findall(".//property") + p.get("name"): p.get("value") + for p in testcase.findall(".//property") + if "CHANNEL" not in p.get("name") + and any(p_listed in p.get("name") for p_listed in PROPERTIES) } - # Extract number of splits, if any - splits = [p.split("_")[-1] for p in properties_found if "split" in p] - splits = list(dict.fromkeys(splits)) # Remove duplicates, keeping order - - if splits: - properties_values = {} - for s in splits: - properties_suffixed = [p + "_" + s for p in PROPERTIES] - properties_values[s] = [ - str(properties_found.get(p)) for p in properties_suffixed - ] - else: - properties_values = [str(properties_found.get(p)) for p in PROPERTIES] - # Identify format and category (mode of operation) # For the format, favor the earliest match in the test case name fmt = get_format_from_fulltestname(fulltestname) @@ -179,20 +171,21 @@ if __name__ == "__main__": testresult = get_testresult(testcase) + # get all present suffixes + pattern = re.compile("|".join(PROPERTIES)) + suffixes = set(pattern.sub("", p) for p in properties_found) + + # record the result for all suffixes # For ERROR cases, both a FAIL and an ERROR result is generated. # Here, a FAIL would be overwritten with an ERROR result since it has the same name. - if splits: - for s in splits: - fulltestname_split = f"{fulltestname}-{s}" - results[fmt][cat][fulltestname_split] = {"Result": testresult} - for propertyname, propertyvalue in zip( - PROPERTIES, properties_values[s] - ): - results[fmt][cat][fulltestname_split][propertyname] = propertyvalue - else: - results[fmt][cat][fulltestname] = {"Result": testresult} - for propertyname, propertyvalue in zip(PROPERTIES, properties_values): - results[fmt][cat][fulltestname][propertyname] = propertyvalue + for s in suffixes: + fulltestname_suffix = f"{fulltestname}{s}" + results[fmt][cat][fulltestname_suffix] = {"Result": testresult} + for propertyname in PROPERTIES: + results[fmt][cat][fulltestname_suffix][propertyname] = properties_found[ + f"{propertyname}{s}" + ] + count[testresult] += 1 header = ["testcase", "Format", "Category", "Result"] + PROPERTIES @@ -204,11 +197,6 @@ if __name__ == "__main__": for cat in CATEGORIES: results[fmt][cat] = dict(sorted(results[fmt][cat].items())) for test in results[fmt][cat]: - if splits: - if "split1" in test: - count[results[fmt][cat][test]["Result"]] += 1 - else: - count[results[fmt][cat][test]["Result"]] += 1 line = ( ";".join( [test, fmt, cat] + list(results[fmt][cat][test].values()) -- GitLab From 99627bd3542acb204da35097134d0c8b8e77e3cc Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Fri, 21 Mar 2025 11:11:30 +0100 Subject: [PATCH 055/106] adjust histogram bins and run formatter --- scripts/create_histogram_summary.py | 48 +++++++++++++++++++++++------ 1 file changed, 38 insertions(+), 10 deletions(-) diff --git a/scripts/create_histogram_summary.py b/scripts/create_histogram_summary.py index af9a11de2a..bcb514acd5 100644 --- a/scripts/create_histogram_summary.py +++ b/scripts/create_histogram_summary.py @@ -80,10 +80,19 @@ if __name__ == "__main__": else: limits_per_measure = { "MLD": ("MLD", [0, 1, 2, 3, 4, 5, 10, 20, math.inf]), - "DIFF": ("MAXIMUM ABS DIFF", [0, 16, 256, 1024, 2048, 4096, 8192, 16384, 32769]), + "DIFF": ( + "MAXIMUM ABS DIFF", + [0, 16, 256, 1024, 2048, 4096, 8192, 16384, 32769], + ), "SSNR": ("MIN_SSNR", [-math.inf, 0, 10, 20, 30, 40, 40, 50, 60, 100]), - "ODG": ("MIN_ODG", [-5, -2, -1, -0.5, -0.4, -0.3, -0.2, -0.1, 0, 0.1, 0.5]), - "DELTA_ODG": ("DELTA_ODG", [-5, -2, -1, -0.5, -0.4, -0.3, -0.2, -0.1, 0, 0.1, 0.5]), + "ODG": ( + "MIN_ODG", + [-5, -4, -3, -2, -1, -0.5, -0.4, -0.3, -0.2, -0.1, 0, 0.1, 0.5], + ), + "DELTA_ODG": ( + "DELTA_ODG", + [-5, -4, -3, -2, -1, -0.5, -0.4, -0.3, -0.2, -0.1, 0, 0.1, 0.5], + ), } (measure_label, limits) = limits_per_measure[measure] @@ -100,18 +109,28 @@ if __name__ == "__main__": results_sorted[testcase][k] = val if limits is None: - vals = [float(x) for x in [m[measure_label] for m in results_sorted.values() if m[measure_label] != "None" and m[measure_label] != ""]] + vals = [ + float(x) + for x in [ + m[measure_label] + for m in results_sorted.values() + if m[measure_label] != "None" and m[measure_label] != "" + ] + ] start = min(vals) f = 10 ** (2 - int(np.floor(np.log10(abs(start)))) - 1) - start = np.floor(start*f)/f - step = (max(vals) - start)/10 + start = np.floor(start * f) / f + step = (max(vals) - start) / 10 f = 10 ** (2 - int(np.floor(np.log10(abs(step)))) - 1) - step = np.ceil(step*f)/f - limits = np.arange(start, 10*step, step) + step = np.ceil(step * f) / f + limits = np.arange(start, 10 * step, step) # Output CSV file with open(csv_summary, "w") as fp: - limits_labels = [f"{a:g}" for a in limits] + ["","None"] # Put None cases in separate bin + limits_labels = [f"{a:g}" for a in limits] + [ + "", + "None", + ] # Put None cases in separate bin headerline = f"Format;Category;" + ";".join(limits_labels) + "\n" fp.write(headerline) @@ -138,7 +157,16 @@ if __name__ == "__main__": fp.write(line) # Matplotlib histogram - ax.bar(limits_labels, data, 1, align='edge', edgecolor='black', linewidth=0.5, label=cat, bottom=bottom) + ax.bar( + limits_labels, + data, + 1, + align="edge", + edgecolor="black", + linewidth=0.5, + label=cat, + bottom=bottom, + ) bottom += data # Histogram layout -- GitLab From 96a6242140c60df610508c8c8ef6208bda403f0b Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Fri, 21 Mar 2025 11:11:50 +0100 Subject: [PATCH 056/106] add image_dir arg this is needed for handling two sets of histograms in basop ci --- ci/basop-pages/create_summary_page.py | 41 +++++++++++---------------- 1 file changed, 16 insertions(+), 25 deletions(-) diff --git a/ci/basop-pages/create_summary_page.py b/ci/basop-pages/create_summary_page.py index 1c2b464802..3c19bf1a6c 100644 --- a/ci/basop-pages/create_summary_page.py +++ b/ci/basop-pages/create_summary_page.py @@ -4,7 +4,7 @@ from typing import List from create_report_pages import SUBPAGE_TMPL_CSS, FORMATS -title = { +TITLE = { "MLD": "Maximum MLD across channels", "DIFF": "Maximim absolute difference across channels", "SSNR": "Minimum SSNR across channels", @@ -26,8 +26,15 @@ def create_summary_page( id_current: int, job_name: str, measures: List[str], + image_dir: str, ): - images = histogram_summary(job_name, measures) + images = "
" + for m in measures: + images += ( + f"

{TITLE[m]}

\n" + + " ".join([f"" for x in FORMATS]) + + f'\n
summary_{m}.csv
\n\n' + ) new_summary_page = SUBPAGE_TMPL_CSS + SUMMARY_PAGE_TMPL_HTML.format( id_current=id_current, @@ -38,41 +45,25 @@ def create_summary_page( f.write(new_summary_page) -def histogram_summary( - job_name: str, - measures: List[str], -): - images = "
" - for m in measures: - images += ( - f"

{title[m]}

\n" - + " ".join( - [f"" for x in FORMATS] - ) - + f'\n
summary_{m}.csv
\n\n' - ) - return images - - if __name__ == "__main__": parser = argparse.ArgumentParser() parser.add_argument("html_out") parser.add_argument("id_current", type=int) parser.add_argument("job_name") + parser.add_argument("image_dir") parser.add_argument( "--measures", nargs="+", - help=f"List of measures to include in summary. Allowed values: {' '.join(title.keys())}", + help=f"List of measures to include in summary. Allowed values: {' '.join(TITLE.keys())}", default=["MLD", "DIFF", "SSNR", "ODG"], ) args = parser.parse_args() - if not all([m in title for m in args.measures]): - raise ValueError(f"Invalid list of measures: {args.measures}, expected one of {' '.join(title.keys())}") + if not all([m in TITLE for m in args.measures]): + raise ValueError( + f"Invalid list of measures: {args.measures}, expected one of {' '.join(TITLE.keys())}" + ) create_summary_page( - args.html_out, - args.id_current, - args.job_name, - args.measures, + args.html_out, args.id_current, args.job_name, args.measures, args.image_dir ) -- GitLab From d8c74f73411cffcb4d809ecfa169d983da588534 Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Mon, 24 Mar 2025 15:09:32 +0100 Subject: [PATCH 057/106] add assert for sampling rate for debugging --- tests/conftest.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/conftest.py b/tests/conftest.py index 1d5295d599..2a22b90b90 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1264,6 +1264,8 @@ def get_split_idx(input_file: str, sampling_rate_khz: int) -> Optional[np.ndarra If no list of indices is available for the given input file, an empty array is returned. """ + assert sampling_rate_khz in [16, 32, 48] + input_file = input_file.lower() if "omasa" in input_file: format = "_".join(input_file.replace(".wav", "").split("_")[:-1]) -- GitLab From 8f4f93c0901fed6cd054faf586c7cb029a184572 Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Mon, 24 Mar 2025 17:46:54 +0100 Subject: [PATCH 058/106] record format as property --- .../test_param_file.py | 17 ++++++++--- tests/codec_be_on_mr_nonselection/test_sba.py | 9 ++++++ tests/conftest.py | 29 ++++++++++++++++++- tests/renderer/utils.py | 1 + tests/test_26444.py | 1 + 5 files changed, 52 insertions(+), 5 deletions(-) diff --git a/tests/codec_be_on_mr_nonselection/test_param_file.py b/tests/codec_be_on_mr_nonselection/test_param_file.py index 3c9095db79..6146cecdfe 100644 --- a/tests/codec_be_on_mr_nonselection/test_param_file.py +++ b/tests/codec_be_on_mr_nonselection/test_param_file.py @@ -54,6 +54,7 @@ from tests.conftest import ( compare_dmx_signals, log_dbg_msg, get_split_idx, + get_format_from_enc_opts, ) from tests.testconfig import PARAM_FILE from tests.constants import ( @@ -298,6 +299,10 @@ def run_test( "All non-passthrough modes are skipped when --compare-to-input is set" ) + pytest.set_trace() + testcase_props = {} + testcase_props["format"] = get_format_from_enc_opts(enc_opts) + tag_str = convert_test_string_to_tag(test_tag) # evaluate encoder options @@ -375,8 +380,9 @@ def run_test( # avoid double recording of the encoder diff if encoder_only: - props = parse_properties(cmp_result_msg, False, [MAX_ENC_DIFF]) - for k, v in props.items(): + result_props = parse_properties(cmp_result_msg, False, [MAX_ENC_DIFF]) + testcase_props.update(result_props) + for k, v in testcase_props.items(): dut_encoder_frontend.record_property(k, v) if encoder_only: @@ -654,8 +660,11 @@ def run_test( for output_differs, reason, suffix in zip( output_differs_parts, reason_parts, prop_suffix ): - props = parse_properties(reason, output_differs, props_to_record, suffix) - for k, v in props.items(): + result_props = parse_properties( + reason, output_differs, props_to_record, suffix + ) + testcase_props.update(result_props) + for k, v in testcase_props.items(): dut_decoder_frontend.record_property(k, v) metadata_differs = False diff --git a/tests/codec_be_on_mr_nonselection/test_sba.py b/tests/codec_be_on_mr_nonselection/test_sba.py index 76f42471a1..575fefeafe 100644 --- a/tests/codec_be_on_mr_nonselection/test_sba.py +++ b/tests/codec_be_on_mr_nonselection/test_sba.py @@ -251,6 +251,7 @@ def test_sba_enc_system( cut_gain = "1.0" plc_pattern = None cut_testv = True + testcase_props = {"format": "SBA"} if dtx == "1" and bitrate not in ["13200", "16400", "24400", "32000", "64000"]: # skip high bitrates for DTX until DTX issue is resolved @@ -343,6 +344,7 @@ def test_sba_enc_system( props = parse_properties( enc_test_result_msg, enc_test_result != 0, props_to_record ) + props.update(testcase_props) for k, v in props.items(): dut_encoder_frontend.record_property(k, v) @@ -427,6 +429,7 @@ def test_spar_hoa2_enc_system( cut_gain = "1.0" plc_pattern = None cut_testv = False + testcase_props = {"format": "SBA"} if "ltv" in tag: tag = f"ltv{sampling_rate}_HOA2" @@ -491,6 +494,7 @@ def test_spar_hoa2_enc_system( props = parse_properties( enc_test_result_msg, enc_test_result != 0, props_to_record ) + props.update(testcase_props) for k, v in props.items(): dut_encoder_frontend.record_property(k, v) @@ -575,6 +579,7 @@ def test_spar_hoa3_enc_system( cut_gain = "1.0" plc_pattern = None cut_testv = False + testcase_props = {"format": "SBA"} if "ltv" in tag: tag = f"ltv{sampling_rate}_HOA3" @@ -633,6 +638,7 @@ def test_spar_hoa3_enc_system( props = parse_properties( enc_test_result_msg, enc_test_result != 0, props_to_record ) + props.update(testcase_props) for k, v in props.items(): dut_encoder_frontend.record_property(k, v) @@ -795,6 +801,7 @@ def test_sba_enc_BWforce_system( props = parse_properties( enc_test_result_msg, enc_test_result != 0, props_to_record ) + props.update(testcase_props) for k, v in props.items(): dut_encoder_frontend.record_property(k, v) @@ -1160,6 +1167,7 @@ def sba_dec( ref_pkt_dir = f"{reference_path}/sba_bs/pkt" dut_out_dir = f"{dut_base_path}/sba_bs/raw" ref_out_dir = f"{reference_path}/sba_bs/raw" + testcase_props = {"format": "SBA"} check_and_makedir(dut_out_dir) check_and_makedir(ref_out_dir) @@ -1326,6 +1334,7 @@ def sba_dec( output_differs_parts, reason_parts, prop_suffix ): props = parse_properties(reason, output_differs, props_to_record, suffix) + props.update(testcase_props) for k, v in props.items(): dut_decoder_frontend.record_property(k, v) diff --git a/tests/conftest.py b/tests/conftest.py index 2a22b90b90..4b52c85aa4 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -45,7 +45,7 @@ from subprocess import TimeoutExpired, run from tempfile import NamedTemporaryFile from shutil import move import tempfile -from typing import Optional, Union +from typing import Optional, Union, List import numpy as np from .constants import ( DMX_DIFF, @@ -1282,3 +1282,30 @@ def get_split_idx(input_file: str, sampling_rate_khz: int) -> Optional[np.ndarra idx *= sampling_rate_khz // 16 return idx + + +IVAS_ENC_FORMATS = { + "stereo": "Stereo", + "ism": "ISM", + "sba": "SBA", + "masa": "Masa", + "ism_sba": "OSBA", + "ism_masa": "OMASA", + "mc": "Multichannel", + "stereo_dmx_evs": "Stereo DMX EVS", +} +PATTERN_IVAS_ENC_FORMAT = re.compile(r"-(" + r"|".join(IVAS_ENC_FORMATS.keys()) + ")") + + +def get_format_from_enc_opts(enc_opts: str) -> str: + """ + Parse the encoder format from the encoder options by searching for any of the + '-' arguments. If none of them is given, encoder will run in EVS mode. + """ + format = "Mono" + m = re.search(PATTERN_IVAS_ENC_FORMAT, enc_opts) + if m is not None: + enc_format_str = m.groups()[0] + format = IVAS_ENC_FORMATS[enc_format_str] + + return format diff --git a/tests/renderer/utils.py b/tests/renderer/utils.py index d2246abf40..02f7eaecc6 100644 --- a/tests/renderer/utils.py +++ b/tests/renderer/utils.py @@ -393,6 +393,7 @@ def run_renderer( reason = reason[0] props = parse_properties(reason, output_differs, props_to_record) + props["format"] = "renderer" for k, v in props.items(): record_property(k, v) diff --git a/tests/test_26444.py b/tests/test_26444.py index a1579dfec9..d41d22efc3 100644 --- a/tests/test_26444.py +++ b/tests/test_26444.py @@ -180,6 +180,7 @@ def test_evs_26444( reason = reason[0] props = parse_properties(reason, output_differs, props_to_record) + props["format"] = "EVS" for k, v in props.items(): ref_decoder_frontend.record_property(k, v) equal &= not output_differs -- GitLab From 89c65f7176aa9a0b3e3d53c4f5832a5538b0ce52 Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Mon, 24 Mar 2025 18:18:43 +0100 Subject: [PATCH 059/106] record category as property --- .../test_param_file.py | 14 ++++++- tests/codec_be_on_mr_nonselection/test_sba.py | 37 ++++++++++++++++--- tests/constants.py | 6 +++ tests/renderer/utils.py | 2 + 4 files changed, 52 insertions(+), 7 deletions(-) diff --git a/tests/codec_be_on_mr_nonselection/test_param_file.py b/tests/codec_be_on_mr_nonselection/test_param_file.py index 6146cecdfe..86248bc3de 100644 --- a/tests/codec_be_on_mr_nonselection/test_param_file.py +++ b/tests/codec_be_on_mr_nonselection/test_param_file.py @@ -62,6 +62,11 @@ from tests.constants import ( MAX_ENC_STATS_DIFF, SCRIPTS_DIR, MAX_ENC_DIFF, + CAT_NORMAL, + CAT_DTX, + CAT_BITRATE_SWITCHING, + CAT_JBM, + CAT_PLC, ) from tests.renderer.utils import check_and_makedir, binauralize_input_and_output @@ -299,9 +304,9 @@ def run_test( "All non-passthrough modes are skipped when --compare-to-input is set" ) - pytest.set_trace() testcase_props = {} testcase_props["format"] = get_format_from_enc_opts(enc_opts) + testcase_props["category"] = CAT_NORMAL tag_str = convert_test_string_to_tag(test_tag) @@ -322,6 +327,9 @@ def run_test( bitrate = enc_split.pop() in_sr = sampling_rate + if "-dtx" in enc_opts: + testcase_props["category"] = CAT_DTX + # bitrate can be a filename: change it to an absolute path if not bitrate.isdigit(): if compare_enc_dmx: @@ -329,6 +337,7 @@ def run_test( "Rate switching + --compare_enc_dmx currently skipped due to DEBUGGING code limitations with varying number of transport channels" ) bitrate = Path(bitrate[3:]).absolute() + testcase_props["category"] = CAT_BITRATE_SWITCHING testv_base = testv_file.split("/")[-1] if testv_base.endswith(".pcm"): @@ -421,9 +430,9 @@ def run_test( rootdir, decoder_only, ) + testcase_props["category"] = CAT_JBM # check for eid-xor command line - if eid_opts != "": eid_split = eid_opts.split() assert len(eid_split) >= 3, "eid-xor expects at least 3 parameters" @@ -449,6 +458,7 @@ def run_test( rootdir, decoder_only, ) + testcase_props["category"] = CAT_PLC # evaluate decoder options dec_split = dec_opts.split() diff --git a/tests/codec_be_on_mr_nonselection/test_sba.py b/tests/codec_be_on_mr_nonselection/test_sba.py index 575fefeafe..3df1f3a711 100644 --- a/tests/codec_be_on_mr_nonselection/test_sba.py +++ b/tests/codec_be_on_mr_nonselection/test_sba.py @@ -48,7 +48,7 @@ from tests.conftest import ( get_split_idx, ) from ..cmp_stats_files import cmp_stats_files -from ..constants import TESTV_DIR, MAX_ENC_FILE_LENGTH_DIFF, MAX_ENC_STATS_DIFF +from ..constants import CAT_BITRATE_SWITCHING, CAT_DTX, CAT_NORMAL, CAT_PLC, TESTV_DIR, MAX_ENC_FILE_LENGTH_DIFF, MAX_ENC_STATS_DIFF from tests.testconfig import use_ltv from tests.renderer.utils import check_and_makedir, binauralize_input_and_output @@ -190,6 +190,7 @@ def test_pca_enc( gain_flag, keep_files, decoder_only, + testcase_props, cut_gain=cut_gain, cut_testv=cut_testv, plc_pattern=plc_pattern, @@ -251,7 +252,7 @@ def test_sba_enc_system( cut_gain = "1.0" plc_pattern = None cut_testv = True - testcase_props = {"format": "SBA"} + testcase_props = {"format": "SBA", "category": CAT_NORMAL} if dtx == "1" and bitrate not in ["13200", "16400", "24400", "32000", "64000"]: # skip high bitrates for DTX until DTX issue is resolved @@ -289,6 +290,12 @@ def test_sba_enc_system( cut_gain = "1.0" input_config = SBA_FORMAT[abs(int(sba_order))] + if dtx: + testcase_props["category"] = CAT_DTX + try: + int(bitrate) + except ValueError: + testcase_props["category"] = CAT_BITRATE_SWITCHING if not decoder_only: ref_stats_file, dut_stats_file = sba_enc( @@ -373,6 +380,7 @@ def test_sba_enc_system( gain_flag, keep_files, decoder_only, + testcase_props, cut_gain=cut_gain, cut_testv=cut_testv, plc_pattern=plc_pattern, @@ -429,7 +437,7 @@ def test_spar_hoa2_enc_system( cut_gain = "1.0" plc_pattern = None cut_testv = False - testcase_props = {"format": "SBA"} + testcase_props = {"format": "SBA", "category": CAT_NORMAL} if "ltv" in tag: tag = f"ltv{sampling_rate}_HOA2" @@ -523,6 +531,7 @@ def test_spar_hoa2_enc_system( gain_flag, keep_files, decoder_only, + testcase_props, cut_gain=cut_gain, cut_testv=cut_testv, plc_pattern=plc_pattern, @@ -579,7 +588,7 @@ def test_spar_hoa3_enc_system( cut_gain = "1.0" plc_pattern = None cut_testv = False - testcase_props = {"format": "SBA"} + testcase_props = {"format": "SBA", "category": CAT_NORMAL} if "ltv" in tag: tag = f"ltv{sampling_rate}_HOA3" @@ -667,6 +676,7 @@ def test_spar_hoa3_enc_system( gain_flag, keep_files, decoder_only, + testcase_props, cut_gain=cut_gain, cut_testv=cut_testv, plc_pattern=plc_pattern, @@ -726,6 +736,7 @@ def test_sba_enc_BWforce_system( cut_testv = False sampling_rate = sample_rate_bw_idx[0] max_bw = sample_rate_bw_idx[1] + testcase_props = {"format": "SBA", "category": CAT_NORMAL} if dtx == "1" and bitrate not in ["32000", "64000"]: # skip high bitrates for DTX until DTX issue is resolved @@ -746,6 +757,12 @@ def test_sba_enc_BWforce_system( assert 0 input_config = SBA_FORMAT[abs(int(sba_order))] + if dtx: + testcase_props["category"] = CAT_DTX + try: + int(bitrate) + except ValueError: + testcase_props["category"] = CAT_BITRATE_SWITCHING if not decoder_only: ref_stats_file, dut_stats_file = sba_enc( @@ -830,6 +847,7 @@ def test_sba_enc_BWforce_system( gain_flag, keep_files, decoder_only, + testcase_props, cut_gain=cut_gain, cut_testv=cut_testv, plc_pattern=plc_pattern, @@ -889,6 +907,7 @@ def test_sba_plc_system( sba_order = "+1" cut_testv = True output_config = "FOA" + testcase_props = {"format": "SBA", "category": CAT_NORMAL} if dtx == "1" and bitrate not in ["13200", "16400", "24400", "32000", "64000"]: # skip high bitrates for DTX until DTX issue is resolved @@ -925,6 +944,12 @@ def test_sba_plc_system( cut_gain = "1.0" input_config = SBA_FORMAT[abs(int(sba_order))] + if dtx: + testcase_props["category"] = CAT_DTX + try: + int(bitrate) + except ValueError: + testcase_props["category"] = CAT_BITRATE_SWITCHING if not decoder_only: sba_enc( @@ -973,6 +998,7 @@ def test_sba_plc_system( gain_flag, keep_files, decoder_only, + testcase_props, cut_gain=cut_gain, cut_testv=cut_testv, plc_pattern=plc_pattern, @@ -1150,6 +1176,7 @@ def sba_dec( gain_flag, keep_files, decoder_only, + testcase_props cut_gain="1.0", cut_testv=False, plc_pattern=None, @@ -1167,7 +1194,6 @@ def sba_dec( ref_pkt_dir = f"{reference_path}/sba_bs/pkt" dut_out_dir = f"{dut_base_path}/sba_bs/raw" ref_out_dir = f"{reference_path}/sba_bs/raw" - testcase_props = {"format": "SBA"} check_and_makedir(dut_out_dir) check_and_makedir(ref_out_dir) @@ -1194,6 +1220,7 @@ def sba_dec( plc_file = None if plc_pattern is not None: + testcase_props["category"] = CAT_PLC plc_file = f"{TESTV_DIR}/{plc_pattern}.g192" if sid == 1: diff --git a/tests/constants.py b/tests/constants.py index caa2c67247..18e44317ab 100644 --- a/tests/constants.py +++ b/tests/constants.py @@ -66,6 +66,12 @@ ENC_AUX_FILES = [ ["vad_flag", np.int16, "fs/50"], ] +CAT_NORMAL = "normal operation" +CAT_DTX = "DTX" +CAT_PLC = "PLC" +CAT_BITRATE_SWITCHING = "bitrate switching" +CAT_JBM = "JBM" + # lists of indices for splitting of output files # values are in SAMPLES for 16kHz (!). For higher rates, need to multiply SPLIT_IDX_LTV_STEREO_16KHZ = np.asarray( diff --git a/tests/renderer/utils.py b/tests/renderer/utils.py index 02f7eaecc6..6da0a276ab 100644 --- a/tests/renderer/utils.py +++ b/tests/renderer/utils.py @@ -57,6 +57,7 @@ from .constants import ( BIN_SUFFIX_MERGETARGET, PEAQ_SUPPORTED_FMT, ) +from ..constants import CAT_NORMAL sys.path.append(SCRIPTS_DIR) from pyaudio3dtools.audiofile import readfile @@ -394,6 +395,7 @@ def run_renderer( props = parse_properties(reason, output_differs, props_to_record) props["format"] = "renderer" + props["category"] = CAT_NORMAL for k, v in props.items(): record_property(k, v) -- GitLab From 5089e568a95c451ddfb93da453a22486d93185cd Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Tue, 25 Mar 2025 08:38:56 +0100 Subject: [PATCH 060/106] rewrite parse_xml_report.py - only parse properties for all testcases and create dataframe - histogram creation will move to other script --- scripts/parse_xml_report.py | 238 ++++++++---------------------------- 1 file changed, 53 insertions(+), 185 deletions(-) diff --git a/scripts/parse_xml_report.py b/scripts/parse_xml_report.py index 313d2682a3..5c856a0a90 100644 --- a/scripts/parse_xml_report.py +++ b/scripts/parse_xml_report.py @@ -1,76 +1,25 @@ #!/usr/bin/env python3 import argparse -import re +import pandas as pd from xml.etree import ElementTree +from collections import Counter -""" -Parse a junit report and create a summary report. -""" - -PROPERTIES = ["MLD", "MAXIMUM ABS DIFF", "MIN_SSNR", "MIN_ODG"] - -IVAS_FORMATS = { - "Stereo": r"stereo", - "ISM": r"ISM", - "Multichannel": r"Multi-channel|MC", - "MASA": r"(? str: - # For the format, favor the earliest match in the test case name - fmt = min( - [ - (f, re.search(FORMATS[f], fulltestname, re.IGNORECASE).end()) - for f in FORMATS - if re.search(FORMATS[f], fulltestname, re.IGNORECASE) is not None - ], - key=lambda x: x[1], - )[0] - return fmt - - -def get_category_from_fulltestname(fulltestname: str) -> str: - cat = [ - c for c in CATEGORIES if re.search(CATEGORIES[c], fulltestname, re.IGNORECASE) - ][-1] - return cat - - -def get_testresult(testcase: ElementTree.Element) -> str: + +def xml_to_dataframe(xml_report: str) -> pd.DataFrame: + tree = ElementTree.parse(xml_report) + root = tree.getroot() + + testcases = root[0].findall("testcase") + testcases = [tc for tc in testcases if tc.find("skipped") is None] + + parsed_testcases = [parse_testcase(tc) for tc in testcases] + testcase_df = pd.DataFrame(parsed_testcases) + + return testcase_df + + +def get_result_from_testcase(testcase: ElementTree.Element) -> str: if testcase.find("failure") is not None: testresult = "FAIL" elif testcase.find("error") is not None: @@ -81,10 +30,42 @@ def get_testresult(testcase: ElementTree.Element) -> str: return testresult -# Main routine +def parse_testcase(testcase) -> dict: + """ + Get all properties + name for a testcase + """ + ret = {} + + filename = testcase.get("file", testcase.get("classname").replace(".", "/") + ".py") + fulltestname = filename + "::" + testcase.get("name") + ret["testcase"] = fulltestname + + result = get_result_from_testcase(testcase) + ret["result"] = result + + properties = { + p.get("name"): p.get("value") for p in testcase.findall(".//property") + } + ret.update(properties) + + return ret + + +def main(xml_report, csv_file): + df = xml_to_dataframe(xml_report) + df.to_csv(csv_file) + + n_testcases = len(df) + count = Counter(df["result"]) + + print( + f"Parsed testsuite with {n_testcases} tests: {count['PASS']} passes, {count['FAIL']} failures and {count['ERROR']} errors." + ) + + if __name__ == "__main__": parser = argparse.ArgumentParser( - description="Parse a junit report and create an MLD summary report." + description="Parse junit report from IVAS pytest suite and convert to csv file" ) parser.add_argument( "xml_report", @@ -92,119 +73,6 @@ if __name__ == "__main__": help="XML junit report input file, e.g. report-junit.xml", ) parser.add_argument("csv_file", type=str, help="Output CSV file, e.g. report.csv") - parser.add_argument( - "--evs", - action="store_true", - help="Parse using EVS 26.444 formats", - ) - parser.add_argument( - "--clipping", - action="store_true", - help="Extract clipping information. Available if encoder has been run with DEBUGGING active.", - ) - parser.add_argument( - "--delta_odg", - action="store_true", - help="Extract Delta ODG information.", - ) - parser.add_argument( - "--skip_formats", - action="store_true", - help="Parse without formats and categories. Suitable for general tests which do not match the IVAS categories.", - ) - args = parser.parse_args() - xml_report = args.xml_report - csv_file = args.csv_file - FORMATS = IVAS_FORMATS - CATEGORIES = IVAS_CATEGORIES - if args.evs: - FORMATS = EVS_FORMATS - CATEGORIES = EVS_CATEGORIES - else: - FORMATS = IVAS_FORMATS - CATEGORIES = IVAS_CATEGORIES - if args.clipping: - PROPERTIES += ["ENC_CORE_OVL", "MAX_OVL", "MIN_OVL"] - if args.delta_odg: - PROPERTIES += ["DELTA_ODG"] - if args.skip_formats: - FORMATS = NO_FORMATS - CATEGORIES = NO_CATEGORIES - - tree = ElementTree.parse(xml_report) - testsuite = tree.find(".//testsuite") - testcases = tree.findall(".//testcase") - - # Prepare result structure - results = {} - for fmt in FORMATS: - results[fmt] = {} - for cat in CATEGORIES: - results[fmt][cat] = {} - count = {"PASS": 0, "FAIL": 0, "ERROR": 0} - - # filter out skipped testcases - testcases = [tc for tc in testcases if tc.find(".//skipped") is None] - - for testcase in testcases: - filename = testcase.get( - "file", testcase.get("classname").replace(".", "/") + ".py" - ) - fulltestname = filename + "::" + testcase.get("name") - - # only include the properties listed above - # we need to find all occurences with any suffixes to also handle the split-comparison - # runs correctly - properties_found = { - p.get("name"): p.get("value") - for p in testcase.findall(".//property") - if "CHANNEL" not in p.get("name") - and any(p_listed in p.get("name") for p_listed in PROPERTIES) - } - - # Identify format and category (mode of operation) - # For the format, favor the earliest match in the test case name - fmt = get_format_from_fulltestname(fulltestname) - # Note that only one category is selected, even though several may match, e.g. bitrate switching + JBM. Here the last match is picked. - cat = get_category_from_fulltestname(fulltestname) - - testresult = get_testresult(testcase) - - # get all present suffixes - pattern = re.compile("|".join(PROPERTIES)) - suffixes = set(pattern.sub("", p) for p in properties_found) - - # record the result for all suffixes - # For ERROR cases, both a FAIL and an ERROR result is generated. - # Here, a FAIL would be overwritten with an ERROR result since it has the same name. - for s in suffixes: - fulltestname_suffix = f"{fulltestname}{s}" - results[fmt][cat][fulltestname_suffix] = {"Result": testresult} - for propertyname in PROPERTIES: - results[fmt][cat][fulltestname_suffix][propertyname] = properties_found[ - f"{propertyname}{s}" - ] - count[testresult] += 1 - - header = ["testcase", "Format", "Category", "Result"] + PROPERTIES - - # Write CSV file - with open(csv_file, "w") as outfile: - headerline = ";".join(header) + "\n" - outfile.write(headerline) - for fmt in FORMATS: - for cat in CATEGORIES: - results[fmt][cat] = dict(sorted(results[fmt][cat].items())) - for test in results[fmt][cat]: - line = ( - ";".join( - [test, fmt, cat] + list(results[fmt][cat][test].values()) - ) - + "\n" - ) - outfile.write(line) - - print( - f"Parsed testsuite with {count['PASS']+count['FAIL']+count['ERROR']} tests: {count['PASS']} passes, {count['FAIL']} failures and {count['ERROR']} errors." - ) + args = parser.parse_args() + main(args.xml_report, args.csv_file) -- GitLab From 32f381ebcd0be8577ff78024d7e7304869ee6659 Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Tue, 25 Mar 2025 14:30:30 +0100 Subject: [PATCH 061/106] handle duplicate testcase for ERROR results --- scripts/parse_xml_report.py | 68 ++++++++++++++++++++++++------------- 1 file changed, 44 insertions(+), 24 deletions(-) diff --git a/scripts/parse_xml_report.py b/scripts/parse_xml_report.py index 5c856a0a90..f326d13e70 100644 --- a/scripts/parse_xml_report.py +++ b/scripts/parse_xml_report.py @@ -6,6 +6,47 @@ from xml.etree import ElementTree from collections import Counter +class TestcaseParser(dict): + def __init__(self, testcases: list): + super().__init__() + + for tc in testcases: + self.parse_testcase(tc) + + def parse_testcase(self, testcase): + """ + Get all properties + name for a testcase + """ + + filename = testcase.get( + "file", testcase.get("classname").replace(".", "/") + ".py" + ) + fulltestname = filename + "::" + testcase.get("name") + + result = get_result_from_testcase(testcase) + # for ERRORS, two testcases are recorded, one with FAIL and one with ERROR + # if we already have this testcase, do a sanity check and set result to ERROR + if fulltestname in self: + results = [self[fulltestname]["result"], result] + assert any(r == "ERROR" for r in results) + self[fulltestname]["result"] = "ERROR" + return + + ret = {} + ret["testcase"] = fulltestname + ret["result"] = result + properties = { + p.get("name"): p.get("value") for p in testcase.findall(".//property") + } + ret.update(properties) + self[fulltestname] = ret + + def to_df(self) -> pd.DataFrame: + testcases = list(self.values()) + df = pd.DataFrame(testcases) + return df + + def xml_to_dataframe(xml_report: str) -> pd.DataFrame: tree = ElementTree.parse(xml_report) root = tree.getroot() @@ -13,8 +54,8 @@ def xml_to_dataframe(xml_report: str) -> pd.DataFrame: testcases = root[0].findall("testcase") testcases = [tc for tc in testcases if tc.find("skipped") is None] - parsed_testcases = [parse_testcase(tc) for tc in testcases] - testcase_df = pd.DataFrame(parsed_testcases) + testcase_parser = TestcaseParser(testcases) + testcase_df = testcase_parser.to_df() return testcase_df @@ -30,30 +71,9 @@ def get_result_from_testcase(testcase: ElementTree.Element) -> str: return testresult -def parse_testcase(testcase) -> dict: - """ - Get all properties + name for a testcase - """ - ret = {} - - filename = testcase.get("file", testcase.get("classname").replace(".", "/") + ".py") - fulltestname = filename + "::" + testcase.get("name") - ret["testcase"] = fulltestname - - result = get_result_from_testcase(testcase) - ret["result"] = result - - properties = { - p.get("name"): p.get("value") for p in testcase.findall(".//property") - } - ret.update(properties) - - return ret - - def main(xml_report, csv_file): df = xml_to_dataframe(xml_report) - df.to_csv(csv_file) + df.to_csv(csv_file, index=False) n_testcases = len(df) count = Counter(df["result"]) -- GitLab From 02958130f2480a20a5001823484278f49d24f419 Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Wed, 26 Mar 2025 08:04:23 +0100 Subject: [PATCH 062/106] add new histogram script with minimal support for now --- scripts/create_histograms.py | 75 ++++++++++++++++++++++++++++++++++++ 1 file changed, 75 insertions(+) create mode 100644 scripts/create_histograms.py diff --git a/scripts/create_histograms.py b/scripts/create_histograms.py new file mode 100644 index 0000000000..5c10c5545e --- /dev/null +++ b/scripts/create_histograms.py @@ -0,0 +1,75 @@ +#!/usr/bin/env python3 + +import argparse +import math +import pandas as pd +import numpy as np +import matplotlib.pyplot as plt + + +BINS_4_COLS = { + "MLD": [0, 1, 2, 3, 4, 5, 10, 20, math.inf], + "MAXIMUM ABS DIFF": [0, 16, 256, 1024, 2048, 4096, 8192, 16384, 32769], + "SSNR": [-math.inf, 0, 10, 20, 30, 40, 40, 50, 60, 100], + "ODG": [-5, -4, -3, -2, -1, -0.5, -0.4, -0.3, -0.2, -0.1, 0, 0.1, 0.5], + "DELTA_ODG": [-5, -4, -3, -2, -1, -0.5, -0.4, -0.3, -0.2, -0.1, 0, 0.1, 0.5], +} + + +if __name__ == "__main__": + parser = argparse.ArgumentParser( + description="Parses a CSV report and creates a summary report." + ) + parser.add_argument( + "csv_report", + type=str, + help="CSV report file of test cases, e.g. report.csv", + ) + args = parser.parse_args() + df = pd.read_csv(args.csv_report) + + measures = ["MAXIMUM ABS DIFF"] + formats = df["format"].unique() + categories = df["category"].unique() + + for measure in measures: + bins = BINS_4_COLS[measure] + x = [f"{x}" for x in bins] + ["", "ERROR"] + for fmt in formats: + fig, ax = plt.subplots() + bottom = np.zeros(len(x)) + for cat in categories: + data_mask = np.logical_and(df["format"] == fmt, df["category"] == cat) + df_slice = df[data_mask] + error_mask = df_slice["result"] == "ERROR" + n_errors = np.sum(error_mask) + df_hist = df_slice[np.logical_not(error_mask)] + + counts, _ = np.histogram(df_hist[measure], bins) + + data = np.concat([counts, [0], [n_errors], [0]]) + ax.bar( + x, + data, + 1, + align="edge", + edgecolor="black", + linewidth=0.5, + label=cat, + bottom=bottom, + ) + bottom += data + + # Histogram layout + ax.set_title(fmt) + ax.legend(loc="best") + ax.set_xlabel(measure) + if "DIFF" in measure: + ax.set_xticks(range(len(x)), x, rotation=35) + ax.set_ylabel("Number of test cases") + + fig.set_figheight(4) + fig.set_figwidth(6) + plt.tight_layout() + + plt.show() -- GitLab From 3a3a23dad798c89fe96515212c28460674ea048b Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Wed, 26 Mar 2025 08:21:24 +0100 Subject: [PATCH 063/106] record "format" and "category" properties before encoder is run this ensures that these are recorded even if one of the binaries fails --- .../test_param_file.py | 16 +++++--- tests/codec_be_on_mr_nonselection/test_sba.py | 37 ++++++++++++++----- 2 files changed, 37 insertions(+), 16 deletions(-) diff --git a/tests/codec_be_on_mr_nonselection/test_param_file.py b/tests/codec_be_on_mr_nonselection/test_param_file.py index 86248bc3de..8158d3581a 100644 --- a/tests/codec_be_on_mr_nonselection/test_param_file.py +++ b/tests/codec_be_on_mr_nonselection/test_param_file.py @@ -343,6 +343,14 @@ def run_test( if testv_base.endswith(".pcm"): testv_base = testv_base[:-4] + if sim_opts != "": + testcase_props["category"] = CAT_JBM + if eid_opts != "": + testcase_props["category"] = CAT_PLC + + for k, v in testcase_props.items(): + dut_encoder_frontend.record_property(k, v) + assert bitstream_file == "bit" # in the parameter file, only "bit" is used as bitstream file name # -> construct bitstream filename @@ -390,8 +398,7 @@ def run_test( # avoid double recording of the encoder diff if encoder_only: result_props = parse_properties(cmp_result_msg, False, [MAX_ENC_DIFF]) - testcase_props.update(result_props) - for k, v in testcase_props.items(): + for k, v in result_props.items(): dut_encoder_frontend.record_property(k, v) if encoder_only: @@ -430,7 +437,6 @@ def run_test( rootdir, decoder_only, ) - testcase_props["category"] = CAT_JBM # check for eid-xor command line if eid_opts != "": @@ -458,7 +464,6 @@ def run_test( rootdir, decoder_only, ) - testcase_props["category"] = CAT_PLC # evaluate decoder options dec_split = dec_opts.split() @@ -673,8 +678,7 @@ def run_test( result_props = parse_properties( reason, output_differs, props_to_record, suffix ) - testcase_props.update(result_props) - for k, v in testcase_props.items(): + for k, v in result_props.items(): dut_decoder_frontend.record_property(k, v) metadata_differs = False diff --git a/tests/codec_be_on_mr_nonselection/test_sba.py b/tests/codec_be_on_mr_nonselection/test_sba.py index 3df1f3a711..39c12e6131 100644 --- a/tests/codec_be_on_mr_nonselection/test_sba.py +++ b/tests/codec_be_on_mr_nonselection/test_sba.py @@ -48,7 +48,15 @@ from tests.conftest import ( get_split_idx, ) from ..cmp_stats_files import cmp_stats_files -from ..constants import CAT_BITRATE_SWITCHING, CAT_DTX, CAT_NORMAL, CAT_PLC, TESTV_DIR, MAX_ENC_FILE_LENGTH_DIFF, MAX_ENC_STATS_DIFF +from ..constants import ( + CAT_BITRATE_SWITCHING, + CAT_DTX, + CAT_NORMAL, + CAT_PLC, + TESTV_DIR, + MAX_ENC_FILE_LENGTH_DIFF, + MAX_ENC_STATS_DIFF, +) from tests.testconfig import use_ltv from tests.renderer.utils import check_and_makedir, binauralize_input_and_output @@ -131,6 +139,7 @@ def test_pca_enc( cut_testv = True cut_gain = "1.0" plc_pattern = None + testcase_props = {"format": "SBA", "category": CAT_NORMAL} if "ltv" in tag: tag = f"ltv{sampling_rate}_FOA" @@ -142,6 +151,8 @@ def test_pca_enc( assert 0 input_config = SBA_FORMAT[abs(int(sba_order))] + for k, v in testcase_props.items(): + dut_encoder_frontend.record_property(k, v) if not decoder_only: sba_enc( @@ -190,7 +201,6 @@ def test_pca_enc( gain_flag, keep_files, decoder_only, - testcase_props, cut_gain=cut_gain, cut_testv=cut_testv, plc_pattern=plc_pattern, @@ -297,6 +307,9 @@ def test_sba_enc_system( except ValueError: testcase_props["category"] = CAT_BITRATE_SWITCHING + for k, v in testcase_props.items(): + dut_encoder_frontend.record_property(k, v) + if not decoder_only: ref_stats_file, dut_stats_file = sba_enc( dut_encoder_frontend, @@ -380,7 +393,6 @@ def test_sba_enc_system( gain_flag, keep_files, decoder_only, - testcase_props, cut_gain=cut_gain, cut_testv=cut_testv, plc_pattern=plc_pattern, @@ -447,6 +459,8 @@ def test_spar_hoa2_enc_system( assert 0 input_config = SBA_FORMAT[abs(int(sba_order))] + for k, v in testcase_props.items(): + dut_encoder_frontend.record_property(k, v) if not decoder_only: ref_stats_file, dut_stats_file = sba_enc( @@ -531,7 +545,6 @@ def test_spar_hoa2_enc_system( gain_flag, keep_files, decoder_only, - testcase_props, cut_gain=cut_gain, cut_testv=cut_testv, plc_pattern=plc_pattern, @@ -598,6 +611,8 @@ def test_spar_hoa3_enc_system( assert 0 input_config = SBA_FORMAT[abs(int(sba_order))] + for k, v in testcase_props.items(): + dut_encoder_frontend.record_property(k, v) if not decoder_only: ref_stats_file, dut_stats_file = sba_enc( @@ -676,7 +691,6 @@ def test_spar_hoa3_enc_system( gain_flag, keep_files, decoder_only, - testcase_props, cut_gain=cut_gain, cut_testv=cut_testv, plc_pattern=plc_pattern, @@ -764,6 +778,9 @@ def test_sba_enc_BWforce_system( except ValueError: testcase_props["category"] = CAT_BITRATE_SWITCHING + for k, v in testcase_props.items(): + dut_encoder_frontend.record_property(k, v) + if not decoder_only: ref_stats_file, dut_stats_file = sba_enc( dut_encoder_frontend, @@ -847,7 +864,6 @@ def test_sba_enc_BWforce_system( gain_flag, keep_files, decoder_only, - testcase_props, cut_gain=cut_gain, cut_testv=cut_testv, plc_pattern=plc_pattern, @@ -950,6 +966,11 @@ def test_sba_plc_system( int(bitrate) except ValueError: testcase_props["category"] = CAT_BITRATE_SWITCHING + if plc_pattern is not None: + testcase_props["category"] = CAT_PLC + + for k, v in testcase_props.items(): + dut_encoder_frontend.record_property(k, v) if not decoder_only: sba_enc( @@ -998,7 +1019,6 @@ def test_sba_plc_system( gain_flag, keep_files, decoder_only, - testcase_props, cut_gain=cut_gain, cut_testv=cut_testv, plc_pattern=plc_pattern, @@ -1176,7 +1196,6 @@ def sba_dec( gain_flag, keep_files, decoder_only, - testcase_props cut_gain="1.0", cut_testv=False, plc_pattern=None, @@ -1220,7 +1239,6 @@ def sba_dec( plc_file = None if plc_pattern is not None: - testcase_props["category"] = CAT_PLC plc_file = f"{TESTV_DIR}/{plc_pattern}.g192" if sid == 1: @@ -1361,7 +1379,6 @@ def sba_dec( output_differs_parts, reason_parts, prop_suffix ): props = parse_properties(reason, output_differs, props_to_record, suffix) - props.update(testcase_props) for k, v in props.items(): dut_decoder_frontend.record_property(k, v) -- GitLab From 15a4ee2059e673b1d9690c7afee226f31acd9019 Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Wed, 26 Mar 2025 08:35:15 +0100 Subject: [PATCH 064/106] harmonize naming of properties --- tests/constants.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/tests/constants.py b/tests/constants.py index 18e44317ab..60a9f6e846 100644 --- a/tests/constants.py +++ b/tests/constants.py @@ -8,17 +8,17 @@ TESTV_DIR = SCRIPTS_DIR.joinpath("testv") # Properties to record MLD = "MLD" -MAX_ABS_DIFF = "MAXIMUM ABS DIFF" +MAX_ABS_DIFF = "MAX_ABS_DIFF" SSNR = "SSNR" ODG = "ODG" -DELTA_ODG = "Delta-ODG" -MAX_ENC_DIFF = "MAXIMUM ENC DIFF" -MAX_ENC_DIFF_PARAM_NAME = "MAXIMUM ENC DIFF PARAM" +DELTA_ODG = "DELTA_ODG" +MAX_ENC_DIFF = "MAXIMUM_ENC_DIFF" +MAX_ENC_DIFF_PARAM_NAME = "MAXIMUM_ENC_DIFF_PARAM" ENC_CORE_OVL = "ENC_CORE_OVL" MAX_OVL = "MAX_OVL" MIN_OVL = "MIN_OVL" -DMX_DIFF = "DMX MAXIMUM ABS DIFF" -DMX_MLD = "DMX MLD" +DMX_DIFF = "DMX_MAX_ABS_DIFF" +DMX_MLD = "DMX_MLD" DMX_SSNR = "DMX_SSNR" # regex patterns for parsing the output from comparisons -> mainly for BASOP ci -- GitLab From 1eb32d9e4c003ff7eaf9011cfa4cc9f11b8130b3 Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Wed, 26 Mar 2025 08:42:05 +0100 Subject: [PATCH 065/106] add MEASURES_TO_PLOT list --- scripts/create_histograms.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/scripts/create_histograms.py b/scripts/create_histograms.py index 5c10c5545e..20d67c4828 100644 --- a/scripts/create_histograms.py +++ b/scripts/create_histograms.py @@ -9,12 +9,14 @@ import matplotlib.pyplot as plt BINS_4_COLS = { "MLD": [0, 1, 2, 3, 4, 5, 10, 20, math.inf], - "MAXIMUM ABS DIFF": [0, 16, 256, 1024, 2048, 4096, 8192, 16384, 32769], - "SSNR": [-math.inf, 0, 10, 20, 30, 40, 40, 50, 60, 100], - "ODG": [-5, -4, -3, -2, -1, -0.5, -0.4, -0.3, -0.2, -0.1, 0, 0.1, 0.5], + "MAX_ABS_DIFF": [0, 16, 256, 1024, 2048, 4096, 8192, 16384, 32769], + "MIN_SSNR": [-math.inf, 0, 10, 20, 30, 40, 40, 50, 60, 100], + "MIN_ODG": [-5, -4, -3, -2, -1, -0.5, -0.4, -0.3, -0.2, -0.1, 0, 0.1, 0.5], "DELTA_ODG": [-5, -4, -3, -2, -1, -0.5, -0.4, -0.3, -0.2, -0.1, 0, 0.1, 0.5], } +MEASURES_TO_PLOT = ["MAX_ABS_DIFF", "MLD", "MIN_SSNR", "MIN_ODG"] + if __name__ == "__main__": parser = argparse.ArgumentParser( @@ -28,7 +30,7 @@ if __name__ == "__main__": args = parser.parse_args() df = pd.read_csv(args.csv_report) - measures = ["MAXIMUM ABS DIFF"] + measures = MEASURES_TO_PLOT formats = df["format"].unique() categories = df["category"].unique() -- GitLab From 4c855275b3c17624fb2a6fb60627ab82e2c96a2c Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Wed, 26 Mar 2025 11:23:02 +0100 Subject: [PATCH 066/106] add saving of histograms to file --- scripts/create_histograms.py | 57 +++++++++++++++++++++++++++--------- 1 file changed, 43 insertions(+), 14 deletions(-) diff --git a/scripts/create_histograms.py b/scripts/create_histograms.py index 20d67c4828..880be9ac2d 100644 --- a/scripts/create_histograms.py +++ b/scripts/create_histograms.py @@ -2,9 +2,11 @@ import argparse import math +import pathlib import pandas as pd import numpy as np import matplotlib.pyplot as plt +from typing import List BINS_4_COLS = { @@ -18,22 +20,18 @@ BINS_4_COLS = { MEASURES_TO_PLOT = ["MAX_ABS_DIFF", "MLD", "MIN_SSNR", "MIN_ODG"] -if __name__ == "__main__": - parser = argparse.ArgumentParser( - description="Parses a CSV report and creates a summary report." - ) - parser.add_argument( - "csv_report", - type=str, - help="CSV report file of test cases, e.g. report.csv", - ) - args = parser.parse_args() - df = pd.read_csv(args.csv_report) - - measures = MEASURES_TO_PLOT +def create_histograms( + df: pd.DataFrame, + measures: List[str], + output_folder: pathlib.Path, + display_only: bool, +): formats = df["format"].unique() categories = df["category"].unique() + if not display_only: + output_folder.mkdir(exist_ok=True, parents=True) + for measure in measures: bins = BINS_4_COLS[measure] x = [f"{x}" for x in bins] + ["", "ERROR"] @@ -74,4 +72,35 @@ if __name__ == "__main__": fig.set_figwidth(6) plt.tight_layout() - plt.show() + if not display_only: + image_file = f"histogram_{measure}_{fmt}.png" + image_path = output_folder.joinpath(image_file) + plt.savefig(image_path) + + if display_only: + plt.show() + + +if __name__ == "__main__": + parser = argparse.ArgumentParser( + description="Parses a csv file generated by parse_xml_report and creates histograms for the given measures." + ) + parser.add_argument( + "csv_report", + type=str, + help="CSV report file as generated by parse_xml_report.py", + ) + parser.add_argument( + "output_folder", type=pathlib.Path, help="Output folder for writing the " + ) + parser.add_argument( + "--display-only", + action="store_true", + help="Do not write the output files, but display the graphs instead.", + ) + args = parser.parse_args() + df = pd.read_csv(args.csv_report) + + measures = MEASURES_TO_PLOT + + create_histograms(df, measures, args.output_folder, args.display_only) -- GitLab From 00b447a1fd94f9779aa4c5892c3c85a5dfc227a3 Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Wed, 26 Mar 2025 12:11:45 +0100 Subject: [PATCH 067/106] add --no-bins arg --- scripts/create_histograms.py | 24 +++++++++++++++++++++--- 1 file changed, 21 insertions(+), 3 deletions(-) diff --git a/scripts/create_histograms.py b/scripts/create_histograms.py index 880be9ac2d..6181142e86 100644 --- a/scripts/create_histograms.py +++ b/scripts/create_histograms.py @@ -9,7 +9,7 @@ import matplotlib.pyplot as plt from typing import List -BINS_4_COLS = { +BINS_FOR_MEASURES = { "MLD": [0, 1, 2, 3, 4, 5, 10, 20, math.inf], "MAX_ABS_DIFF": [0, 16, 256, 1024, 2048, 4096, 8192, 16384, 32769], "MIN_SSNR": [-math.inf, 0, 10, 20, 30, 40, 40, 50, 60, 100], @@ -20,11 +20,16 @@ BINS_4_COLS = { MEASURES_TO_PLOT = ["MAX_ABS_DIFF", "MLD", "MIN_SSNR", "MIN_ODG"] +def get_bins_for_diff(data: pd.Series): + return np.linspace(data.min(), data.max(), num=10) + + def create_histograms( df: pd.DataFrame, measures: List[str], output_folder: pathlib.Path, display_only: bool, + bins_for_measures=BINS_FOR_MEASURES, ): formats = df["format"].unique() categories = df["category"].unique() @@ -33,10 +38,11 @@ def create_histograms( output_folder.mkdir(exist_ok=True, parents=True) for measure in measures: - bins = BINS_4_COLS[measure] + bins = bins_for_measures.get(measure, get_bins_for_diff(df[measure])) x = [f"{x}" for x in bins] + ["", "ERROR"] for fmt in formats: fig, ax = plt.subplots() + ax.xaxis.set_major_formatter("{x:.1f}") bottom = np.zeros(len(x)) for cat in categories: data_mask = np.logical_and(df["format"] == fmt, df["category"] == cat) @@ -98,9 +104,21 @@ if __name__ == "__main__": action="store_true", help="Do not write the output files, but display the graphs instead.", ) + parser.add_argument( + "--no-bins", + action="store_true", + help="""Do not use the hardcoded bins for creating the spectrograms. +Use this for visualising diff scores.""", + ) args = parser.parse_args() df = pd.read_csv(args.csv_report) measures = MEASURES_TO_PLOT - create_histograms(df, measures, args.output_folder, args.display_only) + bins_for_measures = BINS_FOR_MEASURES + if args.no_bins: + bins_for_measures = {} + + create_histograms( + df, measures, args.output_folder, args.display_only, bins_for_measures + ) -- GitLab From e6d2a6ac472eea2436bc328322ddc68a5817f2aa Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Wed, 26 Mar 2025 12:20:48 +0100 Subject: [PATCH 068/106] add --measures arg --- scripts/create_histograms.py | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/scripts/create_histograms.py b/scripts/create_histograms.py index 6181142e86..691ef4f05a 100644 --- a/scripts/create_histograms.py +++ b/scripts/create_histograms.py @@ -17,7 +17,7 @@ BINS_FOR_MEASURES = { "DELTA_ODG": [-5, -4, -3, -2, -1, -0.5, -0.4, -0.3, -0.2, -0.1, 0, 0.1, 0.5], } -MEASURES_TO_PLOT = ["MAX_ABS_DIFF", "MLD", "MIN_SSNR", "MIN_ODG"] +DEFAULT_MEASURES = ["MAX_ABS_DIFF", "MLD", "MIN_SSNR", "MIN_ODG"] def get_bins_for_diff(data: pd.Series): @@ -110,15 +110,20 @@ if __name__ == "__main__": help="""Do not use the hardcoded bins for creating the spectrograms. Use this for visualising diff scores.""", ) + allowed_measures = " ".join(BINS_FOR_MEASURES.keys()) + parser.add_argument( + "--measures", + nargs="+", + default=DEFAULT_MEASURES, + help=f"Measures to plot from the csv file. One of {allowed_measures}", + ) args = parser.parse_args() df = pd.read_csv(args.csv_report) - measures = MEASURES_TO_PLOT - bins_for_measures = BINS_FOR_MEASURES if args.no_bins: bins_for_measures = {} create_histograms( - df, measures, args.output_folder, args.display_only, bins_for_measures + df, args.measures, args.output_folder, args.display_only, bins_for_measures ) -- GitLab From cab4e0f4d031dc6ab69a5f3f638101bb36623bf0 Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Wed, 26 Mar 2025 12:44:41 +0100 Subject: [PATCH 069/106] add format and category properties for EVS test --- tests/test_26444.py | 37 ++++++++++++++++++++++++++++++++++--- 1 file changed, 34 insertions(+), 3 deletions(-) diff --git a/tests/test_26444.py b/tests/test_26444.py index d41d22efc3..d3305e9984 100644 --- a/tests/test_26444.py +++ b/tests/test_26444.py @@ -40,11 +40,12 @@ import shutil from tests.cmp_pcm import cmp_pcm from tests.conftest import DecoderFrontend, EncoderFrontend, parse_properties +from tests.constants import CAT_BITRATE_SWITCHING, CAT_DTX, CAT_JBM, CAT_NORMAL, CAT_PLC test_dict = {} TEST_DIR = "evs_be_test" -scripts = [ +SCRIPTS = [ "Readme_AMRWB_IO_dec.txt", "Readme_AMRWB_IO_enc.txt", "Readme_EVS_dec.txt", @@ -52,7 +53,16 @@ scripts = [ "Readme_JBM_dec.txt", ] -for s in scripts: +FORMATS_4_SCRIPTS = dict( + zip( + [s.replace(".txt", "") for s in SCRIPTS], + ["AMRWBIO_dec", "AMRWBIO_enc", "EVS_dec", "EVS_enc", "EVS_JBM_dec"], + ) +) + +PATTERN_EVS_FORMAT = re.compile(r"-(" + r"|".join(FORMATS_4_SCRIPTS.keys()) + ")") + +for s in SCRIPTS: with open(os.path.join(TEST_DIR, s), "r", encoding="UTF-8") as fp: tag = "" enc_opts = "" @@ -88,11 +98,33 @@ def test_evs_26444( abs_tol, get_ssnr, get_odg, + record_property, ): enc_opts, dec_opts, diff_opts = test_dict[test_tag] + testcase_props = {} + + # get format prop from test_tag + m = re.search(PATTERN_EVS_FORMAT, test_tag) + assert m is not None + testcase_props["format"] = FORMATS_4_SCRIPTS[m.groups()[0]] diff_opts = diff_opts.replace("./", TEST_DIR + "/") + category = CAT_NORMAL + if "JBM" in test_tag: + category = CAT_JBM + elif "br sw" in test_tag or "bitrate switching" in test_tag: + category = CAT_BITRATE_SWITCHING + elif "%" in test_tag: + category = CAT_PLC + elif "DTX" in test_tag: + category = CAT_DTX + + testcase_props["category"] = category + + for k, v in testcase_props.items(): + record_property(k, v) + if enc_opts: args = enc_opts.split()[1:] @@ -180,7 +212,6 @@ def test_evs_26444( reason = reason[0] props = parse_properties(reason, output_differs, props_to_record) - props["format"] = "EVS" for k, v in props.items(): ref_decoder_frontend.record_property(k, v) equal &= not output_differs -- GitLab From 61bf5f1b9727c615553805e8ed3ef903e5bd962b Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Wed, 26 Mar 2025 12:52:35 +0100 Subject: [PATCH 070/106] remove old histogram script --- scripts/create_histogram_summary.py | 184 ---------------------------- 1 file changed, 184 deletions(-) delete mode 100644 scripts/create_histogram_summary.py diff --git a/scripts/create_histogram_summary.py b/scripts/create_histogram_summary.py deleted file mode 100644 index bcb514acd5..0000000000 --- a/scripts/create_histogram_summary.py +++ /dev/null @@ -1,184 +0,0 @@ -#!/usr/bin/env python3 - -import argparse -import math -import numpy as np - -# These next three lines are added as a precaution in case the gitlab runner -# needs DISPLAY to render the plots, even if they are written to file. -import matplotlib - -matplotlib.use("Agg") -import matplotlib.pyplot as plt -import csv -import os -from parse_xml_report import IVAS_FORMATS, EVS_FORMATS, IVAS_CATEGORIES, EVS_CATEGORIES - -""" -Parses a CSV report and creates a summary report. -""" - - -# Main routine -if __name__ == "__main__": - parser = argparse.ArgumentParser( - description="Parses a CSV report and creates a summary report." - ) - parser.add_argument( - "csv_report", - type=str, - help="CSV report file of test cases, e.g. report.csv", - ) - parser.add_argument( - "csv_summary", type=str, help="Output CSV file, e.g. summary.csv" - ) - parser.add_argument( - "csv_image", - type=str, - nargs="?", - help="Summary image file, e.g. summary.png", - default=None, - ) - parser.add_argument( - "--measure", - type=str, - nargs=1, - help="Measure, any of: MLD, DIFF, SSNR, ODG, default: MLD", - default=["MLD"], - ) - parser.add_argument( - "--evs", - action="store_true", - help="Parse using EVS 26.444 formats", - default=False, - ) - parser.add_argument( - "--diff", - action="store_true", - help="Use limits for diff scores", - default=False, - ) - args = parser.parse_args() - csv_report = args.csv_report - csv_summary = args.csv_summary - csv_image = args.csv_image - measure = args.measure[0] - if args.evs: - FORMATS = EVS_FORMATS - CATEGORIES = EVS_CATEGORIES - else: - FORMATS = IVAS_FORMATS - CATEGORIES = IVAS_CATEGORIES - if args.diff: - limits_per_measure = { - "MLD": ("MLD", None), - "DIFF": ("MAXIMUM ABS DIFF", None), - "SSNR": ("MIN_SSNR", None), - "ODG": ("MIN_ODG", None), - "DELTA_ODG": ("DELTA_ODG", None), - } - else: - limits_per_measure = { - "MLD": ("MLD", [0, 1, 2, 3, 4, 5, 10, 20, math.inf]), - "DIFF": ( - "MAXIMUM ABS DIFF", - [0, 16, 256, 1024, 2048, 4096, 8192, 16384, 32769], - ), - "SSNR": ("MIN_SSNR", [-math.inf, 0, 10, 20, 30, 40, 40, 50, 60, 100]), - "ODG": ( - "MIN_ODG", - [-5, -4, -3, -2, -1, -0.5, -0.4, -0.3, -0.2, -0.1, 0, 0.1, 0.5], - ), - "DELTA_ODG": ( - "DELTA_ODG", - [-5, -4, -3, -2, -1, -0.5, -0.4, -0.3, -0.2, -0.1, 0, 0.1, 0.5], - ), - } - (measure_label, limits) = limits_per_measure[measure] - - # Load CSV report - results_sorted = {} - with open(csv_report, "r") as fp: - reader = csv.reader(fp, delimiter=";") - header = next(reader) - keys = header[1:] - for row in reader: - testcase = row[0] - results_sorted[testcase] = {} - for k, val in zip(keys, row[1:]): - results_sorted[testcase][k] = val - - if limits is None: - vals = [ - float(x) - for x in [ - m[measure_label] - for m in results_sorted.values() - if m[measure_label] != "None" and m[measure_label] != "" - ] - ] - start = min(vals) - f = 10 ** (2 - int(np.floor(np.log10(abs(start)))) - 1) - start = np.floor(start * f) / f - step = (max(vals) - start) / 10 - f = 10 ** (2 - int(np.floor(np.log10(abs(step)))) - 1) - step = np.ceil(step * f) / f - limits = np.arange(start, 10 * step, step) - - # Output CSV file - with open(csv_summary, "w") as fp: - limits_labels = [f"{a:g}" for a in limits] + [ - "", - "None", - ] # Put None cases in separate bin - headerline = f"Format;Category;" + ";".join(limits_labels) + "\n" - fp.write(headerline) - - for fmt in FORMATS: - fig, ax = plt.subplots() - bottom = np.zeros(len(limits_labels)) - for cat in CATEGORIES: - values = [ - x - for x in [ - m[measure_label] - for m in results_sorted.values() - if m["Format"] == fmt and m["Category"] == cat - ] - ] - # Create separate bin for None (errors) - val = [float(x) for x in values if x != "None" and x != ""] - none = [sum([1 for x in values if x == "None" or x == ""])] - hist, _ = np.histogram(val, limits) - data = np.array(list(hist) + [0] + none + [0]) - - # CSV output - line = f"{fmt};{cat};{'; '.join(map(str,data))}\n" - fp.write(line) - - # Matplotlib histogram - ax.bar( - limits_labels, - data, - 1, - align="edge", - edgecolor="black", - linewidth=0.5, - label=cat, - bottom=bottom, - ) - bottom += data - - # Histogram layout - ax.set_title(fmt) - ax.legend(loc="best") - ax.set_xlabel(measure_label) - if "DIFF" in measure_label: - ax.set_xticks(range(len(limits_labels)), limits_labels, rotation=35) - ax.set_ylabel("Number of test cases") - - fig.set_figheight(4) - fig.set_figwidth(6) - if csv_image: - base, ext = os.path.splitext(csv_image) - plt.savefig(f"{base}_{fmt}{ext}") -- GitLab From f227980b1163d2bfaf9eacb15ca4ab483c4d7b06 Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Wed, 26 Mar 2025 17:27:23 +0100 Subject: [PATCH 071/106] handle DMX measures via prefix in script --- scripts/create_histograms.py | 18 +++++++++++++++--- tests/conftest.py | 8 +++++--- 2 files changed, 20 insertions(+), 6 deletions(-) diff --git a/scripts/create_histograms.py b/scripts/create_histograms.py index 691ef4f05a..1aeb7b0a7d 100644 --- a/scripts/create_histograms.py +++ b/scripts/create_histograms.py @@ -30,6 +30,7 @@ def create_histograms( output_folder: pathlib.Path, display_only: bool, bins_for_measures=BINS_FOR_MEASURES, + prefix="", ): formats = df["format"].unique() categories = df["category"].unique() @@ -38,7 +39,8 @@ def create_histograms( output_folder.mkdir(exist_ok=True, parents=True) for measure in measures: - bins = bins_for_measures.get(measure, get_bins_for_diff(df[measure])) + measure_in_df = prefix + measure + bins = bins_for_measures.get(measure, get_bins_for_diff(df[measure_in_df])) x = [f"{x}" for x in bins] + ["", "ERROR"] for fmt in formats: fig, ax = plt.subplots() @@ -51,7 +53,7 @@ def create_histograms( n_errors = np.sum(error_mask) df_hist = df_slice[np.logical_not(error_mask)] - counts, _ = np.histogram(df_hist[measure], bins) + counts, _ = np.histogram(df_hist[measure_in_df], bins) data = np.concat([counts, [0], [n_errors], [0]]) ax.bar( @@ -117,6 +119,11 @@ Use this for visualising diff scores.""", default=DEFAULT_MEASURES, help=f"Measures to plot from the csv file. One of {allowed_measures}", ) + parser.add_argument( + "--prefix", + default="", + help="Common suffix to use when collecting measures from the input csv file", + ) args = parser.parse_args() df = pd.read_csv(args.csv_report) @@ -125,5 +132,10 @@ Use this for visualising diff scores.""", bins_for_measures = {} create_histograms( - df, args.measures, args.output_folder, args.display_only, bins_for_measures + df, + args.measures, + args.output_folder, + args.display_only, + bins_for_measures, + args.prefix, ) diff --git a/tests/conftest.py b/tests/conftest.py index 4b52c85aa4..5fd95996a6 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1195,9 +1195,11 @@ def parse_properties( min_ssnr = min(ssnrs) min_ssnr_channel = ssnrs.index(min_ssnr) - prefix = "MIN" if prop == SSNR else "DMX" - props[f"{prefix}_SSNR" + suffix] = min_ssnr - props[f"{prefix}_SSNR_CHANNEL" + suffix] = min_ssnr_channel + propname = "MIN_SSNR" + if prop == DMX_SSNR: + propname = "DMX_MIN_SSNR" + props[propname + suffix] = min_ssnr + props[f"{propname}_CHANNEL" + suffix] = min_ssnr_channel elif prop == ODG: odgs = re.findall(ODG_PATTERN, text_to_parse) min_odg = min(odgs) -- GitLab From e96ede2c8b8c6840ac5b3fcb7d73f1baa098bc88 Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Wed, 26 Mar 2025 19:35:51 +0100 Subject: [PATCH 072/106] handle split comparison results in parse_xml_report.py --- scripts/parse_xml_report.py | 48 +++++++++++++++++++++++++++++++++++++ 1 file changed, 48 insertions(+) diff --git a/scripts/parse_xml_report.py b/scripts/parse_xml_report.py index f326d13e70..77eb857334 100644 --- a/scripts/parse_xml_report.py +++ b/scripts/parse_xml_report.py @@ -6,6 +6,10 @@ from xml.etree import ElementTree from collections import Counter +SPLIT_STRING = "_split" +WHOLE_STRING = "_whole" + + class TestcaseParser(dict): def __init__(self, testcases: list): super().__init__() @@ -38,6 +42,50 @@ class TestcaseParser(dict): properties = { p.get("name"): p.get("value") for p in testcase.findall(".//property") } + + ### handle split comparison results + split_props = {k: v for k, v in properties.items() if SPLIT_STRING in k} + whole_props = {k: v for k, v in properties.items() if WHOLE_STRING in k} + + if len(split_props) > 0 and len(whole_props) > 0: + measures_from_split = set( + [m.split(SPLIT_STRING)[0] for m in split_props.keys()] + ) + measures_from_whole = set( + [m.split(WHOLE_STRING)[0] for m in whole_props.keys()] + ) + assert measures_from_split == measures_from_whole + measures = measures_from_whole + + # collect existing split suffixes by evaluating one of the measures only + m_tmp = measures.pop() + splits = sorted( + [ + k.split(SPLIT_STRING)[-1] + for k in split_props.keys() + if k.startswith(m_tmp) + ] + ) + + # record each split under a separate key + # the dict per key has the same fulltestname and an additional key "split" + # this way, the resulting DataFrame in the end can be split by testnames + for s in splits: + split_key = f"{fulltestname} - {s}" + ret_split = {"testcase": fulltestname, "split": s} + for m in measures: + ret_split.update({m: split_props[m + SPLIT_STRING + f"{s}"]}) + self[split_key] = ret_split + + # it can be the case that there are no splits defined in the pytest suite, e.g. for the renderer + # then, there are only "_whole" values recorded where we only need to remove the suffix + # this if also handles the split case - if there are splits, there was also a "_whole" comparison done + if len(whole_props) > 0: + properties = { + k.replace(WHOLE_STRING, ""): v for k, v in whole_props.items() + } + properties["split"] = "whole" + ret.update(properties) self[fulltestname] = ret -- GitLab From 8a842204bcce34bd0a6d6dfe7556f3fcc400cf7e Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Thu, 27 Mar 2025 11:32:41 +0100 Subject: [PATCH 073/106] fix xticks --- scripts/create_histograms.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/scripts/create_histograms.py b/scripts/create_histograms.py index 1aeb7b0a7d..b202a7c847 100644 --- a/scripts/create_histograms.py +++ b/scripts/create_histograms.py @@ -74,6 +74,8 @@ def create_histograms( ax.set_xlabel(measure) if "DIFF" in measure: ax.set_xticks(range(len(x)), x, rotation=35) + else: + ax.set_xticks(range(len(x)), x) ax.set_ylabel("Number of test cases") fig.set_figheight(4) -- GitLab From d12b82730eb5550361dba8fdd0c3a8154aaebc73 Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Thu, 27 Mar 2025 16:07:50 +0100 Subject: [PATCH 074/106] add separate output file for split cases --- scripts/parse_xml_report.py | 45 ++++++++++++++++++++++++++++--------- 1 file changed, 34 insertions(+), 11 deletions(-) diff --git a/scripts/parse_xml_report.py b/scripts/parse_xml_report.py index 77eb857334..c55f4c02d9 100644 --- a/scripts/parse_xml_report.py +++ b/scripts/parse_xml_report.py @@ -4,12 +4,20 @@ import argparse import pandas as pd from xml.etree import ElementTree from collections import Counter +from typing import Optional +from enum import Enum SPLIT_STRING = "_split" WHOLE_STRING = "_whole" +class Result(str, Enum): + ERROR = "ERROR" + FAIL = "FAIL" + PASS = "PASS" + + class TestcaseParser(dict): def __init__(self, testcases: list): super().__init__() @@ -32,8 +40,8 @@ class TestcaseParser(dict): # if we already have this testcase, do a sanity check and set result to ERROR if fulltestname in self: results = [self[fulltestname]["result"], result] - assert any(r == "ERROR" for r in results) - self[fulltestname]["result"] = "ERROR" + assert any(r == Result.ERROR for r in results) + self[fulltestname]["result"] = Result.ERROR return ret = {} @@ -110,24 +118,33 @@ def xml_to_dataframe(xml_report: str) -> pd.DataFrame: def get_result_from_testcase(testcase: ElementTree.Element) -> str: if testcase.find("failure") is not None: - testresult = "FAIL" + testresult = Result.FAIL elif testcase.find("error") is not None: - testresult = "ERROR" + testresult = Result.ERROR else: - testresult = "PASS" + testresult = Result.PASS return testresult -def main(xml_report, csv_file): +def main(xml_report: str, csv_file: str, split_csv_file: Optional[str]): df = xml_to_dataframe(xml_report) - df.to_csv(csv_file, index=False) - n_testcases = len(df) count = Counter(df["result"]) + if split_csv_file is not None: + mask_errors = df["result"] == Result.ERROR + mask_whole = df["split"] == "whole" + mask_single = mask_errors | mask_whole + df_split = df[~mask_single] + df_split.to_csv(split_csv_file, index=False) + + df = df[mask_single] + + df.to_csv(csv_file, index=False) + print( - f"Parsed testsuite with {n_testcases} tests: {count['PASS']} passes, {count['FAIL']} failures and {count['ERROR']} errors." + f"Parsed testsuite with {n_testcases} tests: {count[Result.PASS]} passes, {count[Result.FAIL]} failures and {count[Result.ERROR]} errors." ) @@ -140,7 +157,13 @@ if __name__ == "__main__": type=str, help="XML junit report input file, e.g. report-junit.xml", ) - parser.add_argument("csv_file", type=str, help="Output CSV file, e.g. report.csv") + parser.add_argument("csv_file", help="Output CSV file, e.g. report.csv") + parser.add_argument( + "--split-csv-file", + type=str, + default=None, + help="If given, write the split comparison values to this file separately", + ) args = parser.parse_args() - main(args.xml_report, args.csv_file) + main(args.xml_report, args.csv_file, args.split_csv_file) -- GitLab From a50fb3b82a5b8f9c9378844de919f9fbc6524d93 Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Thu, 27 Mar 2025 16:08:25 +0100 Subject: [PATCH 075/106] filter out empty format and category to avoid "nan" in output --- scripts/create_histograms.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/scripts/create_histograms.py b/scripts/create_histograms.py index b202a7c847..0e1443eb99 100644 --- a/scripts/create_histograms.py +++ b/scripts/create_histograms.py @@ -129,6 +129,11 @@ Use this for visualising diff scores.""", args = parser.parse_args() df = pd.read_csv(args.csv_report) + # filter out missing format/category values + mask_format_missing = df["format"].isna() + mask_category_missing = df["category"].isna() + df = df[~mask_format_missing | ~mask_category_missing] + bins_for_measures = BINS_FOR_MEASURES if args.no_bins: bins_for_measures = {} -- GitLab From ab4eafa7144bd1a9b878209febe98df924bdd5cd Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Thu, 27 Mar 2025 16:58:57 +0100 Subject: [PATCH 076/106] use copy to prevent modification of object --- tests/conftest.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/conftest.py b/tests/conftest.py index 2a22b90b90..16992d92d2 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1276,7 +1276,7 @@ def get_split_idx(input_file: str, sampling_rate_khz: int) -> Optional[np.ndarra format = "osba_hoa" else: format = input_file.replace(".wav", "").split("_")[-1].lower() - idx = SPLIT_IDX.get(format, np.empty(0)) + idx = SPLIT_IDX.get(format, np.empty(0)).copy() if len(idx) > 0 and sampling_rate_khz != 16: idx *= sampling_rate_khz // 16 -- GitLab From d79d26e32132ac35ed287850313959b7ad2afb7d Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Thu, 27 Mar 2025 17:15:30 +0100 Subject: [PATCH 077/106] fix format and category keys missing in split case --- scripts/create_histograms.py | 2 +- scripts/parse_xml_report.py | 8 ++++++++ 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/scripts/create_histograms.py b/scripts/create_histograms.py index 0e1443eb99..d5c0bce38f 100644 --- a/scripts/create_histograms.py +++ b/scripts/create_histograms.py @@ -12,7 +12,7 @@ from typing import List BINS_FOR_MEASURES = { "MLD": [0, 1, 2, 3, 4, 5, 10, 20, math.inf], "MAX_ABS_DIFF": [0, 16, 256, 1024, 2048, 4096, 8192, 16384, 32769], - "MIN_SSNR": [-math.inf, 0, 10, 20, 30, 40, 40, 50, 60, 100], + "MIN_SSNR": [-math.inf, 0, 10, 20, 30, 40, 40, 50, 60, 100, math.inf], "MIN_ODG": [-5, -4, -3, -2, -1, -0.5, -0.4, -0.3, -0.2, -0.1, 0, 0.1, 0.5], "DELTA_ODG": [-5, -4, -3, -2, -1, -0.5, -0.4, -0.3, -0.2, -0.1, 0, 0.1, 0.5], } diff --git a/scripts/parse_xml_report.py b/scripts/parse_xml_report.py index c55f4c02d9..aa4baadeb5 100644 --- a/scripts/parse_xml_report.py +++ b/scripts/parse_xml_report.py @@ -54,6 +54,11 @@ class TestcaseParser(dict): ### handle split comparison results split_props = {k: v for k, v in properties.items() if SPLIT_STRING in k} whole_props = {k: v for k, v in properties.items() if WHOLE_STRING in k} + other_props = { + k: v + for k, v in properties.items() + if WHOLE_STRING not in k and SPLIT_STRING not in k + } if len(split_props) > 0 and len(whole_props) > 0: measures_from_split = set( @@ -83,6 +88,8 @@ class TestcaseParser(dict): ret_split = {"testcase": fulltestname, "split": s} for m in measures: ret_split.update({m: split_props[m + SPLIT_STRING + f"{s}"]}) + + ret_split.update(other_props) self[split_key] = ret_split # it can be the case that there are no splits defined in the pytest suite, e.g. for the renderer @@ -93,6 +100,7 @@ class TestcaseParser(dict): k.replace(WHOLE_STRING, ""): v for k, v in whole_props.items() } properties["split"] = "whole" + properties.update(other_props) ret.update(properties) self[fulltestname] = ret -- GitLab From 4112ded2ec9e8e5ddba8c94fb3e7b1a3db449acd Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Fri, 28 Mar 2025 09:46:22 +0100 Subject: [PATCH 078/106] fix bug with measure missing due to removal form set --- scripts/parse_xml_report.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/scripts/parse_xml_report.py b/scripts/parse_xml_report.py index aa4baadeb5..948bda4308 100644 --- a/scripts/parse_xml_report.py +++ b/scripts/parse_xml_report.py @@ -71,7 +71,10 @@ class TestcaseParser(dict): measures = measures_from_whole # collect existing split suffixes by evaluating one of the measures only + # get one measure from set and add it back immediately m_tmp = measures.pop() + measures.add(m_tmp) + splits = sorted( [ k.split(SPLIT_STRING)[-1] -- GitLab From 8a8d18809b75b7658ce3f33a75f50564a7dd2090 Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Fri, 28 Mar 2025 10:18:40 +0100 Subject: [PATCH 079/106] add the DISPLAY var missing hack --- scripts/create_histograms.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/scripts/create_histograms.py b/scripts/create_histograms.py index d5c0bce38f..13b51d4869 100644 --- a/scripts/create_histograms.py +++ b/scripts/create_histograms.py @@ -5,10 +5,16 @@ import math import pathlib import pandas as pd import numpy as np -import matplotlib.pyplot as plt from typing import List +# hack for avoiding missing DISPLAY variable in headless CI runners +import matplotlib + +matplotlib.use("Agg") +import matplotlib.pyplot as plt + + BINS_FOR_MEASURES = { "MLD": [0, 1, 2, 3, 4, 5, 10, 20, math.inf], "MAX_ABS_DIFF": [0, 16, 256, 1024, 2048, 4096, 8192, 16384, 32769], -- GitLab From 46037ef006220a0173cc9472674f47a9c2f169f9 Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Fri, 28 Mar 2025 10:44:28 +0100 Subject: [PATCH 080/106] fix warning about too many open figures when not displaying --- scripts/create_histograms.py | 1 + 1 file changed, 1 insertion(+) diff --git a/scripts/create_histograms.py b/scripts/create_histograms.py index 13b51d4869..b0923ddbd8 100644 --- a/scripts/create_histograms.py +++ b/scripts/create_histograms.py @@ -92,6 +92,7 @@ def create_histograms( image_file = f"histogram_{measure}_{fmt}.png" image_path = output_folder.joinpath(image_file) plt.savefig(image_path) + plt.close(fig) if display_only: plt.show() -- GitLab From 7b4d7fe2dcac5a0f2fc0f6710b3b6f16aa051a48 Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Fri, 28 Mar 2025 11:14:00 +0100 Subject: [PATCH 081/106] fix regex pattern to prevent incorrect partial matches --- tests/conftest.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/conftest.py b/tests/conftest.py index f572ac3bdb..e8cdc229d0 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1296,7 +1296,8 @@ IVAS_ENC_FORMATS = { "mc": "Multichannel", "stereo_dmx_evs": "Stereo DMX EVS", } -PATTERN_IVAS_ENC_FORMAT = re.compile(r"-(" + r"|".join(IVAS_ENC_FORMATS.keys()) + ")") +# NOTE: the blanks at start and end are important to prevent e.g. "-ism_masa" matching on "-ism" only +PATTERN_IVAS_ENC_FORMAT = re.compile(r"-( " + r"|".join(IVAS_ENC_FORMATS.keys()) + ") ") def get_format_from_enc_opts(enc_opts: str) -> str: -- GitLab From 87691838a0edd0424f2d0b0366ba5eac01655b70 Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Fri, 28 Mar 2025 11:14:33 +0100 Subject: [PATCH 082/106] adapt summary page creation script to new data format --- ci/basop-pages/create_summary_page.py | 32 +++++++++++++++------------ 1 file changed, 18 insertions(+), 14 deletions(-) diff --git a/ci/basop-pages/create_summary_page.py b/ci/basop-pages/create_summary_page.py index 3c19bf1a6c..199b35bb51 100644 --- a/ci/basop-pages/create_summary_page.py +++ b/ci/basop-pages/create_summary_page.py @@ -4,11 +4,11 @@ from typing import List from create_report_pages import SUBPAGE_TMPL_CSS, FORMATS -TITLE = { +TITLE_4_MEASURE = { "MLD": "Maximum MLD across channels", - "DIFF": "Maximim absolute difference across channels", - "SSNR": "Minimum SSNR across channels", - "ODG": "Minimum PEAQ ODG across channels", + "MAX_ABS_DIFF": "Maximim absolute difference across channels", + "MIN_SSNR": "Minimum SSNR across channels", + "MIN_ODG": "Minimum PEAQ ODG across channels", "DELTA_ODG": "PEAQ ODG using binauralized input and output", } @@ -28,18 +28,21 @@ def create_summary_page( measures: List[str], image_dir: str, ): - images = "
" + html = "
" for m in measures: - images += ( - f"

{TITLE[m]}

\n" - + " ".join([f"" for x in FORMATS]) - + f'\n
summary_{m}.csv
\n\n' + image_filename_tmpl = "histogram_{measure}_{format}.png" + image_html = " ".join( + [ + f"" + for f in FORMATS + ] ) + html += f"

{TITLE_4_MEASURE[m]}

\n" + image_html new_summary_page = SUBPAGE_TMPL_CSS + SUMMARY_PAGE_TMPL_HTML.format( id_current=id_current, job_name=job_name, - images=images, + images=html, ) with open(html_out, "w") as f: f.write(new_summary_page) @@ -54,14 +57,15 @@ if __name__ == "__main__": parser.add_argument( "--measures", nargs="+", - help=f"List of measures to include in summary. Allowed values: {' '.join(TITLE.keys())}", - default=["MLD", "DIFF", "SSNR", "ODG"], + help=f"List of measures to include in summary. Allowed values: {' '.join(TITLE_4_MEASURE.keys())}", + # exclude DELTA_ODG here + default=list(TITLE_4_MEASURE.keys())[:-1], ) args = parser.parse_args() - if not all([m in TITLE for m in args.measures]): + if not all([m in TITLE_4_MEASURE for m in args.measures]): raise ValueError( - f"Invalid list of measures: {args.measures}, expected one of {' '.join(TITLE.keys())}" + f"Invalid list of measures: {args.measures}, expected one of {' '.join(TITLE_4_MEASURE.keys())}" ) create_summary_page( -- GitLab From 08d8466aeb3c9c39890bcbf20505af0cc396fa88 Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Fri, 28 Mar 2025 11:37:30 +0100 Subject: [PATCH 083/106] small refactoring + change to layout --- ci/basop-pages/create_summary_page.py | 28 ++++++++++++++++----------- 1 file changed, 17 insertions(+), 11 deletions(-) diff --git a/ci/basop-pages/create_summary_page.py b/ci/basop-pages/create_summary_page.py index 199b35bb51..32855fce25 100644 --- a/ci/basop-pages/create_summary_page.py +++ b/ci/basop-pages/create_summary_page.py @@ -6,7 +6,7 @@ from create_report_pages import SUBPAGE_TMPL_CSS, FORMATS TITLE_4_MEASURE = { "MLD": "Maximum MLD across channels", - "MAX_ABS_DIFF": "Maximim absolute difference across channels", + "MAX_ABS_DIFF": "Maximum absolute difference across channels", "MIN_SSNR": "Minimum SSNR across channels", "MIN_ODG": "Minimum PEAQ ODG across channels", "DELTA_ODG": "PEAQ ODG using binauralized input and output", @@ -16,9 +16,13 @@ SUMMARY_PAGE_TMPL_HTML = """

Summary for job {job_name}, ID: {id_current}

+
+ {images} """ +IMAGE_HTML_TMPL = "" +SUBHEADING_HTML_TMP = "

{subtitle}

\n" def create_summary_page( @@ -28,16 +32,18 @@ def create_summary_page( measures: List[str], image_dir: str, ): - html = "
" - for m in measures: - image_filename_tmpl = "histogram_{measure}_{format}.png" - image_html = " ".join( - [ - f"" - for f in FORMATS - ] - ) - html += f"

{TITLE_4_MEASURE[m]}

\n" + image_html + html = "\n
\n".join( + [ + SUBHEADING_HTML_TMP.format(subtitle=TITLE_4_MEASURE[m]) + + " ".join( + [ + IMAGE_HTML_TMPL.format(measure=m, format=f, image_dir=image_dir) + for f in FORMATS + ] + ) + for m in measures + ] + ) new_summary_page = SUBPAGE_TMPL_CSS + SUMMARY_PAGE_TMPL_HTML.format( id_current=id_current, -- GitLab From 3438f11d5382c724f5014cac793ce97238b7cc61 Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Fri, 28 Mar 2025 12:04:26 +0100 Subject: [PATCH 084/106] use concatenate instead of concat --- scripts/create_histograms.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/create_histograms.py b/scripts/create_histograms.py index b0923ddbd8..160ea95419 100644 --- a/scripts/create_histograms.py +++ b/scripts/create_histograms.py @@ -61,7 +61,7 @@ def create_histograms( counts, _ = np.histogram(df_hist[measure_in_df], bins) - data = np.concat([counts, [0], [n_errors], [0]]) + data = np.concatenate([counts, [0], [n_errors], [0]]) ax.bar( x, data, -- GitLab From bb5350544752226803f1bb341d39063165236988 Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Fri, 28 Mar 2025 14:07:18 +0100 Subject: [PATCH 085/106] fix format property recording --- tests/conftest.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/conftest.py b/tests/conftest.py index e8cdc229d0..027b191414 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1287,17 +1287,17 @@ def get_split_idx(input_file: str, sampling_rate_khz: int) -> Optional[np.ndarra IVAS_ENC_FORMATS = { - "stereo": "Stereo", - "ism": "ISM", "sba": "SBA", "masa": "Masa", "ism_sba": "OSBA", "ism_masa": "OMASA", + "ism": "ISM", "mc": "Multichannel", "stereo_dmx_evs": "Stereo DMX EVS", + "stereo": "Stereo", } -# NOTE: the blanks at start and end are important to prevent e.g. "-ism_masa" matching on "-ism" only -PATTERN_IVAS_ENC_FORMAT = re.compile(r"-( " + r"|".join(IVAS_ENC_FORMATS.keys()) + ") ") +# NOTE: the blank at the end is important to prevent e.g. "-ism_masa" matching on "-ism" only +PATTERN_IVAS_ENC_FORMAT = re.compile(r"-(" + r"|".join(IVAS_ENC_FORMATS.keys()) + ") ") def get_format_from_enc_opts(enc_opts: str) -> str: -- GitLab From 741ca23874670a83a79deceb0b0b9e7a12d194aa Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Mon, 31 Mar 2025 09:38:47 +0200 Subject: [PATCH 086/106] Correct format names to fit with html page naming --- tests/conftest.py | 2 +- tests/renderer/utils.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/conftest.py b/tests/conftest.py index 027b191414..0d102b16bf 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1288,7 +1288,7 @@ def get_split_idx(input_file: str, sampling_rate_khz: int) -> Optional[np.ndarra IVAS_ENC_FORMATS = { "sba": "SBA", - "masa": "Masa", + "masa": "MASA", "ism_sba": "OSBA", "ism_masa": "OMASA", "ism": "ISM", diff --git a/tests/renderer/utils.py b/tests/renderer/utils.py index 6da0a276ab..ec5fe7f0cb 100644 --- a/tests/renderer/utils.py +++ b/tests/renderer/utils.py @@ -394,7 +394,7 @@ def run_renderer( reason = reason[0] props = parse_properties(reason, output_differs, props_to_record) - props["format"] = "renderer" + props["format"] = "Renderer" props["category"] = CAT_NORMAL for k, v in props.items(): record_property(k, v) -- GitLab From 3d03e4d7357ad847885586e9fd86cf6cd307fd39 Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Mon, 31 Mar 2025 09:48:18 +0200 Subject: [PATCH 087/106] fix split idx selection for OMASA --- tests/conftest.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/conftest.py b/tests/conftest.py index 0d102b16bf..69a13e6d7f 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1270,14 +1270,14 @@ def get_split_idx(input_file: str, sampling_rate_khz: int) -> Optional[np.ndarra input_file = input_file.lower() if "omasa" in input_file: - format = "_".join(input_file.replace(".wav", "").split("_")[:-1]) + format = "_".join(input_file.split("_")[1:-1]) elif "osba" in input_file: if "foa" in input_file: format = "osba_foa" else: format = "osba_hoa" else: - format = input_file.replace(".wav", "").split("_")[-1].lower() + format = input_file.split("_")[-1].lower() idx = SPLIT_IDX.get(format, np.empty(0)).copy() if len(idx) > 0 and sampling_rate_khz != 16: -- GitLab From 1e38c12627f30eddea678a1b5d248903c5468992 Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Mon, 31 Mar 2025 15:24:18 +0200 Subject: [PATCH 088/106] make renderer tests record props correctly for split-comparison --- tests/renderer/test_renderer.py | 134 +++++++++++++++++++++++--------- tests/renderer/utils.py | 12 ++- 2 files changed, 110 insertions(+), 36 deletions(-) diff --git a/tests/renderer/test_renderer.py b/tests/renderer/test_renderer.py index b5758d6f06..956f773666 100644 --- a/tests/renderer/test_renderer.py +++ b/tests/renderer/test_renderer.py @@ -62,7 +62,6 @@ from ..conftest import props_to_record """ Ambisonics """ - @pytest.mark.parametrize("out_fmt", OUTPUT_FORMATS) @pytest.mark.parametrize("in_fmt", INPUT_FORMATS_AMBI) @pytest.mark.parametrize("frame_size", FRAMING_TO_TEST) @@ -78,6 +77,7 @@ def test_ambisonics( get_ssnr, get_odg, get_odg_bin, + split_comparison, ): run_renderer( record_property, @@ -92,10 +92,10 @@ def test_ambisonics( get_ssnr=get_ssnr, get_odg=get_odg, get_odg_bin=get_odg_bin, + split_comparison=split_comparison, ) - @pytest.mark.parametrize("out_fmt", OUTPUT_FORMATS_BINAURAL) @pytest.mark.parametrize("in_fmt", INPUT_FORMATS_AMBI) @pytest.mark.parametrize("frame_size", FRAMING_TO_TEST) @@ -111,6 +111,7 @@ def test_ambisonics_binaural_static( get_ssnr, get_odg, get_odg_bin, + split_comparison, ): run_renderer( record_property, @@ -125,10 +126,10 @@ def test_ambisonics_binaural_static( get_ssnr=get_ssnr, get_odg=get_odg, get_odg_bin=get_odg_bin, + split_comparison=split_comparison, ) - @pytest.mark.parametrize("trj_file", HR_TRAJECTORIES_TO_TEST) @pytest.mark.parametrize("out_fmt", OUTPUT_FORMATS_BINAURAL) @pytest.mark.parametrize("in_fmt", INPUT_FORMATS_AMBI) @@ -146,6 +147,7 @@ def test_ambisonics_binaural_headrotation( get_ssnr, get_odg, get_odg_bin, + split_comparison, ): run_renderer( record_property, @@ -161,11 +163,11 @@ def test_ambisonics_binaural_headrotation( get_ssnr=get_ssnr, get_odg=get_odg, get_odg_bin=get_odg_bin, + split_comparison=split_comparison, ) @pytest.mark.skip(reason="Not supported for BASOP code currently") - @pytest.mark.parametrize("out_fmt", OUTPUT_FORMATS_BINAURAL[2:]) @pytest.mark.parametrize("in_fmt", INPUT_FORMATS_AMBI) @pytest.mark.parametrize("frame_size", FRAMING_TO_TEST) @@ -183,6 +185,7 @@ def test_dynamic_acoustic_environment( get_odg, get_odg_bin, aeid, + split_comparison, ): rend_config_path = TEST_VECTOR_DIR.joinpath(f"rend_config_combined.cfg") rend_config_path.with_stem(f"rend_config") @@ -202,11 +205,11 @@ def test_dynamic_acoustic_environment( get_odg_bin=get_odg_bin, config_file=rend_config_path, aeid=aeid, + split_comparison=split_comparison, ) @pytest.mark.skip(reason="Not supported for BASOP code currently") - @pytest.mark.parametrize("out_fmt", OUTPUT_FORMATS_BINAURAL[2:]) @pytest.mark.parametrize("in_fmt", INPUT_FORMATS_AMBI) @pytest.mark.parametrize("frame_size", FRAMING_TO_TEST) @@ -222,6 +225,7 @@ def test_dynamic_acoustic_environment_file( get_ssnr, get_odg, get_odg_bin, + split_comparison, ): rend_config_path = TEST_VECTOR_DIR.joinpath(f"rend_config_combined.cfg") rend_config_path.with_stem(f"rend_config") @@ -243,13 +247,13 @@ def test_dynamic_acoustic_environment_file( get_odg_bin=get_odg_bin, config_file=rend_config_path, aeid=aeid, + split_comparison=split_comparison, ) """ Multichannel """ - @pytest.mark.parametrize("out_fmt", OUTPUT_FORMATS) @pytest.mark.parametrize("in_fmt", INPUT_FORMATS_MC) @pytest.mark.parametrize("frame_size", FRAMING_TO_TEST) @@ -265,6 +269,7 @@ def test_multichannel( get_ssnr, get_odg, get_odg_bin, + split_comparison, ): run_renderer( record_property, @@ -279,10 +284,10 @@ def test_multichannel( get_ssnr=get_ssnr, get_odg=get_odg, get_odg_bin=get_odg_bin, + split_comparison=split_comparison, ) - @pytest.mark.parametrize("out_fmt", OUTPUT_FORMATS_BINAURAL) @pytest.mark.parametrize("in_fmt", INPUT_FORMATS_MC) @pytest.mark.parametrize("frame_size", FRAMING_TO_TEST) @@ -298,6 +303,7 @@ def test_multichannel_binaural_static( get_ssnr, get_odg, get_odg_bin, + split_comparison, ): if in_fmt in ["MONO", "STEREO"]: pytest.skip("MONO or STEREO to Binaural rendering unsupported") @@ -315,10 +321,10 @@ def test_multichannel_binaural_static( get_ssnr=get_ssnr, get_odg=get_odg, get_odg_bin=get_odg_bin, + split_comparison=split_comparison, ) - @pytest.mark.parametrize("trj_file", HR_TRAJECTORIES_TO_TEST) @pytest.mark.parametrize("out_fmt", OUTPUT_FORMATS_BINAURAL) @pytest.mark.parametrize("in_fmt", INPUT_FORMATS_MC) @@ -336,6 +342,7 @@ def test_multichannel_binaural_headrotation( get_ssnr, get_odg, get_odg_bin, + split_comparison, ): if in_fmt in ["MONO", "STEREO"]: pytest.skip("MONO or STEREO to Binaural rendering unsupported") @@ -354,13 +361,13 @@ def test_multichannel_binaural_headrotation( get_ssnr=get_ssnr, get_odg=get_odg, get_odg_bin=get_odg_bin, + split_comparison=split_comparison, ) """ ISM """ - @pytest.mark.parametrize("out_fmt", OUTPUT_FORMATS) @pytest.mark.parametrize("in_fmt", INPUT_FORMATS_ISM) @pytest.mark.parametrize("frame_size", FRAMING_TO_TEST) @@ -376,6 +383,7 @@ def test_ism( get_ssnr, get_odg, get_odg_bin, + split_comparison, ): run_renderer( record_property, @@ -391,10 +399,10 @@ def test_ism( get_ssnr=get_ssnr, get_odg=get_odg, get_odg_bin=get_odg_bin, + split_comparison=split_comparison, ) - @pytest.mark.parametrize("out_fmt", OUTPUT_FORMATS_BINAURAL) @pytest.mark.parametrize("in_fmt", INPUT_FORMATS_ISM) @pytest.mark.parametrize("frame_size", FRAMING_TO_TEST) @@ -410,6 +418,7 @@ def test_ism_binaural_static( get_ssnr, get_odg, get_odg_bin, + split_comparison, ): try: in_meta_files = FORMAT_TO_METADATA_FILES[in_fmt] @@ -430,10 +439,10 @@ def test_ism_binaural_static( get_ssnr=get_ssnr, get_odg=get_odg, get_odg_bin=get_odg_bin, + split_comparison=split_comparison, ) - @pytest.mark.parametrize("trj_file", HR_TRAJECTORIES_TO_TEST) @pytest.mark.parametrize("out_fmt", OUTPUT_FORMATS_BINAURAL) @pytest.mark.parametrize("in_fmt", INPUT_FORMATS_ISM) @@ -451,6 +460,7 @@ def test_ism_binaural_headrotation( get_ssnr, get_odg, get_odg_bin, + split_comparison, ): try: in_meta_files = FORMAT_TO_METADATA_FILES[in_fmt] @@ -472,13 +482,13 @@ def test_ism_binaural_headrotation( get_ssnr=get_ssnr, get_odg=get_odg, get_odg_bin=get_odg_bin, + split_comparison=split_comparison, ) """ MASA """ - @pytest.mark.parametrize("out_fmt", OUTPUT_FORMATS) @pytest.mark.parametrize("in_fmt", INPUT_FORMATS_MASA) @pytest.mark.parametrize("frame_size", FRAMING_TO_TEST) @@ -494,6 +504,7 @@ def test_masa( get_ssnr, get_odg, get_odg_bin, + split_comparison, ): run_renderer( record_property, @@ -509,10 +520,10 @@ def test_masa( get_ssnr=get_ssnr, get_odg=get_odg, get_odg_bin=get_odg_bin, + split_comparison=split_comparison, ) - @pytest.mark.parametrize("out_fmt", OUTPUT_FORMATS_BINAURAL) @pytest.mark.parametrize("in_fmt", INPUT_FORMATS_MASA) @pytest.mark.parametrize("frame_size", FRAMING_TO_TEST) @@ -528,6 +539,7 @@ def test_masa_binaural_static( get_ssnr, get_odg, get_odg_bin, + split_comparison, ): if out_fmt in ["BINAURAL_ROOM_IR", "BINAURAL_ROOM_REVERB"]: pytest.skip("Skipping binaural room outputs for MASA as unimplemented.") @@ -546,10 +558,10 @@ def test_masa_binaural_static( get_ssnr=get_ssnr, get_odg=get_odg, get_odg_bin=get_odg_bin, + split_comparison=split_comparison, ) - @pytest.mark.parametrize("trj_file", HR_TRAJECTORIES_TO_TEST) @pytest.mark.parametrize("out_fmt", OUTPUT_FORMATS_BINAURAL) @pytest.mark.parametrize("in_fmt", INPUT_FORMATS_MASA) @@ -567,6 +579,7 @@ def test_masa_binaural_headrotation( get_ssnr, get_odg, get_odg_bin, + split_comparison, ): if out_fmt in ["BINAURAL_ROOM_IR", "BINAURAL_ROOM_REVERB"]: pytest.skip("Skipping binaural room outputs for MASA as unimplemented.") @@ -586,10 +599,10 @@ def test_masa_binaural_headrotation( get_ssnr=get_ssnr, get_odg=get_odg, get_odg_bin=get_odg_bin, + split_comparison=split_comparison, ) - @pytest.mark.parametrize("in_fmt", METADATA_SCENES_TO_TEST_MASA_PREREND) def test_masa_prerend( record_property, @@ -601,6 +614,7 @@ def test_masa_prerend( get_ssnr, get_odg, get_odg_bin, + split_comparison, ): run_renderer( record_property, @@ -615,13 +629,13 @@ def test_masa_prerend( get_ssnr=get_ssnr, get_odg=get_odg, get_odg_bin=get_odg_bin, + split_comparison=split_comparison, ) """ Custom loudspeaker layouts """ - @pytest.mark.parametrize("out_fmt", OUTPUT_FORMATS) @pytest.mark.parametrize("in_layout", CUSTOM_LS_TO_TEST) @pytest.mark.parametrize("frame_size", FRAMING_TO_TEST) @@ -637,6 +651,7 @@ def test_custom_ls_input( get_ssnr, get_odg, get_odg_bin, + split_comparison, ): run_renderer( record_property, @@ -651,10 +666,10 @@ def test_custom_ls_input( get_ssnr=get_ssnr, get_odg=get_odg, get_odg_bin=get_odg_bin, + split_comparison=split_comparison, ) - @pytest.mark.parametrize("out_fmt", CUSTOM_LS_TO_TEST) @pytest.mark.parametrize("in_fmt", OUTPUT_FORMATS) def test_custom_ls_output( @@ -668,6 +683,7 @@ def test_custom_ls_output( get_ssnr, get_odg, get_odg_bin, + split_comparison, ): run_renderer( record_property, @@ -681,10 +697,10 @@ def test_custom_ls_output( get_ssnr=get_ssnr, get_odg=get_odg, get_odg_bin=get_odg_bin, + split_comparison=split_comparison, ) - @pytest.mark.parametrize("out_fmt", CUSTOM_LS_TO_TEST) @pytest.mark.parametrize("in_fmt", CUSTOM_LS_TO_TEST) def test_custom_ls_input_output( @@ -698,6 +714,7 @@ def test_custom_ls_input_output( get_ssnr, get_odg, get_odg_bin, + split_comparison, ): run_renderer( record_property, @@ -711,10 +728,10 @@ def test_custom_ls_input_output( get_ssnr=get_ssnr, get_odg=get_odg, get_odg_bin=get_odg_bin, + split_comparison=split_comparison, ) - @pytest.mark.parametrize("out_fmt", OUTPUT_FORMATS_BINAURAL) @pytest.mark.parametrize("in_layout", CUSTOM_LS_TO_TEST) @pytest.mark.parametrize("frame_size", FRAMING_TO_TEST) @@ -730,6 +747,7 @@ def test_custom_ls_input_binaural( get_ssnr, get_odg, get_odg_bin, + split_comparison, ): run_renderer( record_property, @@ -744,10 +762,10 @@ def test_custom_ls_input_binaural( get_ssnr=get_ssnr, get_odg=get_odg, get_odg_bin=get_odg_bin, + split_comparison=split_comparison, ) - @pytest.mark.parametrize("trj_file", HR_TRAJECTORIES_TO_TEST) @pytest.mark.parametrize("out_fmt", OUTPUT_FORMATS_BINAURAL) @pytest.mark.parametrize("in_layout", CUSTOM_LS_TO_TEST) @@ -765,6 +783,7 @@ def test_custom_ls_input_binaural_headrotation( get_ssnr, get_odg, get_odg_bin, + split_comparison, ): run_renderer( record_property, @@ -780,13 +799,13 @@ def test_custom_ls_input_binaural_headrotation( get_ssnr=get_ssnr, get_odg=get_odg, get_odg_bin=get_odg_bin, + split_comparison=split_comparison, ) """ Metadata / scene description input """ - @pytest.mark.parametrize("out_fmt", OUTPUT_FORMATS) @pytest.mark.parametrize("in_fmt", METADATA_SCENES_TO_TEST) @pytest.mark.parametrize("frame_size", FRAMING_TO_TEST) @@ -802,6 +821,7 @@ def test_metadata( get_ssnr, get_odg, get_odg_bin, + split_comparison, ): run_renderer( record_property, @@ -817,13 +837,13 @@ def test_metadata( get_ssnr=get_ssnr, get_odg=get_odg, get_odg_bin=get_odg_bin, + split_comparison=split_comparison, ) """ non diegetic pan """ - @pytest.mark.parametrize("out_fmt", ["STEREO"]) @pytest.mark.parametrize("in_fmt", ["MONO"]) @pytest.mark.parametrize("non_diegetic_pan", ["0", "-30", "45", "90", "-90"]) @@ -839,6 +859,7 @@ def test_non_diegetic_pan_static( get_ssnr, get_odg, get_odg_bin, + split_comparison, ): run_renderer( record_property, @@ -853,10 +874,10 @@ def test_non_diegetic_pan_static( get_ssnr=get_ssnr, get_odg=get_odg, get_odg_bin=get_odg_bin, + split_comparison=split_comparison, ) - @pytest.mark.parametrize("out_fmt", ["STEREO"]) @pytest.mark.parametrize("in_fmt", ["ISM1"]) @pytest.mark.parametrize("non_diegetic_pan", ["0", "-30", "45", "90", "-90"]) @@ -872,6 +893,7 @@ def test_non_diegetic_pan_ism_static( get_ssnr, get_odg, get_odg_bin, + split_comparison, ): run_renderer( record_property, @@ -886,6 +908,7 @@ def test_non_diegetic_pan_ism_static( get_ssnr=get_ssnr, get_odg=get_odg, get_odg_bin=get_odg_bin, + split_comparison=split_comparison, ) @@ -909,9 +932,7 @@ def test_ambisonics_binaural_headrotation_refrotzero( in_fmt, out_fmt, trj_file, - get_mld, - get_mld_lim, - get_ssnr, + split_comparison, ): if test_info.config.option.create_ref or test_info.config.option.create_cut: pytest.skip("OTR tests only run for smoke test") @@ -932,6 +953,7 @@ def test_ambisonics_binaural_headrotation_refrotzero( "refrot_file": HR_TRAJECTORY_DIR.joinpath("const000.csv"), "frame_size": "5", }, + split_comparison=split_comparison, ) @@ -941,7 +963,12 @@ def test_ambisonics_binaural_headrotation_refrotzero( @pytest.mark.parametrize("out_fmt", OUTPUT_FORMATS_BINAURAL) @pytest.mark.parametrize("in_fmt", INPUT_FORMATS_AMBI) def test_ambisonics_binaural_headrotation_refrotequal( - record_property, props_to_record, test_info, in_fmt, out_fmt, get_mld, get_mld_lim + record_property, + props_to_record, + test_info, + in_fmt, + out_fmt, + split_comparison, ): if test_info.config.option.create_ref or test_info.config.option.create_cut: pytest.skip("OTR tests only run for smoke test") @@ -965,6 +992,7 @@ def test_ambisonics_binaural_headrotation_refrotequal( ), "frame_size": "5", }, + split_comparison=split_comparison, ) @@ -982,8 +1010,7 @@ def test_ambisonics_binaural_headrotation_refveczero( in_fmt, out_fmt, trj_file, - get_mld, - get_mld_lim, + split_comparison, ): if test_info.config.option.create_ref or test_info.config.option.create_cut: pytest.skip("OTR tests only run for smoke test") @@ -1004,6 +1031,7 @@ def test_ambisonics_binaural_headrotation_refveczero( "refvec_file": HR_TRAJECTORY_DIR.joinpath("const000-Vector3.csv"), "frame_size": "5", }, + split_comparison=split_comparison, ) @@ -1014,7 +1042,12 @@ def test_ambisonics_binaural_headrotation_refveczero( @pytest.mark.parametrize("out_fmt", OUTPUT_FORMATS_BINAURAL) @pytest.mark.parametrize("in_fmt", INPUT_FORMATS_AMBI) def test_ambisonics_binaural_headrotation_refvecequal( - record_property, props_to_record, test_info, in_fmt, out_fmt, get_mld, get_mld_lim + record_property, + props_to_record, + test_info, + in_fmt, + out_fmt, + split_comparison, ): if test_info.config.option.create_ref or test_info.config.option.create_cut: pytest.skip("OTR tests only run for smoke test") @@ -1042,6 +1075,7 @@ def test_ambisonics_binaural_headrotation_refvecequal( ), "frame_size": "5", }, + split_comparison=split_comparison, ) @@ -1052,7 +1086,12 @@ def test_ambisonics_binaural_headrotation_refvecequal( @pytest.mark.parametrize("out_fmt", OUTPUT_FORMATS_BINAURAL) @pytest.mark.parametrize("in_fmt", INPUT_FORMATS_AMBI) def test_ambisonics_binaural_headrotation_refvec_rotating( - record_property, props_to_record, test_info, in_fmt, out_fmt, get_mld, get_mld_lim + record_property, + props_to_record, + test_info, + in_fmt, + out_fmt, + split_comparison, ): if test_info.config.option.create_ref or test_info.config.option.create_cut: pytest.skip("OTR tests only run for smoke test") @@ -1081,6 +1120,7 @@ def test_ambisonics_binaural_headrotation_refvec_rotating( ), "frame_size": "5", }, + split_comparison=split_comparison, ) @@ -1093,7 +1133,12 @@ def test_ambisonics_binaural_headrotation_refvec_rotating( @pytest.mark.parametrize("out_fmt", OUTPUT_FORMATS_BINAURAL) @pytest.mark.parametrize("in_fmt", INPUT_FORMATS_AMBI) def test_ambisonics_binaural_headrotation_refvec_rotating_fixed_pos_offset( - record_property, props_to_record, test_info, in_fmt, out_fmt, get_mld, get_mld_lim + record_property, + props_to_record, + test_info, + in_fmt, + out_fmt, + split_comparison, ): if test_info.config.option.create_ref or test_info.config.option.create_cut: pytest.skip("OTR tests only run for smoke test") @@ -1118,6 +1163,7 @@ def test_ambisonics_binaural_headrotation_refvec_rotating_fixed_pos_offset( ), "frame_size": "5", }, + split_comparison=split_comparison, ) @@ -1129,7 +1175,12 @@ def test_ambisonics_binaural_headrotation_refvec_rotating_fixed_pos_offset( @pytest.mark.parametrize("out_fmt", OUTPUT_FORMATS_BINAURAL) @pytest.mark.parametrize("in_fmt", INPUT_FORMATS_AMBI) def test_ambisonics_binaural_headrotation_refveclev_vs_refvec( - record_property, props_to_record, test_info, in_fmt, out_fmt, get_mld, get_mld_lim + record_property, + props_to_record, + test_info, + in_fmt, + out_fmt, + split_comparison, ): if test_info.config.option.create_ref or test_info.config.option.create_cut: pytest.skip("OTR tests only run for smoke test") @@ -1153,6 +1204,7 @@ def test_ambisonics_binaural_headrotation_refveclev_vs_refvec( "refvec_file": HR_TRAJECTORY_DIR.joinpath("full-circle-4s-Vector3.csv"), "frame_size": "5", }, + split_comparison=split_comparison, ) @@ -1163,7 +1215,12 @@ def test_ambisonics_binaural_headrotation_refveclev_vs_refvec( @pytest.mark.parametrize("out_fmt", OUTPUT_FORMATS_BINAURAL) @pytest.mark.parametrize("in_fmt", INPUT_FORMATS_MC) def test_multichannel_binaural_headrotation_refvec_rotating( - record_property, props_to_record, test_info, in_fmt, out_fmt, get_mld, get_mld_lim + record_property, + props_to_record, + test_info, + in_fmt, + out_fmt, + split_comparison, ): if test_info.config.option.create_ref or test_info.config.option.create_cut: pytest.skip("OTR tests only run for smoke test") @@ -1191,6 +1248,7 @@ def test_multichannel_binaural_headrotation_refvec_rotating( ), "frame_size": "5", }, + split_comparison=split_comparison, ) @@ -1201,7 +1259,12 @@ def test_multichannel_binaural_headrotation_refvec_rotating( @pytest.mark.parametrize("out_fmt", OUTPUT_FORMATS_BINAURAL) @pytest.mark.parametrize("in_fmt", INPUT_FORMATS_ISM) def test_ism_binaural_headrotation_refvec_rotating( - record_property, props_to_record, test_info, in_fmt, out_fmt, get_mld, get_mld_lim + record_property, + props_to_record, + test_info, + in_fmt, + out_fmt, + split_comparison, ): if test_info.config.option.create_ref or test_info.config.option.create_cut: pytest.skip("OTR tests only run for smoke test") @@ -1233,4 +1296,5 @@ def test_ism_binaural_headrotation_refvec_rotating( "in_meta_files": in_meta_files, "frame_size": "5", }, + split_comparison=split_comparison, ) diff --git a/tests/renderer/utils.py b/tests/renderer/utils.py index ec5fe7f0cb..370da4cf89 100644 --- a/tests/renderer/utils.py +++ b/tests/renderer/utils.py @@ -174,6 +174,7 @@ def run_renderer( out_file=None, sr=48, render_for_peaq=False, + split_comparison=False, ) -> str: # prepare arguments and filepaths if trj_file is not None: @@ -393,7 +394,13 @@ def run_renderer( output_differs = output_differs[0] reason = reason[0] - props = parse_properties(reason, output_differs, props_to_record) + # splitting is not implemented for renderer tests yet + # if arg is given, need to record as "whole" so that XML parsing works + suffix = "" + if split_comparison: + suffix = "_whole" + + props = parse_properties(reason, output_differs, props_to_record, suffix) props["format"] = "Renderer" props["category"] = CAT_NORMAL for k, v in props.items(): @@ -420,6 +427,7 @@ def compare_renderer_args( out_fmt, ref_kwargs: Dict, cut_kwargs: Dict, + split_comparison=False, ): out_file_ref = run_renderer( record_property, @@ -428,6 +436,7 @@ def compare_renderer_args( in_fmt, out_fmt, **ref_kwargs, + split_comparison=split_comparison, ) ref, ref_fs = readfile(out_file_ref) out_file_cut = run_renderer( @@ -437,6 +446,7 @@ def compare_renderer_args( in_fmt, out_fmt, **cut_kwargs, + split_comparison=split_comparison, ) cut, cut_fs = readfile(out_file_cut) [diff_found, snr, gain_b, max_diff] = check_BE(test_info, ref, ref_fs, cut, cut_fs) -- GitLab From 13a44da024a6005f9346143b5d1c891fac89af22 Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Tue, 1 Apr 2025 08:46:27 +0200 Subject: [PATCH 089/106] skip split comparison for JBM cases only record whole-file measurements --- tests/codec_be_on_mr_nonselection/test_param_file.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/tests/codec_be_on_mr_nonselection/test_param_file.py b/tests/codec_be_on_mr_nonselection/test_param_file.py index 8158d3581a..cc7b01ee4a 100644 --- a/tests/codec_be_on_mr_nonselection/test_param_file.py +++ b/tests/codec_be_on_mr_nonselection/test_param_file.py @@ -642,7 +642,9 @@ def run_test( ) # 2. run comparison on split files if --split_comparison is given - if split_comparison: + # for JBM cases, comparison will fail because of length mismatch beetween split wav files and tracefiles + # -> skip split comparison for these cases + if split_comparison and not sim_opts: split_idx = get_split_idx(str(Path(testv_file).stem), int(sampling_rate)) output_differs_splits, reason_splits = cmp_pcm( @@ -668,6 +670,8 @@ def run_test( output_differs_parts += output_differs_splits reason_parts += reason_splits + # separate if to also record the whole-file comparison for JBM cases with "_whole" + if split_comparison: prop_suffix = ["_whole"] + [ f"_split{i:03d}" for i in range(1, len(split_idx) + 1) ] -- GitLab From 93d77e5f2109a4bcb7e5feb082d6d2248e674bdc Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Tue, 1 Apr 2025 12:20:20 +0200 Subject: [PATCH 090/106] enable split comparison for renderer tests --- tests/renderer/constants.py | 120 +++++++++++++++++++++++++------- tests/renderer/test_renderer.py | 49 +++++++++++-- tests/renderer/utils.py | 73 ++++++++++++++----- 3 files changed, 194 insertions(+), 48 deletions(-) diff --git a/tests/renderer/constants.py b/tests/renderer/constants.py index 39f57a6d89..c61800a1a2 100644 --- a/tests/renderer/constants.py +++ b/tests/renderer/constants.py @@ -1,33 +1,33 @@ #!/usr/bin/env python3 """ - (C) 2022-2025 IVAS codec Public Collaboration with portions copyright Dolby International AB, Ericsson AB, - Fraunhofer-Gesellschaft zur Foerderung der angewandten Forschung e.V., Huawei Technologies Co. LTD., - Koninklijke Philips N.V., Nippon Telegraph and Telephone Corporation, Nokia Technologies Oy, Orange, - Panasonic Holdings Corporation, Qualcomm Technologies, Inc., VoiceAge Corporation, and other - contributors to this repository. All Rights Reserved. +(C) 2022-2025 IVAS codec Public Collaboration with portions copyright Dolby International AB, Ericsson AB, +Fraunhofer-Gesellschaft zur Foerderung der angewandten Forschung e.V., Huawei Technologies Co. LTD., +Koninklijke Philips N.V., Nippon Telegraph and Telephone Corporation, Nokia Technologies Oy, Orange, +Panasonic Holdings Corporation, Qualcomm Technologies, Inc., VoiceAge Corporation, and other +contributors to this repository. All Rights Reserved. - This software is protected by copyright law and by international treaties. - The IVAS codec Public Collaboration consisting of Dolby International AB, Ericsson AB, - Fraunhofer-Gesellschaft zur Foerderung der angewandten Forschung e.V., Huawei Technologies Co. LTD., - Koninklijke Philips N.V., Nippon Telegraph and Telephone Corporation, Nokia Technologies Oy, Orange, - Panasonic Holdings Corporation, Qualcomm Technologies, Inc., VoiceAge Corporation, and other - contributors to this repository retain full ownership rights in their respective contributions in - the software. This notice grants no license of any kind, including but not limited to patent - license, nor is any license granted by implication, estoppel or otherwise. +This software is protected by copyright law and by international treaties. +The IVAS codec Public Collaboration consisting of Dolby International AB, Ericsson AB, +Fraunhofer-Gesellschaft zur Foerderung der angewandten Forschung e.V., Huawei Technologies Co. LTD., +Koninklijke Philips N.V., Nippon Telegraph and Telephone Corporation, Nokia Technologies Oy, Orange, +Panasonic Holdings Corporation, Qualcomm Technologies, Inc., VoiceAge Corporation, and other +contributors to this repository retain full ownership rights in their respective contributions in +the software. This notice grants no license of any kind, including but not limited to patent +license, nor is any license granted by implication, estoppel or otherwise. - Contributors are required to enter into the IVAS codec Public Collaboration agreement before making - contributions. +Contributors are required to enter into the IVAS codec Public Collaboration agreement before making +contributions. - This software is provided "AS IS", without any express or implied warranties. The software is in the - development stage. It is intended exclusively for experts who have experience with such software and - solely for the purpose of inspection. All implied warranties of non-infringement, merchantability - and fitness for a particular purpose are hereby disclaimed and excluded. +This software is provided "AS IS", without any express or implied warranties. The software is in the +development stage. It is intended exclusively for experts who have experience with such software and +solely for the purpose of inspection. All implied warranties of non-infringement, merchantability +and fitness for a particular purpose are hereby disclaimed and excluded. - Any dispute, controversy or claim arising under or in relation to providing this software shall be - submitted to and settled by the final, binding jurisdiction of the courts of Munich, Germany in - accordance with the laws of the Federal Republic of Germany excluding its conflict of law rules and - the United Nations Convention on Contracts on the International Sales of Goods. +Any dispute, controversy or claim arising under or in relation to providing this software shall be +submitted to and settled by the final, binding jurisdiction of the courts of Munich, Germany in +accordance with the laws of the Federal Republic of Germany excluding its conflict of law rules and +the United Nations Convention on Contracts on the International Sales of Goods. """ from pathlib import Path @@ -298,6 +298,77 @@ FORMAT_TO_METADATA_FILES = { ], } +FORMAT_TO_METADATA_FILES_LTV = { + "ISM1": [str(TESTV_DIR.joinpath("ltvISM1.csv"))], + "ISM2": [ + str(TESTV_DIR.joinpath("ltvISM1.csv")), + str(TESTV_DIR.joinpath("ltvISM2.csv")), + ], + "ISM3": [ + str(TESTV_DIR.joinpath("ltvISM1.csv")), + str(TESTV_DIR.joinpath("ltvISM2.csv")), + str(TESTV_DIR.joinpath("ltvISM3.csv")), + ], + "ISM4": [ + str(TESTV_DIR.joinpath("ltvISM1.csv")), + str(TESTV_DIR.joinpath("ltvISM2.csv")), + str(TESTV_DIR.joinpath("ltvISM3.csv")), + str(TESTV_DIR.joinpath("ltvISM4.csv")), + ], + "NDP_ISM4": [ + str(TESTV_DIR.joinpath("ltvISM1.csv")), + str(TESTV_DIR.joinpath("stvISM2_non-diegetic-pan.csv")), + str(TESTV_DIR.joinpath("ltvISM3.csv")), + str(TESTV_DIR.joinpath("ltvISM4.csv")), + ], + "MASA1": [str(TESTV_DIR.joinpath("ltv48_MASA1TC.met"))], + "MASA2": [str(TESTV_DIR.joinpath("ltv48_MASA2TC.met"))], + "OMASA_1_1": [ + str(TESTV_DIR.joinpath("ltvISM1.csv")), + str(TESTV_DIR.joinpath("ltv48_OMASA_1ISM_1TC.met")), + ], + "OMASA_1_2": [ + str(TESTV_DIR.joinpath("ltvISM1.csv")), + str(TESTV_DIR.joinpath("ltvISM2.csv")), + str(TESTV_DIR.joinpath("ltv48_OMASA_2ISM_1TC.met")), + ], + "OMASA_1_3": [ + str(TESTV_DIR.joinpath("ltvISM1.csv")), + str(TESTV_DIR.joinpath("ltvISM2.csv")), + str(TESTV_DIR.joinpath("ltvISM3.csv")), + str(TESTV_DIR.joinpath("ltv48_OMASA_3ISM_1TC.met")), + ], + "OMASA_1_4": [ + str(TESTV_DIR.joinpath("ltvISM1.csv")), + str(TESTV_DIR.joinpath("ltvISM2.csv")), + str(TESTV_DIR.joinpath("ltvISM3.csv")), + str(TESTV_DIR.joinpath("ltvISM4.csv")), + str(TESTV_DIR.joinpath("ltv48_OMASA_4ISM_1TC.met")), + ], + "OMASA_2_1": [ + str(TESTV_DIR.joinpath("ltvISM1.csv")), + str(TESTV_DIR.joinpath("ltv48_OMASA_1ISM_2TC.met")), + ], + "OMASA_2_2": [ + str(TESTV_DIR.joinpath("ltvISM1.csv")), + str(TESTV_DIR.joinpath("ltvISM2.csv")), + str(TESTV_DIR.joinpath("ltv48_OMASA_2ISM_2TC.met")), + ], + "OMASA_2_3": [ + str(TESTV_DIR.joinpath("ltvISM1.csv")), + str(TESTV_DIR.joinpath("ltvISM2.csv")), + str(TESTV_DIR.joinpath("ltvISM3.csv")), + str(TESTV_DIR.joinpath("ltv48_OMASA_3ISM_2TC.met")), + ], + "OMASA_2_4": [ + str(TESTV_DIR.joinpath("ltvISM1.csv")), + str(TESTV_DIR.joinpath("ltvISM2.csv")), + str(TESTV_DIR.joinpath("ltvISM3.csv")), + str(TESTV_DIR.joinpath("ltvISM4.csv")), + str(TESTV_DIR.joinpath("ltv48_OMASA_4ISM_2TC.met")), + ], +} + """ Input formats """ INPUT_FORMATS_AMBI = ["FOA", "HOA2", "HOA3"] @@ -347,4 +418,5 @@ PEAQ_SUPPORTED_FMT = [ "BINAURAL", "BINAURAL_ROOM_IR", "BINAURAL_ROOM_REVERB", -] \ No newline at end of file +] + diff --git a/tests/renderer/test_renderer.py b/tests/renderer/test_renderer.py index 956f773666..60ec37a69c 100644 --- a/tests/renderer/test_renderer.py +++ b/tests/renderer/test_renderer.py @@ -33,6 +33,7 @@ the United Nations Convention on Contracts on the International Sales of Goods. import pytest from .constants import ( + FORMAT_TO_METADATA_FILES_LTV, OUTPUT_FORMATS, INPUT_FORMATS_AMBI, FRAMING_TO_TEST, @@ -385,13 +386,18 @@ def test_ism( get_odg_bin, split_comparison, ): + md_files = ( + FORMAT_TO_METADATA_FILES_LTV[in_fmt] + if test_info.config.option.use_ltv + else FORMAT_TO_METADATA_FILES[in_fmt] + ) run_renderer( record_property, props_to_record, test_info, in_fmt, out_fmt, - in_meta_files=FORMAT_TO_METADATA_FILES[in_fmt], + in_meta_files=md_files, binary_suffix=EXE_SUFFIX, frame_size=frame_size, get_mld=get_mld, @@ -421,7 +427,11 @@ def test_ism_binaural_static( split_comparison, ): try: - in_meta_files = FORMAT_TO_METADATA_FILES[in_fmt] + in_meta_files = ( + FORMAT_TO_METADATA_FILES_LTV[in_fmt] + if test_info.config.option.use_ltv + else FORMAT_TO_METADATA_FILES[in_fmt] + ) except KeyError: in_meta_files = None @@ -463,7 +473,11 @@ def test_ism_binaural_headrotation( split_comparison, ): try: - in_meta_files = FORMAT_TO_METADATA_FILES[in_fmt] + in_meta_files = ( + FORMAT_TO_METADATA_FILES_LTV[in_fmt] + if test_info.config.option.use_ltv + else FORMAT_TO_METADATA_FILES[in_fmt] + ) except KeyError: in_meta_files = None @@ -506,13 +520,19 @@ def test_masa( get_odg_bin, split_comparison, ): + md_files = ( + FORMAT_TO_METADATA_FILES_LTV[in_fmt] + if test_info.config.option.use_ltv + else FORMAT_TO_METADATA_FILES[in_fmt] + ) + run_renderer( record_property, props_to_record, test_info, in_fmt, out_fmt, - in_meta_files=FORMAT_TO_METADATA_FILES[in_fmt], + in_meta_files=md_files, binary_suffix=EXE_SUFFIX, frame_size=frame_size, get_mld=get_mld, @@ -544,13 +564,18 @@ def test_masa_binaural_static( if out_fmt in ["BINAURAL_ROOM_IR", "BINAURAL_ROOM_REVERB"]: pytest.skip("Skipping binaural room outputs for MASA as unimplemented.") + md_files = ( + FORMAT_TO_METADATA_FILES_LTV[in_fmt] + if test_info.config.option.use_ltv + else FORMAT_TO_METADATA_FILES[in_fmt] + ) run_renderer( record_property, props_to_record, test_info, in_fmt, out_fmt, - in_meta_files=FORMAT_TO_METADATA_FILES[in_fmt], + in_meta_files=md_files, binary_suffix=EXE_SUFFIX, frame_size=frame_size, get_mld=get_mld, @@ -584,6 +609,12 @@ def test_masa_binaural_headrotation( if out_fmt in ["BINAURAL_ROOM_IR", "BINAURAL_ROOM_REVERB"]: pytest.skip("Skipping binaural room outputs for MASA as unimplemented.") + md_files = ( + FORMAT_TO_METADATA_FILES_LTV[in_fmt] + if test_info.config.option.use_ltv + else FORMAT_TO_METADATA_FILES[in_fmt] + ) + run_renderer( record_property, props_to_record, @@ -591,7 +622,7 @@ def test_masa_binaural_headrotation( in_fmt, out_fmt, trj_file=HR_TRAJECTORY_DIR.joinpath(f"{trj_file}.csv"), - in_meta_files=FORMAT_TO_METADATA_FILES[in_fmt], + in_meta_files=md_files, binary_suffix=EXE_SUFFIX, frame_size=frame_size, get_mld=get_mld, @@ -1270,7 +1301,11 @@ def test_ism_binaural_headrotation_refvec_rotating( pytest.skip("OTR tests only run for smoke test") try: - in_meta_files = FORMAT_TO_METADATA_FILES[in_fmt] + in_meta_files = ( + FORMAT_TO_METADATA_FILES_LTV[in_fmt] + if test_info.config.option.use_ltv + else FORMAT_TO_METADATA_FILES[in_fmt] + ) except KeyError: in_meta_files = None diff --git a/tests/renderer/utils.py b/tests/renderer/utils.py index 370da4cf89..e9a8f27345 100644 --- a/tests/renderer/utils.py +++ b/tests/renderer/utils.py @@ -62,7 +62,7 @@ from ..constants import CAT_NORMAL sys.path.append(SCRIPTS_DIR) from pyaudio3dtools.audiofile import readfile from ..cmp_pcm import cmp_pcm -from ..conftest import parse_properties +from ..conftest import parse_properties, get_split_idx def run_cmd(cmd, test_info, env=None): @@ -233,7 +233,8 @@ def run_renderer( # if in REF or CUT creation mode use the comparetestv if test_info.config.option.create_ref or test_info.config.option.create_cut: FORMAT_TO_FILE = FORMAT_TO_FILE_COMPARETEST - elif test_info.config.option.use_ltv: + + if test_info.config.option.use_ltv: if test_info.config.option.ltv_dir: FORMAT_TO_FILE = dict() for k, v in FORMAT_TO_FILE_LTV.items(): @@ -310,6 +311,14 @@ def run_renderer( env["UBSAN_OPTIONS"] + f",log_path=usan_log_{test_info.node.name}" ) + testcase_props = { + "format": "Renderer", + "category": CAT_NORMAL, + } + + for k, v in testcase_props.items(): + record_property(k, v) + # run the renderer run_cmd(cmd, test_info, env) @@ -373,9 +382,13 @@ def run_renderer( else: odg_input = in_file - # see constants.py + ### run the comparison tools + split_idx = np.empty(0) + prop_suffix = [""] + + # 1. run comparison on whole files - this is done always, regardless of the presence of --split_comparison ref_fs = int(cmd[10]) * 1000 - output_differs, reason = cmp_pcm( + output_differs_parts, reason_parts = cmp_pcm( out_file, out_file_ref, out_fmt, @@ -390,24 +403,50 @@ def run_renderer( odg_test=odg_test, odg_ref=odg_ref, scalefac=test_info.config.option.scalefac, + split_idx=split_idx, ) - output_differs = output_differs[0] - reason = reason[0] - # splitting is not implemented for renderer tests yet - # if arg is given, need to record as "whole" so that XML parsing works - suffix = "" + # 2. run comparison on split files if --split_comparison is given + # for JBM cases, comparison will fail because of length mismatch beetween split wav files and tracefiles + # -> skip split comparison for these cases if split_comparison: - suffix = "_whole" + split_idx = get_split_idx(str(Path(in_file).stem), ref_fs // 1000) - props = parse_properties(reason, output_differs, props_to_record, suffix) - props["format"] = "Renderer" - props["category"] = CAT_NORMAL - for k, v in props.items(): - record_property(k, v) + output_differs_splits, reason_splits = cmp_pcm( + out_file, + out_file_ref, + out_fmt, + ref_fs, + get_mld=get_mld, + mld_lim=get_mld_lim, + abs_tol=abs_tol, + get_ssnr=get_ssnr, + get_odg=get_odg, + get_odg_bin=get_odg_bin, + odg_input=odg_input, + odg_test=odg_test, + odg_ref=odg_ref, + scalefac=test_info.config.option.scalefac, + split_idx=split_idx, + ) + output_differs_parts += output_differs_splits + reason_parts += reason_splits + + prop_suffix = ["_whole"] + [ + f"_split{i:03d}" for i in range(1, len(split_idx) + 1) + ] + + for output_differs, reason, suffix in zip( + output_differs_parts, reason_parts, prop_suffix + ): + result_props = parse_properties( + reason, output_differs, props_to_record, suffix + ) + for k, v in result_props.items(): + record_property(k, v) - if output_differs: - pytest.fail(f"Output differs: ({reason})") + if output_differs_parts[0]: + pytest.fail(f"Output differs: ({reason_parts[0]})") # compare metadata files in case of MASA prerendering if "MASA" in str(out_fmt): -- GitLab From 119ef09829f25e325c4752057a9207f329159b59 Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Tue, 1 Apr 2025 13:51:25 +0200 Subject: [PATCH 091/106] add write out option for histograms to csv --- scripts/create_histograms.py | 26 +++++++++++++++++++++++--- 1 file changed, 23 insertions(+), 3 deletions(-) diff --git a/scripts/create_histograms.py b/scripts/create_histograms.py index 160ea95419..7b05d93f8f 100644 --- a/scripts/create_histograms.py +++ b/scripts/create_histograms.py @@ -37,17 +37,22 @@ def create_histograms( display_only: bool, bins_for_measures=BINS_FOR_MEASURES, prefix="", + write_out_histograms=False, ): formats = df["format"].unique() categories = df["category"].unique() - if not display_only: + if not display_only or write_out_histograms: output_folder.mkdir(exist_ok=True, parents=True) for measure in measures: measure_in_df = prefix + measure bins = bins_for_measures.get(measure, get_bins_for_diff(df[measure_in_df])) x = [f"{x}" for x in bins] + ["", "ERROR"] + + df_hist = pd.DataFrame(columns=["format", "category"] + x) + hist_row_count = 0 + for fmt in formats: fig, ax = plt.subplots() ax.xaxis.set_major_formatter("{x:.1f}") @@ -57,9 +62,9 @@ def create_histograms( df_slice = df[data_mask] error_mask = df_slice["result"] == "ERROR" n_errors = np.sum(error_mask) - df_hist = df_slice[np.logical_not(error_mask)] + df_slice = df_slice[np.logical_not(error_mask)] - counts, _ = np.histogram(df_hist[measure_in_df], bins) + counts, _ = np.histogram(df_slice[measure_in_df], bins) data = np.concatenate([counts, [0], [n_errors], [0]]) ax.bar( @@ -74,6 +79,10 @@ def create_histograms( ) bottom += data + hist_row = [fmt, cat] + list(counts) + [0] + [0, n_errors] + df_hist.loc[hist_row_count] = hist_row + hist_row_count += 1 + # Histogram layout ax.set_title(fmt) ax.legend(loc="best") @@ -94,6 +103,11 @@ def create_histograms( plt.savefig(image_path) plt.close(fig) + if write_out_histograms: + df_hist.to_csv( + output_folder.joinpath(f"histogram_{measure}.csv"), index=False + ) + if display_only: plt.show() @@ -133,6 +147,11 @@ Use this for visualising diff scores.""", default="", help="Common suffix to use when collecting measures from the input csv file", ) + parser.add_argument( + "--write-out-histograms", + action="store_true", + help="Write out the histogram values to csv", + ) args = parser.parse_args() df = pd.read_csv(args.csv_report) @@ -152,4 +171,5 @@ Use this for visualising diff scores.""", args.display_only, bins_for_measures, args.prefix, + args.write_out_histograms, ) -- GitLab From e69f994532b2d64bdbcbab5b1d3b8304caa5e0f1 Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Tue, 1 Apr 2025 14:06:43 +0200 Subject: [PATCH 092/106] add consistent color assignment this helps keep split and whole page in synch wrt colors, even if a format/category/mode is omitted in one of them (e.g. JBM in split) --- scripts/create_histograms.py | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/scripts/create_histograms.py b/scripts/create_histograms.py index 7b05d93f8f..5ec2f0e599 100644 --- a/scripts/create_histograms.py +++ b/scripts/create_histograms.py @@ -3,6 +3,7 @@ import argparse import math import pathlib +import sys import pandas as pd import numpy as np from typing import List @@ -25,6 +26,19 @@ BINS_FOR_MEASURES = { DEFAULT_MEASURES = ["MAX_ABS_DIFF", "MLD", "MIN_SSNR", "MIN_ODG"] +HERE = pathlib.Path(__file__).parent +ROOT_DIR = HERE.parent +sys.path.append(str(ROOT_DIR)) +from tests.constants import CAT_NORMAL, CAT_BITRATE_SWITCHING, CAT_DTX, CAT_JBM, CAT_PLC + +COLORS_FOR_CATEGORIES = { + CAT_DTX: "tab:blue", + CAT_PLC: "tab:orange", + CAT_NORMAL: "tab:green", + CAT_JBM: "tab:red", + CAT_BITRATE_SWITCHING: "tab:purple", +} + def get_bins_for_diff(data: pd.Series): return np.linspace(data.min(), data.max(), num=10) @@ -76,6 +90,7 @@ def create_histograms( linewidth=0.5, label=cat, bottom=bottom, + color=COLORS_FOR_CATEGORIES[cat], ) bottom += data -- GitLab From 2830ade5340c7dfb28d2335bfb072bad32f3bfdf Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Tue, 1 Apr 2025 14:16:52 +0200 Subject: [PATCH 093/106] fix wrong ref/cut file order in cmp_pcm call --- tests/renderer/utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/renderer/utils.py b/tests/renderer/utils.py index 6c17370678..e24dc40e2e 100644 --- a/tests/renderer/utils.py +++ b/tests/renderer/utils.py @@ -413,8 +413,8 @@ def run_renderer( split_idx = get_split_idx(str(Path(in_file).stem), ref_fs // 1000) output_differs_splits, reason_splits = cmp_pcm( - out_file, out_file_ref, + out_file, out_fmt, ref_fs, get_mld=get_mld, -- GitLab From 883fa28c8675c72e80586ce81542577be56c9cc4 Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Tue, 1 Apr 2025 14:44:32 +0200 Subject: [PATCH 094/106] fix calling record_property on being None --- tests/renderer/utils.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/tests/renderer/utils.py b/tests/renderer/utils.py index e24dc40e2e..82622fafe7 100644 --- a/tests/renderer/utils.py +++ b/tests/renderer/utils.py @@ -316,8 +316,9 @@ def run_renderer( "category": CAT_NORMAL, } - for k, v in testcase_props.items(): - record_property(k, v) + if record_property is not None: + for k, v in testcase_props.items(): + record_property(k, v) # run the renderer run_cmd(cmd, test_info, env) -- GitLab From 244927acd7e08a33c190d7fc5350d2713fc8fc72 Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Tue, 1 Apr 2025 14:49:04 +0200 Subject: [PATCH 095/106] do not record properties when reference is being run --- .../test_param_file.py | 5 +-- tests/codec_be_on_mr_nonselection/test_sba.py | 31 ++++++++++++------- 2 files changed, 22 insertions(+), 14 deletions(-) diff --git a/tests/codec_be_on_mr_nonselection/test_param_file.py b/tests/codec_be_on_mr_nonselection/test_param_file.py index cc7b01ee4a..047644e8c0 100644 --- a/tests/codec_be_on_mr_nonselection/test_param_file.py +++ b/tests/codec_be_on_mr_nonselection/test_param_file.py @@ -348,8 +348,9 @@ def run_test( if eid_opts != "": testcase_props["category"] = CAT_PLC - for k, v in testcase_props.items(): - dut_encoder_frontend.record_property(k, v) + if update_ref != 1: + for k, v in testcase_props.items(): + dut_encoder_frontend.record_property(k, v) assert bitstream_file == "bit" # in the parameter file, only "bit" is used as bitstream file name diff --git a/tests/codec_be_on_mr_nonselection/test_sba.py b/tests/codec_be_on_mr_nonselection/test_sba.py index 3d2c586158..17d340d484 100644 --- a/tests/codec_be_on_mr_nonselection/test_sba.py +++ b/tests/codec_be_on_mr_nonselection/test_sba.py @@ -151,8 +151,10 @@ def test_pca_enc( assert 0 input_config = SBA_FORMAT[abs(int(sba_order))] - for k, v in testcase_props.items(): - dut_encoder_frontend.record_property(k, v) + + if update_ref != 1: + for k, v in testcase_props.items(): + dut_encoder_frontend.record_property(k, v) if not decoder_only: sba_enc( @@ -307,8 +309,9 @@ def test_sba_enc_system( except ValueError: testcase_props["category"] = CAT_BITRATE_SWITCHING - for k, v in testcase_props.items(): - dut_encoder_frontend.record_property(k, v) + if update_ref != 1: + for k, v in testcase_props.items(): + dut_encoder_frontend.record_property(k, v) if not decoder_only: ref_stats_file, dut_stats_file = sba_enc( @@ -459,8 +462,9 @@ def test_spar_hoa2_enc_system( assert 0 input_config = SBA_FORMAT[abs(int(sba_order))] - for k, v in testcase_props.items(): - dut_encoder_frontend.record_property(k, v) + if update_ref != 1: + for k, v in testcase_props.items(): + dut_encoder_frontend.record_property(k, v) if not decoder_only: ref_stats_file, dut_stats_file = sba_enc( @@ -611,8 +615,9 @@ def test_spar_hoa3_enc_system( assert 0 input_config = SBA_FORMAT[abs(int(sba_order))] - for k, v in testcase_props.items(): - dut_encoder_frontend.record_property(k, v) + if update_ref != 1: + for k, v in testcase_props.items(): + dut_encoder_frontend.record_property(k, v) if not decoder_only: ref_stats_file, dut_stats_file = sba_enc( @@ -778,8 +783,9 @@ def test_sba_enc_BWforce_system( except ValueError: testcase_props["category"] = CAT_BITRATE_SWITCHING - for k, v in testcase_props.items(): - dut_encoder_frontend.record_property(k, v) + if update_ref != 1: + for k, v in testcase_props.items(): + dut_encoder_frontend.record_property(k, v) if not decoder_only: ref_stats_file, dut_stats_file = sba_enc( @@ -969,8 +975,9 @@ def test_sba_plc_system( if plc_pattern is not None: testcase_props["category"] = CAT_PLC - for k, v in testcase_props.items(): - dut_encoder_frontend.record_property(k, v) + if update_ref != 1: + for k, v in testcase_props.items(): + dut_encoder_frontend.record_property(k, v) if not decoder_only: sba_enc( -- GitLab From 814385b175eba0f4e5aa9cc84bc1d6300cf97d97 Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Wed, 2 Apr 2025 11:17:10 +0200 Subject: [PATCH 096/106] some adjustments to basop compat CI --- .gitlab-ci.yml | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 1721018369..b1d952ddad 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -134,8 +134,11 @@ stages: - sed -i.bak -e "s/\/\/\ *\(#define\ *DEBUGGING\ *\)/\1/g" lib_com/options.h .get-basop-float-reference: &get-basop-float-reference - - git clone -b $BASOP_REFERENCE_BRANCH https://forge.3gpp.org/rep/sa4/audio/ivas-basop.git --single-branch - - make -j -C ivas-basop + - git clone -b $BASOP_REFERENCE_BRANCH https://forge.3gpp.org/rep/sa4/audio/ivas-basop.git --single-branch --depth 1 ivas-basop + - cd ivas-basop + - git status + - make -j + - cd - - cp ivas-basop/IVAS_cod ./ - cp ivas-basop/IVAS_dec ./ @@ -463,7 +466,7 @@ branch-is-up-to-date-with-main-post: - .test-job-linux - .rules-merge-request-to-basop-ci-branch tags: - - ivas-linux-fast + - ivas-linux before_script: - python3 ci/remove_unsupported_testcases.py $PARAM_FILE - python3 tests/create_short_testvectors.py @@ -502,6 +505,8 @@ basop-ref-compat-stv: basop-ref-compat-ltv: extends: - .basop-ci-branch-compat-template + tags: + - ivas-linux-fast variables: PARAM_FILE: scripts/config/self_test_ltv.prm script: @@ -518,6 +523,8 @@ basop-ref-compat-encoder-stv: basop-ref-compat-encoder-ltv: extends: - .basop-ci-branch-compat-template + tags: + - ivas-linux-fast variables: PARAM_FILE: scripts/config/self_test_ltv_basop_encoder.prm script: -- GitLab From 772458c220d25506fecc839b7d5cf7d74dc80c33 Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Wed, 2 Apr 2025 11:44:00 +0200 Subject: [PATCH 097/106] add timeouts for basop-compat tests + new unsupported case --- .gitlab-ci.yml | 7 ++++++- ci/remove_unsupported_testcases.py | 2 ++ 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index b1d952ddad..b6e4433136 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -477,7 +477,7 @@ branch-is-up-to-date-with-main-post: - *get-basop-float-reference - - python3 -m pytest tests/codec_be_on_mr_nonselection $PYTEST_ARGS --param_file $PARAM_FILE --update_ref 1 --ref_encoder_path IVAS_cod --ref_decoder_path IVAS_dec --html=report.html --self-contained-html --junit-xml=report-junit.xml || true + - python3 -m pytest tests/codec_be_on_mr_nonselection $PYTEST_ARGS --param_file $PARAM_FILE --update_ref 1 --ref_encoder_path IVAS_cod --ref_decoder_path IVAS_dec --html=report.html --self-contained-html --junit-xml=report-junit.xml --testcase_timeout=$TESTCASE_TIMEOUT || true - zero_errors=$(cat report-junit.xml | grep -c 'errors="0"') || true - if [ $zero_errors != 1 ]; then echo "Run errors encountered with $PARAM_FILE !"; exit $EXIT_CODE_FAIL; fi @@ -499,6 +499,7 @@ basop-ref-compat-stv: - .basop-ci-branch-compat-template variables: PARAM_FILE: scripts/config/self_test.prm + TESTCASE_TIMEOUT: $TESTCASE_TIMEOUT_STV_SANITIZERS script: - exit 0 @@ -509,6 +510,7 @@ basop-ref-compat-ltv: - ivas-linux-fast variables: PARAM_FILE: scripts/config/self_test_ltv.prm + TESTCASE_TIMEOUT: $TESTCASE_TIMEOUT_LTV_SANITIZERS script: - exit 0 @@ -517,6 +519,7 @@ basop-ref-compat-encoder-stv: - .basop-ci-branch-compat-template variables: PARAM_FILE: scripts/config/self_test_basop_encoder.prm + TESTCASE_TIMEOUT: $TESTCASE_TIMEOUT_STV_SANITIZERS script: - exit 0 @@ -527,6 +530,7 @@ basop-ref-compat-encoder-ltv: - ivas-linux-fast variables: PARAM_FILE: scripts/config/self_test_ltv_basop_encoder.prm + TESTCASE_TIMEOUT: $TESTCASE_TIMEOUT_LTV_SANITIZERS script: - exit 0 @@ -537,6 +541,7 @@ basop-ref-compat-encoder-dmx-comp: PARAM_FILE: scripts/config/self_test_basop_encoder.prm # USING PYTEST_ADDOPTS env var did not work for some reason when testing locally - maybe because this is a custom option PYTEST_ARGS: "--compare_enc_dmx" + TESTCASE_TIMEOUT: $TESTCASE_TIMEOUT_STV_SANITIZERS script: - exit 0 diff --git a/ci/remove_unsupported_testcases.py b/ci/remove_unsupported_testcases.py index e5e184fa83..667463ec7a 100644 --- a/ci/remove_unsupported_testcases.py +++ b/ci/remove_unsupported_testcases.py @@ -66,6 +66,8 @@ TESTCASES_MAIN = [ "MASA 1TC at 256kbps, 48kHz in, 48 kHz out, BINAURAL_ROOM_REVERB out custom configuration", "MASA 1TC at 256 kbps, 48kHz in, 48kHz out, BINAURAL_ROOM_REVERB out, HR custom configuration", "OMASA 2TC 4ISM at br sw techs 13.2 to 512 kbps start 80 kbps, 48kHz in, 48kHz out, EXT out", + "Multi-channel 7_1_4 bitrate switching, 48kHz in, 48kHz out, BINAURAL out, HR, JBM Prof 5", + "Multi-channel 7_1 bitrate switching, 48kHz in, 32kHz out, BINAURAL_ROOM_REVERB out, HR, JBM Prof 5", # object editing starts here "OMASA 2Dir2TC 4ISM at 80 kbps, 48kHz in, 48kHz out, BINAURAL out, default object editing, 1SEP-PARAM", "OMASA 2Dir2TC 4ISM at 256 kbps, 48kHz in, 48kHz out, BINAURAL out, object editing, JBM Prof 5, DISC", -- GitLab From 5f6b3922a80a7139913ded0da11f1582e275053e Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Thu, 3 Apr 2025 13:59:21 +0200 Subject: [PATCH 098/106] add testcase to list of unsupported ones see https://forge.3gpp.org/rep/sa4/audio/ivas-basop/-/issues/1458 --- ci/remove_unsupported_testcases.py | 1 + 1 file changed, 1 insertion(+) diff --git a/ci/remove_unsupported_testcases.py b/ci/remove_unsupported_testcases.py index 667463ec7a..870d513111 100644 --- a/ci/remove_unsupported_testcases.py +++ b/ci/remove_unsupported_testcases.py @@ -68,6 +68,7 @@ TESTCASES_MAIN = [ "OMASA 2TC 4ISM at br sw techs 13.2 to 512 kbps start 80 kbps, 48kHz in, 48kHz out, EXT out", "Multi-channel 7_1_4 bitrate switching, 48kHz in, 48kHz out, BINAURAL out, HR, JBM Prof 5", "Multi-channel 7_1 bitrate switching, 48kHz in, 32kHz out, BINAURAL_ROOM_REVERB out, HR, JBM Prof 5", + "3 ISM with metadata bitrate switching from 48 kbps to 32 kbps, 48 kHz in, 32 kHz out, DTX, BINAURAL_ROOM_IR out, JBM Prof 5", # object editing starts here "OMASA 2Dir2TC 4ISM at 80 kbps, 48kHz in, 48kHz out, BINAURAL out, default object editing, 1SEP-PARAM", "OMASA 2Dir2TC 4ISM at 256 kbps, 48kHz in, 48kHz out, BINAURAL out, object editing, JBM Prof 5, DISC", -- GitLab From 19fe6270cb5d566ffcb85e6daa223e823752e0c8 Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Thu, 3 Apr 2025 15:37:43 +0200 Subject: [PATCH 099/106] add explicit fail if no split indices found --- tests/conftest.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/tests/conftest.py b/tests/conftest.py index 69a13e6d7f..7f3788e209 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1278,7 +1278,14 @@ def get_split_idx(input_file: str, sampling_rate_khz: int) -> Optional[np.ndarra format = "osba_hoa" else: format = input_file.split("_")[-1].lower() - idx = SPLIT_IDX.get(format, np.empty(0)).copy() + idx = SPLIT_IDX.get(format) + + if idx is None: + pytest.fail( + f"Could not get split indices for file {input_file} with infered format {format}." + ) + + idx = idx.copy() if len(idx) > 0 and sampling_rate_khz != 16: idx *= sampling_rate_khz // 16 -- GitLab From 6dc93d57d61bfed75d1d0e732c9e527cd0c26111 Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Mon, 7 Apr 2025 11:45:16 +0200 Subject: [PATCH 100/106] back to empty split idx array as default and no fail also avoid double comparison of unsplitted files --- tests/codec_be_on_mr_nonselection/test_param_file.py | 3 +++ tests/codec_be_on_mr_nonselection/test_sba.py | 4 ++++ tests/conftest.py | 9 +++------ tests/renderer/utils.py | 4 ++++ 4 files changed, 14 insertions(+), 6 deletions(-) diff --git a/tests/codec_be_on_mr_nonselection/test_param_file.py b/tests/codec_be_on_mr_nonselection/test_param_file.py index 047644e8c0..d2acacd39e 100644 --- a/tests/codec_be_on_mr_nonselection/test_param_file.py +++ b/tests/codec_be_on_mr_nonselection/test_param_file.py @@ -648,6 +648,9 @@ def run_test( if split_comparison and not sim_opts: split_idx = get_split_idx(str(Path(testv_file).stem), int(sampling_rate)) + # this extra if takes care of cases where no splits are found, e.g. the "NOOP" case in the self_test_ltv prm file + # if this would not be there, then the comparison of the whole file would run twice + if len(split_idx) > 0: output_differs_splits, reason_splits = cmp_pcm( ref_file, dut_output_file, diff --git a/tests/codec_be_on_mr_nonselection/test_sba.py b/tests/codec_be_on_mr_nonselection/test_sba.py index 17d340d484..563f8ac9ad 100644 --- a/tests/codec_be_on_mr_nonselection/test_sba.py +++ b/tests/codec_be_on_mr_nonselection/test_sba.py @@ -1357,6 +1357,9 @@ def sba_dec( input_file = f"{test_vector_path}/{tag}.wav" split_idx = get_split_idx(str(Path(input_file).stem), int(sampling_rate)) + # this extra if takes care of cases where no splits are found, e.g. the "NOOP" case in the self_test_ltv prm file + # if this would not be there, then the comparison of the whole file would run twice + if len(split_idx) > 0: output_differs_splits, reason_splits = cmp_pcm( ref_out_file, dut_out_file, @@ -1378,6 +1381,7 @@ def sba_dec( output_differs_parts += output_differs_splits reason_parts += reason_splits + if split_comparison: prop_suffix = ["_whole"] + [ f"_split{i:03d}" for i in range(1, len(split_idx) + 1) ] diff --git a/tests/conftest.py b/tests/conftest.py index 7f3788e209..5d3632a762 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1278,13 +1278,10 @@ def get_split_idx(input_file: str, sampling_rate_khz: int) -> Optional[np.ndarra format = "osba_hoa" else: format = input_file.split("_")[-1].lower() - idx = SPLIT_IDX.get(format) - - if idx is None: - pytest.fail( - f"Could not get split indices for file {input_file} with infered format {format}." - ) + idx = SPLIT_IDX.get(format, np.empty(0)) + # copy is important because we modify the array below for fs != 16 + # without copy, the constant would be modified and future split values would be wrong idx = idx.copy() if len(idx) > 0 and sampling_rate_khz != 16: diff --git a/tests/renderer/utils.py b/tests/renderer/utils.py index 82622fafe7..f6f10e6ea7 100644 --- a/tests/renderer/utils.py +++ b/tests/renderer/utils.py @@ -413,6 +413,9 @@ def run_renderer( if split_comparison: split_idx = get_split_idx(str(Path(in_file).stem), ref_fs // 1000) + # this extra if takes care of cases where no splits are found, e.g. the "NOOP" case in the self_test_ltv prm file + # if this would not be there, then the comparison of the whole file would run twice + if len(split_idx) > 0: output_differs_splits, reason_splits = cmp_pcm( out_file_ref, out_file, @@ -433,6 +436,7 @@ def run_renderer( output_differs_parts += output_differs_splits reason_parts += reason_splits + if split_comparison: prop_suffix = ["_whole"] + [ f"_split{i:03d}" for i in range(1, len(split_idx) + 1) ] -- GitLab From 978f5e327dcce5e77aa16ecf976d33cd858e4870 Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Mon, 7 Apr 2025 12:21:37 +0200 Subject: [PATCH 101/106] add split summary link --- ci/basop-pages/create_report_pages.py | 1 + 1 file changed, 1 insertion(+) diff --git a/ci/basop-pages/create_report_pages.py b/ci/basop-pages/create_report_pages.py index 2f01db7bde..41ee81af1b 100644 --- a/ci/basop-pages/create_report_pages.py +++ b/ci/basop-pages/create_report_pages.py @@ -37,6 +37,7 @@ Comparing:

Summary page

+

Split comparison summary page



-- GitLab From 068146d4f72fa7d96ef686fc0a754fbf39c24ee6 Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Mon, 7 Apr 2025 14:17:50 +0200 Subject: [PATCH 102/106] adapt diff_report.py to new csv format --- scripts/diff_report.py | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/scripts/diff_report.py b/scripts/diff_report.py index 5ab64e956f..0aefeaccdc 100644 --- a/scripts/diff_report.py +++ b/scripts/diff_report.py @@ -33,25 +33,29 @@ the United Nations Convention on Contracts on the International Sales of Goods. import pandas as pd import argparse import sys -import os -import pathlib COLUMNS_TO_COMPARE = [ "MLD", - "MAXIMUM ABS DIFF", + "MAX_ABS_DIFF", "MIN_SSNR", "MIN_ODG", ] + def main(args): - df_ref = pd.read_csv(args.csv_ref, sep=";") - df_test = pd.read_csv(args.csv_test, sep=";") + df_ref = pd.read_csv(args.csv_ref).sort_values( + by=["testcase", "format", "category"] + ) + df_test = pd.read_csv(args.csv_test).sort_values( + by=["testcase", "format", "category"] + ) for col in COLUMNS_TO_COMPARE: df_ref[col] = df_test[col] - df_ref[col] - df_ref.to_csv(args.csv_diff, index=False, sep=";") + df_ref.to_csv(args.csv_diff, index=False) return 0 + if __name__ == "__main__": parser = argparse.ArgumentParser() parser.add_argument("csv_ref") -- GitLab From 36e66ba8c261835f5be9305f8c69695d2f097aa1 Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Wed, 9 Apr 2025 16:02:38 +0200 Subject: [PATCH 103/106] no split-comp for SID cut cases --- tests/codec_be_on_mr_nonselection/test_sba.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/codec_be_on_mr_nonselection/test_sba.py b/tests/codec_be_on_mr_nonselection/test_sba.py index 563f8ac9ad..6b1746f858 100644 --- a/tests/codec_be_on_mr_nonselection/test_sba.py +++ b/tests/codec_be_on_mr_nonselection/test_sba.py @@ -1353,7 +1353,8 @@ def sba_dec( ) # 2. run comparison on split files if --split_comparison is given - if split_comparison: + # in the "cut until bitstream starts with SID" case, we can't do the split comp as the indices would not match anymore + if split_comparison and sid != 1: input_file = f"{test_vector_path}/{tag}.wav" split_idx = get_split_idx(str(Path(input_file).stem), int(sampling_rate)) -- GitLab From b8de8b4ba4b1a3fb039bd6a72eeb7dfdfaea4869 Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Wed, 9 Apr 2025 16:38:56 +0200 Subject: [PATCH 104/106] some tweaking for --no-bins --- scripts/create_histograms.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/create_histograms.py b/scripts/create_histograms.py index 5ec2f0e599..977af57777 100644 --- a/scripts/create_histograms.py +++ b/scripts/create_histograms.py @@ -41,7 +41,7 @@ COLORS_FOR_CATEGORIES = { def get_bins_for_diff(data: pd.Series): - return np.linspace(data.min(), data.max(), num=10) + return np.round(np.linspace(data.min(), data.max(), num=10), decimals=2) def create_histograms( @@ -102,7 +102,7 @@ def create_histograms( ax.set_title(fmt) ax.legend(loc="best") ax.set_xlabel(measure) - if "DIFF" in measure: + if "DIFF" in measure or len(bins_for_measures) == 0: ax.set_xticks(range(len(x)), x, rotation=35) else: ax.set_xticks(range(len(x)), x) -- GitLab From 321715be17b2452ba12554ec0b3b828bafc00ca8 Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Thu, 10 Apr 2025 14:04:35 +0200 Subject: [PATCH 105/106] fix regex for EVS format --- tests/test_26444.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_26444.py b/tests/test_26444.py index d3305e9984..25c72421f5 100644 --- a/tests/test_26444.py +++ b/tests/test_26444.py @@ -60,7 +60,7 @@ FORMATS_4_SCRIPTS = dict( ) ) -PATTERN_EVS_FORMAT = re.compile(r"-(" + r"|".join(FORMATS_4_SCRIPTS.keys()) + ")") +PATTERN_EVS_FORMAT = re.compile(r"(" + r"|".join(FORMATS_4_SCRIPTS.keys()) + ")") for s in SCRIPTS: with open(os.path.join(TEST_DIR, s), "r", encoding="UTF-8") as fp: -- GitLab From 852028809b3d269c3135d1cba6a8de253f996925 Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Thu, 10 Apr 2025 14:23:37 +0200 Subject: [PATCH 106/106] add workaround for category import not working sometimes --- scripts/create_histograms.py | 18 ++++++++++++++---- tests/constants.py | 2 ++ 2 files changed, 16 insertions(+), 4 deletions(-) diff --git a/scripts/create_histograms.py b/scripts/create_histograms.py index 977af57777..b2a9f0ec1c 100644 --- a/scripts/create_histograms.py +++ b/scripts/create_histograms.py @@ -26,10 +26,20 @@ BINS_FOR_MEASURES = { DEFAULT_MEASURES = ["MAX_ABS_DIFF", "MLD", "MIN_SSNR", "MIN_ODG"] -HERE = pathlib.Path(__file__).parent -ROOT_DIR = HERE.parent -sys.path.append(str(ROOT_DIR)) -from tests.constants import CAT_NORMAL, CAT_BITRATE_SWITCHING, CAT_DTX, CAT_JBM, CAT_PLC +### !!! Note: this is duplicated in tests/constatns.py. If you change this here, ALSO ADAPT IT THERE!!! +### (importing from there failed for unknown reasons in some jobs on some runners and I don't have time to properly investigate this...) +### below lines are the original solution, kept here for reference + +# HERE = pathlib.Path(__file__).parent +# ROOT_DIR = HERE.parent +# sys.path.append(str(ROOT_DIR)) +# from tests.constants import CAT_NORMAL, CAT_BITRATE_SWITCHING, CAT_DTX, CAT_JBM, CAT_PLC + +CAT_NORMAL = "normal operation" +CAT_DTX = "DTX" +CAT_PLC = "PLC" +CAT_BITRATE_SWITCHING = "bitrate switching" +CAT_JBM = "JBM" COLORS_FOR_CATEGORIES = { CAT_DTX: "tab:blue", diff --git a/tests/constants.py b/tests/constants.py index 60a9f6e846..d70d125194 100644 --- a/tests/constants.py +++ b/tests/constants.py @@ -66,6 +66,8 @@ ENC_AUX_FILES = [ ["vad_flag", np.int16, "fs/50"], ] +### !!! Note: this is duplicated in scripts/create_histogram.py. If you change this here, ALSO ADAPT IT THERE!!! +### (importing from here failed for unknown reasons in some jobs on some runners and I don't have time to properly investigate this...) CAT_NORMAL = "normal operation" CAT_DTX = "DTX" CAT_PLC = "PLC" -- GitLab