From e688d6eb0b6ce4377022bf84649086ccdf7c9396 Mon Sep 17 00:00:00 2001 From: Erik Norvell Date: Wed, 28 Aug 2024 07:44:47 +0200 Subject: [PATCH 1/8] Added encoder call in test_sba_plc_system --- tests/codec_be_on_mr_nonselection/test_sba.py | 41 ++++++++++++++++++- 1 file changed, 39 insertions(+), 2 deletions(-) diff --git a/tests/codec_be_on_mr_nonselection/test_sba.py b/tests/codec_be_on_mr_nonselection/test_sba.py index 66fe1f5891..99aecf291d 100644 --- a/tests/codec_be_on_mr_nonselection/test_sba.py +++ b/tests/codec_be_on_mr_nonselection/test_sba.py @@ -595,11 +595,14 @@ def test_sba_enc_BWforce_system( def test_sba_plc_system( record_property, props_to_record, + dut_encoder_frontend: EncoderFrontend, dut_decoder_frontend: DecoderFrontend, - ref_decoder_frontend, + test_vector_path, reference_path, dut_base_path, - ref_decoder_path, + ref_encoder_frontend, + ref_decoder_frontend, + br_switch_file_path, update_ref, keep_files, ivas_br, @@ -632,11 +635,45 @@ def test_sba_plc_system( pytest.skip() if "ltv" in tag: tag = f"ltv{fs}_FOA" + cut_testv = False elif "stv" in tag: tag = tag + fs + "c" + cut_testv = True else: assert 0 + # added enc call + max_bw = "FB" + sba_order = "+1" + if gain_flag == 1: + cut_gain = "16.0" + elif dtx == "1": + cut_gain = ".004" + else: + cut_gain = "1.0" + + sba_enc( + dut_encoder_frontend, + test_vector_path, + ref_encoder_frontend, + reference_path, + dut_base_path, + br_switch_file_path, + tag, + fs, + ivas_br, + dtx, + SID, + max_bw, + sba_order, + update_ref, + gain_flag, + keep_files, + cut_gain=cut_gain, + create_dutenc=True, + cut_testv=cut_testv, + ) + # dec sba_dec( record_property, -- GitLab From a8ee727e1391c1b15a4ed1fbf5a2f108c40f5288 Mon Sep 17 00:00:00 2001 From: Erik Norvell Date: Wed, 28 Aug 2024 08:01:05 +0200 Subject: [PATCH 2/8] Disable dutenc creation in test_sba_enc_system --- tests/codec_be_on_mr_nonselection/test_sba.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/codec_be_on_mr_nonselection/test_sba.py b/tests/codec_be_on_mr_nonselection/test_sba.py index 99aecf291d..36919d45af 100644 --- a/tests/codec_be_on_mr_nonselection/test_sba.py +++ b/tests/codec_be_on_mr_nonselection/test_sba.py @@ -276,7 +276,7 @@ def test_sba_enc_system( gain_flag, keep_files, cut_gain=cut_gain, - create_dutenc=True, + create_dutenc=False, cut_testv=cut_testv, ) -- GitLab From 253d2250c3f01b49cd91c5512880d7b452d09348 Mon Sep 17 00:00:00 2001 From: Erik Norvell Date: Thu, 29 Aug 2024 17:52:59 +0200 Subject: [PATCH 3/8] Move DuT encoding out of reference creation. Put fer pattern in BS name to avoid conflict during parallel processing --- tests/codec_be_on_mr_nonselection/test_sba.py | 22 ++++++++++++------- tests/conftest.py | 2 +- 2 files changed, 15 insertions(+), 9 deletions(-) diff --git a/tests/codec_be_on_mr_nonselection/test_sba.py b/tests/codec_be_on_mr_nonselection/test_sba.py index 36919d45af..caf76d4d57 100644 --- a/tests/codec_be_on_mr_nonselection/test_sba.py +++ b/tests/codec_be_on_mr_nonselection/test_sba.py @@ -670,8 +670,9 @@ def test_sba_plc_system( gain_flag, keep_files, cut_gain=cut_gain, - create_dutenc=True, + create_dutenc=False, cut_testv=cut_testv, + plc_pattern=plc_pattern, ) # dec @@ -725,6 +726,7 @@ def sba_enc( create_dutenc=False, cut_testv=False, pca=False, + plc_pattern=None, ): # ------------ run cmd ------------ dut_out_dir = f"{dut_base_path}/sba_bs/pkt" @@ -757,12 +759,14 @@ def sba_enc( long_tag_ext += f"_Gain{gain_flag}" if SID == 1: long_tag_ext += "_SID" + if plc_pattern is not None: + long_tag_ext += "_SID" + "_" + plc_pattern dut_pkt_file = f"{dut_out_dir}/{tag_out}{long_tag_ext}.192" - ref_pkt_file = f"{ref_out_dir}/{tag_out}{short_tag_ext}.192" + ref_pkt_file = f"{ref_out_dir}/{tag_out}{long_tag_ext}.192" ref_pkt_file_dutenc = f"{ref_out_dir}/{tag_out}{short_tag_ext}_dutenc.192" if SID == 1: dut_pkt_file_cut = f"{dut_out_dir}/{tag_out}{long_tag_ext}_cut.192" - ref_pkt_file_cut = f"{ref_out_dir}/{tag_out}{short_tag_ext}_cut.192" + ref_pkt_file_cut = f"{ref_out_dir}/{tag_out}{long_tag_ext}_cut.192" ref_pkt_file_dutenc_cut = ( f"{ref_out_dir}/{tag_out}{short_tag_ext}_dutenc_cut.192" ) @@ -883,11 +887,13 @@ def sba_dec( long_tag_ext += f"_Gain{gain_flag}" if SID == 1: long_tag_ext += "_SID_cut" + if plc_pattern is not None: + long_tag_ext += "_SID" + "_" + plc_pattern dut_out_dir = f"{dut_base_path}/sba_bs/raw" ref_out_dir = f"{reference_path}/sba_bs/raw" dut_in_pkt = f"{dut_base_path}/sba_bs/pkt/{tag_out}{long_tag_ext}.192" - ref_in_pkt = f"{reference_path}/sba_bs/pkt/{tag_out}{short_tag_ext}.192" + ref_in_pkt = f"{reference_path}/sba_bs/pkt/{tag_out}{long_tag_ext}.192" ref_in_pkt_dutenc = ( f"{reference_path}/sba_bs/pkt/{tag_out}{short_tag_ext}_dutenc.192" ) @@ -902,11 +908,11 @@ def sba_dec( plc_file = None if plc_pattern is not None: plc_file = f"{TESTV_DIR}/{plc_pattern}.g192" - long_tag_ext = f"{long_tag_ext}_{plc_pattern}" - short_tag_ext = f"{short_tag_ext}_{plc_pattern}" + # long_tag_ext = f"{long_tag_ext}_{plc_pattern}" + # short_tag_ext = f"{short_tag_ext}_{plc_pattern}" dut_out_raw = f"{dut_out_dir}/{tag_out}{long_tag_ext}.wav" - ref_out_raw = f"{ref_out_dir}/{tag_out}{short_tag_ext}.wav" + ref_out_raw = f"{ref_out_dir}/{tag_out}{long_tag_ext}.wav" check_and_makedir(dut_out_dir) check_and_makedir(ref_out_dir) @@ -923,7 +929,7 @@ def sba_dec( if update_ref == 0: if plc_file is not None: - dut_in_pkt = ref_in_pkt_dutenc + dut_in_pkt = dut_in_pkt elif decoder_only: dut_in_pkt = ref_in_pkt diff --git a/tests/conftest.py b/tests/conftest.py index 0e2fb9be87..977541e95d 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -564,7 +564,7 @@ class DecoderFrontend: eid_command = [eid_path] eid_command.extend(["-fer", "-vbr", "-bs", "g192", "-ep", "g192"]) - eid_output_suffix = "." + os.path.basename(plc_file) + ".fer" + eid_output_suffix = ".fer" eid_command += [ str(input_bitstream_path), str(plc_file), -- GitLab From f0f9db94078371017187d3c65703ac9ea5ac4e46 Mon Sep 17 00:00:00 2001 From: Erik Norvell Date: Fri, 30 Aug 2024 08:34:15 +0200 Subject: [PATCH 4/8] Fixes for SID+PLC cases --- tests/codec_be_on_mr_nonselection/test_sba.py | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/tests/codec_be_on_mr_nonselection/test_sba.py b/tests/codec_be_on_mr_nonselection/test_sba.py index caf76d4d57..af4d24f2b1 100644 --- a/tests/codec_be_on_mr_nonselection/test_sba.py +++ b/tests/codec_be_on_mr_nonselection/test_sba.py @@ -760,7 +760,7 @@ def sba_enc( if SID == 1: long_tag_ext += "_SID" if plc_pattern is not None: - long_tag_ext += "_SID" + "_" + plc_pattern + long_tag_ext += "_" + plc_pattern dut_pkt_file = f"{dut_out_dir}/{tag_out}{long_tag_ext}.192" ref_pkt_file = f"{ref_out_dir}/{tag_out}{long_tag_ext}.192" ref_pkt_file_dutenc = f"{ref_out_dir}/{tag_out}{short_tag_ext}_dutenc.192" @@ -826,12 +826,8 @@ def sba_enc( with open(ref_pkt_file, "rb") as fp_in: with open(ref_pkt_file_cut, "wb") as fp_out: fr_cnt, cut_cnt = cut_from_start(fp_in, fp_out, 0, True) - with open(ref_pkt_file_dutenc, "rb") as fp_in: - with open(ref_pkt_file_dutenc_cut, "wb") as fp_out: - fr_cnt, cut_cnt = cut_from_start(fp_in, fp_out, 0, True) if not keep_files: os.remove(ref_pkt_file) - os.remove(ref_pkt_file_dutenc) if update_ref == 0: with open(dut_pkt_file, "rb") as fp_in: with open(dut_pkt_file_cut, "wb") as fp_out: @@ -886,9 +882,12 @@ def sba_dec( if gain_flag != -1: long_tag_ext += f"_Gain{gain_flag}" if SID == 1: - long_tag_ext += "_SID_cut" + long_tag_ext += "_SID" if plc_pattern is not None: - long_tag_ext += "_SID" + "_" + plc_pattern + long_tag_ext += "_" + plc_pattern + if SID == 1: + long_tag_ext += "_cut" + dut_out_dir = f"{dut_base_path}/sba_bs/raw" ref_out_dir = f"{reference_path}/sba_bs/raw" -- GitLab From 45ba0940a920d9d4231eb90693995901ec09ff7e Mon Sep 17 00:00:00 2001 From: Erik Norvell Date: Fri, 30 Aug 2024 09:22:24 +0200 Subject: [PATCH 5/8] Remove marks create_ref and create_ref_part2 and correct coverage job --- .gitlab-ci.yml | 64 ++++++++----------- .../test_param_file.py | 1 - tests/codec_be_on_mr_nonselection/test_sba.py | 43 ------------- .../test_experiments.py | 4 +- tests/conftest.py | 7 -- tests/prepare_pytests.py | 4 +- tests/renderer/test_renderer.py | 46 ++++++------- tests/split_rendering/test_split_rendering.py | 30 ++++----- 8 files changed, 68 insertions(+), 131 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index e5ff5f1a7c..3403219dc7 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -228,7 +228,7 @@ stages: - if [ $CLANG_NUM -eq 1 ]; then sanitizer_type="MemorySanitizer"; elif [ $CLANG_NUM -eq 2 ]; then sanitizer_type="AddressSanitizer"; elif [ $CLANG_NUM -eq 3 ]; then sanitizer_type="UndefinedBehaviorSanitizer"; else echo "Wrong CLANG_NUM $CLANG_NUM given!"; exit 1; fi # run encoder and decoder with 20ms renderer framesize first, use reference creation mode - - python3 -m pytest tests/codec_be_on_mr_nonselection/test_param_file.py --param_file $SELF_TEST_PRM_FILE -v --update_ref 1 -m create_ref --html=report-20ms.html --self-contained-html --junit-xml=report-junit-20ms.xml --testcase_timeout=$testcase_timeout --ref_encoder_path ./IVAS_cod --ref_decoder_path ./IVAS_dec || exit_code20=$? + - python3 -m pytest tests/codec_be_on_mr_nonselection/test_param_file.py --param_file $SELF_TEST_PRM_FILE -v --update_ref 1 --html=report-20ms.html --self-contained-html --junit-xml=report-junit-20ms.xml --testcase_timeout=$testcase_timeout --ref_encoder_path ./IVAS_cod --ref_decoder_path ./IVAS_dec || exit_code20=$? # for 10ms and 5ms renderer framesize, we only need to run the decoder part as renderer framesize is a decoder-only option # set tolerance very high do ignore any BE differences due to the different renderer framesizes, those can appear due to the limiter being active # we are only interested in runtime errors from the sanitizers and ignore the diffs @@ -441,25 +441,22 @@ check-compatibility-with-basop-reference-branch: - python3 tests/create_short_testvectors.py - exit_code1=0 - exit_code2=0 - - python3 -m pytest tests/codec_be_on_mr_nonselection -v --update_ref 1 -m create_ref --html=report1.html --self-contained-html --junit-xml=report-junit1.xml || exit_code1=$? - - python3 -m pytest tests/codec_be_on_mr_nonselection -v --update_ref 1 -m create_ref_part2 --html=report2.html --self-contained-html --junit-xml=report-junit2.xml || exit_code2=$? - - zero_errors=$(cat report-junit1.xml report-junit2.xml | grep -c 'errors="0"') || true - - if [ $zero_errors != 2 ]; then echo "Run errors encountered!"; exit $EXIT_CODE_FAIL; fi + - python3 -m pytest tests/codec_be_on_mr_nonselection -v --update_ref 1 --html=report.html --self-contained-html --junit-xml=report-junit.xml || exit_code1=$? + - zero_errors=$(cat report-junit.xml | grep -c 'errors="0"') || true + - if [ $zero_errors != 1 ]; then echo "Run errors encountered!"; exit $EXIT_CODE_FAIL; fi - exit 0 artifacts: name: "check-float-reference--sha-$CI_COMMIT_SHORT_SHA--results" when: always expire_in: 1 month paths: - - report-junit1.xml - - report-junit2.xml - - report1.html - - report2.html + - report-junit.xml + - report.html + expose_as: "check-float-reference results" reports: junit: - - report-junit1.xml - - report-junit2.xml + - report-junit.xml # --------------------------------------------------------------- # Build jobs @@ -613,8 +610,7 @@ pytest-compare-20ms-and-5ms-rendering: - cp IVAS_cod IVAS_cod_ref - cp IVAS_dec IVAS_dec_ref # create references - - python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR -v --update_ref 1 -m create_ref - - python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR -v --update_ref 1 -m create_ref_part2 + - python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR -v --update_ref 1 ### run pytest - exit_code=0 - exit_code5=0 @@ -925,8 +921,7 @@ ivas-pytest-on-merge-request: ### prepare pytest # create references - - python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR -v --update_ref 1 -m create_ref - - python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR -v --update_ref 1 -m create_ref_part2 + - python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR -v --update_ref 1 ### Run test using branch scripts and input - if [ $ref_using_main == 1 ]; then git checkout $source_branch_commit_sha; fi @@ -983,9 +978,8 @@ ivas-interop-on-merge-request: - exit_code2=0 # set timeout for individual testcase runs to 60 seconds - testcase_timeout=60 - - python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR -v --html=report.html --self-contained-html --junit-xml=report-junit.xml --update_ref 1 -m create_ref --ref_encoder_path ./IVAS_cod --ref_decoder_path ./IVAS_dec_ref --testcase_timeout=$testcase_timeout || exit_code=$? - - python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR -v --html=report2.html --self-contained-html --junit-xml=report2-junit.xml --update_ref 1 -m create_ref_part2 --ref_encoder_path ./IVAS_cod --ref_decoder_path ./IVAS_dec_ref --testcase_timeout=$testcase_timeout || exit_code2=$? - - zero_failures=$(cat report-junit.xml report2-junit.xml | grep -c 'failures="0"') || true + - python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR -v --html=report.html --self-contained-html --junit-xml=report-junit.xml --update_ref 1 --ref_encoder_path ./IVAS_cod --ref_decoder_path ./IVAS_dec_ref --testcase_timeout=$testcase_timeout || exit_code=$? + - zero_failures=$(cat report-junit.xml | grep -c 'failures="0"') || true - if [ $zero_failures != 2 ] && [ $non_interop_flag == 0 ]; then echo "Non-interop cases without non-interop flag encountered!"; exit $EXIT_CODE_FAIL; fi - if [ $zero_failures != 2 ] && [ $non_interop_flag == 1 ]; then echo "Non-interop cases with non-interop flag encountered"; exit $EXIT_CODE_NON_BE; fi @@ -1001,8 +995,6 @@ ivas-interop-on-merge-request: paths: - report-junit.xml - report.html - - report2-junit.xml - - report2.html expose_as: "interop test results" reports: junit: @@ -1033,7 +1025,7 @@ evs-pytest-on-merge-request: ### prepare pytest # create references - testcase_timeout=60 - - python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR/test_param_file.py -v --update_ref 1 -m create_ref --param_file scripts/config/self_test_evs.prm --testcase_timeout=$testcase_timeout + - python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR/test_param_file.py -v --update_ref 1 --param_file scripts/config/self_test_evs.prm --testcase_timeout=$testcase_timeout ### Run test using branch scripts and input - if [ $ref_using_main == 1 ]; then git checkout $source_branch_commit_sha; fi @@ -1317,8 +1309,7 @@ codec-comparison-on-main-push: - mv IVAS_dec_test IVAS_dec # create references - testcase_timeout=60 - - python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR -v --update_ref 1 -m create_ref --testcase_timeout=$testcase_timeout - - python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR -v --update_ref 1 -m create_ref_part2 --testcase_timeout=$testcase_timeout + - python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR -v --update_ref 1 --testcase_timeout=$testcase_timeout ### re-checkout the latest commit here, if ref_using_main is set - if [ $ref_using_main -eq 1 ]; then git checkout $latest_commit;fi @@ -1397,8 +1388,7 @@ ivas-conformance: # Reference creation - python scripts/prepare_combined_format_inputs.py - $TEST_SET = "tests/codec_be_on_mr_nonselection", "tests/renderer/test_renderer.py", "tests/split_rendering/test_split_rendering.py" - - python -m pytest $TEST_SET -v -n auto --update_ref 1 -m create_ref --create_ref --keep_files - - python -m pytest $TEST_SET -v -n auto --update_ref 1 -m create_ref_part2 --keep_files + - python -m pytest $TEST_SET -v -n auto --update_ref 1 --create_ref --keep_files # Output creation - python -m pytest $TEST_SET -v -n auto --keep_files --create_cut --html=report_cmd.html --self-contained-html @@ -1490,8 +1480,7 @@ ivas-conformance-linux: # Reference creation - python3 scripts/prepare_combined_format_inputs.py - TEST_SET="tests/codec_be_on_mr_nonselection tests/renderer/test_renderer.py tests/split_rendering/test_split_rendering.py" - - python3 -m pytest $TEST_SET -v -n auto --update_ref 1 -m create_ref --create_ref --keep_files - - python3 -m pytest $TEST_SET -v -n auto --update_ref 1 -m create_ref_part2 --keep_files + - python3 -m pytest $TEST_SET -v -n auto --update_ref 1 --create_ref --keep_files # Output creation - python3 -m pytest $TEST_SET -v -n auto --keep_files --create_cut --html=report_cmd.html --self-contained-html @@ -1623,7 +1612,7 @@ test-long-self-test: # create references - exit_code_ref=0 - testcase_timeout=300 - - python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR/test_param_file.py -v --update_ref 1 -m create_ref --param_file scripts/config/self_test_ltv.prm --testcase_timeout=$testcase_timeout || exit_code_ref=$? + - python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR/test_param_file.py -v --update_ref 1 --param_file scripts/config/self_test_ltv.prm --testcase_timeout=$testcase_timeout || exit_code_ref=$? ### run pytest self-test using long test vectors - exit_code=0 @@ -2246,24 +2235,27 @@ coverage-test-on-main-scheduled: - *copy-ltv-files-to-testv-dir - make GCOV=1 -j - cp IVAS_rend IVAS_rend_ref # Copy exec to be able to run renderer script - - python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR -v -n auto --update_ref 1 -m create_ref --ref_encoder_path ./IVAS_cod --ref_decoder_path ./IVAS_dec - - python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR -v -n auto --update_ref 1 -m create_ref_part2 --ref_encoder_path ./IVAS_cod --ref_decoder_path ./IVAS_dec - - python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR -v -n auto --update_ref 1 -m create_ref --ref_encoder_path ./IVAS_cod --ref_decoder_path ./IVAS_dec --use_ltv - - python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR -v -n auto --update_ref 1 -m create_ref_part2 --ref_encoder_path ./IVAS_cod --ref_decoder_path ./IVAS_dec --use_ltv + + # -- Run short test vector suite to check coverage of that first + - python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR -v -n auto --update_ref 1 --ref_encoder_path ./IVAS_cod --ref_decoder_path ./IVAS_dec + # need to ignore non-zero exit codes as limiter is active and thus the different framesiszes will not be BE in all cases - python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR -v --dut_encoder_path ./IVAS_cod --dut_decoder_path ./IVAS_dec --dut_fr 5 --decoder_only || true - python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR -v --dut_encoder_path ./IVAS_cod --dut_decoder_path ./IVAS_dec --dut_fr 10 --decoder_only || true - - python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR -v --dut_encoder_path ./IVAS_cod --dut_decoder_path ./IVAS_dec --dut_fr 5 --decoder_only --use_ltv || true - - python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR -v --dut_encoder_path ./IVAS_cod --dut_decoder_path ./IVAS_dec --dut_fr 10 --decoder_only --use_ltv || true - python3 -m pytest -q -n auto tests/renderer/test_renderer.py --create_ref - python3 -m pytest -q -n auto tests/renderer/test_renderer.py --create_cut - lcov -c -d obj -o coverage_stv.info # extract coverage of short test vectors here # remove apps and lib_util files from coverage - lcov -r coverage_stv.info "*apps*" -o coverage_stv.info - lcov -r coverage_stv.info "*lib_util*" -o coverage_stv.info - - python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR/test_param_file.py -v -n auto --update_ref 1 -m create_ref --param_file scripts/config/self_test_evs.prm --ref_encoder_path ./IVAS_cod --ref_decoder_path ./IVAS_dec + + # -- Add extended coverage of EVS test and long test vectors + - python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR/test_param_file.py -v -n auto --update_ref 1 --param_file scripts/config/self_test_evs.prm --ref_encoder_path ./IVAS_cod --ref_decoder_path ./IVAS_dec - bash ci/smoke_test.sh coverage - - python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR/test_param_file.py -v -n auto --update_ref 1 -m create_ref --param_file scripts/config/self_test_ltv.prm --ref_encoder_path ./IVAS_cod --ref_decoder_path ./IVAS_dec + - python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR/test_param_file.py -v -n auto --update_ref 1 --param_file scripts/config/self_test_ltv.prm --ref_encoder_path ./IVAS_cod --ref_decoder_path ./IVAS_dec + - python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR/test_sba.py -v -n auto --update_ref 1 --ref_encoder_path ./IVAS_cod --ref_decoder_path ./IVAS_dec --use_ltv + - python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR/test_sba.py -v --dut_encoder_path ./IVAS_cod --dut_decoder_path ./IVAS_dec --dut_fr 5 --decoder_only --use_ltv || true + - python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR/test_sba.py -v --dut_encoder_path ./IVAS_cod --dut_decoder_path ./IVAS_dec --dut_fr 10 --decoder_only --use_ltv || true - python3 -m pytest tests/test_be_for_jbm_neutral_dly_profile.py - lcov -c -d obj -o coverage.info # remove apps and lib_util files from coverage diff --git a/tests/codec_be_on_mr_nonselection/test_param_file.py b/tests/codec_be_on_mr_nonselection/test_param_file.py index 61f40072cd..1feae1de39 100644 --- a/tests/codec_be_on_mr_nonselection/test_param_file.py +++ b/tests/codec_be_on_mr_nonselection/test_param_file.py @@ -128,7 +128,6 @@ def convert_test_string_to_tag(test_string): return tag_str -@pytest.mark.create_ref @pytest.mark.parametrize("test_tag", list(param_file_test_dict.keys())) # hack to have stv/ltv/evs in the test name @pytest.mark.parametrize("param_file_id", [PARAM_FILE_ID]) diff --git a/tests/codec_be_on_mr_nonselection/test_sba.py b/tests/codec_be_on_mr_nonselection/test_sba.py index af4d24f2b1..8532de30fa 100644 --- a/tests/codec_be_on_mr_nonselection/test_sba.py +++ b/tests/codec_be_on_mr_nonselection/test_sba.py @@ -93,7 +93,6 @@ def check_and_makedir(dir_path): raise # raises the error again -@pytest.mark.create_ref @pytest.mark.parametrize("tag", tag_list) @pytest.mark.parametrize("fs", sample_rate_list) def test_pca_enc( @@ -185,7 +184,6 @@ def test_pca_enc( ) -@pytest.mark.create_ref @pytest.mark.parametrize("ivas_br", ivas_br_FOA) @pytest.mark.parametrize("dtx", dtx_set) @pytest.mark.parametrize("tag", tag_list) @@ -276,7 +274,6 @@ def test_sba_enc_system( gain_flag, keep_files, cut_gain=cut_gain, - create_dutenc=False, cut_testv=cut_testv, ) @@ -307,7 +304,6 @@ def test_sba_enc_system( ) -@pytest.mark.create_ref @pytest.mark.parametrize("ivas_br", ivas_br_HOA2) @pytest.mark.parametrize("tag", tag_list_HOA2) def test_spar_hoa2_enc_system( @@ -394,7 +390,6 @@ def test_spar_hoa2_enc_system( ) -@pytest.mark.create_ref @pytest.mark.parametrize("ivas_br", ivas_br_HOA3) @pytest.mark.parametrize("tag", tag_list_HOA3) def test_spar_hoa3_enc_system( @@ -481,7 +476,6 @@ def test_spar_hoa3_enc_system( ) -@pytest.mark.create_ref @pytest.mark.parametrize("ivas_br", ivas_br_FOA) @pytest.mark.parametrize("dtx", dtx_set) @pytest.mark.parametrize("tag", tag_list_bw_force) @@ -579,13 +573,6 @@ def test_sba_enc_BWforce_system( ) -# assumption: -# - the needed reference bitstreams are created by test_sba_enc_system -# -> reference bitstreams are not any longer created as part of this test -# -> the parameters of this test (except additional parameter plc_pattern) need to be a subset of the parameters in test_sba_enc_system -# -> the reference generation for this test (reference decoder output) needs to be done after completion of test_sba_enc_system -# -> therefore the marker create_ref_part2 -@pytest.mark.create_ref_part2 @pytest.mark.parametrize("ivas_br", ivas_br_plc) @pytest.mark.parametrize("dtx", dtx_set) @pytest.mark.parametrize("tag", tag_list) @@ -670,7 +657,6 @@ def test_sba_plc_system( gain_flag, keep_files, cut_gain=cut_gain, - create_dutenc=False, cut_testv=cut_testv, plc_pattern=plc_pattern, ) @@ -723,7 +709,6 @@ def sba_enc( gain_flag, keep_files=False, cut_gain="1.0", - create_dutenc=False, cut_testv=False, pca=False, plc_pattern=None, @@ -763,13 +748,9 @@ def sba_enc( long_tag_ext += "_" + plc_pattern dut_pkt_file = f"{dut_out_dir}/{tag_out}{long_tag_ext}.192" ref_pkt_file = f"{ref_out_dir}/{tag_out}{long_tag_ext}.192" - ref_pkt_file_dutenc = f"{ref_out_dir}/{tag_out}{short_tag_ext}_dutenc.192" if SID == 1: dut_pkt_file_cut = f"{dut_out_dir}/{tag_out}{long_tag_ext}_cut.192" ref_pkt_file_cut = f"{ref_out_dir}/{tag_out}{long_tag_ext}_cut.192" - ref_pkt_file_dutenc_cut = ( - f"{ref_out_dir}/{tag_out}{short_tag_ext}_dutenc_cut.192" - ) input_path = f"{test_vector_path}/{tag_in}{in_extension}" dtx_mode = dtx == "1" @@ -795,18 +776,6 @@ def sba_enc( pca=pca, dtx_mode=dtx_mode, ) - if create_dutenc: - # for PLC decoder tests, create bitstream using DUT encoder - encoder_frontend.run( - ivas_br, - sampling_rate, - input_path, - ref_pkt_file_dutenc, - sba_order=sba_order, - max_band=ivas_max_bw, - pca=pca, - dtx_mode=dtx_mode, - ) if update_ref == 0: # call DUT encoder @@ -893,22 +862,10 @@ def sba_dec( dut_in_pkt = f"{dut_base_path}/sba_bs/pkt/{tag_out}{long_tag_ext}.192" ref_in_pkt = f"{reference_path}/sba_bs/pkt/{tag_out}{long_tag_ext}.192" - ref_in_pkt_dutenc = ( - f"{reference_path}/sba_bs/pkt/{tag_out}{short_tag_ext}_dutenc.192" - ) - if SID == 1: - if gain_flag != -1: - ref_in_pkt_dutenc = f"{reference_path}/sba_bs/pkt/{tag_out}_Gain{gain_flag}_SID_dutenc_cut.192" - else: - ref_in_pkt_dutenc = ( - f"{reference_path}/sba_bs/pkt/{tag_out}_SID_dutenc_cut.192" - ) plc_file = None if plc_pattern is not None: plc_file = f"{TESTV_DIR}/{plc_pattern}.g192" - # long_tag_ext = f"{long_tag_ext}_{plc_pattern}" - # short_tag_ext = f"{short_tag_ext}_{plc_pattern}" dut_out_raw = f"{dut_out_dir}/{tag_out}{long_tag_ext}.wav" ref_out_raw = f"{ref_out_dir}/{tag_out}{long_tag_ext}.wav" diff --git a/tests/codec_be_on_mr_selection/test_experiments.py b/tests/codec_be_on_mr_selection/test_experiments.py index 35152955e9..71940c7850 100644 --- a/tests/codec_be_on_mr_selection/test_experiments.py +++ b/tests/codec_be_on_mr_selection/test_experiments.py @@ -40,7 +40,6 @@ from .constants import ( ) -@pytest.mark.create_ref @pytest.mark.parametrize("experiment,bitrate,dtx,fer,testset", P800_PARAMS_UNIFIED) @pytest.mark.parametrize("category", P800_CATEGORIES) def test_p800( @@ -71,7 +70,6 @@ def test_p800( ) -@pytest.mark.create_ref @pytest.mark.parametrize("experiment,bitrate,dtx,fer,testset", BS1534_PARAMS_UNIFIED) @pytest.mark.parametrize("input_file_num", range(1, 1 + BS1534_N_FILES)) def test_bs1534_no_masa( @@ -104,7 +102,7 @@ def test_bs1534_no_masa( ) -@pytest.mark.create_ref + @pytest.mark.parametrize( "experiment,bitrate,dtx,fer,testset,category,input_file_num", BS1534_MASA_PARAMS_UNIFIED, diff --git a/tests/conftest.py b/tests/conftest.py index 977541e95d..06658dd7e1 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -824,13 +824,6 @@ def decoder_only(request) -> bool: def pytest_configure(config): config.addinivalue_line("markers", "serial: mark test to run only in serial") - config.addinivalue_line( - "markers", "create_ref: mark test capable of producing references" - ) - config.addinivalue_line( - "markers", - "create_ref_part2: reference creation test that depends on create_ref references", - ) if config.option.param_file: testconfig.PARAM_FILE = config.option.param_file if config.option.use_ltv: diff --git a/tests/prepare_pytests.py b/tests/prepare_pytests.py index 4cd6d07120..fde34f8581 100755 --- a/tests/prepare_pytests.py +++ b/tests/prepare_pytests.py @@ -128,9 +128,7 @@ def main(argv): DEFAULT_DECODER_DUT, ] - result = subprocess.run(base_cmd + ["-m", "create_ref"], check=False) - if not args.param_file: - result = subprocess.run(base_cmd + ["-m", "create_ref_part2"], check=False) + result = subprocess.run(base_cmd, check=False) return result.returncode diff --git a/tests/renderer/test_renderer.py b/tests/renderer/test_renderer.py index 31a125f9a8..80c935c4f6 100644 --- a/tests/renderer/test_renderer.py +++ b/tests/renderer/test_renderer.py @@ -62,7 +62,7 @@ from ..conftest import props_to_record """ Ambisonics """ -@pytest.mark.create_ref + @pytest.mark.parametrize("out_fmt", OUTPUT_FORMATS) @pytest.mark.parametrize("in_fmt", INPUT_FORMATS_AMBI) @pytest.mark.parametrize("frame_size", FRAMING_TO_TEST) @@ -93,7 +93,7 @@ def test_ambisonics( ) -@pytest.mark.create_ref + @pytest.mark.parametrize("out_fmt", OUTPUT_FORMATS_BINAURAL) @pytest.mark.parametrize("in_fmt", INPUT_FORMATS_AMBI) @pytest.mark.parametrize("frame_size", FRAMING_TO_TEST) @@ -124,7 +124,7 @@ def test_ambisonics_binaural_static( ) -@pytest.mark.create_ref + @pytest.mark.parametrize("trj_file", HR_TRAJECTORIES_TO_TEST) @pytest.mark.parametrize("out_fmt", OUTPUT_FORMATS_BINAURAL) @pytest.mark.parametrize("in_fmt", INPUT_FORMATS_AMBI) @@ -159,7 +159,7 @@ def test_ambisonics_binaural_headrotation( @pytest.mark.skip(reason="Not supported for BASOP code currently") -@pytest.mark.create_ref + @pytest.mark.parametrize("out_fmt", OUTPUT_FORMATS_BINAURAL[2:]) @pytest.mark.parametrize("in_fmt", INPUT_FORMATS_AMBI) @pytest.mark.parametrize("frame_size", FRAMING_TO_TEST) @@ -197,7 +197,7 @@ def test_dynamic_acoustic_environment( @pytest.mark.skip(reason="Not supported for BASOP code currently") -@pytest.mark.create_ref + @pytest.mark.parametrize("out_fmt", OUTPUT_FORMATS_BINAURAL[2:]) @pytest.mark.parametrize("in_fmt", INPUT_FORMATS_AMBI) @pytest.mark.parametrize("frame_size", FRAMING_TO_TEST) @@ -238,7 +238,7 @@ def test_dynamic_acoustic_environment_file( """ Multichannel """ -@pytest.mark.create_ref + @pytest.mark.parametrize("out_fmt", OUTPUT_FORMATS) @pytest.mark.parametrize("in_fmt", INPUT_FORMATS_MC) @pytest.mark.parametrize("frame_size", FRAMING_TO_TEST) @@ -269,7 +269,7 @@ def test_multichannel( ) -@pytest.mark.create_ref + @pytest.mark.parametrize("out_fmt", OUTPUT_FORMATS_BINAURAL) @pytest.mark.parametrize("in_fmt", INPUT_FORMATS_MC) @pytest.mark.parametrize("frame_size", FRAMING_TO_TEST) @@ -303,7 +303,7 @@ def test_multichannel_binaural_static( ) -@pytest.mark.create_ref + @pytest.mark.parametrize("trj_file", HR_TRAJECTORIES_TO_TEST) @pytest.mark.parametrize("out_fmt", OUTPUT_FORMATS_BINAURAL) @pytest.mark.parametrize("in_fmt", INPUT_FORMATS_MC) @@ -343,7 +343,7 @@ def test_multichannel_binaural_headrotation( """ ISM """ -@pytest.mark.create_ref + @pytest.mark.parametrize("out_fmt", OUTPUT_FORMATS) @pytest.mark.parametrize("in_fmt", INPUT_FORMATS_ISM) @pytest.mark.parametrize("frame_size", FRAMING_TO_TEST) @@ -375,7 +375,7 @@ def test_ism( ) -@pytest.mark.create_ref + @pytest.mark.parametrize("out_fmt", OUTPUT_FORMATS_BINAURAL) @pytest.mark.parametrize("in_fmt", INPUT_FORMATS_ISM) @pytest.mark.parametrize("frame_size", FRAMING_TO_TEST) @@ -412,7 +412,7 @@ def test_ism_binaural_static( ) -@pytest.mark.create_ref + @pytest.mark.parametrize("trj_file", HR_TRAJECTORIES_TO_TEST) @pytest.mark.parametrize("out_fmt", OUTPUT_FORMATS_BINAURAL) @pytest.mark.parametrize("in_fmt", INPUT_FORMATS_ISM) @@ -455,7 +455,7 @@ def test_ism_binaural_headrotation( """ MASA """ -@pytest.mark.create_ref + @pytest.mark.parametrize("out_fmt", OUTPUT_FORMATS) @pytest.mark.parametrize("in_fmt", INPUT_FORMATS_MASA) @pytest.mark.parametrize("frame_size", FRAMING_TO_TEST) @@ -487,7 +487,7 @@ def test_masa( ) -@pytest.mark.create_ref + @pytest.mark.parametrize("out_fmt", OUTPUT_FORMATS_BINAURAL) @pytest.mark.parametrize("in_fmt", INPUT_FORMATS_MASA) @pytest.mark.parametrize("frame_size", FRAMING_TO_TEST) @@ -522,7 +522,7 @@ def test_masa_binaural_static( ) -@pytest.mark.create_ref + @pytest.mark.parametrize("trj_file", HR_TRAJECTORIES_TO_TEST) @pytest.mark.parametrize("out_fmt", OUTPUT_FORMATS_BINAURAL) @pytest.mark.parametrize("in_fmt", INPUT_FORMATS_MASA) @@ -560,7 +560,7 @@ def test_masa_binaural_headrotation( ) -@pytest.mark.create_ref + @pytest.mark.parametrize("in_fmt", METADATA_SCENES_TO_TEST_MASA_PREREND) def test_masa_prerend( record_property, @@ -590,7 +590,7 @@ def test_masa_prerend( """ Custom loudspeaker layouts """ -@pytest.mark.create_ref + @pytest.mark.parametrize("out_fmt", OUTPUT_FORMATS) @pytest.mark.parametrize("in_layout", CUSTOM_LS_TO_TEST) @pytest.mark.parametrize("frame_size", FRAMING_TO_TEST) @@ -621,7 +621,7 @@ def test_custom_ls_input( ) -@pytest.mark.create_ref + @pytest.mark.parametrize("out_fmt", CUSTOM_LS_TO_TEST) @pytest.mark.parametrize("in_fmt", OUTPUT_FORMATS) def test_custom_ls_output( @@ -649,7 +649,7 @@ def test_custom_ls_output( ) -@pytest.mark.create_ref + @pytest.mark.parametrize("out_fmt", CUSTOM_LS_TO_TEST) @pytest.mark.parametrize("in_fmt", CUSTOM_LS_TO_TEST) def test_custom_ls_input_output( @@ -677,7 +677,7 @@ def test_custom_ls_input_output( ) -@pytest.mark.create_ref + @pytest.mark.parametrize("out_fmt", OUTPUT_FORMATS_BINAURAL) @pytest.mark.parametrize("in_layout", CUSTOM_LS_TO_TEST) @pytest.mark.parametrize("frame_size", FRAMING_TO_TEST) @@ -708,7 +708,7 @@ def test_custom_ls_input_binaural( ) -@pytest.mark.create_ref + @pytest.mark.parametrize("trj_file", HR_TRAJECTORIES_TO_TEST) @pytest.mark.parametrize("out_fmt", OUTPUT_FORMATS_BINAURAL) @pytest.mark.parametrize("in_layout", CUSTOM_LS_TO_TEST) @@ -745,7 +745,7 @@ def test_custom_ls_input_binaural_headrotation( """ Metadata / scene description input """ -@pytest.mark.create_ref + @pytest.mark.parametrize("out_fmt", OUTPUT_FORMATS) @pytest.mark.parametrize("in_fmt", METADATA_SCENES_TO_TEST) @pytest.mark.parametrize("frame_size", FRAMING_TO_TEST) @@ -780,7 +780,7 @@ def test_metadata( """ non diegetic pan """ -@pytest.mark.create_ref + @pytest.mark.parametrize("out_fmt", ["STEREO"]) @pytest.mark.parametrize("in_fmt", ["MONO"]) @pytest.mark.parametrize("non_diegetic_pan", ["0", "-30", "45", "90", "-90"]) @@ -811,7 +811,7 @@ def test_non_diegetic_pan_static( ) -@pytest.mark.create_ref + @pytest.mark.parametrize("out_fmt", ["STEREO"]) @pytest.mark.parametrize("in_fmt", ["ISM1"]) @pytest.mark.parametrize("non_diegetic_pan", ["0", "-30", "45", "90", "-90"]) diff --git a/tests/split_rendering/test_split_rendering.py b/tests/split_rendering/test_split_rendering.py index b038d591b0..24360208ab 100644 --- a/tests/split_rendering/test_split_rendering.py +++ b/tests/split_rendering/test_split_rendering.py @@ -38,7 +38,7 @@ from tests.split_rendering.utils import * """ Ambisonics """ -@pytest.mark.create_ref + @pytest.mark.parametrize("trajectory", SPLIT_REND_HR_TRAJECTORIES_TO_TEST) @pytest.mark.parametrize("render_config", RENDERER_CONFIGS_TO_TEST_AMBI) @pytest.mark.parametrize("bitrate", IVAS_BITRATES_AMBI) @@ -60,7 +60,7 @@ def test_ambisonics_full_chain_split( ) -@pytest.mark.create_ref + @pytest.mark.parametrize("trajectory", SPLIT_REND_HR_TRAJECTORIES_TO_TEST) @pytest.mark.parametrize("render_config", RENDERER_CONFIGS_TO_TEST_AMBI) @pytest.mark.parametrize("in_fmt", INPUT_FORMATS_AMBI_SPLIT_REND) @@ -80,7 +80,7 @@ def test_ambisonics_external_split(test_info, in_fmt, render_config, trajectory) """ Multichannel """ -@pytest.mark.create_ref + @pytest.mark.parametrize("trajectory", SPLIT_REND_HR_TRAJECTORIES_TO_TEST) @pytest.mark.parametrize("render_config", RENDERER_CONFIGS_TO_TEST_MC) @pytest.mark.parametrize("bitrate", IVAS_BITRATES_MC) @@ -102,7 +102,7 @@ def test_multichannel_full_chain_split( ) -@pytest.mark.create_ref + @pytest.mark.parametrize("trajectory", SPLIT_REND_HR_TRAJECTORIES_TO_TEST) @pytest.mark.parametrize("render_config", RENDERER_CONFIGS_TO_TEST_MC) @pytest.mark.parametrize("in_fmt", INPUT_FORMATS_MC_SPLIT_REND) @@ -122,7 +122,7 @@ def test_multichannel_external_split(test_info, in_fmt, render_config, trajector """ ISM """ -@pytest.mark.create_ref + @pytest.mark.parametrize("trajectory", SPLIT_REND_HR_TRAJECTORIES_TO_TEST) @pytest.mark.parametrize("render_config", RENDERER_CONFIGS_TO_TEST_ISM) @pytest.mark.parametrize("bitrate", IVAS_BITRATES_ISM) @@ -142,7 +142,7 @@ def test_ism_full_chain_split(test_info, in_fmt, bitrate, render_config, traject ) -@pytest.mark.create_ref + @pytest.mark.parametrize("trajectory", SPLIT_REND_HR_TRAJECTORIES_TO_TEST) @pytest.mark.parametrize("render_config", RENDERER_CONFIGS_TO_TEST_ISM) @pytest.mark.parametrize("in_fmt", INPUT_FORMATS_ISM_SPLIT_REND) @@ -162,7 +162,7 @@ def test_ism_external_split(test_info, in_fmt, render_config, trajectory): """ MASA """ -@pytest.mark.create_ref + @pytest.mark.parametrize("trajectory", SPLIT_REND_HR_TRAJECTORIES_TO_TEST) @pytest.mark.parametrize("render_config", RENDERER_CONFIGS_TO_TEST_MASA) @pytest.mark.parametrize("bitrate", IVAS_BITRATES_MASA) @@ -182,7 +182,7 @@ def test_masa_full_chain_split(test_info, in_fmt, bitrate, render_config, trajec ) -@pytest.mark.create_ref + @pytest.mark.parametrize("trajectory", SPLIT_REND_HR_TRAJECTORIES_TO_TEST) @pytest.mark.parametrize("render_config", RENDERER_CONFIGS_TO_TEST_MASA) @pytest.mark.parametrize("in_fmt", INPUT_FORMATS_MASA_SPLIT_REND) @@ -202,7 +202,7 @@ def test_masa_external_split(test_info, in_fmt, render_config, trajectory): """ OMASA """ -@pytest.mark.create_ref + @pytest.mark.parametrize("trajectory", SPLIT_REND_HR_TRAJECTORIES_TO_TEST) @pytest.mark.parametrize("render_config", RENDERER_CONFIGS_TO_TEST_OMASA) @pytest.mark.parametrize("bitrate", IVAS_BITRATES_OMASA) @@ -225,7 +225,7 @@ def test_omasa_full_chain_split(test_info, in_fmt, bitrate, render_config, traje """ OSBA """ -@pytest.mark.create_ref + @pytest.mark.parametrize("trajectory", SPLIT_REND_HR_TRAJECTORIES_TO_TEST) @pytest.mark.parametrize("render_config", RENDERER_CONFIGS_TO_TEST_OSBA) @pytest.mark.parametrize("bitrate", IVAS_BITRATES_OSBA) @@ -248,7 +248,7 @@ def test_osba_full_chain_split(test_info, in_fmt, bitrate, render_config, trajec """ PLC """ -@pytest.mark.create_ref + @pytest.mark.parametrize("error_pattern", PLC_ERROR_PATTERNS) @pytest.mark.parametrize("trajectory", SPLIT_REND_HR_TRAJECTORIES_TO_TEST) @pytest.mark.parametrize("render_config", RENDERER_CONFIGS_TO_TEST_PLC) @@ -279,7 +279,7 @@ full_chain_split_pcm_params = [ ] -@pytest.mark.create_ref + @pytest.mark.parametrize("in_fmt,bitrate,render_config", full_chain_split_pcm_params) def test_full_chain_split_pcm(test_info, in_fmt, bitrate, render_config): trajectory = SPLIT_REND_HR_TRAJECTORIES_TO_TEST[0] @@ -305,7 +305,7 @@ external_split_pcm_params = [ ] -@pytest.mark.create_ref + @pytest.mark.parametrize("in_fmt,render_config", external_split_pcm_params) def test_external_split_pcm(test_info, in_fmt, render_config): trajectory = SPLIT_REND_HR_TRAJECTORIES_TO_TEST[0] @@ -321,7 +321,7 @@ def test_external_split_pcm(test_info, in_fmt, render_config): renderer_fmt="BINAURAL_SPLIT_PCM", ) -@pytest.mark.create_ref + @pytest.mark.parametrize("trajectory", SPLIT_REND_HR_TRAJECTORIES_TO_TEST) @pytest.mark.parametrize("render_config", RENDERER_CONFIGS_FRAMING) @pytest.mark.parametrize("in_fmt", ["5_1"]) @@ -340,7 +340,7 @@ def test_framing_combinations_external_split(test_info, in_fmt, render_config, t post_rend_fr=post_rend_fr, pre_rend_fr=pre_rend_fr, ) -@pytest.mark.create_ref + @pytest.mark.parametrize("trajectory", SPLIT_REND_HR_TRAJECTORIES_TO_TEST) @pytest.mark.parametrize("render_config", RENDERER_CONFIGS_FRAMING) @pytest.mark.parametrize("in_fmt", ["5_1"]) -- GitLab From 124dbf2c45f08d278b736d3a6b81afda49b5e250 Mon Sep 17 00:00:00 2001 From: Erik Norvell Date: Fri, 30 Aug 2024 09:44:52 +0200 Subject: [PATCH 6/8] Fix in ivas-interop-on-merge-request since only one report is generated --- .gitlab-ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 3403219dc7..d850bf262a 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -981,8 +981,8 @@ ivas-interop-on-merge-request: - python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR -v --html=report.html --self-contained-html --junit-xml=report-junit.xml --update_ref 1 --ref_encoder_path ./IVAS_cod --ref_decoder_path ./IVAS_dec_ref --testcase_timeout=$testcase_timeout || exit_code=$? - zero_failures=$(cat report-junit.xml | grep -c 'failures="0"') || true - - if [ $zero_failures != 2 ] && [ $non_interop_flag == 0 ]; then echo "Non-interop cases without non-interop flag encountered!"; exit $EXIT_CODE_FAIL; fi - - if [ $zero_failures != 2 ] && [ $non_interop_flag == 1 ]; then echo "Non-interop cases with non-interop flag encountered"; exit $EXIT_CODE_NON_BE; fi + - if [ $zero_failures != 1 ] && [ $non_interop_flag == 0 ]; then echo "Non-interop cases without non-interop flag encountered!"; exit $EXIT_CODE_FAIL; fi + - if [ $zero_failures != 1 ] && [ $non_interop_flag == 1 ]; then echo "Non-interop cases with non-interop flag encountered"; exit $EXIT_CODE_NON_BE; fi - exit 0 allow_failure: -- GitLab From 064ebc4cd181bfc30d2c5ca1cbb533105034f6f2 Mon Sep 17 00:00:00 2001 From: Erik Norvell Date: Mon, 2 Sep 2024 08:19:18 +0200 Subject: [PATCH 7/8] Cleanup sba tag generation and add pca tag --- tests/codec_be_on_mr_nonselection/test_sba.py | 53 ++++++++----------- 1 file changed, 22 insertions(+), 31 deletions(-) diff --git a/tests/codec_be_on_mr_nonselection/test_sba.py b/tests/codec_be_on_mr_nonselection/test_sba.py index 8532de30fa..b9ce5d9699 100644 --- a/tests/codec_be_on_mr_nonselection/test_sba.py +++ b/tests/codec_be_on_mr_nonselection/test_sba.py @@ -731,26 +731,22 @@ def sba_enc( tag_out = f"{tag}_ivasbr{ivas_br[:-3]}k_DTX{dtx}" if ivas_br == "sw_24k4_256k.bin": ivas_br = f"{br_switch_file_path}/sw_24k4_256k.bin" - short_tag_ext = "" - if gain_flag != -1: - short_tag_ext += f"_Gain{gain_flag}" - if SID == 1: - short_tag_ext += "_SID" - if pca: - short_tag_ext += "_pca" + # to avoid conflicting names in case of parallel test execution, differentiate all cases - long_tag_ext = "" + tag_ext = "" + if pca: + tag_ext += "_pca" if gain_flag != -1: - long_tag_ext += f"_Gain{gain_flag}" + tag_ext += f"_Gain{gain_flag}" if SID == 1: - long_tag_ext += "_SID" + tag_ext += "_SID" if plc_pattern is not None: - long_tag_ext += "_" + plc_pattern - dut_pkt_file = f"{dut_out_dir}/{tag_out}{long_tag_ext}.192" - ref_pkt_file = f"{ref_out_dir}/{tag_out}{long_tag_ext}.192" + tag_ext += "_" + plc_pattern + dut_pkt_file = f"{dut_out_dir}/{tag_out}{tag_ext}.192" + ref_pkt_file = f"{ref_out_dir}/{tag_out}{tag_ext}.192" if SID == 1: - dut_pkt_file_cut = f"{dut_out_dir}/{tag_out}{long_tag_ext}_cut.192" - ref_pkt_file_cut = f"{ref_out_dir}/{tag_out}{long_tag_ext}_cut.192" + dut_pkt_file_cut = f"{dut_out_dir}/{tag_out}{tag_ext}_cut.192" + ref_pkt_file_cut = f"{ref_out_dir}/{tag_out}{tag_ext}_cut.192" input_path = f"{test_vector_path}/{tag_in}{in_extension}" dtx_mode = dtx == "1" @@ -839,36 +835,31 @@ def sba_dec( tag_out = f"{tag}_ivasbr{ivas_br[:-3]}k_DTX{dtx}" - short_tag_ext = "" - if gain_flag != -1: - short_tag_ext += f"_Gain{gain_flag}" - if pca: - short_tag_ext += "_pca" - if SID == 1: - short_tag_ext += "_SID_cut" # to avoid conflicting names in case of parallel test execution, differentiate all cases - long_tag_ext = "" + tag_ext = "" + if pca: + tag_ext += "_pca" if gain_flag != -1: - long_tag_ext += f"_Gain{gain_flag}" + tag_ext += f"_Gain{gain_flag}" if SID == 1: - long_tag_ext += "_SID" + tag_ext += "_SID" if plc_pattern is not None: - long_tag_ext += "_" + plc_pattern + tag_ext += "_" + plc_pattern if SID == 1: - long_tag_ext += "_cut" + tag_ext += "_cut" dut_out_dir = f"{dut_base_path}/sba_bs/raw" ref_out_dir = f"{reference_path}/sba_bs/raw" - dut_in_pkt = f"{dut_base_path}/sba_bs/pkt/{tag_out}{long_tag_ext}.192" - ref_in_pkt = f"{reference_path}/sba_bs/pkt/{tag_out}{long_tag_ext}.192" + dut_in_pkt = f"{dut_base_path}/sba_bs/pkt/{tag_out}{tag_ext}.192" + ref_in_pkt = f"{reference_path}/sba_bs/pkt/{tag_out}{tag_ext}.192" plc_file = None if plc_pattern is not None: plc_file = f"{TESTV_DIR}/{plc_pattern}.g192" - dut_out_raw = f"{dut_out_dir}/{tag_out}{long_tag_ext}.wav" - ref_out_raw = f"{ref_out_dir}/{tag_out}{long_tag_ext}.wav" + dut_out_raw = f"{dut_out_dir}/{tag_out}{tag_ext}.wav" + ref_out_raw = f"{ref_out_dir}/{tag_out}{tag_ext}.wav" check_and_makedir(dut_out_dir) check_and_makedir(ref_out_dir) -- GitLab From a466424afd5aec11fc497bb11ad7b816e3bf5d39 Mon Sep 17 00:00:00 2001 From: Erik Norvell Date: Mon, 2 Sep 2024 09:54:09 +0200 Subject: [PATCH 8/8] Call ref encoder only if update_ref == 1 --- tests/codec_be_on_mr_nonselection/test_sba.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/codec_be_on_mr_nonselection/test_sba.py b/tests/codec_be_on_mr_nonselection/test_sba.py index b9ce5d9699..b3041bba4f 100644 --- a/tests/codec_be_on_mr_nonselection/test_sba.py +++ b/tests/codec_be_on_mr_nonselection/test_sba.py @@ -760,7 +760,7 @@ def sba_enc( input_path = cut_file - if ref_encoder_frontend: + if update_ref == 1: # call REF encoder ref_encoder_frontend.run( ivas_br, @@ -787,7 +787,7 @@ def sba_enc( ) if SID == 1: - if ref_encoder_frontend: + if update_ref == 1: with open(ref_pkt_file, "rb") as fp_in: with open(ref_pkt_file_cut, "wb") as fp_out: fr_cnt, cut_cnt = cut_from_start(fp_in, fp_out, 0, True) @@ -864,7 +864,7 @@ def sba_dec( check_and_makedir(dut_out_dir) check_and_makedir(ref_out_dir) - if ref_decoder_frontend: + if update_ref == 1: # call REF decoder ref_decoder_frontend.run( output_config, -- GitLab