diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index e5ff5f1a7c72fb81ad235c0181e4f87111f00bc5..d850bf262a6c6ac2d9e48402c0551fc0eea8bfc2 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -228,7 +228,7 @@ stages: - if [ $CLANG_NUM -eq 1 ]; then sanitizer_type="MemorySanitizer"; elif [ $CLANG_NUM -eq 2 ]; then sanitizer_type="AddressSanitizer"; elif [ $CLANG_NUM -eq 3 ]; then sanitizer_type="UndefinedBehaviorSanitizer"; else echo "Wrong CLANG_NUM $CLANG_NUM given!"; exit 1; fi # run encoder and decoder with 20ms renderer framesize first, use reference creation mode - - python3 -m pytest tests/codec_be_on_mr_nonselection/test_param_file.py --param_file $SELF_TEST_PRM_FILE -v --update_ref 1 -m create_ref --html=report-20ms.html --self-contained-html --junit-xml=report-junit-20ms.xml --testcase_timeout=$testcase_timeout --ref_encoder_path ./IVAS_cod --ref_decoder_path ./IVAS_dec || exit_code20=$? + - python3 -m pytest tests/codec_be_on_mr_nonselection/test_param_file.py --param_file $SELF_TEST_PRM_FILE -v --update_ref 1 --html=report-20ms.html --self-contained-html --junit-xml=report-junit-20ms.xml --testcase_timeout=$testcase_timeout --ref_encoder_path ./IVAS_cod --ref_decoder_path ./IVAS_dec || exit_code20=$? # for 10ms and 5ms renderer framesize, we only need to run the decoder part as renderer framesize is a decoder-only option # set tolerance very high do ignore any BE differences due to the different renderer framesizes, those can appear due to the limiter being active # we are only interested in runtime errors from the sanitizers and ignore the diffs @@ -441,25 +441,22 @@ check-compatibility-with-basop-reference-branch: - python3 tests/create_short_testvectors.py - exit_code1=0 - exit_code2=0 - - python3 -m pytest tests/codec_be_on_mr_nonselection -v --update_ref 1 -m create_ref --html=report1.html --self-contained-html --junit-xml=report-junit1.xml || exit_code1=$? - - python3 -m pytest tests/codec_be_on_mr_nonselection -v --update_ref 1 -m create_ref_part2 --html=report2.html --self-contained-html --junit-xml=report-junit2.xml || exit_code2=$? - - zero_errors=$(cat report-junit1.xml report-junit2.xml | grep -c 'errors="0"') || true - - if [ $zero_errors != 2 ]; then echo "Run errors encountered!"; exit $EXIT_CODE_FAIL; fi + - python3 -m pytest tests/codec_be_on_mr_nonselection -v --update_ref 1 --html=report.html --self-contained-html --junit-xml=report-junit.xml || exit_code1=$? + - zero_errors=$(cat report-junit.xml | grep -c 'errors="0"') || true + - if [ $zero_errors != 1 ]; then echo "Run errors encountered!"; exit $EXIT_CODE_FAIL; fi - exit 0 artifacts: name: "check-float-reference--sha-$CI_COMMIT_SHORT_SHA--results" when: always expire_in: 1 month paths: - - report-junit1.xml - - report-junit2.xml - - report1.html - - report2.html + - report-junit.xml + - report.html + expose_as: "check-float-reference results" reports: junit: - - report-junit1.xml - - report-junit2.xml + - report-junit.xml # --------------------------------------------------------------- # Build jobs @@ -613,8 +610,7 @@ pytest-compare-20ms-and-5ms-rendering: - cp IVAS_cod IVAS_cod_ref - cp IVAS_dec IVAS_dec_ref # create references - - python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR -v --update_ref 1 -m create_ref - - python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR -v --update_ref 1 -m create_ref_part2 + - python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR -v --update_ref 1 ### run pytest - exit_code=0 - exit_code5=0 @@ -925,8 +921,7 @@ ivas-pytest-on-merge-request: ### prepare pytest # create references - - python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR -v --update_ref 1 -m create_ref - - python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR -v --update_ref 1 -m create_ref_part2 + - python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR -v --update_ref 1 ### Run test using branch scripts and input - if [ $ref_using_main == 1 ]; then git checkout $source_branch_commit_sha; fi @@ -983,12 +978,11 @@ ivas-interop-on-merge-request: - exit_code2=0 # set timeout for individual testcase runs to 60 seconds - testcase_timeout=60 - - python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR -v --html=report.html --self-contained-html --junit-xml=report-junit.xml --update_ref 1 -m create_ref --ref_encoder_path ./IVAS_cod --ref_decoder_path ./IVAS_dec_ref --testcase_timeout=$testcase_timeout || exit_code=$? - - python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR -v --html=report2.html --self-contained-html --junit-xml=report2-junit.xml --update_ref 1 -m create_ref_part2 --ref_encoder_path ./IVAS_cod --ref_decoder_path ./IVAS_dec_ref --testcase_timeout=$testcase_timeout || exit_code2=$? - - zero_failures=$(cat report-junit.xml report2-junit.xml | grep -c 'failures="0"') || true + - python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR -v --html=report.html --self-contained-html --junit-xml=report-junit.xml --update_ref 1 --ref_encoder_path ./IVAS_cod --ref_decoder_path ./IVAS_dec_ref --testcase_timeout=$testcase_timeout || exit_code=$? + - zero_failures=$(cat report-junit.xml | grep -c 'failures="0"') || true - - if [ $zero_failures != 2 ] && [ $non_interop_flag == 0 ]; then echo "Non-interop cases without non-interop flag encountered!"; exit $EXIT_CODE_FAIL; fi - - if [ $zero_failures != 2 ] && [ $non_interop_flag == 1 ]; then echo "Non-interop cases with non-interop flag encountered"; exit $EXIT_CODE_NON_BE; fi + - if [ $zero_failures != 1 ] && [ $non_interop_flag == 0 ]; then echo "Non-interop cases without non-interop flag encountered!"; exit $EXIT_CODE_FAIL; fi + - if [ $zero_failures != 1 ] && [ $non_interop_flag == 1 ]; then echo "Non-interop cases with non-interop flag encountered"; exit $EXIT_CODE_NON_BE; fi - exit 0 allow_failure: @@ -1001,8 +995,6 @@ ivas-interop-on-merge-request: paths: - report-junit.xml - report.html - - report2-junit.xml - - report2.html expose_as: "interop test results" reports: junit: @@ -1033,7 +1025,7 @@ evs-pytest-on-merge-request: ### prepare pytest # create references - testcase_timeout=60 - - python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR/test_param_file.py -v --update_ref 1 -m create_ref --param_file scripts/config/self_test_evs.prm --testcase_timeout=$testcase_timeout + - python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR/test_param_file.py -v --update_ref 1 --param_file scripts/config/self_test_evs.prm --testcase_timeout=$testcase_timeout ### Run test using branch scripts and input - if [ $ref_using_main == 1 ]; then git checkout $source_branch_commit_sha; fi @@ -1317,8 +1309,7 @@ codec-comparison-on-main-push: - mv IVAS_dec_test IVAS_dec # create references - testcase_timeout=60 - - python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR -v --update_ref 1 -m create_ref --testcase_timeout=$testcase_timeout - - python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR -v --update_ref 1 -m create_ref_part2 --testcase_timeout=$testcase_timeout + - python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR -v --update_ref 1 --testcase_timeout=$testcase_timeout ### re-checkout the latest commit here, if ref_using_main is set - if [ $ref_using_main -eq 1 ]; then git checkout $latest_commit;fi @@ -1397,8 +1388,7 @@ ivas-conformance: # Reference creation - python scripts/prepare_combined_format_inputs.py - $TEST_SET = "tests/codec_be_on_mr_nonselection", "tests/renderer/test_renderer.py", "tests/split_rendering/test_split_rendering.py" - - python -m pytest $TEST_SET -v -n auto --update_ref 1 -m create_ref --create_ref --keep_files - - python -m pytest $TEST_SET -v -n auto --update_ref 1 -m create_ref_part2 --keep_files + - python -m pytest $TEST_SET -v -n auto --update_ref 1 --create_ref --keep_files # Output creation - python -m pytest $TEST_SET -v -n auto --keep_files --create_cut --html=report_cmd.html --self-contained-html @@ -1490,8 +1480,7 @@ ivas-conformance-linux: # Reference creation - python3 scripts/prepare_combined_format_inputs.py - TEST_SET="tests/codec_be_on_mr_nonselection tests/renderer/test_renderer.py tests/split_rendering/test_split_rendering.py" - - python3 -m pytest $TEST_SET -v -n auto --update_ref 1 -m create_ref --create_ref --keep_files - - python3 -m pytest $TEST_SET -v -n auto --update_ref 1 -m create_ref_part2 --keep_files + - python3 -m pytest $TEST_SET -v -n auto --update_ref 1 --create_ref --keep_files # Output creation - python3 -m pytest $TEST_SET -v -n auto --keep_files --create_cut --html=report_cmd.html --self-contained-html @@ -1623,7 +1612,7 @@ test-long-self-test: # create references - exit_code_ref=0 - testcase_timeout=300 - - python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR/test_param_file.py -v --update_ref 1 -m create_ref --param_file scripts/config/self_test_ltv.prm --testcase_timeout=$testcase_timeout || exit_code_ref=$? + - python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR/test_param_file.py -v --update_ref 1 --param_file scripts/config/self_test_ltv.prm --testcase_timeout=$testcase_timeout || exit_code_ref=$? ### run pytest self-test using long test vectors - exit_code=0 @@ -2246,24 +2235,27 @@ coverage-test-on-main-scheduled: - *copy-ltv-files-to-testv-dir - make GCOV=1 -j - cp IVAS_rend IVAS_rend_ref # Copy exec to be able to run renderer script - - python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR -v -n auto --update_ref 1 -m create_ref --ref_encoder_path ./IVAS_cod --ref_decoder_path ./IVAS_dec - - python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR -v -n auto --update_ref 1 -m create_ref_part2 --ref_encoder_path ./IVAS_cod --ref_decoder_path ./IVAS_dec - - python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR -v -n auto --update_ref 1 -m create_ref --ref_encoder_path ./IVAS_cod --ref_decoder_path ./IVAS_dec --use_ltv - - python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR -v -n auto --update_ref 1 -m create_ref_part2 --ref_encoder_path ./IVAS_cod --ref_decoder_path ./IVAS_dec --use_ltv + + # -- Run short test vector suite to check coverage of that first + - python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR -v -n auto --update_ref 1 --ref_encoder_path ./IVAS_cod --ref_decoder_path ./IVAS_dec + # need to ignore non-zero exit codes as limiter is active and thus the different framesiszes will not be BE in all cases - python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR -v --dut_encoder_path ./IVAS_cod --dut_decoder_path ./IVAS_dec --dut_fr 5 --decoder_only || true - python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR -v --dut_encoder_path ./IVAS_cod --dut_decoder_path ./IVAS_dec --dut_fr 10 --decoder_only || true - - python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR -v --dut_encoder_path ./IVAS_cod --dut_decoder_path ./IVAS_dec --dut_fr 5 --decoder_only --use_ltv || true - - python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR -v --dut_encoder_path ./IVAS_cod --dut_decoder_path ./IVAS_dec --dut_fr 10 --decoder_only --use_ltv || true - python3 -m pytest -q -n auto tests/renderer/test_renderer.py --create_ref - python3 -m pytest -q -n auto tests/renderer/test_renderer.py --create_cut - lcov -c -d obj -o coverage_stv.info # extract coverage of short test vectors here # remove apps and lib_util files from coverage - lcov -r coverage_stv.info "*apps*" -o coverage_stv.info - lcov -r coverage_stv.info "*lib_util*" -o coverage_stv.info - - python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR/test_param_file.py -v -n auto --update_ref 1 -m create_ref --param_file scripts/config/self_test_evs.prm --ref_encoder_path ./IVAS_cod --ref_decoder_path ./IVAS_dec + + # -- Add extended coverage of EVS test and long test vectors + - python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR/test_param_file.py -v -n auto --update_ref 1 --param_file scripts/config/self_test_evs.prm --ref_encoder_path ./IVAS_cod --ref_decoder_path ./IVAS_dec - bash ci/smoke_test.sh coverage - - python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR/test_param_file.py -v -n auto --update_ref 1 -m create_ref --param_file scripts/config/self_test_ltv.prm --ref_encoder_path ./IVAS_cod --ref_decoder_path ./IVAS_dec + - python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR/test_param_file.py -v -n auto --update_ref 1 --param_file scripts/config/self_test_ltv.prm --ref_encoder_path ./IVAS_cod --ref_decoder_path ./IVAS_dec + - python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR/test_sba.py -v -n auto --update_ref 1 --ref_encoder_path ./IVAS_cod --ref_decoder_path ./IVAS_dec --use_ltv + - python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR/test_sba.py -v --dut_encoder_path ./IVAS_cod --dut_decoder_path ./IVAS_dec --dut_fr 5 --decoder_only --use_ltv || true + - python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR/test_sba.py -v --dut_encoder_path ./IVAS_cod --dut_decoder_path ./IVAS_dec --dut_fr 10 --decoder_only --use_ltv || true - python3 -m pytest tests/test_be_for_jbm_neutral_dly_profile.py - lcov -c -d obj -o coverage.info # remove apps and lib_util files from coverage diff --git a/tests/codec_be_on_mr_nonselection/test_param_file.py b/tests/codec_be_on_mr_nonselection/test_param_file.py index 61f40072cd3913d007c4dbb677f8f8c931f36498..1feae1de3991c2539e83441c8f732aa684ff73df 100644 --- a/tests/codec_be_on_mr_nonselection/test_param_file.py +++ b/tests/codec_be_on_mr_nonselection/test_param_file.py @@ -128,7 +128,6 @@ def convert_test_string_to_tag(test_string): return tag_str -@pytest.mark.create_ref @pytest.mark.parametrize("test_tag", list(param_file_test_dict.keys())) # hack to have stv/ltv/evs in the test name @pytest.mark.parametrize("param_file_id", [PARAM_FILE_ID]) diff --git a/tests/codec_be_on_mr_nonselection/test_sba.py b/tests/codec_be_on_mr_nonselection/test_sba.py index 66fe1f58910c700174735dbcfdee1c69d55d008e..b3041bba4f184e377f471569ac05917c24b01442 100644 --- a/tests/codec_be_on_mr_nonselection/test_sba.py +++ b/tests/codec_be_on_mr_nonselection/test_sba.py @@ -93,7 +93,6 @@ def check_and_makedir(dir_path): raise # raises the error again -@pytest.mark.create_ref @pytest.mark.parametrize("tag", tag_list) @pytest.mark.parametrize("fs", sample_rate_list) def test_pca_enc( @@ -185,7 +184,6 @@ def test_pca_enc( ) -@pytest.mark.create_ref @pytest.mark.parametrize("ivas_br", ivas_br_FOA) @pytest.mark.parametrize("dtx", dtx_set) @pytest.mark.parametrize("tag", tag_list) @@ -276,7 +274,6 @@ def test_sba_enc_system( gain_flag, keep_files, cut_gain=cut_gain, - create_dutenc=True, cut_testv=cut_testv, ) @@ -307,7 +304,6 @@ def test_sba_enc_system( ) -@pytest.mark.create_ref @pytest.mark.parametrize("ivas_br", ivas_br_HOA2) @pytest.mark.parametrize("tag", tag_list_HOA2) def test_spar_hoa2_enc_system( @@ -394,7 +390,6 @@ def test_spar_hoa2_enc_system( ) -@pytest.mark.create_ref @pytest.mark.parametrize("ivas_br", ivas_br_HOA3) @pytest.mark.parametrize("tag", tag_list_HOA3) def test_spar_hoa3_enc_system( @@ -481,7 +476,6 @@ def test_spar_hoa3_enc_system( ) -@pytest.mark.create_ref @pytest.mark.parametrize("ivas_br", ivas_br_FOA) @pytest.mark.parametrize("dtx", dtx_set) @pytest.mark.parametrize("tag", tag_list_bw_force) @@ -579,13 +573,6 @@ def test_sba_enc_BWforce_system( ) -# assumption: -# - the needed reference bitstreams are created by test_sba_enc_system -# -> reference bitstreams are not any longer created as part of this test -# -> the parameters of this test (except additional parameter plc_pattern) need to be a subset of the parameters in test_sba_enc_system -# -> the reference generation for this test (reference decoder output) needs to be done after completion of test_sba_enc_system -# -> therefore the marker create_ref_part2 -@pytest.mark.create_ref_part2 @pytest.mark.parametrize("ivas_br", ivas_br_plc) @pytest.mark.parametrize("dtx", dtx_set) @pytest.mark.parametrize("tag", tag_list) @@ -595,11 +582,14 @@ def test_sba_enc_BWforce_system( def test_sba_plc_system( record_property, props_to_record, + dut_encoder_frontend: EncoderFrontend, dut_decoder_frontend: DecoderFrontend, - ref_decoder_frontend, + test_vector_path, reference_path, dut_base_path, - ref_decoder_path, + ref_encoder_frontend, + ref_decoder_frontend, + br_switch_file_path, update_ref, keep_files, ivas_br, @@ -632,11 +622,45 @@ def test_sba_plc_system( pytest.skip() if "ltv" in tag: tag = f"ltv{fs}_FOA" + cut_testv = False elif "stv" in tag: tag = tag + fs + "c" + cut_testv = True else: assert 0 + # added enc call + max_bw = "FB" + sba_order = "+1" + if gain_flag == 1: + cut_gain = "16.0" + elif dtx == "1": + cut_gain = ".004" + else: + cut_gain = "1.0" + + sba_enc( + dut_encoder_frontend, + test_vector_path, + ref_encoder_frontend, + reference_path, + dut_base_path, + br_switch_file_path, + tag, + fs, + ivas_br, + dtx, + SID, + max_bw, + sba_order, + update_ref, + gain_flag, + keep_files, + cut_gain=cut_gain, + cut_testv=cut_testv, + plc_pattern=plc_pattern, + ) + # dec sba_dec( record_property, @@ -685,9 +709,9 @@ def sba_enc( gain_flag, keep_files=False, cut_gain="1.0", - create_dutenc=False, cut_testv=False, pca=False, + plc_pattern=None, ): # ------------ run cmd ------------ dut_out_dir = f"{dut_base_path}/sba_bs/pkt" @@ -707,28 +731,22 @@ def sba_enc( tag_out = f"{tag}_ivasbr{ivas_br[:-3]}k_DTX{dtx}" if ivas_br == "sw_24k4_256k.bin": ivas_br = f"{br_switch_file_path}/sw_24k4_256k.bin" - short_tag_ext = "" - if gain_flag != -1: - short_tag_ext += f"_Gain{gain_flag}" - if SID == 1: - short_tag_ext += "_SID" - if pca: - short_tag_ext += "_pca" + # to avoid conflicting names in case of parallel test execution, differentiate all cases - long_tag_ext = "" + tag_ext = "" + if pca: + tag_ext += "_pca" if gain_flag != -1: - long_tag_ext += f"_Gain{gain_flag}" + tag_ext += f"_Gain{gain_flag}" if SID == 1: - long_tag_ext += "_SID" - dut_pkt_file = f"{dut_out_dir}/{tag_out}{long_tag_ext}.192" - ref_pkt_file = f"{ref_out_dir}/{tag_out}{short_tag_ext}.192" - ref_pkt_file_dutenc = f"{ref_out_dir}/{tag_out}{short_tag_ext}_dutenc.192" + tag_ext += "_SID" + if plc_pattern is not None: + tag_ext += "_" + plc_pattern + dut_pkt_file = f"{dut_out_dir}/{tag_out}{tag_ext}.192" + ref_pkt_file = f"{ref_out_dir}/{tag_out}{tag_ext}.192" if SID == 1: - dut_pkt_file_cut = f"{dut_out_dir}/{tag_out}{long_tag_ext}_cut.192" - ref_pkt_file_cut = f"{ref_out_dir}/{tag_out}{short_tag_ext}_cut.192" - ref_pkt_file_dutenc_cut = ( - f"{ref_out_dir}/{tag_out}{short_tag_ext}_dutenc_cut.192" - ) + dut_pkt_file_cut = f"{dut_out_dir}/{tag_out}{tag_ext}_cut.192" + ref_pkt_file_cut = f"{ref_out_dir}/{tag_out}{tag_ext}_cut.192" input_path = f"{test_vector_path}/{tag_in}{in_extension}" dtx_mode = dtx == "1" @@ -742,7 +760,7 @@ def sba_enc( input_path = cut_file - if ref_encoder_frontend: + if update_ref == 1: # call REF encoder ref_encoder_frontend.run( ivas_br, @@ -754,18 +772,6 @@ def sba_enc( pca=pca, dtx_mode=dtx_mode, ) - if create_dutenc: - # for PLC decoder tests, create bitstream using DUT encoder - encoder_frontend.run( - ivas_br, - sampling_rate, - input_path, - ref_pkt_file_dutenc, - sba_order=sba_order, - max_band=ivas_max_bw, - pca=pca, - dtx_mode=dtx_mode, - ) if update_ref == 0: # call DUT encoder @@ -781,16 +787,12 @@ def sba_enc( ) if SID == 1: - if ref_encoder_frontend: + if update_ref == 1: with open(ref_pkt_file, "rb") as fp_in: with open(ref_pkt_file_cut, "wb") as fp_out: fr_cnt, cut_cnt = cut_from_start(fp_in, fp_out, 0, True) - with open(ref_pkt_file_dutenc, "rb") as fp_in: - with open(ref_pkt_file_dutenc_cut, "wb") as fp_out: - fr_cnt, cut_cnt = cut_from_start(fp_in, fp_out, 0, True) if not keep_files: os.remove(ref_pkt_file) - os.remove(ref_pkt_file_dutenc) if update_ref == 0: with open(dut_pkt_file, "rb") as fp_in: with open(dut_pkt_file_cut, "wb") as fp_out: @@ -833,48 +835,36 @@ def sba_dec( tag_out = f"{tag}_ivasbr{ivas_br[:-3]}k_DTX{dtx}" - short_tag_ext = "" - if gain_flag != -1: - short_tag_ext += f"_Gain{gain_flag}" - if pca: - short_tag_ext += "_pca" - if SID == 1: - short_tag_ext += "_SID_cut" # to avoid conflicting names in case of parallel test execution, differentiate all cases - long_tag_ext = "" + tag_ext = "" + if pca: + tag_ext += "_pca" if gain_flag != -1: - long_tag_ext += f"_Gain{gain_flag}" + tag_ext += f"_Gain{gain_flag}" if SID == 1: - long_tag_ext += "_SID_cut" + tag_ext += "_SID" + if plc_pattern is not None: + tag_ext += "_" + plc_pattern + if SID == 1: + tag_ext += "_cut" + dut_out_dir = f"{dut_base_path}/sba_bs/raw" ref_out_dir = f"{reference_path}/sba_bs/raw" - dut_in_pkt = f"{dut_base_path}/sba_bs/pkt/{tag_out}{long_tag_ext}.192" - ref_in_pkt = f"{reference_path}/sba_bs/pkt/{tag_out}{short_tag_ext}.192" - ref_in_pkt_dutenc = ( - f"{reference_path}/sba_bs/pkt/{tag_out}{short_tag_ext}_dutenc.192" - ) - if SID == 1: - if gain_flag != -1: - ref_in_pkt_dutenc = f"{reference_path}/sba_bs/pkt/{tag_out}_Gain{gain_flag}_SID_dutenc_cut.192" - else: - ref_in_pkt_dutenc = ( - f"{reference_path}/sba_bs/pkt/{tag_out}_SID_dutenc_cut.192" - ) + dut_in_pkt = f"{dut_base_path}/sba_bs/pkt/{tag_out}{tag_ext}.192" + ref_in_pkt = f"{reference_path}/sba_bs/pkt/{tag_out}{tag_ext}.192" plc_file = None if plc_pattern is not None: plc_file = f"{TESTV_DIR}/{plc_pattern}.g192" - long_tag_ext = f"{long_tag_ext}_{plc_pattern}" - short_tag_ext = f"{short_tag_ext}_{plc_pattern}" - dut_out_raw = f"{dut_out_dir}/{tag_out}{long_tag_ext}.wav" - ref_out_raw = f"{ref_out_dir}/{tag_out}{short_tag_ext}.wav" + dut_out_raw = f"{dut_out_dir}/{tag_out}{tag_ext}.wav" + ref_out_raw = f"{ref_out_dir}/{tag_out}{tag_ext}.wav" check_and_makedir(dut_out_dir) check_and_makedir(ref_out_dir) - if ref_decoder_frontend: + if update_ref == 1: # call REF decoder ref_decoder_frontend.run( output_config, @@ -886,7 +876,7 @@ def sba_dec( if update_ref == 0: if plc_file is not None: - dut_in_pkt = ref_in_pkt_dutenc + dut_in_pkt = dut_in_pkt elif decoder_only: dut_in_pkt = ref_in_pkt diff --git a/tests/codec_be_on_mr_selection/test_experiments.py b/tests/codec_be_on_mr_selection/test_experiments.py index 35152955e9cb771c2af774fbc5c99d7da9186fc6..71940c7850a1c2ee964cdd5cfa1a0a4106936143 100644 --- a/tests/codec_be_on_mr_selection/test_experiments.py +++ b/tests/codec_be_on_mr_selection/test_experiments.py @@ -40,7 +40,6 @@ from .constants import ( ) -@pytest.mark.create_ref @pytest.mark.parametrize("experiment,bitrate,dtx,fer,testset", P800_PARAMS_UNIFIED) @pytest.mark.parametrize("category", P800_CATEGORIES) def test_p800( @@ -71,7 +70,6 @@ def test_p800( ) -@pytest.mark.create_ref @pytest.mark.parametrize("experiment,bitrate,dtx,fer,testset", BS1534_PARAMS_UNIFIED) @pytest.mark.parametrize("input_file_num", range(1, 1 + BS1534_N_FILES)) def test_bs1534_no_masa( @@ -104,7 +102,7 @@ def test_bs1534_no_masa( ) -@pytest.mark.create_ref + @pytest.mark.parametrize( "experiment,bitrate,dtx,fer,testset,category,input_file_num", BS1534_MASA_PARAMS_UNIFIED, diff --git a/tests/conftest.py b/tests/conftest.py index 0e2fb9be874a1d0f6338d08b02c2e26443605718..06658dd7e195a1ca5686ad75f10902d70cf62dce 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -564,7 +564,7 @@ class DecoderFrontend: eid_command = [eid_path] eid_command.extend(["-fer", "-vbr", "-bs", "g192", "-ep", "g192"]) - eid_output_suffix = "." + os.path.basename(plc_file) + ".fer" + eid_output_suffix = ".fer" eid_command += [ str(input_bitstream_path), str(plc_file), @@ -824,13 +824,6 @@ def decoder_only(request) -> bool: def pytest_configure(config): config.addinivalue_line("markers", "serial: mark test to run only in serial") - config.addinivalue_line( - "markers", "create_ref: mark test capable of producing references" - ) - config.addinivalue_line( - "markers", - "create_ref_part2: reference creation test that depends on create_ref references", - ) if config.option.param_file: testconfig.PARAM_FILE = config.option.param_file if config.option.use_ltv: diff --git a/tests/prepare_pytests.py b/tests/prepare_pytests.py index 4cd6d071209127dc682fa1360d0199a67a2cfd67..fde34f8581eb310c9861e9d5aa1531915ab65ffd 100755 --- a/tests/prepare_pytests.py +++ b/tests/prepare_pytests.py @@ -128,9 +128,7 @@ def main(argv): DEFAULT_DECODER_DUT, ] - result = subprocess.run(base_cmd + ["-m", "create_ref"], check=False) - if not args.param_file: - result = subprocess.run(base_cmd + ["-m", "create_ref_part2"], check=False) + result = subprocess.run(base_cmd, check=False) return result.returncode diff --git a/tests/renderer/test_renderer.py b/tests/renderer/test_renderer.py index 31a125f9a81e2d56d61322151d0342638cf932f0..80c935c4f6525e15da1eb217b151f74c89189f82 100644 --- a/tests/renderer/test_renderer.py +++ b/tests/renderer/test_renderer.py @@ -62,7 +62,7 @@ from ..conftest import props_to_record """ Ambisonics """ -@pytest.mark.create_ref + @pytest.mark.parametrize("out_fmt", OUTPUT_FORMATS) @pytest.mark.parametrize("in_fmt", INPUT_FORMATS_AMBI) @pytest.mark.parametrize("frame_size", FRAMING_TO_TEST) @@ -93,7 +93,7 @@ def test_ambisonics( ) -@pytest.mark.create_ref + @pytest.mark.parametrize("out_fmt", OUTPUT_FORMATS_BINAURAL) @pytest.mark.parametrize("in_fmt", INPUT_FORMATS_AMBI) @pytest.mark.parametrize("frame_size", FRAMING_TO_TEST) @@ -124,7 +124,7 @@ def test_ambisonics_binaural_static( ) -@pytest.mark.create_ref + @pytest.mark.parametrize("trj_file", HR_TRAJECTORIES_TO_TEST) @pytest.mark.parametrize("out_fmt", OUTPUT_FORMATS_BINAURAL) @pytest.mark.parametrize("in_fmt", INPUT_FORMATS_AMBI) @@ -159,7 +159,7 @@ def test_ambisonics_binaural_headrotation( @pytest.mark.skip(reason="Not supported for BASOP code currently") -@pytest.mark.create_ref + @pytest.mark.parametrize("out_fmt", OUTPUT_FORMATS_BINAURAL[2:]) @pytest.mark.parametrize("in_fmt", INPUT_FORMATS_AMBI) @pytest.mark.parametrize("frame_size", FRAMING_TO_TEST) @@ -197,7 +197,7 @@ def test_dynamic_acoustic_environment( @pytest.mark.skip(reason="Not supported for BASOP code currently") -@pytest.mark.create_ref + @pytest.mark.parametrize("out_fmt", OUTPUT_FORMATS_BINAURAL[2:]) @pytest.mark.parametrize("in_fmt", INPUT_FORMATS_AMBI) @pytest.mark.parametrize("frame_size", FRAMING_TO_TEST) @@ -238,7 +238,7 @@ def test_dynamic_acoustic_environment_file( """ Multichannel """ -@pytest.mark.create_ref + @pytest.mark.parametrize("out_fmt", OUTPUT_FORMATS) @pytest.mark.parametrize("in_fmt", INPUT_FORMATS_MC) @pytest.mark.parametrize("frame_size", FRAMING_TO_TEST) @@ -269,7 +269,7 @@ def test_multichannel( ) -@pytest.mark.create_ref + @pytest.mark.parametrize("out_fmt", OUTPUT_FORMATS_BINAURAL) @pytest.mark.parametrize("in_fmt", INPUT_FORMATS_MC) @pytest.mark.parametrize("frame_size", FRAMING_TO_TEST) @@ -303,7 +303,7 @@ def test_multichannel_binaural_static( ) -@pytest.mark.create_ref + @pytest.mark.parametrize("trj_file", HR_TRAJECTORIES_TO_TEST) @pytest.mark.parametrize("out_fmt", OUTPUT_FORMATS_BINAURAL) @pytest.mark.parametrize("in_fmt", INPUT_FORMATS_MC) @@ -343,7 +343,7 @@ def test_multichannel_binaural_headrotation( """ ISM """ -@pytest.mark.create_ref + @pytest.mark.parametrize("out_fmt", OUTPUT_FORMATS) @pytest.mark.parametrize("in_fmt", INPUT_FORMATS_ISM) @pytest.mark.parametrize("frame_size", FRAMING_TO_TEST) @@ -375,7 +375,7 @@ def test_ism( ) -@pytest.mark.create_ref + @pytest.mark.parametrize("out_fmt", OUTPUT_FORMATS_BINAURAL) @pytest.mark.parametrize("in_fmt", INPUT_FORMATS_ISM) @pytest.mark.parametrize("frame_size", FRAMING_TO_TEST) @@ -412,7 +412,7 @@ def test_ism_binaural_static( ) -@pytest.mark.create_ref + @pytest.mark.parametrize("trj_file", HR_TRAJECTORIES_TO_TEST) @pytest.mark.parametrize("out_fmt", OUTPUT_FORMATS_BINAURAL) @pytest.mark.parametrize("in_fmt", INPUT_FORMATS_ISM) @@ -455,7 +455,7 @@ def test_ism_binaural_headrotation( """ MASA """ -@pytest.mark.create_ref + @pytest.mark.parametrize("out_fmt", OUTPUT_FORMATS) @pytest.mark.parametrize("in_fmt", INPUT_FORMATS_MASA) @pytest.mark.parametrize("frame_size", FRAMING_TO_TEST) @@ -487,7 +487,7 @@ def test_masa( ) -@pytest.mark.create_ref + @pytest.mark.parametrize("out_fmt", OUTPUT_FORMATS_BINAURAL) @pytest.mark.parametrize("in_fmt", INPUT_FORMATS_MASA) @pytest.mark.parametrize("frame_size", FRAMING_TO_TEST) @@ -522,7 +522,7 @@ def test_masa_binaural_static( ) -@pytest.mark.create_ref + @pytest.mark.parametrize("trj_file", HR_TRAJECTORIES_TO_TEST) @pytest.mark.parametrize("out_fmt", OUTPUT_FORMATS_BINAURAL) @pytest.mark.parametrize("in_fmt", INPUT_FORMATS_MASA) @@ -560,7 +560,7 @@ def test_masa_binaural_headrotation( ) -@pytest.mark.create_ref + @pytest.mark.parametrize("in_fmt", METADATA_SCENES_TO_TEST_MASA_PREREND) def test_masa_prerend( record_property, @@ -590,7 +590,7 @@ def test_masa_prerend( """ Custom loudspeaker layouts """ -@pytest.mark.create_ref + @pytest.mark.parametrize("out_fmt", OUTPUT_FORMATS) @pytest.mark.parametrize("in_layout", CUSTOM_LS_TO_TEST) @pytest.mark.parametrize("frame_size", FRAMING_TO_TEST) @@ -621,7 +621,7 @@ def test_custom_ls_input( ) -@pytest.mark.create_ref + @pytest.mark.parametrize("out_fmt", CUSTOM_LS_TO_TEST) @pytest.mark.parametrize("in_fmt", OUTPUT_FORMATS) def test_custom_ls_output( @@ -649,7 +649,7 @@ def test_custom_ls_output( ) -@pytest.mark.create_ref + @pytest.mark.parametrize("out_fmt", CUSTOM_LS_TO_TEST) @pytest.mark.parametrize("in_fmt", CUSTOM_LS_TO_TEST) def test_custom_ls_input_output( @@ -677,7 +677,7 @@ def test_custom_ls_input_output( ) -@pytest.mark.create_ref + @pytest.mark.parametrize("out_fmt", OUTPUT_FORMATS_BINAURAL) @pytest.mark.parametrize("in_layout", CUSTOM_LS_TO_TEST) @pytest.mark.parametrize("frame_size", FRAMING_TO_TEST) @@ -708,7 +708,7 @@ def test_custom_ls_input_binaural( ) -@pytest.mark.create_ref + @pytest.mark.parametrize("trj_file", HR_TRAJECTORIES_TO_TEST) @pytest.mark.parametrize("out_fmt", OUTPUT_FORMATS_BINAURAL) @pytest.mark.parametrize("in_layout", CUSTOM_LS_TO_TEST) @@ -745,7 +745,7 @@ def test_custom_ls_input_binaural_headrotation( """ Metadata / scene description input """ -@pytest.mark.create_ref + @pytest.mark.parametrize("out_fmt", OUTPUT_FORMATS) @pytest.mark.parametrize("in_fmt", METADATA_SCENES_TO_TEST) @pytest.mark.parametrize("frame_size", FRAMING_TO_TEST) @@ -780,7 +780,7 @@ def test_metadata( """ non diegetic pan """ -@pytest.mark.create_ref + @pytest.mark.parametrize("out_fmt", ["STEREO"]) @pytest.mark.parametrize("in_fmt", ["MONO"]) @pytest.mark.parametrize("non_diegetic_pan", ["0", "-30", "45", "90", "-90"]) @@ -811,7 +811,7 @@ def test_non_diegetic_pan_static( ) -@pytest.mark.create_ref + @pytest.mark.parametrize("out_fmt", ["STEREO"]) @pytest.mark.parametrize("in_fmt", ["ISM1"]) @pytest.mark.parametrize("non_diegetic_pan", ["0", "-30", "45", "90", "-90"]) diff --git a/tests/split_rendering/test_split_rendering.py b/tests/split_rendering/test_split_rendering.py index b038d591b0d40e8646de3c501164fbe8423a77eb..24360208ab79c981293496c256cb4341d57a5018 100644 --- a/tests/split_rendering/test_split_rendering.py +++ b/tests/split_rendering/test_split_rendering.py @@ -38,7 +38,7 @@ from tests.split_rendering.utils import * """ Ambisonics """ -@pytest.mark.create_ref + @pytest.mark.parametrize("trajectory", SPLIT_REND_HR_TRAJECTORIES_TO_TEST) @pytest.mark.parametrize("render_config", RENDERER_CONFIGS_TO_TEST_AMBI) @pytest.mark.parametrize("bitrate", IVAS_BITRATES_AMBI) @@ -60,7 +60,7 @@ def test_ambisonics_full_chain_split( ) -@pytest.mark.create_ref + @pytest.mark.parametrize("trajectory", SPLIT_REND_HR_TRAJECTORIES_TO_TEST) @pytest.mark.parametrize("render_config", RENDERER_CONFIGS_TO_TEST_AMBI) @pytest.mark.parametrize("in_fmt", INPUT_FORMATS_AMBI_SPLIT_REND) @@ -80,7 +80,7 @@ def test_ambisonics_external_split(test_info, in_fmt, render_config, trajectory) """ Multichannel """ -@pytest.mark.create_ref + @pytest.mark.parametrize("trajectory", SPLIT_REND_HR_TRAJECTORIES_TO_TEST) @pytest.mark.parametrize("render_config", RENDERER_CONFIGS_TO_TEST_MC) @pytest.mark.parametrize("bitrate", IVAS_BITRATES_MC) @@ -102,7 +102,7 @@ def test_multichannel_full_chain_split( ) -@pytest.mark.create_ref + @pytest.mark.parametrize("trajectory", SPLIT_REND_HR_TRAJECTORIES_TO_TEST) @pytest.mark.parametrize("render_config", RENDERER_CONFIGS_TO_TEST_MC) @pytest.mark.parametrize("in_fmt", INPUT_FORMATS_MC_SPLIT_REND) @@ -122,7 +122,7 @@ def test_multichannel_external_split(test_info, in_fmt, render_config, trajector """ ISM """ -@pytest.mark.create_ref + @pytest.mark.parametrize("trajectory", SPLIT_REND_HR_TRAJECTORIES_TO_TEST) @pytest.mark.parametrize("render_config", RENDERER_CONFIGS_TO_TEST_ISM) @pytest.mark.parametrize("bitrate", IVAS_BITRATES_ISM) @@ -142,7 +142,7 @@ def test_ism_full_chain_split(test_info, in_fmt, bitrate, render_config, traject ) -@pytest.mark.create_ref + @pytest.mark.parametrize("trajectory", SPLIT_REND_HR_TRAJECTORIES_TO_TEST) @pytest.mark.parametrize("render_config", RENDERER_CONFIGS_TO_TEST_ISM) @pytest.mark.parametrize("in_fmt", INPUT_FORMATS_ISM_SPLIT_REND) @@ -162,7 +162,7 @@ def test_ism_external_split(test_info, in_fmt, render_config, trajectory): """ MASA """ -@pytest.mark.create_ref + @pytest.mark.parametrize("trajectory", SPLIT_REND_HR_TRAJECTORIES_TO_TEST) @pytest.mark.parametrize("render_config", RENDERER_CONFIGS_TO_TEST_MASA) @pytest.mark.parametrize("bitrate", IVAS_BITRATES_MASA) @@ -182,7 +182,7 @@ def test_masa_full_chain_split(test_info, in_fmt, bitrate, render_config, trajec ) -@pytest.mark.create_ref + @pytest.mark.parametrize("trajectory", SPLIT_REND_HR_TRAJECTORIES_TO_TEST) @pytest.mark.parametrize("render_config", RENDERER_CONFIGS_TO_TEST_MASA) @pytest.mark.parametrize("in_fmt", INPUT_FORMATS_MASA_SPLIT_REND) @@ -202,7 +202,7 @@ def test_masa_external_split(test_info, in_fmt, render_config, trajectory): """ OMASA """ -@pytest.mark.create_ref + @pytest.mark.parametrize("trajectory", SPLIT_REND_HR_TRAJECTORIES_TO_TEST) @pytest.mark.parametrize("render_config", RENDERER_CONFIGS_TO_TEST_OMASA) @pytest.mark.parametrize("bitrate", IVAS_BITRATES_OMASA) @@ -225,7 +225,7 @@ def test_omasa_full_chain_split(test_info, in_fmt, bitrate, render_config, traje """ OSBA """ -@pytest.mark.create_ref + @pytest.mark.parametrize("trajectory", SPLIT_REND_HR_TRAJECTORIES_TO_TEST) @pytest.mark.parametrize("render_config", RENDERER_CONFIGS_TO_TEST_OSBA) @pytest.mark.parametrize("bitrate", IVAS_BITRATES_OSBA) @@ -248,7 +248,7 @@ def test_osba_full_chain_split(test_info, in_fmt, bitrate, render_config, trajec """ PLC """ -@pytest.mark.create_ref + @pytest.mark.parametrize("error_pattern", PLC_ERROR_PATTERNS) @pytest.mark.parametrize("trajectory", SPLIT_REND_HR_TRAJECTORIES_TO_TEST) @pytest.mark.parametrize("render_config", RENDERER_CONFIGS_TO_TEST_PLC) @@ -279,7 +279,7 @@ full_chain_split_pcm_params = [ ] -@pytest.mark.create_ref + @pytest.mark.parametrize("in_fmt,bitrate,render_config", full_chain_split_pcm_params) def test_full_chain_split_pcm(test_info, in_fmt, bitrate, render_config): trajectory = SPLIT_REND_HR_TRAJECTORIES_TO_TEST[0] @@ -305,7 +305,7 @@ external_split_pcm_params = [ ] -@pytest.mark.create_ref + @pytest.mark.parametrize("in_fmt,render_config", external_split_pcm_params) def test_external_split_pcm(test_info, in_fmt, render_config): trajectory = SPLIT_REND_HR_TRAJECTORIES_TO_TEST[0] @@ -321,7 +321,7 @@ def test_external_split_pcm(test_info, in_fmt, render_config): renderer_fmt="BINAURAL_SPLIT_PCM", ) -@pytest.mark.create_ref + @pytest.mark.parametrize("trajectory", SPLIT_REND_HR_TRAJECTORIES_TO_TEST) @pytest.mark.parametrize("render_config", RENDERER_CONFIGS_FRAMING) @pytest.mark.parametrize("in_fmt", ["5_1"]) @@ -340,7 +340,7 @@ def test_framing_combinations_external_split(test_info, in_fmt, render_config, t post_rend_fr=post_rend_fr, pre_rend_fr=pre_rend_fr, ) -@pytest.mark.create_ref + @pytest.mark.parametrize("trajectory", SPLIT_REND_HR_TRAJECTORIES_TO_TEST) @pytest.mark.parametrize("render_config", RENDERER_CONFIGS_FRAMING) @pytest.mark.parametrize("in_fmt", ["5_1"])