Commit e214f0b0 authored by Jan Kiene's avatar Jan Kiene
Browse files

Revert "remove all -v 's from pytest calls" 

This reverts commit cda08e34.
parent cda08e34

main-basop.yml

+19 −19
Original line number Diff line number Diff line
@@ -487,7 +487,7 @@ workflow: 
    - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/basop/build-binaries.sh float-ref-merge-source



    - exit_code_target=0

    - python3 -m pytest $TEST_SUITE--update_ref 1 --create_ref -n auto --ref_encoder_path $MERGE_TARGET_ENCODER_PATH --ref_decoder_path $MERGE_TARGET_DECODER_PATH > $PYTEST_LOG_TARGET_BRANCH || exit_code_target=$?

    - python3 -m pytest $TEST_SUITE -v --update_ref 1 --create_ref -n auto --ref_encoder_path $MERGE_TARGET_ENCODER_PATH --ref_decoder_path $MERGE_TARGET_DECODER_PATH > $PYTEST_LOG_TARGET_BRANCH || exit_code_target=$?



    - exit_code=0

    - rm -rf .pytest_cache || true
@@ -625,7 +625,7 @@ workflow: 


    ### ----- run pytest for branch (dut) first -----

    # create float reference outputs with corresponding branch

    - python3 -m pytest $TEST_SUITE--update_ref 1 $enc_stats_arg $enc_dmx_arg --create_ref -n auto --ref_encoder_path $MERGE_SOURCE_FLOAT_REF_ENCODER_PATH --ref_decoder_path $MERGE_SOURCE_FLOAT_REF_DECODER_PATH || exit_code=$?

    - python3 -m pytest $TEST_SUITE -v --update_ref 1 $enc_stats_arg $enc_dmx_arg --create_ref -n auto --ref_encoder_path $MERGE_SOURCE_FLOAT_REF_ENCODER_PATH --ref_decoder_path $MERGE_SOURCE_FLOAT_REF_DECODER_PATH || exit_code=$?

    - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/basop/overwrite-pytest-cache-with-artifact.sh

    # create dut outputs

    - python3 -m pytest --tb=no -q $TEST_SUITE --keep_files --create_cut --html=$HTML_REPORT_BRANCH --self-contained-html --junit-xml=$XML_REPORT_BRANCH $comp_args --dut_encoder_path $DUT_ENCODER_PATH --dut_decoder_path $DUT_DECODER_PATH -n auto --testcase_timeout $testcase_timeout || true
@@ -640,7 +640,7 @@ workflow: 


    ### ----- run pytest for merge target now -----

    # create float reference outputs with corresponding branch

    - python3 -m pytest $TEST_SUITE--update_ref 1 $enc_stats_arg $enc_dmx_arg --create_ref -n auto --ref_encoder_path $REF_ENCODER_PATH --ref_decoder_path $REF_DECODER_PATH || exit_code=$?

    - python3 -m pytest $TEST_SUITE -v --update_ref 1 $enc_stats_arg $enc_dmx_arg --create_ref -n auto --ref_encoder_path $REF_ENCODER_PATH --ref_decoder_path $REF_DECODER_PATH || exit_code=$?

    - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/basop/overwrite-pytest-cache-with-artifact.sh

    # create merge-target outputs

    - python3 -m pytest --tb=no -q $TEST_SUITE --keep_files --create_cut --html=$HTML_REPORT_MAIN --self-contained-html --junit-xml=$XML_REPORT_MAIN $comp_args --dut_encoder_path $MERGE_TARGET_ENCODER_PATH --dut_decoder_path $MERGE_TARGET_DECODER_PATH -n auto --testcase_timeout $testcase_timeout || true
@@ -742,8 +742,8 @@ workflow: 


    ### run pytest

    - exit_code=0

    - python3 -m pytest --tb=no $TEST_SUITE--create_cut --html=report-ref.html --self-contained-html --junit-xml=report-junit-ref.xml $comp_args -n auto --testcase_timeout $testcase_timeout --dut_encoder_path $REF_ENCODER_PATH --dut_decoder_path $REF_DECODER_PATH  --compare_to_input || exit_code=$?

    - python3 -m pytest --tb=no $TEST_SUITE--create_cut --html=report-dut.html --self-contained-html --junit-xml=report-junit-dut.xml $comp_args -n auto --testcase_timeout $testcase_timeout --dut_encoder_path $DUT_ENCODER_PATH --dut_decoder_path $DUT_DECODER_PATH --compare_to_input || exit_code=$?

    - python3 -m pytest --tb=no $TEST_SUITE -v --create_cut --html=report-ref.html --self-contained-html --junit-xml=report-junit-ref.xml $comp_args -n auto --testcase_timeout $testcase_timeout --dut_encoder_path $REF_ENCODER_PATH --dut_decoder_path $REF_DECODER_PATH  --compare_to_input || exit_code=$?

    - python3 -m pytest --tb=no $TEST_SUITE -v --create_cut --html=report-dut.html --self-contained-html --junit-xml=report-junit-dut.xml $comp_args -n auto --testcase_timeout $testcase_timeout --dut_encoder_path $DUT_ENCODER_PATH --dut_decoder_path $DUT_DECODER_PATH --compare_to_input || exit_code=$?

    - zero_errors_ref=$(cat report-junit-ref.xml | grep -c 'errors="0"') || true

    - zero_errors_dut=$(cat report-junit-dut.xml | grep -c 'errors="0"') || true

    - python3 scripts/parse_xml_report.py report-junit-ref.xml report-ref.csv
@@ -1194,7 +1194,7 @@ ivas-pytest-on-merge-request: 


    ### prepare pytest

    # create references

    - python3 -m pytest "$TESTS_DIR_CODEC_BE_ON_MR"/"$PYTEST_SCRIPT"--update_ref 1

    - python3 -m pytest "$TESTS_DIR_CODEC_BE_ON_MR"/"$PYTEST_SCRIPT" -v --update_ref 1



    ### Run test using branch scripts and input

    - if [ $ref_using_target == 1 ]; then git checkout $source_branch_commit_sha; fi
@@ -1202,7 +1202,7 @@ ivas-pytest-on-merge-request: 
    ### run pytest

    - exit_code=0

    - testcase_timeout=600

    - python3 -m pytest "$TESTS_DIR_CODEC_BE_ON_MR"/"$PYTEST_SCRIPT"--mld --ssnr --odg --html=$REPORT_HTML --self-contained-html --junit-xml=$REPORT_XML --testcase_timeout=$testcase_timeout || exit_code=$?

    - python3 -m pytest "$TESTS_DIR_CODEC_BE_ON_MR"/"$PYTEST_SCRIPT" -v --mld --ssnr --odg --html=$REPORT_HTML --self-contained-html --junit-xml=$REPORT_XML --testcase_timeout=$testcase_timeout || exit_code=$?

    - zero_errors=$(cat $REPORT_XML | grep -c 'errors="0"') || true



    - python3 scripts/parse_xml_report.py $REPORT_XML $REPORT_CSV
@@ -1372,7 +1372,7 @@ split-rendering-pytest-on-merge-request: 
    - exit_code2=0

    # set timeout for individual testcase runs to 60 seconds

    - testcase_timeout=60

    - python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR--html=report.html --self-contained-html --junit-xml=report-junit.xml --update_ref 1 --ref_encoder_path ./IVAS_cod --ref_decoder_path ./IVAS_dec_ref --testcase_timeout=$testcase_timeout || exit_code=$?

    - python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR -v --html=report.html --self-contained-html --junit-xml=report-junit.xml --update_ref 1 --ref_encoder_path ./IVAS_cod --ref_decoder_path ./IVAS_dec_ref --testcase_timeout=$testcase_timeout || exit_code=$?

    - zero_failures=$(cat report-junit.xml | grep -c 'failures="0"') || true



    - if [ $zero_failures != 1 ] && [ $non_interop_flag == 0 ]; then echo "Non-interop cases without non-interop flag encountered!"; exit $EXIT_CODE_FAIL; fi
@@ -1617,14 +1617,14 @@ renderer-smoke-test: 
    - make -j $make_args



    # run encoder and decoder with 20ms renderer framesize first, use reference creation mode

    - python3 -m pytest tests/codec_be_on_mr_nonselection $USE_LTV --param_file $SELF_TEST_PRM_FILE--update_ref 1 --html=report-20ms.html --self-contained-html --junit-xml=report-junit-20ms.xml --testcase_timeout=$testcase_timeout --ref_encoder_path ./IVAS_cod --ref_decoder_path ./IVAS_dec || exit_code20=$?

    - python3 -m pytest tests/codec_be_on_mr_nonselection $USE_LTV --param_file $SELF_TEST_PRM_FILE -v --update_ref 1 --html=report-20ms.html --self-contained-html --junit-xml=report-junit-20ms.xml --testcase_timeout=$testcase_timeout --ref_encoder_path ./IVAS_cod --ref_decoder_path ./IVAS_dec || exit_code20=$?



    # for 10ms and 5ms renderer framesize, we only need to run the decoder part as renderer framesize is a decoder-only option

    # set tolerance very high do ignore any BE differences due to the different renderer framesizes, those can appear due to the limiter being active

    # we are only interested in runtime errors from the sanitizers and ignore the diffs

    # TODO: was there for ivas-float-update, but not for main -> takes too long

    # - python3 -m pytest tests/codec_be_on_mr_nonselection $USE_LTV --param_file $SELF_TEST_PRM_FILE--html=report-5ms.html --self-contained-html --junit-xml=report-junit-5ms.xml --dut_fr 5 --decoder_only --abs_tol 100000 || exit_code5=$?

    # - python3 -m pytest tests/codec_be_on_mr_nonselection $USE_LTV --param_file $SELF_TEST_PRM_FILE--html=report-10ms.html --self-contained-html --junit-xml=report-junit-10ms.xml --dut_fr 10 --decoder_only --abs_tol 100000 || exit_code10=$?

    # - python3 -m pytest tests/codec_be_on_mr_nonselection $USE_LTV --param_file $SELF_TEST_PRM_FILE -v --html=report-5ms.html --self-contained-html --junit-xml=report-junit-5ms.xml --dut_fr 5 --decoder_only --abs_tol 100000 || exit_code5=$?

    # - python3 -m pytest tests/codec_be_on_mr_nonselection $USE_LTV --param_file $SELF_TEST_PRM_FILE -v --html=report-10ms.html --self-contained-html --junit-xml=report-junit-10ms.xml --dut_fr 10 --decoder_only --abs_tol 100000 || exit_code10=$?



    - if [ $exit_code20 -ne 0 ] || [ $exit_code10 -ne 0 ] || [ $exit_code5 -ne 0 ]; then exit 1; fi
@@ -2054,7 +2054,7 @@ ivas-pytest-compare_ref-long-fx-fx-lev+10: 
    - fi



    # NOTE: here we do not use the "DUT_..CODER_PATH" variables because we do not build via a script, but directly from the make file (which is done because of the "make_args" - no way to inject that into build-binaries.sh)

    - python3 -m pytest $TEST_SUITE--tb=no --update_ref 1 --html=report.html --self-contained-html --junit-xml=report-junit.xml $testcase_timeout_arg --ref_encoder_path IVAS_cod --ref_decoder_path IVAS_dec

    - python3 -m pytest $TEST_SUITE -v --tb=no --update_ref 1 --html=report.html --self-contained-html --junit-xml=report-junit.xml $testcase_timeout_arg --ref_encoder_path IVAS_cod --ref_decoder_path IVAS_dec



ivas-pytest-msan:

  extends:
@@ -2223,11 +2223,11 @@ test-long-self-test: 


    # create references

    - exit_code_ref=0

    - python3 -m pytest tests/codec_be_on_mr_nonselection/test_param_file.py--update_ref 1 --param_file scripts/config/self_test_ltv.prm --use_ltv --testcase_timeout=$testcase_timeout || exit_code_ref=$?

    - python3 -m pytest tests/codec_be_on_mr_nonselection/test_param_file.py -v --update_ref 1 --param_file scripts/config/self_test_ltv.prm --use_ltv --testcase_timeout=$testcase_timeout || exit_code_ref=$?



    ### run pytest self-test using long test vectors

    - exit_code=0

    - python3 -m pytest tests/codec_be_on_mr_nonselection/test_param_file.py--param_file scripts/config/self_test_ltv.prm --use_ltv --html=report-ltv.html --self-contained-html --junit-xml=report-junit-ltv.xml $comp_args --testcase_timeout=$testcase_timeout || exit_code=$?

    - python3 -m pytest tests/codec_be_on_mr_nonselection/test_param_file.py -v --param_file scripts/config/self_test_ltv.prm --use_ltv --html=report-ltv.html --self-contained-html --junit-xml=report-junit-ltv.xml $comp_args --testcase_timeout=$testcase_timeout || exit_code=$?

    - zero_errors=$(cat report-junit-ltv.xml | grep -c 'errors="0"') || true



    - if [ $zero_errors != 1 ]; then echo "Run errors encountered!"; exit $EXIT_CODE_FAIL; fi
@@ -2307,7 +2307,7 @@ coverage-test-on-main-scheduled: 


    - testcase_timeout=$TESTCASE_TIMEOUT_LTV

    - exit_code=0

    - python3 -m pytest --tb=no tests/codec_be_on_mr_nonselection tests/renderer_short --update_ref 1--create_ref --html=report.html --self-contained-html --junit-xml=report-junit.xml -n auto --testcase_timeout $testcase_timeout --ref_encoder_path IVAS_cod --ref_decoder_path IVAS_dec || exit_code=$?

    - python3 -m pytest --tb=no tests/codec_be_on_mr_nonselection tests/renderer_short --update_ref 1 -v --create_ref --html=report.html --self-contained-html --junit-xml=report-junit.xml -n auto --testcase_timeout $testcase_timeout --ref_encoder_path IVAS_cod --ref_decoder_path IVAS_dec || exit_code=$?

    - lcov -c -d obj -o coverage_enc_dec_rend.info # extract coverage



    # remove apps and lib_util files from coverage
@@ -2356,7 +2356,7 @@ be-2-evs-26444: 
    - mkdir -p ./evs_be_test/output/decoded ./evs_be_test/output/bitstreams



    - exit_code=0

    - python3 -m pytest tests/test_26444.py--html=report.html --self-contained-html --junit-xml=report-junit.xml -n auto || exit_code=$?

    - python3 -m pytest tests/test_26444.py -v --html=report.html --self-contained-html --junit-xml=report-junit.xml -n auto || exit_code=$?

    - if [ $exit_code -eq 1 ]; then echo "Differences encountered"; exit $EXIT_CODE_FAIL; fi

    - exit 0
@@ -2436,12 +2436,12 @@ check-be-between-renderer-framesizes: 


    - exit_code=0

    # NOTE: here we do not use the "DUT_..CODER_PATH" variables because we do not build via a script, but directly from the make file (which is done because of the "make_args" - no way to inject that into build-binaries.sh)

    - python3 -m pytest tests/codec_be_on_mr_nonselection --tb=no--ref_encoder_path $REF_ENCODER_PATH --ref_decoder_path ./IVAS_dec --update_ref 1 --html=report-20ms.html --self-contained-html --junit-xml=report-junit-20ms.xml || exit_code=$?

    - python3 -m pytest tests/codec_be_on_mr_nonselection --tb=no -v --ref_encoder_path $REF_ENCODER_PATH --ref_decoder_path ./IVAS_dec --update_ref 1 --html=report-20ms.html --self-contained-html --junit-xml=report-junit-20ms.xml || exit_code=$?



    - exit_code5=0

    - exit_code10=0

    - python3 -m pytest tests/codec_be_on_mr_nonselection --tb=no--ref_encoder_path $REF_ENCODER_PATH --ref_decoder_path ./IVAS_dec --html=report-5ms.html --self-contained-html --junit-xml=report-junit-5ms.xml --dut_fr 5 --decoder_only || exit_code5=$?

    - python3 -m pytest tests/codec_be_on_mr_nonselection --tb=no--ref_encoder_path $REF_ENCODER_PATH --ref_decoder_path ./IVAS_dec --html=report-10ms.html --self-contained-html --junit-xml=report-junit-10ms.xml --dut_fr 10 --decoder_only || exit_code10=$?

    - python3 -m pytest tests/codec_be_on_mr_nonselection --tb=no -v --ref_encoder_path $REF_ENCODER_PATH --ref_decoder_path ./IVAS_dec --html=report-5ms.html --self-contained-html --junit-xml=report-junit-5ms.xml --dut_fr 5 --decoder_only || exit_code5=$?

    - python3 -m pytest tests/codec_be_on_mr_nonselection --tb=no -v --ref_encoder_path $REF_ENCODER_PATH --ref_decoder_path ./IVAS_dec --html=report-10ms.html --self-contained-html --junit-xml=report-junit-10ms.xml --dut_fr 10 --decoder_only || exit_code10=$?



    - zero_errors5=$(cat report-junit-5ms.xml | grep -c 'errors="0"') || true

    - zero_errors10=$(cat report-junit-10ms.xml | grep -c 'errors="0"') || true

main-float.yml

+17 −17
Original line number Diff line number Diff line
@@ -178,12 +178,12 @@ workflow: 
    - if [ $CLANG_NUM -eq 1 ]; then sanitizer_type="MemorySanitizer"; elif [ $CLANG_NUM -eq 2 ]; then sanitizer_type="AddressSanitizer"; elif [ $CLANG_NUM -eq 3 ]; then sanitizer_type="UndefinedBehaviorSanitizer"; else echo "Wrong CLANG_NUM $CLANG_NUM given!"; exit 1; fi



    # run encoder and decoder with 20ms renderer framesize first, use reference creation mode

    - python3 -m pytest tests/codec_be_on_mr_nonselection $USE_LTV --param_file $SELF_TEST_PRM_FILE--update_ref 1 --html=report-20ms.html --self-contained-html --junit-xml=report-junit-20ms.xml --testcase_timeout=$testcase_timeout --ref_encoder_path ./IVAS_cod --ref_decoder_path ./IVAS_dec || exit_code20=$?

    - python3 -m pytest tests/codec_be_on_mr_nonselection $USE_LTV --param_file $SELF_TEST_PRM_FILE -v --update_ref 1 --html=report-20ms.html --self-contained-html --junit-xml=report-junit-20ms.xml --testcase_timeout=$testcase_timeout --ref_encoder_path ./IVAS_cod --ref_decoder_path ./IVAS_dec || exit_code20=$?

    # for 10ms and 5ms renderer framesize, we only need to run the decoder part as renderer framesize is a decoder-only option

    # set tolerance very high do ignore any BE differences due to the different renderer framesizes, those can appear due to the limiter being active

    # we are only interested in runtime errors from the sanitizers and ignore the diffs

    - python3 -m pytest tests/codec_be_on_mr_nonselection $USE_LTV --param_file $SELF_TEST_PRM_FILE--html=report-5ms.html --self-contained-html --junit-xml=report-junit-5ms.xml --dut_fr 5 --decoder_only --abs_tol 100000 || exit_code5=$?

    - python3 -m pytest tests/codec_be_on_mr_nonselection $USE_LTV --param_file $SELF_TEST_PRM_FILE--html=report-10ms.html --self-contained-html --junit-xml=report-junit-10ms.xml --dut_fr 10 --decoder_only --abs_tol 100000 || exit_code10=$?

    - python3 -m pytest tests/codec_be_on_mr_nonselection $USE_LTV --param_file $SELF_TEST_PRM_FILE -v --html=report-5ms.html --self-contained-html --junit-xml=report-junit-5ms.xml --dut_fr 5 --decoder_only --abs_tol 100000 || exit_code5=$?

    - python3 -m pytest tests/codec_be_on_mr_nonselection $USE_LTV --param_file $SELF_TEST_PRM_FILE -v --html=report-10ms.html --self-contained-html --junit-xml=report-junit-10ms.xml --dut_fr 10 --decoder_only --abs_tol 100000 || exit_code10=$?



    - if [ $exit_code20 -ne 0 ] || [ $exit_code10 -ne 0 ] || [ $exit_code5 -ne 0 ]; then exit 1; fi

  after_script:
@@ -475,13 +475,13 @@ pytest-compare-20ms-and-5ms-rendering: 
    - cp IVAS_cod IVAS_cod_ref

    - cp IVAS_dec IVAS_dec_ref

    # create references

    - python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR--update_ref 1

    - python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR -v --update_ref 1

    ### run pytest

    - exit_code=0

    - exit_code5=0

    - exit_code10=0

    - python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR--html=report-5ms.html --self-contained-html --junit-xml=report-junit-5ms.xml --dut_fr 5 --decoder_only || exit_code5=$?

    - python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR--html=report-10ms.html --self-contained-html --junit-xml=report-junit-10ms.xml --dut_fr 10 --decoder_only || exit_code10=$?

    - python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR -v --html=report-5ms.html --self-contained-html --junit-xml=report-junit-5ms.xml --dut_fr 5 --decoder_only || exit_code5=$?

    - python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR -v --html=report-10ms.html --self-contained-html --junit-xml=report-junit-10ms.xml --dut_fr 10 --decoder_only || exit_code10=$?

    - zero_errors5=$(cat report-junit-5ms.xml | grep -c 'errors="0"') || true

    - zero_errors10=$(cat report-junit-10ms.xml | grep -c 'errors="0"') || true

    - zero_errors=1
@@ -821,7 +821,7 @@ ivas-pytest-on-merge-request: 


    ### prepare pytest

    # create references

    - python3 -m pytest "$TESTS_DIR_CODEC_BE_ON_MR"/"$PYTEST_SCRIPT"--update_ref 1

    - python3 -m pytest "$TESTS_DIR_CODEC_BE_ON_MR"/"$PYTEST_SCRIPT" -v --update_ref 1



    ### Run test using branch scripts and input

    - if [ $ref_using_main == 1 ]; then git checkout $source_branch_commit_sha; fi
@@ -829,7 +829,7 @@ ivas-pytest-on-merge-request: 
    ### run pytest

    - exit_code=0

    - testcase_timeout=60

    - python3 -m pytest "$TESTS_DIR_CODEC_BE_ON_MR"/"$PYTEST_SCRIPT"--mld --ssnr --html=report.html --self-contained-html --junit-xml=report-junit.xml --testcase_timeout=$testcase_timeout || exit_code=$?

    - python3 -m pytest "$TESTS_DIR_CODEC_BE_ON_MR"/"$PYTEST_SCRIPT" -v --mld --ssnr --html=report.html --self-contained-html --junit-xml=report-junit.xml --testcase_timeout=$testcase_timeout || exit_code=$?

    - zero_errors=$(cat report-junit.xml | grep -c 'errors="0"') || true



    - *merge-request-comparison-check
@@ -877,7 +877,7 @@ ivas-interop-on-merge-request: 
    - exit_code2=0

    # set timeout for individual testcase runs to 60 seconds

    - testcase_timeout=60

    - python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR--mld --ssnr --html=report.html --self-contained-html --junit-xml=report-junit.xml --update_ref 1 --ref_encoder_path ./IVAS_cod --ref_decoder_path ./IVAS_dec_ref --testcase_timeout=$testcase_timeout || exit_code=$?

    - python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR -v --mld --ssnr --html=report.html --self-contained-html --junit-xml=report-junit.xml --update_ref 1 --ref_encoder_path ./IVAS_cod --ref_decoder_path ./IVAS_dec_ref --testcase_timeout=$testcase_timeout || exit_code=$?

    - zero_failures=$(cat report-junit.xml | grep -c 'failures="0"') || true



    - if [ $zero_failures != 1 ] && [ $non_interop_flag == 0 ]; then echo "Non-interop cases without non-interop flag encountered!"; exit $EXIT_CODE_FAIL; fi
@@ -924,14 +924,14 @@ evs-pytest-on-merge-request: 
    ### prepare pytest

    # create references

    - testcase_timeout=60

    - python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR/test_param_file.py--update_ref 1 --param_file scripts/config/self_test_evs.prm --testcase_timeout=$testcase_timeout

    - python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR/test_param_file.py -v --update_ref 1 --param_file scripts/config/self_test_evs.prm --testcase_timeout=$testcase_timeout



    ### Run test using branch scripts and input

    - if [ $ref_using_main == 1 ]; then git checkout $source_branch_commit_sha; fi



    ### run pytest for EVS cases

    - exit_code=0

    - python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR/test_param_file.py--param_file scripts/config/self_test_evs.prm --html=report.html --self-contained-html --junit-xml=report-junit-evs.xml --testcase_timeout=$testcase_timeout || exit_code=$?

    - python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR/test_param_file.py -v --param_file scripts/config/self_test_evs.prm --html=report.html --self-contained-html --junit-xml=report-junit-evs.xml --testcase_timeout=$testcase_timeout || exit_code=$?

    - zero_errors=$(cat report-junit-evs.xml | grep -c 'errors="0"') || true



    - *merge-request-comparison-check
@@ -1182,14 +1182,14 @@ be-2-evs-windows: 
#     - mv IVAS_dec_test IVAS_dec

#     # create references

#     - testcase_timeout=60

#     - python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR--update_ref 1 --testcase_timeout=$testcase_timeout

#     - python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR -v --update_ref 1 --testcase_timeout=$testcase_timeout

#

#     ### re-checkout the latest commit here, if ref_using_main is set

#     - if [ $ref_using_main -eq 1 ]; then git checkout $latest_commit;fi

#

#     ### run pytest

#     - exit_code=0

#     - python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR--html=report.html --self-contained-html --junit-xml=report-junit.xml --testcase_timeout=$testcase_timeout || exit_code=$?

#     - python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR -v --html=report.html --self-contained-html --junit-xml=report-junit.xml --testcase_timeout=$testcase_timeout || exit_code=$?

#     - if [ $exit_code -ne 0 ] && [ $non_be_flag == 0 ]; then echo "pytest run had failures and non-BE flag not present"; exit $EXIT_CODE_FAIL; fi

#     - zero_errors=$(cat report-junit.xml | grep -c 'errors="0"') || true

#     - if [ $exit_code -ne 0 ] && [ $zero_errors == 1 ]; then echo "pytest run had failures, but no errors and non-BE flag present"; exit $EXIT_CODE_NON_BE; fi
@@ -1483,11 +1483,11 @@ test-long-self-test: 
    # create references

    - exit_code_ref=0

    - testcase_timeout=300

    - python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR --use_ltv--update_ref 1 --param_file scripts/config/self_test_ltv.prm --testcase_timeout=$testcase_timeout || exit_code_ref=$?

    - python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR --use_ltv -v --update_ref 1 --param_file scripts/config/self_test_ltv.prm --testcase_timeout=$testcase_timeout || exit_code_ref=$?



    ### run pytest self-test using long test vectors

    - exit_code=0

    - python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR --use_ltv--mld --ssnr --param_file scripts/config/self_test_ltv.prm --html=report-ltv.html --self-contained-html --junit-xml=report-junit-ltv.xml --testcase_timeout=$testcase_timeout || exit_code=$?

    - python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR --use_ltv -v --mld --ssnr --param_file scripts/config/self_test_ltv.prm --html=report-ltv.html --self-contained-html --junit-xml=report-junit-ltv.xml --testcase_timeout=$testcase_timeout || exit_code=$?

    - zero_errors=$(cat report-junit-ltv.xml | grep -c 'errors="0"') || true



    - if [ $zero_errors != 1 ]; then echo "Run errors encountered!"; exit $EXIT_CODE_FAIL; fi
@@ -1521,7 +1521,7 @@ check-clipping: 
    - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/enable-debugging-macro.sh

    - make -j

    - tests/scale_pcm.py ./scripts/testv/ $SCALE_FACTOR # Default: 3.162 (+10 dB). Can be set in manual trigger.

    - python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR--update_ref 1 --html=report.html --self-contained-html --junit-xml=report-junit.xml --ref_encoder_path ./IVAS_cod --ref_decoder_path ./IVAS_dec

    - python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR -v --update_ref 1 --html=report.html --self-contained-html --junit-xml=report-junit.xml --ref_encoder_path ./IVAS_cod --ref_decoder_path ./IVAS_dec

    - python3 scripts/parse_xml_report.py report-junit.xml report.csv --clipping

  artifacts:

    name: "check-clipping--sha-$CI_COMMIT_SHORT_SHA--results"
@@ -1551,7 +1551,7 @@ test-branch-vs-input-passthrough: 
    - make -j



    - exit_code=0

    - python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR--html=report.html --self-contained-html --junit-xml=report-junit.xml --compare_to_input --mld --ssnr --odg || exit_code=$?

    - python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR -v --html=report.html --self-contained-html --junit-xml=report-junit.xml --compare_to_input --mld --ssnr --odg || exit_code=$?

    - zero_errors=$(cat report-junit.xml | grep -c 'errors="0"') || true

    - python3 scripts/parse_xml_report.py report-junit.xml report.csv

    - if [ $zero_errors != 1 ]; then echo "Run errors encountered!"; exit $EXIT_CODE_FAIL; fi