Commit 83331711 authored by Jan Kiene's avatar Jan Kiene
Browse files

[revert-me] shorten pytest-based tests for faster testing

parent 6bc16129

main-basop.yml

+22 −22
Original line number Diff line number Diff line
@@ -299,7 +299,7 @@ workflow: 
    - mv $DUT_RENDERER_PATH IVAS_rend



    - exit_code=0

    - python3 -m pytest --tb=no $TEST_SUITE -v --create_cut --html=report.html --self-contained-html --junit-xml=report-junit.xml $comp_args -n auto --testcase_timeout $testcase_timeout --ref_encoder_path $REF_ENCODER_PATH --ref_decoder_path $REF_DECODER_PATH --dut_encoder_path $DUT_ENCODER_PATH --dut_decoder_path $DUT_DECODER_PATH || exit_code=$?

    - python3 -m pytest -k stereo --tb=no $TEST_SUITE -v --create_cut --html=report.html --self-contained-html --junit-xml=report-junit.xml $comp_args -n auto --testcase_timeout $testcase_timeout --ref_encoder_path $REF_ENCODER_PATH --ref_decoder_path $REF_DECODER_PATH --dut_encoder_path $DUT_ENCODER_PATH --dut_decoder_path $DUT_DECODER_PATH || exit_code=$?

    - zero_errors=$(cat report-junit.xml | grep -c 'errors="0"') || true



    ### create histograms
@@ -429,11 +429,11 @@ workflow: 
    - python3 ci/remove_unsupported_testcases.py $PRM_FILES



    - exit_code_target=0

    - python3 -m pytest $TEST_SUITE -v --update_ref 1 --create_ref -n auto --ref_encoder_path $MERGE_TARGET_ENCODER_PATH --ref_decoder_path $MERGE_TARGET_DECODER_PATH > $PYTEST_LOG_TARGET_BRANCH || exit_code_target=$?

    - python3 -m pytest -k stereo $TEST_SUITE -v --update_ref 1 --create_ref -n auto --ref_encoder_path $MERGE_TARGET_ENCODER_PATH --ref_decoder_path $MERGE_TARGET_DECODER_PATH > $PYTEST_LOG_TARGET_BRANCH || exit_code_target=$?



    - exit_code=0

    - rm -rf .pytest_cache || true

    - python3 -m pytest --tb=no -q $TEST_SUITE -v --keep_files --create_cut --html=$HTML_REPORT --self-contained-html --junit-xml=$XML_REPORT --dut_encoder_path $DUT_ENCODER_PATH --dut_decoder_path $DUT_DECODER_PATH -n auto --testcase_timeout $testcase_timeout > pytest_log.txt || exit_code=$?

    - python3 -m pytest -k stereo --tb=no -q $TEST_SUITE -v --keep_files --create_cut --html=$HTML_REPORT --self-contained-html --junit-xml=$XML_REPORT --dut_encoder_path $DUT_ENCODER_PATH --dut_decoder_path $DUT_DECODER_PATH -n auto --testcase_timeout $testcase_timeout > pytest_log.txt || exit_code=$?



    - if [ $exit_code -ne 0 ]; then

    -   exit_code=$EXIT_CODE_NON_BE
@@ -574,10 +574,10 @@ workflow: 


    ### ----- run pytest for branch (dut) first -----

    # create float reference outputs with corresponding branch

    - python3 -m pytest $TEST_SUITE -v --update_ref 1 $enc_stats_arg $enc_dmx_arg --create_ref -n auto --ref_encoder_path $MERGE_SOURCE_FLOAT_REF_ENCODER_PATH --ref_decoder_path $MERGE_SOURCE_FLOAT_REF_DECODER_PATH || exit_code=$?

    - python3 -m pytest -k stereo $TEST_SUITE -v --update_ref 1 $enc_stats_arg $enc_dmx_arg --create_ref -n auto --ref_encoder_path $MERGE_SOURCE_FLOAT_REF_ENCODER_PATH --ref_decoder_path $MERGE_SOURCE_FLOAT_REF_DECODER_PATH || exit_code=$?

    - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/basop/overwrite-pytest-cache-with-artifact.sh

    # create dut outputs

    - python3 -m pytest --tb=no -q $TEST_SUITE -v --keep_files --create_cut --html=$HTML_REPORT_BRANCH --self-contained-html --junit-xml=$XML_REPORT_BRANCH $comp_args --dut_encoder_path $DUT_ENCODER_PATH --dut_decoder_path $DUT_DECODER_PATH -n auto --testcase_timeout $testcase_timeout || true

    - python3 -m pytest -k stereo --tb=no -q $TEST_SUITE -v --keep_files --create_cut --html=$HTML_REPORT_BRANCH --self-contained-html --junit-xml=$XML_REPORT_BRANCH $comp_args --dut_encoder_path $DUT_ENCODER_PATH --dut_decoder_path $DUT_DECODER_PATH -n auto --testcase_timeout $testcase_timeout || true

    - zero_errors_branch=$(cat $XML_REPORT_BRANCH | grep -c 'errors="0"') || true

    - python3 scripts/parse_xml_report.py $XML_REPORT_BRANCH $CSV_BRANCH

    # Store branch outputs for later comparison
@@ -589,10 +589,10 @@ workflow: 


    ### ----- run pytest for merge target now -----

    # create float reference outputs with corresponding branch

    - python3 -m pytest $TEST_SUITE -v --update_ref 1 $enc_stats_arg $enc_dmx_arg --create_ref -n auto --ref_encoder_path $REF_ENCODER_PATH --ref_decoder_path $REF_DECODER_PATH || exit_code=$?

    - python3 -m pytest -k stereo $TEST_SUITE -v --update_ref 1 $enc_stats_arg $enc_dmx_arg --create_ref -n auto --ref_encoder_path $REF_ENCODER_PATH --ref_decoder_path $REF_DECODER_PATH || exit_code=$?

    - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/basop/overwrite-pytest-cache-with-artifact.sh

    # create merge-target outputs

    - python3 -m pytest --tb=no -q $TEST_SUITE -v --keep_files --create_cut --html=$HTML_REPORT_MAIN --self-contained-html --junit-xml=$XML_REPORT_MAIN $comp_args --dut_encoder_path $MERGE_TARGET_ENCODER_PATH --dut_decoder_path $MERGE_TARGET_DECODER_PATH -n auto --testcase_timeout $testcase_timeout || true

    - python3 -m pytest -k stereo --tb=no -q $TEST_SUITE -v --keep_files --create_cut --html=$HTML_REPORT_MAIN --self-contained-html --junit-xml=$XML_REPORT_MAIN $comp_args --dut_encoder_path $MERGE_TARGET_ENCODER_PATH --dut_decoder_path $MERGE_TARGET_DECODER_PATH -n auto --testcase_timeout $testcase_timeout || true

    - python3 scripts/parse_xml_report.py $XML_REPORT_MAIN $CSV_MAIN



    ### compare the two csv files for regressions
@@ -1252,13 +1252,13 @@ renderer-pytest-on-merge-request: 


    - exit_code=0

    - testcase_timeout=60

    - python3 -m pytest -q --log-level ERROR -n auto -rA tests/renderer/test_renderer.py --create_ref --testcase_timeout=$testcase_timeout || exit_code=$?

    - python3 -m pytest -k mono -q --log-level ERROR -n auto -rA tests/renderer/test_renderer.py --create_ref --testcase_timeout=$testcase_timeout || exit_code=$?



    ### Run test using branch scripts and input

    - if [ $ref_using_target == 1 ]; then git checkout $source_branch_commit_sha; fi



    # run test

    - python3 -m pytest -q --log-level ERROR -n auto -rA --mld --ssnr --odg --html=$REPORT_HTML --self-contained-html --junit-xml=$REPORT_XML tests/renderer/test_renderer.py --create_cut --testcase_timeout=$testcase_timeout || exit_code=$?

    - python3 -m pytest -k mono -q --log-level ERROR -n auto -rA --mld --ssnr --odg --html=$REPORT_HTML --self-contained-html --junit-xml=$REPORT_XML tests/renderer/test_renderer.py --create_cut --testcase_timeout=$testcase_timeout || exit_code=$?

    - zero_errors=$(cat $REPORT_XML | grep -c 'errors="0"') || true



    - python3 scripts/parse_xml_report.py $REPORT_XML $REPORT_CSV
@@ -1321,7 +1321,7 @@ ivas-pytest-on-merge-request: 


    ### prepare pytest

    # create references

    - python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR -v --update_ref 1

    - python3 -m pytest -k stereo $TESTS_DIR_CODEC_BE_ON_MR -v --update_ref 1



    ### Run test using branch scripts and input

    - if [ $ref_using_target == 1 ]; then git checkout $source_branch_commit_sha; fi
@@ -1329,7 +1329,7 @@ ivas-pytest-on-merge-request: 
    ### run pytest

    - exit_code=0

    - testcase_timeout=600

    - python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR -v --mld --ssnr --odg --html=$REPORT_HTML --self-contained-html --junit-xml=$REPORT_XML --testcase_timeout=$testcase_timeout || exit_code=$?

    - python3 -m pytest -k stereo $TESTS_DIR_CODEC_BE_ON_MR -v --mld --ssnr --odg --html=$REPORT_HTML --self-contained-html --junit-xml=$REPORT_XML --testcase_timeout=$testcase_timeout || exit_code=$?

    - zero_errors=$(cat $REPORT_XML | grep -c 'errors="0"') || true



    - python3 scripts/parse_xml_report.py $REPORT_XML $REPORT_CSV
@@ -1389,7 +1389,7 @@ split-rendering-pytest-on-merge-request: 
    # remporary solution to make it work with and without https://forge.3gpp.org/rep/ivas-codec-pc/ivas-codec/-/merge_requests/2181

    - mv ISAR_post_rend ISAR_post_rend_tmp

    - cp $REF_POST_RENDERER_PATH ISAR_post_rend

    - python3 -m pytest -q --log-level ERROR -n auto -rA tests/split_rendering/test_split_rendering.py --create_ref --testcase_timeout=$testcase_timeout || exit_code=$?

    - python3 -m pytest -k mono -q --log-level ERROR -n auto -rA tests/split_rendering/test_split_rendering.py --create_ref --testcase_timeout=$testcase_timeout || exit_code=$?

    - mv ISAR_post_rend_tmp ISAR_post_rend



    ### Run test using scripts and input from main
@@ -1400,7 +1400,7 @@ split-rendering-pytest-on-merge-request: 
    - echo "$comp_args"



    # run test

    - python3 -m pytest -q --log-level ERROR -n auto -rA --html=$REPORT_HTML --self-contained-html --junit-xml=$REPORT_XML tests/split_rendering/test_split_rendering.py $comp_args --create_cut --testcase_timeout=$testcase_timeout || exit_code=$?

    - python3 -m pytest -k mono -q --log-level ERROR -n auto -rA --html=$REPORT_HTML --self-contained-html --junit-xml=$REPORT_XML tests/split_rendering/test_split_rendering.py $comp_args --create_cut --testcase_timeout=$testcase_timeout || exit_code=$?

    - zero_errors=$(cat $REPORT_XML | grep -c 'errors="0"') || true



    - python3 scripts/parse_xml_report.py $REPORT_XML $REPORT_CSV
@@ -1533,7 +1533,7 @@ ivas-interop-on-merge-request: 
    - exit_code2=0

    # set timeout for individual testcase runs to 60 seconds

    - testcase_timeout=60

    - python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR -v --html=report.html --self-contained-html --junit-xml=report-junit.xml --update_ref 1 --ref_encoder_path ./IVAS_cod --ref_decoder_path ./IVAS_dec_ref --testcase_timeout=$testcase_timeout || exit_code=$?

    - python3 -m pytest -k stereo $TESTS_DIR_CODEC_BE_ON_MR -v --html=report.html --self-contained-html --junit-xml=report-junit.xml --update_ref 1 --ref_encoder_path ./IVAS_cod --ref_decoder_path ./IVAS_dec_ref --testcase_timeout=$testcase_timeout || exit_code=$?

    - zero_failures=$(cat report-junit.xml | grep -c 'failures="0"') || true



    - if [ $zero_failures != 1 ] && [ $non_interop_flag == 0 ]; then echo "Non-interop cases without non-interop flag encountered!"; exit $EXIT_CODE_FAIL; fi
@@ -1568,7 +1568,7 @@ split-rendering-smoke-test: 


    - make -j

    - testcase_timeout=10

    - python3 -m pytest -q -n auto -rA --junit-xml=report-junit.xml tests/split_rendering/test_split_rendering.py --testcase_timeout=$testcase_timeout

    - python3 -m pytest -k mono -q -n auto -rA --junit-xml=report-junit.xml tests/split_rendering/test_split_rendering.py --testcase_timeout=$testcase_timeout

  artifacts:

    name: "mr-$CI_MERGE_REQUEST_IID--sha-$CI_COMMIT_SHORT_SHA--job-$CI_JOB_NAME--results"

    expire_in: 1 week
@@ -1614,7 +1614,7 @@ renderer-smoke-test: 
    - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/basop/update-scripts-repo.sh

    - make -j IVAS_rend

    - testcase_timeout=60

    - python3 -m pytest -q -n auto -rA --junit-xml=report-junit.xml tests/renderer/test_renderer.py --testcase_timeout=$testcase_timeout

    - python3 -m pytest -k mono -q -n auto -rA --junit-xml=report-junit.xml tests/renderer/test_renderer.py --testcase_timeout=$testcase_timeout

  artifacts:

    name: "mr-$CI_MERGE_REQUEST_IID--sha-$CI_COMMIT_SHORT_SHA--job-$CI_JOB_NAME--results"

    expire_in: 1 week
@@ -1640,7 +1640,7 @@ renderer-asan: 
    - cmake -B cmake-build -G "Unix Makefiles" -DCLANG=asan -DCOPY_EXECUTABLES_FROM_BUILD_DIR=true

    - cmake --build cmake-build -- -j

    - testcase_timeout=180

    - python3 -m pytest -q -n auto -rA --junit-xml=report-junit.xml tests/renderer/test_renderer.py --testcase_timeout=$testcase_timeout

    - python3 -m pytest -k mono -q -n auto -rA --junit-xml=report-junit.xml tests/renderer/test_renderer.py --testcase_timeout=$testcase_timeout



  artifacts:

    name: "mr-$CI_MERGE_REQUEST_IID--sha-$CI_COMMIT_SHORT_SHA--job-$CI_JOB_NAME--results"
@@ -1669,7 +1669,7 @@ renderer-msan: 
    - cmake -B cmake-build -G "Unix Makefiles" -DCLANG=msan -DCOPY_EXECUTABLES_FROM_BUILD_DIR=true

    - cmake --build cmake-build -- -j

    - testcase_timeout=180

    - python3 -m pytest -q -n auto -rA --junit-xml=report-junit.xml tests/renderer/test_renderer.py --testcase_timeout=$testcase_timeout

    - python3 -m pytest -k mono -q -n auto -rA --junit-xml=report-junit.xml tests/renderer/test_renderer.py --testcase_timeout=$testcase_timeout



  artifacts:

    name: "mr-$CI_MERGE_REQUEST_IID--sha-$CI_COMMIT_SHORT_SHA--job-$CI_JOB_NAME--results"
@@ -1698,7 +1698,7 @@ renderer-usan: 
    - cmake -B cmake-build -G "Unix Makefiles" -DCLANG=usan -DCOPY_EXECUTABLES_FROM_BUILD_DIR=true

    - cmake --build cmake-build -- -j

    - testcase_timeout=180

    - UBSAN_OPTIONS=suppressions=scripts/ubsan.supp,report_error_type=1,log_path=usan_log_catchall python3 -m pytest -q -n auto -rA --junit-xml=report-junit.xml tests/renderer/test_renderer.py --testcase_timeout=$testcase_timeout

    - UBSAN_OPTIONS=suppressions=scripts/ubsan.supp,report_error_type=1,log_path=usan_log_catchall python3 -m pytest -k mono -q -n auto -rA --junit-xml=report-junit.xml tests/renderer/test_renderer.py --testcase_timeout=$testcase_timeout

    - grep_exit_code=0

    - touch usan_log_empty # Creates an empty file, this is to avoid "grep: usan_log_*: No such file or directory" in case no USAN failures are reported from pytest

    - grep UndefinedBehaviorSanitizer usan_log_* || grep_exit_code=$?
@@ -1759,12 +1759,12 @@ renderer-usan: 
    - if [ $CLANG_NUM -eq 1 ]; then sanitizer_type="MemorySanitizer"; elif [ $CLANG_NUM -eq 2 ]; then sanitizer_type="AddressSanitizer"; elif [ $CLANG_NUM -eq 3 ]; then sanitizer_type="UndefinedBehaviorSanitizer"; else echo "Wrong CLANG_NUM $CLANG_NUM given!"; exit 1; fi



    # run encoder and decoder with 20ms renderer framesize first, use reference creation mode

    - python3 -m pytest tests/codec_be_on_mr_nonselection $USE_LTV --param_file $SELF_TEST_PRM_FILE -v --update_ref 1 --html=report-20ms.html --self-contained-html --junit-xml=report-junit-20ms.xml --testcase_timeout=$testcase_timeout --ref_encoder_path ./IVAS_cod --ref_decoder_path ./IVAS_dec || exit_code20=$?

    - python3 -m pytest -k stereo tests/codec_be_on_mr_nonselection $USE_LTV --param_file $SELF_TEST_PRM_FILE -v --update_ref 1 --html=report-20ms.html --self-contained-html --junit-xml=report-junit-20ms.xml --testcase_timeout=$testcase_timeout --ref_encoder_path ./IVAS_cod --ref_decoder_path ./IVAS_dec || exit_code20=$?

    # for 10ms and 5ms renderer framesize, we only need to run the decoder part as renderer framesize is a decoder-only option

    # set tolerance very high do ignore any BE differences due to the different renderer framesizes, those can appear due to the limiter being active

    # we are only interested in runtime errors from the sanitizers and ignore the diffs

    - python3 -m pytest tests/codec_be_on_mr_nonselection $USE_LTV --param_file $SELF_TEST_PRM_FILE -v --html=report-5ms.html --self-contained-html --junit-xml=report-junit-5ms.xml --dut_fr 5 --decoder_only --abs_tol 100000 || exit_code5=$?

    - python3 -m pytest tests/codec_be_on_mr_nonselection $USE_LTV --param_file $SELF_TEST_PRM_FILE -v --html=report-10ms.html --self-contained-html --junit-xml=report-junit-10ms.xml --dut_fr 10 --decoder_only --abs_tol 100000 || exit_code10=$?

    - python3 -m pytest -k stereo tests/codec_be_on_mr_nonselection $USE_LTV --param_file $SELF_TEST_PRM_FILE -v --html=report-5ms.html --self-contained-html --junit-xml=report-junit-5ms.xml --dut_fr 5 --decoder_only --abs_tol 100000 || exit_code5=$?

    - python3 -m pytest -k stereo tests/codec_be_on_mr_nonselection $USE_LTV --param_file $SELF_TEST_PRM_FILE -v --html=report-10ms.html --self-contained-html --junit-xml=report-junit-10ms.xml --dut_fr 10 --decoder_only --abs_tol 100000 || exit_code10=$?



    - if [ $exit_code20 -ne 0 ] || [ $exit_code10 -ne 0 ] || [ $exit_code5 -ne 0 ]; then exit 1; fi