diff --git a/includes/default-variables.yml b/includes/default-variables.yml index c8c6761e84906b9d8b7ff6c108be06f7ab857171..58c89e65b4c4e3010c14b58c691c19cdbf80097c 100644 --- a/includes/default-variables.yml +++ b/includes/default-variables.yml @@ -23,6 +23,7 @@ variables: SHORT_TEST_SUITE: "tests/codec_be_on_mr_nonselection" SHORT_TEST_SUITE_ENCODER: "tests/codec_be_on_mr_nonselection/test_param_file.py --param_file scripts/config/self_test_basop_encoder.prm" LONG_TEST_SUITE_ENCODER: "tests/codec_be_on_mr_nonselection/test_param_file.py --param_file scripts/config/self_test_ltv_basop_encoder.prm" + TEST_SUITE_RENDERER_SHORT: "tests/renderer/test_renderer_short.py" TEST_SUITE: "" INSTR_DIR: "scripts/c-code_instrument" # workaround for job failures due to forge being slow to respond when uploading artifacts diff --git a/main-basop.yml b/main-basop.yml index 0f4a95eaaa79956c70e69fd42d8317a299ab0f27..fe90d56681ed9d65eaeb25182adf56aa21cfb721 100644 --- a/main-basop.yml +++ b/main-basop.yml @@ -624,7 +624,7 @@ workflow: - comp_args="--mld --ssnr --odg --mld-playback-level $MLD_PLAYBACK_LEVEL" - # build merge target branch and correpsonding float reference + # build merge target branch and corresponding float reference - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/basop/build-binaries.sh float-ref - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/basop/build-binaries.sh merge-target # build to-be-merged branch and corresponding float ref branch @@ -643,14 +643,20 @@ workflow: ### ----- run pytest for branch (dut) first ----- # create float reference outputs with corresponding branch - - python3 -m pytest $TEST_SUITE --update_ref 1 $enc_stats_arg $enc_dmx_arg --create_ref -n auto --ref_encoder_path $MERGE_SOURCE_FLOAT_REF_ENCODER_PATH --ref_decoder_path $MERGE_SOURCE_FLOAT_REF_DECODER_PATH || exit_code=$? + - python3 -m pytest $TEST_SUITE --update_ref 1 $enc_stats_arg $enc_dmx_arg --create_ref -n auto --ref_encoder_path $MERGE_SOURCE_FLOAT_REF_ENCODER_PATH --ref_decoder_path $MERGE_SOURCE_FLOAT_REF_DECODER_PATH --ref_renderer_path $MERGE_SOURCE_FLOAT_REF_RENDERER_PATH || exit_code=$? - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/basop/overwrite-pytest-cache-with-artifact.sh # create dut outputs - - python3 -m pytest --tb=no -q $TEST_SUITE --keep_files --create_cut --html=$HTML_REPORT_BRANCH --self-contained-html --junit-xml=$XML_REPORT_BRANCH $comp_args --dut_encoder_path $DUT_ENCODER_PATH --dut_decoder_path $DUT_DECODER_PATH -n auto --testcase_timeout $testcase_timeout || true + - python3 -m pytest --tb=no -q $TEST_SUITE --keep_files --create_cut --html=$HTML_REPORT_BRANCH --self-contained-html --junit-xml=$XML_REPORT_BRANCH $comp_args --dut_encoder_path $DUT_ENCODER_PATH --dut_decoder_path $DUT_DECODER_PATH --dut_renderer_path $DUT_RENDERER_PATH -n auto --testcase_timeout $testcase_timeout || true - zero_errors_branch=$(cat $XML_REPORT_BRANCH | grep -c 'errors="0"') || true - python3 scripts/parse_xml_report.py $XML_REPORT_BRANCH $CSV_BRANCH # Store branch outputs for later comparison - - mv tests/dut tests/dut_branch + - | + if [ "$TEST_SUITE" == "$TEST_SUITE_RENDERER_SHORT" ]; then + mv tests/renderer/cut tests/renderer/cut_branch + mkdir tests/renderer/cut + else + mv tests/dut tests/dut_branch + fi # create the summary based on the branch only - python3 scripts/create_histograms.py $CSV_BRANCH $IMAGES_ARTIFACT_NAME --measures $MEASURES_FOR_REPORT - python3 ci/basop-pages/create_summary_page.py $SUMMARY_HTML_ARTIFACT_NAME $CI_JOB_ID $CI_JOB_NAME $IMAGES_ARTIFACT_NAME --measures $MEASURES_FOR_REPORT @@ -658,10 +664,10 @@ workflow: ### ----- run pytest for merge target now ----- # create float reference outputs with corresponding branch - - python3 -m pytest $TEST_SUITE --update_ref 1 $enc_stats_arg $enc_dmx_arg --create_ref -n auto --ref_encoder_path $REF_ENCODER_PATH --ref_decoder_path $REF_DECODER_PATH || exit_code=$? + - python3 -m pytest $TEST_SUITE --update_ref 1 $enc_stats_arg $enc_dmx_arg --create_ref -n auto --ref_encoder_path $REF_ENCODER_PATH --ref_decoder_path $REF_DECODER_PATH --ref_renderer_path $REF_RENDERER_PATH || exit_code=$? - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/basop/overwrite-pytest-cache-with-artifact.sh # create merge-target outputs - - python3 -m pytest --tb=no -q $TEST_SUITE --keep_files --create_cut --html=$HTML_REPORT_MAIN --self-contained-html --junit-xml=$XML_REPORT_MAIN $comp_args --dut_encoder_path $MERGE_TARGET_ENCODER_PATH --dut_decoder_path $MERGE_TARGET_DECODER_PATH -n auto --testcase_timeout $testcase_timeout || true + - python3 -m pytest --tb=no -q $TEST_SUITE --keep_files --create_cut --html=$HTML_REPORT_MAIN --self-contained-html --junit-xml=$XML_REPORT_MAIN $comp_args --dut_encoder_path $MERGE_TARGET_ENCODER_PATH --dut_decoder_path $MERGE_TARGET_DECODER_PATH --dut_renderer_path $MERGE_TARGET_RENDERER_PATH -n auto --testcase_timeout $testcase_timeout || true - python3 scripts/parse_xml_report.py $XML_REPORT_MAIN $CSV_MAIN ### compare the two csv files for regressions @@ -1263,6 +1269,20 @@ check-regressions-short-dec--10db: MERGE_TARGET_ENCODER_PATH: "$REF_ENCODER_PATH" USE_LTV: 0 +check-regressions-short-renderer-0db: + extends: + - .check-regressions-pytest-job + needs: + - job: "renderer-pytest-on-merge-request" + artifacts: true + - !reference [.test-job-linux-compares-to-fixed-target, needs] + # this is needed because renderer-pytest-on-merge-request can fail (if non-BE and no non-BE tag set) + # for all the other check-regressions jobs, their parent (check-be-...) is allowed to fail (yellow instead of red), hence this is not needed for them + when: always + variables: + TEST_SUITE: $TEST_SUITE_RENDERER_SHORT + USE_LTV: 0 + check-regressions-long-enc-0db: extends: - .check-regressions-ltv-job @@ -1378,11 +1398,22 @@ renderer-pytest-on-merge-request: - if [ $ref_using_target == 1 ]; then git checkout $source_branch_commit_sha; fi # run test - - python3 -m pytest -q --log-level ERROR -n auto -rA --mld --ssnr --odg --html=$REPORT_HTML --self-contained-html --junit-xml=$REPORT_XML tests/renderer/test_renderer_short.py --create_cut --testcase_timeout=$testcase_timeout || exit_code=$? + - python3 -m pytest -q --log-level ERROR -n auto -rA --mld --ssnr --odg --html=$REPORT_HTML --self-contained-html --junit-xml=$REPORT_XML tests/renderer/test_renderer_short.py --create_cut --testcase_timeout=$testcase_timeout > pytest_log.txt || exit_code=$? - zero_errors=$(cat $REPORT_XML | grep -c 'errors="0"') || true - python3 scripts/parse_xml_report.py $REPORT_XML $REPORT_CSV + - | + if [ $exit_code -ne 0 ] || [ $zero_errors != 1 ]; then + zip -r $PYTEST_CACHE_ARTIFACT .pytest_cache + grep "^FAILED" pytest_log.txt | sed "s/^FAILED /'/" | sed "s/] - .*/]'/" | tr "\n" " " > $FAILED_TESTCASES_LIST || true + else + # create empty files to not have errors at artifact stage + touch $PYTEST_CACHE_ARTIFACT + touch $FAILED_TESTCASES_LIST + touch pytest_log.txt + fi + - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/print-sizes.sh $REPORT_XML $REPORT_HTML $REPORT_CSV $MAKE_BUILD_LOG_FILE_DEFAULT - source "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/merge-request-comparison-check.sh @@ -1398,6 +1429,9 @@ renderer-pytest-on-merge-request: - $REPORT_HTML - $REPORT_CSV - $MAKE_BUILD_LOG_FILE_DEFAULT + - $PYTEST_CACHE_ARTIFACT + - $FAILED_TESTCASES_LIST + - pytest_log.txt expose_as: "pytest renderer results" reports: junit: