add regression test job for renderer (b5a0433f) · Commits · IVAS Codec Public Collaboration / IVAS Codec CI

includes/default-variables.yml

+1 −0

Original line number	Diff line number	Diff line
		@@ -23,6 +23,7 @@ variables:
		SHORT_TEST_SUITE: "tests/codec_be_on_mr_nonselection"
		SHORT_TEST_SUITE_ENCODER: "tests/codec_be_on_mr_nonselection/test_param_file.py --param_file scripts/config/self_test_basop_encoder.prm"
		LONG_TEST_SUITE_ENCODER: "tests/codec_be_on_mr_nonselection/test_param_file.py --param_file scripts/config/self_test_ltv_basop_encoder.prm"
		TEST_SUITE_RENDERER_SHORT: "tests/renderer/test_renderer_short.py"
		TEST_SUITE: ""
		INSTR_DIR: "scripts/c-code_instrument"
		# workaround for job failures due to forge being slow to respond when uploading artifacts

main-basop.yml

+40 −6

Original line number	Diff line number	Diff line
		@@ -643,14 +643,20 @@ workflow:

		### ----- run pytest for branch (dut) first -----
		# create float reference outputs with corresponding branch
		- python3 -m pytest $TEST_SUITE --update_ref 1 $enc_stats_arg $enc_dmx_arg --create_ref -n auto --ref_encoder_path $MERGE_SOURCE_FLOAT_REF_ENCODER_PATH --ref_decoder_path $MERGE_SOURCE_FLOAT_REF_DECODER_PATH \|\| exit_code=$?
		- python3 -m pytest $TEST_SUITE --update_ref 1 $enc_stats_arg $enc_dmx_arg --create_ref -n auto --ref_encoder_path $MERGE_SOURCE_FLOAT_REF_ENCODER_PATH --ref_decoder_path $MERGE_SOURCE_FLOAT_REF_DECODER_PATH --ref_renderer_path $MERGE_SOURCE_FLOAT_REF_RENDERER_PATH \|\| exit_code=$?
		- bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/basop/overwrite-pytest-cache-with-artifact.sh
		# create dut outputs
		- python3 -m pytest --tb=no -q $TEST_SUITE --keep_files --create_cut --html=$HTML_REPORT_BRANCH --self-contained-html --junit-xml=$XML_REPORT_BRANCH $comp_args --dut_encoder_path $DUT_ENCODER_PATH --dut_decoder_path $DUT_DECODER_PATH -n auto --testcase_timeout $testcase_timeout \|\| true
		- python3 -m pytest --tb=no -q $TEST_SUITE --keep_files --create_cut --html=$HTML_REPORT_BRANCH --self-contained-html --junit-xml=$XML_REPORT_BRANCH $comp_args --dut_encoder_path $DUT_ENCODER_PATH --dut_decoder_path $DUT_DECODER_PATH --dut_renderer_path $DUT_RENDERER_PATH -n auto --testcase_timeout $testcase_timeout \|\| true
		- zero_errors_branch=$(cat $XML_REPORT_BRANCH \| grep -c 'errors="0"') \|\| true
		- python3 scripts/parse_xml_report.py $XML_REPORT_BRANCH $CSV_BRANCH
		# Store branch outputs for later comparison
		- mv tests/dut tests/dut_branch
		- \|
		if [ "$TEST_SUITE" == "$TEST_SUITE_RENDERER_SHORT" ]; then
		mv tests/renderer/cut tests/renderer/cut_branch
		mkdir tests/renderer/cut
		else
		mv tests/dut tests/dut_branch
		fi
		# create the summary based on the branch only
		- python3 scripts/create_histograms.py $CSV_BRANCH $IMAGES_ARTIFACT_NAME --measures $MEASURES_FOR_REPORT
		- python3 ci/basop-pages/create_summary_page.py $SUMMARY_HTML_ARTIFACT_NAME $CI_JOB_ID $CI_JOB_NAME $IMAGES_ARTIFACT_NAME --measures $MEASURES_FOR_REPORT
		@@ -658,10 +664,10 @@ workflow:

		### ----- run pytest for merge target now -----
		# create float reference outputs with corresponding branch
		- python3 -m pytest $TEST_SUITE --update_ref 1 $enc_stats_arg $enc_dmx_arg --create_ref -n auto --ref_encoder_path $REF_ENCODER_PATH --ref_decoder_path $REF_DECODER_PATH \|\| exit_code=$?
		- python3 -m pytest $TEST_SUITE --update_ref 1 $enc_stats_arg $enc_dmx_arg --create_ref -n auto --ref_encoder_path $REF_ENCODER_PATH --ref_decoder_path $REF_DECODER_PATH --ref_renderer_path $REF_RENDERER_PATH \|\| exit_code=$?
		- bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/basop/overwrite-pytest-cache-with-artifact.sh
		# create merge-target outputs
		- python3 -m pytest --tb=no -q $TEST_SUITE --keep_files --create_cut --html=$HTML_REPORT_MAIN --self-contained-html --junit-xml=$XML_REPORT_MAIN $comp_args --dut_encoder_path $MERGE_TARGET_ENCODER_PATH --dut_decoder_path $MERGE_TARGET_DECODER_PATH -n auto --testcase_timeout $testcase_timeout \|\| true
		- python3 -m pytest --tb=no -q $TEST_SUITE --keep_files --create_cut --html=$HTML_REPORT_MAIN --self-contained-html --junit-xml=$XML_REPORT_MAIN $comp_args --dut_encoder_path $MERGE_TARGET_ENCODER_PATH --dut_decoder_path $MERGE_TARGET_DECODER_PATH --dut_renderer_path $MERGE_TARGET_RENDERER_PATH -n auto --testcase_timeout $testcase_timeout \|\| true
		- python3 scripts/parse_xml_report.py $XML_REPORT_MAIN $CSV_MAIN

		### compare the two csv files for regressions
		@@ -1263,6 +1269,20 @@ check-regressions-short-dec--10db:
		MERGE_TARGET_ENCODER_PATH: "$REF_ENCODER_PATH"
		USE_LTV: 0

		check-regressions-short-renderer-0db:
		extends:
		- .check-regressions-pytest-job
		needs:
		- job: "renderer-pytest-on-merge-request"
		artifacts: true
		- !reference [.test-job-linux-compares-to-fixed-target, needs]
		# this is needed because renderer-pytest-on-merge-request can fail (if non-BE and no non-BE tag set)
		# for all the other check-regressions jobs, their parent (check-be-...) is allowed to fail (yellow instead of red), hence this is not needed for them
		when: always
		variables:
		TEST_SUITE: $TEST_SUITE_RENDERER_SHORT
		USE_LTV: 0

		check-regressions-long-enc-0db:
		extends:
		- .check-regressions-ltv-job
		@@ -1378,11 +1398,22 @@ renderer-pytest-on-merge-request:
		- if [ $ref_using_target == 1 ]; then git checkout $source_branch_commit_sha; fi

		# run test
		- python3 -m pytest -q --log-level ERROR -n auto -rA --mld --ssnr --odg --html=$REPORT_HTML --self-contained-html --junit-xml=$REPORT_XML tests/renderer/test_renderer_short.py --create_cut --testcase_timeout=$testcase_timeout \|\| exit_code=$?
		- python3 -m pytest -q --log-level ERROR -n auto -rA --mld --ssnr --odg --html=$REPORT_HTML --self-contained-html --junit-xml=$REPORT_XML tests/renderer/test_renderer_short.py --create_cut --testcase_timeout=$testcase_timeout > pytest_log.txt \|\| exit_code=$?
		- zero_errors=$(cat $REPORT_XML \| grep -c 'errors="0"') \|\| true

		- python3 scripts/parse_xml_report.py $REPORT_XML $REPORT_CSV

		- \|
		if [ $exit_code -ne 0 ] \|\| [ $zero_errors != 1 ]; then
		zip -r $PYTEST_CACHE_ARTIFACT .pytest_cache
		grep "^FAILED" pytest_log.txt \| sed "s/^FAILED /'/" \| sed "s/] - .*/]'/" \| tr "\n" " " > $FAILED_TESTCASES_LIST \|\| true
		else
		# create empty files to not have errors at artifact stage
		touch $PYTEST_CACHE_ARTIFACT
		touch $FAILED_TESTCASES_LIST
		touch pytest_log.txt
		fi

		- bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/print-sizes.sh $REPORT_XML $REPORT_HTML $REPORT_CSV $MAKE_BUILD_LOG_FILE_DEFAULT
		- source "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/merge-request-comparison-check.sh

		@@ -1398,6 +1429,9 @@ renderer-pytest-on-merge-request:
		- $REPORT_HTML
		- $REPORT_CSV
		- $MAKE_BUILD_LOG_FILE_DEFAULT
		- $PYTEST_CACHE_ARTIFACT
		- $FAILED_TESTCASES_LIST
		- pytest_log.txt
		expose_as: "pytest renderer results"
		reports:
		junit: