Merge branch 'ci/add-split-comparison-for-ltv-clean' into 'main' (c1f32b7f) · Commits · SA4 / Audio / IVAS BASOP

.gitlab-ci.yml

+37 −11

Original line number	Diff line number	Diff line
		@@ -44,10 +44,12 @@ variables:
		ENCODER_TEST: ""
		DELTA_ODG: ""
		COMPARE_DMX: ""
		SPLIT_COMPARISON: ""
		SKIP_REGRESSION_CHECK: ""
		FAILED_TESTCASES_LIST: "failed-testcases.txt"
		ERRORS_TESTCASES_LIST: "errors-testcases.txt"
		PYTEST_CACHE_ARTIFACT: "pytest_cache.zip"
		MEASURES_FOR_REPORT: "MLD MAX_ABS_DIFF MIN_SSNR MIN_ODG"
		FLOAT_REF_COMMIT_FILE: "float-ref-git-sha.txt"
		CUT_COMMIT_FILE: "CuT-git-sha.txt"
		MERGE_TARGET_COMMIT_FILE: "merge-target-git-sha.txt"
		@@ -370,10 +372,13 @@ stages:
		variables:
		# keep "mld" in artifact name for backwards compatibility reasons
		CSV_ARTIFACT_NAME: "mld--$CI_JOB_NAME-$CI_JOB_ID--sha-$CI_COMMIT_SHORT_SHA.csv"
		CSV_ARTIFACT_SPLIT: "mld--split--$CI_JOB_NAME-$CI_JOB_ID--sha-$CI_COMMIT_SHORT_SHA.csv"
		MERGED_CSV_ARTIFACT_NAME: "$CI_JOB_NAME--merged_csv--$CI_JOB_ID.csv"
		PAGES_HTML_ARTIFACT_NAME: "$CI_JOB_NAME-index.html"
		SUMMARY_HTML_ARTIFACT_NAME: "summary_$CI_JOB_NAME.html"
		SUMMARY_HTML_ARTIFACT_SPLIT: "summary_split_$CI_JOB_NAME.html"
		IMAGES_ARTIFACT_NAME: "images_$CI_JOB_NAME"
		IMAGES_ARTIFACT_SPLIT: "images_split_$CI_JOB_NAME"
		script:
		- set -euxo pipefail
		- *print-common-info
		@@ -394,14 +399,16 @@ stages:
		- if [ "$COMPARE_DMX" = "true" ] \|\| [ "$ENCODER_TEST" = "true" ]; then
		- BUILD_WITH_DEBUG_MODE_INFO="true"
		- fi
		- *build-and-create-float-ref-outputs

		- INV_LEVEL_SCALING=$(awk "BEGIN {print 1.0 / $LEVEL_SCALING}")
		- comp_args="--mld --ssnr --odg --scalefac $INV_LEVEL_SCALING"
		- summary_args="MLD DIFF SSNR ODG"
		- REPORT_ARG=""
		- if [ "$ENCODER_TEST" = "true" ]; then comp_args="${comp_args} --enc_stats"; fi
		- if [ "$DELTA_ODG" = "true" ]; then comp_args="${comp_args} --odg_bin"; summary_args="${summary_args} DELTA_ODG"; REPORT_ARG="--delta_odg"; fi
		- if [ "$DELTA_ODG" = "true" ]; then comp_args="${comp_args} --odg_bin"; MEASURES_FOR_REPORT="$MEASURES_FOR_REPORT DELTA_ODG"; fi
		- if [ "$SPLIT_COMPARISON" = "true" ]; then comp_args="${comp_args} --split-comparison"; fi

		- *build-and-create-float-ref-outputs

		# DMX comparison only in manual job with no other metrics
		- if [ "$COMPARE_DMX" = "true" ]; then
		@@ -415,10 +422,20 @@ stages:
		- python3 -m pytest --tb=no $TEST_SUITE -v --create_cut --html=report.html --self-contained-html --junit-xml=report-junit.xml $comp_args -n auto --testcase_timeout $testcase_timeout --ref_encoder_path $REF_ENCODER_PATH --ref_decoder_path $REF_DECODER_PATH --dut_encoder_path $DUT_ENCODER_PATH --dut_decoder_path $DUT_DECODER_PATH \|\| exit_code=$?
		- zero_errors=$(cat report-junit.xml \| grep -c 'errors="0"') \|\| true

		- python3 scripts/parse_xml_report.py report-junit.xml $CSV_ARTIFACT_NAME $REPORT_ARG
		- mkdir $IMAGES_ARTIFACT_NAME
		- for MEASURE in $summary_args;do python3 scripts/create_histogram_summary.py $CSV_ARTIFACT_NAME $IMAGES_ARTIFACT_NAME/summary_"$MEASURE".csv $IMAGES_ARTIFACT_NAME/summary_"$MEASURE".png --measure $MEASURE; done
		- python3 ci/basop-pages/create_summary_page.py $SUMMARY_HTML_ARTIFACT_NAME $CI_JOB_ID $CI_JOB_NAME --measures $summary_args
		### create histograms
		- python3 scripts/parse_xml_report.py report-junit.xml $CSV_ARTIFACT_NAME --split-csv-file $CSV_ARTIFACT_SPLIT

		# first for "whole" files comparison
		- python3 scripts/create_histograms.py $CSV_ARTIFACT_NAME $IMAGES_ARTIFACT_NAME --measures $MEASURES_FOR_REPORT --write-out-histograms
		- python3 ci/basop-pages/create_summary_page.py $SUMMARY_HTML_ARTIFACT_NAME $CI_JOB_ID $CI_JOB_NAME $IMAGES_ARTIFACT_NAME --measures $MEASURES_FOR_REPORT

		- if [ "$SPLIT_COMPARISON" = "true" ]; then
		- python3 scripts/create_histograms.py $CSV_ARTIFACT_SPLIT $IMAGES_ARTIFACT_SPLIT --measures $MEASURES_FOR_REPORT --write-out-histograms
		- python3 ci/basop-pages/create_summary_page.py $SUMMARY_HTML_ARTIFACT_SPLIT $CI_JOB_ID $CI_JOB_NAME $IMAGES_ARTIFACT_SPLIT --measures $MEASURES_FOR_REPORT
		- else
		# touch files to suppress warning for missing artifacts
		- touch $CSV_ARTIFACT_SPLIT $IMAGES_ARTIFACT_SPLIT
		- fi

		- if [ $USE_LTV -eq 1 ] && [ "$CI_COMMIT_BRANCH" == "$CI_DEFAULT_BRANCH" ]; then
		- id_previous=$(python3 ci/get_id_of_last_job_occurence.py $CI_DEFAULT_BRANCH $CI_JOB_NAME $CI_PROJECT_ID)
		@@ -458,9 +475,12 @@ stages:
		- report.html
		- $PAGES_HTML_ARTIFACT_NAME
		- $CSV_ARTIFACT_NAME
		- $CSV_ARTIFACT_SPLIT
		- $MERGED_CSV_ARTIFACT_NAME
		- $SUMMARY_HTML_ARTIFACT_NAME
		- $SUMMARY_HTML_ARTIFACT_SPLIT
		- $IMAGES_ARTIFACT_NAME
		- $IMAGES_ARTIFACT_SPLIT
		expose_as: "pytest compare results"
		reports:
		junit:
		@@ -652,9 +672,9 @@ stages:
		# Store branch outputs for comparison
		- mv tests/dut tests/dut_branch

		# create the summary based on the branch
		- for MEASURE in MLD DIFF SSNR ODG;do python3 scripts/create_histogram_summary.py $CSV_BRANCH $IMAGES_ARTIFACT_NAME/summary_"$MEASURE".csv $IMAGES_ARTIFACT_NAME/summary_"$MEASURE".png --measure $MEASURE; done
		- python3 ci/basop-pages/create_summary_page.py $SUMMARY_HTML_ARTIFACT_NAME $CI_JOB_ID $CI_JOB_NAME
		# create the summary based on the branch only
		- python3 scripts/create_histograms.py $CSV_BRANCH $IMAGES_ARTIFACT_NAME --measures $MEASURES_FOR_REPORT
		- python3 ci/basop-pages/create_summary_page.py $SUMMARY_HTML_ARTIFACT_NAME $CI_JOB_ID $CI_JOB_NAME $IMAGES_ARTIFACT_NAME --measures $MEASURES_FOR_REPORT

		### run main now
		- git checkout $CI_MERGE_REQUEST_TARGET_BRANCH_NAME
		@@ -816,8 +836,8 @@ stages:

		# create summary
		- mkdir $IMAGES_ARTIFACT_NAME
		- for MEASURE in MLD DIFF SSNR ODG;do python3 scripts/create_histogram_summary.py report-diff.csv $IMAGES_ARTIFACT_NAME/summary_"$MEASURE".csv $IMAGES_ARTIFACT_NAME/summary_"$MEASURE".png --measure $MEASURE --diff; done
		- python3 ci/basop-pages/create_summary_page.py $SUMMARY_HTML_ARTIFACT_NAME $CI_JOB_ID $CI_JOB_NAME
		- python3 scripts/create_histograms.py report-diff.csv $IMAGES_ARTIFACT_NAME --measures $MEASURES_FOR_REPORT --write-out-histograms --no-bins
		- python3 ci/basop-pages/create_summary_page.py $SUMMARY_HTML_ARTIFACT_NAME $CI_JOB_ID $CI_JOB_NAME $IMAGES_ARTIFACT_NAME --measures $MEASURES_FOR_REPORT

		- exit 0

		@@ -1422,6 +1442,7 @@ ivas-pytest-compare_ref-long-enc:
		- DUT_DECODER_PATH=./$REF_DECODER_PATH
		- TEST_SUITE="$LONG_TEST_SUITE_ENCODER"
		- LEVEL_SCALING=1.0
		- SPLIT_COMPARISON="true"
		<<: *ivas-pytest-anchor

		ivas-pytest-compare_ref-long-dec:
		@@ -1433,6 +1454,7 @@ ivas-pytest-compare_ref-long-dec:
		- DUT_ENCODER_PATH=./$REF_ENCODER_PATH
		- TEST_SUITE="$LONG_TEST_SUITE"
		- LEVEL_SCALING=1.0
		- SPLIT_COMPARISON="true"
		<<: *ivas-pytest-anchor

		ivas-pytest-compare_ref-long-enc-lev-10:
		@@ -1444,6 +1466,7 @@ ivas-pytest-compare_ref-long-enc-lev-10:
		- DUT_DECODER_PATH=./$REF_DECODER_PATH
		- TEST_SUITE="$LONG_TEST_SUITE_ENCODER"
		- LEVEL_SCALING=0.3162
		- SPLIT_COMPARISON="true"
		<<: *ivas-pytest-anchor

		ivas-pytest-compare_ref-long-dec-lev-10:
		@@ -1455,6 +1478,7 @@ ivas-pytest-compare_ref-long-dec-lev-10:
		- DUT_ENCODER_PATH=./$REF_ENCODER_PATH
		- TEST_SUITE="$LONG_TEST_SUITE"
		- LEVEL_SCALING=0.3162
		- SPLIT_COMPARISON="true"
		<<: *ivas-pytest-anchor

		ivas-pytest-compare_ref-long-enc-lev+10:
		@@ -1466,6 +1490,7 @@ ivas-pytest-compare_ref-long-enc-lev+10:
		- DUT_DECODER_PATH=./$REF_DECODER_PATH
		- TEST_SUITE="$LONG_TEST_SUITE_ENCODER"
		- LEVEL_SCALING=3.162
		- SPLIT_COMPARISON="true"
		<<: *ivas-pytest-anchor

		ivas-pytest-compare_ref-long-dec-lev+10:
		@@ -1477,6 +1502,7 @@ ivas-pytest-compare_ref-long-dec-lev+10:
		- DUT_ENCODER_PATH=./$REF_ENCODER_PATH
		- TEST_SUITE="$LONG_TEST_SUITE"
		- LEVEL_SCALING=3.162
		- SPLIT_COMPARISON="true"
		<<: *ivas-pytest-anchor

		ivas-smoke-test-saturation: