Merge branch 'ci/check-for-regressions-in-mr-pipelines' into 'main' (23c07bf8) · Commits · SA4 / Audio / IVAS BASOP

.gitlab-ci.yml

+259 −54

Original line number	Diff line number	Diff line
		@@ -21,14 +21,18 @@ variables:
		TESTCASE_TIMEOUT_STV: 900
		TESTCASE_TIMEOUT_LTV: 2400
		TESTCASE_TIMEOUT_STV_SANITIZERS: 1800
		CI_REGRESSION_THRESH_MLD: "0.1"
		CI_REGRESSION_THRESH_MAX_ABS_DIFF: "50"
		CI_REGRESSION_THRESH_SSNR: "-1"
		CI_REGRESSION_THRESH_ODG: "-0.05"
		GIT_CLEAN_FLAGS: -ffdxq
		MANUAL_PIPELINE_TYPE:
		description: "Type for the manual pipeline run. Use 'pytest-mld' to run MLD test against reference float codec." # Not implemented yet, but may be good to have a manual pipeline trigger
		description: "Type for the manual pipeline run. Use 'pytest-compare' to run comparison test against reference float codec."
		value: 'default'
		options:
		- 'default'
		- 'pytest-mld'
		- 'pytest-mld-long'
		- 'pytest-compare'
		- 'pytest-compare-long'
		- 'pytest-saturation-smoke-test'
		- 'evs-26444'
		- 'sanitizer-stv'
		@@ -54,12 +58,12 @@ workflow:
		- if: $CI_PIPELINE_SOURCE == 'web' && $MANUAL_PIPELINE_TYPE == 'default' # for testing
		variables:
		IVAS_PIPELINE_NAME: 'Web run pipeline: $CI_COMMIT_BRANCH'
		- if: $CI_PIPELINE_SOURCE == 'web' && $MANUAL_PIPELINE_TYPE == 'pytest-mld'
		- if: $CI_PIPELINE_SOURCE == 'web' && $MANUAL_PIPELINE_TYPE == 'pytest-compare'
		variables:
		IVAS_PIPELINE_NAME: 'Run MLD tool against float ref: $CI_COMMIT_BRANCH'
		- if: $CI_PIPELINE_SOURCE == 'web' && $MANUAL_PIPELINE_TYPE == 'pytest-mld-long'
		IVAS_PIPELINE_NAME: 'Run comparison tools against float ref: $CI_COMMIT_BRANCH'
		- if: $CI_PIPELINE_SOURCE == 'web' && $MANUAL_PIPELINE_TYPE == 'pytest-compare-long'
		variables:
		IVAS_PIPELINE_NAME: 'Run MLD tool against float ref (long test vectors): $CI_COMMIT_BRANCH'
		IVAS_PIPELINE_NAME: 'Run comparison tools against float ref (long test vectors): $CI_COMMIT_BRANCH'
		- if: $CI_PIPELINE_SOURCE == 'web' && $MANUAL_PIPELINE_TYPE == 'pytest-saturation-smoke-test'
		variables:
		IVAS_PIPELINE_NAME: 'Run saturation smoke-test: $CI_COMMIT_BRANCH'
		@@ -97,7 +101,7 @@ stages:
		echo "Commit time was $CI_COMMIT_TIMESTAMP"
		date \| xargs echo "System time is"

		.setup-codec: &setup-codec
		.build-reference-and-dut-binaries: &build-reference-and-dut-binaries
		- current_commit_sha=$(git rev-parse HEAD)
		### build reference binaries
		- git checkout $REFERENCE_BRANCH
		@@ -113,9 +117,8 @@ stages:
		- make clean
		- make -j


		.mld-test-setup-codec: &mld-test-setup-codec
		- *setup-codec
		.build-and-create-reference-outputs: &build-and-create-reference-outputs
		- *build-reference-and-dut-binaries

		### prepare pytest
		# create short test vectors
		@@ -157,9 +160,19 @@ stages:
		.activate-Werror-linux: &activate-Werror-linux
		- sed -i.bak "s/^# $CFLAGS += -Werror$/\1/" Makefile

		.rules-pytest-mld:
		.rules-pytest-to-ref-short:
		rules:
		- if: $CI_PIPELINE_SOURCE == 'web' && $MANUAL_PIPELINE_TYPE == "pytest-compare"
		- if: $CI_PIPELINE_SOURCE == 'push' && $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH
		- if: $CI_PIPELINE_SOURCE == 'push'
		when: never
		- if: $CI_PIPELINE_SOURCE == 'schedule'
		when: never

		# TODO: only temporary as long the MR encoder tests should not compare to main
		.rules-pytest-to-ref-enc-short-temp:
		rules:
		- if: $CI_PIPELINE_SOURCE == 'web' && $MANUAL_PIPELINE_TYPE == "pytest-mld"
		- if: $CI_PIPELINE_SOURCE == 'web' && $MANUAL_PIPELINE_TYPE == "pytest-compare"
		- if: $CI_PIPELINE_SOURCE == 'push' && $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH
		- if: $CI_PIPELINE_SOURCE == 'merge_request_event' && $CI_MERGE_REQUEST_TARGET_BRANCH_NAME == "main" # only have MR pipelines for MRs to main
		- if: $CI_PIPELINE_SOURCE == 'push'
		@@ -167,10 +180,16 @@ stages:
		- if: $CI_PIPELINE_SOURCE == 'schedule'
		when: never

		.rules-pytest-mld-long:
		.rules-pytest-to-main-short:
		rules:
		- if: $CI_PIPELINE_SOURCE == 'merge_request_event' && $CI_MERGE_REQUEST_TARGET_BRANCH_NAME == "main" # only have MR pipelines for MRs to main
		- if: $CI_PIPELINE_SOURCE == 'push'
		when: never

		.rules-pytest-long:
		rules:
		- if: $PYTEST_MLD_LONG # Set by scheduled pipeline
		- if: $CI_PIPELINE_SOURCE == 'web' && $MANUAL_PIPELINE_TYPE == "pytest-mld-long"
		- if: $CI_PIPELINE_SOURCE == 'web' && $MANUAL_PIPELINE_TYPE == "pytest-compare-long"
		- if: $CI_PIPELINE_SOURCE == 'push'
		when: never
		- if: $CI_PIPELINE_SOURCE == 'merge_request_event'
		@@ -214,7 +233,8 @@ stages:
		needs: ["build-codec-linux-make"]
		timeout: "240 minutes"
		variables:
		MLD_ARTIFACT_NAME: "mld--$CI_JOB_NAME-$CI_JOB_ID--sha-$CI_COMMIT_SHORT_SHA.csv"
		# keep "mld" in artifact name for backwards compatibility reasons
		CSV_ARTIFACT_NAME: "mld--$CI_JOB_NAME-$CI_JOB_ID--sha-$CI_COMMIT_SHORT_SHA.csv"
		MERGED_CSV_ARTIFACT_NAME: "$CI_JOB_NAME--merged_csv--$CI_JOB_ID.csv"
		PAGES_HTML_ARTIFACT_NAME: "$CI_JOB_NAME-index.html"
		SUMMARY_HTML_ARTIFACT_NAME: "summary_$CI_JOB_NAME.html"
		@@ -229,20 +249,21 @@ stages:
		- else
		- testcase_timeout=$TESTCASE_TIMEOUT_STV
		- fi

		- python3 ci/remove_unsupported_testcases.py $PRM_FILES
		- if [ $LEVEL_SCALING != "1.0" ];then
		- *apply-testv-scaling
		- fi
		- *mld-test-setup-codec
		- *build-and-create-reference-outputs

		### run pytest
		- exit_code=0
		- python3 -m pytest --tb=no $TEST_SUITE -v --create_cut --html=report.html --self-contained-html --junit-xml=report-junit.xml --mld --ssnr --odg -n auto --testcase_timeout $testcase_timeout --ref_encoder_path $REF_ENCODER_PATH --ref_decoder_path $REF_DECODER_PATH --dut_encoder_path $DUT_ENCODER_PATH --dut_decoder_path $DUT_DECODER_PATH \|\| exit_code=$?
		- zero_errors=$(cat report-junit.xml \| grep -c 'errors="0"') \|\| true

		- python3 scripts/parse_xml_report.py report-junit.xml $MLD_ARTIFACT_NAME
		- python3 scripts/parse_xml_report.py report-junit.xml $CSV_ARTIFACT_NAME
		- mkdir $IMAGES_ARTIFACT_NAME
		- for MEASURE in MLD DIFF SSNR;do python3 scripts/create_histogram_summary.py $MLD_ARTIFACT_NAME images_"$CI_JOB_NAME"/summary_"$MEASURE".csv images_"$CI_JOB_NAME"/summary_"$MEASURE".png --measure $MEASURE; done
		- for MEASURE in MLD DIFF SSNR;do python3 scripts/create_histogram_summary.py $CSV_ARTIFACT_NAME $IMAGES_ARTIFACT_NAME/summary_"$MEASURE".csv $IMAGES_ARTIFACT_NAME/summary_"$MEASURE".png --measure $MEASURE; done
		- python3 ci/basop-pages/create_summary_page.py $SUMMARY_HTML_ARTIFACT_NAME $CI_JOB_ID $CI_JOB_NAME

		- if [ $USE_LTV -eq 1 ] && [ "$CI_COMMIT_BRANCH" == "$CI_DEFAULT_BRANCH" ]; then
		@@ -252,14 +273,22 @@ stages:
		- unzip artifacts.zip -d previous_artifacts
		# This wildcard thingy relies on only one csv file being present per job
		- file_previous="previous_artifacts/mld--$CI_JOB_NAME-$id_previous--sha-*.csv"
		- python3 ci/basop-pages/create_report_pages.py $PAGES_HTML_ARTIFACT_NAME $MERGED_CSV_ARTIFACT_NAME $MLD_ARTIFACT_NAME $file_previous $CI_JOB_ID $id_previous $CI_JOB_NAME
		- python3 ci/basop-pages/create_report_pages.py $PAGES_HTML_ARTIFACT_NAME $MERGED_CSV_ARTIFACT_NAME $CSV_ARTIFACT_NAME $file_previous $CI_JOB_ID $id_previous $CI_JOB_NAME
		- else
		# create empty file for artifacts to avoid errors
		- touch $PAGES_HTML_ARTIFACT_NAME
		- touch $MERGED_CSV_ARTIFACT_NAME
		- fi

		- if [ $zero_errors != 1 ]; then echo "Run errors encountered!"; exit $EXIT_CODE_FAIL; fi
		- if [ $zero_errors != 1 ]; then
		- echo "Run errors encountered!"
		# TODO: temporary only to not fail pipelines on crashes in encoder
		- if [[ "$CI_JOB_NAME" =~ "ivas-pytest-compare_to_ref-short-enc" ]]; then
		- exit $EXIT_CODE_NON_BE
		- else
		- exit $EXIT_CODE_FAIL
		- fi
		- fi
		- if [ $exit_code -eq 1 ]; then echo "Differences encountered"; exit $EXIT_CODE_NON_BE; fi
		- exit 0

		@@ -274,16 +303,113 @@ stages:
		- report-junit.xml
		- report.html
		- $PAGES_HTML_ARTIFACT_NAME
		- $MLD_ARTIFACT_NAME
		- $CSV_ARTIFACT_NAME
		- $MERGED_CSV_ARTIFACT_NAME
		- images
		- $SUMMARY_HTML_ARTIFACT_NAME
		- $IMAGES_ARTIFACT_NAME
		expose_as: "pytest mld results"
		expose_as: "pytest compare results"
		reports:
		junit:
		- report-junit.xml

		.ivas-pytest-on-merge-request-anchor: &ivas-pytest-on-merge-request-anchor
		stage: test
		needs: ["build-codec-linux-make"]
		timeout: "300 minutes"
		variables:
		XML_REPORT_BRANCH: "report-junit-branch-$CI_JOB_NAME--sha-$CI_COMMIT_SHORT_SHA.xml"
		XML_REPORT_MAIN: "report-junit-main-$CI_JOB_NAME--sha-$CI_COMMIT_SHORT_SHA.xml"
		HTML_REPORT_BRANCH: "report-junit-branch-$CI_JOB_NAME--sha-$CI_COMMIT_SHORT_SHA.html"
		HTML_REPORT_MAIN: "report-junit-main-$CI_JOB_NAME--sha-$CI_COMMIT_SHORT_SHA.html"
		CSV_BRANCH: "scores-branch-$CI_JOB_NAME--sha-$CI_COMMIT_SHORT_SHA.csv"
		CSV_MAIN: "scores-main-$CI_JOB_NAME--sha-$CI_COMMIT_SHORT_SHA.csv"
		IMAGES_ARTIFACT_NAME: "images_$CI_JOB_NAME"
		SUMMARY_HTML_ARTIFACT_NAME: "summary_$CI_JOB_NAME.html"
		script:
		- *print-common-info
		- *update-scripts-repo
		- if [ $USE_LTV -eq 1 ]; then
		- *update-ltv-repo
		- *copy-ltv-files-to-testv-dir
		- testcase_timeout=$TESTCASE_TIMEOUT_LTV
		- else
		- testcase_timeout=$TESTCASE_TIMEOUT_STV
		- fi

		- python3 ci/remove_unsupported_testcases.py $PRM_FILES
		- if [ $LEVEL_SCALING != "1.0" ];then
		- *apply-testv-scaling
		- fi

		# check MR title for flag that allows regressions to be mergable
		- echo $CI_MERGE_REQUEST_TITLE > tmp.txt
		- allow_regressions_flag=$(grep -c --ignore-case "\[allow[ -]*regression\]" tmp.txt) \|\| true

		### run branch first
		# this per default builds the branch and the reference and creates the reference outputs
		- *build-and-create-reference-outputs
		- exit_code=0
		- python3 -m pytest --tb=no $TEST_SUITE -v --create_cut --html=$HTML_REPORT_BRANCH --self-contained-html --junit-xml=$XML_REPORT_BRANCH --mld --ssnr --odg --ref_encoder_path $REF_ENCODER_PATH --ref_decoder_path $REF_DECODER_PATH --dut_encoder_path $DUT_ENCODER_PATH --dut_decoder_path $DUT_DECODER_PATH -n auto --testcase_timeout $testcase_timeout \|\| exit_code=$?
		- zero_errors_branch=$(cat $XML_REPORT_BRANCH \| grep -c 'errors="0"') \|\| true
		- python3 scripts/parse_xml_report.py $XML_REPORT_BRANCH $CSV_BRANCH

		# create the summary based on the branch
		- mkdir $IMAGES_ARTIFACT_NAME
		- for MEASURE in MLD DIFF SSNR;do python3 scripts/create_histogram_summary.py $CSV_BRANCH $IMAGES_ARTIFACT_NAME/summary_"$MEASURE".csv $IMAGES_ARTIFACT_NAME/summary_"$MEASURE".png --measure $MEASURE; done
		- python3 ci/basop-pages/create_summary_page.py $SUMMARY_HTML_ARTIFACT_NAME $CI_JOB_ID $CI_JOB_NAME

		### run main now
		- git checkout $CI_MERGE_REQUEST_TARGET_BRANCH_NAME
		- git pull
		- make clean
		- make -j
		- python3 -m pytest --tb=no $TEST_SUITE -v --create_cut --html=$HTML_REPORT_MAIN --self-contained-html --junit-xml=$XML_REPORT_MAIN --mld --ssnr --odg --ref_encoder_path $REF_ENCODER_PATH --ref_decoder_path $REF_DECODER_PATH --dut_encoder_path $DUT_ENCODER_PATH --dut_decoder_path $DUT_DECODER_PATH -n auto --testcase_timeout $testcase_timeout \|\| true
		- python3 scripts/parse_xml_report.py $XML_REPORT_MAIN $CSV_MAIN

		### compare the two csv files for regressions
		- regressions_found=0
		- python3 scripts/basop_check_for_changes_in_testcases.py $CSV_BRANCH $CSV_MAIN \|\| regressions_found=$?

		- if [ $regressions_found != 0 ]; then
		- echo "Detected regression wrt to main!"
		- if [ $allow_regressions_flag == 0 ]; then
		- exit $EXIT_CODE_FAIL;
		- else
		- exit $EXIT_CODE_NON_BE;
		- fi
		- fi
		- if [ $zero_errors_branch != 1 ]; then echo "Run errors encountered!"; exit $EXIT_CODE_NON_BE; fi
		- if [ $exit_code -eq 1 ]; then echo "Differences encountered"; exit $EXIT_CODE_NON_BE; fi
		- exit 0
		after_script:
		- rm -rf tests/dut tests/ref
		allow_failure:
		exit_codes:
		- 123
		artifacts:
		name: "$CI_JOB_NAME--sha-$CI_COMMIT_SHORT_SHA--results"
		expire_in: 1 week
		when: always
		paths:
		- $XML_REPORT_BRANCH
		- $XML_REPORT_MAIN
		- $HTML_REPORT_BRANCH
		- $HTML_REPORT_MAIN
		- $CSV_BRANCH
		- $CSV_MAIN
		- $SUMMARY_HTML_ARTIFACT_NAME
		- $IMAGES_ARTIFACT_NAME
		- changes_crashes.csv
		- changes_MLD.csv
		- changes_MAXIMUM_ABS_DIFF.csv
		- changes_MIN_SSNR.csv
		- changes_MIN_ODG.csv
		expose_as: "pytest compare results"
		reports:
		junit:
		- $XML_REPORT_BRANCH
		- $XML_REPORT_MAIN


		.ivas-pytest-sanitizers-anchor: &ivas-pytest-sanitizers-anchor
		stage: test
		@@ -300,7 +426,7 @@ stages:
		- *print-common-info
		- *update-scripts-repo
		- python3 ci/remove_unsupported_testcases.py $PRM_FILES
		- *setup-codec
		- *build-reference-and-dut-binaries
		- make clean
		- make -j CLANG=$CLANG_NUM
		- if [[ $CLANG_NUM == 3 ]]; then export UBSAN_OPTIONS="suppressions=scripts/ubsan.supp,report_error_type=1"; fi
		@@ -410,13 +536,90 @@ build-codec-linux-instrumented-make:
		- make -j

		# ---------------------------------------------------------------
		# Short test jobs
		# Short test jobs that run in merge request pipelines
		# ---------------------------------------------------------------

		### jobs that test fx encoder -> flt decoder
		# TODO: reenable once encoder tests shall compare to main
		# ivas-pytest-compare_to_main-short-enc:
		# extends:
		# - .rules-pytest-to-main-short
		# - .test-job-linux
		# before_script:
		# - USE_LTV=0
		# - DUT_DECODER_PATH=./IVAS_dec_ref
		# - TEST_SUITE="$SHORT_TEST_SUITE"
		# - LEVEL_SCALING=1.0
		# <<: *ivas-pytest-on-merge-request-anchor

		# ivas-pytest-compare_to_main-short-enc-lev-10:
		# extends:
		# - .rules-pytest-to-main-short
		# - .test-job-linux
		# before_script:
		# - USE_LTV=0
		# - DUT_DECODER_PATH=./IVAS_dec_ref
		# - TEST_SUITE="$SHORT_TEST_SUITE"
		# - LEVEL_SCALING=0.3162
		# <<: *ivas-pytest-on-merge-request-anchor

		# ivas-pytest-compare_to_main-short-enc-lev+10:
		# extends:
		# - .rules-pytest-to-main-short
		# - .test-job-linux
		# before_script:
		# - USE_LTV=0
		# - DUT_DECODER_PATH=./IVAS_dec_ref
		# - TEST_SUITE="$SHORT_TEST_SUITE"
		# - LEVEL_SCALING=3.162
		# <<: *ivas-pytest-on-merge-request-anchor

		### jobs that test flt encoder -> fx decoder
		ivas-pytest-compare_to_main-short-dec:
		extends:
		- .rules-pytest-to-main-short
		- .test-job-linux
		before_script:
		- USE_LTV=0
		- DUT_ENCODER_PATH=./IVAS_cod_ref
		- TEST_SUITE="$SHORT_TEST_SUITE"
		- LEVEL_SCALING=1.0
		- rm -rf tests/dut tests/ref
		<<: *ivas-pytest-on-merge-request-anchor

		ivas-pytest-compare_to_main-short-dec-lev-10:
		extends:
		- .rules-pytest-to-main-short
		- .test-job-linux
		before_script:
		- USE_LTV=0
		- DUT_ENCODER_PATH=./IVAS_cod_ref
		- TEST_SUITE="$SHORT_TEST_SUITE"
		- LEVEL_SCALING=0.3162
		- rm -rf tests/dut tests/ref
		<<: *ivas-pytest-on-merge-request-anchor

		ivas-pytest-compare_to_main-short-dec-lev+10:
		extends:
		- .rules-pytest-to-main-short
		- .test-job-linux
		before_script:
		- USE_LTV=0
		- DUT_ENCODER_PATH=./IVAS_cod_ref
		- TEST_SUITE="$SHORT_TEST_SUITE"
		- LEVEL_SCALING=3.162
		- rm -rf tests/dut tests/ref
		<<: *ivas-pytest-on-merge-request-anchor

		# ---------------------------------------------------------------
		# Short test jobs for running from web interface or schedule
		# ---------------------------------------------------------------

		### jobs that test fx encoder -> flt decoder
		ivas-pytest-mld-enc:
		ivas-pytest-compare_to_ref-short-enc:
		extends:
		- .rules-pytest-mld
		#- .rules-pytest-to-ref-short
		- .rules-pytest-to-ref-enc-short-temp
		- .test-job-linux
		before_script:
		- USE_LTV=0
		@@ -425,9 +628,10 @@ ivas-pytest-mld-enc:
		- LEVEL_SCALING=1.0
		<<: *ivas-pytest-anchor

		ivas-pytest-mld-enc-lev-10:
		ivas-pytest-compare_to_ref-short-enc-lev-10:
		extends:
		- .rules-pytest-mld
		#- .rules-pytest-to-ref-short
		- .rules-pytest-to-ref-enc-short-temp
		- .test-job-linux
		before_script:
		- USE_LTV=0
		@@ -436,9 +640,10 @@ ivas-pytest-mld-enc-lev-10:
		- LEVEL_SCALING=0.3162
		<<: *ivas-pytest-anchor

		ivas-pytest-mld-enc-lev+10:
		ivas-pytest-compare_to_ref-short-enc-lev+10:
		extends:
		- .rules-pytest-mld
		#- .rules-pytest-to-ref-short
		- .rules-pytest-to-ref-enc-short-temp
		- .test-job-linux
		before_script:
		- USE_LTV=0
		@@ -472,9 +677,9 @@ ivas-pytest-enc-usan:
		<<: *ivas-pytest-sanitizers-anchor

		### jobs that test flt encoder -> fx decoder
		ivas-pytest-mld-dec:
		ivas-pytest-compare_to_ref-short-dec:
		extends:
		- .rules-pytest-mld
		- .rules-pytest-to-ref-short
		- .test-job-linux
		before_script:
		- USE_LTV=0
		@@ -483,9 +688,9 @@ ivas-pytest-mld-dec:
		- LEVEL_SCALING=1.0
		<<: *ivas-pytest-anchor

		ivas-pytest-mld-dec-lev-10:
		ivas-pytest-compare_to_ref-short-lev-10:
		extends:
		- .rules-pytest-mld
		- .rules-pytest-to-ref-short
		- .test-job-linux
		before_script:
		- USE_LTV=0
		@@ -494,9 +699,9 @@ ivas-pytest-mld-dec-lev-10:
		- LEVEL_SCALING=0.3162
		<<: *ivas-pytest-anchor

		ivas-pytest-mld-dec-lev+10:
		ivas-pytest-compare_to_ref-short-dec-lev+10:
		extends:
		- .rules-pytest-mld
		- .rules-pytest-to-ref-short
		- .test-job-linux
		before_script:
		- USE_LTV=0
		@@ -505,7 +710,7 @@ ivas-pytest-mld-dec-lev+10:
		- LEVEL_SCALING=3.162
		<<: *ivas-pytest-anchor

		ivas-pytest-dec-msan:
		ivas-pytest-dec-msan-short:
		extends:
		- .test-job-linux
		before_script:
		@@ -513,7 +718,7 @@ ivas-pytest-dec-msan:
		- DUT_ENCODER_PATH=./$REF_ENCODER_PATH
		<<: *ivas-pytest-sanitizers-anchor

		ivas-pytest-dec-asan:
		ivas-pytest-dec-asan-short:
		extends:
		- .test-job-linux
		before_script:
		@@ -521,7 +726,7 @@ ivas-pytest-dec-asan:
		- DUT_ENCODER_PATH=./$REF_ENCODER_PATH
		<<: *ivas-pytest-sanitizers-anchor

		ivas-pytest-dec-usan:
		ivas-pytest-dec-usan-short:
		extends:
		- .test-job-linux
		before_script:
		@@ -533,9 +738,9 @@ ivas-pytest-dec-usan:
		# Long test jobs
		# ---------------------------------------------------------------

		ivas-pytest-mld-long-enc:
		ivas-pytest-compare_ref-long-enc:
		extends:
		- .rules-pytest-mld-long
		- .rules-pytest-long
		- .test-job-linux
		before_script:
		- USE_LTV=1
		@@ -544,9 +749,9 @@ ivas-pytest-mld-long-enc:
		- LEVEL_SCALING=1.0
		<<: *ivas-pytest-anchor

		ivas-pytest-mld-long-dec:
		ivas-pytest-compare_ref-long-dec:
		extends:
		- .rules-pytest-mld-long
		- .rules-pytest-long
		- .test-job-linux
		before_script:
		- USE_LTV=1
		@@ -555,9 +760,9 @@ ivas-pytest-mld-long-dec:
		- LEVEL_SCALING=1.0
		<<: *ivas-pytest-anchor

		ivas-pytest-mld-long-enc-lev-10:
		ivas-pytest-compare_ref-long-enc-lev-10:
		extends:
		- .rules-pytest-mld-long
		- .rules-pytest-long
		- .test-job-linux
		before_script:
		- USE_LTV=1
		@@ -566,9 +771,9 @@ ivas-pytest-mld-long-enc-lev-10:
		- LEVEL_SCALING=0.3162
		<<: *ivas-pytest-anchor

		ivas-pytest-mld-long-dec-lev-10:
		ivas-pytest-compare_ref-long-dec-lev-10:
		extends:
		- .rules-pytest-mld-long
		- .rules-pytest-long
		- .test-job-linux
		before_script:
		- USE_LTV=1
		@@ -577,9 +782,9 @@ ivas-pytest-mld-long-dec-lev-10:
		- LEVEL_SCALING=0.3162
		<<: *ivas-pytest-anchor

		ivas-pytest-mld-long-enc-lev+10:
		ivas-pytest-compare_ref-long-enc-lev+10:
		extends:
		- .rules-pytest-mld-long
		- .rules-pytest-long
		- .test-job-linux
		before_script:
		- USE_LTV=1
		@@ -588,9 +793,9 @@ ivas-pytest-mld-long-enc-lev+10:
		- LEVEL_SCALING=3.162
		<<: *ivas-pytest-anchor

		ivas-pytest-mld-long-dec-lev+10:
		ivas-pytest-compare_ref-long-dec-lev+10:
		extends:
		- .rules-pytest-mld-long
		- .rules-pytest-long
		- .test-job-linux
		before_script:
		- USE_LTV=1