diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 78c4d078fcbf882ab53a80688a096619ad4a89e0..6a77da6faa189ff5f89f4143798c8cbfd755cf80 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -21,14 +21,18 @@ variables: TESTCASE_TIMEOUT_STV: 900 TESTCASE_TIMEOUT_LTV: 2400 TESTCASE_TIMEOUT_STV_SANITIZERS: 1800 + CI_REGRESSION_THRESH_MLD: "0.1" + CI_REGRESSION_THRESH_MAX_ABS_DIFF: "50" + CI_REGRESSION_THRESH_SSNR: "-1" + CI_REGRESSION_THRESH_ODG: "-0.05" GIT_CLEAN_FLAGS: -ffdxq MANUAL_PIPELINE_TYPE: - description: "Type for the manual pipeline run. Use 'pytest-mld' to run MLD test against reference float codec." # Not implemented yet, but may be good to have a manual pipeline trigger + description: "Type for the manual pipeline run. Use 'pytest-compare' to run comparison test against reference float codec." value: 'default' options: - 'default' - - 'pytest-mld' - - 'pytest-mld-long' + - 'pytest-compare' + - 'pytest-compare-long' - 'pytest-saturation-smoke-test' - 'evs-26444' - 'sanitizer-stv' @@ -54,12 +58,12 @@ workflow: - if: $CI_PIPELINE_SOURCE == 'web' && $MANUAL_PIPELINE_TYPE == 'default' # for testing variables: IVAS_PIPELINE_NAME: 'Web run pipeline: $CI_COMMIT_BRANCH' - - if: $CI_PIPELINE_SOURCE == 'web' && $MANUAL_PIPELINE_TYPE == 'pytest-mld' + - if: $CI_PIPELINE_SOURCE == 'web' && $MANUAL_PIPELINE_TYPE == 'pytest-compare' variables: - IVAS_PIPELINE_NAME: 'Run MLD tool against float ref: $CI_COMMIT_BRANCH' - - if: $CI_PIPELINE_SOURCE == 'web' && $MANUAL_PIPELINE_TYPE == 'pytest-mld-long' + IVAS_PIPELINE_NAME: 'Run comparison tools against float ref: $CI_COMMIT_BRANCH' + - if: $CI_PIPELINE_SOURCE == 'web' && $MANUAL_PIPELINE_TYPE == 'pytest-compare-long' variables: - IVAS_PIPELINE_NAME: 'Run MLD tool against float ref (long test vectors): $CI_COMMIT_BRANCH' + IVAS_PIPELINE_NAME: 'Run comparison tools against float ref (long test vectors): $CI_COMMIT_BRANCH' - if: $CI_PIPELINE_SOURCE == 'web' && $MANUAL_PIPELINE_TYPE == 'pytest-saturation-smoke-test' variables: IVAS_PIPELINE_NAME: 'Run saturation smoke-test: $CI_COMMIT_BRANCH' @@ -97,7 +101,7 @@ stages: echo "Commit time was $CI_COMMIT_TIMESTAMP" date | xargs echo "System time is" -.setup-codec: &setup-codec +.build-reference-and-dut-binaries: &build-reference-and-dut-binaries - current_commit_sha=$(git rev-parse HEAD) ### build reference binaries - git checkout $REFERENCE_BRANCH @@ -113,9 +117,8 @@ stages: - make clean - make -j - -.mld-test-setup-codec: &mld-test-setup-codec - - *setup-codec +.build-and-create-reference-outputs: &build-and-create-reference-outputs + - *build-reference-and-dut-binaries ### prepare pytest # create short test vectors @@ -157,9 +160,19 @@ stages: .activate-Werror-linux: &activate-Werror-linux - sed -i.bak "s/^# \(CFLAGS += -Werror\)/\1/" Makefile -.rules-pytest-mld: +.rules-pytest-to-ref-short: rules: - - if: $CI_PIPELINE_SOURCE == 'web' && $MANUAL_PIPELINE_TYPE == "pytest-mld" + - if: $CI_PIPELINE_SOURCE == 'web' && $MANUAL_PIPELINE_TYPE == "pytest-compare" + - if: $CI_PIPELINE_SOURCE == 'push' && $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH + - if: $CI_PIPELINE_SOURCE == 'push' + when: never + - if: $CI_PIPELINE_SOURCE == 'schedule' + when: never + +# TODO: only temporary as long the MR encoder tests should not compare to main +.rules-pytest-to-ref-enc-short-temp: + rules: + - if: $CI_PIPELINE_SOURCE == 'web' && $MANUAL_PIPELINE_TYPE == "pytest-compare" - if: $CI_PIPELINE_SOURCE == 'push' && $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH - if: $CI_PIPELINE_SOURCE == 'merge_request_event' && $CI_MERGE_REQUEST_TARGET_BRANCH_NAME == "main" # only have MR pipelines for MRs to main - if: $CI_PIPELINE_SOURCE == 'push' @@ -167,10 +180,16 @@ stages: - if: $CI_PIPELINE_SOURCE == 'schedule' when: never -.rules-pytest-mld-long: +.rules-pytest-to-main-short: + rules: + - if: $CI_PIPELINE_SOURCE == 'merge_request_event' && $CI_MERGE_REQUEST_TARGET_BRANCH_NAME == "main" # only have MR pipelines for MRs to main + - if: $CI_PIPELINE_SOURCE == 'push' + when: never + +.rules-pytest-long: rules: - if: $PYTEST_MLD_LONG # Set by scheduled pipeline - - if: $CI_PIPELINE_SOURCE == 'web' && $MANUAL_PIPELINE_TYPE == "pytest-mld-long" + - if: $CI_PIPELINE_SOURCE == 'web' && $MANUAL_PIPELINE_TYPE == "pytest-compare-long" - if: $CI_PIPELINE_SOURCE == 'push' when: never - if: $CI_PIPELINE_SOURCE == 'merge_request_event' @@ -214,7 +233,8 @@ stages: needs: ["build-codec-linux-make"] timeout: "240 minutes" variables: - MLD_ARTIFACT_NAME: "mld--$CI_JOB_NAME-$CI_JOB_ID--sha-$CI_COMMIT_SHORT_SHA.csv" + # keep "mld" in artifact name for backwards compatibility reasons + CSV_ARTIFACT_NAME: "mld--$CI_JOB_NAME-$CI_JOB_ID--sha-$CI_COMMIT_SHORT_SHA.csv" MERGED_CSV_ARTIFACT_NAME: "$CI_JOB_NAME--merged_csv--$CI_JOB_ID.csv" PAGES_HTML_ARTIFACT_NAME: "$CI_JOB_NAME-index.html" SUMMARY_HTML_ARTIFACT_NAME: "summary_$CI_JOB_NAME.html" @@ -229,20 +249,21 @@ stages: - else - testcase_timeout=$TESTCASE_TIMEOUT_STV - fi + - python3 ci/remove_unsupported_testcases.py $PRM_FILES - if [ $LEVEL_SCALING != "1.0" ];then - *apply-testv-scaling - fi - - *mld-test-setup-codec + - *build-and-create-reference-outputs ### run pytest - exit_code=0 - python3 -m pytest --tb=no $TEST_SUITE -v --create_cut --html=report.html --self-contained-html --junit-xml=report-junit.xml --mld --ssnr --odg -n auto --testcase_timeout $testcase_timeout --ref_encoder_path $REF_ENCODER_PATH --ref_decoder_path $REF_DECODER_PATH --dut_encoder_path $DUT_ENCODER_PATH --dut_decoder_path $DUT_DECODER_PATH || exit_code=$? - zero_errors=$(cat report-junit.xml | grep -c 'errors="0"') || true - - python3 scripts/parse_xml_report.py report-junit.xml $MLD_ARTIFACT_NAME + - python3 scripts/parse_xml_report.py report-junit.xml $CSV_ARTIFACT_NAME - mkdir $IMAGES_ARTIFACT_NAME - - for MEASURE in MLD DIFF SSNR;do python3 scripts/create_histogram_summary.py $MLD_ARTIFACT_NAME images_"$CI_JOB_NAME"/summary_"$MEASURE".csv images_"$CI_JOB_NAME"/summary_"$MEASURE".png --measure $MEASURE; done + - for MEASURE in MLD DIFF SSNR;do python3 scripts/create_histogram_summary.py $CSV_ARTIFACT_NAME $IMAGES_ARTIFACT_NAME/summary_"$MEASURE".csv $IMAGES_ARTIFACT_NAME/summary_"$MEASURE".png --measure $MEASURE; done - python3 ci/basop-pages/create_summary_page.py $SUMMARY_HTML_ARTIFACT_NAME $CI_JOB_ID $CI_JOB_NAME - if [ $USE_LTV -eq 1 ] && [ "$CI_COMMIT_BRANCH" == "$CI_DEFAULT_BRANCH" ]; then @@ -252,14 +273,22 @@ stages: - unzip artifacts.zip -d previous_artifacts # This wildcard thingy relies on only one csv file being present per job - file_previous="previous_artifacts/mld--$CI_JOB_NAME-$id_previous--sha-*.csv" - - python3 ci/basop-pages/create_report_pages.py $PAGES_HTML_ARTIFACT_NAME $MERGED_CSV_ARTIFACT_NAME $MLD_ARTIFACT_NAME $file_previous $CI_JOB_ID $id_previous $CI_JOB_NAME + - python3 ci/basop-pages/create_report_pages.py $PAGES_HTML_ARTIFACT_NAME $MERGED_CSV_ARTIFACT_NAME $CSV_ARTIFACT_NAME $file_previous $CI_JOB_ID $id_previous $CI_JOB_NAME - else # create empty file for artifacts to avoid errors - touch $PAGES_HTML_ARTIFACT_NAME - touch $MERGED_CSV_ARTIFACT_NAME - fi - - if [ $zero_errors != 1 ]; then echo "Run errors encountered!"; exit $EXIT_CODE_FAIL; fi + - if [ $zero_errors != 1 ]; then + - echo "Run errors encountered!" + # TODO: temporary only to not fail pipelines on crashes in encoder + - if [[ "$CI_JOB_NAME" =~ "ivas-pytest-compare_to_ref-short-enc" ]]; then + - exit $EXIT_CODE_NON_BE + - else + - exit $EXIT_CODE_FAIL + - fi + - fi - if [ $exit_code -eq 1 ]; then echo "Differences encountered"; exit $EXIT_CODE_NON_BE; fi - exit 0 @@ -274,16 +303,113 @@ stages: - report-junit.xml - report.html - $PAGES_HTML_ARTIFACT_NAME - - $MLD_ARTIFACT_NAME + - $CSV_ARTIFACT_NAME - $MERGED_CSV_ARTIFACT_NAME - - images - $SUMMARY_HTML_ARTIFACT_NAME - $IMAGES_ARTIFACT_NAME - expose_as: "pytest mld results" + expose_as: "pytest compare results" reports: junit: - report-junit.xml - + +.ivas-pytest-on-merge-request-anchor: &ivas-pytest-on-merge-request-anchor + stage: test + needs: ["build-codec-linux-make"] + timeout: "300 minutes" + variables: + XML_REPORT_BRANCH: "report-junit-branch-$CI_JOB_NAME--sha-$CI_COMMIT_SHORT_SHA.xml" + XML_REPORT_MAIN: "report-junit-main-$CI_JOB_NAME--sha-$CI_COMMIT_SHORT_SHA.xml" + HTML_REPORT_BRANCH: "report-junit-branch-$CI_JOB_NAME--sha-$CI_COMMIT_SHORT_SHA.html" + HTML_REPORT_MAIN: "report-junit-main-$CI_JOB_NAME--sha-$CI_COMMIT_SHORT_SHA.html" + CSV_BRANCH: "scores-branch-$CI_JOB_NAME--sha-$CI_COMMIT_SHORT_SHA.csv" + CSV_MAIN: "scores-main-$CI_JOB_NAME--sha-$CI_COMMIT_SHORT_SHA.csv" + IMAGES_ARTIFACT_NAME: "images_$CI_JOB_NAME" + SUMMARY_HTML_ARTIFACT_NAME: "summary_$CI_JOB_NAME.html" + script: + - *print-common-info + - *update-scripts-repo + - if [ $USE_LTV -eq 1 ]; then + - *update-ltv-repo + - *copy-ltv-files-to-testv-dir + - testcase_timeout=$TESTCASE_TIMEOUT_LTV + - else + - testcase_timeout=$TESTCASE_TIMEOUT_STV + - fi + + - python3 ci/remove_unsupported_testcases.py $PRM_FILES + - if [ $LEVEL_SCALING != "1.0" ];then + - *apply-testv-scaling + - fi + + # check MR title for flag that allows regressions to be mergable + - echo $CI_MERGE_REQUEST_TITLE > tmp.txt + - allow_regressions_flag=$(grep -c --ignore-case "\[allow[ -]*regression\]" tmp.txt) || true + + ### run branch first + # this per default builds the branch and the reference and creates the reference outputs + - *build-and-create-reference-outputs + - exit_code=0 + - python3 -m pytest --tb=no $TEST_SUITE -v --create_cut --html=$HTML_REPORT_BRANCH --self-contained-html --junit-xml=$XML_REPORT_BRANCH --mld --ssnr --odg --ref_encoder_path $REF_ENCODER_PATH --ref_decoder_path $REF_DECODER_PATH --dut_encoder_path $DUT_ENCODER_PATH --dut_decoder_path $DUT_DECODER_PATH -n auto --testcase_timeout $testcase_timeout || exit_code=$? + - zero_errors_branch=$(cat $XML_REPORT_BRANCH | grep -c 'errors="0"') || true + - python3 scripts/parse_xml_report.py $XML_REPORT_BRANCH $CSV_BRANCH + + # create the summary based on the branch + - mkdir $IMAGES_ARTIFACT_NAME + - for MEASURE in MLD DIFF SSNR;do python3 scripts/create_histogram_summary.py $CSV_BRANCH $IMAGES_ARTIFACT_NAME/summary_"$MEASURE".csv $IMAGES_ARTIFACT_NAME/summary_"$MEASURE".png --measure $MEASURE; done + - python3 ci/basop-pages/create_summary_page.py $SUMMARY_HTML_ARTIFACT_NAME $CI_JOB_ID $CI_JOB_NAME + + ### run main now + - git checkout $CI_MERGE_REQUEST_TARGET_BRANCH_NAME + - git pull + - make clean + - make -j + - python3 -m pytest --tb=no $TEST_SUITE -v --create_cut --html=$HTML_REPORT_MAIN --self-contained-html --junit-xml=$XML_REPORT_MAIN --mld --ssnr --odg --ref_encoder_path $REF_ENCODER_PATH --ref_decoder_path $REF_DECODER_PATH --dut_encoder_path $DUT_ENCODER_PATH --dut_decoder_path $DUT_DECODER_PATH -n auto --testcase_timeout $testcase_timeout || true + - python3 scripts/parse_xml_report.py $XML_REPORT_MAIN $CSV_MAIN + + ### compare the two csv files for regressions + - regressions_found=0 + - python3 scripts/basop_check_for_changes_in_testcases.py $CSV_BRANCH $CSV_MAIN || regressions_found=$? + + - if [ $regressions_found != 0 ]; then + - echo "Detected regression wrt to main!" + - if [ $allow_regressions_flag == 0 ]; then + - exit $EXIT_CODE_FAIL; + - else + - exit $EXIT_CODE_NON_BE; + - fi + - fi + - if [ $zero_errors_branch != 1 ]; then echo "Run errors encountered!"; exit $EXIT_CODE_NON_BE; fi + - if [ $exit_code -eq 1 ]; then echo "Differences encountered"; exit $EXIT_CODE_NON_BE; fi + - exit 0 + after_script: + - rm -rf tests/dut tests/ref + allow_failure: + exit_codes: + - 123 + artifacts: + name: "$CI_JOB_NAME--sha-$CI_COMMIT_SHORT_SHA--results" + expire_in: 1 week + when: always + paths: + - $XML_REPORT_BRANCH + - $XML_REPORT_MAIN + - $HTML_REPORT_BRANCH + - $HTML_REPORT_MAIN + - $CSV_BRANCH + - $CSV_MAIN + - $SUMMARY_HTML_ARTIFACT_NAME + - $IMAGES_ARTIFACT_NAME + - changes_crashes.csv + - changes_MLD.csv + - changes_MAXIMUM_ABS_DIFF.csv + - changes_MIN_SSNR.csv + - changes_MIN_ODG.csv + expose_as: "pytest compare results" + reports: + junit: + - $XML_REPORT_BRANCH + - $XML_REPORT_MAIN + .ivas-pytest-sanitizers-anchor: &ivas-pytest-sanitizers-anchor stage: test @@ -300,7 +426,7 @@ stages: - *print-common-info - *update-scripts-repo - python3 ci/remove_unsupported_testcases.py $PRM_FILES - - *setup-codec + - *build-reference-and-dut-binaries - make clean - make -j CLANG=$CLANG_NUM - if [[ $CLANG_NUM == 3 ]]; then export UBSAN_OPTIONS="suppressions=scripts/ubsan.supp,report_error_type=1"; fi @@ -410,13 +536,90 @@ build-codec-linux-instrumented-make: - make -j # --------------------------------------------------------------- -# Short test jobs +# Short test jobs that run in merge request pipelines +# --------------------------------------------------------------- + +### jobs that test fx encoder -> flt decoder +# TODO: reenable once encoder tests shall compare to main +# ivas-pytest-compare_to_main-short-enc: +# extends: +# - .rules-pytest-to-main-short +# - .test-job-linux +# before_script: +# - USE_LTV=0 +# - DUT_DECODER_PATH=./IVAS_dec_ref +# - TEST_SUITE="$SHORT_TEST_SUITE" +# - LEVEL_SCALING=1.0 +# <<: *ivas-pytest-on-merge-request-anchor + +# ivas-pytest-compare_to_main-short-enc-lev-10: +# extends: +# - .rules-pytest-to-main-short +# - .test-job-linux +# before_script: +# - USE_LTV=0 +# - DUT_DECODER_PATH=./IVAS_dec_ref +# - TEST_SUITE="$SHORT_TEST_SUITE" +# - LEVEL_SCALING=0.3162 +# <<: *ivas-pytest-on-merge-request-anchor + +# ivas-pytest-compare_to_main-short-enc-lev+10: +# extends: +# - .rules-pytest-to-main-short +# - .test-job-linux +# before_script: +# - USE_LTV=0 +# - DUT_DECODER_PATH=./IVAS_dec_ref +# - TEST_SUITE="$SHORT_TEST_SUITE" +# - LEVEL_SCALING=3.162 +# <<: *ivas-pytest-on-merge-request-anchor + +### jobs that test flt encoder -> fx decoder +ivas-pytest-compare_to_main-short-dec: + extends: + - .rules-pytest-to-main-short + - .test-job-linux + before_script: + - USE_LTV=0 + - DUT_ENCODER_PATH=./IVAS_cod_ref + - TEST_SUITE="$SHORT_TEST_SUITE" + - LEVEL_SCALING=1.0 + - rm -rf tests/dut tests/ref + <<: *ivas-pytest-on-merge-request-anchor + +ivas-pytest-compare_to_main-short-dec-lev-10: + extends: + - .rules-pytest-to-main-short + - .test-job-linux + before_script: + - USE_LTV=0 + - DUT_ENCODER_PATH=./IVAS_cod_ref + - TEST_SUITE="$SHORT_TEST_SUITE" + - LEVEL_SCALING=0.3162 + - rm -rf tests/dut tests/ref + <<: *ivas-pytest-on-merge-request-anchor + +ivas-pytest-compare_to_main-short-dec-lev+10: + extends: + - .rules-pytest-to-main-short + - .test-job-linux + before_script: + - USE_LTV=0 + - DUT_ENCODER_PATH=./IVAS_cod_ref + - TEST_SUITE="$SHORT_TEST_SUITE" + - LEVEL_SCALING=3.162 + - rm -rf tests/dut tests/ref + <<: *ivas-pytest-on-merge-request-anchor + +# --------------------------------------------------------------- +# Short test jobs for running from web interface or schedule # --------------------------------------------------------------- ### jobs that test fx encoder -> flt decoder -ivas-pytest-mld-enc: +ivas-pytest-compare_to_ref-short-enc: extends: - - .rules-pytest-mld + #- .rules-pytest-to-ref-short + - .rules-pytest-to-ref-enc-short-temp - .test-job-linux before_script: - USE_LTV=0 @@ -425,9 +628,10 @@ ivas-pytest-mld-enc: - LEVEL_SCALING=1.0 <<: *ivas-pytest-anchor -ivas-pytest-mld-enc-lev-10: +ivas-pytest-compare_to_ref-short-enc-lev-10: extends: - - .rules-pytest-mld + #- .rules-pytest-to-ref-short + - .rules-pytest-to-ref-enc-short-temp - .test-job-linux before_script: - USE_LTV=0 @@ -436,9 +640,10 @@ ivas-pytest-mld-enc-lev-10: - LEVEL_SCALING=0.3162 <<: *ivas-pytest-anchor -ivas-pytest-mld-enc-lev+10: +ivas-pytest-compare_to_ref-short-enc-lev+10: extends: - - .rules-pytest-mld + #- .rules-pytest-to-ref-short + - .rules-pytest-to-ref-enc-short-temp - .test-job-linux before_script: - USE_LTV=0 @@ -472,9 +677,9 @@ ivas-pytest-enc-usan: <<: *ivas-pytest-sanitizers-anchor ### jobs that test flt encoder -> fx decoder -ivas-pytest-mld-dec: +ivas-pytest-compare_to_ref-short-dec: extends: - - .rules-pytest-mld + - .rules-pytest-to-ref-short - .test-job-linux before_script: - USE_LTV=0 @@ -483,9 +688,9 @@ ivas-pytest-mld-dec: - LEVEL_SCALING=1.0 <<: *ivas-pytest-anchor -ivas-pytest-mld-dec-lev-10: +ivas-pytest-compare_to_ref-short-lev-10: extends: - - .rules-pytest-mld + - .rules-pytest-to-ref-short - .test-job-linux before_script: - USE_LTV=0 @@ -494,9 +699,9 @@ ivas-pytest-mld-dec-lev-10: - LEVEL_SCALING=0.3162 <<: *ivas-pytest-anchor -ivas-pytest-mld-dec-lev+10: +ivas-pytest-compare_to_ref-short-dec-lev+10: extends: - - .rules-pytest-mld + - .rules-pytest-to-ref-short - .test-job-linux before_script: - USE_LTV=0 @@ -505,7 +710,7 @@ ivas-pytest-mld-dec-lev+10: - LEVEL_SCALING=3.162 <<: *ivas-pytest-anchor -ivas-pytest-dec-msan: +ivas-pytest-dec-msan-short: extends: - .test-job-linux before_script: @@ -513,7 +718,7 @@ ivas-pytest-dec-msan: - DUT_ENCODER_PATH=./$REF_ENCODER_PATH <<: *ivas-pytest-sanitizers-anchor -ivas-pytest-dec-asan: +ivas-pytest-dec-asan-short: extends: - .test-job-linux before_script: @@ -521,7 +726,7 @@ ivas-pytest-dec-asan: - DUT_ENCODER_PATH=./$REF_ENCODER_PATH <<: *ivas-pytest-sanitizers-anchor -ivas-pytest-dec-usan: +ivas-pytest-dec-usan-short: extends: - .test-job-linux before_script: @@ -533,9 +738,9 @@ ivas-pytest-dec-usan: # Long test jobs # --------------------------------------------------------------- -ivas-pytest-mld-long-enc: +ivas-pytest-compare_ref-long-enc: extends: - - .rules-pytest-mld-long + - .rules-pytest-long - .test-job-linux before_script: - USE_LTV=1 @@ -544,9 +749,9 @@ ivas-pytest-mld-long-enc: - LEVEL_SCALING=1.0 <<: *ivas-pytest-anchor -ivas-pytest-mld-long-dec: +ivas-pytest-compare_ref-long-dec: extends: - - .rules-pytest-mld-long + - .rules-pytest-long - .test-job-linux before_script: - USE_LTV=1 @@ -555,9 +760,9 @@ ivas-pytest-mld-long-dec: - LEVEL_SCALING=1.0 <<: *ivas-pytest-anchor -ivas-pytest-mld-long-enc-lev-10: +ivas-pytest-compare_ref-long-enc-lev-10: extends: - - .rules-pytest-mld-long + - .rules-pytest-long - .test-job-linux before_script: - USE_LTV=1 @@ -566,9 +771,9 @@ ivas-pytest-mld-long-enc-lev-10: - LEVEL_SCALING=0.3162 <<: *ivas-pytest-anchor -ivas-pytest-mld-long-dec-lev-10: +ivas-pytest-compare_ref-long-dec-lev-10: extends: - - .rules-pytest-mld-long + - .rules-pytest-long - .test-job-linux before_script: - USE_LTV=1 @@ -577,9 +782,9 @@ ivas-pytest-mld-long-dec-lev-10: - LEVEL_SCALING=0.3162 <<: *ivas-pytest-anchor -ivas-pytest-mld-long-enc-lev+10: +ivas-pytest-compare_ref-long-enc-lev+10: extends: - - .rules-pytest-mld-long + - .rules-pytest-long - .test-job-linux before_script: - USE_LTV=1 @@ -588,9 +793,9 @@ ivas-pytest-mld-long-enc-lev+10: - LEVEL_SCALING=3.162 <<: *ivas-pytest-anchor -ivas-pytest-mld-long-dec-lev+10: +ivas-pytest-compare_ref-long-dec-lev+10: extends: - - .rules-pytest-mld-long + - .rules-pytest-long - .test-job-linux before_script: - USE_LTV=1