diff --git a/main-basop.yml b/main-basop.yml index 826b1eb454232a4f10d681e9fc9c361cc3f4f533..eb75aebc03e993fa1afb56c1299eab63f6acfb4e 100644 --- a/main-basop.yml +++ b/main-basop.yml @@ -119,6 +119,7 @@ variables: - 'long-term-logs' - 'backup-long-term-logs' - 'test-long-self-test' + - 'check-regressions-long' - 'ivas-conformance-linux' # This sets when pipelines are created. Jobs have more specific rules to restrict them. @@ -186,6 +187,9 @@ workflow: - if: $CI_PIPELINE_SOURCE == 'web' && $MANUAL_PIPELINE_TYPE == 'test-long-self-test' variables: IVAS_PIPELINE_NAME: 'Test long self-test against main pipeline: $CI_COMMIT_BRANCH' + - if: $CI_PIPELINE_SOURCE == 'web' && $MANUAL_PIPELINE_TYPE == 'check-regressions-long' + variables: + IVAS_PIPELINE_NAME: 'LTV regression check: $CI_COMMIT_BRANCH' - if: $CI_PIPELINE_SOURCE == 'web' && $MANUAL_PIPELINE_TYPE == 'ivas-conformance-linux' variables: IVAS_PIPELINE_NAME: 'IVAS Conformance Linux: $CI_COMMIT_BRANCH' @@ -208,6 +212,8 @@ workflow: rules: - if: $CI_PIPELINE_SOURCE == 'web' && $MANUAL_PIPELINE_TYPE == "ivas-conformance-linux" when: never + - if: $CI_PIPELINE_SOURCE == 'web' && $MANUAL_PIPELINE_TYPE == "check-regressions-long" + when: never - if: $CI_PIPELINE_SOURCE == 'web' - if: $CI_PIPELINE_SOURCE == 'schedule' - if: $CI_PIPELINE_SOURCE == 'push' @@ -257,6 +263,14 @@ workflow: - if: $CI_PIPELINE_SOURCE == 'merge_request_event' when: never +.rules-check-regressions-long: + rules: + - if: $CI_PIPELINE_SOURCE == 'web' && $MANUAL_PIPELINE_TYPE == "check-regressions-long" + - if: $CI_PIPELINE_SOURCE == 'push' + when: never + - if: $CI_PIPELINE_SOURCE == 'merge_request_event' + when: never + .rules-coverage: rules: - if: $COVERAGE_TEST # Set by scheduled pipeline @@ -713,6 +727,136 @@ workflow: - $XML_REPORT_BRANCH - $XML_REPORT_MAIN +.check-regressions-ltv-job: + extends: + - .test-job-linux + - .rules-check-regressions-long + stage: test + needs: ["build-codec-linux-clang-make", "check-regressions-long-pre"] + timeout: "900 minutes" + variables: + XML_REPORT_BRANCH: "report-junit-branch-$CI_JOB_NAME--sha-$CI_COMMIT_SHORT_SHA.xml" + XML_REPORT_MAIN: "report-junit-main-$CI_JOB_NAME--sha-$CI_COMMIT_SHORT_SHA.xml" + HTML_REPORT_BRANCH: "report-junit-branch-$CI_JOB_NAME--sha-$CI_COMMIT_SHORT_SHA.html" + HTML_REPORT_MAIN: "report-junit-main-$CI_JOB_NAME--sha-$CI_COMMIT_SHORT_SHA.html" + CSV_BRANCH: "scores-branch-$CI_JOB_NAME--sha-$CI_COMMIT_SHORT_SHA.csv" + CSV_MAIN: "scores-main-$CI_JOB_NAME--sha-$CI_COMMIT_SHORT_SHA.csv" + IMAGES_ARTIFACT_NAME: "images_$CI_JOB_NAME" + SUMMARY_HTML_ARTIFACT_NAME: "summary_$CI_JOB_NAME.html" + USE_LTV: 1 + before_script: + - !reference [ .test-job-linux, before_script ] + - rm -rf tests/dut tests/ref + - if [ "$DISABLE_HRTF" = "true" ]; then + - export PYTEST_ADDOPTS="${PYTEST_ADDOPTS} -k 'not model'" + - fi + - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/basop/merge-request-print-pinned-commits.sh + script: + - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/print-common-info.sh + + # create empty files for all artifacts to suppress warnings in case of no regressions found or all is BE + - touch $XML_REPORT_BRANCH $XML_REPORT_MAIN $HTML_REPORT_BRANCH $HTML_REPORT_MAIN $CSV_BRANCH $CSV_MAIN $SUMMARY_HTML_ARTIFACT_NAME $FLOAT_REF_COMMIT_FILE $CUT_COMMIT_FILE $MERGE_TARGET_COMMIT_FILE $MERGE_SOURCE_FLOAT_REF_COMMIT_FILE regressions_crashes.csv regressions_MLD.csv regressions_MAX_ABS_DIFF.csv regressions_MIN_SSNR.csv regressions_MIN_ODG.csv improvements_crashes.csv improvements_MLD.csv improvements_MAX_ABS_DIFF.csv improvements_MIN_SSNR.csv improvements_MIN_ODG.csv + - mkdir $IMAGES_ARTIFACT_NAME + + - set -euxo pipefail + + - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/update-ltv-repo.sh + - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/copy-ltv-files-to-testv-dir.sh + - testcase_timeout=$TESTCASE_TIMEOUT_LTV + + - if [ $LEVEL_SCALING != "1.0" ];then + - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/basop/apply-testv-scaling.sh + - fi + + - comp_args="--mld --ssnr --odg --mld-playback-level $MLD_PLAYBACK_LEVEL" + + # build merge target branch and correpsonding float reference + - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/basop/build-binaries.sh float-ref + - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/basop/build-binaries.sh merge-target + # build to-be-merged branch and corresponding float ref branch + - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/basop/build-binaries.sh dut + - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/basop/build-binaries.sh float-ref-merge-source + + # set pytest args for ref creation + - enc_stats_arg="" + - if [ "$ENCODER_TEST" = "true" ]; then + - enc_stats_arg="--enc_stats" + - fi + + ### ----- run pytest for branch (dut) first ----- + # create float reference outputs with corresponding branch + - python3 -m pytest $TEST_SUITE --update_ref 1 $enc_stats_arg --create_ref -n auto --ref_encoder_path $MERGE_SOURCE_FLOAT_REF_ENCODER_PATH --ref_decoder_path $MERGE_SOURCE_FLOAT_REF_DECODER_PATH || exit_code=$? + # create dut outputs + - python3 -m pytest --tb=no -q $TEST_SUITE --keep_files --create_cut --html=$HTML_REPORT_BRANCH --self-contained-html --junit-xml=$XML_REPORT_BRANCH $comp_args --dut_encoder_path $DUT_ENCODER_PATH --dut_decoder_path $DUT_DECODER_PATH -n auto --testcase_timeout $testcase_timeout || true + - zero_errors_branch=$(cat $XML_REPORT_BRANCH | grep -c 'errors="0"') || true + - python3 scripts/parse_xml_report.py $XML_REPORT_BRANCH $CSV_BRANCH + # Store branch outputs for later comparison + - mv tests/dut tests/dut_branch + # create the summary based on the branch only + - python3 scripts/create_histograms.py $CSV_BRANCH $IMAGES_ARTIFACT_NAME --measures $MEASURES_FOR_REPORT + - python3 ci/basop-pages/create_summary_page.py $SUMMARY_HTML_ARTIFACT_NAME $CI_JOB_ID $CI_JOB_NAME $IMAGES_ARTIFACT_NAME --measures $MEASURES_FOR_REPORT + + ### ----- run pytest for merge target now ----- + # create float reference outputs with corresponding branch + - python3 -m pytest $TEST_SUITE --update_ref 1 $enc_stats_arg --create_ref -n auto --ref_encoder_path $REF_ENCODER_PATH --ref_decoder_path $REF_DECODER_PATH || exit_code=$? + # create merge-target outputs + - python3 -m pytest --tb=no -q $TEST_SUITE --keep_files --create_cut --html=$HTML_REPORT_MAIN --self-contained-html --junit-xml=$XML_REPORT_MAIN $comp_args --dut_encoder_path $MERGE_TARGET_ENCODER_PATH --dut_decoder_path $MERGE_TARGET_DECODER_PATH -n auto --testcase_timeout $testcase_timeout || true + - python3 scripts/parse_xml_report.py $XML_REPORT_MAIN $CSV_MAIN + + ### compare the two csv files for regressions + - regressions_found=0 + - python3 scripts/basop_check_for_changes_in_testcases.py --show_improvements --xml_report $XML_REPORT_BRANCH $CSV_BRANCH $CSV_MAIN > regression_log.txt || regressions_found=$? + + - exit_code=0 + - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/basop/print-results-banner.sh + - if [ $zero_errors_branch != 1 ]; then + - echo "Run errors encountered!" + - exit_code=$EXIT_CODE_FAIL + - elif [ $regressions_found != 0 ]; then + - cat regression_log.txt + - echo "Detected regressions wrt to basop main" + - exit_code=$EXIT_CODE_NON_BE + - fi + + - exit $exit_code + after_script: + - rm -rf tests/dut tests/ref + allow_failure: + exit_codes: + - 123 + artifacts: + name: "$CI_JOB_NAME--sha-$CI_COMMIT_SHORT_SHA--results" + expire_in: 1 week + when: always + paths: + - $XML_REPORT_BRANCH + - $XML_REPORT_MAIN + - $HTML_REPORT_BRANCH + - $HTML_REPORT_MAIN + - $CSV_BRANCH + - $CSV_MAIN + - $SUMMARY_HTML_ARTIFACT_NAME + - $IMAGES_ARTIFACT_NAME + - $FLOAT_REF_COMMIT_FILE + - $CUT_COMMIT_FILE + - $MERGE_TARGET_COMMIT_FILE + - $MERGE_SOURCE_FLOAT_REF_COMMIT_FILE + - regressions_crashes.csv + - regressions_MLD.csv + - regressions_MAX_ABS_DIFF.csv + - regressions_MIN_SSNR.csv + - regressions_MIN_ODG.csv + - improvements_crashes.csv + - improvements_MLD.csv + - improvements_MAX_ABS_DIFF.csv + - improvements_MIN_SSNR.csv + - improvements_MIN_ODG.csv + expose_as: "LTV regression test results" + reports: + junit: + - $XML_REPORT_BRANCH + - $XML_REPORT_MAIN + .ivas-pytest-compare-to-input-anchor: &ivas-pytest-compare-to-input-anchor stage: test needs: ["build-codec-linux-clang-make", "pytest-to-ref-pre"] @@ -908,6 +1052,32 @@ pytest-to-ref-pre: reports: dotenv: commits.env + +# This job replicates the functionality of branch-is-up-to-date-with-target-pre job from MR pipeline +# to run in manual pipeline for long regression testing. Note that we are using merge related commit +# names although this not part of MR pipeline. This helps sharing code now but could be improved later. +check-regressions-long-pre: + extends: + - .job-linux + - .rules-check-regressions-long + stage: prevalidate + needs: [] + script: + - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/print-common-info.sh + - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/basop/update-scripts-repo.sh + # This writes the values to the non-existing MR variables to share the script. + - export CI_MERGE_REQUEST_SOURCE_BRANCH_NAME="$CI_COMMIT_REF_NAME" + - export CI_MERGE_REQUEST_TARGET_BRANCH_NAME="main" + - source "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/basop/branch-is-up-to-date-with-target-pre.sh + artifacts: + paths: + - $MERGE_TARGET_COMMIT_FILE + - $FLOAT_REF_COMMIT_FILE + - $MERGE_SOURCE_FLOAT_REF_COMMIT_FILE + reports: + dotenv: commits.env + + # --------------------------------------------------------------- # Test jobs for merge requests # --------------------------------------------------------------- @@ -1082,6 +1252,78 @@ check-regressions-short-dec--10db: MERGE_TARGET_ENCODER_PATH: "$REF_ENCODER_PATH" USE_LTV: 0 +check-regressions-long-enc-0db: + extends: + - .check-regressions-ltv-job + resource_group: "long-regression-0db" + variables: + TEST_SUITE: "$LONG_TEST_SUITE_ENCODER" + # overwrite decoder with float reference one + DUT_DECODER_PATH: "$MERGE_SOURCE_FLOAT_REF_DECODER_PATH_FOR_BUILD_DO_NOT_MODIFY" + MERGE_TARGET_DECODER_PATH: "$REF_DECODER_PATH" + +check-regressions-long-enc-+10db: + extends: + - .check-regressions-ltv-job + resource_group: "long-regression-plus10db" + variables: + TEST_SUITE: "$LONG_TEST_SUITE_ENCODER" + # +10dB + LEVEL_SCALING: "3.162" + MLD_PLAYBACK_LEVEL: "82.0" + # overwrite decoder with float reference one + DUT_DECODER_PATH: "$MERGE_SOURCE_FLOAT_REF_DECODER_PATH_FOR_BUILD_DO_NOT_MODIFY" + MERGE_TARGET_DECODER_PATH: "$REF_DECODER_PATH" + +check-regressions-long-enc--10db: + extends: + - .check-regressions-ltv-job + resource_group: "long-regression-minus10db" + variables: + TEST_SUITE: "$LONG_TEST_SUITE_ENCODER" + # -10dB + LEVEL_SCALING: "0.3162" + MLD_PLAYBACK_LEVEL: "102.0" + # overwrite decoder with float reference one + DUT_DECODER_PATH: "$MERGE_SOURCE_FLOAT_REF_DECODER_PATH_FOR_BUILD_DO_NOT_MODIFY" + MERGE_TARGET_DECODER_PATH: "$REF_DECODER_PATH" + +check-regressions-long-dec-0db: + extends: + - .check-regressions-ltv-job + resource_group: "long-regression-0db" + variables: + TEST_SUITE: "$LONG_TEST_SUITE_NO_RENDERER" + # overwrite encoder with float reference one + DUT_ENCODER_PATH: "$MERGE_SOURCE_FLOAT_REF_ENCODER_PATH_FOR_BUILD_DO_NOT_MODIFY" + MERGE_TARGET_ENCODER_PATH: "$REF_ENCODER_PATH" + +check-regressions-long-dec-+10db: + extends: + - .check-regressions-ltv-job + resource_group: "long-regression-plus10db" + variables: + TEST_SUITE: "$LONG_TEST_SUITE_NO_RENDERER" + # +10dB + LEVEL_SCALING: "3.162" + MLD_PLAYBACK_LEVEL: "82.0" + # overwrite encoder with float reference one + DUT_ENCODER_PATH: "$MERGE_SOURCE_FLOAT_REF_ENCODER_PATH_FOR_BUILD_DO_NOT_MODIFY" + MERGE_TARGET_ENCODER_PATH: "$REF_ENCODER_PATH" + +check-regressions-long-dec--10db: + extends: + - .check-regressions-ltv-job + resource_group: "long-regression-minus10db" + variables: + TEST_SUITE: "$LONG_TEST_SUITE_NO_RENDERER" + # -10dB + LEVEL_SCALING: "0.3162" + MLD_PLAYBACK_LEVEL: "102.0" + # overwrite encoder with float reference one + DUT_ENCODER_PATH: "$MERGE_SOURCE_FLOAT_REF_ENCODER_PATH_FOR_BUILD_DO_NOT_MODIFY" + MERGE_TARGET_ENCODER_PATH: "$REF_ENCODER_PATH" + ### jobs that compare bitexactness between merge target and source renderer-pytest-on-merge-request: extends: