From d9342024801c8ecaddbbd9812c5b4b94ba0f135c Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Wed, 24 Jul 2024 08:16:36 +0200 Subject: [PATCH 01/26] remove use of "mld" from ci file and enable comparison to main --- .gitlab-ci.yml | 114 +++++++++++++++++++++++++++---------------------- 1 file changed, 63 insertions(+), 51 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 34b1755a4..defde0ac2 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -21,12 +21,12 @@ variables: TESTCASE_TIMEOUT_STV_SANITIZERS: 1800 GIT_CLEAN_FLAGS: -ffdxq MANUAL_PIPELINE_TYPE: - description: "Type for the manual pipeline run. Use 'pytest-mld' to run MLD test against reference float codec." # Not implemented yet, but may be good to have a manual pipeline trigger + description: "Type for the manual pipeline run. Use 'pytest-compare' to run comparison test against reference float codec." # Not implemented yet, but may be good to have a manual pipeline trigger value: 'default' options: - 'default' - - 'pytest-mld' - - 'pytest-mld-long' + - 'pytest-compare' + - 'pytest-compare-long' - 'pytest-saturation-smoke-test' - 'evs-26444' - 'sanitizer-stv' @@ -52,12 +52,12 @@ workflow: - if: $CI_PIPELINE_SOURCE == 'web' && $MANUAL_PIPELINE_TYPE == 'default' # for testing variables: IVAS_PIPELINE_NAME: 'Web run pipeline: $CI_COMMIT_BRANCH' - - if: $CI_PIPELINE_SOURCE == 'web' && $MANUAL_PIPELINE_TYPE == 'pytest-mld' + - if: $CI_PIPELINE_SOURCE == 'web' && $MANUAL_PIPELINE_TYPE == 'pytest-compare' variables: - IVAS_PIPELINE_NAME: 'Run MLD tool against float ref: $CI_COMMIT_BRANCH' - - if: $CI_PIPELINE_SOURCE == 'web' && $MANUAL_PIPELINE_TYPE == 'pytest-mld-long' + IVAS_PIPELINE_NAME: 'Run comparison tools against float ref: $CI_COMMIT_BRANCH' + - if: $CI_PIPELINE_SOURCE == 'web' && $MANUAL_PIPELINE_TYPE == 'pytest-comapre-long' variables: - IVAS_PIPELINE_NAME: 'Run MLD tool against float ref (long test vectors): $CI_COMMIT_BRANCH' + IVAS_PIPELINE_NAME: 'Run comparison tools against float ref (long test vectors): $CI_COMMIT_BRANCH' - if: $CI_PIPELINE_SOURCE == 'web' && $MANUAL_PIPELINE_TYPE == 'pytest-saturation-smoke-test' variables: IVAS_PIPELINE_NAME: 'Run saturation smoke-test: $CI_COMMIT_BRANCH' @@ -95,10 +95,10 @@ stages: echo "Commit time was $CI_COMMIT_TIMESTAMP" date | xargs echo "System time is" -.setup-codec: &setup-codec +.build-reference-and-dut-binaries: &build-reference-and-dut-binaries - current_commit_sha=$(git rev-parse HEAD) ### build reference binaries - - git checkout $REFERENCE_BRANCH + - git checkout $REFERENCE_BRANCH_LOCAL - git pull - make clean - make -j @@ -111,9 +111,8 @@ stages: - make clean - make -j - -.mld-test-setup-codec: &mld-test-setup-codec - - *setup-codec +.build-and-create-reference-outputs: &build-and-create-reference-outputs + - *build-reference-and-dut-binaries ### prepare pytest # create short test vectors @@ -155,9 +154,9 @@ stages: .activate-Werror-linux: &activate-Werror-linux - sed -i.bak "s/^# \(CFLAGS += -Werror\)/\1/" Makefile -.rules-pytest-mld: +.rules-pytest-short: rules: - - if: $CI_PIPELINE_SOURCE == 'web' && $MANUAL_PIPELINE_TYPE == "pytest-mld" + - if: $CI_PIPELINE_SOURCE == 'web' && $MANUAL_PIPELINE_TYPE == "pytest-compare" - if: $CI_PIPELINE_SOURCE == 'push' && $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH - if: $CI_PIPELINE_SOURCE == 'merge_request_event' && $CI_MERGE_REQUEST_TARGET_BRANCH_NAME == "main" # only have MR pipelines for MRs to main - if: $CI_PIPELINE_SOURCE == 'push' @@ -165,10 +164,10 @@ stages: - if: $CI_PIPELINE_SOURCE == 'schedule' when: never -.rules-pytest-mld-long: +.rules-pytest-long: rules: - if: $PYTEST_MLD_LONG # Set by scheduled pipeline - - if: $CI_PIPELINE_SOURCE == 'web' && $MANUAL_PIPELINE_TYPE == "pytest-mld-long" + - if: $CI_PIPELINE_SOURCE == 'web' && $MANUAL_PIPELINE_TYPE == "pytest-compare-long" - if: $CI_PIPELINE_SOURCE == 'push' when: never - if: $CI_PIPELINE_SOURCE == 'merge_request_event' @@ -212,7 +211,8 @@ stages: needs: ["build-codec-linux-make"] timeout: "240 minutes" variables: - MLD_ARTIFACT_NAME: "mld--$CI_JOB_NAME-$CI_JOB_ID--sha-$CI_COMMIT_SHORT_SHA.csv" + # keep "mld" in artifact name for backwards compatibility reasons + CSV_ARTIFACT_NAME: "mld--$CI_JOB_NAME-$CI_JOB_ID--sha-$CI_COMMIT_SHORT_SHA.csv" MERGED_CSV_ARTIFACT_NAME: "$CI_JOB_NAME--merged_csv--$CI_JOB_ID.csv" PAGES_HTML_ARTIFACT_NAME: "$CI_JOB_NAME-index.html" SUMMARY_HTML_ARTIFACT_NAME: "summary_$CI_JOB_NAME.html" @@ -231,16 +231,16 @@ stages: - if [ $LEVEL_SCALING != "1.0" ];then - *apply-testv-scaling - fi - - *mld-test-setup-codec + - *build-and-create-reference-outputs ### run pytest - exit_code=0 - python3 -m pytest --tb=no $TEST_SUITE -v --create_cut --html=report.html --self-contained-html --junit-xml=report-junit.xml --mld --ssnr --odg --dut_encoder_path $DUT_ENCODER_PATH --dut_decoder_path $DUT_DECODER_PATH -n auto --testcase_timeout $testcase_timeout || exit_code=$? - zero_errors=$(cat report-junit.xml | grep -c 'errors="0"') || true - - python3 scripts/parse_xml_report.py report-junit.xml $MLD_ARTIFACT_NAME + - python3 scripts/parse_xml_report.py report-junit.xml $CSV_ARTIFACT_NAME - mkdir $IMAGES_ARTIFACT_NAME - - for MEASURE in MLD DIFF SSNR;do python3 scripts/create_histogram_summary.py $MLD_ARTIFACT_NAME images_"$CI_JOB_NAME"/summary_"$MEASURE".csv images_"$CI_JOB_NAME"/summary_"$MEASURE".png --measure $MEASURE; done + - for MEASURE in MLD DIFF SSNR;do python3 scripts/create_histogram_summary.py $CSV_ARTIFACT_NAME images_"$CI_JOB_NAME"/summary_"$MEASURE".csv images_"$CI_JOB_NAME"/summary_"$MEASURE".png --measure $MEASURE; done - python3 ci/basop-pages/create_summary_page.py $SUMMARY_HTML_ARTIFACT_NAME $CI_JOB_ID $CI_JOB_NAME - if [ $USE_LTV -eq 1 ] && [ "$CI_COMMIT_BRANCH" == "$CI_DEFAULT_BRANCH" ]; then @@ -250,7 +250,7 @@ stages: - unzip artifacts.zip -d previous_artifacts # This wildcard thingy relies on only one csv file being present per job - file_previous="previous_artifacts/mld--$CI_JOB_NAME-$id_previous--sha-*.csv" - - python3 ci/basop-pages/create_report_pages.py $PAGES_HTML_ARTIFACT_NAME $MERGED_CSV_ARTIFACT_NAME $MLD_ARTIFACT_NAME $file_previous $CI_JOB_ID $id_previous $CI_JOB_NAME + - python3 ci/basop-pages/create_report_pages.py $PAGES_HTML_ARTIFACT_NAME $MERGED_CSV_ARTIFACT_NAME $CSV_ARTIFACT_NAME $file_previous $CI_JOB_ID $id_previous $CI_JOB_NAME - else # create empty file for artifacts to avoid errors - touch $PAGES_HTML_ARTIFACT_NAME @@ -272,12 +272,12 @@ stages: - report-junit.xml - report.html - $PAGES_HTML_ARTIFACT_NAME - - $MLD_ARTIFACT_NAME + - $CSV_ARTIFACT_NAME - $MERGED_CSV_ARTIFACT_NAME - images - $SUMMARY_HTML_ARTIFACT_NAME - $IMAGES_ARTIFACT_NAME - expose_as: "pytest mld results" + expose_as: "pytest compare results" reports: junit: - report-junit.xml @@ -298,7 +298,7 @@ stages: - *print-common-info - *update-scripts-repo - python3 ci/remove_unsupported_testcases.py $PRM_FILES - - *setup-codec + - *build-reference-and-dut-binaries - make clean - make -j CLANG=$CLANG_NUM - if [[ $CLANG_NUM == 3 ]]; then export UBSAN_OPTIONS="suppressions=scripts/ubsan.supp,report_error_type=1"; fi @@ -412,37 +412,40 @@ build-codec-linux-instrumented-make: # --------------------------------------------------------------- ### jobs that test fx encoder -> flt decoder -ivas-pytest-mld-enc: +ivas-pytest-compare-enc: extends: - - .rules-pytest-mld + - .rules-pytest-short - .test-job-linux before_script: - USE_LTV=0 - DUT_DECODER_PATH=./IVAS_dec_ref - TEST_SUITE="$SHORT_TEST_SUITE" - LEVEL_SCALING=1.0 + - REFERENCE_BRANCH_LOCAL="$REFERENCE_BRANCH" <<: *ivas-pytest-anchor -ivas-pytest-mld-enc-lev-10: +ivas-pytest-compare-enc-lev-10: extends: - - .rules-pytest-mld + - .rules-pytest-short - .test-job-linux before_script: - USE_LTV=0 - DUT_DECODER_PATH=./IVAS_dec_ref - TEST_SUITE="$SHORT_TEST_SUITE" - LEVEL_SCALING=0.3162 + - REFERENCE_BRANCH_LOCAL="$REFERENCE_BRANCH" <<: *ivas-pytest-anchor -ivas-pytest-mld-enc-lev+10: +ivas-pytest-compare-enc-lev+10: extends: - - .rules-pytest-mld + - .rules-pytest-short - .test-job-linux before_script: - USE_LTV=0 - DUT_DECODER_PATH=./IVAS_dec_ref - TEST_SUITE="$SHORT_TEST_SUITE" - LEVEL_SCALING=3.162 + - REFERENCE_BRANCH_LOCAL="$REFERENCE_BRANCH" <<: *ivas-pytest-anchor ivas-pytest-enc-msan: @@ -470,40 +473,43 @@ ivas-pytest-enc-usan: <<: *ivas-pytest-sanitizers-anchor ### jobs that test flt encoder -> fx decoder -ivas-pytest-mld-dec: +ivas-pytest-compare_main-short-dec: extends: - - .rules-pytest-mld + - .rules-pytest-short - .test-job-linux before_script: - USE_LTV=0 - DUT_ENCODER_PATH=./IVAS_cod_ref - TEST_SUITE="$SHORT_TEST_SUITE" - LEVEL_SCALING=1.0 + - REFERENCE_BRANCH_LOCAL="main" <<: *ivas-pytest-anchor -ivas-pytest-mld-dec-lev-10: +ivas-pytest-compare_main-short-lev-10: extends: - - .rules-pytest-mld + - .rules-pytest-short - .test-job-linux before_script: - USE_LTV=0 - DUT_ENCODER_PATH=./IVAS_cod_ref - TEST_SUITE="$SHORT_TEST_SUITE" - LEVEL_SCALING=0.3162 + - REFERENCE_BRANCH_LOCAL="main" <<: *ivas-pytest-anchor -ivas-pytest-mld-dec-lev+10: +ivas-pytest-compare_main-short-dec-lev+10: extends: - - .rules-pytest-mld + - .rules-pytest-short - .test-job-linux before_script: - USE_LTV=0 - DUT_ENCODER_PATH=./IVAS_cod_ref - TEST_SUITE="$SHORT_TEST_SUITE" - LEVEL_SCALING=3.162 + - REFERENCE_BRANCH_LOCAL="main" <<: *ivas-pytest-anchor -ivas-pytest-dec-msan: +ivas-pytest-dec-msan-short: extends: - .test-job-linux before_script: @@ -511,7 +517,7 @@ ivas-pytest-dec-msan: - DUT_ENCODER_PATH=./IVAS_cod_ref <<: *ivas-pytest-sanitizers-anchor -ivas-pytest-dec-asan: +ivas-pytest-dec-asan-short: extends: - .test-job-linux before_script: @@ -519,7 +525,7 @@ ivas-pytest-dec-asan: - DUT_ENCODER_PATH=./IVAS_cod_ref <<: *ivas-pytest-sanitizers-anchor -ivas-pytest-dec-usan: +ivas-pytest-dec-usan-short: extends: - .test-job-linux before_script: @@ -531,70 +537,76 @@ ivas-pytest-dec-usan: # Long test jobs # --------------------------------------------------------------- -ivas-pytest-mld-long-enc: +ivas-pytest-compare_ref-long-enc: extends: - - .rules-pytest-mld-long + - .rules-pytest-long - .test-job-linux before_script: - USE_LTV=1 - DUT_DECODER_PATH=./IVAS_dec_ref - TEST_SUITE="$LONG_TEST_SUITE" - LEVEL_SCALING=1.0 + - REFERENCE_BRANCH_LOCAL="$REFERENCE_BRANCH" <<: *ivas-pytest-anchor -ivas-pytest-mld-long-dec: +ivas-pytest-compare_ref-long-dec: extends: - - .rules-pytest-mld-long + - .rules-pytest-long - .test-job-linux before_script: - USE_LTV=1 - DUT_ENCODER_PATH=./IVAS_cod_ref - TEST_SUITE="$LONG_TEST_SUITE" - LEVEL_SCALING=1.0 + - REFERENCE_BRANCH_LOCAL="$REFERENCE_BRANCH" <<: *ivas-pytest-anchor -ivas-pytest-mld-long-enc-lev-10: +ivas-pytest-compare_ref-long-enc-lev-10: extends: - - .rules-pytest-mld-long + - .rules-pytest-long - .test-job-linux before_script: - USE_LTV=1 - DUT_DECODER_PATH=./IVAS_dec_ref - TEST_SUITE="$LONG_TEST_SUITE" - LEVEL_SCALING=0.3162 + - REFERENCE_BRANCH_LOCAL="$REFERENCE_BRANCH" <<: *ivas-pytest-anchor -ivas-pytest-mld-long-dec-lev-10: +ivas-pytest-compare_ref-long-dec-lev-10: extends: - - .rules-pytest-mld-long + - .rules-pytest-long - .test-job-linux before_script: - USE_LTV=1 - DUT_ENCODER_PATH=./IVAS_cod_ref - TEST_SUITE="$LONG_TEST_SUITE" - LEVEL_SCALING=0.3162 + - REFERENCE_BRANCH_LOCAL="$REFERENCE_BRANCH" <<: *ivas-pytest-anchor -ivas-pytest-mld-long-enc-lev+10: +ivas-pytest-compare_ref-long-enc-lev+10: extends: - - .rules-pytest-mld-long + - .rules-pytest-long - .test-job-linux before_script: - USE_LTV=1 - DUT_DECODER_PATH=./IVAS_dec_ref - TEST_SUITE="$LONG_TEST_SUITE" - LEVEL_SCALING=3.162 + - REFERENCE_BRANCH_LOCAL="$REFERENCE_BRANCH" <<: *ivas-pytest-anchor -ivas-pytest-mld-long-dec-lev+10: +ivas-pytest-compare_ref-long-dec-lev+10: extends: - - .rules-pytest-mld-long + - .rules-pytest-long - .test-job-linux before_script: - USE_LTV=1 - DUT_ENCODER_PATH=./IVAS_cod_ref - TEST_SUITE="$LONG_TEST_SUITE" - LEVEL_SCALING=3.162 + - REFERENCE_BRANCH_LOCAL="$REFERENCE_BRANCH" <<: *ivas-pytest-anchor ivas-smoke-test-saturation: -- GitLab From a39fa5bfdca0910d891099b3556ae262acc399f7 Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Wed, 24 Jul 2024 08:29:56 +0200 Subject: [PATCH 02/26] correct reference branch selection --- .gitlab-ci.yml | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index defde0ac2..249505213 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -3,6 +3,8 @@ variables: LTV_DIR: "/usr/local/ltv" EVS_BE_TEST_DIR_BASOP: "/usr/local/be_2_evs_basop" REFERENCE_BRANCH: "ivas-float-update" + # this is intentionally set to a nonsense value to trigger a clear error if there is a flaw in the variable setting logic + REFERENCE_BRANCH_LOCAL: "xa34wshgaweedfaerssfgsfdg" BUILD_OUTPUT: "build_output.txt" SCRIPTS_DIR: "/usr/local/scripts" EXIT_CODE_NON_BE: 123 @@ -55,7 +57,7 @@ workflow: - if: $CI_PIPELINE_SOURCE == 'web' && $MANUAL_PIPELINE_TYPE == 'pytest-compare' variables: IVAS_PIPELINE_NAME: 'Run comparison tools against float ref: $CI_COMMIT_BRANCH' - - if: $CI_PIPELINE_SOURCE == 'web' && $MANUAL_PIPELINE_TYPE == 'pytest-comapre-long' + - if: $CI_PIPELINE_SOURCE == 'web' && $MANUAL_PIPELINE_TYPE == 'pytest-compare-long' variables: IVAS_PIPELINE_NAME: 'Run comparison tools against float ref (long test vectors): $CI_COMMIT_BRANCH' - if: $CI_PIPELINE_SOURCE == 'web' && $MANUAL_PIPELINE_TYPE == 'pytest-saturation-smoke-test' @@ -99,6 +101,11 @@ stages: - current_commit_sha=$(git rev-parse HEAD) ### build reference binaries - git checkout $REFERENCE_BRANCH_LOCAL + # safeguard check to detect possible mistakes in the reference branch selection logic + - if [ "$?" != "0" ]; then + - echo "REFERENCE_BRANCH_LOCAL variable is set incorrectly: $REFERENCE_BRANCH_LOCAL!" + - exit(1) + - fi - git pull - make clean - make -j @@ -227,6 +234,11 @@ stages: - else - testcase_timeout=$TESTCASE_TIMEOUT_STV - fi + + - if [ $CI_PIPELINE_SOURCE == "web" || $CI_PIPELINE_SOURCE == "schedule" ]; then + - REFERENCE_BRANCH_LOCAL="$REFERENCE_BRANCH" + - fi + - python3 ci/remove_unsupported_testcases.py $PRM_FILES - if [ $LEVEL_SCALING != "1.0" ];then - *apply-testv-scaling @@ -421,7 +433,6 @@ ivas-pytest-compare-enc: - DUT_DECODER_PATH=./IVAS_dec_ref - TEST_SUITE="$SHORT_TEST_SUITE" - LEVEL_SCALING=1.0 - - REFERENCE_BRANCH_LOCAL="$REFERENCE_BRANCH" <<: *ivas-pytest-anchor ivas-pytest-compare-enc-lev-10: @@ -433,7 +444,6 @@ ivas-pytest-compare-enc-lev-10: - DUT_DECODER_PATH=./IVAS_dec_ref - TEST_SUITE="$SHORT_TEST_SUITE" - LEVEL_SCALING=0.3162 - - REFERENCE_BRANCH_LOCAL="$REFERENCE_BRANCH" <<: *ivas-pytest-anchor ivas-pytest-compare-enc-lev+10: @@ -445,7 +455,6 @@ ivas-pytest-compare-enc-lev+10: - DUT_DECODER_PATH=./IVAS_dec_ref - TEST_SUITE="$SHORT_TEST_SUITE" - LEVEL_SCALING=3.162 - - REFERENCE_BRANCH_LOCAL="$REFERENCE_BRANCH" <<: *ivas-pytest-anchor ivas-pytest-enc-msan: @@ -482,7 +491,6 @@ ivas-pytest-compare_main-short-dec: - DUT_ENCODER_PATH=./IVAS_cod_ref - TEST_SUITE="$SHORT_TEST_SUITE" - LEVEL_SCALING=1.0 - - REFERENCE_BRANCH_LOCAL="main" <<: *ivas-pytest-anchor ivas-pytest-compare_main-short-lev-10: @@ -494,7 +502,6 @@ ivas-pytest-compare_main-short-lev-10: - DUT_ENCODER_PATH=./IVAS_cod_ref - TEST_SUITE="$SHORT_TEST_SUITE" - LEVEL_SCALING=0.3162 - - REFERENCE_BRANCH_LOCAL="main" <<: *ivas-pytest-anchor ivas-pytest-compare_main-short-dec-lev+10: @@ -506,7 +513,6 @@ ivas-pytest-compare_main-short-dec-lev+10: - DUT_ENCODER_PATH=./IVAS_cod_ref - TEST_SUITE="$SHORT_TEST_SUITE" - LEVEL_SCALING=3.162 - - REFERENCE_BRANCH_LOCAL="main" <<: *ivas-pytest-anchor ivas-pytest-dec-msan-short: @@ -546,7 +552,6 @@ ivas-pytest-compare_ref-long-enc: - DUT_DECODER_PATH=./IVAS_dec_ref - TEST_SUITE="$LONG_TEST_SUITE" - LEVEL_SCALING=1.0 - - REFERENCE_BRANCH_LOCAL="$REFERENCE_BRANCH" <<: *ivas-pytest-anchor ivas-pytest-compare_ref-long-dec: @@ -558,7 +563,6 @@ ivas-pytest-compare_ref-long-dec: - DUT_ENCODER_PATH=./IVAS_cod_ref - TEST_SUITE="$LONG_TEST_SUITE" - LEVEL_SCALING=1.0 - - REFERENCE_BRANCH_LOCAL="$REFERENCE_BRANCH" <<: *ivas-pytest-anchor ivas-pytest-compare_ref-long-enc-lev-10: @@ -570,7 +574,6 @@ ivas-pytest-compare_ref-long-enc-lev-10: - DUT_DECODER_PATH=./IVAS_dec_ref - TEST_SUITE="$LONG_TEST_SUITE" - LEVEL_SCALING=0.3162 - - REFERENCE_BRANCH_LOCAL="$REFERENCE_BRANCH" <<: *ivas-pytest-anchor ivas-pytest-compare_ref-long-dec-lev-10: @@ -582,7 +585,6 @@ ivas-pytest-compare_ref-long-dec-lev-10: - DUT_ENCODER_PATH=./IVAS_cod_ref - TEST_SUITE="$LONG_TEST_SUITE" - LEVEL_SCALING=0.3162 - - REFERENCE_BRANCH_LOCAL="$REFERENCE_BRANCH" <<: *ivas-pytest-anchor ivas-pytest-compare_ref-long-enc-lev+10: @@ -594,7 +596,6 @@ ivas-pytest-compare_ref-long-enc-lev+10: - DUT_DECODER_PATH=./IVAS_dec_ref - TEST_SUITE="$LONG_TEST_SUITE" - LEVEL_SCALING=3.162 - - REFERENCE_BRANCH_LOCAL="$REFERENCE_BRANCH" <<: *ivas-pytest-anchor ivas-pytest-compare_ref-long-dec-lev+10: @@ -606,7 +607,6 @@ ivas-pytest-compare_ref-long-dec-lev+10: - DUT_ENCODER_PATH=./IVAS_cod_ref - TEST_SUITE="$LONG_TEST_SUITE" - LEVEL_SCALING=3.162 - - REFERENCE_BRANCH_LOCAL="$REFERENCE_BRANCH" <<: *ivas-pytest-anchor ivas-smoke-test-saturation: -- GitLab From 6952a41067d8053733e080272086961a9ab51642 Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Thu, 25 Jul 2024 09:50:53 +0200 Subject: [PATCH 03/26] add anchor for mr pl regression test --- .gitlab-ci.yml | 43 +++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 41 insertions(+), 2 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 249505213..07178384f 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -104,7 +104,7 @@ stages: # safeguard check to detect possible mistakes in the reference branch selection logic - if [ "$?" != "0" ]; then - echo "REFERENCE_BRANCH_LOCAL variable is set incorrectly: $REFERENCE_BRANCH_LOCAL!" - - exit(1) + - exit 1 - fi - git pull - make clean @@ -293,7 +293,46 @@ stages: reports: junit: - report-junit.xml - + +.ivas-pytest-on-merge-request-anchor: $ivas-pytest-on-merge-request-anchor + stage: test + needs: ["build-codec-linux-make"] + timeout: "300 minutes" + script: + - *print-common-info + - *update-scripts-repo + - if [ $USE_LTV -eq 1 ]; then + - *update-ltv-repo + - *copy-ltv-files-to-testv-dir + - testcase_timeout=$TESTCASE_TIMEOUT_LTV + - else + - testcase_timeout=$TESTCASE_TIMEOUT_STV + - fi + + - python3 ci/remove_unsupported_testcases.py $PRM_FILES + - if [ $LEVEL_SCALING != "1.0" ];then + - *apply-testv-scaling + - fi + + ### run branch first + # this per default builds the branch and the reference and creates the reference outputs + - *build-and-create-reference-outputs + - exit_code=0 + - python3 -m pytest --tb=no $TEST_SUITE -v --create_cut --html=report.html --self-contained-html --junit-xml=report-junit.xml --mld --ssnr --odg --dut_encoder_path $DUT_ENCODER_PATH --dut_decoder_path $DUT_DECODER_PATH -n auto --testcase_timeout $testcase_timeout || exit_code=$? + - zero_errors_branch=$(cat report-junit.xml | grep -c 'errors="0"') || true + - python3 scripts/parse_xml_report.py report-junit.xml scores_branch.csv + + ### run main now + git checkout main + - make clean + - make -j + - python3 -m pytest --tb=no $TEST_SUITE -v --create_cut --html=report.html --self-contained-html --junit-xml=report-junit.xml --mld --ssnr --odg --dut_encoder_path $DUT_ENCODER_PATH --dut_decoder_path $DUT_DECODER_PATH -n auto --testcase_timeout $testcase_timeout || exit_code=$? + - zero_errors_main=$(cat report-junit.xml | grep -c 'errors="0"') || true + - python3 scripts/parse_xml_report.py report-junit.xml scores_main.csv + + ### compare the two csv files for regressions + - python3 ci/basop-pages/create_report_pages.py /dev/null scores_merged.csv scores_branch.csv scores_main.csv branch main $CI_JOB_NAME + .ivas-pytest-sanitizers-anchor: &ivas-pytest-sanitizers-anchor stage: test -- GitLab From f0f70b0c69c839decfa996be4b3f1e242c8191f6 Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Thu, 25 Jul 2024 15:20:47 +0200 Subject: [PATCH 04/26] finish anchor for merge request jobs --- .gitlab-ci.yml | 61 +++++++++++++++++++++++++++++++++++++++++--------- 1 file changed, 51 insertions(+), 10 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 07178384f..4c18deb82 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -21,6 +21,10 @@ variables: TESTCASE_TIMEOUT_STV: 900 TESTCASE_TIMEOUT_LTV: 2400 TESTCASE_TIMEOUT_STV_SANITIZERS: 1800 + CI_REGRESSION_THRESH_MLD: 0.1 + CI_REGRESSION_THRESH_MAX_ABS_DIFF: 5 + CI_REGRESSION_THRESH_SSNR: -1 + CI_REGRESSION_THRESH_ODG: -0.05 GIT_CLEAN_FLAGS: -ffdxq MANUAL_PIPELINE_TYPE: description: "Type for the manual pipeline run. Use 'pytest-compare' to run comparison test against reference float codec." # Not implemented yet, but may be good to have a manual pipeline trigger @@ -252,7 +256,7 @@ stages: - python3 scripts/parse_xml_report.py report-junit.xml $CSV_ARTIFACT_NAME - mkdir $IMAGES_ARTIFACT_NAME - - for MEASURE in MLD DIFF SSNR;do python3 scripts/create_histogram_summary.py $CSV_ARTIFACT_NAME images_"$CI_JOB_NAME"/summary_"$MEASURE".csv images_"$CI_JOB_NAME"/summary_"$MEASURE".png --measure $MEASURE; done + - for MEASURE in MLD DIFF SSNR;do python3 scripts/create_histogram_summary.py $CSV_ARTIFACT_NAME $IMAGES_ARTIFACT_NAME/summary_"$MEASURE".csv $IMAGES_ARTIFACT_NAME/summary_"$MEASURE".png --measure $MEASURE; done - python3 ci/basop-pages/create_summary_page.py $SUMMARY_HTML_ARTIFACT_NAME $CI_JOB_ID $CI_JOB_NAME - if [ $USE_LTV -eq 1 ] && [ "$CI_COMMIT_BRANCH" == "$CI_DEFAULT_BRANCH" ]; then @@ -286,7 +290,6 @@ stages: - $PAGES_HTML_ARTIFACT_NAME - $CSV_ARTIFACT_NAME - $MERGED_CSV_ARTIFACT_NAME - - images - $SUMMARY_HTML_ARTIFACT_NAME - $IMAGES_ARTIFACT_NAME expose_as: "pytest compare results" @@ -298,6 +301,15 @@ stages: stage: test needs: ["build-codec-linux-make"] timeout: "300 minutes" + variables: + XML_REPORT_BRANCH: "report-junit-branch-$CI_JOB_NAME--sha-$CI_COMMIT_SHORT_SHA.xml" + XML_REPORT_MAIN: "report-junit-main-$CI_JOB_NAME--sha-$CI_COMMIT_SHORT_SHA.xml" + HTML_REPORT_BRANCH: "report-junit-branch-$CI_JOB_NAME--sha-$CI_COMMIT_SHORT_SHA.html" + HTML_REPORT_MAIN: "report-junit-main-$CI_JOB_NAME--sha-$CI_COMMIT_SHORT_SHA.html" + CSV_BRANCH: "scores-branch-$CI_JOB_NAME--sha-$CI_COMMIT_SHORT_SHA.csv" + CSV_MAIN: "scores-main-$CI_JOB_NAME--sha-$CI_COMMIT_SHORT_SHA.csv" + IMAGES_ARTIFACT_NAME: "images_$CI_JOB_NAME" + SUMMARY_HTML_ARTIFACT_NAME: "summary_$CI_JOB_NAME.html" script: - *print-common-info - *update-scripts-repo @@ -318,20 +330,49 @@ stages: # this per default builds the branch and the reference and creates the reference outputs - *build-and-create-reference-outputs - exit_code=0 - - python3 -m pytest --tb=no $TEST_SUITE -v --create_cut --html=report.html --self-contained-html --junit-xml=report-junit.xml --mld --ssnr --odg --dut_encoder_path $DUT_ENCODER_PATH --dut_decoder_path $DUT_DECODER_PATH -n auto --testcase_timeout $testcase_timeout || exit_code=$? - - zero_errors_branch=$(cat report-junit.xml | grep -c 'errors="0"') || true - - python3 scripts/parse_xml_report.py report-junit.xml scores_branch.csv + - python3 -m pytest --tb=no $TEST_SUITE -v --create_cut --html=$HTML_REPORT_BRANCH --self-contained-html --junit-xml=$XML_REPORT_BRANCH --mld --ssnr --odg --dut_encoder_path $DUT_ENCODER_PATH --dut_decoder_path $DUT_DECODER_PATH -n auto --testcase_timeout $testcase_timeout || exit_code=$? + - zero_errors_branch=$(cat $XML_REPORT_BRANCH | grep -c 'errors="0"') || true + - python3 scripts/parse_xml_report.py $XML_REPORT_BRANCH $CSV_BRANCH + + # create the summary based on the branch + - mkdir $IMAGES_ARTIFACT_NAME + - for MEASURE in MLD DIFF SSNR;do python3 scripts/create_histogram_summary.py $CSV_BRANCH $IMAGES_ARTIFACT_NAME/summary_"$MEASURE".csv $IMAGES_ARTIFACT_NAME/summary_"$MEASURE".png --measure $MEASURE; done + - python3 ci/basop-pages/create_summary_page.py $SUMMARY_HTML_ARTIFACT_NAME $CI_JOB_ID $CI_JOB_NAME ### run main now - git checkout main + - git checkout main - make clean - make -j - - python3 -m pytest --tb=no $TEST_SUITE -v --create_cut --html=report.html --self-contained-html --junit-xml=report-junit.xml --mld --ssnr --odg --dut_encoder_path $DUT_ENCODER_PATH --dut_decoder_path $DUT_DECODER_PATH -n auto --testcase_timeout $testcase_timeout || exit_code=$? - - zero_errors_main=$(cat report-junit.xml | grep -c 'errors="0"') || true - - python3 scripts/parse_xml_report.py report-junit.xml scores_main.csv + - python3 -m pytest --tb=no $TEST_SUITE -v --create_cut --html=$HTML_REPORT_MAIN --self-contained-html --junit-xml=$XML_REPORT_MAIN --mld --ssnr --odg --dut_encoder_path $DUT_ENCODER_PATH --dut_decoder_path $DUT_DECODER_PATH -n auto --testcase_timeout $testcase_timeout || true + - python3 scripts/parse_xml_report.py $XML_REPORT_MAIN $CSV_MAIN ### compare the two csv files for regressions - - python3 ci/basop-pages/create_report_pages.py /dev/null scores_merged.csv scores_branch.csv scores_main.csv branch main $CI_JOB_NAME + - regressions_found=$(python3 scripts/basop_check_for_changes_in_testcases.py $CSV_BRANCH $CSV_MAIN + + - if [ $regressions_found != 0 ]; then echo "Detected regression wrt to main!"; exit $EXIT_CODE_FAIL; fi + - if [ $zero_errors_branch != 1 ]; then echo "Run errors encountered!"; exit $EXIT_CODE_FAIL; fi + - if [ $exit_code -eq 1 ]; then echo "Differences encountered"; exit $EXIT_CODE_NON_BE; fi + - exit 0 + allow_failure: + exit_codes: + - 123 + artifacts: + name: "$CI_JOB_NAME--sha-$CI_COMMIT_SHORT_SHA--results" + expire_in: 1 week + when: always + paths: + - $XML_REPORT_BRANCH + - $XML_REPORT_MAIN + - $HTML_REPORT_BRANCH + - $HTML_REPORT_MAIN + - $CSV_BRANCH + - $CSV_MAIN + - $SUMMARY_HTML_ARTIFACT_NAME + - $IMAGES_ARTIFACT_NAME + expose_as: "pytest compare results" + reports: + junit: + - report-junit.xml .ivas-pytest-sanitizers-anchor: &ivas-pytest-sanitizers-anchor -- GitLab From 49703087821a8863e0ddd597173b57ab8fb2f02a Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Thu, 25 Jul 2024 15:32:51 +0200 Subject: [PATCH 05/26] add new jobs --- .gitlab-ci.yml | 107 ++++++++++++++++++++++++++++++++++++++++++------- 1 file changed, 92 insertions(+), 15 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 4c18deb82..386cb0c1f 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -165,16 +165,21 @@ stages: .activate-Werror-linux: &activate-Werror-linux - sed -i.bak "s/^# \(CFLAGS += -Werror\)/\1/" Makefile -.rules-pytest-short: +.rules-pytest-to-ref-short: rules: - if: $CI_PIPELINE_SOURCE == 'web' && $MANUAL_PIPELINE_TYPE == "pytest-compare" - if: $CI_PIPELINE_SOURCE == 'push' && $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH - - if: $CI_PIPELINE_SOURCE == 'merge_request_event' && $CI_MERGE_REQUEST_TARGET_BRANCH_NAME == "main" # only have MR pipelines for MRs to main - if: $CI_PIPELINE_SOURCE == 'push' when: never - if: $CI_PIPELINE_SOURCE == 'schedule' when: never +.rules-pytest-to-main-short: + rules: + - if: $CI_PIPELINE_SOURCE == 'merge_request_event' && $CI_MERGE_REQUEST_TARGET_BRANCH_NAME == "main" # only have MR pipelines for MRs to main + - if: $CI_PIPELINE_SOURCE == 'push' + when: never + .rules-pytest-long: rules: - if: $PYTEST_MLD_LONG # Set by scheduled pipeline @@ -500,13 +505,85 @@ build-codec-linux-instrumented-make: - make -j # --------------------------------------------------------------- -# Short test jobs +# Short test jobs that run in merge request pipelines +# --------------------------------------------------------------- + +### jobs that test fx encoder -> flt decoder +ivas-pytest-compare_to_main-enc: + extends: + - .rules-pytest-to-main-short + - .test-job-linux + before_script: + - USE_LTV=0 + - DUT_DECODER_PATH=./IVAS_dec_ref + - TEST_SUITE="$SHORT_TEST_SUITE" + - LEVEL_SCALING=1.0 + <<: *ivas-pytest-on-merge-request-anchor + +ivas-pytest-compare_to_main-enc-lev-10: + extends: + - .rules-pytest-to-main-short + - .test-job-linux + before_script: + - USE_LTV=0 + - DUT_DECODER_PATH=./IVAS_dec_ref + - TEST_SUITE="$SHORT_TEST_SUITE" + - LEVEL_SCALING=0.3162 + <<: *ivas-pytest-on-merge-request-anchor + +ivas-pytest-compare_to_main-enc-lev+10: + extends: + - .rules-pytest-to-main-short + - .test-job-linux + before_script: + - USE_LTV=0 + - DUT_DECODER_PATH=./IVAS_dec_ref + - TEST_SUITE="$SHORT_TEST_SUITE" + - LEVEL_SCALING=3.162 + <<: *ivas-pytest-on-merge-request-anchor + +### jobs that test flt encoder -> fx decoder +ivas-pytest-compare_to_main-short-dec: + extends: + - .rules-pytest-to-main-short + - .test-job-linux + before_script: + - USE_LTV=0 + - DUT_ENCODER_PATH=./IVAS_cod_ref + - TEST_SUITE="$SHORT_TEST_SUITE" + - LEVEL_SCALING=1.0 + <<: *ivas-pytest-on-merge-request-anchor + +ivas-pytest-compare_to_main-short-lev-10: + extends: + - .rules-pytest-to-main-short + - .test-job-linux + before_script: + - USE_LTV=0 + - DUT_ENCODER_PATH=./IVAS_cod_ref + - TEST_SUITE="$SHORT_TEST_SUITE" + - LEVEL_SCALING=0.3162 + <<: *ivas-pytest-on-merge-request-anchor + +ivas-pytest-compare_to_main-short-dec-lev+10: + extends: + - .rules-pytest-to-main-short + - .test-job-linux + before_script: + - USE_LTV=0 + - DUT_ENCODER_PATH=./IVAS_cod_ref + - TEST_SUITE="$SHORT_TEST_SUITE" + - LEVEL_SCALING=3.162 + <<: *ivas-pytest-on-merge-request-anchor + +# --------------------------------------------------------------- +# Short test jobs for running from web interface or schedule # --------------------------------------------------------------- ### jobs that test fx encoder -> flt decoder -ivas-pytest-compare-enc: +ivas-pytest-compare_to_ref-enc: extends: - - .rules-pytest-short + - .rules-pytest-to-ref-short - .test-job-linux before_script: - USE_LTV=0 @@ -515,9 +592,9 @@ ivas-pytest-compare-enc: - LEVEL_SCALING=1.0 <<: *ivas-pytest-anchor -ivas-pytest-compare-enc-lev-10: +ivas-pytest-compare_to_ref-enc-lev-10: extends: - - .rules-pytest-short + - .rules-pytest-to-ref-short - .test-job-linux before_script: - USE_LTV=0 @@ -526,9 +603,9 @@ ivas-pytest-compare-enc-lev-10: - LEVEL_SCALING=0.3162 <<: *ivas-pytest-anchor -ivas-pytest-compare-enc-lev+10: +ivas-pytest-compare_to_ref-enc-lev+10: extends: - - .rules-pytest-short + - .rules-pytest-to-ref-short - .test-job-linux before_script: - USE_LTV=0 @@ -562,9 +639,9 @@ ivas-pytest-enc-usan: <<: *ivas-pytest-sanitizers-anchor ### jobs that test flt encoder -> fx decoder -ivas-pytest-compare_main-short-dec: +ivas-pytest-compare_to_ref-short-dec: extends: - - .rules-pytest-short + - .rules-pytest-to-ref-short - .test-job-linux before_script: - USE_LTV=0 @@ -573,9 +650,9 @@ ivas-pytest-compare_main-short-dec: - LEVEL_SCALING=1.0 <<: *ivas-pytest-anchor -ivas-pytest-compare_main-short-lev-10: +ivas-pytest-compare_to_ref-short-lev-10: extends: - - .rules-pytest-short + - .rules-pytest-to-ref-short - .test-job-linux before_script: - USE_LTV=0 @@ -584,9 +661,9 @@ ivas-pytest-compare_main-short-lev-10: - LEVEL_SCALING=0.3162 <<: *ivas-pytest-anchor -ivas-pytest-compare_main-short-dec-lev+10: +ivas-pytest-compare_to_ref-short-dec-lev+10: extends: - - .rules-pytest-short + - .rules-pytest-to-ref-short - .test-job-linux before_script: - USE_LTV=0 -- GitLab From 125e9d0cf54f9c73ea6e5c848c157dd6816ce0fd Mon Sep 17 00:00:00 2001 From: kiene Date: Thu, 25 Jul 2024 13:34:18 +0000 Subject: [PATCH 06/26] Update .gitlab-ci.yml file --- .gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 386cb0c1f..d67d0160a 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -302,7 +302,7 @@ stages: junit: - report-junit.xml -.ivas-pytest-on-merge-request-anchor: $ivas-pytest-on-merge-request-anchor +.ivas-pytest-on-merge-request-anchor: &ivas-pytest-on-merge-request-anchor stage: test needs: ["build-codec-linux-make"] timeout: "300 minutes" -- GitLab From e1cb7e7a6c7e0de505f039ae8a4e495f90ee4701 Mon Sep 17 00:00:00 2001 From: kiene Date: Thu, 25 Jul 2024 13:37:01 +0000 Subject: [PATCH 07/26] Update .gitlab-ci.yml file --- .gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index d67d0160a..14f873096 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -107,7 +107,7 @@ stages: - git checkout $REFERENCE_BRANCH_LOCAL # safeguard check to detect possible mistakes in the reference branch selection logic - if [ "$?" != "0" ]; then - - echo "REFERENCE_BRANCH_LOCAL variable is set incorrectly: $REFERENCE_BRANCH_LOCAL!" + - echo "REFERENCE_BRANCH_LOCAL variable is set incorrectly ($REFERENCE_BRANCH_LOCAL)!" - exit 1 - fi - git pull -- GitLab From e97d12b05f5a866825a1f21d45ed05f4939738bd Mon Sep 17 00:00:00 2001 From: kiene Date: Thu, 25 Jul 2024 13:40:36 +0000 Subject: [PATCH 08/26] Update .gitlab-ci.yml file --- .gitlab-ci.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 14f873096..956410d19 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -21,10 +21,10 @@ variables: TESTCASE_TIMEOUT_STV: 900 TESTCASE_TIMEOUT_LTV: 2400 TESTCASE_TIMEOUT_STV_SANITIZERS: 1800 - CI_REGRESSION_THRESH_MLD: 0.1 - CI_REGRESSION_THRESH_MAX_ABS_DIFF: 5 - CI_REGRESSION_THRESH_SSNR: -1 - CI_REGRESSION_THRESH_ODG: -0.05 + CI_REGRESSION_THRESH_MLD: "0.1" + CI_REGRESSION_THRESH_MAX_ABS_DIFF: "5" + CI_REGRESSION_THRESH_SSNR: "-1" + CI_REGRESSION_THRESH_ODG: "-0.05" GIT_CLEAN_FLAGS: -ffdxq MANUAL_PIPELINE_TYPE: description: "Type for the manual pipeline run. Use 'pytest-compare' to run comparison test against reference float codec." # Not implemented yet, but may be good to have a manual pipeline trigger -- GitLab From bc9c1826e715b67f259e1a6b0b67ed82914746cf Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Fri, 26 Jul 2024 09:41:01 +0200 Subject: [PATCH 09/26] clean out old approach with setting reference branch --- .gitlab-ci.yml | 13 +------------ 1 file changed, 1 insertion(+), 12 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 956410d19..e2081564f 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -3,8 +3,6 @@ variables: LTV_DIR: "/usr/local/ltv" EVS_BE_TEST_DIR_BASOP: "/usr/local/be_2_evs_basop" REFERENCE_BRANCH: "ivas-float-update" - # this is intentionally set to a nonsense value to trigger a clear error if there is a flaw in the variable setting logic - REFERENCE_BRANCH_LOCAL: "xa34wshgaweedfaerssfgsfdg" BUILD_OUTPUT: "build_output.txt" SCRIPTS_DIR: "/usr/local/scripts" EXIT_CODE_NON_BE: 123 @@ -104,12 +102,7 @@ stages: .build-reference-and-dut-binaries: &build-reference-and-dut-binaries - current_commit_sha=$(git rev-parse HEAD) ### build reference binaries - - git checkout $REFERENCE_BRANCH_LOCAL - # safeguard check to detect possible mistakes in the reference branch selection logic - - if [ "$?" != "0" ]; then - - echo "REFERENCE_BRANCH_LOCAL variable is set incorrectly ($REFERENCE_BRANCH_LOCAL)!" - - exit 1 - - fi + - git checkout $REFERENCE_BRANCH - git pull - make clean - make -j @@ -244,10 +237,6 @@ stages: - testcase_timeout=$TESTCASE_TIMEOUT_STV - fi - - if [ $CI_PIPELINE_SOURCE == "web" || $CI_PIPELINE_SOURCE == "schedule" ]; then - - REFERENCE_BRANCH_LOCAL="$REFERENCE_BRANCH" - - fi - - python3 ci/remove_unsupported_testcases.py $PRM_FILES - if [ $LEVEL_SCALING != "1.0" ];then - *apply-testv-scaling -- GitLab From 954414bc901c7f1fdcf7b236ef14c6f971907937 Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Fri, 26 Jul 2024 10:00:21 +0200 Subject: [PATCH 10/26] add missing paranthesis --- .gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index e2081564f..11ea22478 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -341,7 +341,7 @@ stages: - python3 scripts/parse_xml_report.py $XML_REPORT_MAIN $CSV_MAIN ### compare the two csv files for regressions - - regressions_found=$(python3 scripts/basop_check_for_changes_in_testcases.py $CSV_BRANCH $CSV_MAIN + - regressions_found=$(python3 scripts/basop_check_for_changes_in_testcases.py $CSV_BRANCH $CSV_MAIN) - if [ $regressions_found != 0 ]; then echo "Detected regression wrt to main!"; exit $EXIT_CODE_FAIL; fi - if [ $zero_errors_branch != 1 ]; then echo "Run errors encountered!"; exit $EXIT_CODE_FAIL; fi -- GitLab From a7e93eaec8531cfd98bbb1f1b62ef0e7840d13aa Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Fri, 26 Jul 2024 10:23:18 +0200 Subject: [PATCH 11/26] correctly get the return value from the check script --- .gitlab-ci.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 11ea22478..22129ac0a 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -341,7 +341,8 @@ stages: - python3 scripts/parse_xml_report.py $XML_REPORT_MAIN $CSV_MAIN ### compare the two csv files for regressions - - regressions_found=$(python3 scripts/basop_check_for_changes_in_testcases.py $CSV_BRANCH $CSV_MAIN) + - regressions_found=0 + - python3 scripts/basop_check_for_changes_in_testcases.py $CSV_BRANCH $CSV_MAIN || regressions_found=$? - if [ $regressions_found != 0 ]; then echo "Detected regression wrt to main!"; exit $EXIT_CODE_FAIL; fi - if [ $zero_errors_branch != 1 ]; then echo "Run errors encountered!"; exit $EXIT_CODE_FAIL; fi -- GitLab From 7c39f53f7fa459de4f2c47469a56b4aa2bb8868d Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Fri, 26 Jul 2024 10:30:49 +0200 Subject: [PATCH 12/26] harmonize job names --- .gitlab-ci.yml | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 22129ac0a..1716b3ef5 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -499,7 +499,7 @@ build-codec-linux-instrumented-make: # --------------------------------------------------------------- ### jobs that test fx encoder -> flt decoder -ivas-pytest-compare_to_main-enc: +ivas-pytest-compare_to_main-short-enc: extends: - .rules-pytest-to-main-short - .test-job-linux @@ -510,7 +510,7 @@ ivas-pytest-compare_to_main-enc: - LEVEL_SCALING=1.0 <<: *ivas-pytest-on-merge-request-anchor -ivas-pytest-compare_to_main-enc-lev-10: +ivas-pytest-compare_to_main-short-enc-lev-10: extends: - .rules-pytest-to-main-short - .test-job-linux @@ -521,7 +521,7 @@ ivas-pytest-compare_to_main-enc-lev-10: - LEVEL_SCALING=0.3162 <<: *ivas-pytest-on-merge-request-anchor -ivas-pytest-compare_to_main-enc-lev+10: +ivas-pytest-compare_to_main-short-enc-lev+10: extends: - .rules-pytest-to-main-short - .test-job-linux @@ -544,7 +544,7 @@ ivas-pytest-compare_to_main-short-dec: - LEVEL_SCALING=1.0 <<: *ivas-pytest-on-merge-request-anchor -ivas-pytest-compare_to_main-short-lev-10: +ivas-pytest-compare_to_main-short-dec-lev-10: extends: - .rules-pytest-to-main-short - .test-job-linux @@ -571,7 +571,7 @@ ivas-pytest-compare_to_main-short-dec-lev+10: # --------------------------------------------------------------- ### jobs that test fx encoder -> flt decoder -ivas-pytest-compare_to_ref-enc: +ivas-pytest-compare_to_ref-short-enc: extends: - .rules-pytest-to-ref-short - .test-job-linux @@ -582,7 +582,7 @@ ivas-pytest-compare_to_ref-enc: - LEVEL_SCALING=1.0 <<: *ivas-pytest-anchor -ivas-pytest-compare_to_ref-enc-lev-10: +ivas-pytest-compare_to_ref-short-enc-lev-10: extends: - .rules-pytest-to-ref-short - .test-job-linux @@ -593,7 +593,7 @@ ivas-pytest-compare_to_ref-enc-lev-10: - LEVEL_SCALING=0.3162 <<: *ivas-pytest-anchor -ivas-pytest-compare_to_ref-enc-lev+10: +ivas-pytest-compare_to_ref-short-enc-lev+10: extends: - .rules-pytest-to-ref-short - .test-job-linux -- GitLab From 250e2b981152d2161154630ae7b27d54f1af774e Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Wed, 31 Jul 2024 09:29:59 +0200 Subject: [PATCH 13/26] add MR title flag mechanism for allowing regressions --- .gitlab-ci.yml | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 1716b3ef5..32f42f489 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -320,6 +320,10 @@ stages: - *apply-testv-scaling - fi + # check MR title for flag that allows regressions to be mergable + - echo $CI_MERGE_REQUEST_TITLE > tmp.txt + - allow_regressions_flag=$(grep -c --ignore-case "\[allow[ -]*regression\]" tmp.txt) || true + ### run branch first # this per default builds the branch and the reference and creates the reference outputs - *build-and-create-reference-outputs @@ -344,8 +348,15 @@ stages: - regressions_found=0 - python3 scripts/basop_check_for_changes_in_testcases.py $CSV_BRANCH $CSV_MAIN || regressions_found=$? - - if [ $regressions_found != 0 ]; then echo "Detected regression wrt to main!"; exit $EXIT_CODE_FAIL; fi - - if [ $zero_errors_branch != 1 ]; then echo "Run errors encountered!"; exit $EXIT_CODE_FAIL; fi + - if [ $regressions_found != 0 ]; then + - echo "Detected regression wrt to main!" + - if [ $allow_regressions_flag == 0 ]; then + - exit $EXIT_CODE_FAIL; + - else + - exit $EXIT_CODE_NON_BE; + - fi + - fi + - if [ $zero_errors_branch != 1 ]; then echo "Run errors encountered!"; exit $EXIT_CODE_NON_BE; fi - if [ $exit_code -eq 1 ]; then echo "Differences encountered"; exit $EXIT_CODE_NON_BE; fi - exit 0 allow_failure: -- GitLab From 2f1d28560ac45043a7fff9131bcdbd3ffb8eee06 Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Wed, 31 Jul 2024 10:26:56 +0200 Subject: [PATCH 14/26] add new artifact --- .gitlab-ci.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 32f42f489..696dcac49 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -375,6 +375,7 @@ stages: - $CSV_MAIN - $SUMMARY_HTML_ARTIFACT_NAME - $IMAGES_ARTIFACT_NAME + - changes*.csv expose_as: "pytest compare results" reports: junit: -- GitLab From 40ba58c4750675c712c111c72b9fc142efd58b1b Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Wed, 31 Jul 2024 10:30:37 +0200 Subject: [PATCH 15/26] fix artifact names --- .gitlab-ci.yml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 696dcac49..65389e157 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -375,7 +375,11 @@ stages: - $CSV_MAIN - $SUMMARY_HTML_ARTIFACT_NAME - $IMAGES_ARTIFACT_NAME - - changes*.csv + - changes_crashes.csv + - changes_MLD.csv + - changes_MAXIMUM_ABS_DIFF.csv + - changes_MIN_SSNR.csv + - changes_MIN_ODG.csv expose_as: "pytest compare results" reports: junit: -- GitLab From 319467bb0ed8952359171b308a95b2fb3ec88080 Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Wed, 31 Jul 2024 15:16:52 +0200 Subject: [PATCH 16/26] for encoder test jobs, allow any regression for now until it is stable --- .gitlab-ci.yml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 65389e157..d81f421fc 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -324,6 +324,12 @@ stages: - echo $CI_MERGE_REQUEST_TITLE > tmp.txt - allow_regressions_flag=$(grep -c --ignore-case "\[allow[ -]*regression\]" tmp.txt) || true + # TODO: temporary patch, remove once encoder is stable + # allow regressions for encoder tests + - if [[ "$CI_JOB_NAME" =~ "ivas-pytest-compare_to_main-short-enc" ]]; then + - allow_regressions_flag=1 + - fi + ### run branch first # this per default builds the branch and the reference and creates the reference outputs - *build-and-create-reference-outputs -- GitLab From 79f53b959df2ba805d456ba284a8d01f0fe86bf7 Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Wed, 31 Jul 2024 15:51:50 +0200 Subject: [PATCH 17/26] remove obsolete comment --- .gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index d81f421fc..5083eff85 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -25,7 +25,7 @@ variables: CI_REGRESSION_THRESH_ODG: "-0.05" GIT_CLEAN_FLAGS: -ffdxq MANUAL_PIPELINE_TYPE: - description: "Type for the manual pipeline run. Use 'pytest-compare' to run comparison test against reference float codec." # Not implemented yet, but may be good to have a manual pipeline trigger + description: "Type for the manual pipeline run. Use 'pytest-compare' to run comparison test against reference float codec." value: 'default' options: - 'default' -- GitLab From 6ecfc4217b3bab5e0404e067d386c177af60f9ac Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Wed, 7 Aug 2024 14:36:57 +0200 Subject: [PATCH 18/26] add explicitly codec binary paths to new pytest calls --- .gitlab-ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 6ab00966c..2866cbea8 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -336,7 +336,7 @@ stages: # this per default builds the branch and the reference and creates the reference outputs - *build-and-create-reference-outputs - exit_code=0 - - python3 -m pytest --tb=no $TEST_SUITE -v --create_cut --html=$HTML_REPORT_BRANCH --self-contained-html --junit-xml=$XML_REPORT_BRANCH --mld --ssnr --odg --dut_encoder_path $DUT_ENCODER_PATH --dut_decoder_path $DUT_DECODER_PATH -n auto --testcase_timeout $testcase_timeout || exit_code=$? + - python3 -m pytest --tb=no $TEST_SUITE -v --create_cut --html=$HTML_REPORT_BRANCH --self-contained-html --junit-xml=$XML_REPORT_BRANCH --mld --ssnr --odg --ref_encoder_path $REF_ENCODER_PATH --ref_decoder_path $REF_DECODER_PATH --dut_encoder_path $DUT_ENCODER_PATH --dut_decoder_path $DUT_DECODER_PATH -n auto --testcase_timeout $testcase_timeout || exit_code=$? - zero_errors_branch=$(cat $XML_REPORT_BRANCH | grep -c 'errors="0"') || true - python3 scripts/parse_xml_report.py $XML_REPORT_BRANCH $CSV_BRANCH @@ -349,7 +349,7 @@ stages: - git checkout main - make clean - make -j - - python3 -m pytest --tb=no $TEST_SUITE -v --create_cut --html=$HTML_REPORT_MAIN --self-contained-html --junit-xml=$XML_REPORT_MAIN --mld --ssnr --odg --dut_encoder_path $DUT_ENCODER_PATH --dut_decoder_path $DUT_DECODER_PATH -n auto --testcase_timeout $testcase_timeout || true + - python3 -m pytest --tb=no $TEST_SUITE -v --create_cut --html=$HTML_REPORT_MAIN --self-contained-html --junit-xml=$XML_REPORT_MAIN --mld --ssnr --odg --ref_encoder_path $REF_ENCODER_PATH --ref_decoder_path $REF_DECODER_PATH --dut_encoder_path $DUT_ENCODER_PATH --dut_decoder_path $DUT_DECODER_PATH -n auto --testcase_timeout $testcase_timeout || true - python3 scripts/parse_xml_report.py $XML_REPORT_MAIN $CSV_MAIN ### compare the two csv files for regressions -- GitLab From f1fbc9a84e3e43a4aa8285e0313846d8535b73f6 Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Wed, 7 Aug 2024 14:49:31 +0200 Subject: [PATCH 19/26] switch encoder MR pipeline jobs to compare_to_ref only --- .gitlab-ci.yml | 22 ++++++++++++++++++---- 1 file changed, 18 insertions(+), 4 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 2866cbea8..304eb0b68 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -269,7 +269,15 @@ stages: - touch $MERGED_CSV_ARTIFACT_NAME - fi - - if [ $zero_errors != 1 ]; then echo "Run errors encountered!"; exit $EXIT_CODE_FAIL; fi + - if [ $zero_errors != 1 ]; then + - echo "Run errors encountered!"; + - # TODO: temporary only to not fail pipelines on crashes in encoder + - if [[ "$CI_JOB_NAME" =~ "ivas-pytest-compare_to_main-short-enc" ]]; then + - exit $EXIT_CODE_NON_BE + - else + - exit $EXIT_CODE_FAIL + - fi + - fi - if [ $exit_code -eq 1 ]; then echo "Differences encountered"; exit $EXIT_CODE_NON_BE; fi - exit 0 @@ -532,7 +540,9 @@ ivas-pytest-compare_to_main-short-enc: - DUT_DECODER_PATH=./IVAS_dec_ref - TEST_SUITE="$SHORT_TEST_SUITE" - LEVEL_SCALING=1.0 - <<: *ivas-pytest-on-merge-request-anchor + # TODO: use other anchor once encoder is stable + <<: *ivas-pytest-anchor + # <<: *ivas-pytest-on-merge-request-anchor ivas-pytest-compare_to_main-short-enc-lev-10: extends: @@ -543,7 +553,9 @@ ivas-pytest-compare_to_main-short-enc-lev-10: - DUT_DECODER_PATH=./IVAS_dec_ref - TEST_SUITE="$SHORT_TEST_SUITE" - LEVEL_SCALING=0.3162 - <<: *ivas-pytest-on-merge-request-anchor + # TODO: use other anchor once encoder is stable + <<: *ivas-pytest-anchor + # <<: *ivas-pytest-on-merge-request-anchor ivas-pytest-compare_to_main-short-enc-lev+10: extends: @@ -554,7 +566,9 @@ ivas-pytest-compare_to_main-short-enc-lev+10: - DUT_DECODER_PATH=./IVAS_dec_ref - TEST_SUITE="$SHORT_TEST_SUITE" - LEVEL_SCALING=3.162 - <<: *ivas-pytest-on-merge-request-anchor + # TODO: use other anchor once encoder is stable + <<: *ivas-pytest-anchor + # <<: *ivas-pytest-on-merge-request-anchor ### jobs that test flt encoder -> fx decoder ivas-pytest-compare_to_main-short-dec: -- GitLab From e9c37e99342d4174653ef2cf5bed9f0aaf958941 Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Wed, 7 Aug 2024 14:52:42 +0200 Subject: [PATCH 20/26] correct comment --- .gitlab-ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 304eb0b68..2913a316f 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -270,8 +270,8 @@ stages: - fi - if [ $zero_errors != 1 ]; then - - echo "Run errors encountered!"; - - # TODO: temporary only to not fail pipelines on crashes in encoder + - echo "Run errors encountered!" + # TODO: temporary only to not fail pipelines on crashes in encoder - if [[ "$CI_JOB_NAME" =~ "ivas-pytest-compare_to_main-short-enc" ]]; then - exit $EXIT_CODE_NON_BE - else -- GitLab From 33cdc8fbea4593445884292fcff7b2ddfc2abad0 Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Wed, 7 Aug 2024 15:00:59 +0200 Subject: [PATCH 21/26] use other jobs for encoder for naming clarity --- .gitlab-ci.yml | 99 ++++++++++++++++++++++++++------------------------ 1 file changed, 51 insertions(+), 48 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 2913a316f..8dc0374ea 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -169,6 +169,17 @@ stages: - if: $CI_PIPELINE_SOURCE == 'schedule' when: never +# TODO: only temporary as long the MR encoder tests should not compare to main +.rules-pytest-to-ref-enc-short-temp: + rules: + - if: $CI_PIPELINE_SOURCE == 'web' && $MANUAL_PIPELINE_TYPE == "pytest-compare" + - if: $CI_PIPELINE_SOURCE == 'push' && $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH + - if: $CI_PIPELINE_SOURCE == 'merge_request_event' && $CI_MERGE_REQUEST_TARGET_BRANCH_NAME == "main" # only have MR pipelines for MRs to main + - if: $CI_PIPELINE_SOURCE == 'push' + when: never + - if: $CI_PIPELINE_SOURCE == 'schedule' + when: never + .rules-pytest-to-main-short: rules: - if: $CI_PIPELINE_SOURCE == 'merge_request_event' && $CI_MERGE_REQUEST_TARGET_BRANCH_NAME == "main" # only have MR pipelines for MRs to main @@ -272,7 +283,7 @@ stages: - if [ $zero_errors != 1 ]; then - echo "Run errors encountered!" # TODO: temporary only to not fail pipelines on crashes in encoder - - if [[ "$CI_JOB_NAME" =~ "ivas-pytest-compare_to_main-short-enc" ]]; then + - if [[ "$CI_JOB_NAME" =~ "ivas-pytest-compare_to_ref-short-enc" ]]; then - exit $EXIT_CODE_NON_BE - else - exit $EXIT_CODE_FAIL @@ -334,12 +345,6 @@ stages: - echo $CI_MERGE_REQUEST_TITLE > tmp.txt - allow_regressions_flag=$(grep -c --ignore-case "\[allow[ -]*regression\]" tmp.txt) || true - # TODO: temporary patch, remove once encoder is stable - # allow regressions for encoder tests - - if [[ "$CI_JOB_NAME" =~ "ivas-pytest-compare_to_main-short-enc" ]]; then - - allow_regressions_flag=1 - - fi - ### run branch first # this per default builds the branch and the reference and creates the reference outputs - *build-and-create-reference-outputs @@ -531,44 +536,39 @@ build-codec-linux-instrumented-make: # --------------------------------------------------------------- ### jobs that test fx encoder -> flt decoder -ivas-pytest-compare_to_main-short-enc: - extends: - - .rules-pytest-to-main-short - - .test-job-linux - before_script: - - USE_LTV=0 - - DUT_DECODER_PATH=./IVAS_dec_ref - - TEST_SUITE="$SHORT_TEST_SUITE" - - LEVEL_SCALING=1.0 - # TODO: use other anchor once encoder is stable - <<: *ivas-pytest-anchor - # <<: *ivas-pytest-on-merge-request-anchor - -ivas-pytest-compare_to_main-short-enc-lev-10: - extends: - - .rules-pytest-to-main-short - - .test-job-linux - before_script: - - USE_LTV=0 - - DUT_DECODER_PATH=./IVAS_dec_ref - - TEST_SUITE="$SHORT_TEST_SUITE" - - LEVEL_SCALING=0.3162 - # TODO: use other anchor once encoder is stable - <<: *ivas-pytest-anchor - # <<: *ivas-pytest-on-merge-request-anchor - -ivas-pytest-compare_to_main-short-enc-lev+10: - extends: - - .rules-pytest-to-main-short - - .test-job-linux - before_script: - - USE_LTV=0 - - DUT_DECODER_PATH=./IVAS_dec_ref - - TEST_SUITE="$SHORT_TEST_SUITE" - - LEVEL_SCALING=3.162 - # TODO: use other anchor once encoder is stable - <<: *ivas-pytest-anchor - # <<: *ivas-pytest-on-merge-request-anchor +# TODO: reenable once encoder tests shall compare to main +# ivas-pytest-compare_to_main-short-enc: +# extends: +# - .rules-pytest-to-main-short +# - .test-job-linux +# before_script: +# - USE_LTV=0 +# - DUT_DECODER_PATH=./IVAS_dec_ref +# - TEST_SUITE="$SHORT_TEST_SUITE" +# - LEVEL_SCALING=1.0 +# <<: *ivas-pytest-on-merge-request-anchor + +# ivas-pytest-compare_to_main-short-enc-lev-10: +# extends: +# - .rules-pytest-to-main-short +# - .test-job-linux +# before_script: +# - USE_LTV=0 +# - DUT_DECODER_PATH=./IVAS_dec_ref +# - TEST_SUITE="$SHORT_TEST_SUITE" +# - LEVEL_SCALING=0.3162 +# <<: *ivas-pytest-on-merge-request-anchor + +# ivas-pytest-compare_to_main-short-enc-lev+10: +# extends: +# - .rules-pytest-to-main-short +# - .test-job-linux +# before_script: +# - USE_LTV=0 +# - DUT_DECODER_PATH=./IVAS_dec_ref +# - TEST_SUITE="$SHORT_TEST_SUITE" +# - LEVEL_SCALING=3.162 +# <<: *ivas-pytest-on-merge-request-anchor ### jobs that test flt encoder -> fx decoder ivas-pytest-compare_to_main-short-dec: @@ -611,7 +611,8 @@ ivas-pytest-compare_to_main-short-dec-lev+10: ### jobs that test fx encoder -> flt decoder ivas-pytest-compare_to_ref-short-enc: extends: - - .rules-pytest-to-ref-short + #- .rules-pytest-to-ref-short + - .rules-pytest-to-ref-enc-short-temp - .test-job-linux before_script: - USE_LTV=0 @@ -622,7 +623,8 @@ ivas-pytest-compare_to_ref-short-enc: ivas-pytest-compare_to_ref-short-enc-lev-10: extends: - - .rules-pytest-to-ref-short + #- .rules-pytest-to-ref-short + - .rules-pytest-to-ref-enc-short-temp - .test-job-linux before_script: - USE_LTV=0 @@ -633,7 +635,8 @@ ivas-pytest-compare_to_ref-short-enc-lev-10: ivas-pytest-compare_to_ref-short-enc-lev+10: extends: - - .rules-pytest-to-ref-short + #- .rules-pytest-to-ref-short + - .rules-pytest-to-ref-enc-short-temp - .test-job-linux before_script: - USE_LTV=0 -- GitLab From 7717cb374a8ba8d9ec72154da47471dcca1e2b2f Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Wed, 7 Aug 2024 17:06:25 +0200 Subject: [PATCH 22/26] when on main, do git pull to be up-to-date --- .gitlab-ci.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 8dc0374ea..d83916e0f 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -359,7 +359,8 @@ stages: - python3 ci/basop-pages/create_summary_page.py $SUMMARY_HTML_ARTIFACT_NAME $CI_JOB_ID $CI_JOB_NAME ### run main now - - git checkout main + - git checkout $CI_MERGE_REQUEST_TARGET_BRANCH_NAME + - git pull - make clean - make -j - python3 -m pytest --tb=no $TEST_SUITE -v --create_cut --html=$HTML_REPORT_MAIN --self-contained-html --junit-xml=$XML_REPORT_MAIN --mld --ssnr --odg --ref_encoder_path $REF_ENCODER_PATH --ref_decoder_path $REF_DECODER_PATH --dut_encoder_path $DUT_ENCODER_PATH --dut_decoder_path $DUT_DECODER_PATH -n auto --testcase_timeout $testcase_timeout || true -- GitLab From 713dbcf50e6593598414b3d243db37b53a5f2f36 Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Wed, 7 Aug 2024 17:34:27 +0200 Subject: [PATCH 23/26] fix report exposure --- .gitlab-ci.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index d83916e0f..a0fc6bf45 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -405,7 +405,8 @@ stages: expose_as: "pytest compare results" reports: junit: - - report-junit.xml + - $XML_REPORT_BRANCH + - $XML_REPORT_MAIN .ivas-pytest-sanitizers-anchor: &ivas-pytest-sanitizers-anchor -- GitLab From 8a63f4a949ee156f7400a048bf070e11606e43e3 Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Wed, 7 Aug 2024 17:37:19 +0200 Subject: [PATCH 24/26] increase threshold for abs diff --- .gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index a0fc6bf45..41c591617 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -22,7 +22,7 @@ variables: TESTCASE_TIMEOUT_LTV: 2400 TESTCASE_TIMEOUT_STV_SANITIZERS: 1800 CI_REGRESSION_THRESH_MLD: "0.1" - CI_REGRESSION_THRESH_MAX_ABS_DIFF: "5" + CI_REGRESSION_THRESH_MAX_ABS_DIFF: "50" CI_REGRESSION_THRESH_SSNR: "-1" CI_REGRESSION_THRESH_ODG: "-0.05" GIT_CLEAN_FLAGS: -ffdxq -- GitLab From daf37c760bbd90a5679cfe05c2f8075f6a72a008 Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Wed, 7 Aug 2024 18:16:11 +0200 Subject: [PATCH 25/26] add extra cleanup in tests --- .gitlab-ci.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 41c591617..72c45077c 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -381,6 +381,8 @@ stages: - if [ $zero_errors_branch != 1 ]; then echo "Run errors encountered!"; exit $EXIT_CODE_NON_BE; fi - if [ $exit_code -eq 1 ]; then echo "Differences encountered"; exit $EXIT_CODE_NON_BE; fi - exit 0 + after_script: + t- rm -rf tests/dut tests/ref allow_failure: exit_codes: - 123 @@ -582,6 +584,7 @@ ivas-pytest-compare_to_main-short-dec: - DUT_ENCODER_PATH=./IVAS_cod_ref - TEST_SUITE="$SHORT_TEST_SUITE" - LEVEL_SCALING=1.0 + - rm -rf tests/dut tests/ref <<: *ivas-pytest-on-merge-request-anchor ivas-pytest-compare_to_main-short-dec-lev-10: @@ -593,6 +596,7 @@ ivas-pytest-compare_to_main-short-dec-lev-10: - DUT_ENCODER_PATH=./IVAS_cod_ref - TEST_SUITE="$SHORT_TEST_SUITE" - LEVEL_SCALING=0.3162 + - rm -rf tests/dut tests/ref <<: *ivas-pytest-on-merge-request-anchor ivas-pytest-compare_to_main-short-dec-lev+10: @@ -604,6 +608,7 @@ ivas-pytest-compare_to_main-short-dec-lev+10: - DUT_ENCODER_PATH=./IVAS_cod_ref - TEST_SUITE="$SHORT_TEST_SUITE" - LEVEL_SCALING=3.162 + - rm -rf tests/dut tests/ref <<: *ivas-pytest-on-merge-request-anchor # --------------------------------------------------------------- -- GitLab From e234feb51432c3e7bcb732decefc0472ccd09094 Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Wed, 7 Aug 2024 19:25:23 +0200 Subject: [PATCH 26/26] fix typo --- .gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 72c45077c..6a77da6fa 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -382,7 +382,7 @@ stages: - if [ $exit_code -eq 1 ]; then echo "Differences encountered"; exit $EXIT_CODE_NON_BE; fi - exit 0 after_script: - t- rm -rf tests/dut tests/ref + - rm -rf tests/dut tests/ref allow_failure: exit_codes: - 123 -- GitLab