From 30c8e5080a56e7a17e3630ce96e492524a160f1c Mon Sep 17 00:00:00 2001 From: Erik Norvell Date: Fri, 11 Oct 2024 14:26:40 +0200 Subject: [PATCH 1/3] Add draft job pytest-compare-to-input-short-dec --- .gitlab-ci.yml | 105 +++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 105 insertions(+) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 169688653..8e3331224 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -34,6 +34,7 @@ variables: - 'default' - 'pytest-compare' - 'pytest-compare-long' + - 'pytest-compare-to-input' - 'pytest-saturation-smoke-test' - 'evs-26444' - 'sanitizer-stv' @@ -66,6 +67,9 @@ workflow: - if: $CI_PIPELINE_SOURCE == 'web' && $MANUAL_PIPELINE_TYPE == 'pytest-compare-long' variables: IVAS_PIPELINE_NAME: 'Run comparison tools against float ref (long test vectors): $CI_COMMIT_BRANCH' + - if: $CI_PIPELINE_SOURCE == 'web' && $MANUAL_PIPELINE_TYPE == 'pytest-compare-to-input' + variables: + IVAS_PIPELINE_NAME: 'Run comparison tools against input ref (pass-through only): $CI_COMMIT_BRANCH' - if: $CI_PIPELINE_SOURCE == 'web' && $MANUAL_PIPELINE_TYPE == 'pytest-saturation-smoke-test' variables: IVAS_PIPELINE_NAME: 'Run saturation smoke-test: $CI_COMMIT_BRANCH' @@ -174,6 +178,14 @@ stages: - if: $CI_PIPELINE_SOURCE == 'schedule' when: never +.rules-pytest-to-input-short: + rules: + - if: $CI_PIPELINE_SOURCE == 'web' && $MANUAL_PIPELINE_TYPE == "pytest-compare-to-input" + - if: $CI_PIPELINE_SOURCE == 'push' + when: never + - if: $CI_PIPELINE_SOURCE == 'schedule' + when: never + # TODO: only temporary as long the MR encoder tests should not compare to main .rules-pytest-to-ref-enc-short-temp: rules: @@ -471,6 +483,88 @@ stages: junit: - report-junit.xml +.ivas-pytest-compare-to-input-anchor: &ivas-pytest-compare-to-input-anchor + stage: test + needs: ["build-codec-linux-make"] + timeout: "240 minutes" + variables: + + script: + - *print-common-info + - *update-scripts-repo + - if [ $USE_LTV -eq 1 ]; then + - *update-ltv-repo + - *copy-ltv-files-to-testv-dir + - testcase_timeout=$TESTCASE_TIMEOUT_LTV + - else + - testcase_timeout=$TESTCASE_TIMEOUT_STV + - fi + + - python3 ci/remove_unsupported_testcases.py $PRM_FILES + - if [ $LEVEL_SCALING != "1.0" ];then + - *apply-testv-scaling + - fi + - *build-reference-and-dut-binaries + + ### run pytest + - exit_code=0 + - python3 -m pytest --tb=no $TEST_SUITE -v --create_cut --html=report-ref.html --self-contained-html --junit-xml=report-junit-ref.xml --mld --ssnr --odg -n auto --testcase_timeout $testcase_timeout --dut_encoder_path $REF_ENCODER_PATH --dut_decoder_path $REF_DECODER_PATH --compare_to_input || exit_code=$? + - python3 -m pytest --tb=no $TEST_SUITE -v --create_cut --html=report-dut.html --self-contained-html --junit-xml=report-junit-dut.xml --mld --ssnr --odg -n auto --testcase_timeout $testcase_timeout --dut_encoder_path $DUT_ENCODER_PATH --dut_decoder_path $DUT_DECODER_PATH --compare_to_input || exit_code=$? + - zero_errors_ref=$(cat report-junit-ref.xml | grep -c 'errors="0"') || true + - zero_errors_dut=$(cat report-junit-dut.xml | grep -c 'errors="0"') || true + - python3 scripts/parse_xml_report.py report-junit-ref.xml report-ref.csv + - python3 scripts/parse_xml_report.py report-junit-dut.xml report-dut.csv + + # - mkdir $IMAGES_ARTIFACT_NAME + # - for MEASURE in MLD DIFF SSNR ODG;do python3 scripts/create_histogram_summary.py $CSV_ARTIFACT_NAME $IMAGES_ARTIFACT_NAME/summary_"$MEASURE".csv $IMAGES_ARTIFACT_NAME/summary_"$MEASURE".png --measure $MEASURE; done + # - python3 ci/basop-pages/create_summary_page.py $SUMMARY_HTML_ARTIFACT_NAME $CI_JOB_ID $CI_JOB_NAME + + # - if [ $USE_LTV -eq 1 ] && [ "$CI_COMMIT_BRANCH" == "$CI_DEFAULT_BRANCH" ]; then + # - id_previous=$(python3 ci/get_id_of_last_job_occurence.py $CI_DEFAULT_BRANCH $CI_JOB_NAME $CI_PROJECT_ID) + # - echo "Job ID from variables - $CI_JOB_ID, Job ID from script - $id_previous" + # - curl --request GET "https://forge.3gpp.org/rep/api/v4/projects/$CI_PROJECT_ID/jobs/$id_previous/artifacts" --output artifacts.zip + # - unzip artifacts.zip -d previous_artifacts + # # This wildcard thingy relies on only one csv file being present per job + # - file_previous="previous_artifacts/mld--$CI_JOB_NAME-$id_previous--sha-*.csv" + # - python3 ci/basop-pages/create_report_pages.py $PAGES_HTML_ARTIFACT_NAME $MERGED_CSV_ARTIFACT_NAME $CSV_ARTIFACT_NAME $file_previous $CI_JOB_ID $id_previous $CI_JOB_NAME + # - else + # # create empty file for artifacts to avoid errors + # - touch $PAGES_HTML_ARTIFACT_NAME + # - touch $MERGED_CSV_ARTIFACT_NAME + # - fi + + # - if [ $zero_errors != 1 ]; then + # - echo "Run errors encountered!" + # # TODO: temporary only to not fail MR pipelines on crashes + # - if [ $CI_PIPELINE_SOURCE == 'merge_request_event' ]; then + # - exit $EXIT_CODE_NON_BE + # - else + # - exit $EXIT_CODE_FAIL + # - fi + # - fi + # - if [ $exit_code -eq 1 ]; then echo "Differences encountered"; exit $EXIT_CODE_NON_BE; fi + - exit 0 + + allow_failure: + exit_codes: + - 123 + artifacts: + name: "$CI_JOB_NAME--sha-$CI_COMMIT_SHORT_SHA--results" + expire_in: 1 week + when: always + paths: + - report-junit-ref.xml + - report-ref.html + - report-ref.csv + - report-junit-dut.xml + - report-dut.html + - report-dut.csv + expose_as: "pytest compare to input results" + reports: + junit: + - report-junit-ref.xml + - report-junit-dut.xml + # --------------------------------------------------------------- # .pre jobs for setting up things # --------------------------------------------------------------- @@ -762,6 +856,17 @@ ivas-pytest-compare_to_ref-short-dec-lev+10: - LEVEL_SCALING=3.162 <<: *ivas-pytest-anchor +ivas-pytest-compare-to-input-short-dec: + extends: + - .rules-pytest-to-input-short + - .test-job-linux + before_script: + - USE_LTV=0 + - DUT_ENCODER_PATH=./$REF_ENCODER_PATH + - TEST_SUITE="$SHORT_TEST_SUITE" + - LEVEL_SCALING=1.0 + <<: *ivas-pytest-compare-to-input-anchor + ivas-pytest-dec-msan-short: extends: - .test-job-linux -- GitLab From e1619a1a18aa022853db6006e4fee902b3715cff Mon Sep 17 00:00:00 2001 From: Erik Norvell Date: Fri, 11 Oct 2024 17:44:43 +0200 Subject: [PATCH 2/3] Add report-diff.csv to output --- .gitlab-ci.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 8e3331224..4139688e5 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -69,7 +69,7 @@ workflow: IVAS_PIPELINE_NAME: 'Run comparison tools against float ref (long test vectors): $CI_COMMIT_BRANCH' - if: $CI_PIPELINE_SOURCE == 'web' && $MANUAL_PIPELINE_TYPE == 'pytest-compare-to-input' variables: - IVAS_PIPELINE_NAME: 'Run comparison tools against input ref (pass-through only): $CI_COMMIT_BRANCH' + IVAS_PIPELINE_NAME: 'Run comparison tools against input (pass-through only): $CI_COMMIT_BRANCH' - if: $CI_PIPELINE_SOURCE == 'web' && $MANUAL_PIPELINE_TYPE == 'pytest-saturation-smoke-test' variables: IVAS_PIPELINE_NAME: 'Run saturation smoke-test: $CI_COMMIT_BRANCH' @@ -514,6 +514,7 @@ stages: - zero_errors_dut=$(cat report-junit-dut.xml | grep -c 'errors="0"') || true - python3 scripts/parse_xml_report.py report-junit-ref.xml report-ref.csv - python3 scripts/parse_xml_report.py report-junit-dut.xml report-dut.csv + - python3 scripts/diff_report.py report-ref.csv report-dut.csv report-diff.csv # - mkdir $IMAGES_ARTIFACT_NAME # - for MEASURE in MLD DIFF SSNR ODG;do python3 scripts/create_histogram_summary.py $CSV_ARTIFACT_NAME $IMAGES_ARTIFACT_NAME/summary_"$MEASURE".csv $IMAGES_ARTIFACT_NAME/summary_"$MEASURE".png --measure $MEASURE; done @@ -559,6 +560,7 @@ stages: - report-junit-dut.xml - report-dut.html - report-dut.csv + - report-diff.csv expose_as: "pytest compare to input results" reports: junit: -- GitLab From a5df33e34188511d556bbb6faf6333086f1228c8 Mon Sep 17 00:00:00 2001 From: Erik Norvell Date: Sat, 12 Oct 2024 15:07:31 +0200 Subject: [PATCH 3/3] Cleanup of commented out code --- .gitlab-ci.yml | 28 ---------------------------- 1 file changed, 28 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 4139688e5..7aead521c 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -516,34 +516,6 @@ stages: - python3 scripts/parse_xml_report.py report-junit-dut.xml report-dut.csv - python3 scripts/diff_report.py report-ref.csv report-dut.csv report-diff.csv - # - mkdir $IMAGES_ARTIFACT_NAME - # - for MEASURE in MLD DIFF SSNR ODG;do python3 scripts/create_histogram_summary.py $CSV_ARTIFACT_NAME $IMAGES_ARTIFACT_NAME/summary_"$MEASURE".csv $IMAGES_ARTIFACT_NAME/summary_"$MEASURE".png --measure $MEASURE; done - # - python3 ci/basop-pages/create_summary_page.py $SUMMARY_HTML_ARTIFACT_NAME $CI_JOB_ID $CI_JOB_NAME - - # - if [ $USE_LTV -eq 1 ] && [ "$CI_COMMIT_BRANCH" == "$CI_DEFAULT_BRANCH" ]; then - # - id_previous=$(python3 ci/get_id_of_last_job_occurence.py $CI_DEFAULT_BRANCH $CI_JOB_NAME $CI_PROJECT_ID) - # - echo "Job ID from variables - $CI_JOB_ID, Job ID from script - $id_previous" - # - curl --request GET "https://forge.3gpp.org/rep/api/v4/projects/$CI_PROJECT_ID/jobs/$id_previous/artifacts" --output artifacts.zip - # - unzip artifacts.zip -d previous_artifacts - # # This wildcard thingy relies on only one csv file being present per job - # - file_previous="previous_artifacts/mld--$CI_JOB_NAME-$id_previous--sha-*.csv" - # - python3 ci/basop-pages/create_report_pages.py $PAGES_HTML_ARTIFACT_NAME $MERGED_CSV_ARTIFACT_NAME $CSV_ARTIFACT_NAME $file_previous $CI_JOB_ID $id_previous $CI_JOB_NAME - # - else - # # create empty file for artifacts to avoid errors - # - touch $PAGES_HTML_ARTIFACT_NAME - # - touch $MERGED_CSV_ARTIFACT_NAME - # - fi - - # - if [ $zero_errors != 1 ]; then - # - echo "Run errors encountered!" - # # TODO: temporary only to not fail MR pipelines on crashes - # - if [ $CI_PIPELINE_SOURCE == 'merge_request_event' ]; then - # - exit $EXIT_CODE_NON_BE - # - else - # - exit $EXIT_CODE_FAIL - # - fi - # - fi - # - if [ $exit_code -eq 1 ]; then echo "Differences encountered"; exit $EXIT_CODE_NON_BE; fi - exit 0 allow_failure: -- GitLab