diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index d3f7ba99996f94ea5a5f40b1f3d685e6d4053d03..275b91bfbb4ff8d5994d938242e5f017bdc292e2 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -44,10 +44,12 @@ variables: ENCODER_TEST: "" DELTA_ODG: "" COMPARE_DMX: "" + SPLIT_COMPARISON: "" SKIP_REGRESSION_CHECK: "" FAILED_TESTCASES_LIST: "failed-testcases.txt" ERRORS_TESTCASES_LIST: "errors-testcases.txt" PYTEST_CACHE_ARTIFACT: "pytest_cache.zip" + MEASURES_FOR_REPORT: "MLD MAX_ABS_DIFF MIN_SSNR MIN_ODG" FLOAT_REF_COMMIT_FILE: "float-ref-git-sha.txt" CUT_COMMIT_FILE: "CuT-git-sha.txt" MERGE_TARGET_COMMIT_FILE: "merge-target-git-sha.txt" @@ -370,10 +372,13 @@ stages: variables: # keep "mld" in artifact name for backwards compatibility reasons CSV_ARTIFACT_NAME: "mld--$CI_JOB_NAME-$CI_JOB_ID--sha-$CI_COMMIT_SHORT_SHA.csv" + CSV_ARTIFACT_SPLIT: "mld--split--$CI_JOB_NAME-$CI_JOB_ID--sha-$CI_COMMIT_SHORT_SHA.csv" MERGED_CSV_ARTIFACT_NAME: "$CI_JOB_NAME--merged_csv--$CI_JOB_ID.csv" PAGES_HTML_ARTIFACT_NAME: "$CI_JOB_NAME-index.html" SUMMARY_HTML_ARTIFACT_NAME: "summary_$CI_JOB_NAME.html" + SUMMARY_HTML_ARTIFACT_SPLIT: "summary_split_$CI_JOB_NAME.html" IMAGES_ARTIFACT_NAME: "images_$CI_JOB_NAME" + IMAGES_ARTIFACT_SPLIT: "images_split_$CI_JOB_NAME" script: - set -euxo pipefail - *print-common-info @@ -394,14 +399,16 @@ stages: - if [ "$COMPARE_DMX" = "true" ] || [ "$ENCODER_TEST" = "true" ]; then - BUILD_WITH_DEBUG_MODE_INFO="true" - fi - - *build-and-create-float-ref-outputs - INV_LEVEL_SCALING=$(awk "BEGIN {print 1.0 / $LEVEL_SCALING}") - comp_args="--mld --ssnr --odg --scalefac $INV_LEVEL_SCALING" - summary_args="MLD DIFF SSNR ODG" - REPORT_ARG="" - if [ "$ENCODER_TEST" = "true" ]; then comp_args="${comp_args} --enc_stats"; fi - - if [ "$DELTA_ODG" = "true" ]; then comp_args="${comp_args} --odg_bin"; summary_args="${summary_args} DELTA_ODG"; REPORT_ARG="--delta_odg"; fi + - if [ "$DELTA_ODG" = "true" ]; then comp_args="${comp_args} --odg_bin"; MEASURES_FOR_REPORT="$MEASURES_FOR_REPORT DELTA_ODG"; fi + - if [ "$SPLIT_COMPARISON" = "true" ]; then comp_args="${comp_args} --split-comparison"; fi + + - *build-and-create-float-ref-outputs # DMX comparison only in manual job with no other metrics - if [ "$COMPARE_DMX" = "true" ]; then @@ -415,10 +422,20 @@ stages: - python3 -m pytest --tb=no $TEST_SUITE -v --create_cut --html=report.html --self-contained-html --junit-xml=report-junit.xml $comp_args -n auto --testcase_timeout $testcase_timeout --ref_encoder_path $REF_ENCODER_PATH --ref_decoder_path $REF_DECODER_PATH --dut_encoder_path $DUT_ENCODER_PATH --dut_decoder_path $DUT_DECODER_PATH || exit_code=$? - zero_errors=$(cat report-junit.xml | grep -c 'errors="0"') || true - - python3 scripts/parse_xml_report.py report-junit.xml $CSV_ARTIFACT_NAME $REPORT_ARG - - mkdir $IMAGES_ARTIFACT_NAME - - for MEASURE in $summary_args;do python3 scripts/create_histogram_summary.py $CSV_ARTIFACT_NAME $IMAGES_ARTIFACT_NAME/summary_"$MEASURE".csv $IMAGES_ARTIFACT_NAME/summary_"$MEASURE".png --measure $MEASURE; done - - python3 ci/basop-pages/create_summary_page.py $SUMMARY_HTML_ARTIFACT_NAME $CI_JOB_ID $CI_JOB_NAME --measures $summary_args + ### create histograms + - python3 scripts/parse_xml_report.py report-junit.xml $CSV_ARTIFACT_NAME --split-csv-file $CSV_ARTIFACT_SPLIT + + # first for "whole" files comparison + - python3 scripts/create_histograms.py $CSV_ARTIFACT_NAME $IMAGES_ARTIFACT_NAME --measures $MEASURES_FOR_REPORT --write-out-histograms + - python3 ci/basop-pages/create_summary_page.py $SUMMARY_HTML_ARTIFACT_NAME $CI_JOB_ID $CI_JOB_NAME $IMAGES_ARTIFACT_NAME --measures $MEASURES_FOR_REPORT + + - if [ "$SPLIT_COMPARISON" = "true" ]; then + - python3 scripts/create_histograms.py $CSV_ARTIFACT_SPLIT $IMAGES_ARTIFACT_SPLIT --measures $MEASURES_FOR_REPORT --write-out-histograms + - python3 ci/basop-pages/create_summary_page.py $SUMMARY_HTML_ARTIFACT_SPLIT $CI_JOB_ID $CI_JOB_NAME $IMAGES_ARTIFACT_SPLIT --measures $MEASURES_FOR_REPORT + - else + # touch files to suppress warning for missing artifacts + - touch $CSV_ARTIFACT_SPLIT $IMAGES_ARTIFACT_SPLIT + - fi - if [ $USE_LTV -eq 1 ] && [ "$CI_COMMIT_BRANCH" == "$CI_DEFAULT_BRANCH" ]; then - id_previous=$(python3 ci/get_id_of_last_job_occurence.py $CI_DEFAULT_BRANCH $CI_JOB_NAME $CI_PROJECT_ID) @@ -458,9 +475,12 @@ stages: - report.html - $PAGES_HTML_ARTIFACT_NAME - $CSV_ARTIFACT_NAME + - $CSV_ARTIFACT_SPLIT - $MERGED_CSV_ARTIFACT_NAME - $SUMMARY_HTML_ARTIFACT_NAME + - $SUMMARY_HTML_ARTIFACT_SPLIT - $IMAGES_ARTIFACT_NAME + - $IMAGES_ARTIFACT_SPLIT expose_as: "pytest compare results" reports: junit: @@ -652,9 +672,9 @@ stages: # Store branch outputs for comparison - mv tests/dut tests/dut_branch - # create the summary based on the branch - - for MEASURE in MLD DIFF SSNR ODG;do python3 scripts/create_histogram_summary.py $CSV_BRANCH $IMAGES_ARTIFACT_NAME/summary_"$MEASURE".csv $IMAGES_ARTIFACT_NAME/summary_"$MEASURE".png --measure $MEASURE; done - - python3 ci/basop-pages/create_summary_page.py $SUMMARY_HTML_ARTIFACT_NAME $CI_JOB_ID $CI_JOB_NAME + # create the summary based on the branch only + - python3 scripts/create_histograms.py $CSV_BRANCH $IMAGES_ARTIFACT_NAME --measures $MEASURES_FOR_REPORT + - python3 ci/basop-pages/create_summary_page.py $SUMMARY_HTML_ARTIFACT_NAME $CI_JOB_ID $CI_JOB_NAME $IMAGES_ARTIFACT_NAME --measures $MEASURES_FOR_REPORT ### run main now - git checkout $CI_MERGE_REQUEST_TARGET_BRANCH_NAME @@ -816,8 +836,8 @@ stages: # create summary - mkdir $IMAGES_ARTIFACT_NAME - - for MEASURE in MLD DIFF SSNR ODG;do python3 scripts/create_histogram_summary.py report-diff.csv $IMAGES_ARTIFACT_NAME/summary_"$MEASURE".csv $IMAGES_ARTIFACT_NAME/summary_"$MEASURE".png --measure $MEASURE --diff; done - - python3 ci/basop-pages/create_summary_page.py $SUMMARY_HTML_ARTIFACT_NAME $CI_JOB_ID $CI_JOB_NAME + - python3 scripts/create_histograms.py report-diff.csv $IMAGES_ARTIFACT_NAME --measures $MEASURES_FOR_REPORT --write-out-histograms --no-bins + - python3 ci/basop-pages/create_summary_page.py $SUMMARY_HTML_ARTIFACT_NAME $CI_JOB_ID $CI_JOB_NAME $IMAGES_ARTIFACT_NAME --measures $MEASURES_FOR_REPORT - exit 0 @@ -1422,6 +1442,7 @@ ivas-pytest-compare_ref-long-enc: - DUT_DECODER_PATH=./$REF_DECODER_PATH - TEST_SUITE="$LONG_TEST_SUITE_ENCODER" - LEVEL_SCALING=1.0 + - SPLIT_COMPARISON="true" <<: *ivas-pytest-anchor ivas-pytest-compare_ref-long-dec: @@ -1433,6 +1454,7 @@ ivas-pytest-compare_ref-long-dec: - DUT_ENCODER_PATH=./$REF_ENCODER_PATH - TEST_SUITE="$LONG_TEST_SUITE" - LEVEL_SCALING=1.0 + - SPLIT_COMPARISON="true" <<: *ivas-pytest-anchor ivas-pytest-compare_ref-long-enc-lev-10: @@ -1444,6 +1466,7 @@ ivas-pytest-compare_ref-long-enc-lev-10: - DUT_DECODER_PATH=./$REF_DECODER_PATH - TEST_SUITE="$LONG_TEST_SUITE_ENCODER" - LEVEL_SCALING=0.3162 + - SPLIT_COMPARISON="true" <<: *ivas-pytest-anchor ivas-pytest-compare_ref-long-dec-lev-10: @@ -1455,6 +1478,7 @@ ivas-pytest-compare_ref-long-dec-lev-10: - DUT_ENCODER_PATH=./$REF_ENCODER_PATH - TEST_SUITE="$LONG_TEST_SUITE" - LEVEL_SCALING=0.3162 + - SPLIT_COMPARISON="true" <<: *ivas-pytest-anchor ivas-pytest-compare_ref-long-enc-lev+10: @@ -1466,6 +1490,7 @@ ivas-pytest-compare_ref-long-enc-lev+10: - DUT_DECODER_PATH=./$REF_DECODER_PATH - TEST_SUITE="$LONG_TEST_SUITE_ENCODER" - LEVEL_SCALING=3.162 + - SPLIT_COMPARISON="true" <<: *ivas-pytest-anchor ivas-pytest-compare_ref-long-dec-lev+10: @@ -1477,6 +1502,7 @@ ivas-pytest-compare_ref-long-dec-lev+10: - DUT_ENCODER_PATH=./$REF_ENCODER_PATH - TEST_SUITE="$LONG_TEST_SUITE" - LEVEL_SCALING=3.162 + - SPLIT_COMPARISON="true" <<: *ivas-pytest-anchor ivas-smoke-test-saturation: