From e538224816eff3d9455bc28e81be5d5eb29c18d0 Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Thu, 20 Mar 2025 11:24:07 +0100 Subject: [PATCH 01/23] [revert-me] change branch for getting the scripts --- .gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 91fbd0569..a42d849c6 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -29,7 +29,7 @@ variables: MERGE_TARGET_DECODER_PATH_FOR_BUILD_DO_NOT_MODIFY: "./IVAS_dec_merge_target" LEVEL_SCALING: "1.0" IVAS_PIPELINE_NAME: '' - BASOP_CI_BRANCH_PC_REPO: "basop-ci-branch" + BASOP_CI_BRANCH_PC_REPO: "ci/split-output-files-b4-comparison" PRM_FILES: "scripts/config/self_test.prm scripts/config/self_test_ltv.prm" TESTCASE_TIMEOUT_STV: 900 TESTCASE_TIMEOUT_LTV: 2400 -- GitLab From 7cd676169f61f8c3016e85673416355e6d3833cb Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Thu, 20 Mar 2025 11:28:53 +0100 Subject: [PATCH 02/23] add split comparison support to CI config --- .gitlab-ci.yml | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index a42d849c6..7dc0f45f6 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -44,6 +44,7 @@ variables: ENCODER_TEST: "" DELTA_ODG: "" COMPARE_DMX: "" + SPLIT_COMPARISON: "" SKIP_REGRESSION_CHECK: "" FAILED_TESTCASES_LIST: "failed-testcases.txt" ERRORS_TESTCASES_LIST: "errors-testcases.txt" @@ -402,6 +403,7 @@ stages: - REPORT_ARG="" - if [ "$ENCODER_TEST" = "true" ]; then comp_args="${comp_args} --enc_stats"; fi - if [ "$DELTA_ODG" = "true" ]; then comp_args="${comp_args} --odg_bin"; summary_args="${summary_args} DELTA_ODG"; REPORT_ARG="--delta_odg"; fi + - if [ "$SPLIT_COMPARISON" = "true" ]; then comp_args="${comp_args} --split-comparison"; fi # DMX comparison only in manual job with no other metrics - if [ "$COMPARE_DMX" = "true" ]; then @@ -1416,6 +1418,7 @@ ivas-pytest-compare_ref-long-enc: - DUT_DECODER_PATH=./$REF_DECODER_PATH - TEST_SUITE="$LONG_TEST_SUITE_ENCODER" - LEVEL_SCALING=1.0 + - SPLIT_COMPARISON="true" <<: *ivas-pytest-anchor ivas-pytest-compare_ref-long-dec: @@ -1427,6 +1430,7 @@ ivas-pytest-compare_ref-long-dec: - DUT_ENCODER_PATH=./$REF_ENCODER_PATH - TEST_SUITE="$LONG_TEST_SUITE" - LEVEL_SCALING=1.0 + - SPLIT_COMPARISON="true" <<: *ivas-pytest-anchor ivas-pytest-compare_ref-long-enc-lev-10: @@ -1438,6 +1442,7 @@ ivas-pytest-compare_ref-long-enc-lev-10: - DUT_DECODER_PATH=./$REF_DECODER_PATH - TEST_SUITE="$LONG_TEST_SUITE_ENCODER" - LEVEL_SCALING=0.3162 + - SPLIT_COMPARISON="true" <<: *ivas-pytest-anchor ivas-pytest-compare_ref-long-dec-lev-10: @@ -1449,6 +1454,7 @@ ivas-pytest-compare_ref-long-dec-lev-10: - DUT_ENCODER_PATH=./$REF_ENCODER_PATH - TEST_SUITE="$LONG_TEST_SUITE" - LEVEL_SCALING=0.3162 + - SPLIT_COMPARISON="true" <<: *ivas-pytest-anchor ivas-pytest-compare_ref-long-enc-lev+10: @@ -1460,6 +1466,7 @@ ivas-pytest-compare_ref-long-enc-lev+10: - DUT_DECODER_PATH=./$REF_DECODER_PATH - TEST_SUITE="$LONG_TEST_SUITE_ENCODER" - LEVEL_SCALING=3.162 + - SPLIT_COMPARISON="true" <<: *ivas-pytest-anchor ivas-pytest-compare_ref-long-dec-lev+10: @@ -1471,6 +1478,7 @@ ivas-pytest-compare_ref-long-dec-lev+10: - DUT_ENCODER_PATH=./$REF_ENCODER_PATH - TEST_SUITE="$LONG_TEST_SUITE" - LEVEL_SCALING=3.162 + - SPLIT_COMPARISON="true" <<: *ivas-pytest-anchor ivas-smoke-test-saturation: -- GitLab From e73fb419ee9c8c74d68cd744eb9df07a92f68a0b Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Fri, 21 Mar 2025 11:16:02 +0100 Subject: [PATCH 03/23] create two sets of histograms + respective webpages --- .gitlab-ci.yml | 27 +++++++++++++++++++++++++-- 1 file changed, 25 insertions(+), 2 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 7dc0f45f6..15f70e917 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -371,10 +371,12 @@ stages: variables: # keep "mld" in artifact name for backwards compatibility reasons CSV_ARTIFACT_NAME: "mld--$CI_JOB_NAME-$CI_JOB_ID--sha-$CI_COMMIT_SHORT_SHA.csv" + CSV_ARTIFACT_SPLIT: "mld--split--$CI_JOB_NAME-$CI_JOB_ID--sha-$CI_COMMIT_SHORT_SHA.csv" MERGED_CSV_ARTIFACT_NAME: "$CI_JOB_NAME--merged_csv--$CI_JOB_ID.csv" PAGES_HTML_ARTIFACT_NAME: "$CI_JOB_NAME-index.html" SUMMARY_HTML_ARTIFACT_NAME: "summary_$CI_JOB_NAME.html" IMAGES_ARTIFACT_NAME: "images_$CI_JOB_NAME" + IMAGES_ARTIFACT_SPLIT: "images_split_$CI_JOB_NAME" script: - set -euxo pipefail - *print-common-info @@ -417,10 +419,29 @@ stages: - python3 -m pytest --tb=no $TEST_SUITE -v --create_cut --html=report.html --self-contained-html --junit-xml=report-junit.xml $comp_args -n auto --testcase_timeout $testcase_timeout --ref_encoder_path $REF_ENCODER_PATH --ref_decoder_path $REF_DECODER_PATH --dut_encoder_path $DUT_ENCODER_PATH --dut_decoder_path $DUT_DECODER_PATH || exit_code=$? - zero_errors=$(cat report-junit.xml | grep -c 'errors="0"') || true + ### create histograms - python3 scripts/parse_xml_report.py report-junit.xml $CSV_ARTIFACT_NAME $REPORT_ARG + # if split comparison is done, we need to create two pages, one for the "whole file" values and one for the "split" values + - if [ "$SPLIT_COMPARISON" = "true" ]; then + # use -v flag ("inverse") to also have the header in the output file + - grep -v "_whole" $CSV_ARTIFACT_NAME > $CSV_ARTIFACT_SPLIT + - grep -v "_split" $CSV_ARTIFACT_NAME > $CSV_ARTIFACT_NAME + - mkdir $IMAGES_ARTIFACT_SPLIT + - for MEASURE in $summary_args;do + - python3 scripts/create_histogram_summary.py $CSV_ARTIFACT_SPLIT $IMAGES_ARTIFACT_SPLIT/summary_"$MEASURE".csv $IMAGES_ARTIFACT_SPLIT/summary_"$MEASURE".png --measure $MEASURE + - done + - python3 ci/basop-pages/create_summary_page.py $SUMMARY_HTML_ARTIFACT_NAME $CI_JOB_ID $CI_JOB_NAME $IMAGES_ARTIFACT_SPLIT --measures $summary_args + - else + # touch files to suppress warning for missing artifacts + - touch $CSV_ARTIFACT_SPLIT $IMAGES_ARTIFACT_SPLIT + - fi + + # this part creates the "whole file" histograms - mkdir $IMAGES_ARTIFACT_NAME - - for MEASURE in $summary_args;do python3 scripts/create_histogram_summary.py $CSV_ARTIFACT_NAME $IMAGES_ARTIFACT_NAME/summary_"$MEASURE".csv $IMAGES_ARTIFACT_NAME/summary_"$MEASURE".png --measure $MEASURE; done - - python3 ci/basop-pages/create_summary_page.py $SUMMARY_HTML_ARTIFACT_NAME $CI_JOB_ID $CI_JOB_NAME --measures $summary_args + - for MEASURE in $summary_args;do + - python3 scripts/create_histogram_summary.py $CSV_ARTIFACT_NAME $IMAGES_ARTIFACT_NAME/summary_"$MEASURE".csv $IMAGES_ARTIFACT_NAME/summary_"$MEASURE".png --measure $MEASURE; + - done + - python3 ci/basop-pages/create_summary_page.py $SUMMARY_HTML_ARTIFACT_NAME $CI_JOB_ID $CI_JOB_NAME $IMAGES_ARTIFACT_NAME --measures $summary_args - if [ $USE_LTV -eq 1 ] && [ "$CI_COMMIT_BRANCH" == "$CI_DEFAULT_BRANCH" ]; then - id_previous=$(python3 ci/get_id_of_last_job_occurence.py $CI_DEFAULT_BRANCH $CI_JOB_NAME $CI_PROJECT_ID) @@ -460,9 +481,11 @@ stages: - report.html - $PAGES_HTML_ARTIFACT_NAME - $CSV_ARTIFACT_NAME + - $CSV_ARTIFACT_SPLIT - $MERGED_CSV_ARTIFACT_NAME - $SUMMARY_HTML_ARTIFACT_NAME - $IMAGES_ARTIFACT_NAME + - $IMAGES_ARTIFACT_SPLIT expose_as: "pytest compare results" reports: junit: -- GitLab From 7699604cbeba5c4b0b36310220d689d2b157f8d4 Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Fri, 21 Mar 2025 14:32:20 +0100 Subject: [PATCH 04/23] workaround for grep with input=output --- .gitlab-ci.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 15f70e917..33a38214e 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -425,7 +425,8 @@ stages: - if [ "$SPLIT_COMPARISON" = "true" ]; then # use -v flag ("inverse") to also have the header in the output file - grep -v "_whole" $CSV_ARTIFACT_NAME > $CSV_ARTIFACT_SPLIT - - grep -v "_split" $CSV_ARTIFACT_NAME > $CSV_ARTIFACT_NAME + - cp $CSV_ARTIFACT_NAME tmp.csv + - grep -v "_split" tmp.csv > $CSV_ARTIFACT_NAME - mkdir $IMAGES_ARTIFACT_SPLIT - for MEASURE in $summary_args;do - python3 scripts/create_histogram_summary.py $CSV_ARTIFACT_SPLIT $IMAGES_ARTIFACT_SPLIT/summary_"$MEASURE".csv $IMAGES_ARTIFACT_SPLIT/summary_"$MEASURE".png --measure $MEASURE -- GitLab From 9c244dff1eb59d3e8b6a8eebddf12a295ddf02d4 Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Fri, 21 Mar 2025 14:34:00 +0100 Subject: [PATCH 05/23] [revert-me] limit to stereo testcases for faster testing --- .gitlab-ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 33a38214e..57378accb 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -224,7 +224,7 @@ stages: - enc_dmx_arg="--compare_enc_dmx" - fi - - python3 -m pytest $TEST_SUITE -v --update_ref 1 $enc_stats_arg $enc_dmx_arg --create_ref -n auto --ref_encoder_path $REF_ENCODER_PATH --ref_decoder_path $REF_DECODER_PATH --dut_encoder_path $DUT_ENCODER_PATH --dut_decoder_path $DUT_DECODER_PATH || exit_code=$? + - python3 -m pytest $TEST_SUITE -k stereo -v --update_ref 1 $enc_stats_arg $enc_dmx_arg --create_ref -n auto --ref_encoder_path $REF_ENCODER_PATH --ref_decoder_path $REF_DECODER_PATH --dut_encoder_path $DUT_ENCODER_PATH --dut_decoder_path $DUT_DECODER_PATH || exit_code=$? .update-scripts-repo: &update-scripts-repo - cd $SCRIPTS_DIR @@ -416,7 +416,7 @@ stages: ### run pytest - exit_code=0 - - python3 -m pytest --tb=no $TEST_SUITE -v --create_cut --html=report.html --self-contained-html --junit-xml=report-junit.xml $comp_args -n auto --testcase_timeout $testcase_timeout --ref_encoder_path $REF_ENCODER_PATH --ref_decoder_path $REF_DECODER_PATH --dut_encoder_path $DUT_ENCODER_PATH --dut_decoder_path $DUT_DECODER_PATH || exit_code=$? + - python3 -m pytest -k stereo --tb=no $TEST_SUITE -v --create_cut --html=report.html --self-contained-html --junit-xml=report-junit.xml $comp_args -n auto --testcase_timeout $testcase_timeout --ref_encoder_path $REF_ENCODER_PATH --ref_decoder_path $REF_DECODER_PATH --dut_encoder_path $DUT_ENCODER_PATH --dut_decoder_path $DUT_DECODER_PATH || exit_code=$? - zero_errors=$(cat report-junit.xml | grep -c 'errors="0"') || true ### create histograms -- GitLab From cbaf8b9d93e19d7127a83fac9d21b7336ce35768 Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Fri, 21 Mar 2025 16:00:36 +0100 Subject: [PATCH 06/23] Revert "[revert-me] limit to stereo testcases for faster testing" This reverts commit 9c244dff1eb59d3e8b6a8eebddf12a295ddf02d4. --- .gitlab-ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 57378accb..33a38214e 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -224,7 +224,7 @@ stages: - enc_dmx_arg="--compare_enc_dmx" - fi - - python3 -m pytest $TEST_SUITE -k stereo -v --update_ref 1 $enc_stats_arg $enc_dmx_arg --create_ref -n auto --ref_encoder_path $REF_ENCODER_PATH --ref_decoder_path $REF_DECODER_PATH --dut_encoder_path $DUT_ENCODER_PATH --dut_decoder_path $DUT_DECODER_PATH || exit_code=$? + - python3 -m pytest $TEST_SUITE -v --update_ref 1 $enc_stats_arg $enc_dmx_arg --create_ref -n auto --ref_encoder_path $REF_ENCODER_PATH --ref_decoder_path $REF_DECODER_PATH --dut_encoder_path $DUT_ENCODER_PATH --dut_decoder_path $DUT_DECODER_PATH || exit_code=$? .update-scripts-repo: &update-scripts-repo - cd $SCRIPTS_DIR @@ -416,7 +416,7 @@ stages: ### run pytest - exit_code=0 - - python3 -m pytest -k stereo --tb=no $TEST_SUITE -v --create_cut --html=report.html --self-contained-html --junit-xml=report-junit.xml $comp_args -n auto --testcase_timeout $testcase_timeout --ref_encoder_path $REF_ENCODER_PATH --ref_decoder_path $REF_DECODER_PATH --dut_encoder_path $DUT_ENCODER_PATH --dut_decoder_path $DUT_DECODER_PATH || exit_code=$? + - python3 -m pytest --tb=no $TEST_SUITE -v --create_cut --html=report.html --self-contained-html --junit-xml=report-junit.xml $comp_args -n auto --testcase_timeout $testcase_timeout --ref_encoder_path $REF_ENCODER_PATH --ref_decoder_path $REF_DECODER_PATH --dut_encoder_path $DUT_ENCODER_PATH --dut_decoder_path $DUT_DECODER_PATH || exit_code=$? - zero_errors=$(cat report-junit.xml | grep -c 'errors="0"') || true ### create histograms -- GitLab From ca78d64b8655295422fafd8100f4bd609b71a8c9 Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Mon, 24 Mar 2025 10:53:47 +0100 Subject: [PATCH 07/23] [revert-me] use other branch for testing --- .gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 33a38214e..7f9c4b622 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -29,7 +29,7 @@ variables: MERGE_TARGET_DECODER_PATH_FOR_BUILD_DO_NOT_MODIFY: "./IVAS_dec_merge_target" LEVEL_SCALING: "1.0" IVAS_PIPELINE_NAME: '' - BASOP_CI_BRANCH_PC_REPO: "ci/split-output-files-b4-comparison" + BASOP_CI_BRANCH_PC_REPO: "kiene/tmp-branch-for-ltv-split-testing" PRM_FILES: "scripts/config/self_test.prm scripts/config/self_test_ltv.prm" TESTCASE_TIMEOUT_STV: 900 TESTCASE_TIMEOUT_LTV: 2400 -- GitLab From 542102d19ee03f87649465f81b357f905e50d55b Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Mon, 24 Mar 2025 10:56:26 +0100 Subject: [PATCH 08/23] exclude JBM from split comparison splitting is only done on wav outputs, but not on tracefiles --- .gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 7f9c4b622..18ee8f878 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -405,7 +405,7 @@ stages: - REPORT_ARG="" - if [ "$ENCODER_TEST" = "true" ]; then comp_args="${comp_args} --enc_stats"; fi - if [ "$DELTA_ODG" = "true" ]; then comp_args="${comp_args} --odg_bin"; summary_args="${summary_args} DELTA_ODG"; REPORT_ARG="--delta_odg"; fi - - if [ "$SPLIT_COMPARISON" = "true" ]; then comp_args="${comp_args} --split-comparison"; fi + - if [ "$SPLIT_COMPARISON" = "true" ]; then comp_args="${comp_args} --split-comparison"; export PYTEST_ADDOPTS="-k 'not JBM'" fi # DMX comparison only in manual job with no other metrics - if [ "$COMPARE_DMX" = "true" ]; then -- GitLab From 80cef4c62c104565540aed445c886e6d3876fc24 Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Mon, 24 Mar 2025 12:25:04 +0100 Subject: [PATCH 09/23] try different env var export --- .gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 18ee8f878..ee9864b39 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -405,7 +405,7 @@ stages: - REPORT_ARG="" - if [ "$ENCODER_TEST" = "true" ]; then comp_args="${comp_args} --enc_stats"; fi - if [ "$DELTA_ODG" = "true" ]; then comp_args="${comp_args} --odg_bin"; summary_args="${summary_args} DELTA_ODG"; REPORT_ARG="--delta_odg"; fi - - if [ "$SPLIT_COMPARISON" = "true" ]; then comp_args="${comp_args} --split-comparison"; export PYTEST_ADDOPTS="-k 'not JBM'" fi + - if [ "$SPLIT_COMPARISON" = "true" ]; then comp_args="${comp_args} --split-comparison"; export PYTEST_ADDOPTS="-k not JBM" fi # DMX comparison only in manual job with no other metrics - if [ "$COMPARE_DMX" = "true" ]; then -- GitLab From 1c525e918c69a670c88b58cf98bc4e7e088eedcd Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Mon, 24 Mar 2025 12:31:59 +0100 Subject: [PATCH 10/23] move reference creation to also exclude JBM cases there --- .gitlab-ci.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index ee9864b39..95e4fa282 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -397,7 +397,6 @@ stages: - if [ "$COMPARE_DMX" = "true" ] || [ "$ENCODER_TEST" = "true" ]; then - BUILD_WITH_DEBUG_MODE_INFO="true" - fi - - *build-and-create-float-ref-outputs - comp_args="--mld --ssnr --odg" @@ -407,6 +406,8 @@ stages: - if [ "$DELTA_ODG" = "true" ]; then comp_args="${comp_args} --odg_bin"; summary_args="${summary_args} DELTA_ODG"; REPORT_ARG="--delta_odg"; fi - if [ "$SPLIT_COMPARISON" = "true" ]; then comp_args="${comp_args} --split-comparison"; export PYTEST_ADDOPTS="-k not JBM" fi + - *build-and-create-float-ref-outputs + # DMX comparison only in manual job with no other metrics - if [ "$COMPARE_DMX" = "true" ]; then - comp_args="--compare_enc_dmx" -- GitLab From 5c3730f6c0ac44821c85cc9af520ef5f3b368806 Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Mon, 24 Mar 2025 12:37:23 +0100 Subject: [PATCH 11/23] fix missing ; --- .gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 95e4fa282..5d9caaa88 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -404,7 +404,7 @@ stages: - REPORT_ARG="" - if [ "$ENCODER_TEST" = "true" ]; then comp_args="${comp_args} --enc_stats"; fi - if [ "$DELTA_ODG" = "true" ]; then comp_args="${comp_args} --odg_bin"; summary_args="${summary_args} DELTA_ODG"; REPORT_ARG="--delta_odg"; fi - - if [ "$SPLIT_COMPARISON" = "true" ]; then comp_args="${comp_args} --split-comparison"; export PYTEST_ADDOPTS="-k not JBM" fi + - if [ "$SPLIT_COMPARISON" = "true" ]; then comp_args="${comp_args} --split-comparison"; export PYTEST_ADDOPTS="-k not JBM"; fi - *build-and-create-float-ref-outputs -- GitLab From 86bf628facb296a90adc606fe831a8167539b17d Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Mon, 24 Mar 2025 12:47:04 +0100 Subject: [PATCH 12/23] add escaped qoutes --- .gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 5d9caaa88..836829bbc 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -404,7 +404,7 @@ stages: - REPORT_ARG="" - if [ "$ENCODER_TEST" = "true" ]; then comp_args="${comp_args} --enc_stats"; fi - if [ "$DELTA_ODG" = "true" ]; then comp_args="${comp_args} --odg_bin"; summary_args="${summary_args} DELTA_ODG"; REPORT_ARG="--delta_odg"; fi - - if [ "$SPLIT_COMPARISON" = "true" ]; then comp_args="${comp_args} --split-comparison"; export PYTEST_ADDOPTS="-k not JBM"; fi + - if [ "$SPLIT_COMPARISON" = "true" ]; then comp_args="${comp_args} --split-comparison"; export PYTEST_ADDOPTS="-k \"not JBM\""; fi - *build-and-create-float-ref-outputs -- GitLab From 47bf67bd01e16f22d9733aab5bc358091631e41d Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Mon, 24 Mar 2025 15:15:28 +0100 Subject: [PATCH 13/23] fewer testcases for testing --- .gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 836829bbc..f7db000f7 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -404,7 +404,7 @@ stages: - REPORT_ARG="" - if [ "$ENCODER_TEST" = "true" ]; then comp_args="${comp_args} --enc_stats"; fi - if [ "$DELTA_ODG" = "true" ]; then comp_args="${comp_args} --odg_bin"; summary_args="${summary_args} DELTA_ODG"; REPORT_ARG="--delta_odg"; fi - - if [ "$SPLIT_COMPARISON" = "true" ]; then comp_args="${comp_args} --split-comparison"; export PYTEST_ADDOPTS="-k \"not JBM\""; fi + - if [ "$SPLIT_COMPARISON" = "true" ]; then comp_args="${comp_args} --split-comparison"; export PYTEST_ADDOPTS="-k \"not JBM\ and stereo and at""; fi - *build-and-create-float-ref-outputs -- GitLab From 6d8e151543207ae117f5624b4bd4ef201b003884 Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Mon, 7 Apr 2025 13:06:15 +0200 Subject: [PATCH 14/23] fix PYTEST_ADDOPTS string --- .gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index f7db000f7..fcaa06d57 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -404,7 +404,7 @@ stages: - REPORT_ARG="" - if [ "$ENCODER_TEST" = "true" ]; then comp_args="${comp_args} --enc_stats"; fi - if [ "$DELTA_ODG" = "true" ]; then comp_args="${comp_args} --odg_bin"; summary_args="${summary_args} DELTA_ODG"; REPORT_ARG="--delta_odg"; fi - - if [ "$SPLIT_COMPARISON" = "true" ]; then comp_args="${comp_args} --split-comparison"; export PYTEST_ADDOPTS="-k \"not JBM\ and stereo and at""; fi + - if [ "$SPLIT_COMPARISON" = "true" ]; then comp_args="${comp_args} --split-comparison"; export PYTEST_ADDOPTS="-k \"not JBM and stereo and at\""; fi - *build-and-create-float-ref-outputs -- GitLab From 5f5ee69577d1095ccf087de5ff12ea933b19466f Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Fri, 28 Mar 2025 11:41:17 +0100 Subject: [PATCH 15/23] adapt CI config to new versions of scripts --- .gitlab-ci.yml | 33 ++++++++++++--------------------- 1 file changed, 12 insertions(+), 21 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index fcaa06d57..37bd9c601 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -375,6 +375,7 @@ stages: MERGED_CSV_ARTIFACT_NAME: "$CI_JOB_NAME--merged_csv--$CI_JOB_ID.csv" PAGES_HTML_ARTIFACT_NAME: "$CI_JOB_NAME-index.html" SUMMARY_HTML_ARTIFACT_NAME: "summary_$CI_JOB_NAME.html" + SUMMARY_HTML_ARTIFACT_SPLIT: "summary_split_$CI_JOB_NAME.html" IMAGES_ARTIFACT_NAME: "images_$CI_JOB_NAME" IMAGES_ARTIFACT_SPLIT: "images_split_$CI_JOB_NAME" script: @@ -400,10 +401,9 @@ stages: - comp_args="--mld --ssnr --odg" - - summary_args="MLD DIFF SSNR ODG" - - REPORT_ARG="" + - MEASURES_FOR_REPORT="MLD MAX_ABS_DIFF MIN_SSNR MIN_ODG" - if [ "$ENCODER_TEST" = "true" ]; then comp_args="${comp_args} --enc_stats"; fi - - if [ "$DELTA_ODG" = "true" ]; then comp_args="${comp_args} --odg_bin"; summary_args="${summary_args} DELTA_ODG"; REPORT_ARG="--delta_odg"; fi + - if [ "$DELTA_ODG" = "true" ]; then comp_args="${comp_args} --odg_bin"; MEASURES_FOR_REPORT="$MEASURES_FOR_REPORT DELTA_ODG"; fi - if [ "$SPLIT_COMPARISON" = "true" ]; then comp_args="${comp_args} --split-comparison"; export PYTEST_ADDOPTS="-k \"not JBM and stereo and at\""; fi - *build-and-create-float-ref-outputs @@ -421,30 +421,20 @@ stages: - zero_errors=$(cat report-junit.xml | grep -c 'errors="0"') || true ### create histograms - - python3 scripts/parse_xml_report.py report-junit.xml $CSV_ARTIFACT_NAME $REPORT_ARG - # if split comparison is done, we need to create two pages, one for the "whole file" values and one for the "split" values + - python3 scripts/parse_xml_report.py report-junit.xml $CSV_ARTIFACT_NAME --split-csv-file $CSV_ARTIFACT_SPLIT + + # first for "whole" files comparison + - python3 scripts/create_histograms.py $CSV_ARTIFACT_NAME $IMAGES_ARTIFACT_NAME --measures $MEASURES_FOR_REPORT + - python3 ci/basop-pages/create_summary_page.py $SUMMARY_HTML_ARTIFACT_NAME $CI_JOB_ID $CI_JOB_NAME $IMAGES_ARTIFACT_NAME --measures $MEASURES_FOR_REPORT + - if [ "$SPLIT_COMPARISON" = "true" ]; then - # use -v flag ("inverse") to also have the header in the output file - - grep -v "_whole" $CSV_ARTIFACT_NAME > $CSV_ARTIFACT_SPLIT - - cp $CSV_ARTIFACT_NAME tmp.csv - - grep -v "_split" tmp.csv > $CSV_ARTIFACT_NAME - - mkdir $IMAGES_ARTIFACT_SPLIT - - for MEASURE in $summary_args;do - - python3 scripts/create_histogram_summary.py $CSV_ARTIFACT_SPLIT $IMAGES_ARTIFACT_SPLIT/summary_"$MEASURE".csv $IMAGES_ARTIFACT_SPLIT/summary_"$MEASURE".png --measure $MEASURE - - done - - python3 ci/basop-pages/create_summary_page.py $SUMMARY_HTML_ARTIFACT_NAME $CI_JOB_ID $CI_JOB_NAME $IMAGES_ARTIFACT_SPLIT --measures $summary_args + - python3 scripts/create_histograms.py $CSV_ARTIFACT_SPLIT $IMAGES_ARTIFACT_SPLIT --measures $MEASURES_FOR_REPORT + - python3 ci/basop-pages/create_summary_page.py $SUMMARY_HTML_ARTIFACT_SPLIT $CI_JOB_ID $CI_JOB_NAME $IMAGES_ARTIFACT_SPLIT --measures $MEASURES_FOR_REPORT - else # touch files to suppress warning for missing artifacts - touch $CSV_ARTIFACT_SPLIT $IMAGES_ARTIFACT_SPLIT - fi - # this part creates the "whole file" histograms - - mkdir $IMAGES_ARTIFACT_NAME - - for MEASURE in $summary_args;do - - python3 scripts/create_histogram_summary.py $CSV_ARTIFACT_NAME $IMAGES_ARTIFACT_NAME/summary_"$MEASURE".csv $IMAGES_ARTIFACT_NAME/summary_"$MEASURE".png --measure $MEASURE; - - done - - python3 ci/basop-pages/create_summary_page.py $SUMMARY_HTML_ARTIFACT_NAME $CI_JOB_ID $CI_JOB_NAME $IMAGES_ARTIFACT_NAME --measures $summary_args - - if [ $USE_LTV -eq 1 ] && [ "$CI_COMMIT_BRANCH" == "$CI_DEFAULT_BRANCH" ]; then - id_previous=$(python3 ci/get_id_of_last_job_occurence.py $CI_DEFAULT_BRANCH $CI_JOB_NAME $CI_PROJECT_ID) - echo "Job ID from variables - $CI_JOB_ID, Job ID from script - $id_previous" @@ -486,6 +476,7 @@ stages: - $CSV_ARTIFACT_SPLIT - $MERGED_CSV_ARTIFACT_NAME - $SUMMARY_HTML_ARTIFACT_NAME + - $SUMMARY_HTML_ARTIFACT_SPLIT - $IMAGES_ARTIFACT_NAME - $IMAGES_ARTIFACT_SPLIT expose_as: "pytest compare results" -- GitLab From 4b93dfab26d87fc7c321d3734a141fededc753a1 Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Fri, 28 Mar 2025 14:37:12 +0100 Subject: [PATCH 16/23] remove format filter again to run whole test suite --- .gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 37bd9c601..6b70851fb 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -404,7 +404,7 @@ stages: - MEASURES_FOR_REPORT="MLD MAX_ABS_DIFF MIN_SSNR MIN_ODG" - if [ "$ENCODER_TEST" = "true" ]; then comp_args="${comp_args} --enc_stats"; fi - if [ "$DELTA_ODG" = "true" ]; then comp_args="${comp_args} --odg_bin"; MEASURES_FOR_REPORT="$MEASURES_FOR_REPORT DELTA_ODG"; fi - - if [ "$SPLIT_COMPARISON" = "true" ]; then comp_args="${comp_args} --split-comparison"; export PYTEST_ADDOPTS="-k \"not JBM and stereo and at\""; fi + - if [ "$SPLIT_COMPARISON" = "true" ]; then comp_args="${comp_args} --split-comparison"; fi - *build-and-create-float-ref-outputs -- GitLab From de5e4c1b921998e6239c0e8fedd53db59f733ef0 Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Tue, 1 Apr 2025 14:11:58 +0200 Subject: [PATCH 17/23] add --write-out-histograms flag --- .gitlab-ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 6b70851fb..75618d54e 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -424,11 +424,11 @@ stages: - python3 scripts/parse_xml_report.py report-junit.xml $CSV_ARTIFACT_NAME --split-csv-file $CSV_ARTIFACT_SPLIT # first for "whole" files comparison - - python3 scripts/create_histograms.py $CSV_ARTIFACT_NAME $IMAGES_ARTIFACT_NAME --measures $MEASURES_FOR_REPORT + - python3 scripts/create_histograms.py $CSV_ARTIFACT_NAME $IMAGES_ARTIFACT_NAME --measures $MEASURES_FOR_REPORT --write-out-histograms - python3 ci/basop-pages/create_summary_page.py $SUMMARY_HTML_ARTIFACT_NAME $CI_JOB_ID $CI_JOB_NAME $IMAGES_ARTIFACT_NAME --measures $MEASURES_FOR_REPORT - if [ "$SPLIT_COMPARISON" = "true" ]; then - - python3 scripts/create_histograms.py $CSV_ARTIFACT_SPLIT $IMAGES_ARTIFACT_SPLIT --measures $MEASURES_FOR_REPORT + - python3 scripts/create_histograms.py $CSV_ARTIFACT_SPLIT $IMAGES_ARTIFACT_SPLIT --measures $MEASURES_FOR_REPORT --write-out-histograms - python3 ci/basop-pages/create_summary_page.py $SUMMARY_HTML_ARTIFACT_SPLIT $CI_JOB_ID $CI_JOB_NAME $IMAGES_ARTIFACT_SPLIT --measures $MEASURES_FOR_REPORT - else # touch files to suppress warning for missing artifacts -- GitLab From 9140ad05bc5806e66a042ed176a39be6be5cf725 Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Mon, 7 Apr 2025 12:13:38 +0200 Subject: [PATCH 18/23] adapt other uses of parse_xml_report.py to new version --- .gitlab-ci.yml | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 75618d54e..4eb08fe7a 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -52,6 +52,7 @@ variables: FLOAT_REF_COMMIT_FILE: "float-ref-git-sha" CUT_COMMIT_FILE: "CuT-git-sha" MERGE_TARGET_COMMIT_FILE: "merge-target-git-sha" + MEASURES_FOR_REPORT: "MLD MAX_ABS_DIFF MIN_SSNR MIN_ODG" MANUAL_PIPELINE_TYPE: description: "Type for the manual pipeline run. Use 'pytest-compare' to run comparison test against reference float codec." value: 'default' @@ -401,7 +402,8 @@ stages: - comp_args="--mld --ssnr --odg" - - MEASURES_FOR_REPORT="MLD MAX_ABS_DIFF MIN_SSNR MIN_ODG" + - INV_LEVEL_SCALING=$(awk "BEGIN {print 1.0 / $LEVEL_SCALING}") + - comp_args="--mld --ssnr --odg --scalefac $INV_LEVEL_SCALING" - if [ "$ENCODER_TEST" = "true" ]; then comp_args="${comp_args} --enc_stats"; fi - if [ "$DELTA_ODG" = "true" ]; then comp_args="${comp_args} --odg_bin"; MEASURES_FOR_REPORT="$MEASURES_FOR_REPORT DELTA_ODG"; fi - if [ "$SPLIT_COMPARISON" = "true" ]; then comp_args="${comp_args} --split-comparison"; fi @@ -667,9 +669,9 @@ stages: # Store branch outputs for comparison - mv tests/dut tests/dut_branch - # create the summary based on the branch - - for MEASURE in MLD DIFF SSNR ODG;do python3 scripts/create_histogram_summary.py $CSV_BRANCH $IMAGES_ARTIFACT_NAME/summary_"$MEASURE".csv $IMAGES_ARTIFACT_NAME/summary_"$MEASURE".png --measure $MEASURE; done - - python3 ci/basop-pages/create_summary_page.py $SUMMARY_HTML_ARTIFACT_NAME $CI_JOB_ID $CI_JOB_NAME + # create the summary based on the branch only + - python3 scripts/create_histograms.py $CSV_BRANCH $IMAGES_ARTIFACT_NAME --measures $MEASURES_FOR_REPORT + - python3 ci/basop-pages/create_summary_page.py $SUMMARY_HTML_ARTIFACT_NAME $CI_JOB_ID $CI_JOB_NAME $IMAGES_ARTIFACT_NAME --measures $MEASURES_FOR_REPORT ### run main now - git checkout $CI_MERGE_REQUEST_TARGET_BRANCH_NAME -- GitLab From 7436e9696b190bd4af140cd645d42a2c9a033993 Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Wed, 9 Apr 2025 16:40:06 +0200 Subject: [PATCH 19/23] adjust to new scripts in compare-to-input test --- .gitlab-ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index d7de529a5..5950fa19b 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -836,8 +836,8 @@ stages: # create summary - mkdir $IMAGES_ARTIFACT_NAME - - for MEASURE in MLD DIFF SSNR ODG;do python3 scripts/create_histogram_summary.py report-diff.csv $IMAGES_ARTIFACT_NAME/summary_"$MEASURE".csv $IMAGES_ARTIFACT_NAME/summary_"$MEASURE".png --measure $MEASURE --diff; done - - python3 ci/basop-pages/create_summary_page.py $SUMMARY_HTML_ARTIFACT_NAME $CI_JOB_ID $CI_JOB_NAME + - python3 scripts/create_histograms.py $CSV_ARTIFACT_NAME $IMAGES_ARTIFACT_NAME --measures $MEASURES_FOR_REPORT --write-out-histograms --no-bins + - python3 ci/basop-pages/create_summary_page.py $SUMMARY_HTML_ARTIFACT_NAME $CI_JOB_ID $CI_JOB_NAME --measures $MEASURES_FOR_REPORT - exit 0 -- GitLab From f2927e211fa53161b7e9876803c484efddd52577 Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Wed, 9 Apr 2025 16:56:05 +0200 Subject: [PATCH 20/23] fix histogram command --- .gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 5950fa19b..b4a1405cd 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -836,7 +836,7 @@ stages: # create summary - mkdir $IMAGES_ARTIFACT_NAME - - python3 scripts/create_histograms.py $CSV_ARTIFACT_NAME $IMAGES_ARTIFACT_NAME --measures $MEASURES_FOR_REPORT --write-out-histograms --no-bins + - python3 scripts/create_histograms.py report-diff.csv $IMAGES_ARTIFACT_NAME --measures $MEASURES_FOR_REPORT --write-out-histograms --no-bins - python3 ci/basop-pages/create_summary_page.py $SUMMARY_HTML_ARTIFACT_NAME $CI_JOB_ID $CI_JOB_NAME --measures $MEASURES_FOR_REPORT - exit 0 -- GitLab From 4e6d3527c28936f542a435eb303c22c52f0d4e2d Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Wed, 9 Apr 2025 18:20:46 +0200 Subject: [PATCH 21/23] fix command again --- .gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index b4a1405cd..46e941af8 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -837,7 +837,7 @@ stages: # create summary - mkdir $IMAGES_ARTIFACT_NAME - python3 scripts/create_histograms.py report-diff.csv $IMAGES_ARTIFACT_NAME --measures $MEASURES_FOR_REPORT --write-out-histograms --no-bins - - python3 ci/basop-pages/create_summary_page.py $SUMMARY_HTML_ARTIFACT_NAME $CI_JOB_ID $CI_JOB_NAME --measures $MEASURES_FOR_REPORT + - python3 ci/basop-pages/create_summary_page.py $SUMMARY_HTML_ARTIFACT_NAME $CI_JOB_ID $CI_JOB_NAME $IMAGES_ARTIFACT_NAME --measures $MEASURES_FOR_REPORT - exit 0 -- GitLab From f325d96e34c46d7dc59d04289b830371fefed00f Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Tue, 22 Apr 2025 10:37:39 +0200 Subject: [PATCH 22/23] Revert "[revert-me] use other branch for testing" This reverts commit ca78d64b8655295422fafd8100f4bd609b71a8c9. --- .gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index c500107c1..01ac69a92 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -29,7 +29,7 @@ variables: MERGE_TARGET_DECODER_PATH_FOR_BUILD_DO_NOT_MODIFY: "./IVAS_dec_merge_target" LEVEL_SCALING: "1.0" IVAS_PIPELINE_NAME: '' - BASOP_CI_BRANCH_PC_REPO: "kiene/tmp-branch-for-ltv-split-testing" + BASOP_CI_BRANCH_PC_REPO: "ci/split-output-files-b4-comparison" PRM_FILES: "scripts/config/self_test.prm scripts/config/self_test_ltv.prm" TESTCASE_TIMEOUT_STV: 900 TESTCASE_TIMEOUT_LTV: 2400 -- GitLab From 7c5a7682fe42f3b38974c1225ebb53924dea7478 Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Tue, 22 Apr 2025 10:37:53 +0200 Subject: [PATCH 23/23] Revert "[revert-me] change branch for getting the scripts" This reverts commit e538224816eff3d9455bc28e81be5d5eb29c18d0. --- .gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 01ac69a92..275b91bfb 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -29,7 +29,7 @@ variables: MERGE_TARGET_DECODER_PATH_FOR_BUILD_DO_NOT_MODIFY: "./IVAS_dec_merge_target" LEVEL_SCALING: "1.0" IVAS_PIPELINE_NAME: '' - BASOP_CI_BRANCH_PC_REPO: "ci/split-output-files-b4-comparison" + BASOP_CI_BRANCH_PC_REPO: "basop-ci-branch" PRM_FILES: "scripts/config/self_test.prm scripts/config/self_test_ltv.prm" TESTCASE_TIMEOUT_STV: 900 TESTCASE_TIMEOUT_LTV: 2400 -- GitLab