From 26eb37d5626d65f76c852b9d5e49957c76745360 Mon Sep 17 00:00:00 2001 From: Tapani Pihlajakuja Date: Wed, 21 Jan 2026 12:27:48 +0200 Subject: [PATCH 1/3] Adds MLD and SSNR stats to comparisons in the float MR pipeline. --- main-float.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/main-float.yml b/main-float.yml index 84aa197..e91a023 100644 --- a/main-float.yml +++ b/main-float.yml @@ -597,7 +597,7 @@ renderer-pytest-on-merge-request: - if [ $ref_using_main == 1 ]; then git checkout $source_branch_commit_sha; fi # run test - - python3 -m pytest -q --log-level ERROR -n auto -rA --html=report.html --self-contained-html --junit-xml=report-junit.xml tests/renderer/test_renderer.py --create_cut --testcase_timeout=$testcase_timeout || exit_code=$? + - python3 -m pytest -q --log-level ERROR -n auto -rA --mld --ssnr --html=report.html --self-contained-html --junit-xml=report-junit.xml tests/renderer/test_renderer.py --create_cut --testcase_timeout=$testcase_timeout || exit_code=$? - zero_errors=$(cat report-junit.xml | grep -c 'errors="0"') || true - *merge-request-comparison-check @@ -807,7 +807,7 @@ ivas-pytest-on-merge-request: ### run pytest - exit_code=0 - testcase_timeout=60 - - python3 -m pytest "$TESTS_DIR_CODEC_BE_ON_MR"/"$PYTEST_SCRIPT" -v --html=report.html --self-contained-html --junit-xml=report-junit.xml --testcase_timeout=$testcase_timeout || exit_code=$? + - python3 -m pytest "$TESTS_DIR_CODEC_BE_ON_MR"/"$PYTEST_SCRIPT" -v --mld --ssnr --html=report.html --self-contained-html --junit-xml=report-junit.xml --testcase_timeout=$testcase_timeout || exit_code=$? - zero_errors=$(cat report-junit.xml | grep -c 'errors="0"') || true - *merge-request-comparison-check @@ -855,7 +855,7 @@ ivas-interop-on-merge-request: - exit_code2=0 # set timeout for individual testcase runs to 60 seconds - testcase_timeout=60 - - python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR -v --html=report.html --self-contained-html --junit-xml=report-junit.xml --update_ref 1 --ref_encoder_path ./IVAS_cod --ref_decoder_path ./IVAS_dec_ref --testcase_timeout=$testcase_timeout || exit_code=$? + - python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR -v --mld --ssnr --html=report.html --self-contained-html --junit-xml=report-junit.xml --update_ref 1 --ref_encoder_path ./IVAS_cod --ref_decoder_path ./IVAS_dec_ref --testcase_timeout=$testcase_timeout || exit_code=$? - zero_failures=$(cat report-junit.xml | grep -c 'failures="0"') || true - if [ $zero_failures != 1 ] && [ $non_interop_flag == 0 ]; then echo "Non-interop cases without non-interop flag encountered!"; exit $EXIT_CODE_FAIL; fi -- GitLab From 524ca80e6e6af8fec480219ba21876e8580218e5 Mon Sep 17 00:00:00 2001 From: Tapani Pihlajakuja Date: Wed, 21 Jan 2026 12:53:00 +0200 Subject: [PATCH 2/3] Add MLD and SSNR to more places as requested. --- main-basop.yml | 4 ++-- main-float.yml | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/main-basop.yml b/main-basop.yml index b03a998..d638dfa 100644 --- a/main-basop.yml +++ b/main-basop.yml @@ -1316,7 +1316,7 @@ split-rendering-pytest-on-merge-request: - echo "$comp_args" # run test - - python3 -m pytest -q --log-level ERROR -n auto -rA --html=$REPORT_HTML --self-contained-html --junit-xml=$REPORT_XML tests/split_rendering/test_split_rendering.py $comp_args --create_cut --testcase_timeout=$testcase_timeout || exit_code=$? + - python3 -m pytest -q --log-level ERROR -n auto -rA --mld --ssnr --html=$REPORT_HTML --self-contained-html --junit-xml=$REPORT_XML tests/split_rendering/test_split_rendering.py $comp_args --create_cut --testcase_timeout=$testcase_timeout || exit_code=$? - zero_errors=$(cat $REPORT_XML | grep -c 'errors="0"') || true - python3 scripts/parse_xml_report.py $REPORT_XML $REPORT_CSV @@ -2121,7 +2121,7 @@ test-long-self-test: ### run pytest self-test using long test vectors - exit_code=0 - - python3 -m pytest tests/codec_be_on_mr_nonselection/test_param_file.py -v --param_file scripts/config/self_test_ltv.prm --use_ltv --html=report-ltv.html --self-contained-html --junit-xml=report-junit-ltv.xml $comp_args --testcase_timeout=$testcase_timeout || exit_code=$? + - python3 -m pytest tests/codec_be_on_mr_nonselection/test_param_file.py -v --mld --ssnr --param_file scripts/config/self_test_ltv.prm --use_ltv --html=report-ltv.html --self-contained-html --junit-xml=report-junit-ltv.xml $comp_args --testcase_timeout=$testcase_timeout || exit_code=$? - zero_errors=$(cat report-junit-ltv.xml | grep -c 'errors="0"') || true - if [ $zero_errors != 1 ]; then echo "Run errors encountered!"; exit $EXIT_CODE_FAIL; fi diff --git a/main-float.yml b/main-float.yml index e91a023..0c06a42 100644 --- a/main-float.yml +++ b/main-float.yml @@ -748,7 +748,7 @@ split-rendering-pytest-on-merge-request: - if [ $ref_using_main == 1 ]; then git checkout $source_branch_commit_sha; fi # run test - - python3 -m pytest -q --log-level ERROR -n auto -rA --html=report.html --self-contained-html --junit-xml=report-junit.xml tests/split_rendering/test_split_rendering.py --create_cut --testcase_timeout=$testcase_timeout || exit_code=$? + - python3 -m pytest -q --log-level ERROR -n auto -rA --mld --ssnr --html=report.html --self-contained-html --junit-xml=report-junit.xml tests/split_rendering/test_split_rendering.py --create_cut --testcase_timeout=$testcase_timeout || exit_code=$? - zero_errors=$(cat report-junit.xml | grep -c 'errors="0"') || true - *merge-request-comparison-check @@ -1491,7 +1491,7 @@ test-long-self-test: ### run pytest self-test using long test vectors - exit_code=0 - - python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR --use_ltv -v --param_file scripts/config/self_test_ltv.prm --html=report-ltv.html --self-contained-html --junit-xml=report-junit-ltv.xml --testcase_timeout=$testcase_timeout || exit_code=$? + - python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR --use_ltv -v --mld --ssnr --param_file scripts/config/self_test_ltv.prm --html=report-ltv.html --self-contained-html --junit-xml=report-junit-ltv.xml --testcase_timeout=$testcase_timeout || exit_code=$? - zero_errors=$(cat report-junit-ltv.xml | grep -c 'errors="0"') || true - if [ $zero_errors != 1 ]; then echo "Run errors encountered!"; exit $EXIT_CODE_FAIL; fi -- GitLab From 51a52f5996642a93180ea6053de35fcabe24b93f Mon Sep 17 00:00:00 2001 From: Tapani Pihlajakuja Date: Wed, 21 Jan 2026 14:22:53 +0200 Subject: [PATCH 3/3] Remove unnecessary additions for BASOP side. --- main-basop.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/main-basop.yml b/main-basop.yml index d638dfa..b03a998 100644 --- a/main-basop.yml +++ b/main-basop.yml @@ -1316,7 +1316,7 @@ split-rendering-pytest-on-merge-request: - echo "$comp_args" # run test - - python3 -m pytest -q --log-level ERROR -n auto -rA --mld --ssnr --html=$REPORT_HTML --self-contained-html --junit-xml=$REPORT_XML tests/split_rendering/test_split_rendering.py $comp_args --create_cut --testcase_timeout=$testcase_timeout || exit_code=$? + - python3 -m pytest -q --log-level ERROR -n auto -rA --html=$REPORT_HTML --self-contained-html --junit-xml=$REPORT_XML tests/split_rendering/test_split_rendering.py $comp_args --create_cut --testcase_timeout=$testcase_timeout || exit_code=$? - zero_errors=$(cat $REPORT_XML | grep -c 'errors="0"') || true - python3 scripts/parse_xml_report.py $REPORT_XML $REPORT_CSV @@ -2121,7 +2121,7 @@ test-long-self-test: ### run pytest self-test using long test vectors - exit_code=0 - - python3 -m pytest tests/codec_be_on_mr_nonselection/test_param_file.py -v --mld --ssnr --param_file scripts/config/self_test_ltv.prm --use_ltv --html=report-ltv.html --self-contained-html --junit-xml=report-junit-ltv.xml $comp_args --testcase_timeout=$testcase_timeout || exit_code=$? + - python3 -m pytest tests/codec_be_on_mr_nonselection/test_param_file.py -v --param_file scripts/config/self_test_ltv.prm --use_ltv --html=report-ltv.html --self-contained-html --junit-xml=report-junit-ltv.xml $comp_args --testcase_timeout=$testcase_timeout || exit_code=$? - zero_errors=$(cat report-junit-ltv.xml | grep -c 'errors="0"') || true - if [ $zero_errors != 1 ]; then echo "Run errors encountered!"; exit $EXIT_CODE_FAIL; fi -- GitLab