From cb0384fa2667575e09f44a90b971c5f24221a266 Mon Sep 17 00:00:00 2001 From: Archit Tamarapu Date: Thu, 25 Sep 2025 14:07:02 +0200 Subject: [PATCH 01/30] add jobs to coverage test: - LTV tests with differing decoder framesizes - LTV jobs for renderer - Split rendering test - VoIP BE tests - BE vs EXT test - LC3plus wrapper unit test --- .gitlab-ci.yml | 54 ++++++++++++++++++++++++++++++++++++++++++-------- 1 file changed, 46 insertions(+), 8 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 7a02bd69e5..bdcd7eb104 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -2203,31 +2203,69 @@ coverage-test-on-main-scheduled: - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/print-common-info.sh - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/update-ltv-repo.sh - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/copy-ltv-files-to-testv-dir.sh - - make GCOV=1 -j + # compile with coverage enabled + - cmake -B cmake-build -G "Unix Makefiles" -DCOPY_EXECUTABLES_FROM_BUILD_DIR=true -DGCOV=ON + - cmake --build cmake-build -- -j - cp IVAS_rend IVAS_rend_ref # Copy exec to be able to run renderer script # -- Run short test vector suite to check coverage of that first + # codec smoke test + - bash ci/smoke_test.sh coverage + # default renderer framesize is 20ms - python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR -v -n auto --update_ref 1 --ref_encoder_path ./IVAS_cod --ref_decoder_path ./IVAS_dec - + # for different renderer framesizes, only the decoder needs to be tested, so use --decoder_only # need to ignore non-zero exit codes as limiter is active and thus the different framesiszes will not be BE in all cases - python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR -v --dut_encoder_path ./IVAS_cod --dut_decoder_path ./IVAS_dec --dut_fr 5 --decoder_only || true - python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR -v --dut_encoder_path ./IVAS_cod --dut_decoder_path ./IVAS_dec --dut_fr 10 --decoder_only || true + # run renderer smoke test - needed for REFVEC tests + - python3 -m pytest -q -n auto tests/renderer/test_renderer.py + # run renderer test only with REF creation stage - no CUT needed since executables are identical - python3 -m pytest -q -n auto tests/renderer/test_renderer.py --create_ref - - python3 -m pytest -q -n auto tests/renderer/test_renderer.py --create_cut - - lcov -c -d obj -o coverage_stv.info # extract coverage of short test vectors here - # remove apps and lib_util files from coverage + # run split rendering smoke test and REF creation + - python3 -m pytest -n auto tests/split_rendering/test_split_rendering.py + - python3 -m pytest -n auto tests/split_rendering/test_split_rendering.py --create_ref + # VoIP BE test + - python3 -m pytest tests/test_be_for_jbm_neutral_dly_profile.py + # split rendering VoIP BE test + - python3 -m pytest -n auto tests/split_rendering/test_voip_be_splitrend_vs_binaural.py + # TODO first frame SID - this script runs a `make clean` and needs to be checked/adapted + # - bash ci/run-first-frame-is-sid-test.sh + # TODO consider HRTF binary loading test, or probably already covered by PRM files + # - python3 -m pytest tests/hrtf_binary_loading + # run BE vs EXT test + - python3 -m pytest tests/test_be_for_ext_outputs.py + # run LC3plus wrapper unit test + - scripts/split_rendering/lc3plus_float/ivas_lc3plus_unit_test + # TODO consider including check-clipping, but this scales ALL testv files. could be added at end of this job + + # -- Extract coverage of short testvectors + - lcov -c -d cmake-build -o coverage_stv.info + # -- Remove apps and lib_util files from coverage - lcov -r coverage_stv.info "*apps*" -o coverage_stv.info - lcov -r coverage_stv.info "*lib_util*" -o coverage_stv.info # -- Add extended coverage of EVS test and long test vectors - python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR/test_param_file.py -v -n auto --update_ref 1 --param_file scripts/config/self_test_evs.prm --ref_encoder_path ./IVAS_cod --ref_decoder_path ./IVAS_dec + # codec smoke test - bash ci/smoke_test.sh coverage + # default renderer framesize is 20ms - python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR/test_param_file.py -v -n auto --update_ref 1 --param_file scripts/config/self_test_ltv.prm --ref_encoder_path ./IVAS_cod --ref_decoder_path ./IVAS_dec + # for different renderer framesizes, only the decoder needs to be tested, so use --decoder_only + # need to ignore non-zero exit codes as limiter is active and thus the different framesiszes will not be BE in all cases + - python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR/test_param_file.py -v -n auto --param_file scripts/config/self_test_ltv.prm -dut_decoder_path ./IVAS_dec --decoder_only --dut_fr 5 || true + - python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR/test_param_file.py -v -n auto --param_file scripts/config/self_test_ltv.prm -dut_decoder_path ./IVAS_dec --decoder_only --dut_fr 10 || true - python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR/test_sba.py -v -n auto --update_ref 1 --ref_encoder_path ./IVAS_cod --ref_decoder_path ./IVAS_dec --use_ltv - python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR/test_sba.py -v --dut_encoder_path ./IVAS_cod --dut_decoder_path ./IVAS_dec --dut_fr 5 --decoder_only --use_ltv || true - - python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR/test_sba.py -v --dut_encoder_path ./IVAS_cod --dut_decoder_path ./IVAS_dec --dut_fr 10 --decoder_only --use_ltv || true - - python3 -m pytest tests/test_be_for_jbm_neutral_dly_profile.py - - lcov -c -d obj -o coverage.info + - python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR/test_sba.py -v --dut_encoder_path ./IVAS_cod --dut_decoder_path ./IVAS_dec --dut_fr 10 --decoder_only --use_ltv || true + # run renderer smoke test - needed for REFVEC tests + - python3 -m pytest -q -n auto tests/renderer/test_renderer.py --use_ltv + # run renderer test only with REF creation stage - no CUT needed since executables are identical + - python3 -m pytest -q -n auto tests/renderer/test_renderer.py --use_ltv --create_cut + # split rendering test doesn't support LTV - skipped here + # VoIP BE tests don't support LTV - skipped here + + # -- Extract accumulated coverage of STV + EVS + LTV + - lcov -c -d cmake-build -o coverage.info # remove apps and lib_util files from coverage - lcov -r coverage.info "*apps*" -o coverage.info - lcov -r coverage.info "*lib_util*" -o coverage.info -- GitLab From 5646836cf40059c99d00d98d399637fbd812a4e0 Mon Sep 17 00:00:00 2001 From: Archit Tamarapu Date: Thu, 25 Sep 2025 16:11:09 +0200 Subject: [PATCH 02/30] increase coverage test timeout to 12 hours --- .gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index bdcd7eb104..c301f1d624 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -2198,7 +2198,7 @@ coverage-test-on-main-scheduled: rules: # only run in scheduled pipeline that passes this env vars - if: $COVERAGE_TEST - timeout: 3 hours + timeout: 12 hours script: - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/print-common-info.sh - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/update-ltv-repo.sh -- GitLab From 75d0d942b6849504d618454304a1e6d20bc8f87f Mon Sep 17 00:00:00 2001 From: Archit Tamarapu Date: Fri, 26 Sep 2025 08:58:20 +0200 Subject: [PATCH 03/30] [fix] missing executables for coverage job --- .gitlab-ci.yml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index c301f1d624..c37375d3df 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -2206,7 +2206,11 @@ coverage-test-on-main-scheduled: # compile with coverage enabled - cmake -B cmake-build -G "Unix Makefiles" -DCOPY_EXECUTABLES_FROM_BUILD_DIR=true -DGCOV=ON - cmake --build cmake-build -- -j - - cp IVAS_rend IVAS_rend_ref # Copy exec to be able to run renderer script + # copy executables to ref to be able to run some --create_ref tests + - cp IVAS_cod IVAS_cod_ref + - cp IVAS_dec IVAS_dec_ref + - cp IVAS_rend IVAS_rend_ref + - cp ISAR_post_rend ISAR_post_rend_ref # -- Run short test vector suite to check coverage of that first # codec smoke test -- GitLab From d7adc329b9383329c176abf91667484bc9d42f4a Mon Sep 17 00:00:00 2001 From: Archit Tamarapu Date: Fri, 26 Sep 2025 10:14:44 +0200 Subject: [PATCH 04/30] [fix] missing short testvectors for check BE to EXT --- .gitlab-ci.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index c37375d3df..a54e8e6666 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -2237,6 +2237,7 @@ coverage-test-on-main-scheduled: # TODO consider HRTF binary loading test, or probably already covered by PRM files # - python3 -m pytest tests/hrtf_binary_loading # run BE vs EXT test + - python3 tests/create_short_testvectors.py --cut_len 1.0 - python3 -m pytest tests/test_be_for_ext_outputs.py # run LC3plus wrapper unit test - scripts/split_rendering/lc3plus_float/ivas_lc3plus_unit_test -- GitLab From 6e12182e714412bb483b5c5b247a6883ef6d9b3d Mon Sep 17 00:00:00 2001 From: Archit Tamarapu Date: Fri, 26 Sep 2025 14:14:49 +0200 Subject: [PATCH 05/30] [fix] renderer LTV test accidentally using CUT mode --- .gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index a54e8e6666..77603623b4 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -2265,7 +2265,7 @@ coverage-test-on-main-scheduled: # run renderer smoke test - needed for REFVEC tests - python3 -m pytest -q -n auto tests/renderer/test_renderer.py --use_ltv # run renderer test only with REF creation stage - no CUT needed since executables are identical - - python3 -m pytest -q -n auto tests/renderer/test_renderer.py --use_ltv --create_cut + - python3 -m pytest -q -n auto tests/renderer/test_renderer.py --use_ltv --create_ref # split rendering test doesn't support LTV - skipped here # VoIP BE tests don't support LTV - skipped here -- GitLab From 84654f5a1bc87af415974c875a1e7d302a6a9596 Mon Sep 17 00:00:00 2001 From: Archit Tamarapu Date: Mon, 29 Sep 2025 14:43:29 +0200 Subject: [PATCH 06/30] try debugging failed commands --- .gitlab-ci.yml | 42 ++++++++++++++++++++++-------------------- 1 file changed, 22 insertions(+), 20 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 77603623b4..1b67ef29db 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -2200,6 +2200,8 @@ coverage-test-on-main-scheduled: - if: $COVERAGE_TEST timeout: 12 hours script: + - set -e + - "trap 'echo \"Command failed at line $LINENO: $BASH_COMMAND\"' ERR" - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/print-common-info.sh - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/update-ltv-repo.sh - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/copy-ltv-files-to-testv-dir.sh @@ -2216,29 +2218,29 @@ coverage-test-on-main-scheduled: # codec smoke test - bash ci/smoke_test.sh coverage # default renderer framesize is 20ms - - python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR -v -n auto --update_ref 1 --ref_encoder_path ./IVAS_cod --ref_decoder_path ./IVAS_dec + - python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR -v -n auto -q --tb=no --update_ref 1 --ref_encoder_path ./IVAS_cod --ref_decoder_path ./IVAS_dec # for different renderer framesizes, only the decoder needs to be tested, so use --decoder_only # need to ignore non-zero exit codes as limiter is active and thus the different framesiszes will not be BE in all cases - - python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR -v --dut_encoder_path ./IVAS_cod --dut_decoder_path ./IVAS_dec --dut_fr 5 --decoder_only || true - - python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR -v --dut_encoder_path ./IVAS_cod --dut_decoder_path ./IVAS_dec --dut_fr 10 --decoder_only || true + - python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR -v -n auto -q --tb=no --dut_encoder_path ./IVAS_cod --dut_decoder_path ./IVAS_dec --dut_fr 5 --decoder_only || true + - python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR -v -n auto -q --tb=no --dut_encoder_path ./IVAS_cod --dut_decoder_path ./IVAS_dec --dut_fr 10 --decoder_only || true # run renderer smoke test - needed for REFVEC tests - - python3 -m pytest -q -n auto tests/renderer/test_renderer.py + - python3 -m pytest -n auto -q --tb=no tests/renderer/test_renderer.py # run renderer test only with REF creation stage - no CUT needed since executables are identical - - python3 -m pytest -q -n auto tests/renderer/test_renderer.py --create_ref + - python3 -m pytest -n auto -q --tb=no tests/renderer/test_renderer.py --create_ref # run split rendering smoke test and REF creation - - python3 -m pytest -n auto tests/split_rendering/test_split_rendering.py - - python3 -m pytest -n auto tests/split_rendering/test_split_rendering.py --create_ref + - python3 -m pytest -n auto -q --tb=no tests/split_rendering/test_split_rendering.py + - python3 -m pytest -n auto -q --tb=no tests/split_rendering/test_split_rendering.py --create_ref # VoIP BE test - - python3 -m pytest tests/test_be_for_jbm_neutral_dly_profile.py + - python3 -m pytest -q --tb=no tests/test_be_for_jbm_neutral_dly_profile.py # split rendering VoIP BE test - - python3 -m pytest -n auto tests/split_rendering/test_voip_be_splitrend_vs_binaural.py + - python3 -m pytest -n auto -q --tb=no tests/split_rendering/test_voip_be_splitrend_vs_binaural.py # TODO first frame SID - this script runs a `make clean` and needs to be checked/adapted # - bash ci/run-first-frame-is-sid-test.sh # TODO consider HRTF binary loading test, or probably already covered by PRM files - # - python3 -m pytest tests/hrtf_binary_loading + # - python3 -m pytest -q --tb=no tests/hrtf_binary_loading # run BE vs EXT test - python3 tests/create_short_testvectors.py --cut_len 1.0 - - python3 -m pytest tests/test_be_for_ext_outputs.py + - python3 -m pytest -q --tb=no tests/test_be_for_ext_outputs.py # run LC3plus wrapper unit test - scripts/split_rendering/lc3plus_float/ivas_lc3plus_unit_test # TODO consider including check-clipping, but this scales ALL testv files. could be added at end of this job @@ -2250,22 +2252,22 @@ coverage-test-on-main-scheduled: - lcov -r coverage_stv.info "*lib_util*" -o coverage_stv.info # -- Add extended coverage of EVS test and long test vectors - - python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR/test_param_file.py -v -n auto --update_ref 1 --param_file scripts/config/self_test_evs.prm --ref_encoder_path ./IVAS_cod --ref_decoder_path ./IVAS_dec + - python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR/test_param_file.py -v -n auto -q --tb=no --update_ref 1 --param_file scripts/config/self_test_evs.prm --ref_encoder_path ./IVAS_cod --ref_decoder_path ./IVAS_dec # codec smoke test - bash ci/smoke_test.sh coverage # default renderer framesize is 20ms - - python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR/test_param_file.py -v -n auto --update_ref 1 --param_file scripts/config/self_test_ltv.prm --ref_encoder_path ./IVAS_cod --ref_decoder_path ./IVAS_dec + - python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR/test_param_file.py -v -n auto -q --tb=no --update_ref 1 --param_file scripts/config/self_test_ltv.prm --ref_encoder_path ./IVAS_cod --ref_decoder_path ./IVAS_dec # for different renderer framesizes, only the decoder needs to be tested, so use --decoder_only # need to ignore non-zero exit codes as limiter is active and thus the different framesiszes will not be BE in all cases - - python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR/test_param_file.py -v -n auto --param_file scripts/config/self_test_ltv.prm -dut_decoder_path ./IVAS_dec --decoder_only --dut_fr 5 || true - - python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR/test_param_file.py -v -n auto --param_file scripts/config/self_test_ltv.prm -dut_decoder_path ./IVAS_dec --decoder_only --dut_fr 10 || true - - python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR/test_sba.py -v -n auto --update_ref 1 --ref_encoder_path ./IVAS_cod --ref_decoder_path ./IVAS_dec --use_ltv - - python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR/test_sba.py -v --dut_encoder_path ./IVAS_cod --dut_decoder_path ./IVAS_dec --dut_fr 5 --decoder_only --use_ltv || true - - python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR/test_sba.py -v --dut_encoder_path ./IVAS_cod --dut_decoder_path ./IVAS_dec --dut_fr 10 --decoder_only --use_ltv || true + - python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR/test_param_file.py -v -n auto -q --tb=no --param_file scripts/config/self_test_ltv.prm -dut_decoder_path ./IVAS_dec --decoder_only --dut_fr 5 || true + - python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR/test_param_file.py -v -n auto -q --tb=no --param_file scripts/config/self_test_ltv.prm -dut_decoder_path ./IVAS_dec --decoder_only --dut_fr 10 || true + - python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR/test_sba.py -v -n auto -q --tb=no --update_ref 1 --ref_encoder_path ./IVAS_cod --ref_decoder_path ./IVAS_dec --use_ltv + - python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR/test_sba.py -v -n auto -q --tb=no --dut_encoder_path ./IVAS_cod --dut_decoder_path ./IVAS_dec --dut_fr 5 --decoder_only --use_ltv || true + - python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR/test_sba.py -v -n auto -q --tb=no --dut_encoder_path ./IVAS_cod --dut_decoder_path ./IVAS_dec --dut_fr 10 --decoder_only --use_ltv || true # run renderer smoke test - needed for REFVEC tests - - python3 -m pytest -q -n auto tests/renderer/test_renderer.py --use_ltv + - python3 -m pytest -n auto -q --tb=no tests/renderer/test_renderer.py --use_ltv # run renderer test only with REF creation stage - no CUT needed since executables are identical - - python3 -m pytest -q -n auto tests/renderer/test_renderer.py --use_ltv --create_ref + - python3 -m pytest -n auto -q --tb=no tests/renderer/test_renderer.py --use_ltv --create_ref # split rendering test doesn't support LTV - skipped here # VoIP BE tests don't support LTV - skipped here -- GitLab From 4a36b9fcfb1789fd2dc8dd105237fbd0aff7d87f Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Tue, 30 Sep 2025 08:49:55 +0200 Subject: [PATCH 07/30] fix pytest call argument --- .gitlab-ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 1b67ef29db..178d02f7ac 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -2259,8 +2259,8 @@ coverage-test-on-main-scheduled: - python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR/test_param_file.py -v -n auto -q --tb=no --update_ref 1 --param_file scripts/config/self_test_ltv.prm --ref_encoder_path ./IVAS_cod --ref_decoder_path ./IVAS_dec # for different renderer framesizes, only the decoder needs to be tested, so use --decoder_only # need to ignore non-zero exit codes as limiter is active and thus the different framesiszes will not be BE in all cases - - python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR/test_param_file.py -v -n auto -q --tb=no --param_file scripts/config/self_test_ltv.prm -dut_decoder_path ./IVAS_dec --decoder_only --dut_fr 5 || true - - python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR/test_param_file.py -v -n auto -q --tb=no --param_file scripts/config/self_test_ltv.prm -dut_decoder_path ./IVAS_dec --decoder_only --dut_fr 10 || true + - python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR/test_param_file.py -v -n auto -q --tb=no --param_file scripts/config/self_test_ltv.prm --dut_decoder_path ./IVAS_dec --decoder_only --dut_fr 5 || true + - python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR/test_param_file.py -v -n auto -q --tb=no --param_file scripts/config/self_test_ltv.prm --dut_decoder_path ./IVAS_dec --decoder_only --dut_fr 10 || true - python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR/test_sba.py -v -n auto -q --tb=no --update_ref 1 --ref_encoder_path ./IVAS_cod --ref_decoder_path ./IVAS_dec --use_ltv - python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR/test_sba.py -v -n auto -q --tb=no --dut_encoder_path ./IVAS_cod --dut_decoder_path ./IVAS_dec --dut_fr 5 --decoder_only --use_ltv || true - python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR/test_sba.py -v -n auto -q --tb=no --dut_encoder_path ./IVAS_cod --dut_decoder_path ./IVAS_dec --dut_fr 10 --decoder_only --use_ltv || true -- GitLab From eb28fe8b80b786274a899914e238a174724125b3 Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Tue, 30 Sep 2025 08:50:54 +0200 Subject: [PATCH 08/30] add 2 debug printouts --- .gitlab-ci.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 178d02f7ac..297f40275d 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -2205,6 +2205,7 @@ coverage-test-on-main-scheduled: - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/print-common-info.sh - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/update-ltv-repo.sh - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/copy-ltv-files-to-testv-dir.sh + - ls -altr scripts\testv # compile with coverage enabled - cmake -B cmake-build -G "Unix Makefiles" -DCOPY_EXECUTABLES_FROM_BUILD_DIR=true -DGCOV=ON - cmake --build cmake-build -- -j @@ -2265,6 +2266,7 @@ coverage-test-on-main-scheduled: - python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR/test_sba.py -v -n auto -q --tb=no --dut_encoder_path ./IVAS_cod --dut_decoder_path ./IVAS_dec --dut_fr 5 --decoder_only --use_ltv || true - python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR/test_sba.py -v -n auto -q --tb=no --dut_encoder_path ./IVAS_cod --dut_decoder_path ./IVAS_dec --dut_fr 10 --decoder_only --use_ltv || true # run renderer smoke test - needed for REFVEC tests + - ls -altr scripts\testv - python3 -m pytest -n auto -q --tb=no tests/renderer/test_renderer.py --use_ltv # run renderer test only with REF creation stage - no CUT needed since executables are identical - python3 -m pytest -n auto -q --tb=no tests/renderer/test_renderer.py --use_ltv --create_ref -- GitLab From 25138faadca1ab9d804fe15ad1d4665d9677fae1 Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Tue, 30 Sep 2025 09:15:35 +0200 Subject: [PATCH 09/30] fix typo: backslash - back = slash --- .gitlab-ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 297f40275d..0086e802e6 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -2205,7 +2205,7 @@ coverage-test-on-main-scheduled: - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/print-common-info.sh - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/update-ltv-repo.sh - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/copy-ltv-files-to-testv-dir.sh - - ls -altr scripts\testv + - ls -altr scripts/testv # compile with coverage enabled - cmake -B cmake-build -G "Unix Makefiles" -DCOPY_EXECUTABLES_FROM_BUILD_DIR=true -DGCOV=ON - cmake --build cmake-build -- -j @@ -2266,7 +2266,7 @@ coverage-test-on-main-scheduled: - python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR/test_sba.py -v -n auto -q --tb=no --dut_encoder_path ./IVAS_cod --dut_decoder_path ./IVAS_dec --dut_fr 5 --decoder_only --use_ltv || true - python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR/test_sba.py -v -n auto -q --tb=no --dut_encoder_path ./IVAS_cod --dut_decoder_path ./IVAS_dec --dut_fr 10 --decoder_only --use_ltv || true # run renderer smoke test - needed for REFVEC tests - - ls -altr scripts\testv + - ls -altr scripts/testv - python3 -m pytest -n auto -q --tb=no tests/renderer/test_renderer.py --use_ltv # run renderer test only with REF creation stage - no CUT needed since executables are identical - python3 -m pytest -n auto -q --tb=no tests/renderer/test_renderer.py --use_ltv --create_ref -- GitLab From 57bb26d3f1b4dbc69cf6839e4af29e8a6bc2868f Mon Sep 17 00:00:00 2001 From: Archit Tamarapu Date: Tue, 30 Sep 2025 13:18:08 +0200 Subject: [PATCH 10/30] [fix] whitespace in metadata filename --- tests/renderer/constants.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/renderer/constants.py b/tests/renderer/constants.py index 85affb1b3a..2d6a4daee4 100644 --- a/tests/renderer/constants.py +++ b/tests/renderer/constants.py @@ -332,7 +332,7 @@ FORMAT_TO_METADATA_FILES_LTV = { "ISM2MASA1": [ str(LTV_DIR.joinpath("ltvISM1.csv")), str(LTV_DIR.joinpath("ltvISM2.csv")), - str(LTV_DIR.joinpath("ltv48_OMASA_2ISM_1TC.met ")), + str(LTV_DIR.joinpath("ltv48_OMASA_2ISM_1TC.met")), ], "ISM3MASA1": [ str(LTV_DIR.joinpath("ltvISM1.csv")), -- GitLab From d68051e43a846906a2a646b1a145e51e7ffcd058 Mon Sep 17 00:00:00 2001 From: Archit Tamarapu Date: Tue, 30 Sep 2025 13:30:08 +0200 Subject: [PATCH 11/30] split up coverage into two jobs + light formatting --- .gitlab-ci.yml | 253 +++++++++++++++++++++++++------------------------ 1 file changed, 128 insertions(+), 125 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 0086e802e6..aed699e6d5 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -11,43 +11,43 @@ include: # This sets when pipelines are created. Jobs have more specific rules to restrict them. workflow: - name: '$IVAS_PIPELINE_NAME' + name: "$IVAS_PIPELINE_NAME" rules: # see https://docs.gitlab.com/ee/ci/yaml/workflow.html#switch-between-branch-pipelines-and-merge-request-pipelines - if: $CI_COMMIT_BRANCH && $CI_OPEN_MERGE_REQUESTS && $CI_PIPELINE_SOURCE == "push" when: never - if: $CI_PIPELINE_SOURCE == 'merge_request_event' variables: - IVAS_PIPELINE_NAME: 'MR pipeline: $CI_MERGE_REQUEST_SOURCE_BRANCH_NAME' + IVAS_PIPELINE_NAME: "MR pipeline: $CI_MERGE_REQUEST_SOURCE_BRANCH_NAME" - if: $CI_PIPELINE_SOURCE == 'push' && $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH # Pushes to main variables: - IVAS_PIPELINE_NAME: 'Push pipeline: $CI_COMMIT_BRANCH' + IVAS_PIPELINE_NAME: "Push pipeline: $CI_COMMIT_BRANCH" - if: $CI_PIPELINE_SOURCE == 'schedule' && $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH # Scheduled in main variables: - IVAS_PIPELINE_NAME: 'Scheduled pipeline: $CI_COMMIT_BRANCH' + IVAS_PIPELINE_NAME: "Scheduled pipeline: $CI_COMMIT_BRANCH" - if: $CI_PIPELINE_SOURCE == 'web' && $MANUAL_PIPELINE_TYPE == 'default' # for testing variables: - IVAS_PIPELINE_NAME: 'Web run pipeline: $CI_COMMIT_BRANCH' + IVAS_PIPELINE_NAME: "Web run pipeline: $CI_COMMIT_BRANCH" - if: $CI_PIPELINE_SOURCE == 'web' && $MANUAL_PIPELINE_TYPE == 'test-be-release' variables: - IVAS_PIPELINE_NAME: 'Test BE to release pipeline: $CI_COMMIT_BRANCH' + IVAS_PIPELINE_NAME: "Test BE to release pipeline: $CI_COMMIT_BRANCH" - if: $CI_PIPELINE_SOURCE == 'web' && $MANUAL_PIPELINE_TYPE == 'test-long-self-test' variables: - IVAS_PIPELINE_NAME: 'Test long self-test against main pipeline: $CI_COMMIT_BRANCH' + IVAS_PIPELINE_NAME: "Test long self-test against main pipeline: $CI_COMMIT_BRANCH" - if: $CI_PIPELINE_SOURCE == 'web' && $MANUAL_PIPELINE_TYPE == 'ivas-conformance' variables: - IVAS_PIPELINE_NAME: 'Draft IVAS Conformance test: $CI_COMMIT_BRANCH' + IVAS_PIPELINE_NAME: "Draft IVAS Conformance test: $CI_COMMIT_BRANCH" - if: $CI_PIPELINE_SOURCE == 'trigger' - if: $CI_PIPELINE_SOURCE == 'web' && $MANUAL_PIPELINE_TYPE == 'ivas-conformance-linux' variables: - IVAS_PIPELINE_NAME: 'Draft IVAS Conformance test -- Linux: $CI_COMMIT_BRANCH' + IVAS_PIPELINE_NAME: "Draft IVAS Conformance test -- Linux: $CI_COMMIT_BRANCH" - if: $CI_PIPELINE_SOURCE == 'trigger' - if: $CI_PIPELINE_SOURCE == 'web' && $MANUAL_PIPELINE_TYPE == 'check-clipping' variables: - IVAS_PIPELINE_NAME: 'Check core input clipping: $CI_COMMIT_BRANCH' + IVAS_PIPELINE_NAME: "Check core input clipping: $CI_COMMIT_BRANCH" - if: $CI_PIPELINE_SOURCE == 'web' && $MANUAL_PIPELINE_TYPE == 'test-branch-vs-input-passthrough' variables: - IVAS_PIPELINE_NAME: 'Pass-through comparison vs input: $CI_COMMIT_BRANCH' + IVAS_PIPELINE_NAME: "Pass-through comparison vs input: $CI_COMMIT_BRANCH" # --------------------------------------------------------------- # Generic script anchors @@ -71,8 +71,8 @@ workflow: - cp ivas-basop/IVAS_cod ./ - cp ivas-basop/IVAS_dec ./ -.merge-request-comparison-setup-codec: &merge-request-comparison-setup-codec - ### build test binaries, initial clean for paranoia reasons +.merge-request-comparison-setup-codec: + &merge-request-comparison-setup-codec ### build test binaries, initial clean for paranoia reasons - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/disable-debugging-macro.sh - make clean - mkdir build @@ -125,9 +125,9 @@ workflow: - Pop-Location .copy-ltv-files-to-testv-dir-win: ©-ltv-files-to-testv-dir-win - - cp $LTV_DIR_WIN\*.wav scripts\testv - - cp $LTV_DIR_WIN\*.met scripts\testv - - cp $LTV_DIR_WIN\*.csv scripts\testv + - cp $LTV_DIR_WIN\*.wav scripts\testv + - cp $LTV_DIR_WIN\*.met scripts\testv + - cp $LTV_DIR_WIN\*.csv scripts\testv .activate-WX-windows: &activate-WX-windows - (Get-Content -Path "CMakeLists.txt") -replace '# \(add_compile_options\("\/WX"\)\)', '$1' | Set-Content -Path "CMakeLists.txt" @@ -160,7 +160,6 @@ workflow: - if [ $exit_code20 -ne 0 ] || [ $exit_code10 -ne 0 ] || [ $exit_code5 -ne 0 ]; then exit 1; fi - # --------------------------------------------------------------- # .pre jobs for setting up things # --------------------------------------------------------------- @@ -359,7 +358,6 @@ build-codec-instrumented-linux: paths: - wmc_tool_output.txt - # make sure that the codec builds with msan, asan and usan build-codec-sanitizers-linux: extends: @@ -402,7 +400,13 @@ codec-smoke-test: tags: - ivas-linux-fast stage: test - needs: ["build-codec-linux-cmake", "build-codec-linux-make", "build-codec-instrumented-linux", "build-codec-sanitizers-linux"] + needs: + [ + "build-codec-linux-cmake", + "build-codec-linux-make", + "build-codec-instrumented-linux", + "build-codec-sanitizers-linux", + ] script: - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/print-common-info.sh # LTV update needed as ltv ISM metadata files are used @@ -411,11 +415,11 @@ codec-smoke-test: # for MRs to basop-ci-branch, we want to test with the BASOP float reference build # per default, the smoke test builds the current repo again with WMOPS activated to catch unbalanced instrumentation macros - if [ "$CI_MERGE_REQUEST_TARGET_BRANCH_NAME" = "basop-ci-branch" ]; then - - python3 scripts/smoketest-basop-filter.py scripts/config/ivas_modes.json --inline - - *get-basop-float-reference - - bash ci/smoke_test.sh coverage + - python3 scripts/smoketest-basop-filter.py scripts/config/ivas_modes.json --inline + - *get-basop-float-reference + - bash ci/smoke_test.sh coverage - else - - bash ci/smoke_test.sh + - bash ci/smoke_test.sh - fi ### analyze for failures @@ -481,7 +485,13 @@ pytest-compare-20ms-and-5ms-rendering: - .test-job-linux - .rules-merge-request-to-main stage: test - needs: ["build-codec-linux-cmake", "build-codec-linux-make", "build-codec-instrumented-linux", "build-codec-sanitizers-linux"] + needs: + [ + "build-codec-linux-cmake", + "build-codec-linux-make", + "build-codec-instrumented-linux", + "build-codec-sanitizers-linux", + ] script: - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/print-common-info.sh - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/disable-limiter.sh @@ -583,7 +593,6 @@ renderer-asan: artifacts: expose_as: "renderer asan result" - # test renderer executable with cmake + msan renderer-msan: extends: @@ -595,7 +604,6 @@ renderer-msan: artifacts: expose_as: "renderer msan result" - # test renderer executable with cmake + usan renderer-usan: extends: @@ -607,7 +615,6 @@ renderer-usan: artifacts: expose_as: "renderer usan result" - # compare renderer bitexactness between target and source branch renderer-pytest-on-merge-request: extends: @@ -846,7 +853,6 @@ ivas-pytest-on-merge-request: junit: - report-junit.xml - # Check interop IVAS_cod_test -> IVAS_dec_ref ivas-interop-on-merge-request: extends: @@ -1093,7 +1099,6 @@ check-bitexactness-ext-and-transport-format: expose_as: "logs-ext-sanity-check" expire_in: "5 days" - # --------------------------------------------------------------- # Test jobs for main branch # --------------------------------------------------------------- @@ -1230,7 +1235,6 @@ codec-comparison-on-main-push: reports: junit: report-junit.xml - # --------------------------------------------------------------- # Manual jobs # --------------------------------------------------------------- @@ -1244,7 +1248,6 @@ test-be-to-release: rules: - if: $CI_PIPELINE_SOURCE == 'web' && $MANUAL_PIPELINE_TYPE == 'test-be-release' script: - - echo "$CI_COMMIT_BRANCH" - MSBuild.exe -maxcpucount .\Workspace_msvc\Workspace_msvc.sln /property:Configuration=Debug /p:Platform=win32 /m | tee -variable winoutdata @@ -1261,7 +1264,6 @@ test-be-to-release: - logs/ expose_as: "test-be-to-release results" - ivas-conformance: tags: - ivas-windows @@ -1274,13 +1276,13 @@ ivas-conformance: exit_codes: - 123 script: - - *print-common-info-windows - - MSBuild.exe -maxcpucount .\Workspace_msvc\Workspace_msvc.sln /property:Configuration=Debug + - *print-common-info-windows + - MSBuild.exe -maxcpucount .\Workspace_msvc\Workspace_msvc.sln /property:Configuration=Debug - cp -force IVAS_cod.exe IVAS_cod_ref.exe - cp -force IVAS_dec.exe IVAS_dec_ref.exe - - cp -force IVAS_rend.exe IVAS_rend_ref.exe + - cp -force IVAS_rend.exe IVAS_rend_ref.exe - cp -force ISAR_post_rend.exe ISAR_post_rend_ref.exe - + # Reference creation - python scripts/prepare_combined_format_inputs.py - $TEST_SET = "tests/codec_be_on_mr_nonselection", "tests/renderer/test_renderer.py", "tests/split_rendering/test_split_rendering.py" @@ -1289,21 +1291,21 @@ ivas-conformance: # Output creation - python -m pytest $TEST_SET -v -n auto --keep_files --create_cut --html=report_cmd.html --self-contained-html - python scripts/parse_commands.py report_cmd.html Readme_IVAS.txt - + # Copy input data and output ref data - if (Test-Path testvec) {rm -r -force testvec} - if (Test-Path TMP_DEC) {rm -r -force TMP_DEC} - - if (Test-Path TMP_ENC) {rm -r -force TMP_ENC} + - if (Test-Path TMP_ENC) {rm -r -force TMP_ENC} - if (Test-Path TMP_JBM) {rm -r -force TMP_JBM} - if (Test-Path TMP_REND) {rm -r -force TMP_REND} - - if (Test-Path TMP_ISAR_POST_REND) {rm -r -force TMP_ISAR_POST_REND} - - if (Test-Path TMP_DEC_ISAR) {rm -r -force TMP_DEC_ISAR} + - if (Test-Path TMP_ISAR_POST_REND) {rm -r -force TMP_ISAR_POST_REND} + - if (Test-Path TMP_DEC_ISAR) {rm -r -force TMP_DEC_ISAR} - mkdir testvec - mkdir testvec/binauralRenderer_interface - mkdir testvec/testv - mkdir testvec/testv/renderer - - mkdir testvec/testv/split_rendering - - mkdir testvec/bin + - mkdir testvec/testv/split_rendering + - mkdir testvec/bin - cp -force -ErrorAction Ignore scripts/testv/* testvec/testv - cp -r -force -ErrorAction Ignore scripts/ls_layouts testvec - cp -r -force -ErrorAction Ignore scripts/object_edit testvec @@ -1313,29 +1315,28 @@ ivas-conformance: - cp -r -force -ErrorAction Ignore tests/ref testvec/testv/ref - cp -r -force -ErrorAction Ignore tests/dut/* testvec/testv/ref - cp -r -force -ErrorAction Ignore tests/renderer/cut testvec/testv/renderer/ref - - cp -r -force -ErrorAction Ignore tests/split_rendering/cut testvec/testv/split_rendering/ref - - cp -r -force -ErrorAction Ignore tests/split_rendering/renderer_configs testvec/testv/split_rendering/renderer_configs - - cp -r -force -ErrorAction Ignore tests/split_rendering/error_patterns testvec/testv/split_rendering/error_patterns - - # Remove redundant files + - cp -r -force -ErrorAction Ignore tests/split_rendering/cut testvec/testv/split_rendering/ref + - cp -r -force -ErrorAction Ignore tests/split_rendering/renderer_configs testvec/testv/split_rendering/renderer_configs + - cp -r -force -ErrorAction Ignore tests/split_rendering/error_patterns testvec/testv/split_rendering/error_patterns + + # Remove redundant files - python scripts/cleanup_26252.py # Copy test script files - cp -r -force -ErrorAction Ignore tests/conformance-test testvec/ - cp Readme_IVAS_dec.txt testvec - - cp Readme_IVAS_enc.txt testvec + - cp Readme_IVAS_enc.txt testvec - cp Readme_IVAS_rend.txt testvec - cp Readme_IVAS_JBM_dec.txt testvec - - cp Readme_IVAS_ISAR_dec.txt testvec - - cp Readme_IVAS_ISAR_post_rend.txt testvec + - cp Readme_IVAS_ISAR_dec.txt testvec + - cp Readme_IVAS_ISAR_post_rend.txt testvec - cp IVAS_cod.exe testvec/bin - cp IVAS_dec.exe testvec/bin - cp IVAS_rend.exe testvec/bin - - cp ISAR_post_rend.exe testvec/bin - - + - cp ISAR_post_rend.exe testvec/bin + # Test run generated scripts in testvec - - cd testvec + - cd testvec - python -m pytest conformance-test/test_26252.py --junit-xml=report-junit.xml --html=report.html --self-contained-html - mv report.html .. - mv report-junit.xml .. @@ -1348,12 +1349,12 @@ ivas-conformance: - report_cmd.html - report-junit.xml - report.html - - Readme_IVAS_dec.txt - - Readme_IVAS_enc.txt + - Readme_IVAS_dec.txt + - Readme_IVAS_enc.txt - Readme_IVAS_rend.txt - Readme_IVAS_JBM_dec.txt - - Readme_IVAS_ISAR_dec.txt - - Readme_IVAS_ISAR_post_rend.txt + - Readme_IVAS_ISAR_dec.txt + - Readme_IVAS_ISAR_post_rend.txt expose_as: "Draft IVAS conformance" reports: junit: report-junit.xml @@ -1375,7 +1376,7 @@ ivas-conformance-linux: - cp IVAS_dec IVAS_dec_ref - cp IVAS_rend IVAS_rend_ref - cp ISAR_post_rend ISAR_post_rend_ref - + # Reference creation - python3 scripts/prepare_combined_format_inputs.py - TEST_SET="tests/codec_be_on_mr_nonselection tests/renderer/test_renderer.py tests/split_rendering/test_split_rendering.py" @@ -1384,15 +1385,15 @@ ivas-conformance-linux: # Output creation - python3 -m pytest $TEST_SET -v -n auto --keep_files --create_cut --html=report_cmd.html --self-contained-html - python3 scripts/parse_commands.py report_cmd.html Readme_IVAS.txt - + # Copy input data and output ref data - rm -rf testvec - mkdir testvec - mkdir testvec/binauralRenderer_interface - mkdir testvec/testv - mkdir testvec/testv/renderer - - mkdir testvec/testv/split_rendering - - mkdir testvec/bin + - mkdir testvec/testv/split_rendering + - mkdir testvec/bin - cp -r scripts/testv/* testvec/testv - cp -r scripts/ls_layouts testvec - cp -r scripts/object_edit testvec @@ -1402,34 +1403,34 @@ ivas-conformance-linux: - cp -r tests/ref testvec/testv/ref - cp -r tests/dut/* testvec/testv/ref - cp -r tests/renderer/cut testvec/testv/renderer/ref - - cp -r tests/split_rendering/cut testvec/testv/split_rendering/ref - - cp -r tests/split_rendering/renderer_configs testvec/testv/split_rendering/renderer_configs - - cp -r tests/split_rendering/error_patterns testvec/testv/split_rendering/error_patterns - - # Remove redundant files + - cp -r tests/split_rendering/cut testvec/testv/split_rendering/ref + - cp -r tests/split_rendering/renderer_configs testvec/testv/split_rendering/renderer_configs + - cp -r tests/split_rendering/error_patterns testvec/testv/split_rendering/error_patterns + + # Remove redundant files - python3 scripts/cleanup_26252.py - + # Copy test script files - cp -r tests/conformance-test testvec/ - cp Readme_IVAS_dec.txt testvec - - cp Readme_IVAS_enc.txt testvec + - cp Readme_IVAS_enc.txt testvec - cp Readme_IVAS_rend.txt testvec - cp Readme_IVAS_JBM_dec.txt testvec - - cp Readme_IVAS_ISAR_dec.txt testvec - - cp Readme_IVAS_ISAR_post_rend.txt testvec - + - cp Readme_IVAS_ISAR_dec.txt testvec + - cp Readme_IVAS_ISAR_post_rend.txt testvec + # Create GCOV execs for coverage analysis - make clean - make GCOV=1 -j - + - cp IVAS_cod testvec/bin - cp IVAS_dec testvec/bin - cp IVAS_rend testvec/bin - - cp ISAR_post_rend testvec/bin - + - cp ISAR_post_rend testvec/bin + # Test run generated scripts in testvec - - cd testvec - - exit_code=0 + - cd testvec + - exit_code=0 - python3 -m pytest conformance-test/test_26252.py --junit-xml=report-junit.xml --html=report.html --self-contained-html || exit_code=$? - mv report.html .. - mv report-junit.xml .. @@ -1444,7 +1445,7 @@ ivas-conformance-linux: # Check for failures - if [ $exit_code -ne 0 ]; then echo "Test failures encountered"; exit $EXIT_CODE_FAIL; fi - + artifacts: name: "ivas-conformance-linux-$CI_COMMIT_SHORT_SHA" expire_in: 1 week @@ -1453,19 +1454,18 @@ ivas-conformance-linux: - report_cmd.html - report-junit.xml - report.html - - Readme_IVAS_dec.txt - - Readme_IVAS_enc.txt + - Readme_IVAS_dec.txt + - Readme_IVAS_enc.txt - Readme_IVAS_rend.txt - Readme_IVAS_JBM_dec.txt - - Readme_IVAS_ISAR_dec.txt - - Readme_IVAS_ISAR_post_rend.txt + - Readme_IVAS_ISAR_dec.txt + - Readme_IVAS_ISAR_post_rend.txt - coverage.info - coverage expose_as: "Draft IVAS conformance -- Linux" reports: junit: report-junit.xml - test-long-self-test: extends: - .job-linux @@ -1523,7 +1523,6 @@ test-long-self-test: - if [ $exit_code -ne 0 ]; then echo "Non-bitexact cases encountered!"; exit $EXIT_CODE_NON_BE; fi - exit 0 - artifacts: name: "test-long-self-test--sha-$CI_COMMIT_SHORT_SHA--results" when: always @@ -1972,7 +1971,6 @@ sanitizer-test-omasa-ism4: - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/update-ltv-repo.sh - python3 ci/run_scheduled_sanitizer_test.py MASA-ISM4 $OUT_FORMATS_ALL --tests $SANITIZER_TESTS - ### --- sanitizer schedule C --- .sanitizer-test-schedule-C: @@ -2185,10 +2183,8 @@ sanitizer-test-osba-planar-hoa3-ism4: - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/update-ltv-repo.sh - python3 ci/run_scheduled_sanitizer_test.py PlanarHOA3-ISM4 $OUT_FORMATS_ALL --tests $SANITIZER_TESTS - - # GCOV/LCOV coverage analysis of self_test suite -coverage-test-on-main-scheduled: +.coverage-test-template: extends: - .test-job-linux-needs-testv-dir - .rules-main-scheduled @@ -2198,10 +2194,10 @@ coverage-test-on-main-scheduled: rules: # only run in scheduled pipeline that passes this env vars - if: $COVERAGE_TEST - timeout: 12 hours - script: + timeout: 3 hours + before_script: - set -e - - "trap 'echo \"Command failed at line $LINENO: $BASH_COMMAND\"' ERR" + - 'trap ''echo "Command failed at line $LINENO: $BASH_COMMAND"'' ERR' - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/print-common-info.sh - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/update-ltv-repo.sh - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/copy-ltv-files-to-testv-dir.sh @@ -2214,7 +2210,25 @@ coverage-test-on-main-scheduled: - cp IVAS_dec IVAS_dec_ref - cp IVAS_rend IVAS_rend_ref - cp ISAR_post_rend ISAR_post_rend_ref + after_script: + - lcov -c -d cmake-build -o $COVERAGE_OUTPUT_FILE + # remove apps and lib_util files from coverage + - lcov -r coverage.info "*apps*" -o $COVERAGE_OUTPUT_FILE + - lcov -r coverage.info "*lib_util*" -o $COVERAGE_OUTPUT_FILE + - commit_sha=$(git rev-parse HEAD) + - genhtml $COVERAGE_OUTPUT_FILE -o $COVERAGE_OUTPUT_DIR -t "Coverage on main -- $COVERAGE_TITLE @ $commit_sha" + artifacts: + name: "main-coverage-sha-$CI_COMMIT_SHORT_SHA" + when: always + expire_in: 1 week +coverage-test-on-main-scheduled-stv: + extends: .coverage-test-template + variables: + COVERAGE_OUTPUT_FILE: "coverage_stv.info" + COVERAGE_OUTPUT_DIR: "coverage_stv" + COVERAGE_TITLE: "short test vectors" + script: # -- Run short test vector suite to check coverage of that first # codec smoke test - bash ci/smoke_test.sh coverage @@ -2245,17 +2259,20 @@ coverage-test-on-main-scheduled: # run LC3plus wrapper unit test - scripts/split_rendering/lc3plus_float/ivas_lc3plus_unit_test # TODO consider including check-clipping, but this scales ALL testv files. could be added at end of this job + artifacts: + paths: + - coverage_stv.info + - coverage_stv - # -- Extract coverage of short testvectors - - lcov -c -d cmake-build -o coverage_stv.info - # -- Remove apps and lib_util files from coverage - - lcov -r coverage_stv.info "*apps*" -o coverage_stv.info - - lcov -r coverage_stv.info "*lib_util*" -o coverage_stv.info - +coverage-test-on-main-scheduled-ltv: + extends: .coverage-test-template + variables: + COVERAGE_OUTPUT_FILE: "coverage_ltv.info" + COVERAGE_OUTPUT_DIR: "coverage_ltv" + COVERAGE_TITLE: "long test vectors" + script: # -- Add extended coverage of EVS test and long test vectors - python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR/test_param_file.py -v -n auto -q --tb=no --update_ref 1 --param_file scripts/config/self_test_evs.prm --ref_encoder_path ./IVAS_cod --ref_decoder_path ./IVAS_dec - # codec smoke test - - bash ci/smoke_test.sh coverage # default renderer framesize is 20ms - python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR/test_param_file.py -v -n auto -q --tb=no --update_ref 1 --param_file scripts/config/self_test_ltv.prm --ref_encoder_path ./IVAS_cod --ref_decoder_path ./IVAS_dec # for different renderer framesizes, only the decoder needs to be tested, so use --decoder_only @@ -2272,24 +2289,10 @@ coverage-test-on-main-scheduled: - python3 -m pytest -n auto -q --tb=no tests/renderer/test_renderer.py --use_ltv --create_ref # split rendering test doesn't support LTV - skipped here # VoIP BE tests don't support LTV - skipped here - - # -- Extract accumulated coverage of STV + EVS + LTV - - lcov -c -d cmake-build -o coverage.info - # remove apps and lib_util files from coverage - - lcov -r coverage.info "*apps*" -o coverage.info - - lcov -r coverage.info "*lib_util*" -o coverage.info - - commit_sha=$(git rev-parse HEAD) - - genhtml coverage.info -o coverage -t "Coverage on main @ $commit_sha" - - genhtml coverage_stv.info -o coverage_stv -t "Coverage on main -- short test vectors @ $commit_sha" artifacts: - name: "main-coverage-sha-$CI_COMMIT_SHORT_SHA" - when: always - expire_in: 1 week paths: - - coverage.info - - coverage_stv.info - - coverage - - coverage_stv + - coverage_ltv.info + - coverage_ltv # --------------------------------------------------------------- # Complexity measurement jobs @@ -2309,20 +2312,20 @@ coverage-test-on-main-scheduled: # 1. check for public_dir being there as this might not be the case when artifact download failed # 2. check for public dir not being empty - handle job failures in prev job that happen after the dir is created. In that case, the empty dir is in the artifacts - if [ -d $public_dir ] && [ ! -z "$( ls -A $public_dir )" ]; then - - mv $public_dir/* wmops/ + - mv $public_dir/* wmops/ # check here if we have the split-by-levels files present - if not, fake them up with the existing global one # this is needed for the first run with split graphs on a branch where the global version did run previously # NOTE: checking only for level_1 file here as this should already be sufficient # NOTE2: also not chechking for RAM for same reason - - wmops_all_global="wmops/log_wmops_all.txt" - - ram_all_global="wmops/log_ram_all.txt" - - if [ -f "${wmops_all_global}" ] && [ ! -f "wmops/log_wmops_all_level_1.txt" ]; then - - declare -a suffixes=("level_1" "level_2" "level_3" "rate_sw") - - for suffix in "${suffixes[@]}"; do - - cp ${wmops_all_global} wmops/log_wmops_all_${suffix}.txt - - cp ${ram_all_global} wmops/log_ram_all_${suffix}.txt - - done - - fi + - wmops_all_global="wmops/log_wmops_all.txt" + - ram_all_global="wmops/log_ram_all.txt" + - if [ -f "${wmops_all_global}" ] && [ ! -f "wmops/log_wmops_all_level_1.txt" ]; then + - declare -a suffixes=("level_1" "level_2" "level_3" "rate_sw") + - for suffix in "${suffixes[@]}"; do + - cp ${wmops_all_global} wmops/log_wmops_all_${suffix}.txt + - cp ${ram_all_global} wmops/log_ram_all_${suffix}.txt + - done + - fi - fi - rm artifacts.zip -- GitLab From ec9ca95f5727dc05d6076224c1639b2230d47f7f Mon Sep 17 00:00:00 2001 From: Archit Tamarapu Date: Tue, 30 Sep 2025 13:39:10 +0200 Subject: [PATCH 12/30] revert some whitespace changes and fix before_script --- .gitlab-ci.yml | 33 +++++++++++++++++---------------- 1 file changed, 17 insertions(+), 16 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index aed699e6d5..a483a7272a 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -71,8 +71,8 @@ workflow: - cp ivas-basop/IVAS_cod ./ - cp ivas-basop/IVAS_dec ./ -.merge-request-comparison-setup-codec: - &merge-request-comparison-setup-codec ### build test binaries, initial clean for paranoia reasons +.merge-request-comparison-setup-codec: &merge-request-comparison-setup-codec + ### build test binaries, initial clean for paranoia reasons - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/disable-debugging-macro.sh - make clean - mkdir build @@ -415,11 +415,11 @@ codec-smoke-test: # for MRs to basop-ci-branch, we want to test with the BASOP float reference build # per default, the smoke test builds the current repo again with WMOPS activated to catch unbalanced instrumentation macros - if [ "$CI_MERGE_REQUEST_TARGET_BRANCH_NAME" = "basop-ci-branch" ]; then - - python3 scripts/smoketest-basop-filter.py scripts/config/ivas_modes.json --inline - - *get-basop-float-reference - - bash ci/smoke_test.sh coverage + - python3 scripts/smoketest-basop-filter.py scripts/config/ivas_modes.json --inline + - *get-basop-float-reference + - bash ci/smoke_test.sh coverage - else - - bash ci/smoke_test.sh + - bash ci/smoke_test.sh - fi ### analyze for failures @@ -2196,6 +2196,7 @@ sanitizer-test-osba-planar-hoa3-ism4: - if: $COVERAGE_TEST timeout: 3 hours before_script: + - !reference [.job-linux, before_script] - set -e - 'trap ''echo "Command failed at line $LINENO: $BASH_COMMAND"'' ERR' - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/print-common-info.sh @@ -2312,20 +2313,20 @@ coverage-test-on-main-scheduled-ltv: # 1. check for public_dir being there as this might not be the case when artifact download failed # 2. check for public dir not being empty - handle job failures in prev job that happen after the dir is created. In that case, the empty dir is in the artifacts - if [ -d $public_dir ] && [ ! -z "$( ls -A $public_dir )" ]; then - - mv $public_dir/* wmops/ + - mv $public_dir/* wmops/ # check here if we have the split-by-levels files present - if not, fake them up with the existing global one # this is needed for the first run with split graphs on a branch where the global version did run previously # NOTE: checking only for level_1 file here as this should already be sufficient # NOTE2: also not chechking for RAM for same reason - - wmops_all_global="wmops/log_wmops_all.txt" - - ram_all_global="wmops/log_ram_all.txt" - - if [ -f "${wmops_all_global}" ] && [ ! -f "wmops/log_wmops_all_level_1.txt" ]; then - - declare -a suffixes=("level_1" "level_2" "level_3" "rate_sw") - - for suffix in "${suffixes[@]}"; do - - cp ${wmops_all_global} wmops/log_wmops_all_${suffix}.txt - - cp ${ram_all_global} wmops/log_ram_all_${suffix}.txt - - done - - fi + - wmops_all_global="wmops/log_wmops_all.txt" + - ram_all_global="wmops/log_ram_all.txt" + - if [ -f "${wmops_all_global}" ] && [ ! -f "wmops/log_wmops_all_level_1.txt" ]; then + - declare -a suffixes=("level_1" "level_2" "level_3" "rate_sw") + - for suffix in "${suffixes[@]}"; do + - cp ${wmops_all_global} wmops/log_wmops_all_${suffix}.txt + - cp ${ram_all_global} wmops/log_ram_all_${suffix}.txt + - done + - fi - fi - rm artifacts.zip -- GitLab From ba81f075c9a85142c9ee9a21c5b9a0bb9cfcd3fd Mon Sep 17 00:00:00 2001 From: Archit Tamarapu Date: Tue, 30 Sep 2025 13:50:06 +0200 Subject: [PATCH 13/30] remove ls command from debugging --- .gitlab-ci.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index a483a7272a..dd91e1d752 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -2202,7 +2202,6 @@ sanitizer-test-osba-planar-hoa3-ism4: - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/print-common-info.sh - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/update-ltv-repo.sh - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/copy-ltv-files-to-testv-dir.sh - - ls -altr scripts/testv # compile with coverage enabled - cmake -B cmake-build -G "Unix Makefiles" -DCOPY_EXECUTABLES_FROM_BUILD_DIR=true -DGCOV=ON - cmake --build cmake-build -- -j -- GitLab From 0d3c9d1beaa00a44c8e74e07a93b18088bc3d76b Mon Sep 17 00:00:00 2001 From: Archit Tamarapu Date: Tue, 30 Sep 2025 16:53:30 +0200 Subject: [PATCH 14/30] [fix] remove redundant second call to renderer LTV test --- .gitlab-ci.yml | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index dd91e1d752..21997fe6f8 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -2213,8 +2213,8 @@ sanitizer-test-osba-planar-hoa3-ism4: after_script: - lcov -c -d cmake-build -o $COVERAGE_OUTPUT_FILE # remove apps and lib_util files from coverage - - lcov -r coverage.info "*apps*" -o $COVERAGE_OUTPUT_FILE - - lcov -r coverage.info "*lib_util*" -o $COVERAGE_OUTPUT_FILE + - lcov -r $COVERAGE_OUTPUT_FILE "*apps*" -o $COVERAGE_OUTPUT_FILE + - lcov -r $COVERAGE_OUTPUT_FILE "*lib_util*" -o $COVERAGE_OUTPUT_FILE - commit_sha=$(git rev-parse HEAD) - genhtml $COVERAGE_OUTPUT_FILE -o $COVERAGE_OUTPUT_DIR -t "Coverage on main -- $COVERAGE_TITLE @ $commit_sha" artifacts: @@ -2284,9 +2284,8 @@ coverage-test-on-main-scheduled-ltv: - python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR/test_sba.py -v -n auto -q --tb=no --dut_encoder_path ./IVAS_cod --dut_decoder_path ./IVAS_dec --dut_fr 10 --decoder_only --use_ltv || true # run renderer smoke test - needed for REFVEC tests - ls -altr scripts/testv + # LTV smoketest is the same as REF creation stage for renderer - python3 -m pytest -n auto -q --tb=no tests/renderer/test_renderer.py --use_ltv - # run renderer test only with REF creation stage - no CUT needed since executables are identical - - python3 -m pytest -n auto -q --tb=no tests/renderer/test_renderer.py --use_ltv --create_ref # split rendering test doesn't support LTV - skipped here # VoIP BE tests don't support LTV - skipped here artifacts: -- GitLab From 7f775d5aa1d9c706c3dcad7e13a2bcb4728804a3 Mon Sep 17 00:00:00 2001 From: Archit Tamarapu Date: Wed, 1 Oct 2025 11:47:36 +0200 Subject: [PATCH 15/30] [fix] update artifact name and replace hardcoded paths in .info files with relative paths --- .gitlab-ci.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 21997fe6f8..ae97cf7153 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -2216,9 +2216,11 @@ sanitizer-test-osba-planar-hoa3-ism4: - lcov -r $COVERAGE_OUTPUT_FILE "*apps*" -o $COVERAGE_OUTPUT_FILE - lcov -r $COVERAGE_OUTPUT_FILE "*lib_util*" -o $COVERAGE_OUTPUT_FILE - commit_sha=$(git rev-parse HEAD) + # replace hardcoded path with relative path to allow easy merging of .info files + - sed -i "s|$(pwd)|..|g" $COVERAGE_OUTPUT_FILE - genhtml $COVERAGE_OUTPUT_FILE -o $COVERAGE_OUTPUT_DIR -t "Coverage on main -- $COVERAGE_TITLE @ $commit_sha" artifacts: - name: "main-coverage-sha-$CI_COMMIT_SHORT_SHA" + name: "main-$COVERAGE_OUTPUT_DIR-sha-$CI_COMMIT_SHORT_SHA" when: always expire_in: 1 week -- GitLab From 52854f012bf3c5504136aaacee8a8a2c9a6e4755 Mon Sep 17 00:00:00 2001 From: Archit Tamarapu Date: Wed, 1 Oct 2025 11:55:54 +0200 Subject: [PATCH 16/30] try to merge coverage reports --- .gitlab-ci.yml | 30 ++++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index ae97cf7153..985532c3a9 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -2224,6 +2224,36 @@ sanitizer-test-osba-planar-hoa3-ism4: when: always expire_in: 1 week +coverage-merge: + extends: .rules-main-scheduled + tags: + - ivas-linux-fast + stage: test + rules: + # only run in scheduled pipeline that passes this env vars + - if: $COVERAGE_TEST + needs: + - job: coverage-test-on-main-scheduled-stv + artifacts: true + - job: coverage-test-on-main-scheduled-ltv + artifacts: true + - job: ivas-conformance-linux + artifacts: true + # before_script: + # - !reference [.job-linux, before_script] # check if this is really needed + script: + - echo "Merging coverage reports..." + - lcov -a *.info -o coverage_merged.info + - commit_sha=$(git rev-parse HEAD) + - genhtml coverage_merged.info -o coverage_merged -t "Merged Coverage Report @ $commit_sha" + artifacts: + name: "merged-coverage-sha-$CI_COMMIT_SHORT_SHA" + when: always + expire_in: 1 week + paths: + - coverage_merged.info + - coverage_merged + coverage-test-on-main-scheduled-stv: extends: .coverage-test-template variables: -- GitLab From 2d3df7ed6461e42c8e1b8e88180481ae3ed075e9 Mon Sep 17 00:00:00 2001 From: Archit Tamarapu Date: Wed, 1 Oct 2025 12:57:46 +0200 Subject: [PATCH 17/30] [fix] substitute path also for ivas-conformance-linux and replace glob with explicit file list (lcov doesn't like multiple arguments for -a) --- .gitlab-ci.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 985532c3a9..9c80573454 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1441,6 +1441,8 @@ ivas-conformance-linux: - lcov -r coverage.info "*apps*" -o coverage.info - lcov -r coverage.info "*lib_util*" -o coverage.info - commit_sha=$(git rev-parse HEAD) + # replace hardcoded path with relative path to allow easy merging of .info files + - sed -i "s|$(pwd)|..|g" coverage.info - genhtml coverage.info -o coverage -t "Coverage on main @ $commit_sha" # Check for failures @@ -2243,7 +2245,7 @@ coverage-merge: # - !reference [.job-linux, before_script] # check if this is really needed script: - echo "Merging coverage reports..." - - lcov -a *.info -o coverage_merged.info + - lcov -a coverage.info -a coverage_stv.info -a coverage_ltv.info -o coverage_merged.info - commit_sha=$(git rev-parse HEAD) - genhtml coverage_merged.info -o coverage_merged -t "Merged Coverage Report @ $commit_sha" artifacts: -- GitLab From 89596ea7a938a6b232c9cfa9f16a51c3936ee07d Mon Sep 17 00:00:00 2001 From: Archit Tamarapu Date: Wed, 1 Oct 2025 15:08:27 +0200 Subject: [PATCH 18/30] cleanup coverage merge job --- .gitlab-ci.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 9c80573454..6e68613b54 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -2241,8 +2241,6 @@ coverage-merge: artifacts: true - job: ivas-conformance-linux artifacts: true - # before_script: - # - !reference [.job-linux, before_script] # check if this is really needed script: - echo "Merging coverage reports..." - lcov -a coverage.info -a coverage_stv.info -a coverage_ltv.info -o coverage_merged.info -- GitLab From f294a1fcf923682bf145f1bc68e10712a0fd9d73 Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Mon, 6 Oct 2025 09:39:27 +0200 Subject: [PATCH 19/30] reduce pytest verbosity in conformance test otherwise, too much is being printed and the results are not shown --- .gitlab-ci.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 6e68613b54..91017da125 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1380,10 +1380,10 @@ ivas-conformance-linux: # Reference creation - python3 scripts/prepare_combined_format_inputs.py - TEST_SET="tests/codec_be_on_mr_nonselection tests/renderer/test_renderer.py tests/split_rendering/test_split_rendering.py" - - python3 -m pytest $TEST_SET -v -n auto --update_ref 1 --create_ref --keep_files + - python3 -m pytest -q $TEST_SET -v -n auto --update_ref 1 --create_ref --keep_files # Output creation - - python3 -m pytest $TEST_SET -v -n auto --keep_files --create_cut --html=report_cmd.html --self-contained-html + - python3 -m pytest -q $TEST_SET -v -n auto --keep_files --create_cut --html=report_cmd.html --self-contained-html - python3 scripts/parse_commands.py report_cmd.html Readme_IVAS.txt # Copy input data and output ref data @@ -1431,7 +1431,7 @@ ivas-conformance-linux: # Test run generated scripts in testvec - cd testvec - exit_code=0 - - python3 -m pytest conformance-test/test_26252.py --junit-xml=report-junit.xml --html=report.html --self-contained-html || exit_code=$? + - python3 -m pytest -q conformance-test/test_26252.py --junit-xml=report-junit.xml --html=report.html --self-contained-html || exit_code=$? - mv report.html .. - mv report-junit.xml .. -- GitLab From a5f96a1cae65296945a89b1a5c90b4ef77cf41e9 Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Mon, 6 Oct 2025 10:55:08 +0200 Subject: [PATCH 20/30] add pl type "coverage" + trigger conformance test with that --- .gitlab-ci.yml | 9 +++++++-- .gitlab-ci/variables.yml | 1 + 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 91017da125..2053cf2688 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -48,6 +48,9 @@ workflow: - if: $CI_PIPELINE_SOURCE == 'web' && $MANUAL_PIPELINE_TYPE == 'test-branch-vs-input-passthrough' variables: IVAS_PIPELINE_NAME: "Pass-through comparison vs input: $CI_COMMIT_BRANCH" + - if: $MANUAL_PIPELINE_TYPE == 'coverage' + variables: + IVAS_PIPELINE_NAME: "Coverage measurement" # --------------------------------------------------------------- # Generic script anchors @@ -1366,6 +1369,8 @@ ivas-conformance-linux: timeout: "90 minutes" rules: - if: ($CI_PIPELINE_SOURCE == 'web' || $CI_PIPELINE_SOURCE == 'trigger') && $MANUAL_PIPELINE_TYPE == 'ivas-conformance-linux' + - if: $MANUAL_PIPELINE_TYPE == "coverage" + - if: $COVERAGE_TEST allow_failure: exit_codes: - 123 @@ -2194,8 +2199,8 @@ sanitizer-test-osba-planar-hoa3-ism4: - ivas-linux-fast stage: test rules: - # only run in scheduled pipeline that passes this env vars - if: $COVERAGE_TEST + - if: $MANUAL_PIPELINE_TYPE == "coverage" timeout: 3 hours before_script: - !reference [.job-linux, before_script] @@ -2232,8 +2237,8 @@ coverage-merge: - ivas-linux-fast stage: test rules: - # only run in scheduled pipeline that passes this env vars - if: $COVERAGE_TEST + - if: $MANUAL_PIPELINE_TYPE == "coverage" needs: - job: coverage-test-on-main-scheduled-stv artifacts: true diff --git a/.gitlab-ci/variables.yml b/.gitlab-ci/variables.yml index 9e60194a6c..52b811a001 100644 --- a/.gitlab-ci/variables.yml +++ b/.gitlab-ci/variables.yml @@ -18,6 +18,7 @@ variables: - 'ivas-conformance-linux' - 'check-clipping' - 'test-branch-vs-input-passthrough' + - 'coverage' PYTEST_ARGS: "" LONG_TEST_SUITE: "tests/codec_be_on_mr_nonselection tests/renderer --param_file scripts/config/self_test_ltv.prm --use_ltv" -- GitLab From b0e2ba6357c8478edcf8ce780d8150b09ab42e8e Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Mon, 6 Oct 2025 12:51:11 +0200 Subject: [PATCH 21/30] use variables for coverage files/dirs + fix pwd stripping --- .gitlab-ci.yml | 44 ++++++++++++++++++++++++---------------- .gitlab-ci/variables.yml | 4 ++++ 2 files changed, 30 insertions(+), 18 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 2053cf2688..f9e6e15c80 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1374,6 +1374,10 @@ ivas-conformance-linux: allow_failure: exit_codes: - 123 + variables: + COVERAGE_OUTPUT_FILE: "$COVERAGE_OUTPUT_FILE_CONFORMANCE" + COVERAGE_OUTPUT_DIR: "coverage_conformance" + COVERAGE_TITLE: "conformance test test_26252.py" script: - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/print-common-info.sh - make -j @@ -1442,13 +1446,13 @@ ivas-conformance-linux: # Collect coverage - cd - - - lcov -c -d obj -o coverage.info - - lcov -r coverage.info "*apps*" -o coverage.info - - lcov -r coverage.info "*lib_util*" -o coverage.info + - lcov -c -d obj -o $COVERAGE_OUTPUT_FILE + - lcov -r $COVERAGE_OUTPUT_FILE "*apps*" -o $COVERAGE_OUTPUT_FILE + - lcov -r $COVERAGE_OUTPUT_FILE "*lib_util*" -o $COVERAGE_OUTPUT_FILE - commit_sha=$(git rev-parse HEAD) # replace hardcoded path with relative path to allow easy merging of .info files - - sed -i "s|$(pwd)|..|g" coverage.info - - genhtml coverage.info -o coverage -t "Coverage on main @ $commit_sha" + - sed -i "s|$(pwd)|.|g" coverage.info + - genhtml $COVERAGE_OUTPUT_FILE -o $COVERAGE_OUTPUT_DIR -t "Coverage on main -- $COVERAGE_TITLE @ $commit_sha" # Check for failures - if [ $exit_code -ne 0 ]; then echo "Test failures encountered"; exit $EXIT_CODE_FAIL; fi @@ -1467,8 +1471,8 @@ ivas-conformance-linux: - Readme_IVAS_JBM_dec.txt - Readme_IVAS_ISAR_dec.txt - Readme_IVAS_ISAR_post_rend.txt - - coverage.info - - coverage + - $COVERAGE_OUTPUT_FILE + - $COVERAGE_OUTPUT_DIR expose_as: "Draft IVAS conformance -- Linux" reports: junit: report-junit.xml @@ -2224,7 +2228,7 @@ sanitizer-test-osba-planar-hoa3-ism4: - lcov -r $COVERAGE_OUTPUT_FILE "*lib_util*" -o $COVERAGE_OUTPUT_FILE - commit_sha=$(git rev-parse HEAD) # replace hardcoded path with relative path to allow easy merging of .info files - - sed -i "s|$(pwd)|..|g" $COVERAGE_OUTPUT_FILE + - sed -i "s|$(pwd)|.|g" $COVERAGE_OUTPUT_FILE - genhtml $COVERAGE_OUTPUT_FILE -o $COVERAGE_OUTPUT_DIR -t "Coverage on main -- $COVERAGE_TITLE @ $commit_sha" artifacts: name: "main-$COVERAGE_OUTPUT_DIR-sha-$CI_COMMIT_SHORT_SHA" @@ -2246,23 +2250,27 @@ coverage-merge: artifacts: true - job: ivas-conformance-linux artifacts: true + variables: + COVERAGE_OUTPUT_FILE: "$COVERAGE_OUTPUT_FILE_MERGED" + COVERAGE_OUTPUT_DIR: "coverage-merged" + COVERAGE_TITLE: "merged total coverage" script: - echo "Merging coverage reports..." - - lcov -a coverage.info -a coverage_stv.info -a coverage_ltv.info -o coverage_merged.info + - lcov -a $COVERAGE_OUTPUT_FILE_CONFORMANCE -a $COVERAGE_OUTPUT_FILE_STV -a $COVERAGE_OUTPUT_FILE_LTV -o "$COVERAGE_OUTPUT_FILE_MERGED" - commit_sha=$(git rev-parse HEAD) - - genhtml coverage_merged.info -o coverage_merged -t "Merged Coverage Report @ $commit_sha" + - genhtml $COVERAGE_OUTPUT_FILE_MERGED -o $COVERAGE_OUTPUT_DIR -t "Coverage on main -- $COVERAGE_TITLE @ $commit_sha" artifacts: name: "merged-coverage-sha-$CI_COMMIT_SHORT_SHA" when: always expire_in: 1 week paths: - - coverage_merged.info - - coverage_merged + - $COVERAGE_OUTPUT_FILE + - $COVERAGE_OUTPUT_DIR coverage-test-on-main-scheduled-stv: extends: .coverage-test-template variables: - COVERAGE_OUTPUT_FILE: "coverage_stv.info" + COVERAGE_OUTPUT_FILE: "$COVERAGE_OUTPUT_FILE_STV" COVERAGE_OUTPUT_DIR: "coverage_stv" COVERAGE_TITLE: "short test vectors" script: @@ -2298,13 +2306,13 @@ coverage-test-on-main-scheduled-stv: # TODO consider including check-clipping, but this scales ALL testv files. could be added at end of this job artifacts: paths: - - coverage_stv.info - - coverage_stv + - $COVERAGE_OUTPUT_FILE + - $COVERAGE_OUTPUT_DIR coverage-test-on-main-scheduled-ltv: extends: .coverage-test-template variables: - COVERAGE_OUTPUT_FILE: "coverage_ltv.info" + COVERAGE_OUTPUT_FILE: "$COVERAGE_OUTPUT_FILE_LTV" COVERAGE_OUTPUT_DIR: "coverage_ltv" COVERAGE_TITLE: "long test vectors" script: @@ -2327,8 +2335,8 @@ coverage-test-on-main-scheduled-ltv: # VoIP BE tests don't support LTV - skipped here artifacts: paths: - - coverage_ltv.info - - coverage_ltv + - $COVERAGE_OUTPUT_FILE + - $COVERAGE_OUTPUT_DIR # --------------------------------------------------------------- # Complexity measurement jobs diff --git a/.gitlab-ci/variables.yml b/.gitlab-ci/variables.yml index 52b811a001..b70de66596 100644 --- a/.gitlab-ci/variables.yml +++ b/.gitlab-ci/variables.yml @@ -29,3 +29,7 @@ variables: TEST_SUITE: "" # note: currently overwrites default value from ci repo TESTCASE_TIMEOUT_STV_SANITIZERS: 240 + COVERAGE_OUTPUT_FILE_STV: "coverage-stv.info" + COVERAGE_OUTPUT_FILE_LTV: "coverage-ltv.info" + COVERAGE_OUTPUT_FILE_CONFORMANCE: "coverage-conformance.info" + COVERAGE_OUTPUT_FILE_MERGED: "coverage-merged.info" -- GitLab From 40a41ba041a2d9941cc83bcb7a748f33da3fdb61 Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Mon, 6 Oct 2025 13:28:35 +0200 Subject: [PATCH 22/30] fix - use variable also in this spot --- .gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index f9e6e15c80..5682b3205e 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1451,7 +1451,7 @@ ivas-conformance-linux: - lcov -r $COVERAGE_OUTPUT_FILE "*lib_util*" -o $COVERAGE_OUTPUT_FILE - commit_sha=$(git rev-parse HEAD) # replace hardcoded path with relative path to allow easy merging of .info files - - sed -i "s|$(pwd)|.|g" coverage.info + - sed -i "s|$(pwd)|.|g" $COVERAGE_OUTPUT_FILE - genhtml $COVERAGE_OUTPUT_FILE -o $COVERAGE_OUTPUT_DIR -t "Coverage on main -- $COVERAGE_TITLE @ $commit_sha" # Check for failures -- GitLab From 8345f5f0d2b2fbd41ddca93aa035202b480298bd Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Mon, 6 Oct 2025 13:30:47 +0200 Subject: [PATCH 23/30] [revert-me] reduce number of tests for faster iteration --- .gitlab-ci.yml | 86 +++++++++++++++++++++++++------------------------- 1 file changed, 43 insertions(+), 43 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 5682b3205e..9b5b45dd0d 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1389,10 +1389,10 @@ ivas-conformance-linux: # Reference creation - python3 scripts/prepare_combined_format_inputs.py - TEST_SET="tests/codec_be_on_mr_nonselection tests/renderer/test_renderer.py tests/split_rendering/test_split_rendering.py" - - python3 -m pytest -q $TEST_SET -v -n auto --update_ref 1 --create_ref --keep_files + - python3 -m pytest -k stereo -q $TEST_SET -v -n auto --update_ref 1 --create_ref --keep_files # Output creation - - python3 -m pytest -q $TEST_SET -v -n auto --keep_files --create_cut --html=report_cmd.html --self-contained-html + - python3 -m pytest -k stereo -q $TEST_SET -v -n auto --keep_files --create_cut --html=report_cmd.html --self-contained-html - python3 scripts/parse_commands.py report_cmd.html Readme_IVAS.txt # Copy input data and output ref data @@ -1440,7 +1440,7 @@ ivas-conformance-linux: # Test run generated scripts in testvec - cd testvec - exit_code=0 - - python3 -m pytest -q conformance-test/test_26252.py --junit-xml=report-junit.xml --html=report.html --self-contained-html || exit_code=$? + - python3 -m pytest -k stereo -q conformance-test/test_26252.py --junit-xml=report-junit.xml --html=report.html --self-contained-html || exit_code=$? - mv report.html .. - mv report-junit.xml .. @@ -2276,33 +2276,33 @@ coverage-test-on-main-scheduled-stv: script: # -- Run short test vector suite to check coverage of that first # codec smoke test - - bash ci/smoke_test.sh coverage + # - bash ci/smoke_test.sh coverage # default renderer framesize is 20ms - - python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR -v -n auto -q --tb=no --update_ref 1 --ref_encoder_path ./IVAS_cod --ref_decoder_path ./IVAS_dec - # for different renderer framesizes, only the decoder needs to be tested, so use --decoder_only - # need to ignore non-zero exit codes as limiter is active and thus the different framesiszes will not be BE in all cases - - python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR -v -n auto -q --tb=no --dut_encoder_path ./IVAS_cod --dut_decoder_path ./IVAS_dec --dut_fr 5 --decoder_only || true - - python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR -v -n auto -q --tb=no --dut_encoder_path ./IVAS_cod --dut_decoder_path ./IVAS_dec --dut_fr 10 --decoder_only || true - # run renderer smoke test - needed for REFVEC tests - - python3 -m pytest -n auto -q --tb=no tests/renderer/test_renderer.py - # run renderer test only with REF creation stage - no CUT needed since executables are identical - - python3 -m pytest -n auto -q --tb=no tests/renderer/test_renderer.py --create_ref - # run split rendering smoke test and REF creation - - python3 -m pytest -n auto -q --tb=no tests/split_rendering/test_split_rendering.py - - python3 -m pytest -n auto -q --tb=no tests/split_rendering/test_split_rendering.py --create_ref - # VoIP BE test - - python3 -m pytest -q --tb=no tests/test_be_for_jbm_neutral_dly_profile.py - # split rendering VoIP BE test - - python3 -m pytest -n auto -q --tb=no tests/split_rendering/test_voip_be_splitrend_vs_binaural.py - # TODO first frame SID - this script runs a `make clean` and needs to be checked/adapted - # - bash ci/run-first-frame-is-sid-test.sh - # TODO consider HRTF binary loading test, or probably already covered by PRM files - # - python3 -m pytest -q --tb=no tests/hrtf_binary_loading - # run BE vs EXT test - - python3 tests/create_short_testvectors.py --cut_len 1.0 - - python3 -m pytest -q --tb=no tests/test_be_for_ext_outputs.py - # run LC3plus wrapper unit test - - scripts/split_rendering/lc3plus_float/ivas_lc3plus_unit_test + - python3 -m pytest -k stereo $TESTS_DIR_CODEC_BE_ON_MR -v -n auto -q --tb=no --update_ref 1 --ref_encoder_path ./IVAS_cod --ref_decoder_path ./IVAS_dec + # # for different renderer framesizes, only the decoder needs to be tested, so use --decoder_only + # # need to ignore non-zero exit codes as limiter is active and thus the different framesiszes will not be BE in all cases + # - python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR -v -n auto -q --tb=no --dut_encoder_path ./IVAS_cod --dut_decoder_path ./IVAS_dec --dut_fr 5 --decoder_only || true + # - python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR -v -n auto -q --tb=no --dut_encoder_path ./IVAS_cod --dut_decoder_path ./IVAS_dec --dut_fr 10 --decoder_only || true + # # run renderer smoke test - needed for REFVEC tests + # - python3 -m pytest -n auto -q --tb=no tests/renderer/test_renderer.py + # # run renderer test only with REF creation stage - no CUT needed since executables are identical + # - python3 -m pytest -n auto -q --tb=no tests/renderer/test_renderer.py --create_ref + # # run split rendering smoke test and REF creation + # - python3 -m pytest -n auto -q --tb=no tests/split_rendering/test_split_rendering.py + # - python3 -m pytest -n auto -q --tb=no tests/split_rendering/test_split_rendering.py --create_ref + # # VoIP BE test + # - python3 -m pytest -q --tb=no tests/test_be_for_jbm_neutral_dly_profile.py + # # split rendering VoIP BE test + # - python3 -m pytest -n auto -q --tb=no tests/split_rendering/test_voip_be_splitrend_vs_binaural.py + # # TODO first frame SID - this script runs a `make clean` and needs to be checked/adapted + # # - bash ci/run-first-frame-is-sid-test.sh + # # TODO consider HRTF binary loading test, or probably already covered by PRM files + # # - python3 -m pytest -q --tb=no tests/hrtf_binary_loading + # # run BE vs EXT test + # - python3 tests/create_short_testvectors.py --cut_len 1.0 + # - python3 -m pytest -q --tb=no tests/test_be_for_ext_outputs.py + # # run LC3plus wrapper unit test + # - scripts/split_rendering/lc3plus_float/ivas_lc3plus_unit_test # TODO consider including check-clipping, but this scales ALL testv files. could be added at end of this job artifacts: paths: @@ -2317,20 +2317,20 @@ coverage-test-on-main-scheduled-ltv: COVERAGE_TITLE: "long test vectors" script: # -- Add extended coverage of EVS test and long test vectors - - python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR/test_param_file.py -v -n auto -q --tb=no --update_ref 1 --param_file scripts/config/self_test_evs.prm --ref_encoder_path ./IVAS_cod --ref_decoder_path ./IVAS_dec - # default renderer framesize is 20ms - - python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR/test_param_file.py -v -n auto -q --tb=no --update_ref 1 --param_file scripts/config/self_test_ltv.prm --ref_encoder_path ./IVAS_cod --ref_decoder_path ./IVAS_dec - # for different renderer framesizes, only the decoder needs to be tested, so use --decoder_only - # need to ignore non-zero exit codes as limiter is active and thus the different framesiszes will not be BE in all cases - - python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR/test_param_file.py -v -n auto -q --tb=no --param_file scripts/config/self_test_ltv.prm --dut_decoder_path ./IVAS_dec --decoder_only --dut_fr 5 || true - - python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR/test_param_file.py -v -n auto -q --tb=no --param_file scripts/config/self_test_ltv.prm --dut_decoder_path ./IVAS_dec --decoder_only --dut_fr 10 || true - - python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR/test_sba.py -v -n auto -q --tb=no --update_ref 1 --ref_encoder_path ./IVAS_cod --ref_decoder_path ./IVAS_dec --use_ltv - - python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR/test_sba.py -v -n auto -q --tb=no --dut_encoder_path ./IVAS_cod --dut_decoder_path ./IVAS_dec --dut_fr 5 --decoder_only --use_ltv || true - - python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR/test_sba.py -v -n auto -q --tb=no --dut_encoder_path ./IVAS_cod --dut_decoder_path ./IVAS_dec --dut_fr 10 --decoder_only --use_ltv || true - # run renderer smoke test - needed for REFVEC tests - - ls -altr scripts/testv - # LTV smoketest is the same as REF creation stage for renderer - - python3 -m pytest -n auto -q --tb=no tests/renderer/test_renderer.py --use_ltv + - python3 -m pytest -k stereo $TESTS_DIR_CODEC_BE_ON_MR/test_param_file.py -v -n auto -q --tb=no --update_ref 1 --param_file scripts/config/self_test_evs.prm --ref_encoder_path ./IVAS_cod --ref_decoder_path ./IVAS_dec + # # default renderer framesize is 20ms + # - python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR/test_param_file.py -v -n auto -q --tb=no --update_ref 1 --param_file scripts/config/self_test_ltv.prm --ref_encoder_path ./IVAS_cod --ref_decoder_path ./IVAS_dec + # # for different renderer framesizes, only the decoder needs to be tested, so use --decoder_only + # # need to ignore non-zero exit codes as limiter is active and thus the different framesiszes will not be BE in all cases + # - python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR/test_param_file.py -v -n auto -q --tb=no --param_file scripts/config/self_test_ltv.prm --dut_decoder_path ./IVAS_dec --decoder_only --dut_fr 5 || true + # - python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR/test_param_file.py -v -n auto -q --tb=no --param_file scripts/config/self_test_ltv.prm --dut_decoder_path ./IVAS_dec --decoder_only --dut_fr 10 || true + # - python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR/test_sba.py -v -n auto -q --tb=no --update_ref 1 --ref_encoder_path ./IVAS_cod --ref_decoder_path ./IVAS_dec --use_ltv + # - python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR/test_sba.py -v -n auto -q --tb=no --dut_encoder_path ./IVAS_cod --dut_decoder_path ./IVAS_dec --dut_fr 5 --decoder_only --use_ltv || true + # - python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR/test_sba.py -v -n auto -q --tb=no --dut_encoder_path ./IVAS_cod --dut_decoder_path ./IVAS_dec --dut_fr 10 --decoder_only --use_ltv || true + # # run renderer smoke test - needed for REFVEC tests + # - ls -altr scripts/testv + # # LTV smoketest is the same as REF creation stage for renderer + # - python3 -m pytest -n auto -q --tb=no tests/renderer/test_renderer.py --use_ltv # split rendering test doesn't support LTV - skipped here # VoIP BE tests don't support LTV - skipped here artifacts: -- GitLab From 0f6023ff51c0f8802f1e8894b28595e8d8767221 Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Mon, 6 Oct 2025 13:40:02 +0200 Subject: [PATCH 24/30] fix reduction of testcase in LTV test --- .gitlab-ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 9b5b45dd0d..1fcf22f5c1 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -2317,9 +2317,9 @@ coverage-test-on-main-scheduled-ltv: COVERAGE_TITLE: "long test vectors" script: # -- Add extended coverage of EVS test and long test vectors - - python3 -m pytest -k stereo $TESTS_DIR_CODEC_BE_ON_MR/test_param_file.py -v -n auto -q --tb=no --update_ref 1 --param_file scripts/config/self_test_evs.prm --ref_encoder_path ./IVAS_cod --ref_decoder_path ./IVAS_dec + # - python3 -m pytest -k stereo $TESTS_DIR_CODEC_BE_ON_MR/test_param_file.py -v -n auto -q --tb=no --update_ref 1 --param_file scripts/config/self_test_evs.prm --ref_encoder_path ./IVAS_cod --ref_decoder_path ./IVAS_dec # # default renderer framesize is 20ms - # - python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR/test_param_file.py -v -n auto -q --tb=no --update_ref 1 --param_file scripts/config/self_test_ltv.prm --ref_encoder_path ./IVAS_cod --ref_decoder_path ./IVAS_dec + - python3 -m pytest -k stereo $TESTS_DIR_CODEC_BE_ON_MR/test_param_file.py -v -n auto -q --tb=no --update_ref 1 --param_file scripts/config/self_test_ltv.prm --ref_encoder_path ./IVAS_cod --ref_decoder_path ./IVAS_dec # # for different renderer framesizes, only the decoder needs to be tested, so use --decoder_only # # need to ignore non-zero exit codes as limiter is active and thus the different framesiszes will not be BE in all cases # - python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR/test_param_file.py -v -n auto -q --tb=no --param_file scripts/config/self_test_ltv.prm --dut_decoder_path ./IVAS_dec --decoder_only --dut_fr 5 || true -- GitLab From 8a9bdb93103c565c823b059ff42b23c8f335b1e4 Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Mon, 6 Oct 2025 14:18:11 +0200 Subject: [PATCH 25/30] use ci snippet for coverage collection --- .gitlab-ci.yml | 17 ++--------------- 1 file changed, 2 insertions(+), 15 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 1fcf22f5c1..1710466465 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1446,13 +1446,7 @@ ivas-conformance-linux: # Collect coverage - cd - - - lcov -c -d obj -o $COVERAGE_OUTPUT_FILE - - lcov -r $COVERAGE_OUTPUT_FILE "*apps*" -o $COVERAGE_OUTPUT_FILE - - lcov -r $COVERAGE_OUTPUT_FILE "*lib_util*" -o $COVERAGE_OUTPUT_FILE - - commit_sha=$(git rev-parse HEAD) - # replace hardcoded path with relative path to allow easy merging of .info files - - sed -i "s|$(pwd)|.|g" $COVERAGE_OUTPUT_FILE - - genhtml $COVERAGE_OUTPUT_FILE -o $COVERAGE_OUTPUT_DIR -t "Coverage on main -- $COVERAGE_TITLE @ $commit_sha" + - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/collect-coverage.sh --obj-dir obj --output-file $COVERAGE_OUTPUT_FILE --output-dir $COVERAGE_OUTPUT_DIR --title "Coverage on main -- $COVERAGE_TITLE" # Check for failures - if [ $exit_code -ne 0 ]; then echo "Test failures encountered"; exit $EXIT_CODE_FAIL; fi @@ -2222,14 +2216,7 @@ sanitizer-test-osba-planar-hoa3-ism4: - cp IVAS_rend IVAS_rend_ref - cp ISAR_post_rend ISAR_post_rend_ref after_script: - - lcov -c -d cmake-build -o $COVERAGE_OUTPUT_FILE - # remove apps and lib_util files from coverage - - lcov -r $COVERAGE_OUTPUT_FILE "*apps*" -o $COVERAGE_OUTPUT_FILE - - lcov -r $COVERAGE_OUTPUT_FILE "*lib_util*" -o $COVERAGE_OUTPUT_FILE - - commit_sha=$(git rev-parse HEAD) - # replace hardcoded path with relative path to allow easy merging of .info files - - sed -i "s|$(pwd)|.|g" $COVERAGE_OUTPUT_FILE - - genhtml $COVERAGE_OUTPUT_FILE -o $COVERAGE_OUTPUT_DIR -t "Coverage on main -- $COVERAGE_TITLE @ $commit_sha" + - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/collect-coverage.sh --obj-dir cmake-build --output-file $COVERAGE_OUTPUT_FILE --output-dir $COVERAGE_OUTPUT_DIR --title "Coverage on main -- $COVERAGE_TITLE" artifacts: name: "main-$COVERAGE_OUTPUT_DIR-sha-$CI_COMMIT_SHORT_SHA" when: always -- GitLab From cecfb716842d67174996a8e8a984a91a4bef775d Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Mon, 6 Oct 2025 14:18:51 +0200 Subject: [PATCH 26/30] [revert-me] change CI ref for testing new snippet --- .gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 1710466465..22e926b7a9 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1,7 +1,7 @@ variables: # note: GitLab cannot reference variables defined by users in the include ref:, we need to use a YAML anchor for this # see https://docs.gitlab.com/ci/yaml/includes/#use-variables-with-include for more information - IVAS_CODEC_CI_REF: &IVAS_CODEC_CI_REF main + IVAS_CODEC_CI_REF: &IVAS_CODEC_CI_REF add-snippet-for-coverage-collection include: - local: .gitlab-ci/variables.yml -- GitLab From d8e91ec508be3476ab07e5bfa488b6cfd2715a6c Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Mon, 6 Oct 2025 15:07:58 +0200 Subject: [PATCH 27/30] Revert "[revert-me] change CI ref for testing new snippet" This reverts commit cecfb716842d67174996a8e8a984a91a4bef775d. --- .gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 22e926b7a9..1710466465 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1,7 +1,7 @@ variables: # note: GitLab cannot reference variables defined by users in the include ref:, we need to use a YAML anchor for this # see https://docs.gitlab.com/ci/yaml/includes/#use-variables-with-include for more information - IVAS_CODEC_CI_REF: &IVAS_CODEC_CI_REF add-snippet-for-coverage-collection + IVAS_CODEC_CI_REF: &IVAS_CODEC_CI_REF main include: - local: .gitlab-ci/variables.yml -- GitLab From 3cd7de857159c593d28d6106ebf3f83745bd9cd9 Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Mon, 6 Oct 2025 15:09:18 +0200 Subject: [PATCH 28/30] Revert "fix reduction of testcase in LTV test" This reverts commit 0f6023ff51c0f8802f1e8894b28595e8d8767221. --- .gitlab-ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 1710466465..68df34351a 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -2304,9 +2304,9 @@ coverage-test-on-main-scheduled-ltv: COVERAGE_TITLE: "long test vectors" script: # -- Add extended coverage of EVS test and long test vectors - # - python3 -m pytest -k stereo $TESTS_DIR_CODEC_BE_ON_MR/test_param_file.py -v -n auto -q --tb=no --update_ref 1 --param_file scripts/config/self_test_evs.prm --ref_encoder_path ./IVAS_cod --ref_decoder_path ./IVAS_dec + - python3 -m pytest -k stereo $TESTS_DIR_CODEC_BE_ON_MR/test_param_file.py -v -n auto -q --tb=no --update_ref 1 --param_file scripts/config/self_test_evs.prm --ref_encoder_path ./IVAS_cod --ref_decoder_path ./IVAS_dec # # default renderer framesize is 20ms - - python3 -m pytest -k stereo $TESTS_DIR_CODEC_BE_ON_MR/test_param_file.py -v -n auto -q --tb=no --update_ref 1 --param_file scripts/config/self_test_ltv.prm --ref_encoder_path ./IVAS_cod --ref_decoder_path ./IVAS_dec + # - python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR/test_param_file.py -v -n auto -q --tb=no --update_ref 1 --param_file scripts/config/self_test_ltv.prm --ref_encoder_path ./IVAS_cod --ref_decoder_path ./IVAS_dec # # for different renderer framesizes, only the decoder needs to be tested, so use --decoder_only # # need to ignore non-zero exit codes as limiter is active and thus the different framesiszes will not be BE in all cases # - python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR/test_param_file.py -v -n auto -q --tb=no --param_file scripts/config/self_test_ltv.prm --dut_decoder_path ./IVAS_dec --decoder_only --dut_fr 5 || true -- GitLab From fd6dda87d84ac0300ae107a6fd9d2b057e4e7c65 Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Mon, 6 Oct 2025 15:09:26 +0200 Subject: [PATCH 29/30] Revert "[revert-me] reduce number of tests for faster iteration" This reverts commit 8345f5f0d2b2fbd41ddca93aa035202b480298bd. --- .gitlab-ci.yml | 86 +++++++++++++++++++++++++------------------------- 1 file changed, 43 insertions(+), 43 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 68df34351a..8036827b99 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1389,10 +1389,10 @@ ivas-conformance-linux: # Reference creation - python3 scripts/prepare_combined_format_inputs.py - TEST_SET="tests/codec_be_on_mr_nonselection tests/renderer/test_renderer.py tests/split_rendering/test_split_rendering.py" - - python3 -m pytest -k stereo -q $TEST_SET -v -n auto --update_ref 1 --create_ref --keep_files + - python3 -m pytest -q $TEST_SET -v -n auto --update_ref 1 --create_ref --keep_files # Output creation - - python3 -m pytest -k stereo -q $TEST_SET -v -n auto --keep_files --create_cut --html=report_cmd.html --self-contained-html + - python3 -m pytest -q $TEST_SET -v -n auto --keep_files --create_cut --html=report_cmd.html --self-contained-html - python3 scripts/parse_commands.py report_cmd.html Readme_IVAS.txt # Copy input data and output ref data @@ -1440,7 +1440,7 @@ ivas-conformance-linux: # Test run generated scripts in testvec - cd testvec - exit_code=0 - - python3 -m pytest -k stereo -q conformance-test/test_26252.py --junit-xml=report-junit.xml --html=report.html --self-contained-html || exit_code=$? + - python3 -m pytest -q conformance-test/test_26252.py --junit-xml=report-junit.xml --html=report.html --self-contained-html || exit_code=$? - mv report.html .. - mv report-junit.xml .. @@ -2263,33 +2263,33 @@ coverage-test-on-main-scheduled-stv: script: # -- Run short test vector suite to check coverage of that first # codec smoke test - # - bash ci/smoke_test.sh coverage + - bash ci/smoke_test.sh coverage # default renderer framesize is 20ms - - python3 -m pytest -k stereo $TESTS_DIR_CODEC_BE_ON_MR -v -n auto -q --tb=no --update_ref 1 --ref_encoder_path ./IVAS_cod --ref_decoder_path ./IVAS_dec - # # for different renderer framesizes, only the decoder needs to be tested, so use --decoder_only - # # need to ignore non-zero exit codes as limiter is active and thus the different framesiszes will not be BE in all cases - # - python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR -v -n auto -q --tb=no --dut_encoder_path ./IVAS_cod --dut_decoder_path ./IVAS_dec --dut_fr 5 --decoder_only || true - # - python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR -v -n auto -q --tb=no --dut_encoder_path ./IVAS_cod --dut_decoder_path ./IVAS_dec --dut_fr 10 --decoder_only || true - # # run renderer smoke test - needed for REFVEC tests - # - python3 -m pytest -n auto -q --tb=no tests/renderer/test_renderer.py - # # run renderer test only with REF creation stage - no CUT needed since executables are identical - # - python3 -m pytest -n auto -q --tb=no tests/renderer/test_renderer.py --create_ref - # # run split rendering smoke test and REF creation - # - python3 -m pytest -n auto -q --tb=no tests/split_rendering/test_split_rendering.py - # - python3 -m pytest -n auto -q --tb=no tests/split_rendering/test_split_rendering.py --create_ref - # # VoIP BE test - # - python3 -m pytest -q --tb=no tests/test_be_for_jbm_neutral_dly_profile.py - # # split rendering VoIP BE test - # - python3 -m pytest -n auto -q --tb=no tests/split_rendering/test_voip_be_splitrend_vs_binaural.py - # # TODO first frame SID - this script runs a `make clean` and needs to be checked/adapted - # # - bash ci/run-first-frame-is-sid-test.sh - # # TODO consider HRTF binary loading test, or probably already covered by PRM files - # # - python3 -m pytest -q --tb=no tests/hrtf_binary_loading - # # run BE vs EXT test - # - python3 tests/create_short_testvectors.py --cut_len 1.0 - # - python3 -m pytest -q --tb=no tests/test_be_for_ext_outputs.py - # # run LC3plus wrapper unit test - # - scripts/split_rendering/lc3plus_float/ivas_lc3plus_unit_test + - python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR -v -n auto -q --tb=no --update_ref 1 --ref_encoder_path ./IVAS_cod --ref_decoder_path ./IVAS_dec + # for different renderer framesizes, only the decoder needs to be tested, so use --decoder_only + # need to ignore non-zero exit codes as limiter is active and thus the different framesiszes will not be BE in all cases + - python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR -v -n auto -q --tb=no --dut_encoder_path ./IVAS_cod --dut_decoder_path ./IVAS_dec --dut_fr 5 --decoder_only || true + - python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR -v -n auto -q --tb=no --dut_encoder_path ./IVAS_cod --dut_decoder_path ./IVAS_dec --dut_fr 10 --decoder_only || true + # run renderer smoke test - needed for REFVEC tests + - python3 -m pytest -n auto -q --tb=no tests/renderer/test_renderer.py + # run renderer test only with REF creation stage - no CUT needed since executables are identical + - python3 -m pytest -n auto -q --tb=no tests/renderer/test_renderer.py --create_ref + # run split rendering smoke test and REF creation + - python3 -m pytest -n auto -q --tb=no tests/split_rendering/test_split_rendering.py + - python3 -m pytest -n auto -q --tb=no tests/split_rendering/test_split_rendering.py --create_ref + # VoIP BE test + - python3 -m pytest -q --tb=no tests/test_be_for_jbm_neutral_dly_profile.py + # split rendering VoIP BE test + - python3 -m pytest -n auto -q --tb=no tests/split_rendering/test_voip_be_splitrend_vs_binaural.py + # TODO first frame SID - this script runs a `make clean` and needs to be checked/adapted + # - bash ci/run-first-frame-is-sid-test.sh + # TODO consider HRTF binary loading test, or probably already covered by PRM files + # - python3 -m pytest -q --tb=no tests/hrtf_binary_loading + # run BE vs EXT test + - python3 tests/create_short_testvectors.py --cut_len 1.0 + - python3 -m pytest -q --tb=no tests/test_be_for_ext_outputs.py + # run LC3plus wrapper unit test + - scripts/split_rendering/lc3plus_float/ivas_lc3plus_unit_test # TODO consider including check-clipping, but this scales ALL testv files. could be added at end of this job artifacts: paths: @@ -2304,20 +2304,20 @@ coverage-test-on-main-scheduled-ltv: COVERAGE_TITLE: "long test vectors" script: # -- Add extended coverage of EVS test and long test vectors - - python3 -m pytest -k stereo $TESTS_DIR_CODEC_BE_ON_MR/test_param_file.py -v -n auto -q --tb=no --update_ref 1 --param_file scripts/config/self_test_evs.prm --ref_encoder_path ./IVAS_cod --ref_decoder_path ./IVAS_dec - # # default renderer framesize is 20ms - # - python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR/test_param_file.py -v -n auto -q --tb=no --update_ref 1 --param_file scripts/config/self_test_ltv.prm --ref_encoder_path ./IVAS_cod --ref_decoder_path ./IVAS_dec - # # for different renderer framesizes, only the decoder needs to be tested, so use --decoder_only - # # need to ignore non-zero exit codes as limiter is active and thus the different framesiszes will not be BE in all cases - # - python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR/test_param_file.py -v -n auto -q --tb=no --param_file scripts/config/self_test_ltv.prm --dut_decoder_path ./IVAS_dec --decoder_only --dut_fr 5 || true - # - python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR/test_param_file.py -v -n auto -q --tb=no --param_file scripts/config/self_test_ltv.prm --dut_decoder_path ./IVAS_dec --decoder_only --dut_fr 10 || true - # - python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR/test_sba.py -v -n auto -q --tb=no --update_ref 1 --ref_encoder_path ./IVAS_cod --ref_decoder_path ./IVAS_dec --use_ltv - # - python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR/test_sba.py -v -n auto -q --tb=no --dut_encoder_path ./IVAS_cod --dut_decoder_path ./IVAS_dec --dut_fr 5 --decoder_only --use_ltv || true - # - python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR/test_sba.py -v -n auto -q --tb=no --dut_encoder_path ./IVAS_cod --dut_decoder_path ./IVAS_dec --dut_fr 10 --decoder_only --use_ltv || true - # # run renderer smoke test - needed for REFVEC tests - # - ls -altr scripts/testv - # # LTV smoketest is the same as REF creation stage for renderer - # - python3 -m pytest -n auto -q --tb=no tests/renderer/test_renderer.py --use_ltv + - python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR/test_param_file.py -v -n auto -q --tb=no --update_ref 1 --param_file scripts/config/self_test_evs.prm --ref_encoder_path ./IVAS_cod --ref_decoder_path ./IVAS_dec + # default renderer framesize is 20ms + - python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR/test_param_file.py -v -n auto -q --tb=no --update_ref 1 --param_file scripts/config/self_test_ltv.prm --ref_encoder_path ./IVAS_cod --ref_decoder_path ./IVAS_dec + # for different renderer framesizes, only the decoder needs to be tested, so use --decoder_only + # need to ignore non-zero exit codes as limiter is active and thus the different framesiszes will not be BE in all cases + - python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR/test_param_file.py -v -n auto -q --tb=no --param_file scripts/config/self_test_ltv.prm --dut_decoder_path ./IVAS_dec --decoder_only --dut_fr 5 || true + - python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR/test_param_file.py -v -n auto -q --tb=no --param_file scripts/config/self_test_ltv.prm --dut_decoder_path ./IVAS_dec --decoder_only --dut_fr 10 || true + - python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR/test_sba.py -v -n auto -q --tb=no --update_ref 1 --ref_encoder_path ./IVAS_cod --ref_decoder_path ./IVAS_dec --use_ltv + - python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR/test_sba.py -v -n auto -q --tb=no --dut_encoder_path ./IVAS_cod --dut_decoder_path ./IVAS_dec --dut_fr 5 --decoder_only --use_ltv || true + - python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR/test_sba.py -v -n auto -q --tb=no --dut_encoder_path ./IVAS_cod --dut_decoder_path ./IVAS_dec --dut_fr 10 --decoder_only --use_ltv || true + # run renderer smoke test - needed for REFVEC tests + - ls -altr scripts/testv + # LTV smoketest is the same as REF creation stage for renderer + - python3 -m pytest -n auto -q --tb=no tests/renderer/test_renderer.py --use_ltv # split rendering test doesn't support LTV - skipped here # VoIP BE tests don't support LTV - skipped here artifacts: -- GitLab From 35f451dcab00bec6534df1fc19f7ec203f33afa1 Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Mon, 6 Oct 2025 16:15:42 +0200 Subject: [PATCH 30/30] integrate new coverage jobs into gtlab pages setup --- ci/index-pages.html | 5 +---- ci/setup_pages.py | 35 +++++++++++++++++++++++++++-------- 2 files changed, 28 insertions(+), 12 deletions(-) diff --git a/ci/index-pages.html b/ci/index-pages.html index 510e203115..390f06631d 100644 --- a/ci/index-pages.html +++ b/ci/index-pages.html @@ -11,9 +11,6 @@

Test Coverage

- + {} diff --git a/ci/setup_pages.py b/ci/setup_pages.py index df350d8568..1a1c1ea735 100755 --- a/ci/setup_pages.py +++ b/ci/setup_pages.py @@ -40,7 +40,10 @@ JOBS_FLOAT_REPO = { "complexity-osba-in-binaural_room_ir-out": "OSBA in, BINAURAL_ROOM_IR out", "complexity-StereoDmxEVS-stereo-in-mono-out": "StereoDmxEVS, Stereo in, Mono out", # "timeless" jobs (not complexity) - "coverage-test-on-main-scheduled": "Coverage", + "coverage-test-on-main-scheduled-stv": "Short test vector coverage (~ merge request pipelines)", + "coverage-test-on-main-scheduled-ltv": "Long test vector coverage (for test_param_file.py, test_sba.py and test_renderer.py)", + "ivas-conformance-linux": "Conformance test coverage", + "coverage-merge": "Merged coverage (all of the above combined)", } JOBS_BASOP_REPO = { "ivas-pytest-compare_ref-long-dec": "Pytest decoder compare to ref LTV", @@ -77,6 +80,13 @@ JOBS_FOR_PROJECT_ID = { PROJECT_ID_BASOP_REPO: JOBS_BASOP_REPO, } +ARTIFACT_FOLDER_4_COVERAGE_JOBS = { + "coverage-test-on-main-scheduled-stv": "coverage_stv", + "coverage-test-on-main-scheduled-ltv": "coverage_ltv", + "ivas-conformance-linux": "coverage_conformance", + "coverage-merge": "coverage-merged", +} + ARTIFACTS = "artifacts.zip" API_URL_BASE = "https://forge.3gpp.org/rep/api/v4/projects/{}/jobs" PUBLIC_FOLDER = pathlib.Path("./public").absolute() @@ -104,15 +114,20 @@ def main(): def create_landing_page(jobs, index_html, project_id): # dynamically create the complexity links on the landing page - link_html = list() - link_html = ["
    "] + link_html_complexity = ["
      "] + link_html_coverage = ["
        "] for job, link_text in jobs.items(): if job.startswith("complexity"): line = f'
      • {link_text}
        • ' - link_html.append(line) - link_html.append("
      ") - link_html_text = "\n".join(link_html) + link_html_complexity.append(line) + elif job.startswith("coverage") or job == "ivas-conformance-linux": + artifact_folder = ARTIFACT_FOLDER_4_COVERAGE_JOBS[job] + line = f'
    • {link_text}
      • ' + link_html_coverage.append(line) + + link_html_complexity.append("
    ") + link_html_coverage.append("
") if project_id == PROJECT_ID_FLOAT_REPO: index_template = "index-pages.html" @@ -126,14 +141,18 @@ def create_landing_page(jobs, index_html, project_id): with open(index_pages_tmpl_path) as f: index_pages_tmpl = f.read() - index_pages_tmpl = index_pages_tmpl.format(link_html_text) + link_html_complexity_text = "\n".join(link_html_complexity) + link_html_coverage_text = "\n".join(link_html_coverage) + index_pages_tmpl = index_pages_tmpl.format( + link_html_complexity_text, link_html_coverage_text + ) with open(index_html, "w") as f: f.write(index_pages_tmpl) def get_artifacts_for_jobs_and_return_num_failed( - jobs: list, project_id: int, success_only: bool + jobs: dict, project_id: int, success_only: bool ) -> int: """ Get specified artifact folders for all jobs given and put them into the public folder. -- GitLab