From dd2c551db9ea194a12968e35a5122483206fae04 Mon Sep 17 00:00:00 2001 From: Erik Norvell Date: Tue, 5 Nov 2024 11:49:33 +0100 Subject: [PATCH 1/8] Adding coverage test --- .gitlab-ci.yml | 90 ++++++++++++++++++++++++++++++++++++++++++-------- 1 file changed, 77 insertions(+), 13 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 3167243ff..0f26f2ac4 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -8,6 +8,7 @@ variables: EXIT_CODE_NON_BE: 123 EXIT_CODE_FAIL: 1 LONG_TEST_SUITE: "tests/codec_be_on_mr_nonselection tests/renderer --param_file scripts/config/self_test_ltv.prm --use_ltv" + LONG_TEST_SUITE_NO_RENDERER: "tests/codec_be_on_mr_nonselection --param_file scripts/config/self_test_ltv.prm --use_ltv" SHORT_TEST_SUITE: "tests/codec_be_on_mr_nonselection" TEST_SUITE: "" DUT_ENCODER_PATH: "./IVAS_cod" @@ -111,20 +112,25 @@ stages: echo "Commit time was $CI_COMMIT_TIMESTAMP" date | xargs echo "System time is" -.build-reference-and-dut-binaries: &build-reference-and-dut-binaries - - current_commit_sha=$(git rev-parse HEAD) - ### build reference binaries - - git checkout $REFERENCE_BRANCH - - git pull - - make clean - - make -j - - mv ./IVAS_cod ./$REF_ENCODER_PATH - - mv ./IVAS_dec ./$REF_DECODER_PATH - - mv ./IVAS_rend ./IVAS_rend_ref +.build-reference-binaries: &build-reference-binaries +- current_commit_sha=$(git rev-parse HEAD) +### build reference binaries +- git checkout $REFERENCE_BRANCH +- git pull +- make clean +- make -j +- mv ./IVAS_cod ./$REF_ENCODER_PATH +- mv ./IVAS_dec ./$REF_DECODER_PATH +- mv ./IVAS_rend ./IVAS_rend_ref +### Return to current branch +- git restore . +- git checkout $current_commit_sha - ### build test binaries - - git restore . - - git checkout $current_commit_sha + +.build-reference-and-dut-binaries: &build-reference-and-dut-binaries +### build reference binaries + - *build-reference-binaries +### build dut binaries - make clean - make -j @@ -212,6 +218,15 @@ stages: - if: $CI_PIPELINE_SOURCE == 'merge_request_event' when: never +.rules-coverage: + rules: + - if: $COVERAGE_TEST # Set by scheduled pipeline + - if: $CI_PIPELINE_SOURCE == 'web' && $MANUAL_PIPELINE_TYPE == "coverage" + - if: $CI_PIPELINE_SOURCE == 'push' + when: never + - if: $CI_PIPELINE_SOURCE == 'merge_request_event' + when: never + .rules-pytest-saturation-smoke-test: rules: - if: $PYTEST_SMOKE_TEST # Set by scheduled pipeline @@ -996,6 +1011,55 @@ ivas-smoke-test-saturation: - smoke_test_output_hrtf.txt expose_as: "saturation smoke test results" + +# GCOV/LCOV coverage analysis of self_test suite +coverage-test-on-main-scheduled: + extends: + - .rules-coverage + tags: + - ivas-linux-basop + stage: test + timeout: 3 hours + script: + - *print-common-info + - *update-ltv-repo + - *copy-ltv-files-to-testv-dir + - *build-reference-binaries + # Build DuT binaries with GCOV + - make clean + - make GCOV=1 -j + + - exit_code_dec=0 + - python3 -m pytest --tb=no $LONG_TEST_SUITE --update_ref 1 -v --create_ref --html=report-dec.html --self-contained-html --junit-xml=report-junit-dec.xml -n auto --testcase_timeout $testcase_timeout --ref_encoder_path $REF_ENCODER_PATH --ref_decoder_path $DUT_DECODER_PATH || exit_code_dec=$? + - lcov -c -d obj -o coverage_dec_rend.info # extract coverage of decoder/renderer + + - python3 -m pytest --tb=no $LONG_TEST_SUITE_NO_RENDERER --encoder_only -v --html=report-enc.html --self-contained-html --junit-xml=report-junit-enc.xml -n auto --testcase_timeout $testcase_timeout --dut_encoder_path $DUT_ENCODER_PATH || exit_code_enc=$? + - lcov -c -d obj -o coverage_enc_dec_rend.info # extract coverage of encoder/decoder/renderer + + # remove apps and lib_util files from coverage + - lcov -r coverage_dec_rend.info "*apps*" -o coverage_dec_rend.info + - lcov -r coverage_dec_rend.info "*lib_util*" -o coverage_dec_rend.info + - lcov -r coverage_enc_dec_rend.info "*apps*" -o coverage_enc_dec_rend.info + - lcov -r coverage_enc_dec_rend.info "*lib_util*" -o coverage_enc_dec_rend.info + + - commit_sha=$(git rev-parse HEAD) + - genhtml coverage_enc_dec_rend.info -o coverage_enc_dec_rend -t "Coverage on main enc/dec/rend @ $commit_sha" + - genhtml coverage_dec_rend.info -o coverage_dec_rend -t "Coverage on main -- dec/rend @ $commit_sha" + artifacts: + name: "main-coverage-sha-$CI_COMMIT_SHORT_SHA" + when: always + expire_in: 1 week + paths: + - coverage_enc_dec_rend.info + - coverage_dec_rend.info + - coverage_enc_dec_rend + - coverage_dec_rend + expose_as: "Coverage result" + reports: + junit: + - report-junit-dec.xml + - report-junit-enc.xml + # --------------------------------------------------------------- # EVS 26.444 test job # --------------------------------------------------------------- -- GitLab From c9b81367e4b0f24ea93a5573a0b59f55d90baebf Mon Sep 17 00:00:00 2001 From: Erik Norvell Date: Tue, 5 Nov 2024 11:51:47 +0100 Subject: [PATCH 2/8] Add manual job in drop-down for coverage --- .gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 0f26f2ac4..4eefa5b62 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -41,7 +41,7 @@ variables: - 'sanitizer-stv' - 'pytest-renderer' - 'complexity' - + - 'coverage' default: interruptible: true # Make all jobs by default interruptible -- GitLab From c2b230abea3055ee33f5e2a3cb8c04e516942171 Mon Sep 17 00:00:00 2001 From: Erik Norvell Date: Tue, 5 Nov 2024 11:54:30 +0100 Subject: [PATCH 3/8] Fix rules on coverage job --- .gitlab-ci.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 4eefa5b62..6fe04ea76 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1015,9 +1015,8 @@ ivas-smoke-test-saturation: # GCOV/LCOV coverage analysis of self_test suite coverage-test-on-main-scheduled: extends: + - .test-job-linux - .rules-coverage - tags: - - ivas-linux-basop stage: test timeout: 3 hours script: -- GitLab From ed78c6b67a364a0b48504283734a3744088542a6 Mon Sep 17 00:00:00 2001 From: Erik Norvell Date: Tue, 5 Nov 2024 14:54:47 +0100 Subject: [PATCH 4/8] Add *update-scripts-repo to coverage job --- .gitlab-ci.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 6fe04ea76..7f2eec09e 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1023,6 +1023,7 @@ coverage-test-on-main-scheduled: - *print-common-info - *update-ltv-repo - *copy-ltv-files-to-testv-dir + - *update-scripts-repo - *build-reference-binaries # Build DuT binaries with GCOV - make clean -- GitLab From b88a643ae5a025a4f39d0b48aba3d321c92d6c71 Mon Sep 17 00:00:00 2001 From: Erik Norvell Date: Tue, 5 Nov 2024 15:14:40 +0100 Subject: [PATCH 5/8] Fix order of update-scripts-repo in coverage job --- .gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 7f2eec09e..418e5a8c5 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1021,9 +1021,9 @@ coverage-test-on-main-scheduled: timeout: 3 hours script: - *print-common-info + - *update-scripts-repo - *update-ltv-repo - *copy-ltv-files-to-testv-dir - - *update-scripts-repo - *build-reference-binaries # Build DuT binaries with GCOV - make clean -- GitLab From 42dffbbf291fe054663a69c8c52adf51b80962ab Mon Sep 17 00:00:00 2001 From: Erik Norvell Date: Tue, 5 Nov 2024 15:41:13 +0100 Subject: [PATCH 6/8] Fix timeout --- .gitlab-ci.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 418e5a8c5..07a888bf2 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1029,7 +1029,9 @@ coverage-test-on-main-scheduled: - make clean - make GCOV=1 -j + - testcase_timeout=$TESTCASE_TIMEOUT_LTV - exit_code_dec=0 + - exit_code_enc=0 - python3 -m pytest --tb=no $LONG_TEST_SUITE --update_ref 1 -v --create_ref --html=report-dec.html --self-contained-html --junit-xml=report-junit-dec.xml -n auto --testcase_timeout $testcase_timeout --ref_encoder_path $REF_ENCODER_PATH --ref_decoder_path $DUT_DECODER_PATH || exit_code_dec=$? - lcov -c -d obj -o coverage_dec_rend.info # extract coverage of decoder/renderer -- GitLab From a911abb127935591e97d92d676eeb5735aab33f8 Mon Sep 17 00:00:00 2001 From: Erik Norvell Date: Tue, 5 Nov 2024 19:34:11 +0100 Subject: [PATCH 7/8] Switch to STV, LTV takes a long time --- .gitlab-ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 07a888bf2..de5e63dc2 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1032,10 +1032,10 @@ coverage-test-on-main-scheduled: - testcase_timeout=$TESTCASE_TIMEOUT_LTV - exit_code_dec=0 - exit_code_enc=0 - - python3 -m pytest --tb=no $LONG_TEST_SUITE --update_ref 1 -v --create_ref --html=report-dec.html --self-contained-html --junit-xml=report-junit-dec.xml -n auto --testcase_timeout $testcase_timeout --ref_encoder_path $REF_ENCODER_PATH --ref_decoder_path $DUT_DECODER_PATH || exit_code_dec=$? + - python3 -m pytest --tb=no $SHORT_TEST_SUITE --update_ref 1 -v --create_ref --html=report-dec.html --self-contained-html --junit-xml=report-junit-dec.xml -n auto --testcase_timeout $testcase_timeout --ref_encoder_path $REF_ENCODER_PATH --ref_decoder_path $DUT_DECODER_PATH || exit_code_dec=$? - lcov -c -d obj -o coverage_dec_rend.info # extract coverage of decoder/renderer - - python3 -m pytest --tb=no $LONG_TEST_SUITE_NO_RENDERER --encoder_only -v --html=report-enc.html --self-contained-html --junit-xml=report-junit-enc.xml -n auto --testcase_timeout $testcase_timeout --dut_encoder_path $DUT_ENCODER_PATH || exit_code_enc=$? + - python3 -m pytest --tb=no $SHORT_TEST_SUITE --encoder_only -v --html=report-enc.html --self-contained-html --junit-xml=report-junit-enc.xml -n auto --testcase_timeout $testcase_timeout --dut_encoder_path $DUT_ENCODER_PATH || exit_code_enc=$? - lcov -c -d obj -o coverage_enc_dec_rend.info # extract coverage of encoder/decoder/renderer # remove apps and lib_util files from coverage -- GitLab From 137e11f019f606946cc90b7bed90ddbb65749d8b Mon Sep 17 00:00:00 2001 From: Erik Norvell Date: Wed, 6 Nov 2024 15:16:23 +0100 Subject: [PATCH 8/8] Add renderer to coverage --- .gitlab-ci.yml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index de5e63dc2..d71eef6c0 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1032,10 +1032,10 @@ coverage-test-on-main-scheduled: - testcase_timeout=$TESTCASE_TIMEOUT_LTV - exit_code_dec=0 - exit_code_enc=0 - - python3 -m pytest --tb=no $SHORT_TEST_SUITE --update_ref 1 -v --create_ref --html=report-dec.html --self-contained-html --junit-xml=report-junit-dec.xml -n auto --testcase_timeout $testcase_timeout --ref_encoder_path $REF_ENCODER_PATH --ref_decoder_path $DUT_DECODER_PATH || exit_code_dec=$? + - python3 -m pytest --tb=no tests/codec_be_on_mr_nonselection tests/renderer --update_ref 1 -v --create_ref --html=report-dec.html --self-contained-html --junit-xml=report-junit-dec.xml -n auto --testcase_timeout $testcase_timeout --ref_encoder_path $REF_ENCODER_PATH --ref_decoder_path $DUT_DECODER_PATH || exit_code_dec=$? - lcov -c -d obj -o coverage_dec_rend.info # extract coverage of decoder/renderer - - python3 -m pytest --tb=no $SHORT_TEST_SUITE --encoder_only -v --html=report-enc.html --self-contained-html --junit-xml=report-junit-enc.xml -n auto --testcase_timeout $testcase_timeout --dut_encoder_path $DUT_ENCODER_PATH || exit_code_enc=$? + - python3 -m pytest --tb=no tests/codec_be_on_mr_nonselection --encoder_only -v --html=report-enc.html --self-contained-html --junit-xml=report-junit-enc.xml -n auto --testcase_timeout $testcase_timeout --dut_encoder_path $DUT_ENCODER_PATH || exit_code_enc=$? - lcov -c -d obj -o coverage_enc_dec_rend.info # extract coverage of encoder/decoder/renderer # remove apps and lib_util files from coverage @@ -1056,6 +1056,8 @@ coverage-test-on-main-scheduled: - coverage_dec_rend.info - coverage_enc_dec_rend - coverage_dec_rend + - report-dec.html + - report-enc.html expose_as: "Coverage result" reports: junit: -- GitLab