diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 9c53be7cae9ab555b9d164321a198b3907fc95d2..0b735a6df44bddb0773b35bf19b1211b0cacf3aa 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -11,43 +11,46 @@ include: # This sets when pipelines are created. Jobs have more specific rules to restrict them. workflow: - name: '$IVAS_PIPELINE_NAME' + name: "$IVAS_PIPELINE_NAME" rules: # see https://docs.gitlab.com/ee/ci/yaml/workflow.html#switch-between-branch-pipelines-and-merge-request-pipelines - if: $CI_COMMIT_BRANCH && $CI_OPEN_MERGE_REQUESTS && $CI_PIPELINE_SOURCE == "push" when: never - if: $CI_PIPELINE_SOURCE == 'merge_request_event' variables: - IVAS_PIPELINE_NAME: 'MR pipeline: $CI_MERGE_REQUEST_SOURCE_BRANCH_NAME' + IVAS_PIPELINE_NAME: "MR pipeline: $CI_MERGE_REQUEST_SOURCE_BRANCH_NAME" - if: $CI_PIPELINE_SOURCE == 'push' && $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH # Pushes to main variables: - IVAS_PIPELINE_NAME: 'Push pipeline: $CI_COMMIT_BRANCH' + IVAS_PIPELINE_NAME: "Push pipeline: $CI_COMMIT_BRANCH" - if: $CI_PIPELINE_SOURCE == 'schedule' && $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH # Scheduled in main variables: - IVAS_PIPELINE_NAME: 'Scheduled pipeline: $CI_COMMIT_BRANCH' + IVAS_PIPELINE_NAME: "Scheduled pipeline: $CI_COMMIT_BRANCH" - if: $CI_PIPELINE_SOURCE == 'web' && $MANUAL_PIPELINE_TYPE == 'default' # for testing variables: - IVAS_PIPELINE_NAME: 'Web run pipeline: $CI_COMMIT_BRANCH' + IVAS_PIPELINE_NAME: "Web run pipeline: $CI_COMMIT_BRANCH" - if: $CI_PIPELINE_SOURCE == 'web' && $MANUAL_PIPELINE_TYPE == 'test-be-release' variables: - IVAS_PIPELINE_NAME: 'Test BE to release pipeline: $CI_COMMIT_BRANCH' + IVAS_PIPELINE_NAME: "Test BE to release pipeline: $CI_COMMIT_BRANCH" - if: $CI_PIPELINE_SOURCE == 'web' && $MANUAL_PIPELINE_TYPE == 'test-long-self-test' variables: - IVAS_PIPELINE_NAME: 'Test long self-test against main pipeline: $CI_COMMIT_BRANCH' + IVAS_PIPELINE_NAME: "Test long self-test against main pipeline: $CI_COMMIT_BRANCH" - if: $CI_PIPELINE_SOURCE == 'web' && $MANUAL_PIPELINE_TYPE == 'ivas-conformance' variables: - IVAS_PIPELINE_NAME: 'Draft IVAS Conformance test: $CI_COMMIT_BRANCH' + IVAS_PIPELINE_NAME: "Draft IVAS Conformance test: $CI_COMMIT_BRANCH" - if: $CI_PIPELINE_SOURCE == 'trigger' - if: $CI_PIPELINE_SOURCE == 'web' && $MANUAL_PIPELINE_TYPE == 'ivas-conformance-linux' variables: - IVAS_PIPELINE_NAME: 'Draft IVAS Conformance test -- Linux: $CI_COMMIT_BRANCH' + IVAS_PIPELINE_NAME: "Draft IVAS Conformance test -- Linux: $CI_COMMIT_BRANCH" - if: $CI_PIPELINE_SOURCE == 'trigger' - if: $CI_PIPELINE_SOURCE == 'web' && $MANUAL_PIPELINE_TYPE == 'check-clipping' variables: - IVAS_PIPELINE_NAME: 'Check core input clipping: $CI_COMMIT_BRANCH' + IVAS_PIPELINE_NAME: "Check core input clipping: $CI_COMMIT_BRANCH" - if: $CI_PIPELINE_SOURCE == 'web' && $MANUAL_PIPELINE_TYPE == 'test-branch-vs-input-passthrough' variables: - IVAS_PIPELINE_NAME: 'Pass-through comparison vs input: $CI_COMMIT_BRANCH' + IVAS_PIPELINE_NAME: "Pass-through comparison vs input: $CI_COMMIT_BRANCH" + - if: $MANUAL_PIPELINE_TYPE == 'coverage' + variables: + IVAS_PIPELINE_NAME: "Coverage measurement" # --------------------------------------------------------------- # Generic script anchors @@ -125,9 +128,9 @@ workflow: - Pop-Location .copy-ltv-files-to-testv-dir-win: ©-ltv-files-to-testv-dir-win - - cp $LTV_DIR_WIN\*.wav scripts\testv - - cp $LTV_DIR_WIN\*.met scripts\testv - - cp $LTV_DIR_WIN\*.csv scripts\testv + - cp $LTV_DIR_WIN\*.wav scripts\testv + - cp $LTV_DIR_WIN\*.met scripts\testv + - cp $LTV_DIR_WIN\*.csv scripts\testv .activate-WX-windows: &activate-WX-windows - (Get-Content -Path "CMakeLists.txt") -replace '# \(add_compile_options\("\/WX"\)\)', '$1' | Set-Content -Path "CMakeLists.txt" @@ -160,7 +163,6 @@ workflow: - if [ $exit_code20 -ne 0 ] || [ $exit_code10 -ne 0 ] || [ $exit_code5 -ne 0 ]; then exit 1; fi - # --------------------------------------------------------------- # .pre jobs for setting up things # --------------------------------------------------------------- @@ -359,7 +361,6 @@ build-codec-instrumented-linux: paths: - wmc_tool_output.txt - # make sure that the codec builds with msan, asan and usan build-codec-sanitizers-linux: extends: @@ -402,7 +403,13 @@ codec-smoke-test: tags: - ivas-linux-fast stage: test - needs: ["build-codec-linux-cmake", "build-codec-linux-make", "build-codec-instrumented-linux", "build-codec-sanitizers-linux"] + needs: + [ + "build-codec-linux-cmake", + "build-codec-linux-make", + "build-codec-instrumented-linux", + "build-codec-sanitizers-linux", + ] script: - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/print-common-info.sh # LTV update needed as ltv ISM metadata files are used @@ -481,7 +488,13 @@ pytest-compare-20ms-and-5ms-rendering: - .test-job-linux - .rules-merge-request-to-main stage: test - needs: ["build-codec-linux-cmake", "build-codec-linux-make", "build-codec-instrumented-linux", "build-codec-sanitizers-linux"] + needs: + [ + "build-codec-linux-cmake", + "build-codec-linux-make", + "build-codec-instrumented-linux", + "build-codec-sanitizers-linux", + ] script: - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/print-common-info.sh - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/disable-limiter.sh @@ -577,7 +590,6 @@ renderer-asan: artifacts: expose_as: "renderer asan result" - # test renderer executable with cmake + msan renderer-msan: extends: @@ -589,7 +601,6 @@ renderer-msan: artifacts: expose_as: "renderer msan result" - # test renderer executable with cmake + usan renderer-usan: extends: @@ -601,7 +612,6 @@ renderer-usan: artifacts: expose_as: "renderer usan result" - # compare renderer bitexactness between target and source branch renderer-pytest-on-merge-request: extends: @@ -840,7 +850,6 @@ ivas-pytest-on-merge-request: junit: - report-junit.xml - # Check interop IVAS_cod_test -> IVAS_dec_ref ivas-interop-on-merge-request: extends: @@ -1087,7 +1096,6 @@ check-bitexactness-ext-and-transport-format: expose_as: "logs-ext-sanity-check" expire_in: "5 days" - # --------------------------------------------------------------- # Test jobs for main branch # --------------------------------------------------------------- @@ -1226,7 +1234,6 @@ be-2-evs-windows: # reports: # junit: report-junit.xml - # --------------------------------------------------------------- # Manual jobs # --------------------------------------------------------------- @@ -1240,7 +1247,6 @@ test-be-to-release: rules: - if: $CI_PIPELINE_SOURCE == 'web' && $MANUAL_PIPELINE_TYPE == 'test-be-release' script: - - echo "$CI_COMMIT_BRANCH" - MSBuild.exe -maxcpucount .\Workspace_msvc\Workspace_msvc.sln /property:Configuration=Debug /p:Platform=win32 /m | tee -variable winoutdata @@ -1257,7 +1263,6 @@ test-be-to-release: - logs/ expose_as: "test-be-to-release results" - ivas-conformance: tags: - ivas-windows @@ -1270,13 +1275,13 @@ ivas-conformance: exit_codes: - 123 script: - - *print-common-info-windows - - MSBuild.exe -maxcpucount .\Workspace_msvc\Workspace_msvc.sln /property:Configuration=Debug + - *print-common-info-windows + - MSBuild.exe -maxcpucount .\Workspace_msvc\Workspace_msvc.sln /property:Configuration=Debug - cp -force IVAS_cod.exe IVAS_cod_ref.exe - cp -force IVAS_dec.exe IVAS_dec_ref.exe - - cp -force IVAS_rend.exe IVAS_rend_ref.exe + - cp -force IVAS_rend.exe IVAS_rend_ref.exe - cp -force ISAR_post_rend.exe ISAR_post_rend_ref.exe - + # Reference creation - python scripts/prepare_combined_format_inputs.py - $TEST_SET = "tests/codec_be_on_mr_nonselection", "tests/renderer/test_renderer.py", "tests/split_rendering/test_split_rendering.py" @@ -1285,21 +1290,21 @@ ivas-conformance: # Output creation - python -m pytest $TEST_SET -v -n auto --keep_files --create_cut --html=report_cmd.html --self-contained-html - python scripts/parse_commands.py report_cmd.html Readme_IVAS.txt - + # Copy input data and output ref data - if (Test-Path testvec) {rm -r -force testvec} - if (Test-Path TMP_DEC) {rm -r -force TMP_DEC} - - if (Test-Path TMP_ENC) {rm -r -force TMP_ENC} + - if (Test-Path TMP_ENC) {rm -r -force TMP_ENC} - if (Test-Path TMP_JBM) {rm -r -force TMP_JBM} - if (Test-Path TMP_REND) {rm -r -force TMP_REND} - - if (Test-Path TMP_ISAR_POST_REND) {rm -r -force TMP_ISAR_POST_REND} - - if (Test-Path TMP_DEC_ISAR) {rm -r -force TMP_DEC_ISAR} + - if (Test-Path TMP_ISAR_POST_REND) {rm -r -force TMP_ISAR_POST_REND} + - if (Test-Path TMP_DEC_ISAR) {rm -r -force TMP_DEC_ISAR} - mkdir testvec - mkdir testvec/binauralRenderer_interface - mkdir testvec/testv - mkdir testvec/testv/renderer - - mkdir testvec/testv/split_rendering - - mkdir testvec/bin + - mkdir testvec/testv/split_rendering + - mkdir testvec/bin - cp -force -ErrorAction Ignore scripts/testv/* testvec/testv - cp -r -force -ErrorAction Ignore scripts/ls_layouts testvec - cp -r -force -ErrorAction Ignore scripts/object_edit testvec @@ -1309,29 +1314,28 @@ ivas-conformance: - cp -r -force -ErrorAction Ignore tests/ref testvec/testv/ref - cp -r -force -ErrorAction Ignore tests/dut/* testvec/testv/ref - cp -r -force -ErrorAction Ignore tests/renderer/cut testvec/testv/renderer/ref - - cp -r -force -ErrorAction Ignore tests/split_rendering/cut testvec/testv/split_rendering/ref - - cp -r -force -ErrorAction Ignore tests/split_rendering/renderer_configs testvec/testv/split_rendering/renderer_configs - - cp -r -force -ErrorAction Ignore tests/split_rendering/error_patterns testvec/testv/split_rendering/error_patterns - - # Remove redundant files + - cp -r -force -ErrorAction Ignore tests/split_rendering/cut testvec/testv/split_rendering/ref + - cp -r -force -ErrorAction Ignore tests/split_rendering/renderer_configs testvec/testv/split_rendering/renderer_configs + - cp -r -force -ErrorAction Ignore tests/split_rendering/error_patterns testvec/testv/split_rendering/error_patterns + + # Remove redundant files - python scripts/cleanup_26252.py # Copy test script files - cp -r -force -ErrorAction Ignore tests/conformance-test testvec/ - cp Readme_IVAS_dec.txt testvec - - cp Readme_IVAS_enc.txt testvec + - cp Readme_IVAS_enc.txt testvec - cp Readme_IVAS_rend.txt testvec - cp Readme_IVAS_JBM_dec.txt testvec - - cp Readme_IVAS_ISAR_dec.txt testvec - - cp Readme_IVAS_ISAR_post_rend.txt testvec + - cp Readme_IVAS_ISAR_dec.txt testvec + - cp Readme_IVAS_ISAR_post_rend.txt testvec - cp IVAS_cod.exe testvec/bin - cp IVAS_dec.exe testvec/bin - cp IVAS_rend.exe testvec/bin - - cp ISAR_post_rend.exe testvec/bin - - + - cp ISAR_post_rend.exe testvec/bin + # Test run generated scripts in testvec - - cd testvec + - cd testvec - python -m pytest conformance-test/test_26252.py --junit-xml=report-junit.xml --html=report.html --self-contained-html - mv report.html .. - mv report-junit.xml .. @@ -1344,12 +1348,12 @@ ivas-conformance: - report_cmd.html - report-junit.xml - report.html - - Readme_IVAS_dec.txt - - Readme_IVAS_enc.txt + - Readme_IVAS_dec.txt + - Readme_IVAS_enc.txt - Readme_IVAS_rend.txt - Readme_IVAS_JBM_dec.txt - - Readme_IVAS_ISAR_dec.txt - - Readme_IVAS_ISAR_post_rend.txt + - Readme_IVAS_ISAR_dec.txt + - Readme_IVAS_ISAR_post_rend.txt expose_as: "Draft IVAS conformance" reports: junit: report-junit.xml @@ -1361,9 +1365,15 @@ ivas-conformance-linux: timeout: "90 minutes" rules: - if: ($CI_PIPELINE_SOURCE == 'web' || $CI_PIPELINE_SOURCE == 'trigger') && $MANUAL_PIPELINE_TYPE == 'ivas-conformance-linux' + - if: $MANUAL_PIPELINE_TYPE == "coverage" + - if: $COVERAGE_TEST allow_failure: exit_codes: - 123 + variables: + COVERAGE_OUTPUT_FILE: "$COVERAGE_OUTPUT_FILE_CONFORMANCE" + COVERAGE_OUTPUT_DIR: "coverage_conformance" + COVERAGE_TITLE: "conformance test test_26252.py" script: - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/print-common-info.sh - make -j @@ -1371,24 +1381,24 @@ ivas-conformance-linux: - cp IVAS_dec IVAS_dec_ref - cp IVAS_rend IVAS_rend_ref - cp ISAR_post_rend ISAR_post_rend_ref - + # Reference creation - python3 scripts/prepare_combined_format_inputs.py - TEST_SET="tests/codec_be_on_mr_nonselection tests/renderer/test_renderer.py tests/split_rendering/test_split_rendering.py" - - python3 -m pytest $TEST_SET -v -n auto --update_ref 1 --create_ref --keep_files + - python3 -m pytest -q $TEST_SET -v -n auto --update_ref 1 --create_ref --keep_files # Output creation - - python3 -m pytest $TEST_SET -v -n auto --keep_files --create_cut --html=report_cmd.html --self-contained-html + - python3 -m pytest -q $TEST_SET -v -n auto --keep_files --create_cut --html=report_cmd.html --self-contained-html - python3 scripts/parse_commands.py report_cmd.html Readme_IVAS.txt - + # Copy input data and output ref data - rm -rf testvec - mkdir testvec - mkdir testvec/binauralRenderer_interface - mkdir testvec/testv - mkdir testvec/testv/renderer - - mkdir testvec/testv/split_rendering - - mkdir testvec/bin + - mkdir testvec/testv/split_rendering + - mkdir testvec/bin - cp -r scripts/testv/* testvec/testv - cp -r scripts/ls_layouts testvec - cp -r scripts/object_edit testvec @@ -1398,49 +1408,45 @@ ivas-conformance-linux: - cp -r tests/ref testvec/testv/ref - cp -r tests/dut/* testvec/testv/ref - cp -r tests/renderer/cut testvec/testv/renderer/ref - - cp -r tests/split_rendering/cut testvec/testv/split_rendering/ref - - cp -r tests/split_rendering/renderer_configs testvec/testv/split_rendering/renderer_configs - - cp -r tests/split_rendering/error_patterns testvec/testv/split_rendering/error_patterns - - # Remove redundant files + - cp -r tests/split_rendering/cut testvec/testv/split_rendering/ref + - cp -r tests/split_rendering/renderer_configs testvec/testv/split_rendering/renderer_configs + - cp -r tests/split_rendering/error_patterns testvec/testv/split_rendering/error_patterns + + # Remove redundant files - python3 scripts/cleanup_26252.py - + # Copy test script files - cp -r tests/conformance-test testvec/ - cp Readme_IVAS_dec.txt testvec - - cp Readme_IVAS_enc.txt testvec + - cp Readme_IVAS_enc.txt testvec - cp Readme_IVAS_rend.txt testvec - cp Readme_IVAS_JBM_dec.txt testvec - - cp Readme_IVAS_ISAR_dec.txt testvec - - cp Readme_IVAS_ISAR_post_rend.txt testvec - + - cp Readme_IVAS_ISAR_dec.txt testvec + - cp Readme_IVAS_ISAR_post_rend.txt testvec + # Create GCOV execs for coverage analysis - make clean - make GCOV=1 -j - + - cp IVAS_cod testvec/bin - cp IVAS_dec testvec/bin - cp IVAS_rend testvec/bin - - cp ISAR_post_rend testvec/bin - + - cp ISAR_post_rend testvec/bin + # Test run generated scripts in testvec - - cd testvec - - exit_code=0 - - python3 -m pytest conformance-test/test_26252.py --junit-xml=report-junit.xml --html=report.html --self-contained-html || exit_code=$? + - cd testvec + - exit_code=0 + - python3 -m pytest -q conformance-test/test_26252.py --junit-xml=report-junit.xml --html=report.html --self-contained-html || exit_code=$? - mv report.html .. - mv report-junit.xml .. # Collect coverage - cd - - - lcov -c -d obj -o coverage.info - - lcov -r coverage.info "*apps*" -o coverage.info - - lcov -r coverage.info "*lib_util*" -o coverage.info - - commit_sha=$(git rev-parse HEAD) - - genhtml coverage.info -o coverage -t "Coverage on main @ $commit_sha" + - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/collect-coverage.sh --obj-dir obj --output-file $COVERAGE_OUTPUT_FILE --output-dir $COVERAGE_OUTPUT_DIR --title "Coverage on main -- $COVERAGE_TITLE" # Check for failures - if [ $exit_code -ne 0 ]; then echo "Test failures encountered"; exit $EXIT_CODE_FAIL; fi - + artifacts: name: "ivas-conformance-linux-$CI_COMMIT_SHORT_SHA" expire_in: 1 week @@ -1449,19 +1455,18 @@ ivas-conformance-linux: - report_cmd.html - report-junit.xml - report.html - - Readme_IVAS_dec.txt - - Readme_IVAS_enc.txt + - Readme_IVAS_dec.txt + - Readme_IVAS_enc.txt - Readme_IVAS_rend.txt - Readme_IVAS_JBM_dec.txt - - Readme_IVAS_ISAR_dec.txt - - Readme_IVAS_ISAR_post_rend.txt - - coverage.info - - coverage + - Readme_IVAS_ISAR_dec.txt + - Readme_IVAS_ISAR_post_rend.txt + - $COVERAGE_OUTPUT_FILE + - $COVERAGE_OUTPUT_DIR expose_as: "Draft IVAS conformance -- Linux" reports: junit: report-junit.xml - test-long-self-test: extends: - .job-linux @@ -1519,7 +1524,6 @@ test-long-self-test: - if [ $exit_code -ne 0 ]; then echo "Non-bitexact cases encountered!"; exit $EXIT_CODE_NON_BE; fi - exit 0 - artifacts: name: "test-long-self-test--sha-$CI_COMMIT_SHORT_SHA--results" when: always @@ -1968,7 +1972,6 @@ sanitizer-test-omasa-ism4: - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/update-ltv-repo.sh - python3 ci/run_scheduled_sanitizer_test.py MASA-ISM4 $OUT_FORMATS_ALL --tests $SANITIZER_TESTS - ### --- sanitizer schedule C --- .sanitizer-test-schedule-C: @@ -2181,10 +2184,8 @@ sanitizer-test-osba-planar-hoa3-ism4: - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/update-ltv-repo.sh - python3 ci/run_scheduled_sanitizer_test.py PlanarHOA3-ISM4 $OUT_FORMATS_ALL --tests $SANITIZER_TESTS - - # GCOV/LCOV coverage analysis of self_test suite -coverage-test-on-main-scheduled: +.coverage-test-template: extends: - .test-job-linux-needs-testv-dir - .rules-main-scheduled @@ -2192,53 +2193,133 @@ coverage-test-on-main-scheduled: - ivas-linux-fast stage: test rules: - # only run in scheduled pipeline that passes this env vars - if: $COVERAGE_TEST + - if: $MANUAL_PIPELINE_TYPE == "coverage" timeout: 3 hours - script: + before_script: + - !reference [.job-linux, before_script] + - set -e + - 'trap ''echo "Command failed at line $LINENO: $BASH_COMMAND"'' ERR' - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/print-common-info.sh - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/update-ltv-repo.sh - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/copy-ltv-files-to-testv-dir.sh - - make GCOV=1 -j - - cp IVAS_rend IVAS_rend_ref # Copy exec to be able to run renderer script + # compile with coverage enabled + - cmake -B cmake-build -G "Unix Makefiles" -DCOPY_EXECUTABLES_FROM_BUILD_DIR=true -DGCOV=ON + - cmake --build cmake-build -- -j + # copy executables to ref to be able to run some --create_ref tests + - cp IVAS_cod IVAS_cod_ref + - cp IVAS_dec IVAS_dec_ref + - cp IVAS_rend IVAS_rend_ref + - cp ISAR_post_rend ISAR_post_rend_ref + after_script: + - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/collect-coverage.sh --obj-dir cmake-build --output-file $COVERAGE_OUTPUT_FILE --output-dir $COVERAGE_OUTPUT_DIR --title "Coverage on main -- $COVERAGE_TITLE" + artifacts: + name: "main-$COVERAGE_OUTPUT_DIR-sha-$CI_COMMIT_SHORT_SHA" + when: always + expire_in: 1 week - # -- Run short test vector suite to check coverage of that first - - python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR -v -n auto --update_ref 1 --ref_encoder_path ./IVAS_cod --ref_decoder_path ./IVAS_dec +coverage-merge: + extends: .rules-main-scheduled + tags: + - ivas-linux-fast + stage: test + rules: + - if: $COVERAGE_TEST + - if: $MANUAL_PIPELINE_TYPE == "coverage" + needs: + - job: coverage-test-on-main-scheduled-stv + artifacts: true + - job: coverage-test-on-main-scheduled-ltv + artifacts: true + - job: ivas-conformance-linux + artifacts: true + variables: + COVERAGE_OUTPUT_FILE: "$COVERAGE_OUTPUT_FILE_MERGED" + COVERAGE_OUTPUT_DIR: "coverage-merged" + COVERAGE_TITLE: "merged total coverage" + script: + - echo "Merging coverage reports..." + - lcov -a $COVERAGE_OUTPUT_FILE_CONFORMANCE -a $COVERAGE_OUTPUT_FILE_STV -a $COVERAGE_OUTPUT_FILE_LTV -o "$COVERAGE_OUTPUT_FILE_MERGED" + - commit_sha=$(git rev-parse HEAD) + - genhtml $COVERAGE_OUTPUT_FILE_MERGED -o $COVERAGE_OUTPUT_DIR -t "Coverage on main -- $COVERAGE_TITLE @ $commit_sha" + artifacts: + name: "merged-coverage-sha-$CI_COMMIT_SHORT_SHA" + when: always + expire_in: 1 week + paths: + - $COVERAGE_OUTPUT_FILE + - $COVERAGE_OUTPUT_DIR +coverage-test-on-main-scheduled-stv: + extends: .coverage-test-template + variables: + COVERAGE_OUTPUT_FILE: "$COVERAGE_OUTPUT_FILE_STV" + COVERAGE_OUTPUT_DIR: "coverage_stv" + COVERAGE_TITLE: "short test vectors" + script: + # -- Run short test vector suite to check coverage of that first + # codec smoke test + - bash ci/smoke_test.sh coverage + # default renderer framesize is 20ms + - python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR -v -n auto -q --tb=no --update_ref 1 --ref_encoder_path ./IVAS_cod --ref_decoder_path ./IVAS_dec + # for different renderer framesizes, only the decoder needs to be tested, so use --decoder_only # need to ignore non-zero exit codes as limiter is active and thus the different framesiszes will not be BE in all cases - - python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR -v --dut_encoder_path ./IVAS_cod --dut_decoder_path ./IVAS_dec --dut_fr 5 --decoder_only || true - - python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR -v --dut_encoder_path ./IVAS_cod --dut_decoder_path ./IVAS_dec --dut_fr 10 --decoder_only || true - - python3 -m pytest -q -n auto tests/renderer/test_renderer.py --create_ref - - python3 -m pytest -q -n auto tests/renderer/test_renderer.py --create_cut - - lcov -c -d obj -o coverage_stv.info # extract coverage of short test vectors here - # remove apps and lib_util files from coverage - - lcov -r coverage_stv.info "*apps*" -o coverage_stv.info - - lcov -r coverage_stv.info "*lib_util*" -o coverage_stv.info + - python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR -v -n auto -q --tb=no --dut_encoder_path ./IVAS_cod --dut_decoder_path ./IVAS_dec --dut_fr 5 --decoder_only || true + - python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR -v -n auto -q --tb=no --dut_encoder_path ./IVAS_cod --dut_decoder_path ./IVAS_dec --dut_fr 10 --decoder_only || true + # run renderer smoke test - needed for REFVEC tests + - python3 -m pytest -n auto -q --tb=no tests/renderer/test_renderer.py + # run renderer test only with REF creation stage - no CUT needed since executables are identical + - python3 -m pytest -n auto -q --tb=no tests/renderer/test_renderer.py --create_ref + # run split rendering smoke test and REF creation + - python3 -m pytest -n auto -q --tb=no tests/split_rendering/test_split_rendering.py + - python3 -m pytest -n auto -q --tb=no tests/split_rendering/test_split_rendering.py --create_ref + # VoIP BE test + - python3 -m pytest -q --tb=no tests/test_be_for_jbm_neutral_dly_profile.py + # split rendering VoIP BE test + - python3 -m pytest -n auto -q --tb=no tests/split_rendering/test_voip_be_splitrend_vs_binaural.py + # TODO first frame SID - this script runs a `make clean` and needs to be checked/adapted + # - bash ci/run-first-frame-is-sid-test.sh + # TODO consider HRTF binary loading test, or probably already covered by PRM files + # - python3 -m pytest -q --tb=no tests/hrtf_binary_loading + # run BE vs EXT test + - python3 tests/create_short_testvectors.py --cut_len 1.0 + - python3 -m pytest -q --tb=no tests/test_be_for_ext_outputs.py + # run LC3plus wrapper unit test + - scripts/split_rendering/lc3plus_float/ivas_lc3plus_unit_test + # TODO consider including check-clipping, but this scales ALL testv files. could be added at end of this job + artifacts: + paths: + - $COVERAGE_OUTPUT_FILE + - $COVERAGE_OUTPUT_DIR +coverage-test-on-main-scheduled-ltv: + extends: .coverage-test-template + variables: + COVERAGE_OUTPUT_FILE: "$COVERAGE_OUTPUT_FILE_LTV" + COVERAGE_OUTPUT_DIR: "coverage_ltv" + COVERAGE_TITLE: "long test vectors" + script: # -- Add extended coverage of EVS test and long test vectors - - python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR/test_param_file.py -v -n auto --update_ref 1 --param_file scripts/config/self_test_evs.prm --ref_encoder_path ./IVAS_cod --ref_decoder_path ./IVAS_dec - - bash ci/smoke_test.sh coverage - - python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR/test_param_file.py -v -n auto --update_ref 1 --param_file scripts/config/self_test_ltv.prm --ref_encoder_path ./IVAS_cod --ref_decoder_path ./IVAS_dec - - python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR/test_sba.py -v -n auto --update_ref 1 --ref_encoder_path ./IVAS_cod --ref_decoder_path ./IVAS_dec --use_ltv - - python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR/test_sba.py -v --dut_encoder_path ./IVAS_cod --dut_decoder_path ./IVAS_dec --dut_fr 5 --decoder_only --use_ltv || true - - python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR/test_sba.py -v --dut_encoder_path ./IVAS_cod --dut_decoder_path ./IVAS_dec --dut_fr 10 --decoder_only --use_ltv || true - - python3 -m pytest tests/test_be_for_jbm_neutral_dly_profile.py - - lcov -c -d obj -o coverage.info - # remove apps and lib_util files from coverage - - lcov -r coverage.info "*apps*" -o coverage.info - - lcov -r coverage.info "*lib_util*" -o coverage.info - - commit_sha=$(git rev-parse HEAD) - - genhtml coverage.info -o coverage -t "Coverage on main @ $commit_sha" - - genhtml coverage_stv.info -o coverage_stv -t "Coverage on main -- short test vectors @ $commit_sha" + - python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR/test_param_file.py -v -n auto -q --tb=no --update_ref 1 --param_file scripts/config/self_test_evs.prm --ref_encoder_path ./IVAS_cod --ref_decoder_path ./IVAS_dec + # default renderer framesize is 20ms + - python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR/test_param_file.py -v -n auto -q --tb=no --update_ref 1 --param_file scripts/config/self_test_ltv.prm --ref_encoder_path ./IVAS_cod --ref_decoder_path ./IVAS_dec + # for different renderer framesizes, only the decoder needs to be tested, so use --decoder_only + # need to ignore non-zero exit codes as limiter is active and thus the different framesiszes will not be BE in all cases + - python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR/test_param_file.py -v -n auto -q --tb=no --param_file scripts/config/self_test_ltv.prm --dut_decoder_path ./IVAS_dec --decoder_only --dut_fr 5 || true + - python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR/test_param_file.py -v -n auto -q --tb=no --param_file scripts/config/self_test_ltv.prm --dut_decoder_path ./IVAS_dec --decoder_only --dut_fr 10 || true + - python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR/test_sba.py -v -n auto -q --tb=no --update_ref 1 --ref_encoder_path ./IVAS_cod --ref_decoder_path ./IVAS_dec --use_ltv + - python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR/test_sba.py -v -n auto -q --tb=no --dut_encoder_path ./IVAS_cod --dut_decoder_path ./IVAS_dec --dut_fr 5 --decoder_only --use_ltv || true + - python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR/test_sba.py -v -n auto -q --tb=no --dut_encoder_path ./IVAS_cod --dut_decoder_path ./IVAS_dec --dut_fr 10 --decoder_only --use_ltv || true + # run renderer smoke test - needed for REFVEC tests + - ls -altr scripts/testv + # LTV smoketest is the same as REF creation stage for renderer + - python3 -m pytest -n auto -q --tb=no tests/renderer/test_renderer.py --use_ltv + # split rendering test doesn't support LTV - skipped here + # VoIP BE tests don't support LTV - skipped here artifacts: - name: "main-coverage-sha-$CI_COMMIT_SHORT_SHA" - when: always - expire_in: 1 week paths: - - coverage.info - - coverage_stv.info - - coverage - - coverage_stv + - $COVERAGE_OUTPUT_FILE + - $COVERAGE_OUTPUT_DIR # --------------------------------------------------------------- # Complexity measurement jobs diff --git a/.gitlab-ci/variables.yml b/.gitlab-ci/variables.yml index 9e60194a6c7cbe29dba20cbbcc6de430961cb571..b70de66596b69cf9aef46b56d71ec8f392b0fe28 100644 --- a/.gitlab-ci/variables.yml +++ b/.gitlab-ci/variables.yml @@ -18,6 +18,7 @@ variables: - 'ivas-conformance-linux' - 'check-clipping' - 'test-branch-vs-input-passthrough' + - 'coverage' PYTEST_ARGS: "" LONG_TEST_SUITE: "tests/codec_be_on_mr_nonselection tests/renderer --param_file scripts/config/self_test_ltv.prm --use_ltv" @@ -28,3 +29,7 @@ variables: TEST_SUITE: "" # note: currently overwrites default value from ci repo TESTCASE_TIMEOUT_STV_SANITIZERS: 240 + COVERAGE_OUTPUT_FILE_STV: "coverage-stv.info" + COVERAGE_OUTPUT_FILE_LTV: "coverage-ltv.info" + COVERAGE_OUTPUT_FILE_CONFORMANCE: "coverage-conformance.info" + COVERAGE_OUTPUT_FILE_MERGED: "coverage-merged.info" diff --git a/ci/index-pages.html b/ci/index-pages.html index 510e2031155ac1b2a2296b1066d29c37cb1344f7..390f06631d57664149ea2149ad62860cc859058a 100644 --- a/ci/index-pages.html +++ b/ci/index-pages.html @@ -11,9 +11,6 @@

Test Coverage

- + {} diff --git a/ci/setup_pages.py b/ci/setup_pages.py index df350d856890d0a1e0b2f4ff69f676e2b7bea442..1a1c1ea7352a07d907aa838a63b777884d29d333 100755 --- a/ci/setup_pages.py +++ b/ci/setup_pages.py @@ -40,7 +40,10 @@ JOBS_FLOAT_REPO = { "complexity-osba-in-binaural_room_ir-out": "OSBA in, BINAURAL_ROOM_IR out", "complexity-StereoDmxEVS-stereo-in-mono-out": "StereoDmxEVS, Stereo in, Mono out", # "timeless" jobs (not complexity) - "coverage-test-on-main-scheduled": "Coverage", + "coverage-test-on-main-scheduled-stv": "Short test vector coverage (~ merge request pipelines)", + "coverage-test-on-main-scheduled-ltv": "Long test vector coverage (for test_param_file.py, test_sba.py and test_renderer.py)", + "ivas-conformance-linux": "Conformance test coverage", + "coverage-merge": "Merged coverage (all of the above combined)", } JOBS_BASOP_REPO = { "ivas-pytest-compare_ref-long-dec": "Pytest decoder compare to ref LTV", @@ -77,6 +80,13 @@ JOBS_FOR_PROJECT_ID = { PROJECT_ID_BASOP_REPO: JOBS_BASOP_REPO, } +ARTIFACT_FOLDER_4_COVERAGE_JOBS = { + "coverage-test-on-main-scheduled-stv": "coverage_stv", + "coverage-test-on-main-scheduled-ltv": "coverage_ltv", + "ivas-conformance-linux": "coverage_conformance", + "coverage-merge": "coverage-merged", +} + ARTIFACTS = "artifacts.zip" API_URL_BASE = "https://forge.3gpp.org/rep/api/v4/projects/{}/jobs" PUBLIC_FOLDER = pathlib.Path("./public").absolute() @@ -104,15 +114,20 @@ def main(): def create_landing_page(jobs, index_html, project_id): # dynamically create the complexity links on the landing page - link_html = list() - link_html = ["") if project_id == PROJECT_ID_FLOAT_REPO: index_template = "index-pages.html" @@ -126,14 +141,18 @@ def create_landing_page(jobs, index_html, project_id): with open(index_pages_tmpl_path) as f: index_pages_tmpl = f.read() - index_pages_tmpl = index_pages_tmpl.format(link_html_text) + link_html_complexity_text = "\n".join(link_html_complexity) + link_html_coverage_text = "\n".join(link_html_coverage) + index_pages_tmpl = index_pages_tmpl.format( + link_html_complexity_text, link_html_coverage_text + ) with open(index_html, "w") as f: f.write(index_pages_tmpl) def get_artifacts_for_jobs_and_return_num_failed( - jobs: list, project_id: int, success_only: bool + jobs: dict, project_id: int, success_only: bool ) -> int: """ Get specified artifact folders for all jobs given and put them into the public folder. diff --git a/tests/renderer/constants.py b/tests/renderer/constants.py index 85affb1b3acfa05ed6628d5fc9e8a7cb5232e4e2..2d6a4daee41b8ad5187cd1d953f9ff39ae99e862 100644 --- a/tests/renderer/constants.py +++ b/tests/renderer/constants.py @@ -332,7 +332,7 @@ FORMAT_TO_METADATA_FILES_LTV = { "ISM2MASA1": [ str(LTV_DIR.joinpath("ltvISM1.csv")), str(LTV_DIR.joinpath("ltvISM2.csv")), - str(LTV_DIR.joinpath("ltv48_OMASA_2ISM_1TC.met ")), + str(LTV_DIR.joinpath("ltv48_OMASA_2ISM_1TC.met")), ], "ISM3MASA1": [ str(LTV_DIR.joinpath("ltvISM1.csv")),