diff --git a/main-basop.yml b/main-basop.yml index 003f2d50d0ca1d1a5489f277659ddef9569e2015..075f6fda7666d1c1cc51fbf12351d9216878e4e3 100644 --- a/main-basop.yml +++ b/main-basop.yml @@ -80,6 +80,9 @@ workflow: - if: $CI_PIPELINE_SOURCE == 'web' && $MANUAL_PIPELINE_TYPE == 'test-long-self-test' variables: IVAS_PIPELINE_NAME: 'Test long self-test against main pipeline: $CI_COMMIT_BRANCH' + - if: $CI_PIPELINE_SOURCE == 'web' && $MANUAL_PIPELINE_TYPE == 'ivas-conformance-linux' + variables: + IVAS_PIPELINE_NAME: 'IVAS Conformance Linux: $CI_COMMIT_BRANCH' - if: $CI_PIPELINE_SOURCE == 'schedule' # Scheduled in any branch variables: IVAS_PIPELINE_NAME: 'Scheduled pipeline: $CI_COMMIT_BRANCH' @@ -149,6 +152,8 @@ workflow: .rules-pytest-to-ref-pre: rules: + - if: $CI_PIPELINE_SOURCE == 'web' && $MANUAL_PIPELINE_TYPE == "ivas-conformance-linux" + when: never - if: $CI_PIPELINE_SOURCE == 'web' - if: $CI_PIPELINE_SOURCE == 'schedule' - if: $CI_PIPELINE_SOURCE == 'push' @@ -950,7 +955,7 @@ pytest-to-ref-pre: # ensure that codec builds on linux build-codec-linux-make: rules: - - if: $MANUAL_PIPELINE_TYPE == 'long-term-logs' || $MANUAL_PIPELINE_TYPE == 'backup-long-term-logs' || $UPDATE_PAGES + - if: $MANUAL_PIPELINE_TYPE == 'long-term-logs' || $MANUAL_PIPELINE_TYPE == 'ivas-conformance-linux' || $MANUAL_PIPELINE_TYPE == 'backup-long-term-logs' || $UPDATE_PAGES when: never - if: $CI_PIPELINE_SOURCE == 'web' - if: $CI_PIPELINE_SOURCE == 'push' && $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH @@ -970,7 +975,7 @@ build-codec-linux-cmake: rules: - if: $CI_MERGE_REQUEST_TITLE =~ /^(\[Draft\]|\(Draft\)|Draft:)/ when: never - - if: $MANUAL_PIPELINE_TYPE == 'long-term-logs' || $MANUAL_PIPELINE_TYPE == 'backup-long-term-logs' || $UPDATE_PAGES + - if: $MANUAL_PIPELINE_TYPE == 'long-term-logs' || $MANUAL_PIPELINE_TYPE == 'backup-long-term-logs' || $MANUAL_PIPELINE_TYPE == 'ivas-conformance-linux' || $UPDATE_PAGES when: never - if: $CI_PIPELINE_SOURCE == 'push' && $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH - if: $CI_PIPELINE_SOURCE == 'merge_request_event' @@ -1055,7 +1060,7 @@ build-codec-linux-debugging-make: build-codec-windows-msbuild: rules: - - if: $MANUAL_PIPELINE_TYPE == 'long-term-logs' || $MANUAL_PIPELINE_TYPE == 'backup-long-term-logs' || $UPDATE_PAGES + - if: $MANUAL_PIPELINE_TYPE == 'long-term-logs' || $MANUAL_PIPELINE_TYPE == 'backup-long-term-logs' || $MANUAL_PIPELINE_TYPE == 'ivas-conformance-linux' || $UPDATE_PAGES when: never - if: $CI_PIPELINE_SOURCE == 'web' - if: $CI_PIPELINE_SOURCE == 'push' && $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH @@ -3171,6 +3176,49 @@ complexity-osba-in-binaural_room_ir-out: - *complexity-measurements-report-summary - exit $ret_val +ivas-conformance-linux: + extends: + - .job-linux + stage: test + timeout: "6 hours" + rules: + - if: ($CI_PIPELINE_SOURCE == 'web' || $CI_PIPELINE_SOURCE == 'trigger') && $MANUAL_PIPELINE_TYPE == 'ivas-conformance-linux' + - if: $MANUAL_PIPELINE_TYPE == "coverage" + - if: $COVERAGE_TEST + allow_failure: + exit_codes: + - 123 + variables: + COVERAGE_OUTPUT_FILE: "coverage_conformance.info" + COVERAGE_OUTPUT_DIR: "coverage_conformance" + COVERAGE_TITLE: "conformance test test_26252.py" + script: + - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/print-common-info.sh + - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/basop/update-scripts-repo.sh + - python3 ci/remove_unsupported_testcases.py $PRM_FILES # Should not be needed once the porting is done and all test cases are supported in BASOP/float. + - make -j # To be substituted with reference platform build + - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/ivas-conformance.sh + + artifacts: + name: "ivas-conformance-linux-$CI_COMMIT_SHORT_SHA" + expire_in: 1 week + when: always + paths: + - report_cmd.html + - report-junit.xml + - report.html + - Readme_IVAS_dec.txt + - Readme_IVAS_enc.txt + - Readme_IVAS_rend.txt + - Readme_IVAS_JBM_dec.txt + - Readme_IVAS_ISAR_dec.txt + - Readme_IVAS_ISAR_post_rend.txt + - $COVERAGE_OUTPUT_FILE + - $COVERAGE_OUTPUT_DIR + expose_as: "Draft IVAS conformance -- Linux" + reports: + junit: report-junit.xml + # job that sets up gitlab pages website pages: stage: deploy diff --git a/snippets/ivas-conformance.sh b/snippets/ivas-conformance.sh new file mode 100644 index 0000000000000000000000000000000000000000..2a6c359d0f54f3d6864181d123712ec70de81eac --- /dev/null +++ b/snippets/ivas-conformance.sh @@ -0,0 +1,68 @@ +#! /bin/bash + +set -euxo pipefail + +cp IVAS_cod IVAS_cod_ref +cp IVAS_dec IVAS_dec_ref +cp IVAS_rend IVAS_rend_ref +cp ISAR_post_rend ISAR_post_rend_ref + +# Reference creation +python3 -m pytest -q tests/codec_be_on_mr_nonselection tests/renderer_short/test_renderer.py tests/split_rendering/test_split_rendering.py -v -n auto --update_ref 1 --create_ref --keep_files --html=report_cmd.html --self-contained-html +python3 scripts/parse_commands.py report_cmd.html Readme_IVAS.txt + +# Copy input data and output ref data +rm -rf testvec +mkdir testvec +mkdir testvec/binauralRenderer_interface +mkdir testvec/testv +mkdir testvec/testv/renderer_short +mkdir testvec/testv/split_rendering +mkdir testvec/bin +cp -r scripts/testv/* testvec/testv +cp -r scripts/ls_layouts testvec +cp -r scripts/object_edit testvec +cp -r scripts/switchPaths testvec +cp -r scripts/trajectories testvec +cp -r scripts/binauralRenderer_interface/binaural_renderers_hrtf_data testvec/binauralRenderer_interface +cp -r tests/ref testvec/testv/ref +cp -r tests/renderer_short/ref testvec/testv/renderer_short/ref +cp -r tests/split_rendering/ref testvec/testv/split_rendering/ref +cp -r tests/split_rendering/renderer_configs testvec/testv/split_rendering/renderer_configs +cp -r tests/split_rendering/error_patterns testvec/testv/split_rendering/error_patterns + +# Remove redundant files +python3 scripts/cleanup_26252.py + +# Copy test script files +cp -r tests/conformance-test testvec/ +cp Readme_IVAS_dec.txt testvec +cp Readme_IVAS_enc.txt testvec +cp Readme_IVAS_rend.txt testvec +cp Readme_IVAS_JBM_dec.txt testvec +cp Readme_IVAS_ISAR_dec.txt testvec +cp Readme_IVAS_ISAR_post_rend.txt testvec + +# Create GCOV execs for coverage analysis +make clean +make GCOV=1 -j + +cp IVAS_cod testvec/bin +cp IVAS_dec testvec/bin +cp IVAS_rend testvec/bin +cp ISAR_post_rend testvec/bin + +# Test run generated scripts in testvec +cd testvec +exit_code=0 +python3 -m pytest -q conformance-test/test_26252.py --junit-xml=report-junit.xml --html=report.html --self-contained-html || exit_code=$? +mv report.html .. +mv report-junit.xml .. + +# Collect coverage +cd - +bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/collect-coverage.sh --obj-dir obj --output-file $COVERAGE_OUTPUT_FILE --output-dir $COVERAGE_OUTPUT_DIR --title "Coverage on main -- $COVERAGE_TITLE" + +# Check for failures +if [ $exit_code -ne 0 ]; then echo "Test failures encountered"; exit $EXIT_CODE_FAIL; fi +