Loading includes/default-variables.yml +3 −0 Original line number Diff line number Diff line Loading @@ -12,3 +12,6 @@ variables: BASOP_REFERENCE_BRANCH: "ivas-float-update" SCALE_FACTOR: "3.162" BASOP_CI_BRANCH_PC_REPO: "ci/add-inplace-option-to-create-short-testvectory-script" PYTEST_ADDOPTS: "" # overwrite this via .gitlab/variables.yml in the respective other repo to disable "(model from file)" testcases DISABLE_HRTF: "false" main-basop.yml +114 −22 Original line number Diff line number Diff line Loading @@ -76,6 +76,9 @@ workflow: - if: $CI_PIPELINE_SOURCE == 'web' && $MANUAL_PIPELINE_TYPE == 'peaq-enc-passthrough' variables: IVAS_PIPELINE_NAME: 'PEAQ encoder pass-through test: $CI_COMMIT_BRANCH' - if: $CI_PIPELINE_SOURCE == 'web' && $MANUAL_PIPELINE_TYPE == 'test-long-self-test' variables: IVAS_PIPELINE_NAME: 'Test long self-test against main pipeline: $CI_COMMIT_BRANCH' - if: $CI_PIPELINE_SOURCE == 'schedule' # Scheduled in any branch variables: IVAS_PIPELINE_NAME: 'Scheduled pipeline: $CI_COMMIT_BRANCH' Loading Loading @@ -239,7 +242,7 @@ workflow: .ivas-pytest-anchor: &ivas-pytest-anchor stage: test needs: ["build-codec-linux-make"] timeout: "360 minutes" timeout: "480 minutes" variables: # keep "mld" in artifact name for backwards compatibility reasons CSV_ARTIFACT_NAME: "mld--$CI_JOB_NAME-$CI_JOB_ID--sha-$CI_COMMIT_SHORT_SHA.csv" Loading Loading @@ -374,6 +377,9 @@ workflow: before_script: - !reference [ .test-job-linux, before_script ] - rm -rf tests/dut tests/ref - if [ "$DISABLE_HRTF" = "true" ]; then - export PYTEST_ADDOPTS="${PYTEST_ADDOPTS} -k 'not model'" - fi variables: USE_LTV: 0 Loading Loading @@ -1310,6 +1316,10 @@ ivas-pytest-on-merge-request: ### If ref_using_target is not set, checkout the source branch to use scripts and input from there - if [ $ref_using_target == 0 ]; then git checkout $source_branch_commit_sha; fi - if [ "$DISABLE_HRTF" = "true" ]; then - export PYTEST_ADDOPTS="${PYTEST_ADDOPTS} -k 'not model'" - fi ### prepare pytest # create references - python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR -v --update_ref 1 Loading Loading @@ -2117,6 +2127,102 @@ ivas-pytest-usan: - TEST_SUITE=$LONG_TEST_SUITE_NO_RENDERER <<: *ivas-pytest-sanitizers-anchor # Manual long self-test (not including tests_sba.py) against BASOP main test-long-self-test: extends: - .job-linux stage: compare timeout: "360 minutes" rules: - if: $CI_PIPELINE_SOURCE == 'web' && $MANUAL_PIPELINE_TYPE == 'test-long-self-test' allow_failure: exit_codes: - 123 parallel: matrix: - LEVEL: - 0dB - minus10dB - plus10dB script: - set -euxo pipefail - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/print-common-info.sh - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/basop/update-scripts-repo.sh - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/update-ltv-repo.sh - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/copy-ltv-files-to-testv-dir.sh - testcase_timeout=$TESTCASE_TIMEOUT_LTV - case $LEVEL in "0dB") LEVEL_SCALING=1.0 ;; "minus10dB") LEVEL_SCALING=0.3162 ;; "plus10dB") LEVEL_SCALING=3.162 ;; esac - python3 ci/remove_unsupported_testcases.py $PRM_FILES - if [ $LEVEL_SCALING != "1.0" ]; then - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/basop/apply-testv-scaling.sh - fi - INV_LEVEL_SCALING=$(awk "BEGIN {print 1.0 / $LEVEL_SCALING}") - comp_args="--mld --ssnr --odg --scalefac $INV_LEVEL_SCALING" ### store the current commit hash - source_branch_commit_sha=$(git rev-parse HEAD) ### switch to main - git checkout main - git pull origin main - echo "Building reference codec at commit $(git rev-parse HEAD)" ### build main (ref) binaries - make clean - make -j - mv ./IVAS_cod ./IVAS_cod_ref - mv ./IVAS_dec ./IVAS_dec_ref ### Switch back to branch and rename binaries back - git checkout $source_branch_commit_sha - echo "Building test codec at commit $source_branch_commit_sha" ### build branch binaries - make clean - make -j ### prepare pytest # create references - exit_code_ref=0 - python3 -m pytest tests/codec_be_on_mr_nonselection/test_param_file.py -v --update_ref 1 --param_file scripts/config/self_test_ltv.prm --use_ltv --testcase_timeout=$testcase_timeout || exit_code_ref=$? ### run pytest self-test using long test vectors - exit_code=0 - python3 -m pytest tests/codec_be_on_mr_nonselection/test_param_file.py -v --param_file scripts/config/self_test_ltv.prm --use_ltv --html=report-ltv.html --self-contained-html --junit-xml=report-junit-ltv.xml $comp_args --testcase_timeout=$testcase_timeout || exit_code=$? - zero_errors=$(cat report-junit-ltv.xml | grep -c 'errors="0"') || true - if [ $zero_errors != 1 ]; then echo "Run errors encountered!"; exit $EXIT_CODE_FAIL; fi - if [ $exit_code -ne 0 ]; then echo "Non-bitexact cases encountered!"; exit $EXIT_CODE_NON_BE; fi - exit 0 artifacts: name: "$CI_JOB_NAME--sha-$CI_COMMIT_SHORT_SHA--results" when: always expire_in: 4 mos paths: - report-junit-ltv.xml - report-ltv.html expose_as: "test-long-self-test results" reports: junit: - report-junit-ltv.xml ivas-smoke-test-saturation: extends: - .rules-pytest-saturation-smoke-test Loading Loading @@ -2170,48 +2276,34 @@ coverage-test-on-main-scheduled: script: - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/print-common-info.sh - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/basop/update-scripts-repo.sh - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/update-ltv-repo.sh - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/copy-ltv-files-to-testv-dir.sh - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/basop/build-binaries.sh float-ref # Build DuT binaries with GCOV - make clean >> /dev/null - make GCOV=1 -j - cp IVAS_rend IVAS_rend_ref # Copy to ensure instrumented renderer is run in the first pytest call - cp IVAS_rend IVAS_rend_ref # Copy to ensure instrumented renderer is run in ref creation call - testcase_timeout=$TESTCASE_TIMEOUT_LTV - exit_code_dec=0 - exit_code_enc=0 - python3 -m pytest --tb=no tests/codec_be_on_mr_nonselection tests/renderer --update_ref 1 -v --create_ref --html=report-dec.html --self-contained-html --junit-xml=report-junit-dec.xml -n auto --testcase_timeout $testcase_timeout --ref_encoder_path $REF_ENCODER_PATH --ref_decoder_path IVAS_dec || exit_code_dec=$? - lcov -c -d obj -o coverage_dec_rend.info # extract coverage of decoder/renderer - python3 -m pytest --tb=no tests/codec_be_on_mr_nonselection --encoder_only -v --html=report-enc.html --self-contained-html --junit-xml=report-junit-enc.xml -n auto --testcase_timeout $testcase_timeout --dut_encoder_path IVAS_enc || exit_code_enc=$? - lcov -c -d obj -o coverage_enc_dec_rend.info # extract coverage of encoder/decoder/renderer - exit_code=0 - python3 -m pytest --tb=no tests/codec_be_on_mr_nonselection tests/renderer --update_ref 1 -v --create_ref --html=report.html --self-contained-html --junit-xml=report-junit.xml -n auto --testcase_timeout $testcase_timeout --ref_encoder_path IVAS_cod --ref_decoder_path IVAS_dec || exit_code=$? - lcov -c -d obj -o coverage_enc_dec_rend.info # extract coverage # remove apps and lib_util files from coverage - lcov -r coverage_dec_rend.info "*apps*" -o coverage_dec_rend.info - lcov -r coverage_dec_rend.info "*lib_util*" -o coverage_dec_rend.info - lcov -r coverage_enc_dec_rend.info "*apps*" -o coverage_enc_dec_rend.info - lcov -r coverage_enc_dec_rend.info "*lib_util*" -o coverage_enc_dec_rend.info - commit_sha=$(git rev-parse HEAD) - genhtml coverage_enc_dec_rend.info -o coverage_enc_dec_rend -t "Coverage on main enc/dec/rend @ $commit_sha" - genhtml coverage_dec_rend.info -o coverage_dec_rend -t "Coverage on main -- dec/rend @ $commit_sha" - genhtml coverage_enc_dec_rend.info -o coverage_enc_dec_rend -t "Coverage on main @ $commit_sha" artifacts: name: "main-coverage-sha-$CI_COMMIT_SHORT_SHA" when: always expire_in: 1 week paths: - coverage_enc_dec_rend.info - coverage_dec_rend.info - coverage_enc_dec_rend - coverage_dec_rend - report-dec.html - report-enc.html - report.html expose_as: "Coverage result" reports: junit: - report-junit-dec.xml - report-junit-enc.xml - report-junit.xml # --------------------------------------------------------------- # EVS 26.444 test job Loading Loading
includes/default-variables.yml +3 −0 Original line number Diff line number Diff line Loading @@ -12,3 +12,6 @@ variables: BASOP_REFERENCE_BRANCH: "ivas-float-update" SCALE_FACTOR: "3.162" BASOP_CI_BRANCH_PC_REPO: "ci/add-inplace-option-to-create-short-testvectory-script" PYTEST_ADDOPTS: "" # overwrite this via .gitlab/variables.yml in the respective other repo to disable "(model from file)" testcases DISABLE_HRTF: "false"
main-basop.yml +114 −22 Original line number Diff line number Diff line Loading @@ -76,6 +76,9 @@ workflow: - if: $CI_PIPELINE_SOURCE == 'web' && $MANUAL_PIPELINE_TYPE == 'peaq-enc-passthrough' variables: IVAS_PIPELINE_NAME: 'PEAQ encoder pass-through test: $CI_COMMIT_BRANCH' - if: $CI_PIPELINE_SOURCE == 'web' && $MANUAL_PIPELINE_TYPE == 'test-long-self-test' variables: IVAS_PIPELINE_NAME: 'Test long self-test against main pipeline: $CI_COMMIT_BRANCH' - if: $CI_PIPELINE_SOURCE == 'schedule' # Scheduled in any branch variables: IVAS_PIPELINE_NAME: 'Scheduled pipeline: $CI_COMMIT_BRANCH' Loading Loading @@ -239,7 +242,7 @@ workflow: .ivas-pytest-anchor: &ivas-pytest-anchor stage: test needs: ["build-codec-linux-make"] timeout: "360 minutes" timeout: "480 minutes" variables: # keep "mld" in artifact name for backwards compatibility reasons CSV_ARTIFACT_NAME: "mld--$CI_JOB_NAME-$CI_JOB_ID--sha-$CI_COMMIT_SHORT_SHA.csv" Loading Loading @@ -374,6 +377,9 @@ workflow: before_script: - !reference [ .test-job-linux, before_script ] - rm -rf tests/dut tests/ref - if [ "$DISABLE_HRTF" = "true" ]; then - export PYTEST_ADDOPTS="${PYTEST_ADDOPTS} -k 'not model'" - fi variables: USE_LTV: 0 Loading Loading @@ -1310,6 +1316,10 @@ ivas-pytest-on-merge-request: ### If ref_using_target is not set, checkout the source branch to use scripts and input from there - if [ $ref_using_target == 0 ]; then git checkout $source_branch_commit_sha; fi - if [ "$DISABLE_HRTF" = "true" ]; then - export PYTEST_ADDOPTS="${PYTEST_ADDOPTS} -k 'not model'" - fi ### prepare pytest # create references - python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR -v --update_ref 1 Loading Loading @@ -2117,6 +2127,102 @@ ivas-pytest-usan: - TEST_SUITE=$LONG_TEST_SUITE_NO_RENDERER <<: *ivas-pytest-sanitizers-anchor # Manual long self-test (not including tests_sba.py) against BASOP main test-long-self-test: extends: - .job-linux stage: compare timeout: "360 minutes" rules: - if: $CI_PIPELINE_SOURCE == 'web' && $MANUAL_PIPELINE_TYPE == 'test-long-self-test' allow_failure: exit_codes: - 123 parallel: matrix: - LEVEL: - 0dB - minus10dB - plus10dB script: - set -euxo pipefail - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/print-common-info.sh - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/basop/update-scripts-repo.sh - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/update-ltv-repo.sh - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/copy-ltv-files-to-testv-dir.sh - testcase_timeout=$TESTCASE_TIMEOUT_LTV - case $LEVEL in "0dB") LEVEL_SCALING=1.0 ;; "minus10dB") LEVEL_SCALING=0.3162 ;; "plus10dB") LEVEL_SCALING=3.162 ;; esac - python3 ci/remove_unsupported_testcases.py $PRM_FILES - if [ $LEVEL_SCALING != "1.0" ]; then - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/basop/apply-testv-scaling.sh - fi - INV_LEVEL_SCALING=$(awk "BEGIN {print 1.0 / $LEVEL_SCALING}") - comp_args="--mld --ssnr --odg --scalefac $INV_LEVEL_SCALING" ### store the current commit hash - source_branch_commit_sha=$(git rev-parse HEAD) ### switch to main - git checkout main - git pull origin main - echo "Building reference codec at commit $(git rev-parse HEAD)" ### build main (ref) binaries - make clean - make -j - mv ./IVAS_cod ./IVAS_cod_ref - mv ./IVAS_dec ./IVAS_dec_ref ### Switch back to branch and rename binaries back - git checkout $source_branch_commit_sha - echo "Building test codec at commit $source_branch_commit_sha" ### build branch binaries - make clean - make -j ### prepare pytest # create references - exit_code_ref=0 - python3 -m pytest tests/codec_be_on_mr_nonselection/test_param_file.py -v --update_ref 1 --param_file scripts/config/self_test_ltv.prm --use_ltv --testcase_timeout=$testcase_timeout || exit_code_ref=$? ### run pytest self-test using long test vectors - exit_code=0 - python3 -m pytest tests/codec_be_on_mr_nonselection/test_param_file.py -v --param_file scripts/config/self_test_ltv.prm --use_ltv --html=report-ltv.html --self-contained-html --junit-xml=report-junit-ltv.xml $comp_args --testcase_timeout=$testcase_timeout || exit_code=$? - zero_errors=$(cat report-junit-ltv.xml | grep -c 'errors="0"') || true - if [ $zero_errors != 1 ]; then echo "Run errors encountered!"; exit $EXIT_CODE_FAIL; fi - if [ $exit_code -ne 0 ]; then echo "Non-bitexact cases encountered!"; exit $EXIT_CODE_NON_BE; fi - exit 0 artifacts: name: "$CI_JOB_NAME--sha-$CI_COMMIT_SHORT_SHA--results" when: always expire_in: 4 mos paths: - report-junit-ltv.xml - report-ltv.html expose_as: "test-long-self-test results" reports: junit: - report-junit-ltv.xml ivas-smoke-test-saturation: extends: - .rules-pytest-saturation-smoke-test Loading Loading @@ -2170,48 +2276,34 @@ coverage-test-on-main-scheduled: script: - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/print-common-info.sh - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/basop/update-scripts-repo.sh - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/update-ltv-repo.sh - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/copy-ltv-files-to-testv-dir.sh - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/basop/build-binaries.sh float-ref # Build DuT binaries with GCOV - make clean >> /dev/null - make GCOV=1 -j - cp IVAS_rend IVAS_rend_ref # Copy to ensure instrumented renderer is run in the first pytest call - cp IVAS_rend IVAS_rend_ref # Copy to ensure instrumented renderer is run in ref creation call - testcase_timeout=$TESTCASE_TIMEOUT_LTV - exit_code_dec=0 - exit_code_enc=0 - python3 -m pytest --tb=no tests/codec_be_on_mr_nonselection tests/renderer --update_ref 1 -v --create_ref --html=report-dec.html --self-contained-html --junit-xml=report-junit-dec.xml -n auto --testcase_timeout $testcase_timeout --ref_encoder_path $REF_ENCODER_PATH --ref_decoder_path IVAS_dec || exit_code_dec=$? - lcov -c -d obj -o coverage_dec_rend.info # extract coverage of decoder/renderer - python3 -m pytest --tb=no tests/codec_be_on_mr_nonselection --encoder_only -v --html=report-enc.html --self-contained-html --junit-xml=report-junit-enc.xml -n auto --testcase_timeout $testcase_timeout --dut_encoder_path IVAS_enc || exit_code_enc=$? - lcov -c -d obj -o coverage_enc_dec_rend.info # extract coverage of encoder/decoder/renderer - exit_code=0 - python3 -m pytest --tb=no tests/codec_be_on_mr_nonselection tests/renderer --update_ref 1 -v --create_ref --html=report.html --self-contained-html --junit-xml=report-junit.xml -n auto --testcase_timeout $testcase_timeout --ref_encoder_path IVAS_cod --ref_decoder_path IVAS_dec || exit_code=$? - lcov -c -d obj -o coverage_enc_dec_rend.info # extract coverage # remove apps and lib_util files from coverage - lcov -r coverage_dec_rend.info "*apps*" -o coverage_dec_rend.info - lcov -r coverage_dec_rend.info "*lib_util*" -o coverage_dec_rend.info - lcov -r coverage_enc_dec_rend.info "*apps*" -o coverage_enc_dec_rend.info - lcov -r coverage_enc_dec_rend.info "*lib_util*" -o coverage_enc_dec_rend.info - commit_sha=$(git rev-parse HEAD) - genhtml coverage_enc_dec_rend.info -o coverage_enc_dec_rend -t "Coverage on main enc/dec/rend @ $commit_sha" - genhtml coverage_dec_rend.info -o coverage_dec_rend -t "Coverage on main -- dec/rend @ $commit_sha" - genhtml coverage_enc_dec_rend.info -o coverage_enc_dec_rend -t "Coverage on main @ $commit_sha" artifacts: name: "main-coverage-sha-$CI_COMMIT_SHORT_SHA" when: always expire_in: 1 week paths: - coverage_enc_dec_rend.info - coverage_dec_rend.info - coverage_enc_dec_rend - coverage_dec_rend - report-dec.html - report-enc.html - report.html expose_as: "Coverage result" reports: junit: - report-junit-dec.xml - report-junit-enc.xml - report-junit.xml # --------------------------------------------------------------- # EVS 26.444 test job Loading
