diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 7a02bd69e5f46ded24881183f8722e55ce98ef0f..9c53be7cae9ab555b9d164321a198b3907fc95d2 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -564,12 +564,6 @@ renderer-smoke-test: before_script: - cmake -B cmake-build -G "Unix Makefiles" -DCLANG=$SANITIZER_BUILD_STRING -DCOPY_EXECUTABLES_FROM_BUILD_DIR=true - cmake --build cmake-build -- -j - # rename files to fit naming convention - # en- and decoder needed because of split rendering testcases - - mv IVAS_cod IVAS_cod_ref - - mv IVAS_dec IVAS_dec_ref - - mv IVAS_rend IVAS_rend_ref - - mv ISAR_post_rend ISAR_post_rend_ref - testcase_timeout=180 # test renderer executable with cmake + asan @@ -579,7 +573,7 @@ renderer-asan: variables: SANITIZER_BUILD_STRING: "asan" script: - - python3 -m pytest -q --tb=no -n auto --html=report.html --self-contained-html --junit-xml=report-junit.xml tests/renderer/test_renderer.py tests/split_rendering/test_split_rendering.py --create_ref --testcase_timeout=$testcase_timeout + - python3 -m pytest -q --tb=no -n auto --html=report.html --self-contained-html --junit-xml=report-junit.xml tests/renderer/test_renderer.py tests/split_rendering/test_split_rendering.py --testcase_timeout=$testcase_timeout artifacts: expose_as: "renderer asan result" @@ -591,7 +585,7 @@ renderer-msan: variables: SANITIZER_BUILD_STRING: "msan" script: - - python3 -m pytest -q --tb=no -n auto --html=report.html --self-contained-html --junit-xml=report-junit.xml tests/renderer/test_renderer.py tests/split_rendering/test_split_rendering.py --create_ref --testcase_timeout=$testcase_timeout + - python3 -m pytest -q --tb=no -n auto --html=report.html --self-contained-html --junit-xml=report-junit.xml tests/renderer/test_renderer.py tests/split_rendering/test_split_rendering.py --testcase_timeout=$testcase_timeout artifacts: expose_as: "renderer msan result" @@ -603,7 +597,7 @@ renderer-usan: variables: SANITIZER_BUILD_STRING: "usan" script: - - UBSAN_OPTIONS=suppressions=scripts/ubsan.supp,report_error_type=1 python3 -m pytest -q --tb=no -n auto --html=report.html --self-contained-html --junit-xml=report-junit.xml tests/renderer/test_renderer.py tests/split_rendering/test_split_rendering.py --create_ref --testcase_timeout=$testcase_timeout + - UBSAN_OPTIONS=suppressions=scripts/ubsan.supp,report_error_type=1 python3 -m pytest -q --tb=no -n auto --html=report.html --self-contained-html --junit-xml=report-junit.xml tests/renderer/test_renderer.py tests/split_rendering/test_split_rendering.py --testcase_timeout=$testcase_timeout artifacts: expose_as: "renderer usan result" @@ -1122,113 +1116,115 @@ be-2-evs-windows: - cd evs_be_win_test - python ../ci/run_evs_be_win_test.py +# TODO: turn into manual job if needed # check bitexactness to EVS -be-2-evs-linux: - extends: - - .test-job-linux - - .rules-main-push - tags: - - be-2-evs-temp - stage: test - needs: ["build-codec-linux-cmake"] - timeout: "20 minutes" # To be revisited - script: - - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/print-common-info.sh - - - mkdir build - - cd build - - cmake .. - - make -j - - cd .. - - # copy over to never change the testvector dir - - cp -r $EVS_BE_TEST_DIR ./evs_be_test - - cp build/IVAS_cod ./evs_be_test/bin/EVS_cod - - cp build/IVAS_dec ./evs_be_test/bin/EVS_dec - - - cd evs_be_test - - python3 ../ci/run_evs_be_test.py - -codec-comparison-on-main-push: - extends: - - .test-job-linux-needs-testv-dir - - .rules-main-push - stage: compare - needs: ["build-codec-linux-cmake"] - timeout: "30 minutes" # To be revisited - script: - - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/print-common-info.sh - - latest_commit=$(git rev-parse HEAD) # Latest commit - - previous_merge_commit=$(git --no-pager log --merges HEAD~1 -n 1 --pretty=format:%H) - - echo "Comparing changes from $previous_merge_commit to $latest_commit" - - git --no-pager diff --stat $previous_merge_commit..$latest_commit - - # Rest is more or less placeholder adapted from MR self test. This should be replaced with more complex tests. - - ### build test binaries, initial clean for paranoia reasons - - make clean - - mkdir build - - cd build - - cmake .. - - make -j - - mv IVAS_cod ../IVAS_cod_test - - mv IVAS_dec ../IVAS_dec_test - - cd .. - - rm -rf build/* - - ### compare to the previous merge commit in the main branch - - git fetch origin main - - git checkout $previous_merge_commit - - echo "Building reference codec at commit $previous_merge_commit" - - ### build reference binaries - - cd build - - cmake .. - - make -j - - mv IVAS_cod ../IVAS_cod_ref - - mv IVAS_dec ../IVAS_dec_ref - - cd .. - - # helper variable - "|| true" to prevent failures from grep not finding anything - # write to temporary file as workaround for failures observed with piping echo - - echo $CI_COMMIT_MESSAGE > tmp.txt - - non_be_flag=$(grep -c --ignore-case "\[non[ -]*be\]" tmp.txt) || true - - ref_using_main=$(grep -c --ignore-case "\[ref[ -]*using[ -]*main\]" tmp.txt) || true - - ### re-checkout the latest commit in the main branch, if ref_using_main is not set - - if [ $ref_using_main == 0 ]; then git checkout $latest_commit;fi - - ### prepare pytest - # rename test binaries back - - mv IVAS_cod_test IVAS_cod - - mv IVAS_dec_test IVAS_dec - # create references - - testcase_timeout=60 - - python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR -v --update_ref 1 --testcase_timeout=$testcase_timeout - - ### re-checkout the latest commit here, if ref_using_main is set - - if [ $ref_using_main -eq 1 ]; then git checkout $latest_commit;fi - - ### run pytest - - exit_code=0 - - python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR -v --html=report.html --self-contained-html --junit-xml=report-junit.xml --testcase_timeout=$testcase_timeout || exit_code=$? - - if [ $exit_code -ne 0 ] && [ $non_be_flag == 0 ]; then echo "pytest run had failures and non-BE flag not present"; exit $EXIT_CODE_FAIL; fi - - zero_errors=$(cat report-junit.xml | grep -c 'errors="0"') || true - - if [ $exit_code -ne 0 ] && [ $zero_errors == 1 ]; then echo "pytest run had failures, but no errors and non-BE flag present"; exit $EXIT_CODE_NON_BE; fi - - if [ $exit_code -ne 0 ]; then echo "pytest run had errors"; exit $EXIT_CODE_FAIL; fi; - allow_failure: - exit_codes: - - 123 - artifacts: - name: "main-push--sha-$CI_COMMIT_SHORT_SHA--job-$CI_JOB_NAME--results" - expire_in: 1 week - when: always - paths: - - report-junit.xml - - report.html - expose_as: "Results of comparison to previous merge commit" - reports: - junit: report-junit.xml +# be-2-evs-linux: +# extends: +# - .test-job-linux +# - .rules-main-push +# tags: +# - be-2-evs-temp +# stage: test +# needs: ["build-codec-linux-cmake"] +# timeout: "20 minutes" # To be revisited +# script: +# - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/print-common-info.sh +# +# - mkdir build +# - cd build +# - cmake .. +# - make -j +# - cd .. +# +# # copy over to never change the testvector dir +# - cp -r $EVS_BE_TEST_DIR ./evs_be_test +# - cp build/IVAS_cod ./evs_be_test/bin/EVS_cod +# - cp build/IVAS_dec ./evs_be_test/bin/EVS_dec +# +# - cd evs_be_test +# - python3 ../ci/run_evs_be_test.py + +# TODO: do we still need this? +# codec-comparison-on-main-push: +# extends: +# - .test-job-linux-needs-testv-dir +# - .rules-main-push +# stage: compare +# needs: ["build-codec-linux-cmake"] +# timeout: "30 minutes" # To be revisited +# script: +# - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/print-common-info.sh +# - latest_commit=$(git rev-parse HEAD) # Latest commit +# - previous_merge_commit=$(git --no-pager log --merges HEAD~1 -n 1 --pretty=format:%H) +# - echo "Comparing changes from $previous_merge_commit to $latest_commit" +# - git --no-pager diff --stat $previous_merge_commit..$latest_commit +# +# # Rest is more or less placeholder adapted from MR self test. This should be replaced with more complex tests. +# +# ### build test binaries, initial clean for paranoia reasons +# - make clean +# - mkdir build +# - cd build +# - cmake .. +# - make -j +# - mv IVAS_cod ../IVAS_cod_test +# - mv IVAS_dec ../IVAS_dec_test +# - cd .. +# - rm -rf build/* +# +# ### compare to the previous merge commit in the main branch +# - git fetch origin main +# - git checkout $previous_merge_commit +# - echo "Building reference codec at commit $previous_merge_commit" +# +# ### build reference binaries +# - cd build +# - cmake .. +# - make -j +# - mv IVAS_cod ../IVAS_cod_ref +# - mv IVAS_dec ../IVAS_dec_ref +# - cd .. +# +# # helper variable - "|| true" to prevent failures from grep not finding anything +# # write to temporary file as workaround for failures observed with piping echo +# - echo $CI_COMMIT_MESSAGE > tmp.txt +# - non_be_flag=$(grep -c --ignore-case "\[non[ -]*be\]" tmp.txt) || true +# - ref_using_main=$(grep -c --ignore-case "\[ref[ -]*using[ -]*main\]" tmp.txt) || true +# +# ### re-checkout the latest commit in the main branch, if ref_using_main is not set +# - if [ $ref_using_main == 0 ]; then git checkout $latest_commit;fi +# +# ### prepare pytest +# # rename test binaries back +# - mv IVAS_cod_test IVAS_cod +# - mv IVAS_dec_test IVAS_dec +# # create references +# - testcase_timeout=60 +# - python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR -v --update_ref 1 --testcase_timeout=$testcase_timeout +# +# ### re-checkout the latest commit here, if ref_using_main is set +# - if [ $ref_using_main -eq 1 ]; then git checkout $latest_commit;fi +# +# ### run pytest +# - exit_code=0 +# - python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR -v --html=report.html --self-contained-html --junit-xml=report-junit.xml --testcase_timeout=$testcase_timeout || exit_code=$? +# - if [ $exit_code -ne 0 ] && [ $non_be_flag == 0 ]; then echo "pytest run had failures and non-BE flag not present"; exit $EXIT_CODE_FAIL; fi +# - zero_errors=$(cat report-junit.xml | grep -c 'errors="0"') || true +# - if [ $exit_code -ne 0 ] && [ $zero_errors == 1 ]; then echo "pytest run had failures, but no errors and non-BE flag present"; exit $EXIT_CODE_NON_BE; fi +# - if [ $exit_code -ne 0 ]; then echo "pytest run had errors"; exit $EXIT_CODE_FAIL; fi; +# allow_failure: +# exit_codes: +# - 123 +# artifacts: +# name: "main-push--sha-$CI_COMMIT_SHORT_SHA--job-$CI_JOB_NAME--results" +# expire_in: 1 week +# when: always +# paths: +# - report-junit.xml +# - report.html +# expose_as: "Results of comparison to previous merge commit" +# reports: +# junit: report-junit.xml # ---------------------------------------------------------------