From 06e267f96e4c030d68ca586cdbca7a229eac5929 Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Wed, 1 Oct 2025 09:25:29 +0200 Subject: [PATCH 1/5] tag renderer sanitizer tests for fast runners --- .gitlab-ci.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 7a02bd69e5..543332ac81 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -549,6 +549,8 @@ renderer-smoke-test: - .test-job-linux - .rules-merge-request-to-main needs: ["build-codec-linux-cmake"] + tags: + - ivas-linux-fast stage: test timeout: "90 minutes" artifacts: -- GitLab From 96a93fcf342e15c6e7c8d0268552b0924b6ee359 Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Wed, 1 Oct 2025 12:39:35 +0200 Subject: [PATCH 2/5] switch renderer sanity tests to run smoke test only --- .gitlab-ci.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 543332ac81..8c1654e120 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -581,7 +581,7 @@ renderer-asan: variables: SANITIZER_BUILD_STRING: "asan" script: - - python3 -m pytest -q --tb=no -n auto --html=report.html --self-contained-html --junit-xml=report-junit.xml tests/renderer/test_renderer.py tests/split_rendering/test_split_rendering.py --create_ref --testcase_timeout=$testcase_timeout + - python3 -m pytest -q --tb=no -n auto --html=report.html --self-contained-html --junit-xml=report-junit.xml tests/renderer/test_renderer.py tests/split_rendering/test_split_rendering.py --testcase_timeout=$testcase_timeout artifacts: expose_as: "renderer asan result" @@ -593,7 +593,7 @@ renderer-msan: variables: SANITIZER_BUILD_STRING: "msan" script: - - python3 -m pytest -q --tb=no -n auto --html=report.html --self-contained-html --junit-xml=report-junit.xml tests/renderer/test_renderer.py tests/split_rendering/test_split_rendering.py --create_ref --testcase_timeout=$testcase_timeout + - python3 -m pytest -q --tb=no -n auto --html=report.html --self-contained-html --junit-xml=report-junit.xml tests/renderer/test_renderer.py tests/split_rendering/test_split_rendering.py --testcase_timeout=$testcase_timeout artifacts: expose_as: "renderer msan result" @@ -605,7 +605,7 @@ renderer-usan: variables: SANITIZER_BUILD_STRING: "usan" script: - - UBSAN_OPTIONS=suppressions=scripts/ubsan.supp,report_error_type=1 python3 -m pytest -q --tb=no -n auto --html=report.html --self-contained-html --junit-xml=report-junit.xml tests/renderer/test_renderer.py tests/split_rendering/test_split_rendering.py --create_ref --testcase_timeout=$testcase_timeout + - UBSAN_OPTIONS=suppressions=scripts/ubsan.supp,report_error_type=1 python3 -m pytest -q --tb=no -n auto --html=report.html --self-contained-html --junit-xml=report-junit.xml tests/renderer/test_renderer.py tests/split_rendering/test_split_rendering.py --testcase_timeout=$testcase_timeout artifacts: expose_as: "renderer usan result" -- GitLab From ae14c5361dd26fb59eef5df32f0881bc9c45067e Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Wed, 1 Oct 2025 12:58:55 +0200 Subject: [PATCH 3/5] remove renaming from renderer-Xsan tests _ref suffix not needed anymore after switching to smoke test cases --- .gitlab-ci.yml | 6 ------ 1 file changed, 6 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 8c1654e120..dcfe8300ad 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -566,12 +566,6 @@ renderer-smoke-test: before_script: - cmake -B cmake-build -G "Unix Makefiles" -DCLANG=$SANITIZER_BUILD_STRING -DCOPY_EXECUTABLES_FROM_BUILD_DIR=true - cmake --build cmake-build -- -j - # rename files to fit naming convention - # en- and decoder needed because of split rendering testcases - - mv IVAS_cod IVAS_cod_ref - - mv IVAS_dec IVAS_dec_ref - - mv IVAS_rend IVAS_rend_ref - - mv ISAR_post_rend ISAR_post_rend_ref - testcase_timeout=180 # test renderer executable with cmake + asan -- GitLab From c6d94f6b59d9820ab2a40e750a68b04a8ee6f926 Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Thu, 2 Oct 2025 11:23:26 +0200 Subject: [PATCH 4/5] remove evs-be-linux and codec-comparison jobs from main push pl --- .gitlab-ci.yml | 214 +++++++++++++++++++++++++------------------------ 1 file changed, 108 insertions(+), 106 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index dcfe8300ad..0473b241b4 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1118,113 +1118,115 @@ be-2-evs-windows: - cd evs_be_win_test - python ../ci/run_evs_be_win_test.py +# TODO: turn into manual job if needed # check bitexactness to EVS -be-2-evs-linux: - extends: - - .test-job-linux - - .rules-main-push - tags: - - be-2-evs-temp - stage: test - needs: ["build-codec-linux-cmake"] - timeout: "20 minutes" # To be revisited - script: - - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/print-common-info.sh - - - mkdir build - - cd build - - cmake .. - - make -j - - cd .. - - # copy over to never change the testvector dir - - cp -r $EVS_BE_TEST_DIR ./evs_be_test - - cp build/IVAS_cod ./evs_be_test/bin/EVS_cod - - cp build/IVAS_dec ./evs_be_test/bin/EVS_dec - - - cd evs_be_test - - python3 ../ci/run_evs_be_test.py - -codec-comparison-on-main-push: - extends: - - .test-job-linux-needs-testv-dir - - .rules-main-push - stage: compare - needs: ["build-codec-linux-cmake"] - timeout: "30 minutes" # To be revisited - script: - - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/print-common-info.sh - - latest_commit=$(git rev-parse HEAD) # Latest commit - - previous_merge_commit=$(git --no-pager log --merges HEAD~1 -n 1 --pretty=format:%H) - - echo "Comparing changes from $previous_merge_commit to $latest_commit" - - git --no-pager diff --stat $previous_merge_commit..$latest_commit - - # Rest is more or less placeholder adapted from MR self test. This should be replaced with more complex tests. - - ### build test binaries, initial clean for paranoia reasons - - make clean - - mkdir build - - cd build - - cmake .. - - make -j - - mv IVAS_cod ../IVAS_cod_test - - mv IVAS_dec ../IVAS_dec_test - - cd .. - - rm -rf build/* - - ### compare to the previous merge commit in the main branch - - git fetch origin main - - git checkout $previous_merge_commit - - echo "Building reference codec at commit $previous_merge_commit" - - ### build reference binaries - - cd build - - cmake .. - - make -j - - mv IVAS_cod ../IVAS_cod_ref - - mv IVAS_dec ../IVAS_dec_ref - - cd .. - - # helper variable - "|| true" to prevent failures from grep not finding anything - # write to temporary file as workaround for failures observed with piping echo - - echo $CI_COMMIT_MESSAGE > tmp.txt - - non_be_flag=$(grep -c --ignore-case "\[non[ -]*be\]" tmp.txt) || true - - ref_using_main=$(grep -c --ignore-case "\[ref[ -]*using[ -]*main\]" tmp.txt) || true - - ### re-checkout the latest commit in the main branch, if ref_using_main is not set - - if [ $ref_using_main == 0 ]; then git checkout $latest_commit;fi - - ### prepare pytest - # rename test binaries back - - mv IVAS_cod_test IVAS_cod - - mv IVAS_dec_test IVAS_dec - # create references - - testcase_timeout=60 - - python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR -v --update_ref 1 --testcase_timeout=$testcase_timeout - - ### re-checkout the latest commit here, if ref_using_main is set - - if [ $ref_using_main -eq 1 ]; then git checkout $latest_commit;fi - - ### run pytest - - exit_code=0 - - python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR -v --html=report.html --self-contained-html --junit-xml=report-junit.xml --testcase_timeout=$testcase_timeout || exit_code=$? - - if [ $exit_code -ne 0 ] && [ $non_be_flag == 0 ]; then echo "pytest run had failures and non-BE flag not present"; exit $EXIT_CODE_FAIL; fi - - zero_errors=$(cat report-junit.xml | grep -c 'errors="0"') || true - - if [ $exit_code -ne 0 ] && [ $zero_errors == 1 ]; then echo "pytest run had failures, but no errors and non-BE flag present"; exit $EXIT_CODE_NON_BE; fi - - if [ $exit_code -ne 0 ]; then echo "pytest run had errors"; exit $EXIT_CODE_FAIL; fi; - allow_failure: - exit_codes: - - 123 - artifacts: - name: "main-push--sha-$CI_COMMIT_SHORT_SHA--job-$CI_JOB_NAME--results" - expire_in: 1 week - when: always - paths: - - report-junit.xml - - report.html - expose_as: "Results of comparison to previous merge commit" - reports: - junit: report-junit.xml +# be-2-evs-linux: +# extends: +# - .test-job-linux +# - .rules-main-push +# tags: +# - be-2-evs-temp +# stage: test +# needs: ["build-codec-linux-cmake"] +# timeout: "20 minutes" # To be revisited +# script: +# - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/print-common-info.sh +# +# - mkdir build +# - cd build +# - cmake .. +# - make -j +# - cd .. +# +# # copy over to never change the testvector dir +# - cp -r $EVS_BE_TEST_DIR ./evs_be_test +# - cp build/IVAS_cod ./evs_be_test/bin/EVS_cod +# - cp build/IVAS_dec ./evs_be_test/bin/EVS_dec +# +# - cd evs_be_test +# - python3 ../ci/run_evs_be_test.py + +# TODO: do we still need this? +# codec-comparison-on-main-push: +# extends: +# - .test-job-linux-needs-testv-dir +# - .rules-main-push +# stage: compare +# needs: ["build-codec-linux-cmake"] +# timeout: "30 minutes" # To be revisited +# script: +# - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/print-common-info.sh +# - latest_commit=$(git rev-parse HEAD) # Latest commit +# - previous_merge_commit=$(git --no-pager log --merges HEAD~1 -n 1 --pretty=format:%H) +# - echo "Comparing changes from $previous_merge_commit to $latest_commit" +# - git --no-pager diff --stat $previous_merge_commit..$latest_commit +# +# # Rest is more or less placeholder adapted from MR self test. This should be replaced with more complex tests. +# +# ### build test binaries, initial clean for paranoia reasons +# - make clean +# - mkdir build +# - cd build +# - cmake .. +# - make -j +# - mv IVAS_cod ../IVAS_cod_test +# - mv IVAS_dec ../IVAS_dec_test +# - cd .. +# - rm -rf build/* +# +# ### compare to the previous merge commit in the main branch +# - git fetch origin main +# - git checkout $previous_merge_commit +# - echo "Building reference codec at commit $previous_merge_commit" +# +# ### build reference binaries +# - cd build +# - cmake .. +# - make -j +# - mv IVAS_cod ../IVAS_cod_ref +# - mv IVAS_dec ../IVAS_dec_ref +# - cd .. +# +# # helper variable - "|| true" to prevent failures from grep not finding anything +# # write to temporary file as workaround for failures observed with piping echo +# - echo $CI_COMMIT_MESSAGE > tmp.txt +# - non_be_flag=$(grep -c --ignore-case "\[non[ -]*be\]" tmp.txt) || true +# - ref_using_main=$(grep -c --ignore-case "\[ref[ -]*using[ -]*main\]" tmp.txt) || true +# +# ### re-checkout the latest commit in the main branch, if ref_using_main is not set +# - if [ $ref_using_main == 0 ]; then git checkout $latest_commit;fi +# +# ### prepare pytest +# # rename test binaries back +# - mv IVAS_cod_test IVAS_cod +# - mv IVAS_dec_test IVAS_dec +# # create references +# - testcase_timeout=60 +# - python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR -v --update_ref 1 --testcase_timeout=$testcase_timeout +# +# ### re-checkout the latest commit here, if ref_using_main is set +# - if [ $ref_using_main -eq 1 ]; then git checkout $latest_commit;fi +# +# ### run pytest +# - exit_code=0 +# - python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR -v --html=report.html --self-contained-html --junit-xml=report-junit.xml --testcase_timeout=$testcase_timeout || exit_code=$? +# - if [ $exit_code -ne 0 ] && [ $non_be_flag == 0 ]; then echo "pytest run had failures and non-BE flag not present"; exit $EXIT_CODE_FAIL; fi +# - zero_errors=$(cat report-junit.xml | grep -c 'errors="0"') || true +# - if [ $exit_code -ne 0 ] && [ $zero_errors == 1 ]; then echo "pytest run had failures, but no errors and non-BE flag present"; exit $EXIT_CODE_NON_BE; fi +# - if [ $exit_code -ne 0 ]; then echo "pytest run had errors"; exit $EXIT_CODE_FAIL; fi; +# allow_failure: +# exit_codes: +# - 123 +# artifacts: +# name: "main-push--sha-$CI_COMMIT_SHORT_SHA--job-$CI_JOB_NAME--results" +# expire_in: 1 week +# when: always +# paths: +# - report-junit.xml +# - report.html +# expose_as: "Results of comparison to previous merge commit" +# reports: +# junit: report-junit.xml # --------------------------------------------------------------- -- GitLab From 4ef010c7eb38bb957e0dd44e142e861b70c8df28 Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Thu, 2 Oct 2025 11:25:15 +0200 Subject: [PATCH 5/5] Revert "tag renderer sanitizer tests for fast runners" This reverts commit 06e267f96e4c030d68ca586cdbca7a229eac5929. --- .gitlab-ci.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 0473b241b4..9c53be7cae 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -549,8 +549,6 @@ renderer-smoke-test: - .test-job-linux - .rules-merge-request-to-main needs: ["build-codec-linux-cmake"] - tags: - - ivas-linux-fast stage: test timeout: "90 minutes" artifacts: -- GitLab