Merge branch 'kiene/tag-renderer-sanitizer-tests-for-fast-runners' into 'main'

  before_script:

    - cmake -B cmake-build -G "Unix Makefiles" -DCLANG=$SANITIZER_BUILD_STRING -DCOPY_EXECUTABLES_FROM_BUILD_DIR=true

    - cmake --build cmake-build -- -j

    # rename files to fit naming convention

    # en- and decoder needed because of split rendering testcases

    - mv IVAS_cod IVAS_cod_ref

    - mv IVAS_dec IVAS_dec_ref

    - mv IVAS_rend IVAS_rend_ref

    - mv ISAR_post_rend ISAR_post_rend_ref

    - testcase_timeout=180

# test renderer executable with cmake + asan

  variables:

    SANITIZER_BUILD_STRING: "asan"

  script:

    - python3 -m pytest -q --tb=no -n auto --html=report.html --self-contained-html --junit-xml=report-junit.xml tests/renderer/test_renderer.py tests/split_rendering/test_split_rendering.py --create_ref --testcase_timeout=$testcase_timeout

    - python3 -m pytest -q --tb=no -n auto --html=report.html --self-contained-html --junit-xml=report-junit.xml tests/renderer/test_renderer.py tests/split_rendering/test_split_rendering.py --testcase_timeout=$testcase_timeout

  artifacts:

    expose_as: "renderer asan result"

  variables:

    SANITIZER_BUILD_STRING: "msan"

  script:

    - python3 -m pytest -q --tb=no -n auto --html=report.html --self-contained-html --junit-xml=report-junit.xml tests/renderer/test_renderer.py tests/split_rendering/test_split_rendering.py --create_ref --testcase_timeout=$testcase_timeout

    - python3 -m pytest -q --tb=no -n auto --html=report.html --self-contained-html --junit-xml=report-junit.xml tests/renderer/test_renderer.py tests/split_rendering/test_split_rendering.py --testcase_timeout=$testcase_timeout

  artifacts:

    expose_as: "renderer msan result"

  variables:

    SANITIZER_BUILD_STRING: "usan"

  script:

    - UBSAN_OPTIONS=suppressions=scripts/ubsan.supp,report_error_type=1 python3 -m pytest -q --tb=no -n auto --html=report.html --self-contained-html --junit-xml=report-junit.xml tests/renderer/test_renderer.py tests/split_rendering/test_split_rendering.py --create_ref --testcase_timeout=$testcase_timeout

    - UBSAN_OPTIONS=suppressions=scripts/ubsan.supp,report_error_type=1 python3 -m pytest -q --tb=no -n auto --html=report.html --self-contained-html --junit-xml=report-junit.xml tests/renderer/test_renderer.py tests/split_rendering/test_split_rendering.py --testcase_timeout=$testcase_timeout

  artifacts:

    expose_as: "renderer usan result"

    - cd evs_be_win_test

    - python ../ci/run_evs_be_win_test.py

# TODO: turn into manual job if needed

# check bitexactness to EVS

be-2-evs-linux:

  extends:

    - .test-job-linux

    - .rules-main-push

  tags:

    - be-2-evs-temp

  stage: test

  needs: ["build-codec-linux-cmake"]

  timeout: "20 minutes" # To be revisited

  script:

    - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/print-common-info.sh

    - mkdir build

    - cd build

    - cmake ..

    - make -j

    - cd ..

    # copy over to never change the testvector dir

    - cp -r $EVS_BE_TEST_DIR ./evs_be_test

    - cp build/IVAS_cod ./evs_be_test/bin/EVS_cod

    - cp build/IVAS_dec ./evs_be_test/bin/EVS_dec

    - cd evs_be_test

    - python3 ../ci/run_evs_be_test.py

codec-comparison-on-main-push:

  extends:

    - .test-job-linux-needs-testv-dir

    - .rules-main-push

  stage: compare

  needs: ["build-codec-linux-cmake"]

  timeout: "30 minutes" # To be revisited

  script:

    - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/print-common-info.sh

    - latest_commit=$(git rev-parse HEAD) # Latest commit

    - previous_merge_commit=$(git --no-pager log --merges HEAD~1 -n 1 --pretty=format:%H)

    - echo "Comparing changes from $previous_merge_commit to $latest_commit"

    - git --no-pager diff --stat $previous_merge_commit..$latest_commit

    # Rest is more or less placeholder adapted from MR self test. This should be replaced with more complex tests.

    ### build test binaries, initial clean for paranoia reasons

    - make clean

    - mkdir build

    - cd build

    - cmake ..

    - make -j

    - mv IVAS_cod ../IVAS_cod_test

    - mv IVAS_dec ../IVAS_dec_test

    - cd ..

    - rm -rf build/*

    ### compare to the previous merge commit in the main branch

    - git fetch origin main

    - git checkout $previous_merge_commit

    - echo "Building reference codec at commit $previous_merge_commit"

    ### build reference binaries

    - cd build

    - cmake ..

    - make -j

    - mv IVAS_cod ../IVAS_cod_ref

    - mv IVAS_dec ../IVAS_dec_ref

    - cd ..

    # helper variable - "|| true" to prevent failures from grep not finding anything

    # write to temporary file as workaround for failures observed with piping echo

    - echo $CI_COMMIT_MESSAGE > tmp.txt

    - non_be_flag=$(grep -c --ignore-case "\[non[ -]*be\]" tmp.txt) || true

    - ref_using_main=$(grep -c --ignore-case "\[ref[ -]*using[ -]*main\]" tmp.txt) || true

    ### re-checkout the latest commit in the main branch, if ref_using_main is not set

    - if [ $ref_using_main == 0 ]; then git checkout $latest_commit;fi

    ### prepare pytest

    # rename test binaries back

    - mv IVAS_cod_test IVAS_cod

    - mv IVAS_dec_test IVAS_dec

    # create references

    - testcase_timeout=60

    - python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR -v --update_ref 1 --testcase_timeout=$testcase_timeout

    ### re-checkout the latest commit here, if ref_using_main is set

    - if [ $ref_using_main -eq 1 ]; then git checkout $latest_commit;fi

    ### run pytest

    - exit_code=0

    - python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR -v --html=report.html --self-contained-html --junit-xml=report-junit.xml --testcase_timeout=$testcase_timeout || exit_code=$?

    - if [ $exit_code -ne 0 ] && [ $non_be_flag == 0 ]; then echo "pytest run had failures and non-BE flag not present"; exit $EXIT_CODE_FAIL; fi

    - zero_errors=$(cat report-junit.xml | grep -c 'errors="0"') || true

    - if [ $exit_code -ne 0 ] && [ $zero_errors == 1 ]; then echo "pytest run had failures, but no errors and non-BE flag present"; exit $EXIT_CODE_NON_BE; fi

    - if [ $exit_code -ne 0 ]; then echo "pytest run had errors"; exit $EXIT_CODE_FAIL; fi;

  allow_failure:

    exit_codes:

      - 123

  artifacts:

    name: "main-push--sha-$CI_COMMIT_SHORT_SHA--job-$CI_JOB_NAME--results"

    expire_in: 1 week

    when: always

    paths:

      - report-junit.xml

      - report.html

    expose_as: "Results of comparison to previous merge commit"

    reports:

      junit: report-junit.xml

# be-2-evs-linux:

#   extends:

#     - .test-job-linux

#     - .rules-main-push

#   tags:

#     - be-2-evs-temp

#   stage: test

#   needs: ["build-codec-linux-cmake"]

#   timeout: "20 minutes" # To be revisited

#   script:

#     - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/print-common-info.sh

#

#     - mkdir build

#     - cd build

#     - cmake ..

#     - make -j

#     - cd ..

#

#     # copy over to never change the testvector dir

#     - cp -r $EVS_BE_TEST_DIR ./evs_be_test

#     - cp build/IVAS_cod ./evs_be_test/bin/EVS_cod

#     - cp build/IVAS_dec ./evs_be_test/bin/EVS_dec

#

#     - cd evs_be_test

#     - python3 ../ci/run_evs_be_test.py

# TODO: do we still need this?

# codec-comparison-on-main-push:

#   extends:

#     - .test-job-linux-needs-testv-dir

#     - .rules-main-push

#   stage: compare

#   needs: ["build-codec-linux-cmake"]

#   timeout: "30 minutes" # To be revisited

#   script:

#     - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/print-common-info.sh

#     - latest_commit=$(git rev-parse HEAD) # Latest commit

#     - previous_merge_commit=$(git --no-pager log --merges HEAD~1 -n 1 --pretty=format:%H)

#     - echo "Comparing changes from $previous_merge_commit to $latest_commit"

#     - git --no-pager diff --stat $previous_merge_commit..$latest_commit

#

#     # Rest is more or less placeholder adapted from MR self test. This should be replaced with more complex tests.

#

#     ### build test binaries, initial clean for paranoia reasons

#     - make clean

#     - mkdir build

#     - cd build

#     - cmake ..

#     - make -j

#     - mv IVAS_cod ../IVAS_cod_test

#     - mv IVAS_dec ../IVAS_dec_test

#     - cd ..

#     - rm -rf build/*

#

#     ### compare to the previous merge commit in the main branch

#     - git fetch origin main

#     - git checkout $previous_merge_commit

#     - echo "Building reference codec at commit $previous_merge_commit"

#

#     ### build reference binaries

#     - cd build

#     - cmake ..

#     - make -j

#     - mv IVAS_cod ../IVAS_cod_ref

#     - mv IVAS_dec ../IVAS_dec_ref

#     - cd ..

#

#     # helper variable - "|| true" to prevent failures from grep not finding anything

#     # write to temporary file as workaround for failures observed with piping echo

#     - echo $CI_COMMIT_MESSAGE > tmp.txt

#     - non_be_flag=$(grep -c --ignore-case "\[non[ -]*be\]" tmp.txt) || true

#     - ref_using_main=$(grep -c --ignore-case "\[ref[ -]*using[ -]*main\]" tmp.txt) || true

#

#     ### re-checkout the latest commit in the main branch, if ref_using_main is not set

#     - if [ $ref_using_main == 0 ]; then git checkout $latest_commit;fi

#

#     ### prepare pytest

#     # rename test binaries back

#     - mv IVAS_cod_test IVAS_cod

#     - mv IVAS_dec_test IVAS_dec

#     # create references

#     - testcase_timeout=60

#     - python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR -v --update_ref 1 --testcase_timeout=$testcase_timeout

#

#     ### re-checkout the latest commit here, if ref_using_main is set

#     - if [ $ref_using_main -eq 1 ]; then git checkout $latest_commit;fi

#

#     ### run pytest

#     - exit_code=0

#     - python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR -v --html=report.html --self-contained-html --junit-xml=report-junit.xml --testcase_timeout=$testcase_timeout || exit_code=$?

#     - if [ $exit_code -ne 0 ] && [ $non_be_flag == 0 ]; then echo "pytest run had failures and non-BE flag not present"; exit $EXIT_CODE_FAIL; fi

#     - zero_errors=$(cat report-junit.xml | grep -c 'errors="0"') || true

#     - if [ $exit_code -ne 0 ] && [ $zero_errors == 1 ]; then echo "pytest run had failures, but no errors and non-BE flag present"; exit $EXIT_CODE_NON_BE; fi

#     - if [ $exit_code -ne 0 ]; then echo "pytest run had errors"; exit $EXIT_CODE_FAIL; fi;

#   allow_failure:

#     exit_codes:

#       - 123

#   artifacts:

#     name: "main-push--sha-$CI_COMMIT_SHORT_SHA--job-$CI_JOB_NAME--results"

#     expire_in: 1 week

#     when: always

#     paths:

#       - report-junit.xml

#       - report.html

#     expose_as: "Results of comparison to previous merge commit"

#     reports:

#       junit: report-junit.xml

# ---------------------------------------------------------------