Commit 8b774846 authored by Jan Kiene's avatar Jan Kiene
Browse files

long and short scheduled sanitizer tests

parent f28d3fd8

main-basop.yml

+50 −51
Original line number Diff line number Diff line
@@ -714,46 +714,6 @@ workflow: 
        - $XML_REPORT_BRANCH

        - $XML_REPORT_MAIN





.ivas-pytest-sanitizers-anchor: &ivas-pytest-sanitizers-anchor

  extends:

    - .job-linux

  stage: test

  needs: ["build-codec-linux-make"]

  script:

    - !reference [ .test-job-linux, before_script ]



    - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/print-common-info.sh

    - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/copy-ltv-files-to-testv-dir.sh



    - set -euxo pipefail

    - make_args="CLANG=$CLANG_NUM"

    - if [[ $CLANG_NUM == 3 ]]; then

    -   export UBSAN_OPTIONS="suppressions=scripts/ubsan_basop.supp,report_error_type=1,print_stacktrace=1"

    -   python3 scripts/basop_create_ignorelist_for_ubsan.py

    -   make_args="$make_args IGNORELIST=1"

    - fi

    - make clean

    - make -j $make_args >> /dev/null

    - testcase_timeout_arg="--testcase_timeout $TESTCASE_TIMEOUT_LTV_SANITIZERS"

    # disable per-testcase timeout for msan to evaluate what is going on that it takes so long

    - if [[ $CLANG_NUM = 1 ]]; then

    -   testcase_timeout_arg=""

    - fi



    # NOTE: here we do not use the "DUT_..CODER_PATH" variables because we do not build via a script, but directly from the make file (which is done because of the "make_args" - no way to inject that into build-binaries.sh)

    - python3 -m pytest $TEST_SUITE -v --tb=no --update_ref 1 --html=report.html --self-contained-html --junit-xml=report-junit.xml $testcase_timeout_arg --ref_encoder_path IVAS_cod --ref_decoder_path IVAS_dec

  artifacts:

    name: "$CI_JOB_NAME--sha-$CI_COMMIT_SHORT_SHA--results"

    when: always

    expire_in: "2 weeks"

    paths:

      - report-junit.xml

      - report.html

    reports:

      junit:

        - report-junit.xml



.ivas-pytest-compare-to-input-anchor: &ivas-pytest-compare-to-input-anchor

  stage: test

  needs: ["build-codec-linux-make", "pytest-to-ref-pre"]
@@ -2026,9 +1986,53 @@ ivas-pytest-compare_ref-long-fx-fx-lev+10: 


### These next 3 jobs run three different sanitizers for fx -> fx codec chain usin



.ivas-pytest-sanitizers-anchor:

  extends:

    - .job-linux

  stage: test

  needs: ["build-codec-linux-make"]

  timeout: "6 hours"

  parallel:

    matrix:

      - TESTSIGNALS:

        - long

        - short

  before_script:

    - case $TESTSIGNALS in

        "long")

          TEST_SUITE=LONG_TEST_SUITE_NO_RENDERER

        ;;

        "short")

          TEST_SUITE=SHORT_TEST_SUITE

        ;;

      esac

  script:

    - !reference [ .test-job-linux, before_script ]



    - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/print-common-info.sh

    - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/copy-ltv-files-to-testv-dir.sh



    - set -euxo pipefail

    - make_args="CLANG=$CLANG_NUM"

    - if [[ $CLANG_NUM == 3 ]]; then

    -   export UBSAN_OPTIONS="suppressions=scripts/ubsan_basop.supp,report_error_type=1,print_stacktrace=1"

    -   python3 scripts/basop_create_ignorelist_for_ubsan.py

    -   make_args="$make_args IGNORELIST=1"

    - fi

    - make clean

    - make -j $make_args

    - testcase_timeout_arg="--testcase_timeout $TESTCASE_TIMEOUT_LTV_SANITIZERS"

    # disable per-testcase timeout for msan to evaluate what is going on that it takes so long

    - if [[ $CLANG_NUM = 1 ]]; then

    -   testcase_timeout_arg=""

    - fi



    # NOTE: here we do not use the "DUT_..CODER_PATH" variables because we do not build via a script, but directly from the make file (which is done because of the "make_args" - no way to inject that into build-binaries.sh)

    - python3 -m pytest $TEST_SUITE -v --tb=no --update_ref 1 --html=report.html --self-contained-html --junit-xml=report-junit.xml $testcase_timeout_arg --ref_encoder_path IVAS_cod --ref_decoder_path IVAS_dec



ivas-pytest-msan:

  extends:

    - .test-job-linux

    - .ivas-pytest-sanitizers-anchor

  tags:

    - ivas-linux-fast

  resource_group: basop-long-sanitizers
@@ -2041,9 +2045,8 @@ ivas-pytest-msan: 
  - if: $CI_PIPELINE_SOURCE == 'web' && $MANUAL_PIPELINE_TYPE == "sanitizer"

  timeout: "24 hours"

  before_script:

    - !reference [ .ivas-pytest-sanitizers-anchor, before_script ]

    - CLANG_NUM=1

    - TEST_SUITE=$LONG_TEST_SUITE_NO_RENDERER

  <<: *ivas-pytest-sanitizers-anchor

  after_script:

    - python3 scripts/parse_sanitizer_errors_from_xml_report.py report-junit.xml msan-errors.csv

  artifacts:
@@ -2061,7 +2064,7 @@ ivas-pytest-msan: 


ivas-pytest-asan:

  extends:

    - .test-job-linux

    - .ivas-pytest-sanitizers-anchor

  tags:

    - ivas-linux-fast

  resource_group: basop-long-sanitizers
@@ -2072,11 +2075,9 @@ ivas-pytest-asan: 
    when: never

  - if: $CI_PIPELINE_SOURCE == 'schedule' && $IVAS_PYTEST_ASAN_USAN

  - if: $CI_PIPELINE_SOURCE == 'web' && $MANUAL_PIPELINE_TYPE == "sanitizer"

  timeout: "6 hours"

  before_script:

    - !reference [ .ivas-pytest-sanitizers-anchor, before_script ]

    - CLANG_NUM=2

    - TEST_SUITE=$LONG_TEST_SUITE_NO_RENDERER

  <<: *ivas-pytest-sanitizers-anchor

  after_script:

    - python3 scripts/parse_sanitizer_errors_from_xml_report.py report-junit.xml asan-errors.csv

  artifacts:
@@ -2094,7 +2095,7 @@ ivas-pytest-asan: 


ivas-pytest-usan:

  extends:

    - .test-job-linux

    - .ivas-pytest-sanitizers-anchor

  tags:

    - ivas-linux-fast

  resource_group: basop-long-sanitizers
@@ -2105,11 +2106,9 @@ ivas-pytest-usan: 
    when: never

  - if: $CI_PIPELINE_SOURCE == 'schedule' && $IVAS_PYTEST_ASAN_USAN

  - if: $CI_PIPELINE_SOURCE == 'web' && $MANUAL_PIPELINE_TYPE == "sanitizer"

  timeout: "6 hours"

  before_script:

    - !reference [ .ivas-pytest-sanitizers-anchor, before_script ]

    - CLANG_NUM=3

    - TEST_SUITE=$LONG_TEST_SUITE_NO_RENDERER

  <<: *ivas-pytest-sanitizers-anchor

  after_script:

    - python3 scripts/parse_sanitizer_errors_from_xml_report.py report-junit.xml usan-errors.csv

  artifacts: