Commit 42fb494f authored by Jan Kiene's avatar Jan Kiene
Browse files

unify the pytest-based sanitizer tests

parent 0bdbc4de

main-basop.yml

+34 −130
Original line number Diff line number Diff line
@@ -293,6 +293,15 @@ workflow: 
  - if: $CI_PIPELINE_SOURCE == 'schedule' && $IVAS_PYTEST_ASAN_USAN

  - if: $CI_PIPELINE_SOURCE == 'web' && $MANUAL_PIPELINE_TYPE == "sanitizer"



.rules-pytest-sanitizers-on-schedule:

  rules:

  - if: $CI_PIPELINE_SOURCE == 'push'

    when: never

  - if: $CI_PIPELINE_SOURCE == 'merge_request_event'

    when: never

  - if: $CI_PIPELINE_SOURCE == 'schedule' && $IVAS_PYTEST_SANITIZERS

  - if: $CI_PIPELINE_SOURCE == 'web' && $MANUAL_PIPELINE_TYPE == "sanitizer"



# ---------------------------------------------------------------

# Job templates

# ---------------------------------------------------------------
@@ -1510,13 +1519,13 @@ renderer-smoke-test: 
      junit:

        - report-junit.xml



renderer-sanitizer:

ivas-pytest-sanitizers:

  extends:

    - .test-job-linux

    - .rules-pytest-msan-on-schedule

  needs: ["build-codec-linux-clang-make"]

    - .rules-pytest-sanitizers-on-schedule

  needs: ["build-codec-sanitizers-clang-linux"]

  stage: test

  timeout: "300 minutes"

  timeout: "20 hours"

  resource_group: basop-long-sanitizers

  parallel:

    matrix:
@@ -1524,9 +1533,15 @@ renderer-sanitizer: 
        - MSAN

        - ASAN

        - USAN

      - WHAT:

        - CODEC

        - RENDERER

      - TESTSIGNALS:

        - SHORT

        - LONG

  artifacts:

    name: "$CI_JOB_NAME--sha-$CI_COMMIT_SHORT_SHA--results"

    expire_in: 1 week

    expire_in: 2 week

    when: always

    paths:

      - report-junit.xml
@@ -1548,6 +1563,7 @@ renderer-sanitizer: 
    - sanitizer_to_clangnum["ASAN"]=2

    - sanitizer_to_clangnum["USAN"]=3



    # build everything with the right sanitizer

    - make clean

    - make_args="-j CLANG=${sanitizer_to_clangnum["$SANITIZER"]}"

    - |
@@ -1556,7 +1572,19 @@ renderer-sanitizer: 
      fi

    - make -j $make_args 2>&1 >$MAKE_BUILD_LOG_FILE_DEFAULT



    - UBSAN_OPTIONS=suppressions=scripts/ubsan.supp,report_error_type=1,print_stacktrace=1,log_path=usan_log_catchall python3 -m pytest -q -n auto --tb=no --log-level=INFO --html=report.html --self-contained-html --junit-xml=report-junit.xml tests/renderer/test_renderer.py

    # assemble arguments and run pytest

    - pytest_args="-q -n auto --tb=no --log-level=INFO --html=report.html --self-contained-html --junit-xml=report-junit.xml"

    - |

      if [ "$TESTSIGNALS" == "LONG"]; then

          pytest_args="$pytest_args --param_file scripts/config/self_test_ltv.prm --use_ltv"

      fi

    - |

      if [ "$WHAT" == "CODEC" ]; then

          pytest_args="$pytest_args --update_ref 1 tests/codec_be_on_mr_nonselection --ref_encoder_path IVAS_cod --ref_decoder_path IVAS_dec"

      else

          pytest_args="$pytest_args --create_ref tests/renderer/test_renderer.py"

      fi

    - UBSAN_OPTIONS=suppressions=scripts/ubsan.supp,report_error_type=1,print_stacktrace=1,log_path=usan_log_catchall python3 -m pytest $pytest_args



    - python3 scripts/parse_sanitizer_errors_from_xml_report.py report-junit.xml errors.csv
@@ -1997,130 +2025,6 @@ ivas-pytest-compare_ref-long-fx-fx-lev+10: 
    - SPLIT_COMPARISON="true"

  <<: *compare-to-ref-anchor



### These next 3 jobs run three different sanitizers for fx -> fx codec chain usin



.ivas-pytest-sanitizers-anchor:

  extends:

    - .job-linux

  stage: test

  needs: ["build-codec-linux-clang-make"]

  timeout: "6 hours"

  parallel:

    matrix:

      - TESTSIGNALS:

        - long

        - short

  before_script:

    - case $TESTSIGNALS in

        "long")

          TEST_SUITE=$LONG_TEST_SUITE_NO_RENDERER

        ;;

        "short")

          TEST_SUITE=$SHORT_TEST_SUITE

        ;;

      esac

  script:

    - !reference [ .test-job-linux, before_script ]



    - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/print-common-info.sh

    - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/copy-ltv-files-to-testv-dir.sh



    - set -euxo pipefail

    - make_args="CLANG=$CLANG_NUM"

    - if [[ $CLANG_NUM == 3 ]]; then

    -   export UBSAN_OPTIONS="suppressions=scripts/ubsan_basop.supp,report_error_type=1,print_stacktrace=1"

    -   python3 scripts/basop_create_ignorelist_for_ubsan.py

    -   make_args="$make_args IGNORELIST=1"

    - fi

    - make clean

    - make -j $make_args 2>&1 >$MAKE_BUILD_LOG_FILE_DEFAULT

    - testcase_timeout_arg="--testcase_timeout $TESTCASE_TIMEOUT_LTV_SANITIZERS"

    # disable per-testcase timeout for msan to evaluate what is going on that it takes so long

    - if [[ $CLANG_NUM = 1 ]]; then

    -   testcase_timeout_arg=""

    - fi



    # NOTE: here we do not use the "DUT_..CODER_PATH" variables because we do not build via a script, but directly from the make file (which is done because of the "make_args" - no way to inject that into build-binaries.sh)

    - python3 -m pytest $TEST_SUITE --tb=no --update_ref 1 --html=report.html --self-contained-html --junit-xml=report-junit.xml $testcase_timeout_arg --ref_encoder_path IVAS_cod --ref_decoder_path IVAS_dec



ivas-pytest-msan:

  extends:

    - .ivas-pytest-sanitizers-anchor

    - .rules-pytest-msan-on-schedule

  tags:

    - ivas-linux-fast

  resource_group: basop-long-sanitizers

  timeout: "24 hours"

  before_script:

    - !reference [ .ivas-pytest-sanitizers-anchor, before_script ]

    - CLANG_NUM=1

  after_script:

    - python3 scripts/parse_sanitizer_errors_from_xml_report.py report-junit.xml msan-errors.csv

  artifacts:

    name: "$CI_JOB_NAME--sha-$CI_COMMIT_SHORT_SHA--results"

    expire_in: 2 weeks

    when: always

    paths:

      - report-junit.xml

      - report.html

      - msan-errors.csv

      - $MAKE_BUILD_LOG_FILE_DEFAULT

    expose_as: "Sanitizer selftest results"

    reports:

      junit:

        - report-junit.xml



ivas-pytest-asan:

  extends:

    - .ivas-pytest-sanitizers-anchor

    - .rules-pytest-asan-usan-on-schedule

  tags:

    - ivas-linux-fast

  resource_group: basop-long-sanitizers

  before_script:

    - !reference [ .ivas-pytest-sanitizers-anchor, before_script ]

    - CLANG_NUM=2

  after_script:

    - python3 scripts/parse_sanitizer_errors_from_xml_report.py report-junit.xml asan-errors.csv

  artifacts:

    name: "$CI_JOB_NAME--sha-$CI_COMMIT_SHORT_SHA--results"

    expire_in: 2 weeks

    when: always

    paths:

      - report-junit.xml

      - report.html

      - asan-errors.csv

      - $MAKE_BUILD_LOG_FILE_DEFAULT

    expose_as: "Sanitizer selftest results"

    reports:

      junit:

        - report-junit.xml



ivas-pytest-usan:

  extends:

    - .ivas-pytest-sanitizers-anchor

    - .rules-pytest-asan-usan-on-schedule

  tags:

    - ivas-linux-fast

  resource_group: basop-long-sanitizers

  before_script:

    - !reference [ .ivas-pytest-sanitizers-anchor, before_script ]

    - CLANG_NUM=3

  after_script:

    - python3 scripts/parse_sanitizer_errors_from_xml_report.py report-junit.xml usan-errors.csv

  artifacts:

    name: "$CI_JOB_NAME--sha-$CI_COMMIT_SHORT_SHA--results"

    expire_in: 2 weeks

    when: always

    paths:

      - report-junit.xml

      - report.html

      - usan-errors.csv

      - $MAKE_BUILD_LOG_FILE_DEFAULT

    expose_as: "Sanitizer selftest results"

    reports:

      junit:

        - report-junit.xml



# Manual long self-test (not including tests_sba.py) against BASOP main

test-long-self-test: