Commit f28d3fd8 authored by Jan Kiene's avatar Jan Kiene
Browse files

Merge branch 'kiene/ubsan-error-reporting' into 'main' 

sanitizer error reporting

See merge request !69
parents 30751811 65270cef

main-basop.yml

+120 −25
Original line number Diff line number Diff line
@@ -1597,23 +1597,6 @@ lc3-wrapper-unit-test: 
  extends:

    - .rules-merge-request

    - .test-job-linux

  artifacts:

    name: "mr-$CI_MERGE_REQUEST_IID--sha-$CI_COMMIT_SHORT_SHA--job-$CI_JOB_NAME--results"

    expire_in: 1 week

    when: always

    paths:

      - report-junit-20ms.xml

      - report-junit-10ms.xml

      - report-junit-5ms.xml

      - report-20ms.html

      - report-10ms.html

      - report-5ms.html

    expose_as: "Sanitizer selftest results"

    reports:

      junit:

        - report-junit-20ms.xml

        - report-junit-10ms.xml

        - report-junit-5ms.xml



# From float repo

# to be reused in MR test jobs
@@ -1654,14 +1637,12 @@ lc3-wrapper-unit-test: 
    # run encoder and decoder with 20ms renderer framesize first, use reference creation mode

    - python3 -m pytest tests/codec_be_on_mr_nonselection $USE_LTV --param_file $SELF_TEST_PRM_FILE -v --update_ref 1 --html=report-20ms.html --self-contained-html --junit-xml=report-junit-20ms.xml --testcase_timeout=$testcase_timeout --ref_encoder_path ./IVAS_cod --ref_decoder_path ./IVAS_dec || exit_code20=$?



    # check different renderer fremsizes only for float

    - if [ "$CI_MERGE_REQUEST_TARGET_BRANCH_NAME" != "main" ]; then

    # for 10ms and 5ms renderer framesize, we only need to run the decoder part as renderer framesize is a decoder-only option

    # set tolerance very high do ignore any BE differences due to the different renderer framesizes, those can appear due to the limiter being active

    # we are only interested in runtime errors from the sanitizers and ignore the diffs

    -   python3 -m pytest tests/codec_be_on_mr_nonselection $USE_LTV --param_file $SELF_TEST_PRM_FILE -v --html=report-5ms.html --self-contained-html --junit-xml=report-junit-5ms.xml --dut_fr 5 --decoder_only --abs_tol 100000 || exit_code5=$?

    -   python3 -m pytest tests/codec_be_on_mr_nonselection $USE_LTV --param_file $SELF_TEST_PRM_FILE -v --html=report-10ms.html --self-contained-html --junit-xml=report-junit-10ms.xml --dut_fr 10 --decoder_only --abs_tol 100000 || exit_code10=$?

    - fi

    # TODO: was there for ivas-float-update, but not for main -> takes too long

    # - python3 -m pytest tests/codec_be_on_mr_nonselection $USE_LTV --param_file $SELF_TEST_PRM_FILE -v --html=report-5ms.html --self-contained-html --junit-xml=report-junit-5ms.xml --dut_fr 5 --decoder_only --abs_tol 100000 || exit_code5=$?

    # - python3 -m pytest tests/codec_be_on_mr_nonselection $USE_LTV --param_file $SELF_TEST_PRM_FILE -v --html=report-10ms.html --self-contained-html --junit-xml=report-junit-10ms.xml --dut_fr 10 --decoder_only --abs_tol 100000 || exit_code10=$?



    - if [ $exit_code20 -ne 0 ] || [ $exit_code10 -ne 0 ] || [ $exit_code5 -ne 0 ]; then exit 1; fi
@@ -1679,6 +1660,30 @@ codec-msan: 
    - SELF_TEST_PRM_FILE="scripts/config/self_test.prm"

    - USE_LTV=""

    - *sanitizer-selftest-anchor

  after_script:

    - python3 scripts/parse_sanitizer_errors_from_xml_report.py report-junit-20ms.xml msan-errors-20ms.csv

    # - python3 scripts/parse_sanitizer_errors_from_xml_report.py report-junit-10ms.xml msan-errors-10ms.csv

    # - python3 scripts/parse_sanitizer_errors_from_xml_report.py report-junit-5ms.xml msan-errors-5ms.csv

  artifacts:

    name: "mr-$CI_MERGE_REQUEST_IID--sha-$CI_COMMIT_SHORT_SHA--stage-$CI_JOB_STAGE--results"

    expire_in: 1 week

    when: always

    # NOTE: artifacts paths can't contain '*' when used with 'expose_as'

    # --> reminder for next time you wanna make this more concise...

    paths:

      - report-junit-20ms.xml

      - report-junit-10ms.xml

      - report-junit-5ms.xml

      - report-20ms.html

      - report-10ms.html

      - report-5ms.html

      - msan-errors-20ms.csv

    expose_as: "Sanitizer selftest results"

    reports:

      junit:

        - report-junit-20ms.xml

        - report-junit-10ms.xml

        - report-junit-5ms.xml



# code selftest testvectors with address-sanitizer binaries

codec-asan:
@@ -1692,6 +1697,30 @@ codec-asan: 
    - SELF_TEST_PRM_FILE="scripts/config/self_test.prm"

    - USE_LTV=""

    - *sanitizer-selftest-anchor

  after_script:

    - python3 scripts/parse_sanitizer_errors_from_xml_report.py report-junit-20ms.xml asan-errors-20ms.csv

    # - python3 scripts/parse_sanitizer_errors_from_xml_report.py report-junit-10ms.xml asan-errors-10ms.csv

    # - python3 scripts/parse_sanitizer_errors_from_xml_report.py report-junit-5ms.xml asan-errors-5ms.csv

  artifacts:

    name: "mr-$CI_MERGE_REQUEST_IID--sha-$CI_COMMIT_SHORT_SHA--stage-$CI_JOB_STAGE--results"

    expire_in: 1 week

    when: always

    # NOTE: artifacts paths can't contain '*' when used with 'expose_as'

    # --> reminder for next time you wanna make this more concise...

    paths:

      - report-junit-20ms.xml

      - report-junit-10ms.xml

      - report-junit-5ms.xml

      - report-20ms.html

      - report-10ms.html

      - report-5ms.html

      - asan-errors-20ms.csv

    expose_as: "Sanitizer selftest results"

    reports:

      junit:

        - report-junit-20ms.xml

        - report-junit-10ms.xml

        - report-junit-5ms.xml



# code selftest testvectors with undefined-behaviour-sanitizer binaries

codec-usan:
@@ -1705,6 +1734,30 @@ codec-usan: 
    - SELF_TEST_PRM_FILE="scripts/config/self_test.prm"

    - USE_LTV=""

    - *sanitizer-selftest-anchor

  after_script:

    - python3 scripts/parse_sanitizer_errors_from_xml_report.py report-junit-20ms.xml ubsan-errors-20ms.csv

    # - python3 scripts/parse_sanitizer_errors_from_xml_report.py report-junit-10ms.xml ubsan-errors-10ms.csv

    # - python3 scripts/parse_sanitizer_errors_from_xml_report.py report-junit-5ms.xml ubsan-errors-5ms.csv

  artifacts:

    name: "mr-$CI_MERGE_REQUEST_IID--sha-$CI_COMMIT_SHORT_SHA--stage-$CI_JOB_STAGE--results"

    expire_in: 1 week

    when: always

    # NOTE: artifacts paths can't contain '*' when used with 'expose_as'

    # --> reminder for next time you wanna make this more concise...

    paths:

      - report-junit-20ms.xml

      - report-junit-10ms.xml

      - report-junit-5ms.xml

      - report-20ms.html

      - report-10ms.html

      - report-5ms.html

      - ubsan-errors-20ms.csv

    expose_as: "Sanitizer selftest results"

    reports:

      junit:

        - report-junit-20ms.xml

        - report-junit-10ms.xml

        - report-junit-5ms.xml



# ---------------------------------------------------------------

# Short test jobs for running from web interface or schedule
@@ -1991,6 +2044,20 @@ ivas-pytest-msan: 
    - CLANG_NUM=1

    - TEST_SUITE=$LONG_TEST_SUITE_NO_RENDERER

  <<: *ivas-pytest-sanitizers-anchor

  after_script:

    - python3 scripts/parse_sanitizer_errors_from_xml_report.py report-junit.xml msan-errors.csv

  artifacts:

    name: "$CI_JOB_NAME--sha-$CI_COMMIT_SHORT_SHA--results"

    expire_in: 2 weeks

    when: always

    paths:

      - report-junit.xml

      - report.html

      - msan-errors.csv

    expose_as: "Sanitizer selftest results"

    reports:

      junit:

        - report-junit.xml



ivas-pytest-asan:

  extends:
@@ -2010,6 +2077,20 @@ ivas-pytest-asan: 
    - CLANG_NUM=2

    - TEST_SUITE=$LONG_TEST_SUITE_NO_RENDERER

  <<: *ivas-pytest-sanitizers-anchor

  after_script:

    - python3 scripts/parse_sanitizer_errors_from_xml_report.py report-junit.xml asan-errors.csv

  artifacts:

    name: "$CI_JOB_NAME--sha-$CI_COMMIT_SHORT_SHA--results"

    expire_in: 2 weeks

    when: always

    paths:

      - report-junit.xml

      - report.html

      - asan-errors.csv

    expose_as: "Sanitizer selftest results"

    reports:

      junit:

        - report-junit.xml



ivas-pytest-usan:

  extends:
@@ -2029,6 +2110,20 @@ ivas-pytest-usan: 
    - CLANG_NUM=3

    - TEST_SUITE=$LONG_TEST_SUITE_NO_RENDERER

  <<: *ivas-pytest-sanitizers-anchor

  after_script:

    - python3 scripts/parse_sanitizer_errors_from_xml_report.py report-junit.xml usan-errors.csv

  artifacts:

    name: "$CI_JOB_NAME--sha-$CI_COMMIT_SHORT_SHA--results"

    expire_in: 2 weeks

    when: always

    paths:

      - report-junit.xml

      - report.html

      - usan-errors.csv

    expose_as: "Sanitizer selftest results"

    reports:

      junit:

        - report-junit.xml



# Manual long self-test (not including tests_sba.py) against BASOP main

test-long-self-test:

main-float.yml

+36 −7
Original line number Diff line number Diff line
@@ -173,7 +173,7 @@ workflow: 


# to be reused in MR and LTV-scheduled sanitizer test jobs

# set CLANG_NUM, SELFTEST_SANITY_TIMEOUT and SELF_TEST_PRM_FILE in before_script section

.sanitizer-selftest-anchor:

.sanitizer-selftest-template:

  script:

    - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/print-common-info.sh

    - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/copy-ltv-files-to-testv-dir.sh
@@ -197,6 +197,32 @@ workflow: 
    - python3 -m pytest tests/codec_be_on_mr_nonselection $USE_LTV --param_file $SELF_TEST_PRM_FILE -v --html=report-10ms.html --self-contained-html --junit-xml=report-junit-10ms.xml --dut_fr 10 --decoder_only --abs_tol 100000 || exit_code10=$?



    - if [ $exit_code20 -ne 0 ] || [ $exit_code10 -ne 0 ] || [ $exit_code5 -ne 0 ]; then exit 1; fi

  after_script:

    - python3 scripts/parse_sanitizer_errors_from_xml_report.py report-junit-20ms.xml errors-20ms.csv

    - python3 scripts/parse_sanitizer_errors_from_xml_report.py report-junit-10ms.xml errors-10ms.csv

    - python3 scripts/parse_sanitizer_errors_from_xml_report.py report-junit-5ms.xml errors-5ms.csv

  artifacts:

    name: "mr-$CI_MERGE_REQUEST_IID--sha-$CI_COMMIT_SHORT_SHA--stage-$CI_JOB_STAGE--results"

    expire_in: 1 week

    when: always

    # NOTE: artifacts paths can't contain '*' when used with 'expose_as'

    # --> reminder for next time you wanna make this more concise...

    paths:

      - report-junit-20ms.xml

      - report-junit-10ms.xml

      - report-junit-5ms.xml

      - report-20ms.html

      - report-10ms.html

      - report-5ms.html

      - errors-20ms.csv

      - errors-10ms.csv

      - errors-5ms.csv

    expose_as: "Sanitizer selftest results"

    reports:

      junit:

        - report-junit-20ms.xml

        - report-junit-10ms.xml

        - report-junit-5ms.xml



# ---------------------------------------------------------------

# Validation jobs
@@ -402,7 +428,7 @@ basop-compat-smoke-test: 
codec-msan:

  extends:

    - .sanitizer-selftest-on-mr

    - .sanitizer-selftest-anchor

    - .sanitizer-selftest-template

  tags:

    - ivas-linux-fast

  before_script:
@@ -416,7 +442,7 @@ codec-msan: 
codec-asan:

  extends:

    - .sanitizer-selftest-on-mr

    - .sanitizer-selftest-anchor

    - .sanitizer-selftest-template

  before_script:

    - !reference [.job-linux, before_script]

    - CLANG_NUM=2
@@ -428,7 +454,7 @@ codec-asan: 
codec-usan:

  extends:

    - .sanitizer-selftest-on-mr

    - .sanitizer-selftest-anchor

    - .sanitizer-selftest-template

  tags:

    - ivas-linux-fast

  before_script:
@@ -527,6 +553,7 @@ renderer-smoke-test: 
    paths:

      - report-junit.xml

      - report.html

      - errors.csv

    reports:

      junit:

        - report-junit.xml
@@ -534,6 +561,8 @@ renderer-smoke-test: 
    - cmake -B cmake-build -G "Unix Makefiles" -DCLANG=$SANITIZER_BUILD_STRING -DCOPY_EXECUTABLES_FROM_BUILD_DIR=true

    - cmake --build cmake-build -- -j

    - testcase_timeout=180

  after_script:

    - python3 scripts/parse_sanitizer_errors_from_xml_report.py report-junit.xml errors.csv



# test renderer executable with cmake + asan

renderer-asan:
@@ -1558,7 +1587,7 @@ test-branch-vs-input-passthrough: 
ltv-msan:

  extends:

    - .sanitizer-selftest-ltv

    - .sanitizer-selftest-anchor

    - .sanitizer-selftest-template

  rules:

    - if: $SANITIZER_SCHEDULE_E

  timeout: 5 hours
@@ -1575,7 +1604,7 @@ ltv-msan: 
ltv-asan:

  extends:

    - .sanitizer-selftest-ltv

    - .sanitizer-selftest-anchor

    - .sanitizer-selftest-template

  rules:

    - if: $SANITIZER_SCHEDULE_E

      when: delayed
@@ -1594,7 +1623,7 @@ ltv-asan: 
ltv-usan:

  extends:

    - .sanitizer-selftest-ltv

    - .sanitizer-selftest-anchor

    - .sanitizer-selftest-template

  rules:

    - if: $SANITIZER_SCHEDULE_E

      when: delayed