Commit a1facf7c authored by Jan Kiene's avatar Jan Kiene
Browse files

make sanitizer tests more concise + also add scripts for renderer

parent 1a1490d2

main-float.yml

+29 −78
Original line number Diff line number Diff line
@@ -197,6 +197,32 @@ workflow: 
    - python3 -m pytest tests/codec_be_on_mr_nonselection $USE_LTV --param_file $SELF_TEST_PRM_FILE -v --html=report-10ms.html --self-contained-html --junit-xml=report-junit-10ms.xml --dut_fr 10 --decoder_only --abs_tol 100000 || exit_code10=$?



    - if [ $exit_code20 -ne 0 ] || [ $exit_code10 -ne 0 ] || [ $exit_code5 -ne 0 ]; then exit 1; fi

  after_script:

    - python3 scripts/parse_sanitizer_errors_from_xml_report.py report-junit-20ms.xml errors-20ms.csv

    - python3 scripts/parse_sanitizer_errors_from_xml_report.py report-junit-10ms.xml errors-10ms.csv

    - python3 scripts/parse_sanitizer_errors_from_xml_report.py report-junit-5ms.xml errors-5ms.csv

  artifacts:

    name: "mr-$CI_MERGE_REQUEST_IID--sha-$CI_COMMIT_SHORT_SHA--stage-$CI_JOB_STAGE--results"

    expire_in: 1 week

    when: always

    # NOTE: artifacts paths can't contain '*' when used with 'expose_as'

    # --> reminder for next time you wanna make this more concise...

    paths:

      - report-junit-20ms.xml

      - report-junit-10ms.xml

      - report-junit-5ms.xml

      - report-20ms.html

      - report-10ms.html

      - report-5ms.html

      - errors-20ms.csv

      - errors-10ms.csv

      - errors-5ms.csv

    expose_as: "Sanitizer selftest results"

    reports:

      junit:

        - report-junit-20ms.xml

        - report-junit-10ms.xml

        - report-junit-5ms.xml



# ---------------------------------------------------------------

# Validation jobs
@@ -411,32 +437,6 @@ codec-msan: 
    - SELFTEST_SANITY_TIMEOUT=$TESTCASE_TIMEOUT_STV_SANITIZERS

    - SELF_TEST_PRM_FILE="scripts/config/self_test.prm"

    - USE_LTV=""

  after_script:

    - python3 scripts/parse_sanitizer_errors_from_xml_report.py report-junit-20ms.xml msan-errors-20ms.csv

    - python3 scripts/parse_sanitizer_errors_from_xml_report.py report-junit-10ms.xml msan-errors-10ms.csv

    - python3 scripts/parse_sanitizer_errors_from_xml_report.py report-junit-5ms.xml msan-errors-5ms.csv

  artifacts:

    name: "mr-$CI_MERGE_REQUEST_IID--sha-$CI_COMMIT_SHORT_SHA--stage-$CI_JOB_STAGE--results"

    expire_in: 1 week

    when: always

    # NOTE: artifacts paths can't contain '*' when used with 'expose_as'

    # --> reminder for next time you wanna make this more concise...

    paths:

      - report-junit-20ms.xml

      - report-junit-10ms.xml

      - report-junit-5ms.xml

      - report-20ms.html

      - report-10ms.html

      - report-5ms.html

      - msan-errors-20ms.csv

      - msan-errors-10ms.csv

      - msan-errors-5ms.csv

    expose_as: "Sanitizer selftest results"

    reports:

      junit:

        - report-junit-20ms.xml

        - report-junit-10ms.xml

        - report-junit-5ms.xml



# code selftest testvectors with address-sanitizer binaries

codec-asan:
@@ -449,32 +449,6 @@ codec-asan: 
    - SELFTEST_SANITY_TIMEOUT=$TESTCASE_TIMEOUT_STV_SANITIZERS

    - SELF_TEST_PRM_FILE="scripts/config/self_test.prm"

    - USE_LTV=""

  after_script:

    - python3 scripts/parse_sanitizer_errors_from_xml_report.py report-junit-20ms.xml asan-errors-20ms.csv

    - python3 scripts/parse_sanitizer_errors_from_xml_report.py report-junit-10ms.xml asan-errors-10ms.csv

    - python3 scripts/parse_sanitizer_errors_from_xml_report.py report-junit-5ms.xml asan-errors-5ms.csv

  artifacts:

    name: "mr-$CI_MERGE_REQUEST_IID--sha-$CI_COMMIT_SHORT_SHA--stage-$CI_JOB_STAGE--results"

    expire_in: 1 week

    when: always

    # NOTE: artifacts paths can't contain '*' when used with 'expose_as'

    # --> reminder for next time you wanna make this more concise...

    paths:

      - report-junit-20ms.xml

      - report-junit-10ms.xml

      - report-junit-5ms.xml

      - report-20ms.html

      - report-10ms.html

      - report-5ms.html

      - asan-errors-20ms.csv

      - asan-errors-10ms.csv

      - asan-errors-5ms.csv

    expose_as: "Sanitizer selftest results"

    reports:

      junit:

        - report-junit-20ms.xml

        - report-junit-10ms.xml

        - report-junit-5ms.xml



# code selftest testvectors with undefined-behaviour-sanitizer binaries

codec-usan:
@@ -489,32 +463,6 @@ codec-usan: 
    - SELFTEST_SANITY_TIMEOUT=$TESTCASE_TIMEOUT_STV_SANITIZERS

    - SELF_TEST_PRM_FILE="scripts/config/self_test.prm"

    - USE_LTV=""

  after_script:

    - python3 scripts/parse_sanitizer_errors_from_xml_report.py report-junit-20ms.xml ubsan-errors-20ms.csv

    - python3 scripts/parse_sanitizer_errors_from_xml_report.py report-junit-10ms.xml ubsan-errors-10ms.csv

    - python3 scripts/parse_sanitizer_errors_from_xml_report.py report-junit-5ms.xml ubsan-errors-5ms.csv

  artifacts:

    name: "mr-$CI_MERGE_REQUEST_IID--sha-$CI_COMMIT_SHORT_SHA--stage-$CI_JOB_STAGE--results"

    expire_in: 1 week

    when: always

    # NOTE: artifacts paths can't contain '*' when used with 'expose_as'

    # --> reminder for next time you wanna make this more concise...

    paths:

      - report-junit-20ms.xml

      - report-junit-10ms.xml

      - report-junit-5ms.xml

      - report-20ms.html

      - report-10ms.html

      - report-5ms.html

      - ubsan-errors-20ms.csv

      - ubsan-errors-10ms.csv

      - ubsan-errors-5ms.csv

    expose_as: "Sanitizer selftest results"

    reports:

      junit:

        - report-junit-20ms.xml

        - report-junit-10ms.xml

        - report-junit-5ms.xml



# compare bit-exactness between 5ms and 20 on the branch

pytest-compare-20ms-and-5ms-rendering:
@@ -605,6 +553,7 @@ renderer-smoke-test: 
    paths:

      - report-junit.xml

      - report.html

      - errors.csv

    reports:

      junit:

        - report-junit.xml
@@ -612,6 +561,8 @@ renderer-smoke-test: 
    - cmake -B cmake-build -G "Unix Makefiles" -DCLANG=$SANITIZER_BUILD_STRING -DCOPY_EXECUTABLES_FROM_BUILD_DIR=true

    - cmake --build cmake-build -- -j

    - testcase_timeout=180

  after_script:

    - python3 scripts/parse_sanitizer_errors_from_xml_report.py report-junit.xml errors.csv



# test renderer executable with cmake + asan

renderer-asan: