Commit 03839b06 authored by Jan Kiene's avatar Jan Kiene
Browse files

Merge branch '62-add-sanitizer-tests-to-ci' into 'main' 

Resolve "Add sanitizer tests to CI"

See merge request !134
parents 7dedf056 127df93c

.gitlab-ci.yml

+171 −20
Original line number Diff line number Diff line
@@ -2,6 +2,10 @@ variables: 
  TESTV_DIR: "/usr/local/testv"

  BUILD_OUTPUT: "build_output.txt"

  EVS_BE_TEST_DIR: "/usr/local/be_2_evs_test"

  SANITIZER_TESTS: "CLANG1 CLANG2"

  OUT_FORMATS_CHANNEL_BASED: "stereo mono 5_1 5_1_2 5_1_4 7_1 7_1_4"

  OUT_FORMATS_SCENE_BASED: "FOA HOA2 HOA3"

  OUT_FORMATS_BINAURAL: "BINAURAL BINAURAL_ROOM"





# This sets when pipelines are created. Jobs have more specific rules to restrict them.
@@ -462,33 +466,180 @@ codec-comparison-on-main-push: 
      junit: report-junit.xml





# parameterizable job for sanitizer tests per format

# how to set up: create a schedule (CI/CD -> schedules) and enter the respective values for the environment variables:

#   - SANITIZER_TEST_IN_FMT: input format

#   - SANITIZER_TEST_OUT_FMTS: list of output formats, blank-separated, e.g.: stereo mono 5_1

#   - SANITIZER_TEST_TESTS: list of checks to do, can be one of CLANG1, CLANG2, CLANG3, VALGRIND

sanitizer-test-on-main-scheduled:

# ---------------------------------------------------------------

# Scheduled jobs on main

# ---------------------------------------------------------------

.sanitizer-test-template:

  extends:

    - .test-job-linux-needs-testv-dir

    # this next one is maybe not really needed, since there is the rule checking for the existence of the env vars below, but use for clarity

    - .rules-main-scheduled

  stage: test

  tags:

    - sanitizer_test_main



sanitizer-test-mono:

  extends: .sanitizer-test-template

  rules:

    - if: $IS_SANITIZER_TEST_RUN

  script:

    - python3 ci/run_scheduled_sanitizer_test.py mono mono --tests $SANITIZER_TESTS



sanitizer-test-stereo:

  extends: .sanitizer-test-template

  rules:

    - if: $IS_SANITIZER_TEST_RUN

      when: delayed

      start_in: 20 minutes

  script:

    - python3 ci/run_scheduled_sanitizer_test.py stereo $OUT_FORMATS_CHANNEL_BASED --tests $SANITIZER_TESTS



sanitizer-test-stereodmxevs:

  extends: .sanitizer-test-template

  rules:

    - if: $IS_SANITIZER_TEST_RUN

      when: delayed

      start_in: 40 minutes

  script:

    - python3 ci/run_scheduled_sanitizer_test.py StereoDmxEvs mono --tests $SANITIZER_TESTS



sanitizer-test-ism1:

  extends: .sanitizer-test-template

  rules:

    - if: $IS_SANITIZER_TEST_RUN

      when: delayed

      start_in: 1 hour

  script:

    - python3 ci/run_scheduled_sanitizer_test.py ISM1 $OUT_FORMATS_CHANNEL_BASED $OUT_FORMATS_SCENE_BASED $OUT_FORMATS_BINAURAL EXT --tests $SANITIZER_TESTS



sanitizer-test-ism2:

  extends: .sanitizer-test-template

  rules:

    - if: $IS_SANITIZER_TEST_RUN

      when: delayed

      start_in: 1 hour 30 minutes

  script:

    - python3 ci/run_scheduled_sanitizer_test.py ISM2 $OUT_FORMATS_CHANNEL_BASED $OUT_FORMATS_SCENE_BASED $OUT_FORMATS_BINAURAL EXT --tests $SANITIZER_TESTS



sanitizer-test-ism3:

  extends: .sanitizer-test-template

  rules:

    - if: $IS_SANITIZER_TEST_RUN

      when: delayed

      start_in: 2 hours

  script:

    - python3 ci/run_scheduled_sanitizer_test.py ISM3 $OUT_FORMATS_CHANNEL_BASED $OUT_FORMATS_SCENE_BASED $OUT_FORMATS_BINAURAL EXT --tests $SANITIZER_TESTS



sanitizer-test-ism4:

  extends: .sanitizer-test-template

  rules:

    - if: $IS_SANITIZER_TEST_RUN

      when: delayed

      start_in: 2 hours 30 minutes

  script:

    - python3 ci/run_scheduled_sanitizer_test.py ISM4 $OUT_FORMATS_CHANNEL_BASED $OUT_FORMATS_SCENE_BASED $OUT_FORMATS_BINAURAL EXT --tests $SANITIZER_TESTS



sanitizer-test-mc-5_1:

  extends:

    - .test-job-linux-needs-testv-dir

  stage: test

  tags:

    - sanitizer_test_main

  rules:

    # only run in scheduled pipeline that passes this env vars

    - if: $SANITIZER_TEST_IN_FMT && $SANITIZER_TEST_OUT_FMTS && $SANITIZER_TEST_TESTS

    - if: $IS_SANITIZER_TEST_RUN

      when: delayed

      start_in: 3 hours

  script:

    - *print-common-info

    - echo "Running scheduled sanitizer tests $SANITIZER_TEST_TESTS for input format $SANITIZER_TEST_IN_FMT and output format(s) $SANITIZER_TEST_OUT_FMTS"

    - python3 ci/run_scheduled_sanitizer_test.py 5_1 $OUT_FORMATS_CHANNEL_BASED $OUT_FORMATS_SCENE_BASED $OUT_FORMATS_BINAURAL --tests $SANITIZER_TESTS



    - python3 ci/run_scheduled_sanitizer_test.py $SANITIZER_TEST_IN_FMT $SANITIZER_TEST_OUT_FMTS --tests $SANITIZER_TEST_TESTS

  artifacts:

    name: "sanitizer-test-results-and-error_pattern-$SANITIZER_TEST_IN_FMT-in-$SANITIZER_TEST_OUT_FMTS-out"

    when: always

    paths:

      - ep_015.g192

      - "./*/logs"

sanitizer-test-mc-5_1_2:

  extends:

    - .test-job-linux-needs-testv-dir

  stage: test

  tags:

    - sanitizer_test_main

  rules:

    - if: $IS_SANITIZER_TEST_RUN

      when: delayed

      start_in: 4 hours

  script:

    - python3 ci/run_scheduled_sanitizer_test.py 5_1_2 $OUT_FORMATS_CHANNEL_BASED $OUT_FORMATS_SCENE_BASED $OUT_FORMATS_BINAURAL --tests $SANITIZER_TESTS



sanitizer-test-mc-5_1_4:

  extends:

    - .test-job-linux-needs-testv-dir

  stage: test

  tags:

    - sanitizer_test_main

  rules:

    - if: $IS_SANITIZER_TEST_RUN

      when: delayed

      start_in: 5 hours

  script:

    - python3 ci/run_scheduled_sanitizer_test.py 5_1_4 $OUT_FORMATS_CHANNEL_BASED $OUT_FORMATS_SCENE_BASED $OUT_FORMATS_BINAURAL --tests $SANITIZER_TESTS



sanitizer-test-mc-7_1:

  extends:

    - .test-job-linux-needs-testv-dir

  stage: test

  tags:

    - sanitizer_test_main

  rules:

    - if: $IS_SANITIZER_TEST_RUN

      when: delayed

      start_in: 6 hours

  script:

    - python3 ci/run_scheduled_sanitizer_test.py 7_1 $OUT_FORMATS_CHANNEL_BASED $OUT_FORMATS_SCENE_BASED $OUT_FORMATS_BINAURAL --tests $SANITIZER_TESTS



sanitizer-test-mc-7_1_4:

  extends:

    - .test-job-linux-needs-testv-dir

  stage: test

  tags:

    - sanitizer_test_main

  rules:

    - if: $IS_SANITIZER_TEST_RUN

      when: delayed

      start_in: 7 hours

  script:

    - python3 ci/run_scheduled_sanitizer_test.py 7_1_4 $OUT_FORMATS_CHANNEL_BASED $OUT_FORMATS_SCENE_BASED $OUT_FORMATS_BINAURAL --tests $SANITIZER_TESTS



sanitizer-test-masa:

  extends:

    - .test-job-linux-needs-testv-dir

  stage: test

  tags:

    - sanitizer_test_main

  rules:

    - if: $IS_SANITIZER_TEST_RUN

      when: delayed

      start_in: 8 hours

  script:

    - python3 ci/run_scheduled_sanitizer_test.py MASA $OUT_FORMATS_CHANNEL_BASED $OUT_FORMATS_SCENE_BASED $OUT_FORMATS_BINAURAL EXT --tests $SANITIZER_TESTS



sanitizer-test-sba:

  extends:

    - .test-job-linux-needs-testv-dir

  stage: test

  tags:

    - sanitizer_test_main

  rules:

    - if: $IS_SANITIZER_TEST_RUN

      when: delayed

      start_in: 9 hours

  script:

    - python3 ci/run_scheduled_sanitizer_test.py SBA $OUT_FORMATS_CHANNEL_BASED $OUT_FORMATS_SCENE_BASED $OUT_FORMATS_BINAURAL --tests $SANITIZER_TESTS



sanitizer-test-planarsba:

  extends:

    - .test-job-linux-needs-testv-dir

  stage: test

  tags:

    - sanitizer_test_main

  rules:

    - if: $IS_SANITIZER_TEST_RUN

      when: delayed

      start_in: 10 hours

  script:

    - python3 ci/run_scheduled_sanitizer_test.py PlanarSBA $OUT_FORMATS_CHANNEL_BASED $OUT_FORMATS_SCENE_BASED $OUT_FORMATS_BINAURAL --tests $SANITIZER_TESTS



# GCOV/LCOV coverage analysis of self_test suite

coverage-test-on-main-scheduled:

ci/run_scheduled_sanitizer_test.py

+14 −3
Original line number Diff line number Diff line
@@ -32,16 +32,27 @@ def main(args): 




def get_modes(in_format: str) -> list:

    cmd = [SCRIPT_DIR.joinpath("runIvasCodec.py"), "-l"]



    cmd = [

            SCRIPT_DIR.joinpath("runIvasCodec.py"),

            "-C",

            "MC" if in_format in MC_MODES else in_format,

            "-l"

        ]

    list_process = subprocess.run(cmd, capture_output=True)



    output = list_process.stdout.decode("utf8")



    # correction for multichannel modes to avoid selecting some mono modes...

    if in_format in MC_MODES:

        in_format = "MC_" + in_format

        in_format = "MC_" + in_format + "_b"



    mode_list = [m for m in output.splitlines() if in_format in m]

    if "SBA" in in_format:

        # rate switching not implemented yet

        mode_list = [m for m in mode_list if not "_rs" in m]



    return [m for m in output.splitlines() if in_format in m]

    return mode_list





def run_check(modes: list, out_formats: list, tests: list, run_fec: bool = True):