From 580c79fad96dfa6fd6b26c1ea4d384fca4f94937 Mon Sep 17 00:00:00 2001 From: knj Date: Thu, 15 Sep 2022 15:35:32 +0200 Subject: [PATCH 01/11] create test jobs for sanitizer scheduling --- .gitlab-ci.yml | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 624322e87b..8e14dc322f 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -492,6 +492,29 @@ sanitizer-test-on-main-scheduled: - "./*/logs" +# --------------------------------------------------------------- +# Scheduled sanitizer tests on main +# --------------------------------------------------------------- +sanitizer-test-mono: + extends: + - .test-job-linux-needs-testv-dir + stage: test + rules: + - if: $IS_SANITIZER_TEST_RUN + script: + - python3 ci/run_scheduled_sanitizer_test.py mono mono --tests CLANG1 CLANG2 + +sanitizer-test-stereo: + extends: + - .test-job-linux-needs-testv-dir + stage: test + rules: + - if: $IS_SANITIZER_TEST_RUN + when: delayed + start_in: 20 minutes + script: + - python3 ci/run_scheduled_sanitizer_test.py stereo stereo mono 5_1 5_1_2 5_1_4 7_1 7_1_4 --tests CLANG1 CLANG2 + # --------------------------------------------------------------- # Other jobs # --------------------------------------------------------------- -- GitLab From f8b66c791a4ad3796fd925feea78af30d4a6bfac Mon Sep 17 00:00:00 2001 From: knj Date: Thu, 15 Sep 2022 15:36:40 +0200 Subject: [PATCH 02/11] allow scheduled pipelines also on non-main branches --- .gitlab-ci.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 8e14dc322f..3b669889b1 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -13,6 +13,7 @@ workflow: - if: $CI_PIPELINE_SOURCE == 'merge_request_event' # Runs for merge requests - if: $CI_PIPELINE_SOURCE == 'push' && $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH # Pushes to main - if: $CI_PIPELINE_SOURCE == 'schedule' && $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH # Scheduled in main + - if: $CI_PIPELINE_SOURCE == 'schedule' # temporary: any scheduled pipeline (for testing on other branch than main) stages: -- GitLab From 068c61a19fef831ecd89e138d999621c047b858e Mon Sep 17 00:00:00 2001 From: knj Date: Thu, 15 Sep 2022 15:51:12 +0200 Subject: [PATCH 03/11] add tags for scheduled jobs --- .gitlab-ci.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 3b669889b1..5b5f7c52b1 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -500,6 +500,8 @@ sanitizer-test-mono: extends: - .test-job-linux-needs-testv-dir stage: test + tags: + - sanitizer_test_main rules: - if: $IS_SANITIZER_TEST_RUN script: @@ -509,6 +511,8 @@ sanitizer-test-stereo: extends: - .test-job-linux-needs-testv-dir stage: test + tags: + - sanitizer_test_main rules: - if: $IS_SANITIZER_TEST_RUN when: delayed -- GitLab From e80b8a3f2f0a3f205f367988219148ca3c736d25 Mon Sep 17 00:00:00 2001 From: knj Date: Fri, 16 Sep 2022 09:13:08 +0200 Subject: [PATCH 04/11] add jobs for all input formats to sanitizer pipeline --- .gitlab-ci.yml | 176 +++++++++++++++++++++++++++++++++++++++++-------- 1 file changed, 148 insertions(+), 28 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 5b5f7c52b1..36923fb48d 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -2,6 +2,7 @@ variables: TESTV_DIR: "/usr/local/testv" BUILD_OUTPUT: "build_output.txt" EVS_BE_TEST_DIR: "/usr/local/be_2_evs_test" + SANITIZER_TESTS: "CLANG1 CLANG2" # This sets when pipelines are created. Jobs have more specific rules to restrict them. @@ -464,39 +465,117 @@ codec-comparison-on-main-push: junit: report-junit.xml -# parameterizable job for sanitizer tests per format -# how to set up: create a schedule (CI/CD -> schedules) and enter the respective values for the environment variables: -# - SANITIZER_TEST_IN_FMT: input format -# - SANITIZER_TEST_OUT_FMTS: list of output formats, blank-separated, e.g.: stereo mono 5_1 -# - SANITIZER_TEST_TESTS: list of checks to do, can be one of CLANG1, CLANG2, CLANG3, VALGRIND -sanitizer-test-on-main-scheduled: - extends: +# --------------------------------------------------------------- +# Scheduled sanitizer tests on main +# --------------------------------------------------------------- +.sanitizer-test-template: + extends: - .test-job-linux-needs-testv-dir - # this next one is maybe not really needed, since there is the rule checking for the existence of the env vars below, but use for clarity - - .rules-main-scheduled + stage: test tags: - sanitizer_test_main + +sanitizer-test-mono: + extends: .sanitizer-test-template + rules: + - if: $IS_SANITIZER_TEST_RUN + script: + - python3 ci/run_scheduled_sanitizer_test.py mono mono --tests $SANITIZER_TESTS + +sanitizer-test-stereo: + extends: .sanitizer-test-template + rules: + - if: $IS_SANITIZER_TEST_RUN + when: delayed + start_in: 20 minutes + script: + - python3 ci/run_scheduled_sanitizer_test.py stereo stereo mono 5_1 5_1_2 5_1_4 7_1 7_1_4 --tests $SANITIZER_TESTS + +sanitizer-test-stereodmxevs: + extends: .sanitizer-test-template + rules: + - if: $IS_SANITIZER_TEST_RUN + when: delayed + start_in: 40 minutes + script: + - python3 ci/run_scheduled_sanitizer_test.py StereoDmxEvs mono --tests $SANITIZER_TESTS + +sanitizer-test-ism1: + extends: .sanitizer-test-template + rules: + - if: $IS_SANITIZER_TEST_RUN + when: delayed + start_in: 1 hour + script: + - python3 ci/run_scheduled_sanitizer_test.py ISM1 stereo mono 5_1 5_1_2 5_1_4 7_1 7_1_4 FOA HOA2 HOA3 BINAURAL BINAURAL_ROOM EXT --tests $SANITIZER_TESTS + +sanitizer-test-ism2: + extends: .sanitizer-test-template + rules: + - if: $IS_SANITIZER_TEST_RUN + when: delayed + start_in: 1 hour 30 minutes + script: + - python3 ci/run_scheduled_sanitizer_test.py ISM2 stereo mono 5_1 5_1_2 5_1_4 7_1 7_1_4 FOA HOA2 HOA3 BINAURAL BINAURAL_ROOM EXT --tests $SANITIZER_TESTS + +sanitizer-test-ism3: + extends: .sanitizer-test-template + rules: + - if: $IS_SANITIZER_TEST_RUN + when: delayed + start_in: 2 hours + script: + - python3 ci/run_scheduled_sanitizer_test.py ISM3 stereo mono 5_1 5_1_2 5_1_4 7_1 7_1_4 FOA HOA2 HOA3 BINAURAL BINAURAL_ROOM EXT --tests $SANITIZER_TESTS + +sanitizer-test-ism4: + extends: .sanitizer-test-template + rules: + - if: $IS_SANITIZER_TEST_RUN + when: delayed + start_in: 2 hours 30 minutes + script: + - python3 ci/run_scheduled_sanitizer_test.py ISM4 stereo mono 5_1 5_1_2 5_1_4 7_1 7_1_4 FOA HOA2 HOA3 BINAURAL BINAURAL_ROOM EXT --tests $SANITIZER_TESTS + +sanitizer-test-mc-5_1: + extends: + - .test-job-linux-needs-testv-dir stage: test + tags: + - sanitizer_test_main rules: - # only run in scheduled pipeline that passes this env vars - - if: $SANITIZER_TEST_IN_FMT && $SANITIZER_TEST_OUT_FMTS && $SANITIZER_TEST_TESTS + - if: $IS_SANITIZER_TEST_RUN + when: delayed + start_in: 3 hours script: - - *print-common-info - - echo "Running scheduled sanitizer tests $SANITIZER_TEST_TESTS for input format $SANITIZER_TEST_IN_FMT and output format(s) $SANITIZER_TEST_OUT_FMTS" + - python3 ci/run_scheduled_sanitizer_test.py 5_1 stereo mono 5_1 5_1_2 5_1_4 7_1 7_1_4 FOA HOA2 HOA3 BINAURAL BINAURAL_ROOM --tests $SANITIZER_TESTS - - python3 ci/run_scheduled_sanitizer_test.py $SANITIZER_TEST_IN_FMT $SANITIZER_TEST_OUT_FMTS --tests $SANITIZER_TEST_TESTS - artifacts: - name: "sanitizer-test-results-and-error_pattern-$SANITIZER_TEST_IN_FMT-in-$SANITIZER_TEST_OUT_FMTS-out" - when: always - paths: - - ep_015.g192 - - "./*/logs" +sanitizer-test-mc-5_1_2: + extends: + - .test-job-linux-needs-testv-dir + stage: test + tags: + - sanitizer_test_main + rules: + - if: $IS_SANITIZER_TEST_RUN + when: delayed + start_in: 4 hours + script: + - python3 ci/run_scheduled_sanitizer_test.py 5_1_2 stereo mono 5_1 5_1_2 5_1_4 7_1 7_1_4 FOA HOA2 HOA3 BINAURAL BINAURAL_ROOM --tests $SANITIZER_TESTS +sanitizer-test-mc-5_1_4: + extends: + - .test-job-linux-needs-testv-dir + stage: test + tags: + - sanitizer_test_main + rules: + - if: $IS_SANITIZER_TEST_RUN + when: delayed + start_in: 5 hours + script: + - python3 ci/run_scheduled_sanitizer_test.py 5_1_4 stereo mono 5_1 5_1_2 5_1_4 7_1 7_1_4 FOA HOA2 HOA3 BINAURAL BINAURAL_ROOM --tests $SANITIZER_TESTS -# --------------------------------------------------------------- -# Scheduled sanitizer tests on main -# --------------------------------------------------------------- -sanitizer-test-mono: +sanitizer-test-mc-7_1: extends: - .test-job-linux-needs-testv-dir stage: test @@ -504,10 +583,12 @@ sanitizer-test-mono: - sanitizer_test_main rules: - if: $IS_SANITIZER_TEST_RUN + when: delayed + start_in: 6 hours script: - - python3 ci/run_scheduled_sanitizer_test.py mono mono --tests CLANG1 CLANG2 + - python3 ci/run_scheduled_sanitizer_test.py 7_1 stereo mono 5_1 5_1_2 5_1_4 7_1 7_1_4 FOA HOA2 HOA3 BINAURAL BINAURAL_ROOM --tests $SANITIZER_TESTS -sanitizer-test-stereo: +sanitizer-test-mc-7_1_4: extends: - .test-job-linux-needs-testv-dir stage: test @@ -516,9 +597,48 @@ sanitizer-test-stereo: rules: - if: $IS_SANITIZER_TEST_RUN when: delayed - start_in: 20 minutes + start_in: 7 hours + script: + - python3 ci/run_scheduled_sanitizer_test.py 7_1_4 stereo mono 5_1 5_1_2 5_1_4 7_1 7_1_4 FOA HOA2 HOA3 BINAURAL BINAURAL_ROOM --tests $SANITIZER_TESTS + +sanitizer-test-masa: + extends: + - .test-job-linux-needs-testv-dir + stage: test + tags: + - sanitizer_test_main + rules: + - if: $IS_SANITIZER_TEST_RUN + when: delayed + start_in: 8 hours + script: + - python3 ci/run_scheduled_sanitizer_test.py MASA stereo mono 5_1 5_1_2 5_1_4 7_1 7_1_4 FOA HOA2 HOA3 BINAURAL BINAURAL_ROOM EXT --tests $SANITIZER_TESTS + +sanitizer-test-mc-sba: + extends: + - .test-job-linux-needs-testv-dir + stage: test + tags: + - sanitizer_test_main + rules: + - if: $IS_SANITIZER_TEST_RUN + when: delayed + start_in: 9 hours + script: + - python3 ci/run_scheduled_sanitizer_test.py SBA stereo mono 5_1 5_1_2 5_1_4 7_1 7_1_4 FOA HOA2 HOA3 BINAURAL BINAURAL_ROOM --tests $SANITIZER_TESTS + +sanitizer-test-mc-planarsba: + extends: + - .test-job-linux-needs-testv-dir + stage: test + tags: + - sanitizer_test_main + rules: + - if: $IS_SANITIZER_TEST_RUN + when: delayed + start_in: 10 hours script: - - python3 ci/run_scheduled_sanitizer_test.py stereo stereo mono 5_1 5_1_2 5_1_4 7_1 7_1_4 --tests CLANG1 CLANG2 + - python3 ci/run_scheduled_sanitizer_test.py PlanarSBA stereo mono 5_1 5_1_2 5_1_4 7_1 7_1_4 FOA HOA2 HOA3 BINAURAL BINAURAL_ROOM --tests $SANITIZER_TESTS # --------------------------------------------------------------- # Other jobs -- GitLab From c8fb106258d4808be2ad802f6d5cda2521e099a6 Mon Sep 17 00:00:00 2001 From: knj Date: Fri, 16 Sep 2022 11:30:47 +0200 Subject: [PATCH 05/11] test whether pipeline will continue if one of the jobs fails --- .gitlab-ci.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 36923fb48d..5805573311 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -481,13 +481,16 @@ sanitizer-test-mono: - if: $IS_SANITIZER_TEST_RUN script: - python3 ci/run_scheduled_sanitizer_test.py mono mono --tests $SANITIZER_TESTS + - exit 1 # test whether this already fails the whole pipeline (I guess it should) sanitizer-test-stereo: extends: .sanitizer-test-template rules: - if: $IS_SANITIZER_TEST_RUN when: delayed - start_in: 20 minutes + # TODO: change back + # start_in: 20 minutes + start_in: 5 minutes script: - python3 ci/run_scheduled_sanitizer_test.py stereo stereo mono 5_1 5_1_2 5_1_4 7_1 7_1_4 --tests $SANITIZER_TESTS -- GitLab From 4f5c36150f7b49e43f789144bed968e4e858c437 Mon Sep 17 00:00:00 2001 From: knj Date: Fri, 16 Sep 2022 11:56:18 +0200 Subject: [PATCH 06/11] change testing values back to intended ones --- .gitlab-ci.yml | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 5805573311..36923fb48d 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -481,16 +481,13 @@ sanitizer-test-mono: - if: $IS_SANITIZER_TEST_RUN script: - python3 ci/run_scheduled_sanitizer_test.py mono mono --tests $SANITIZER_TESTS - - exit 1 # test whether this already fails the whole pipeline (I guess it should) sanitizer-test-stereo: extends: .sanitizer-test-template rules: - if: $IS_SANITIZER_TEST_RUN when: delayed - # TODO: change back - # start_in: 20 minutes - start_in: 5 minutes + start_in: 20 minutes script: - python3 ci/run_scheduled_sanitizer_test.py stereo stereo mono 5_1 5_1_2 5_1_4 7_1 7_1_4 --tests $SANITIZER_TESTS -- GitLab From 8018526c1bbef914ebe60c204dba83b88d7fada2 Mon Sep 17 00:00:00 2001 From: knj Date: Fri, 16 Sep 2022 12:18:47 +0200 Subject: [PATCH 07/11] use variables for output formats --- .gitlab-ci.yml | 31 ++++++++++++++++++------------- 1 file changed, 18 insertions(+), 13 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 36923fb48d..d3309a1719 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -3,6 +3,11 @@ variables: BUILD_OUTPUT: "build_output.txt" EVS_BE_TEST_DIR: "/usr/local/be_2_evs_test" SANITIZER_TESTS: "CLANG1 CLANG2" + OUT_FORMATS_CHANNEL_BASED: "stereo mono 5_1 5_1_2 5_1_4 7_1 7_1_4" + OUT_FORMATS_SCENE_BASED: "FOA HOA2 HOA3" + OUT_FORMATS_BINAURAL: "BINAURAL BINAURAL_ROOM" + OUT_FORMATS_ALL_NO_EXT: "$OUT_FORMATS_CHANNEL_BASED $OUT_FORMATS_SCENE_BASED $OUT_FORMATS_BINAURAL" + OUT_FORMATS_ALL: "$OUT_FORMATS_ALL_NO_EXT EXT" # This sets when pipelines are created. Jobs have more specific rules to restrict them. @@ -489,7 +494,7 @@ sanitizer-test-stereo: when: delayed start_in: 20 minutes script: - - python3 ci/run_scheduled_sanitizer_test.py stereo stereo mono 5_1 5_1_2 5_1_4 7_1 7_1_4 --tests $SANITIZER_TESTS + - python3 ci/run_scheduled_sanitizer_test.py stereo $OUT_FORMATS_CHANNEL_BASED --tests $SANITIZER_TESTS sanitizer-test-stereodmxevs: extends: .sanitizer-test-template @@ -507,7 +512,7 @@ sanitizer-test-ism1: when: delayed start_in: 1 hour script: - - python3 ci/run_scheduled_sanitizer_test.py ISM1 stereo mono 5_1 5_1_2 5_1_4 7_1 7_1_4 FOA HOA2 HOA3 BINAURAL BINAURAL_ROOM EXT --tests $SANITIZER_TESTS + - python3 ci/run_scheduled_sanitizer_test.py ISM1 $OUT_FORMATS_ALL --tests $SANITIZER_TESTS sanitizer-test-ism2: extends: .sanitizer-test-template @@ -516,7 +521,7 @@ sanitizer-test-ism2: when: delayed start_in: 1 hour 30 minutes script: - - python3 ci/run_scheduled_sanitizer_test.py ISM2 stereo mono 5_1 5_1_2 5_1_4 7_1 7_1_4 FOA HOA2 HOA3 BINAURAL BINAURAL_ROOM EXT --tests $SANITIZER_TESTS + - python3 ci/run_scheduled_sanitizer_test.py ISM2 $OUT_FORMATS_ALL --tests $SANITIZER_TESTS sanitizer-test-ism3: extends: .sanitizer-test-template @@ -525,7 +530,7 @@ sanitizer-test-ism3: when: delayed start_in: 2 hours script: - - python3 ci/run_scheduled_sanitizer_test.py ISM3 stereo mono 5_1 5_1_2 5_1_4 7_1 7_1_4 FOA HOA2 HOA3 BINAURAL BINAURAL_ROOM EXT --tests $SANITIZER_TESTS + - python3 ci/run_scheduled_sanitizer_test.py ISM3 $OUT_FORMATS_ALL --tests $SANITIZER_TESTS sanitizer-test-ism4: extends: .sanitizer-test-template @@ -534,7 +539,7 @@ sanitizer-test-ism4: when: delayed start_in: 2 hours 30 minutes script: - - python3 ci/run_scheduled_sanitizer_test.py ISM4 stereo mono 5_1 5_1_2 5_1_4 7_1 7_1_4 FOA HOA2 HOA3 BINAURAL BINAURAL_ROOM EXT --tests $SANITIZER_TESTS + - python3 ci/run_scheduled_sanitizer_test.py ISM4 $OUT_FORMATS_ALL --tests $SANITIZER_TESTS sanitizer-test-mc-5_1: extends: @@ -547,7 +552,7 @@ sanitizer-test-mc-5_1: when: delayed start_in: 3 hours script: - - python3 ci/run_scheduled_sanitizer_test.py 5_1 stereo mono 5_1 5_1_2 5_1_4 7_1 7_1_4 FOA HOA2 HOA3 BINAURAL BINAURAL_ROOM --tests $SANITIZER_TESTS + - python3 ci/run_scheduled_sanitizer_test.py 5_1 $OUT_FORMATS_ALL_NO_EXT --tests $SANITIZER_TESTS sanitizer-test-mc-5_1_2: extends: @@ -560,7 +565,7 @@ sanitizer-test-mc-5_1_2: when: delayed start_in: 4 hours script: - - python3 ci/run_scheduled_sanitizer_test.py 5_1_2 stereo mono 5_1 5_1_2 5_1_4 7_1 7_1_4 FOA HOA2 HOA3 BINAURAL BINAURAL_ROOM --tests $SANITIZER_TESTS + - python3 ci/run_scheduled_sanitizer_test.py 5_1_2 $OUT_FORMATS_ALL_NO_EXT --tests $SANITIZER_TESTS sanitizer-test-mc-5_1_4: extends: @@ -573,7 +578,7 @@ sanitizer-test-mc-5_1_4: when: delayed start_in: 5 hours script: - - python3 ci/run_scheduled_sanitizer_test.py 5_1_4 stereo mono 5_1 5_1_2 5_1_4 7_1 7_1_4 FOA HOA2 HOA3 BINAURAL BINAURAL_ROOM --tests $SANITIZER_TESTS + - python3 ci/run_scheduled_sanitizer_test.py 5_1_4 $OUT_FORMATS_ALL_NO_EXT --tests $SANITIZER_TESTS sanitizer-test-mc-7_1: extends: @@ -586,7 +591,7 @@ sanitizer-test-mc-7_1: when: delayed start_in: 6 hours script: - - python3 ci/run_scheduled_sanitizer_test.py 7_1 stereo mono 5_1 5_1_2 5_1_4 7_1 7_1_4 FOA HOA2 HOA3 BINAURAL BINAURAL_ROOM --tests $SANITIZER_TESTS + - python3 ci/run_scheduled_sanitizer_test.py 7_1 $OUT_FORMATS_ALL_NO_EXT --tests $SANITIZER_TESTS sanitizer-test-mc-7_1_4: extends: @@ -599,7 +604,7 @@ sanitizer-test-mc-7_1_4: when: delayed start_in: 7 hours script: - - python3 ci/run_scheduled_sanitizer_test.py 7_1_4 stereo mono 5_1 5_1_2 5_1_4 7_1 7_1_4 FOA HOA2 HOA3 BINAURAL BINAURAL_ROOM --tests $SANITIZER_TESTS + - python3 ci/run_scheduled_sanitizer_test.py 7_1_4 $OUT_FORMATS_ALL_NO_EXT --tests $SANITIZER_TESTS sanitizer-test-masa: extends: @@ -612,7 +617,7 @@ sanitizer-test-masa: when: delayed start_in: 8 hours script: - - python3 ci/run_scheduled_sanitizer_test.py MASA stereo mono 5_1 5_1_2 5_1_4 7_1 7_1_4 FOA HOA2 HOA3 BINAURAL BINAURAL_ROOM EXT --tests $SANITIZER_TESTS + - python3 ci/run_scheduled_sanitizer_test.py MASA $OUT_FORMATS_ALL --tests $SANITIZER_TESTS sanitizer-test-mc-sba: extends: @@ -625,7 +630,7 @@ sanitizer-test-mc-sba: when: delayed start_in: 9 hours script: - - python3 ci/run_scheduled_sanitizer_test.py SBA stereo mono 5_1 5_1_2 5_1_4 7_1 7_1_4 FOA HOA2 HOA3 BINAURAL BINAURAL_ROOM --tests $SANITIZER_TESTS + - python3 ci/run_scheduled_sanitizer_test.py SBA $OUT_FORMATS_ALL_NO_EXT --tests $SANITIZER_TESTS sanitizer-test-mc-planarsba: extends: @@ -638,7 +643,7 @@ sanitizer-test-mc-planarsba: when: delayed start_in: 10 hours script: - - python3 ci/run_scheduled_sanitizer_test.py PlanarSBA stereo mono 5_1 5_1_2 5_1_4 7_1 7_1_4 FOA HOA2 HOA3 BINAURAL BINAURAL_ROOM --tests $SANITIZER_TESTS + - python3 ci/run_scheduled_sanitizer_test.py PlanarSBA $OUT_FORMATS_ALL_NO_EXT --tests $SANITIZER_TESTS # --------------------------------------------------------------- # Other jobs -- GitLab From 5c472e0af2d8f58c599c60235e8260ac27a9e718 Mon Sep 17 00:00:00 2001 From: knj Date: Fri, 16 Sep 2022 14:24:17 +0200 Subject: [PATCH 08/11] fix variables in sanitizer jobs --- .gitlab-ci.yml | 26 ++++++++++++-------------- 1 file changed, 12 insertions(+), 14 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index d3309a1719..cb6a0d6bc2 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -6,8 +6,6 @@ variables: OUT_FORMATS_CHANNEL_BASED: "stereo mono 5_1 5_1_2 5_1_4 7_1 7_1_4" OUT_FORMATS_SCENE_BASED: "FOA HOA2 HOA3" OUT_FORMATS_BINAURAL: "BINAURAL BINAURAL_ROOM" - OUT_FORMATS_ALL_NO_EXT: "$OUT_FORMATS_CHANNEL_BASED $OUT_FORMATS_SCENE_BASED $OUT_FORMATS_BINAURAL" - OUT_FORMATS_ALL: "$OUT_FORMATS_ALL_NO_EXT EXT" # This sets when pipelines are created. Jobs have more specific rules to restrict them. @@ -512,7 +510,7 @@ sanitizer-test-ism1: when: delayed start_in: 1 hour script: - - python3 ci/run_scheduled_sanitizer_test.py ISM1 $OUT_FORMATS_ALL --tests $SANITIZER_TESTS + - python3 ci/run_scheduled_sanitizer_test.py ISM1 $OUT_FORMATS_CHANNEL_BASED $OUT_FORMATS_SCENE_BASED $OUT_FORMATS_BINAURAL EXT --tests $SANITIZER_TESTS sanitizer-test-ism2: extends: .sanitizer-test-template @@ -521,7 +519,7 @@ sanitizer-test-ism2: when: delayed start_in: 1 hour 30 minutes script: - - python3 ci/run_scheduled_sanitizer_test.py ISM2 $OUT_FORMATS_ALL --tests $SANITIZER_TESTS + - python3 ci/run_scheduled_sanitizer_test.py ISM2 $OUT_FORMATS_CHANNEL_BASED $OUT_FORMATS_SCENE_BASED $OUT_FORMATS_BINAURAL EXT --tests $SANITIZER_TESTS sanitizer-test-ism3: extends: .sanitizer-test-template @@ -530,7 +528,7 @@ sanitizer-test-ism3: when: delayed start_in: 2 hours script: - - python3 ci/run_scheduled_sanitizer_test.py ISM3 $OUT_FORMATS_ALL --tests $SANITIZER_TESTS + - python3 ci/run_scheduled_sanitizer_test.py ISM3 $OUT_FORMATS_CHANNEL_BASED $OUT_FORMATS_SCENE_BASED $OUT_FORMATS_BINAURAL EXT --tests $SANITIZER_TESTS sanitizer-test-ism4: extends: .sanitizer-test-template @@ -539,7 +537,7 @@ sanitizer-test-ism4: when: delayed start_in: 2 hours 30 minutes script: - - python3 ci/run_scheduled_sanitizer_test.py ISM4 $OUT_FORMATS_ALL --tests $SANITIZER_TESTS + - python3 ci/run_scheduled_sanitizer_test.py ISM4 $OUT_FORMATS_CHANNEL_BASED $OUT_FORMATS_SCENE_BASED $OUT_FORMATS_BINAURAL EXT --tests $SANITIZER_TESTS sanitizer-test-mc-5_1: extends: @@ -552,7 +550,7 @@ sanitizer-test-mc-5_1: when: delayed start_in: 3 hours script: - - python3 ci/run_scheduled_sanitizer_test.py 5_1 $OUT_FORMATS_ALL_NO_EXT --tests $SANITIZER_TESTS + - python3 ci/run_scheduled_sanitizer_test.py 5_1 $OUT_FORMATS_CHANNEL_BASED $OUT_FORMATS_SCENE_BASED $OUT_FORMATS_BINAURAL --tests $SANITIZER_TESTS sanitizer-test-mc-5_1_2: extends: @@ -565,7 +563,7 @@ sanitizer-test-mc-5_1_2: when: delayed start_in: 4 hours script: - - python3 ci/run_scheduled_sanitizer_test.py 5_1_2 $OUT_FORMATS_ALL_NO_EXT --tests $SANITIZER_TESTS + - python3 ci/run_scheduled_sanitizer_test.py 5_1_2 $OUT_FORMATS_CHANNEL_BASED $OUT_FORMATS_SCENE_BASED $OUT_FORMATS_BINAURAL --tests $SANITIZER_TESTS sanitizer-test-mc-5_1_4: extends: @@ -578,7 +576,7 @@ sanitizer-test-mc-5_1_4: when: delayed start_in: 5 hours script: - - python3 ci/run_scheduled_sanitizer_test.py 5_1_4 $OUT_FORMATS_ALL_NO_EXT --tests $SANITIZER_TESTS + - python3 ci/run_scheduled_sanitizer_test.py 5_1_4 $OUT_FORMATS_CHANNEL_BASED $OUT_FORMATS_SCENE_BASED $OUT_FORMATS_BINAURAL --tests $SANITIZER_TESTS sanitizer-test-mc-7_1: extends: @@ -591,7 +589,7 @@ sanitizer-test-mc-7_1: when: delayed start_in: 6 hours script: - - python3 ci/run_scheduled_sanitizer_test.py 7_1 $OUT_FORMATS_ALL_NO_EXT --tests $SANITIZER_TESTS + - python3 ci/run_scheduled_sanitizer_test.py 7_1 $OUT_FORMATS_CHANNEL_BASED $OUT_FORMATS_SCENE_BASED $OUT_FORMATS_BINAURAL --tests $SANITIZER_TESTS sanitizer-test-mc-7_1_4: extends: @@ -604,7 +602,7 @@ sanitizer-test-mc-7_1_4: when: delayed start_in: 7 hours script: - - python3 ci/run_scheduled_sanitizer_test.py 7_1_4 $OUT_FORMATS_ALL_NO_EXT --tests $SANITIZER_TESTS + - python3 ci/run_scheduled_sanitizer_test.py 7_1_4 $OUT_FORMATS_CHANNEL_BASED $OUT_FORMATS_SCENE_BASED $OUT_FORMATS_BINAURAL --tests $SANITIZER_TESTS sanitizer-test-masa: extends: @@ -617,7 +615,7 @@ sanitizer-test-masa: when: delayed start_in: 8 hours script: - - python3 ci/run_scheduled_sanitizer_test.py MASA $OUT_FORMATS_ALL --tests $SANITIZER_TESTS + - python3 ci/run_scheduled_sanitizer_test.py MASA $OUT_FORMATS_CHANNEL_BASED $OUT_FORMATS_SCENE_BASED $OUT_FORMATS_BINAURAL EXT --tests $SANITIZER_TESTS sanitizer-test-mc-sba: extends: @@ -630,7 +628,7 @@ sanitizer-test-mc-sba: when: delayed start_in: 9 hours script: - - python3 ci/run_scheduled_sanitizer_test.py SBA $OUT_FORMATS_ALL_NO_EXT --tests $SANITIZER_TESTS + - python3 ci/run_scheduled_sanitizer_test.py SBA $OUT_FORMATS_CHANNEL_BASED $OUT_FORMATS_SCENE_BASED $OUT_FORMATS_BINAURAL --tests $SANITIZER_TESTS sanitizer-test-mc-planarsba: extends: @@ -643,7 +641,7 @@ sanitizer-test-mc-planarsba: when: delayed start_in: 10 hours script: - - python3 ci/run_scheduled_sanitizer_test.py PlanarSBA $OUT_FORMATS_ALL_NO_EXT --tests $SANITIZER_TESTS + - python3 ci/run_scheduled_sanitizer_test.py PlanarSBA $OUT_FORMATS_CHANNEL_BASED $OUT_FORMATS_SCENE_BASED $OUT_FORMATS_BINAURAL --tests $SANITIZER_TESTS # --------------------------------------------------------------- # Other jobs -- GitLab From 0efbabe9ceab5456297d75f10d81b9ad39717708 Mon Sep 17 00:00:00 2001 From: knj Date: Mon, 19 Sep 2022 10:20:21 +0200 Subject: [PATCH 09/11] fix name of job --- .gitlab-ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index cb6a0d6bc2..316098f06b 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -617,7 +617,7 @@ sanitizer-test-masa: script: - python3 ci/run_scheduled_sanitizer_test.py MASA $OUT_FORMATS_CHANNEL_BASED $OUT_FORMATS_SCENE_BASED $OUT_FORMATS_BINAURAL EXT --tests $SANITIZER_TESTS -sanitizer-test-mc-sba: +sanitizer-test-sba: extends: - .test-job-linux-needs-testv-dir stage: test @@ -630,7 +630,7 @@ sanitizer-test-mc-sba: script: - python3 ci/run_scheduled_sanitizer_test.py SBA $OUT_FORMATS_CHANNEL_BASED $OUT_FORMATS_SCENE_BASED $OUT_FORMATS_BINAURAL --tests $SANITIZER_TESTS -sanitizer-test-mc-planarsba: +sanitizer-test-planarsba: extends: - .test-job-linux-needs-testv-dir stage: test -- GitLab From 3179088aa067474932247e20473475669f78d641 Mon Sep 17 00:00:00 2001 From: knj Date: Mon, 19 Sep 2022 11:39:57 +0200 Subject: [PATCH 10/11] fix mode selection and exclude rate sw for SBA --- ci/run_scheduled_sanitizer_test.py | 17 ++++++++++++++--- 1 file changed, 14 insertions(+), 3 deletions(-) diff --git a/ci/run_scheduled_sanitizer_test.py b/ci/run_scheduled_sanitizer_test.py index 073f9f70bf..1ffaaf55a8 100644 --- a/ci/run_scheduled_sanitizer_test.py +++ b/ci/run_scheduled_sanitizer_test.py @@ -32,16 +32,27 @@ def main(args): def get_modes(in_format: str) -> list: - cmd = [SCRIPT_DIR.joinpath("runIvasCodec.py"), "-l"] + + cmd = [ + SCRIPT_DIR.joinpath("runIvasCodec.py"), + "-C", + "MC" if in_format in MC_MODES else in_format, + "-l" + ] list_process = subprocess.run(cmd, capture_output=True) output = list_process.stdout.decode("utf8") # correction for multichannel modes to avoid selecting some mono modes... if in_format in MC_MODES: - in_format = "MC_" + in_format + in_format = "MC_" + in_format + "_b" + + mode_list = [m for m in output.splitlines() if in_format in m] + if "SBA" in in_format: + # rate switching not implemented yet + mode_list = [m for m in mode_list if not "_rs" in m] - return [m for m in output.splitlines() if in_format in m] + return mode_list def run_check(modes: list, out_formats: list, tests: list, run_fec: bool = True): -- GitLab From 06f1edda3046713f74c6336b2f7393fa50bc3ada Mon Sep 17 00:00:00 2001 From: knj Date: Mon, 19 Sep 2022 11:43:45 +0200 Subject: [PATCH 11/11] remove temporary hack --- .gitlab-ci.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 316098f06b..a82753e8f9 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -17,7 +17,6 @@ workflow: - if: $CI_PIPELINE_SOURCE == 'merge_request_event' # Runs for merge requests - if: $CI_PIPELINE_SOURCE == 'push' && $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH # Pushes to main - if: $CI_PIPELINE_SOURCE == 'schedule' && $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH # Scheduled in main - - if: $CI_PIPELINE_SOURCE == 'schedule' # temporary: any scheduled pipeline (for testing on other branch than main) stages: -- GitLab