From b3db699a300f4a9197a82171960234b20250f1f9 Mon Sep 17 00:00:00 2001 From: knj Date: Wed, 27 Jul 2022 14:59:09 +0200 Subject: [PATCH 01/28] add return value to IvasBuildAndRunChecks.py --- scripts/IvasBuildAndRunChecks.py | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/scripts/IvasBuildAndRunChecks.py b/scripts/IvasBuildAndRunChecks.py index fe1953a603..5933c9fe59 100755 --- a/scripts/IvasBuildAndRunChecks.py +++ b/scripts/IvasBuildAndRunChecks.py @@ -39,6 +39,9 @@ import pyivastest.constants as constants from pyivastest import ivas_svn +RET_CODE_FAILURE = 101 + + class IvasBuildAndRunChecks(IvasScriptsCommon.IvasScript): def __init__(self): super().__init__( @@ -195,7 +198,16 @@ class IvasBuildAndRunChecks(IvasScriptsCommon.IvasScript): self.args["create_complexity_tables"] ) + for check in checks: + runner = br.build_and_run_dict[check]["runner"] + failed_encs = runner.failed_modes["enc"] + failed_decs = runner.failed_modes["dec"] + if len(failed_encs) > 0 or len(failed_decs) > 0: + return RET_CODE_FAILURE + else: + return 0 + if __name__ == "__main__": script = IvasBuildAndRunChecks() - script.run() + sys.exit(script.run()) -- GitLab From f2b55de245012853ad44fbbfb3ff7639254cef4b Mon Sep 17 00:00:00 2001 From: knj Date: Wed, 27 Jul 2022 15:03:20 +0200 Subject: [PATCH 02/28] remove SVN related code from IvasBuildAndRunChecks.py --- scripts/IvasBuildAndRunChecks.py | 11 +++-------- 1 file changed, 3 insertions(+), 8 deletions(-) diff --git a/scripts/IvasBuildAndRunChecks.py b/scripts/IvasBuildAndRunChecks.py index 5933c9fe59..1cf4a868db 100755 --- a/scripts/IvasBuildAndRunChecks.py +++ b/scripts/IvasBuildAndRunChecks.py @@ -36,7 +36,6 @@ import sys from pyivastest.IvasSvnBuilder import * from pyivastest import IvasScriptsCommon import pyivastest.constants as constants -from pyivastest import ivas_svn RET_CODE_FAILURE = 101 @@ -172,14 +171,10 @@ class IvasBuildAndRunChecks(IvasScriptsCommon.IvasScript): for check in checks: br.run(check) if self.args["create_html_output"]: - revision = ivas_svn.get_local_svn_info(self.args["srcdir"], self.logger) - if revision is None: - print("Could not get revision from local copy") - revision = -1 - else: - revision = revision["commit_revision"] + cmd = ["git", "rev-parse", "HEAD"] + commit_hash = subprocess.run(cmd, capture_output=True).stdout.decode("utf8") br.build_and_run_dict[check]["analyzer"].write_html_file( - check, self.args["create_html_output"], revision + check, self.args["create_html_output"], commit_hash ) for r in br.build_and_run_dict[check]["runner"].results: self.logger.console(r[0]) -- GitLab From d9c92338f2fda1ffbd87974f691842c9855fa2df Mon Sep 17 00:00:00 2001 From: knj Date: Wed, 27 Jul 2022 15:42:28 +0200 Subject: [PATCH 03/28] do only delete tmp wav file when it was actually used --- scripts/pyivastest/IvasModeRunner.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/pyivastest/IvasModeRunner.py b/scripts/pyivastest/IvasModeRunner.py index 84616a9331..7a85aefbde 100755 --- a/scripts/pyivastest/IvasModeRunner.py +++ b/scripts/pyivastest/IvasModeRunner.py @@ -638,7 +638,7 @@ class IvasModeRunner(IvasModeCollector.IvasModeCollector): self.lock.acquire() os.remove(pcm_name_lock) # os.remove(pcm_name_res_tmp) - if do_limit_duration: + if do_limit_duration and cut_len_samples < in_len: os.remove(pcm_name_cpy_tmp) self.logger.info( "PCM file {} successfully created!".format(pcm_name) -- GitLab From d55dedef30242f6b9eceff7409604d71e30b4faf Mon Sep 17 00:00:00 2001 From: knj Date: Thu, 28 Jul 2022 08:49:26 +0200 Subject: [PATCH 04/28] add script for running sanitizer tests --- ci/run_scheduled_sanitizer_test.py | 70 ++++++++++++++++++++++++++++++ 1 file changed, 70 insertions(+) create mode 100644 ci/run_scheduled_sanitizer_test.py diff --git a/ci/run_scheduled_sanitizer_test.py b/ci/run_scheduled_sanitizer_test.py new file mode 100644 index 0000000000..a5ebd8f8da --- /dev/null +++ b/ci/run_scheduled_sanitizer_test.py @@ -0,0 +1,70 @@ +#!/usr/bin/env python3 + +import argparse +import sys +import subprocess +import pathlib + + +DURATION = "120" +CFG = "ci_linux.json" +TESTS = [ "CLANG1", "CLANG2" ] +TESTS = [ "CLANG2" ] +EP_FILE = "ep_015.g192" + +SCRIPT_DIR = pathlib.Path("./scripts").resolve() + +def main(args): + in_format = args.in_format + out_formats = args.out_formats + + modes = get_modes(in_format) + returncode = run_check(modes, out_formats) + + sys.exit(returncode) + + +def get_modes(in_format: str) -> list: + cmd = [SCRIPT_DIR.joinpath("runIvasCodec.py"), "-l"] + list_process = subprocess.run(cmd, capture_output=True) + + output = list_process.stdout.decode("utf8") + return [m for m in output.splitlines() if in_format in m] + + +def run_check(modes: list, out_formats: list): + cmd = [ + SCRIPT_DIR.joinpath("IvasBuildAndRunChecks.py"), + "-U", + DURATION, + "-p", + CFG, + "--checks", + *TESTS, + "-m", + *modes, + "--oc", + *out_formats + ] + + proc = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + for c in iter(lambda: proc.stdout.read(1), b""): + sys.stdout.buffer.write(c) + proc.wait() + + if proc.returncode not in [0, 101]: + raise IvasBuildAndRunFailed() + + return proc.returncode + + +class IvasBuildAndRunFailed(Exception): + pass + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument("in_format", type=str) + parser.add_argument("out_formats", type=str, nargs="+") + + sys.exit(main(parser.parse_args())) \ No newline at end of file -- GitLab From 85de53ab29f1d76672826d70bd92c8d24f550be3 Mon Sep 17 00:00:00 2001 From: knj Date: Thu, 28 Jul 2022 08:55:11 +0200 Subject: [PATCH 05/28] test job for sanitizer run on stereo input --- .gitlab-ci.yml | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 7aff1b478a..07f817ac74 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -120,6 +120,14 @@ build-codec-sanitizers-linux: - bash ci/build_codec_sanitizers_linux.sh +sanitizer-test-stereo-linux: + extends: .test-job-linux-needs-testv-dir + stage: test + needs: ["build-codec-sanitizers-linux"] + script: + - python3 ci/run_scheduled_sanitizer_test.py stereo stereo mono + + # test that runs all modes with 1s input signals codec-smoke-test: extends: .test-job-linux-needs-testv-dir -- GitLab From f07c79f7ec71a497f075875d4d22609c681efd14 Mon Sep 17 00:00:00 2001 From: knj Date: Thu, 28 Jul 2022 10:47:20 +0200 Subject: [PATCH 06/28] remove needs for testing --- .gitlab-ci.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 07f817ac74..fca69fd10f 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -123,7 +123,6 @@ build-codec-sanitizers-linux: sanitizer-test-stereo-linux: extends: .test-job-linux-needs-testv-dir stage: test - needs: ["build-codec-sanitizers-linux"] script: - python3 ci/run_scheduled_sanitizer_test.py stereo stereo mono -- GitLab From e77b8c97ac2565a1955426e36de6548d4ce518d1 Mon Sep 17 00:00:00 2001 From: knj Date: Thu, 28 Jul 2022 10:57:04 +0200 Subject: [PATCH 07/28] parameterize sanitizer job for running in a schedule --- .gitlab-ci.yml | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index fca69fd10f..5bf5c17522 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -120,11 +120,14 @@ build-codec-sanitizers-linux: - bash ci/build_codec_sanitizers_linux.sh -sanitizer-test-stereo-linux: +sanitizer-test-scheduled-linux: extends: .test-job-linux-needs-testv-dir stage: test + rules: + # only run in scheduled pipeline that passes this env var + - if $SANITIZER_TEST_IN_FMT script: - - python3 ci/run_scheduled_sanitizer_test.py stereo stereo mono + - python3 ci/run_scheduled_sanitizer_test.py $SANITIZER_TEST_IN_FMT $SANITIZER_TEST_OUT_FMTS # test that runs all modes with 1s input signals -- GitLab From cb59052fae7d3c27dceb832a24502be196b5ce0e Mon Sep 17 00:00:00 2001 From: knj Date: Thu, 28 Jul 2022 10:58:25 +0200 Subject: [PATCH 08/28] fix erro in ci file --- .gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 5bf5c17522..4056d187ff 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -125,7 +125,7 @@ sanitizer-test-scheduled-linux: stage: test rules: # only run in scheduled pipeline that passes this env var - - if $SANITIZER_TEST_IN_FMT + - if: $SANITIZER_TEST_IN_FMT script: - python3 ci/run_scheduled_sanitizer_test.py $SANITIZER_TEST_IN_FMT $SANITIZER_TEST_OUT_FMTS -- GitLab From 00e680e3735eaa7196b00ce0ce9d75dbc4ebe263 Mon Sep 17 00:00:00 2001 From: knj Date: Thu, 28 Jul 2022 16:45:26 +0200 Subject: [PATCH 09/28] do msan as well (was deactivated for local debugging) --- ci/run_scheduled_sanitizer_test.py | 1 - 1 file changed, 1 deletion(-) diff --git a/ci/run_scheduled_sanitizer_test.py b/ci/run_scheduled_sanitizer_test.py index a5ebd8f8da..755fcc73c1 100644 --- a/ci/run_scheduled_sanitizer_test.py +++ b/ci/run_scheduled_sanitizer_test.py @@ -9,7 +9,6 @@ import pathlib DURATION = "120" CFG = "ci_linux.json" TESTS = [ "CLANG1", "CLANG2" ] -TESTS = [ "CLANG2" ] EP_FILE = "ep_015.g192" SCRIPT_DIR = pathlib.Path("./scripts").resolve() -- GitLab From 3e66ea8f8bf83dded4b61c904740433526f0fdb3 Mon Sep 17 00:00:00 2001 From: knj Date: Thu, 1 Sep 2022 17:09:04 +0200 Subject: [PATCH 10/28] allow passing the tests in scheduled sanitizer run --- .gitlab-ci.yml | 4 ++-- ci/run_scheduled_sanitizer_test.py | 8 +++++--- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 68f7497b52..b4abebc9a6 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -425,11 +425,11 @@ sanitizer-test-on-main-scheduled: stage: test rules: # only run in scheduled pipeline that passes this env var - - if: $SANITIZER_TEST_IN_FMT && $SANITIZER_TEST_OUT_FMTS + - if: $SANITIZER_TEST_IN_FMT && $SANITIZER_TEST_OUT_FMTS && SANITIZER_TEST_TESTS script: - *print-common-info - echo "Running scheduled sanitizer" - - python3 ci/run_scheduled_sanitizer_test.py $SANITIZER_TEST_IN_FMT $SANITIZER_TEST_OUT_FMTS + - python3 ci/run_scheduled_sanitizer_test.py $SANITIZER_TEST_IN_FMT $SANITIZER_TEST_OUT_FMTS $SANITIZER_TEST_TESTS # --------------------------------------------------------------- diff --git a/ci/run_scheduled_sanitizer_test.py b/ci/run_scheduled_sanitizer_test.py index 755fcc73c1..16c82dd8e5 100644 --- a/ci/run_scheduled_sanitizer_test.py +++ b/ci/run_scheduled_sanitizer_test.py @@ -8,7 +8,7 @@ import pathlib DURATION = "120" CFG = "ci_linux.json" -TESTS = [ "CLANG1", "CLANG2" ] +SUPPORTED_TESTS = [ "CLANG1", "CLANG2", "CLANG3", "VALGRIND" ] EP_FILE = "ep_015.g192" SCRIPT_DIR = pathlib.Path("./scripts").resolve() @@ -16,9 +16,10 @@ SCRIPT_DIR = pathlib.Path("./scripts").resolve() def main(args): in_format = args.in_format out_formats = args.out_formats + tests = args.tests modes = get_modes(in_format) - returncode = run_check(modes, out_formats) + returncode = run_check(modes, out_formats, tests) sys.exit(returncode) @@ -31,7 +32,7 @@ def get_modes(in_format: str) -> list: return [m for m in output.splitlines() if in_format in m] -def run_check(modes: list, out_formats: list): +def run_check(modes: list, out_formats: list, tests: list): cmd = [ SCRIPT_DIR.joinpath("IvasBuildAndRunChecks.py"), "-U", @@ -65,5 +66,6 @@ if __name__ == "__main__": parser = argparse.ArgumentParser() parser.add_argument("in_format", type=str) parser.add_argument("out_formats", type=str, nargs="+") + parser.add_argument("tests", type=str, nargs="+") sys.exit(main(parser.parse_args())) \ No newline at end of file -- GitLab From cf0ba99e0d0fa7a1cbdf3e9731281ea26daea463 Mon Sep 17 00:00:00 2001 From: knj Date: Thu, 1 Sep 2022 17:12:56 +0200 Subject: [PATCH 11/28] fix error in ci file --- .gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index b4abebc9a6..fd43efd5cc 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -425,7 +425,7 @@ sanitizer-test-on-main-scheduled: stage: test rules: # only run in scheduled pipeline that passes this env var - - if: $SANITIZER_TEST_IN_FMT && $SANITIZER_TEST_OUT_FMTS && SANITIZER_TEST_TESTS + - if: $SANITIZER_TEST_IN_FMT && $SANITIZER_TEST_OUT_FMTS && $SANITIZER_TEST_TESTS script: - *print-common-info - echo "Running scheduled sanitizer" -- GitLab From 751b4581c1bd14ecb7846825706f07bdee7d1835 Mon Sep 17 00:00:00 2001 From: knj Date: Wed, 7 Sep 2022 11:25:18 +0200 Subject: [PATCH 12/28] add fec run to script --- ci/run_scheduled_sanitizer_test.py | 59 +++++++++++++++++++++++++----- 1 file changed, 49 insertions(+), 10 deletions(-) diff --git a/ci/run_scheduled_sanitizer_test.py b/ci/run_scheduled_sanitizer_test.py index 16c82dd8e5..81cce0de75 100644 --- a/ci/run_scheduled_sanitizer_test.py +++ b/ci/run_scheduled_sanitizer_test.py @@ -1,6 +1,7 @@ #!/usr/bin/env python3 import argparse +from distutils.command.clean import clean import sys import subprocess import pathlib @@ -10,6 +11,8 @@ DURATION = "120" CFG = "ci_linux.json" SUPPORTED_TESTS = [ "CLANG1", "CLANG2", "CLANG3", "VALGRIND" ] EP_FILE = "ep_015.g192" +GENPATT_CMD = f"gen-patt -tailstat -fer -g192 -gamma 0 -rate 0.15 -tol 0.001 -reset -n {int(DURATION) * 50} {EP_FILE}" +EIDXOR_CMD = "eid-xor -vbr -fer {bitstream} {ep_file} {out_file}" SCRIPT_DIR = pathlib.Path("./scripts").resolve() @@ -32,30 +35,66 @@ def get_modes(in_format: str) -> list: return [m for m in output.splitlines() if in_format in m] -def run_check(modes: list, out_formats: list, tests: list): - cmd = [ - SCRIPT_DIR.joinpath("IvasBuildAndRunChecks.py"), +def run_check(modes: list, out_formats: list, tests: list, run_fec: bool = True): + + ### always run encoder and decoder with no frameloss + cmd_no_fec = [ + str(SCRIPT_DIR.joinpath("IvasBuildAndRunChecks.py")), "-U", - DURATION, + # DURATION, + "1", + "-t", + "1", "-p", CFG, "--checks", - *TESTS, + *tests, "-m", - *modes, + # *modes, + "stereo_b48_swb_cbr", "--oc", - *out_formats + *out_formats, ] - proc = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + proc = subprocess.Popen(cmd_no_fec, stdout=subprocess.PIPE, stderr=subprocess.PIPE) for c in iter(lambda: proc.stdout.read(1), b""): sys.stdout.buffer.write(c) proc.wait() if proc.returncode not in [0, 101]: - raise IvasBuildAndRunFailed() + raise IvasBuildAndRunFailed("Failed at first run (no PLC)") + + returncode_no_fec = proc.returncode + + if not run_fec: + return returncode_no_fec + + ### second run: decoder only with disturbed bitstream + + # generate error pattern + subprocess.call(GENPATT_CMD.split()) + + # cleanup to avoid script errors + # we want "logs" and "dec" subfolders to be empty -> delete and recreate them + cleanup_folders = ["logs", "dec"] + for t in tests: + for fol in cleanup_folders: + for fi in pathlib.Path(t).joinpath(fol).iterdir(): + fi.unlink() + + cmd_fec = cmd_no_fec + ["--decoder_only", "-f", EP_FILE] + + proc = subprocess.Popen(cmd_fec, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + for c in iter(lambda: proc.stdout.read(1), b""): + sys.stdout.buffer.write(c) + proc.wait() + + returncode_fec = proc.returncode + + if returncode_fec not in [0, 101]: + raise IvasBuildAndRunFailed("failed at second run (PLC)") - return proc.returncode + return 101 if 101 in [returncode_no_fec, returncode_fec] else 0 class IvasBuildAndRunFailed(Exception): -- GitLab From ad54fdcdc84b7468037c7a69b6ba4ca5bf6910bb Mon Sep 17 00:00:00 2001 From: knj Date: Wed, 7 Sep 2022 11:26:49 +0200 Subject: [PATCH 13/28] cleanup and remove modifications for testing --- ci/run_scheduled_sanitizer_test.py | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/ci/run_scheduled_sanitizer_test.py b/ci/run_scheduled_sanitizer_test.py index 81cce0de75..9607dd2235 100644 --- a/ci/run_scheduled_sanitizer_test.py +++ b/ci/run_scheduled_sanitizer_test.py @@ -1,7 +1,6 @@ #!/usr/bin/env python3 import argparse -from distutils.command.clean import clean import sys import subprocess import pathlib @@ -41,17 +40,13 @@ def run_check(modes: list, out_formats: list, tests: list, run_fec: bool = True) cmd_no_fec = [ str(SCRIPT_DIR.joinpath("IvasBuildAndRunChecks.py")), "-U", - # DURATION, - "1", - "-t", - "1", + DURATION, "-p", CFG, "--checks", *tests, "-m", - # *modes, - "stereo_b48_swb_cbr", + *modes, "--oc", *out_formats, ] -- GitLab From 1be8c1315a2352ea865e8e870681f040f06fcfaf Mon Sep 17 00:00:00 2001 From: knj Date: Wed, 7 Sep 2022 11:38:29 +0200 Subject: [PATCH 14/28] add cmd line option for skipping 2nd run with fec --- ci/run_scheduled_sanitizer_test.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/ci/run_scheduled_sanitizer_test.py b/ci/run_scheduled_sanitizer_test.py index 9607dd2235..3d371a276d 100644 --- a/ci/run_scheduled_sanitizer_test.py +++ b/ci/run_scheduled_sanitizer_test.py @@ -19,9 +19,10 @@ def main(args): in_format = args.in_format out_formats = args.out_formats tests = args.tests + run_fec = not args.skip_fec modes = get_modes(in_format) - returncode = run_check(modes, out_formats, tests) + returncode = run_check(modes, out_formats, tests, run_fec=run_fec) sys.exit(returncode) @@ -101,5 +102,6 @@ if __name__ == "__main__": parser.add_argument("in_format", type=str) parser.add_argument("out_formats", type=str, nargs="+") parser.add_argument("tests", type=str, nargs="+") + parser.add_argument("--skip_fec", action="store_true") sys.exit(main(parser.parse_args())) \ No newline at end of file -- GitLab From d411e5e2e65f86de1f2edbb4090d6bf0337a47bd Mon Sep 17 00:00:00 2001 From: knj Date: Wed, 7 Sep 2022 12:00:41 +0200 Subject: [PATCH 15/28] add debug printout to job --- .gitlab-ci.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index fd43efd5cc..f117eb8b43 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -429,6 +429,7 @@ sanitizer-test-on-main-scheduled: script: - *print-common-info - echo "Running scheduled sanitizer" + - echo "$SANITIZER_TEST_IN_FMT && $SANITIZER_TEST_OUT_FMTS && $SANITIZER_TEST_TESTS" - python3 ci/run_scheduled_sanitizer_test.py $SANITIZER_TEST_IN_FMT $SANITIZER_TEST_OUT_FMTS $SANITIZER_TEST_TESTS -- GitLab From fc64b5d46a566035337a93db28a35448c57f1d0f Mon Sep 17 00:00:00 2001 From: kiene Date: Wed, 7 Sep 2022 10:22:25 +0000 Subject: [PATCH 16/28] Update .gitlab-ci.yml file --- .gitlab-ci.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index f117eb8b43..66ba7202da 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -425,7 +425,8 @@ sanitizer-test-on-main-scheduled: stage: test rules: # only run in scheduled pipeline that passes this env var - - if: $SANITIZER_TEST_IN_FMT && $SANITIZER_TEST_OUT_FMTS && $SANITIZER_TEST_TESTS + #- if: $SANITIZER_TEST_IN_FMT && $SANITIZER_TEST_OUT_FMTS && $SANITIZER_TEST_TESTS + - if: $SANITIZER_TEST_IN_FMT script: - *print-common-info - echo "Running scheduled sanitizer" -- GitLab From ecb67fa0f5cf2a62cef49f823266e5a85aae2093 Mon Sep 17 00:00:00 2001 From: kiene Date: Wed, 7 Sep 2022 10:38:35 +0000 Subject: [PATCH 17/28] run on MR pipeline for testing --- .gitlab-ci.yml | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 66ba7202da..a9c5b8de10 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -421,17 +421,20 @@ codec-comparison-on-main-push: sanitizer-test-on-main-scheduled: - extends: .test-job-linux-needs-testv-dir + extends: + - .rules-merge-request + - .test-job-linux-needs-testv-dir stage: test - rules: + #rules: # only run in scheduled pipeline that passes this env var #- if: $SANITIZER_TEST_IN_FMT && $SANITIZER_TEST_OUT_FMTS && $SANITIZER_TEST_TESTS - - if: $SANITIZER_TEST_IN_FMT + #- if: $SANITIZER_TEST_IN_FMT script: - *print-common-info - echo "Running scheduled sanitizer" - echo "$SANITIZER_TEST_IN_FMT && $SANITIZER_TEST_OUT_FMTS && $SANITIZER_TEST_TESTS" - - python3 ci/run_scheduled_sanitizer_test.py $SANITIZER_TEST_IN_FMT $SANITIZER_TEST_OUT_FMTS $SANITIZER_TEST_TESTS + #- python3 ci/run_scheduled_sanitizer_test.py $SANITIZER_TEST_IN_FMT $SANITIZER_TEST_OUT_FMTS $SANITIZER_TEST_TESTS + - python3 ci/run_scheduled_sanitizer_test.py stereo stereo CLANG1 # --------------------------------------------------------------- -- GitLab From 9761af8e4b87ca88beefe87042db3919da374aeb Mon Sep 17 00:00:00 2001 From: knj Date: Wed, 7 Sep 2022 12:46:27 +0200 Subject: [PATCH 18/28] test only on one runner --- .gitlab-ci.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index a9c5b8de10..b07afcb36f 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -424,6 +424,8 @@ sanitizer-test-on-main-scheduled: extends: - .rules-merge-request - .test-job-linux-needs-testv-dir + tags: + - test-fhg-linux-runner1 stage: test #rules: # only run in scheduled pipeline that passes this env var -- GitLab From 6d7276378008555ee1afe34aa4d4844d344a2184 Mon Sep 17 00:00:00 2001 From: knj Date: Wed, 7 Sep 2022 12:51:55 +0200 Subject: [PATCH 19/28] add tools dir in ci_linux.json --- scripts/config/ci_linux.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/config/ci_linux.json b/scripts/config/ci_linux.json index 23aee75ba7..6279d3f89d 100644 --- a/scripts/config/ci_linux.json +++ b/scripts/config/ci_linux.json @@ -1,6 +1,6 @@ { "afspPath": "not_needed", - "utilPath": "not_needed", + "utilPath": "/tools", "inpaths": { "MONO": "/usr/local/testv/test_mono.wav", "STEREO": "/usr/local/testv/test_stereo.wav", -- GitLab From d770d532a0469338a8d4c12c05f4be391148e47d Mon Sep 17 00:00:00 2001 From: knj Date: Wed, 7 Sep 2022 12:59:08 +0200 Subject: [PATCH 20/28] define env vars in ci file for testing --- .gitlab-ci.yml | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index b07afcb36f..c87ab86cb7 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1,6 +1,9 @@ variables: TESTV_DIR: "/usr/local/testv" BUILD_OUTPUT: "build_output.txt" + SANITIZER_TEST_IN_FMT: "stereo" + SANITIZER_TEST_OUT_FMTS: "stereo" + SANITIZER_TEST_TESTS: "CLANG1" # This sets when pipelines are created. Jobs have more specific rules to restrict them. @@ -422,21 +425,18 @@ codec-comparison-on-main-push: sanitizer-test-on-main-scheduled: extends: - - .rules-merge-request - .test-job-linux-needs-testv-dir tags: - test-fhg-linux-runner1 stage: test - #rules: + rules: # only run in scheduled pipeline that passes this env var - #- if: $SANITIZER_TEST_IN_FMT && $SANITIZER_TEST_OUT_FMTS && $SANITIZER_TEST_TESTS - #- if: $SANITIZER_TEST_IN_FMT + - if: $SANITIZER_TEST_IN_FMT && $SANITIZER_TEST_OUT_FMTS && $SANITIZER_TEST_TESTS script: - *print-common-info - echo "Running scheduled sanitizer" - - echo "$SANITIZER_TEST_IN_FMT && $SANITIZER_TEST_OUT_FMTS && $SANITIZER_TEST_TESTS" - #- python3 ci/run_scheduled_sanitizer_test.py $SANITIZER_TEST_IN_FMT $SANITIZER_TEST_OUT_FMTS $SANITIZER_TEST_TESTS - - python3 ci/run_scheduled_sanitizer_test.py stereo stereo CLANG1 + - echo " $SANITIZER_TEST_IN_FMT && $SANITIZER_TEST_OUT_FMTS && $SANITIZER_TEST_TESTS " + - python3 ci/run_scheduled_sanitizer_test.py $SANITIZER_TEST_IN_FMT $SANITIZER_TEST_OUT_FMTS $SANITIZER_TEST_TESTS # --------------------------------------------------------------- -- GitLab From ffd01bed0d37d54c1a99bd69730bcadd0f0d40f6 Mon Sep 17 00:00:00 2001 From: knj Date: Wed, 7 Sep 2022 13:02:10 +0200 Subject: [PATCH 21/28] remove variable definition again --- .gitlab-ci.yml | 3 --- 1 file changed, 3 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index c87ab86cb7..740d85a9e4 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1,9 +1,6 @@ variables: TESTV_DIR: "/usr/local/testv" BUILD_OUTPUT: "build_output.txt" - SANITIZER_TEST_IN_FMT: "stereo" - SANITIZER_TEST_OUT_FMTS: "stereo" - SANITIZER_TEST_TESTS: "CLANG1" # This sets when pipelines are created. Jobs have more specific rules to restrict them. -- GitLab From e4aa8599083d2741f4e235e8148695541fdcc91f Mon Sep 17 00:00:00 2001 From: knj Date: Wed, 7 Sep 2022 13:07:53 +0200 Subject: [PATCH 22/28] allow scheduled pipelines on all branches --- .gitlab-ci.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 740d85a9e4..9124d86a2c 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -11,7 +11,9 @@ workflow: when: never - if: $CI_PIPELINE_SOURCE == 'merge_request_event' # Runs for merge requests - if: $CI_PIPELINE_SOURCE == 'push' && $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH # Pushes to main - - if: $CI_PIPELINE_SOURCE == 'schedule' && $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH # Scheduled in main + # only temporary + # - if: $CI_PIPELINE_SOURCE == 'schedule' && $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH # Scheduled in main + - if: $CI_PIPELINE_SOURCE == 'schedule' stages: -- GitLab From 1e493ebaefb27767091e8a7dbcfc56fb36a25e7e Mon Sep 17 00:00:00 2001 From: knj Date: Wed, 7 Sep 2022 13:18:51 +0200 Subject: [PATCH 23/28] add debug output --- ci/run_scheduled_sanitizer_test.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/ci/run_scheduled_sanitizer_test.py b/ci/run_scheduled_sanitizer_test.py index 3d371a276d..2ff0b52104 100644 --- a/ci/run_scheduled_sanitizer_test.py +++ b/ci/run_scheduled_sanitizer_test.py @@ -51,6 +51,7 @@ def run_check(modes: list, out_formats: list, tests: list, run_fec: bool = True) "--oc", *out_formats, ] + print(cmd_no_fec) proc = subprocess.Popen(cmd_no_fec, stdout=subprocess.PIPE, stderr=subprocess.PIPE) for c in iter(lambda: proc.stdout.read(1), b""): @@ -79,6 +80,7 @@ def run_check(modes: list, out_formats: list, tests: list, run_fec: bool = True) fi.unlink() cmd_fec = cmd_no_fec + ["--decoder_only", "-f", EP_FILE] + print(cmd_fec) proc = subprocess.Popen(cmd_fec, stdout=subprocess.PIPE, stderr=subprocess.PIPE) for c in iter(lambda: proc.stdout.read(1), b""): -- GitLab From 896a26206c540685e862b2482bb0b66a4c5670aa Mon Sep 17 00:00:00 2001 From: knj Date: Wed, 7 Sep 2022 13:31:07 +0200 Subject: [PATCH 24/28] fix command line handling --- ci/run_scheduled_sanitizer_test.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/run_scheduled_sanitizer_test.py b/ci/run_scheduled_sanitizer_test.py index 2ff0b52104..b52bc72fb3 100644 --- a/ci/run_scheduled_sanitizer_test.py +++ b/ci/run_scheduled_sanitizer_test.py @@ -103,7 +103,7 @@ if __name__ == "__main__": parser = argparse.ArgumentParser() parser.add_argument("in_format", type=str) parser.add_argument("out_formats", type=str, nargs="+") - parser.add_argument("tests", type=str, nargs="+") + parser.add_argument("--tests", type=str, nargs="+", default=["CLANG1", "CLANG2"]) parser.add_argument("--skip_fec", action="store_true") sys.exit(main(parser.parse_args())) \ No newline at end of file -- GitLab From ff56aa1964f08007633983e23ac2ef21c6b6f591 Mon Sep 17 00:00:00 2001 From: knj Date: Wed, 7 Sep 2022 13:37:02 +0200 Subject: [PATCH 25/28] add input validation and call script correctly in ci file --- .gitlab-ci.yml | 2 +- ci/run_scheduled_sanitizer_test.py | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 9124d86a2c..3a0ce7ae10 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -435,7 +435,7 @@ sanitizer-test-on-main-scheduled: - *print-common-info - echo "Running scheduled sanitizer" - echo " $SANITIZER_TEST_IN_FMT && $SANITIZER_TEST_OUT_FMTS && $SANITIZER_TEST_TESTS " - - python3 ci/run_scheduled_sanitizer_test.py $SANITIZER_TEST_IN_FMT $SANITIZER_TEST_OUT_FMTS $SANITIZER_TEST_TESTS + - python3 ci/run_scheduled_sanitizer_test.py $SANITIZER_TEST_IN_FMT $SANITIZER_TEST_OUT_FMTS --tests $SANITIZER_TEST_TESTS # --------------------------------------------------------------- diff --git a/ci/run_scheduled_sanitizer_test.py b/ci/run_scheduled_sanitizer_test.py index b52bc72fb3..481ae56473 100644 --- a/ci/run_scheduled_sanitizer_test.py +++ b/ci/run_scheduled_sanitizer_test.py @@ -21,6 +21,8 @@ def main(args): tests = args.tests run_fec = not args.skip_fec + assert all([t in SUPPORTED_TESTS for t in tests]) + modes = get_modes(in_format) returncode = run_check(modes, out_formats, tests, run_fec=run_fec) -- GitLab From 505f05b1097ffdafc2aaba217e335ef8f63d56b7 Mon Sep 17 00:00:00 2001 From: knj Date: Wed, 7 Sep 2022 14:40:33 +0200 Subject: [PATCH 26/28] prevent non-schedule jobs from being run on scheduled pipelines --- .gitlab-ci.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 3a0ce7ae10..42b1eca27c 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -53,6 +53,8 @@ stages: rules: - if: $MIRROR_ACCESS_TOKEN # Don't run in the mirror update pipeline (only then MIRROR_ACCESS_TOKEN is defined) when: never + - if: $CI_PIPELINE_SOURCE == 'schedule' # Don't run in any scheduled pipelines by default (use schedule templates below to enable again for certain conditions) + when: never - when: on_success .rules-merge-request: -- GitLab From 1ce7f7eaf80fce1d9ce7131e5b15ac6eeed8b33a Mon Sep 17 00:00:00 2001 From: knj Date: Wed, 7 Sep 2022 15:30:24 +0200 Subject: [PATCH 27/28] add template to job and comment --- .gitlab-ci.yml | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 42b1eca27c..fa435f8a8a 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -424,14 +424,21 @@ codec-comparison-on-main-push: junit: report-junit.xml +# parameterizable job for sanitizer tests per format +# how to set up: create a schedule (CI/CD -> schedules) and enter the respective values for the environment variables: +# - SANITIZER_TEST_IN_FMT: input format +# - SANITIZER_TEST_OUT_FMTS: list of output formats, blank-separated, e.g.: stereo mono 5_1 +# - SANITIZER_TEST_TESTS: list of checks to do, can be one of CLANG1, CLANG2, CLANG3, VALGRIND sanitizer-test-on-main-scheduled: extends: - .test-job-linux-needs-testv-dir + # this next one is maybe not really needed, since there is the rule checking for the existence of the env vars below, but use for clarity + - .rules-main-scheduled tags: - - test-fhg-linux-runner1 + - sanitizer_test_main stage: test rules: - # only run in scheduled pipeline that passes this env var + # only run in scheduled pipeline that passes this env vars - if: $SANITIZER_TEST_IN_FMT && $SANITIZER_TEST_OUT_FMTS && $SANITIZER_TEST_TESTS script: - *print-common-info -- GitLab From 3cd654c016807132cad4268f59dc7b11ab217a47 Mon Sep 17 00:00:00 2001 From: knj Date: Thu, 8 Sep 2022 17:31:44 +0200 Subject: [PATCH 28/28] final cleanup --- .gitlab-ci.yml | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index fa435f8a8a..7207ded343 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -11,9 +11,7 @@ workflow: when: never - if: $CI_PIPELINE_SOURCE == 'merge_request_event' # Runs for merge requests - if: $CI_PIPELINE_SOURCE == 'push' && $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH # Pushes to main - # only temporary - # - if: $CI_PIPELINE_SOURCE == 'schedule' && $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH # Scheduled in main - - if: $CI_PIPELINE_SOURCE == 'schedule' + - if: $CI_PIPELINE_SOURCE == 'schedule' && $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH # Scheduled in main stages: -- GitLab