diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 56420ae02e377767c88b7f388b6ae77d379474bf..7207ded34333decfc83ef9e4308bce7b05a6d1bb 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -51,6 +51,8 @@ stages: rules: - if: $MIRROR_ACCESS_TOKEN # Don't run in the mirror update pipeline (only then MIRROR_ACCESS_TOKEN is defined) when: never + - if: $CI_PIPELINE_SOURCE == 'schedule' # Don't run in any scheduled pipelines by default (use schedule templates below to enable again for certain conditions) + when: never - when: on_success .rules-merge-request: @@ -420,16 +422,27 @@ codec-comparison-on-main-push: junit: report-junit.xml +# parameterizable job for sanitizer tests per format +# how to set up: create a schedule (CI/CD -> schedules) and enter the respective values for the environment variables: +# - SANITIZER_TEST_IN_FMT: input format +# - SANITIZER_TEST_OUT_FMTS: list of output formats, blank-separated, e.g.: stereo mono 5_1 +# - SANITIZER_TEST_TESTS: list of checks to do, can be one of CLANG1, CLANG2, CLANG3, VALGRIND sanitizer-test-on-main-scheduled: - extends: .test-job-linux-needs-testv-dir + extends: + - .test-job-linux-needs-testv-dir + # this next one is maybe not really needed, since there is the rule checking for the existence of the env vars below, but use for clarity + - .rules-main-scheduled + tags: + - sanitizer_test_main stage: test rules: - # only run in scheduled pipeline that passes this env var - - if: $SANITIZER_TEST_IN_FMT + # only run in scheduled pipeline that passes this env vars + - if: $SANITIZER_TEST_IN_FMT && $SANITIZER_TEST_OUT_FMTS && $SANITIZER_TEST_TESTS script: - *print-common-info - echo "Running scheduled sanitizer" - # - python3 ci/run_scheduled_sanitizer_test.py $SANITIZER_TEST_IN_FMT $SANITIZER_TEST_OUT_FMTS + - echo " $SANITIZER_TEST_IN_FMT && $SANITIZER_TEST_OUT_FMTS && $SANITIZER_TEST_TESTS " + - python3 ci/run_scheduled_sanitizer_test.py $SANITIZER_TEST_IN_FMT $SANITIZER_TEST_OUT_FMTS --tests $SANITIZER_TEST_TESTS # --------------------------------------------------------------- diff --git a/ci/run_scheduled_sanitizer_test.py b/ci/run_scheduled_sanitizer_test.py new file mode 100644 index 0000000000000000000000000000000000000000..481ae56473b9993af0ee6fa44747467239bc1d8c --- /dev/null +++ b/ci/run_scheduled_sanitizer_test.py @@ -0,0 +1,111 @@ +#!/usr/bin/env python3 + +import argparse +import sys +import subprocess +import pathlib + + +DURATION = "120" +CFG = "ci_linux.json" +SUPPORTED_TESTS = [ "CLANG1", "CLANG2", "CLANG3", "VALGRIND" ] +EP_FILE = "ep_015.g192" +GENPATT_CMD = f"gen-patt -tailstat -fer -g192 -gamma 0 -rate 0.15 -tol 0.001 -reset -n {int(DURATION) * 50} {EP_FILE}" +EIDXOR_CMD = "eid-xor -vbr -fer {bitstream} {ep_file} {out_file}" + +SCRIPT_DIR = pathlib.Path("./scripts").resolve() + +def main(args): + in_format = args.in_format + out_formats = args.out_formats + tests = args.tests + run_fec = not args.skip_fec + + assert all([t in SUPPORTED_TESTS for t in tests]) + + modes = get_modes(in_format) + returncode = run_check(modes, out_formats, tests, run_fec=run_fec) + + sys.exit(returncode) + + +def get_modes(in_format: str) -> list: + cmd = [SCRIPT_DIR.joinpath("runIvasCodec.py"), "-l"] + list_process = subprocess.run(cmd, capture_output=True) + + output = list_process.stdout.decode("utf8") + return [m for m in output.splitlines() if in_format in m] + + +def run_check(modes: list, out_formats: list, tests: list, run_fec: bool = True): + + ### always run encoder and decoder with no frameloss + cmd_no_fec = [ + str(SCRIPT_DIR.joinpath("IvasBuildAndRunChecks.py")), + "-U", + DURATION, + "-p", + CFG, + "--checks", + *tests, + "-m", + *modes, + "--oc", + *out_formats, + ] + print(cmd_no_fec) + + proc = subprocess.Popen(cmd_no_fec, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + for c in iter(lambda: proc.stdout.read(1), b""): + sys.stdout.buffer.write(c) + proc.wait() + + if proc.returncode not in [0, 101]: + raise IvasBuildAndRunFailed("Failed at first run (no PLC)") + + returncode_no_fec = proc.returncode + + if not run_fec: + return returncode_no_fec + + ### second run: decoder only with disturbed bitstream + + # generate error pattern + subprocess.call(GENPATT_CMD.split()) + + # cleanup to avoid script errors + # we want "logs" and "dec" subfolders to be empty -> delete and recreate them + cleanup_folders = ["logs", "dec"] + for t in tests: + for fol in cleanup_folders: + for fi in pathlib.Path(t).joinpath(fol).iterdir(): + fi.unlink() + + cmd_fec = cmd_no_fec + ["--decoder_only", "-f", EP_FILE] + print(cmd_fec) + + proc = subprocess.Popen(cmd_fec, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + for c in iter(lambda: proc.stdout.read(1), b""): + sys.stdout.buffer.write(c) + proc.wait() + + returncode_fec = proc.returncode + + if returncode_fec not in [0, 101]: + raise IvasBuildAndRunFailed("failed at second run (PLC)") + + return 101 if 101 in [returncode_no_fec, returncode_fec] else 0 + + +class IvasBuildAndRunFailed(Exception): + pass + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument("in_format", type=str) + parser.add_argument("out_formats", type=str, nargs="+") + parser.add_argument("--tests", type=str, nargs="+", default=["CLANG1", "CLANG2"]) + parser.add_argument("--skip_fec", action="store_true") + + sys.exit(main(parser.parse_args())) \ No newline at end of file diff --git a/scripts/IvasBuildAndRunChecks.py b/scripts/IvasBuildAndRunChecks.py index fe1953a6030a99fbd9af431ff374cee97d41dbc1..1cf4a868db1f197b28a21fee3170cbacb193a19b 100755 --- a/scripts/IvasBuildAndRunChecks.py +++ b/scripts/IvasBuildAndRunChecks.py @@ -36,7 +36,9 @@ import sys from pyivastest.IvasSvnBuilder import * from pyivastest import IvasScriptsCommon import pyivastest.constants as constants -from pyivastest import ivas_svn + + +RET_CODE_FAILURE = 101 class IvasBuildAndRunChecks(IvasScriptsCommon.IvasScript): @@ -169,14 +171,10 @@ class IvasBuildAndRunChecks(IvasScriptsCommon.IvasScript): for check in checks: br.run(check) if self.args["create_html_output"]: - revision = ivas_svn.get_local_svn_info(self.args["srcdir"], self.logger) - if revision is None: - print("Could not get revision from local copy") - revision = -1 - else: - revision = revision["commit_revision"] + cmd = ["git", "rev-parse", "HEAD"] + commit_hash = subprocess.run(cmd, capture_output=True).stdout.decode("utf8") br.build_and_run_dict[check]["analyzer"].write_html_file( - check, self.args["create_html_output"], revision + check, self.args["create_html_output"], commit_hash ) for r in br.build_and_run_dict[check]["runner"].results: self.logger.console(r[0]) @@ -195,7 +193,16 @@ class IvasBuildAndRunChecks(IvasScriptsCommon.IvasScript): self.args["create_complexity_tables"] ) + for check in checks: + runner = br.build_and_run_dict[check]["runner"] + failed_encs = runner.failed_modes["enc"] + failed_decs = runner.failed_modes["dec"] + if len(failed_encs) > 0 or len(failed_decs) > 0: + return RET_CODE_FAILURE + else: + return 0 + if __name__ == "__main__": script = IvasBuildAndRunChecks() - script.run() + sys.exit(script.run()) diff --git a/scripts/config/ci_linux.json b/scripts/config/ci_linux.json index 23aee75ba7f2daeef62481c518216a7f389a29df..6279d3f89d2493ff586bc4064d50e26bfadd623f 100644 --- a/scripts/config/ci_linux.json +++ b/scripts/config/ci_linux.json @@ -1,6 +1,6 @@ { "afspPath": "not_needed", - "utilPath": "not_needed", + "utilPath": "/tools", "inpaths": { "MONO": "/usr/local/testv/test_mono.wav", "STEREO": "/usr/local/testv/test_stereo.wav", diff --git a/scripts/pyivastest/IvasModeRunner.py b/scripts/pyivastest/IvasModeRunner.py index 246e8139ad9775cba9766c92a59da42b81ec28fe..63bf58cb59f8f1965da45b118af5b78285f804af 100755 --- a/scripts/pyivastest/IvasModeRunner.py +++ b/scripts/pyivastest/IvasModeRunner.py @@ -638,7 +638,7 @@ class IvasModeRunner(IvasModeCollector.IvasModeCollector): self.lock.acquire() os.remove(pcm_name_lock) # os.remove(pcm_name_res_tmp) - if do_limit_duration: + if do_limit_duration and cut_len_samples < in_len: os.remove(pcm_name_cpy_tmp) self.logger.info( "PCM file {} successfully created!".format(pcm_name)