diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index c4a429c75d2ac2ff575d112a2aeea6ada7c1c4d7..7192aebe439e5dc593afbbd4300914b360059b7d 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -20,6 +20,7 @@ workflow: - if: $CI_PIPELINE_SOURCE == 'merge_request_event' # Runs for merge requests - if: $CI_PIPELINE_SOURCE == 'push' && $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH # Pushes to main - if: $CI_PIPELINE_SOURCE == 'schedule' && $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH # Scheduled in main + - if: $CI_PIPELINE_SOURCE == "web" stages: - maintenance @@ -722,8 +723,8 @@ codec-comparison-on-main-push: when: always paths: - ep_015.g192 - # second wildcard is necessary to get encoder and no-PLC run logs - - "CLANG*/logs*" + - ./LOGS_PLC + - ./LOGS_noPLC ### --- sanitizer schedule A --- diff --git a/ci/collect_artifacts.py b/ci/collect_artifacts.py new file mode 100755 index 0000000000000000000000000000000000000000..d3228ba2e85e224fa7fb9fdc6415036b6c77f982 --- /dev/null +++ b/ci/collect_artifacts.py @@ -0,0 +1,103 @@ +#! /usr/bin/env python3 +import pathlib +import argparse +import re + + +TEST_TYPES = ["sanitizers"] + + +def main(args): + + test = args.test + file = args.console_out_file + if test == "sanitizers": + collect_for_sanitizer_test(file) + + +def find_failed_files_for_sanitizer_test( + console_log: list, subfolder: str, which="LOGS" +) -> dict(): + + assert which in ["LOGS", "FILE_BASENAMES"] + + pattern_line = "(Encoding|Decoding) failed .*for \/.*(CLANG.|VALGRIND)\/(.*)" + pattern_file = "(.*_b[0-9]*_.*_rs|.*_b[0-9]*_.*_cbr).*" + + files_found = dict() + for line in console_log: + m_line = re.match(pattern_line, line) + + if m_line is not None: + _, test, filename = m_line.groups() + filename = pathlib.Path(filename).name + m_file = re.match(pattern_file, filename) + if m_file is None: + print(f"Unexpected: no match on {filename} with {pattern_file} - skip") + continue + filename_start = m_file.groups()[0] + + if which == "LOGS": + folder = pathlib.Path(f"{test}/{subfolder}/") + files = [ + f for f in folder.iterdir() if f.name.startswith(filename_start) + ] + elif which == "FILE_BASENAMES": + files = [filename_start] + if test in files_found: + files_found[test].extend(files) + else: + files_found[test] = files + + return files_found + + +def collect_for_sanitizer_test(file): + + with open(file) as f: + console_log = f.readlines() + + files_to_archive_noPLC = find_failed_files_for_sanitizer_test( + console_log, "logs_noPLC" + ) + files_to_archive = find_failed_files_for_sanitizer_test(console_log, "logs") + + log_folder = pathlib.Path("./LOGS_PLC") + log_folder.mkdir() + for test in files_to_archive.keys(): + log_folder.joinpath(test).mkdir() + for test, files in files_to_archive.items(): + folder = log_folder.joinpath(test) + for p in files: + source = pathlib.Path(p) + target = folder.joinpath(source.name) + source.rename(target) + + log_folder_noPLC = pathlib.Path("./LOGS_noPLC") + log_folder_noPLC.mkdir() + for test in files_to_archive_noPLC.keys(): + log_folder_noPLC.joinpath(test).mkdir() + for test, files in files_to_archive_noPLC.items(): + folder = log_folder_noPLC.joinpath(test) + for p in files: + source = pathlib.Path(p) + target = folder.joinpath(source.name) + source.rename(target) + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument( + "test", + type=str, + choices=TEST_TYPES, + help="for which test should artifacts be collected?", + ) + parser.add_argument( + "console_out_file", + type=str, + help="file with stdout from IvasBuildAndRunChecks.py", + ) + args = parser.parse_args() + + main(args) diff --git a/ci/run_scheduled_sanitizer_test.py b/ci/run_scheduled_sanitizer_test.py old mode 100644 new mode 100755 index 51e1e73eecaba64426450e069b168810c522eb44..686e61822cf92850ced2624f2388bcaab6ba9575 --- a/ci/run_scheduled_sanitizer_test.py +++ b/ci/run_scheduled_sanitizer_test.py @@ -5,6 +5,13 @@ import sys import subprocess import pathlib +CI_SCRIPT_DIR = "./ci" +sys.path.append(CI_SCRIPT_DIR) +from collect_artifacts import ( + find_failed_files_for_sanitizer_test, + collect_for_sanitizer_test, +) + DURATION = "120" CFG = "ci_linux_ltv.json" @@ -16,6 +23,8 @@ MC_MODES = ["5_1", "5_1_2", "5_1_4", "7_1", "7_1_4"] SCRIPT_DIR = pathlib.Path("./scripts").resolve() +CONSOLE_OUT_FILE = "output_san.txt" + def main(args): in_format = args.in_format @@ -28,17 +37,19 @@ def main(args): modes = get_modes(in_format) returncode = run_check(modes, out_formats, tests, run_fec=run_fec) + collect_for_sanitizer_test(CONSOLE_OUT_FILE) + sys.exit(returncode) def get_modes(in_format: str) -> list: cmd = [ - SCRIPT_DIR.joinpath("runIvasCodec.py"), - "-C", - "MC" if in_format in MC_MODES else in_format, - "-l" - ] + SCRIPT_DIR.joinpath("runIvasCodec.py"), + "-C", + "MC" if in_format in MC_MODES else in_format, + "-l", + ] list_process = subprocess.run(cmd, capture_output=True) output = list_process.stdout.decode("utf8") @@ -74,21 +85,43 @@ def run_check(modes: list, out_formats: list, tests: list, run_fec: bool = True) *out_formats, ] - print("======== Script command line WITHOUT plc: ========\n{}".format(" ".join(cmd_no_fec))) - - proc = subprocess.Popen(cmd_no_fec, stdout=subprocess.PIPE, stderr=subprocess.PIPE) - for c in iter(lambda: proc.stdout.read(1), b""): - sys.stdout.buffer.write(c) - proc.wait() - - if proc.returncode not in [0, 101]: - raise IvasBuildAndRunFailed("Failed at first run (no PLC)") + print( + "======== Script command line WITHOUT plc: ========\n{}".format( + " ".join(cmd_no_fec) + ) + ) + + with open(CONSOLE_OUT_FILE, "a") as f: + proc = subprocess.Popen( + cmd_no_fec, stdout=subprocess.PIPE, stderr=subprocess.PIPE + ) + for c in iter(lambda: proc.stdout.read(1), b""): + sys.stdout.buffer.write(c) + f.write(c.decode("utf8")) + proc.wait() returncode_no_fec = proc.returncode + print("returncode_no_fec:", returncode_no_fec) + if returncode_no_fec not in [0, 101]: + raise IvasBuildAndRunFailed("Failed at first run (no PLC)") if not run_fec: return returncode_no_fec + # delete bitstream files for all failed modes to prevent follow-up errors in decoder-only run + with open(CONSOLE_OUT_FILE) as f: + console_log = f.readlines() + failed_files = find_failed_files_for_sanitizer_test( + console_log, "logs", "FILE_BASENAMES" + ) + for t in failed_files.keys(): + bs_folder = pathlib.Path(f"{t}/enc") + file_starts = failed_files[t] + for f in bs_folder.iterdir(): + for fs in file_starts: + if f.name.startswith(fs): + f.unlink() + ### second run: decoder only with disturbed bitstream # generate error pattern @@ -111,7 +144,11 @@ def run_check(modes: list, out_formats: list, tests: list, run_fec: bool = True) path.mkdir() cmd_fec = cmd_no_fec + ["--decoder_only", "-f", EP_FILE] - print("======== Script command line WITH plc: ========\n{}".format(" ".join(cmd_no_fec))) + print( + "======== Script command line WITH plc: ========\n{}".format( + " ".join(cmd_no_fec) + ) + ) proc = subprocess.Popen(cmd_fec, stdout=subprocess.PIPE, stderr=subprocess.PIPE) for c in iter(lambda: proc.stdout.read(1), b""): @@ -119,6 +156,7 @@ def run_check(modes: list, out_formats: list, tests: list, run_fec: bool = True) proc.wait() returncode_fec = proc.returncode + print("returncode_fec:", returncode_fec) if returncode_fec not in [0, 101]: raise IvasBuildAndRunFailed("failed at second run (PLC)") diff --git a/scripts/IvasBuildAndRunChecks.py b/scripts/IvasBuildAndRunChecks.py index 1cf4a868db1f197b28a21fee3170cbacb193a19b..0b2a2932c010357aa66d3c4d3d343a74fff1d62d 100755 --- a/scripts/IvasBuildAndRunChecks.py +++ b/scripts/IvasBuildAndRunChecks.py @@ -193,15 +193,15 @@ class IvasBuildAndRunChecks(IvasScriptsCommon.IvasScript): self.args["create_complexity_tables"] ) + returncode = 0 for check in checks: runner = br.build_and_run_dict[check]["runner"] failed_encs = runner.failed_modes["enc"] failed_decs = runner.failed_modes["dec"] if len(failed_encs) > 0 or len(failed_decs) > 0: - return RET_CODE_FAILURE - else: - return 0 + returncode = RET_CODE_FAILURE + return returncode if __name__ == "__main__": script = IvasBuildAndRunChecks()