From 2ee7ea13b8a665dc872ab31df307a10875661695 Mon Sep 17 00:00:00 2001 From: knj Date: Wed, 19 Oct 2022 11:05:14 +0200 Subject: [PATCH 01/33] add scripts for artifact collection --- ci/collect_artifacts.py | 66 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 66 insertions(+) create mode 100644 ci/collect_artifacts.py diff --git a/ci/collect_artifacts.py b/ci/collect_artifacts.py new file mode 100644 index 0000000000..616c4c5c6b --- /dev/null +++ b/ci/collect_artifacts.py @@ -0,0 +1,66 @@ +import pathlib +import argparse +import re + + +TEST_TYPES = ["sanitizers"] + + +def main(args): + + test = args.test + file = args.console_out_file + if test == "sanitizers": + collect_for_sanitizer_test(file) + + +def collect_for_sanitizer_test(file): + + with open(file) as f: + console_log = f.readlines() + + files_to_archive = list() + + pattern_line = "(Encoding|Decoding) failed .*for \/.*(CLANG.|VALGRIND)\/(.*)" + pattern_file = "(.*_b[1-9]*_.*_rs|.*_b[1-9]*_.*_cbr).*" + + for line in console_log: + m_line = re.match(pattern_line, line) + + if m_line is not None: + _, test, filename = m_line.groups() + filename = pathlib.Path(filename).name + m_file = re.match(pattern_file, filename) + filename_start = m_file.groups() + + files = [ + f + for f in pathlib.Path(f"{test}/logs/").iterdir() + if f.name.startswith(filename_start) + ] + files_to_archive.extend(files) + + log_folder = pathlib.Path("./LOGS") + log_folder.mkdir() + for p in files_to_archive: + source = pathlib.Path(p) + target = log_folder.joinpath(source.name) + source.rename(target) + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument( + "test", + type=str, + choices=TEST_TYPES, + help="for which test should artifacts be collected?", + ) + parser.add_argument( + "console_out_file", + type=str, + help="file with stdout from IvasBuildAndRunChecks.py", + ) + args = parser.parse_args() + + main(args) -- GitLab From 243b70f81a8ad5ad7fd4694d7fb253bed5b77300 Mon Sep 17 00:00:00 2001 From: knj Date: Wed, 19 Oct 2022 11:09:46 +0200 Subject: [PATCH 02/33] use artifact collection script in wrapper script --- ci/run_scheduled_sanitizer_test.py | 30 +++++++++++++++++++++++------- 1 file changed, 23 insertions(+), 7 deletions(-) diff --git a/ci/run_scheduled_sanitizer_test.py b/ci/run_scheduled_sanitizer_test.py index 4f3cd25c0b..f0f16d6776 100644 --- a/ci/run_scheduled_sanitizer_test.py +++ b/ci/run_scheduled_sanitizer_test.py @@ -16,6 +16,9 @@ MC_MODES = ["5_1", "5_1_2", "5_1_4", "7_1", "7_1_4"] SCRIPT_DIR = pathlib.Path("./scripts").resolve() +COLLECT_ARTIFACTS_SCRIPT = "ci/collect_artifacts.py" +CONSOLE_OUT_FILE = "output_san.txt" + def main(args): in_format = args.in_format @@ -28,17 +31,19 @@ def main(args): modes = get_modes(in_format) returncode = run_check(modes, out_formats, tests, run_fec=run_fec) + cmd_collect_artifacts = [COLLECT_ARTIFACTS_SCRIPT, "sanitizers", CONSOLE_OUT_FILE] + sys.exit(returncode) def get_modes(in_format: str) -> list: cmd = [ - SCRIPT_DIR.joinpath("runIvasCodec.py"), - "-C", - "MC" if in_format in MC_MODES else in_format, - "-l" - ] + SCRIPT_DIR.joinpath("runIvasCodec.py"), + "-C", + "MC" if in_format in MC_MODES else in_format, + "-l", + ] list_process = subprocess.run(cmd, capture_output=True) output = list_process.stdout.decode("utf8") @@ -70,9 +75,16 @@ def run_check(modes: list, out_formats: list, tests: list, run_fec: bool = True) *modes, "--oc", *out_formats, + "|", + "tee", + CONSOLE_OUT_FILE, ] - print("======== Script command line WITHOUT plc: ========\n{}".format(" ".join(cmd_no_fec))) + print( + "======== Script command line WITHOUT plc: ========\n{}".format( + " ".join(cmd_no_fec) + ) + ) proc = subprocess.Popen(cmd_no_fec, stdout=subprocess.PIPE, stderr=subprocess.PIPE) for c in iter(lambda: proc.stdout.read(1), b""): @@ -109,7 +121,11 @@ def run_check(modes: list, out_formats: list, tests: list, run_fec: bool = True) path.mkdir() cmd_fec = cmd_no_fec + ["--decoder_only", "-f", EP_FILE] - print("======== Script command line WITH plc: ========\n{}".format(" ".join(cmd_no_fec))) + print( + "======== Script command line WITH plc: ========\n{}".format( + " ".join(cmd_no_fec) + ) + ) proc = subprocess.Popen(cmd_fec, stdout=subprocess.PIPE, stderr=subprocess.PIPE) for c in iter(lambda: proc.stdout.read(1), b""): -- GitLab From ed8dd7785627f068bcf121229054324edf0a91ae Mon Sep 17 00:00:00 2001 From: knj Date: Wed, 19 Oct 2022 11:10:44 +0200 Subject: [PATCH 03/33] add new artifact dir to sanitizer test --- .gitlab-ci.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 0b2b3ddff9..cfe85f0ee3 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -501,8 +501,7 @@ codec-comparison-on-main-push: when: always paths: - ep_015.g192 - # second wildcard is necessary to get encoder and no-PLC run logs - - "CLANG*/logs*" + - ./LOGS sanitizer-test-mono: extends: .sanitizer-test-template -- GitLab From 3df389a5495eace369215bb5fe4bb241815f3f1a Mon Sep 17 00:00:00 2001 From: knj Date: Wed, 19 Oct 2022 11:13:45 +0200 Subject: [PATCH 04/33] change rules temporarily for testing --- .gitlab-ci.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index cfe85f0ee3..02db1102c4 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -19,6 +19,7 @@ workflow: - if: $CI_PIPELINE_SOURCE == 'merge_request_event' # Runs for merge requests - if: $CI_PIPELINE_SOURCE == 'push' && $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH # Pushes to main - if: $CI_PIPELINE_SOURCE == 'schedule' && $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH # Scheduled in main + - if: $CI_PIPELINE_SOURCE == "web" stages: - maintenance -- GitLab From 59e95b81b2e5359e0789651bf06d3ccc5fb9b840 Mon Sep 17 00:00:00 2001 From: knj Date: Wed, 19 Oct 2022 11:20:48 +0200 Subject: [PATCH 05/33] do not use pipe in subprocess cmd --- ci/run_scheduled_sanitizer_test.py | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/ci/run_scheduled_sanitizer_test.py b/ci/run_scheduled_sanitizer_test.py index f0f16d6776..7638d43452 100644 --- a/ci/run_scheduled_sanitizer_test.py +++ b/ci/run_scheduled_sanitizer_test.py @@ -75,9 +75,6 @@ def run_check(modes: list, out_formats: list, tests: list, run_fec: bool = True) *modes, "--oc", *out_formats, - "|", - "tee", - CONSOLE_OUT_FILE, ] print( @@ -86,10 +83,12 @@ def run_check(modes: list, out_formats: list, tests: list, run_fec: bool = True) ) ) - proc = subprocess.Popen(cmd_no_fec, stdout=subprocess.PIPE, stderr=subprocess.PIPE) - for c in iter(lambda: proc.stdout.read(1), b""): - sys.stdout.buffer.write(c) - proc.wait() + with open(CONSOLE_OUT_FILE, "a") as f: + proc = subprocess.Popen(cmd_no_fec, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + for c in iter(lambda: proc.stdout.read(1), b""): + sys.stdout.buffer.write(c) + f.write(c) + proc.wait() if proc.returncode not in [0, 101]: raise IvasBuildAndRunFailed("Failed at first run (no PLC)") -- GitLab From 04eee7ddaed479553d8ce2321ea4a2865f245fb5 Mon Sep 17 00:00:00 2001 From: knj Date: Fri, 28 Oct 2022 15:25:16 +0200 Subject: [PATCH 06/33] fix type error --- ci/run_scheduled_sanitizer_test.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/run_scheduled_sanitizer_test.py b/ci/run_scheduled_sanitizer_test.py index 7638d43452..80a746bf09 100644 --- a/ci/run_scheduled_sanitizer_test.py +++ b/ci/run_scheduled_sanitizer_test.py @@ -87,7 +87,7 @@ def run_check(modes: list, out_formats: list, tests: list, run_fec: bool = True) proc = subprocess.Popen(cmd_no_fec, stdout=subprocess.PIPE, stderr=subprocess.PIPE) for c in iter(lambda: proc.stdout.read(1), b""): sys.stdout.buffer.write(c) - f.write(c) + f.write(c.decode("utf8")) proc.wait() if proc.returncode not in [0, 101]: -- GitLab From 3a2aebf5c6f6caed38233a569308093bb2e0976e Mon Sep 17 00:00:00 2001 From: knj Date: Fri, 28 Oct 2022 15:31:48 +0200 Subject: [PATCH 07/33] actually run artifact collectionscript --- ci/run_scheduled_sanitizer_test.py | 1 + 1 file changed, 1 insertion(+) diff --git a/ci/run_scheduled_sanitizer_test.py b/ci/run_scheduled_sanitizer_test.py index 80a746bf09..e572c550b0 100644 --- a/ci/run_scheduled_sanitizer_test.py +++ b/ci/run_scheduled_sanitizer_test.py @@ -32,6 +32,7 @@ def main(args): returncode = run_check(modes, out_formats, tests, run_fec=run_fec) cmd_collect_artifacts = [COLLECT_ARTIFACTS_SCRIPT, "sanitizers", CONSOLE_OUT_FILE] + subprocess.call(cmd_collect_artifacts) sys.exit(returncode) -- GitLab From 59a6eb42aa17460991cd0537112903833c2e70d3 Mon Sep 17 00:00:00 2001 From: knj Date: Fri, 28 Oct 2022 15:40:55 +0200 Subject: [PATCH 08/33] makescript executable --- ci/collect_artifacts.py | 0 1 file changed, 0 insertions(+), 0 deletions(-) mode change 100644 => 100755 ci/collect_artifacts.py diff --git a/ci/collect_artifacts.py b/ci/collect_artifacts.py old mode 100644 new mode 100755 -- GitLab From 77478e6b891546f03c0a11225c96e7f2dd70b0c1 Mon Sep 17 00:00:00 2001 From: knj Date: Fri, 4 Nov 2022 15:17:19 +0100 Subject: [PATCH 09/33] make scripts executable --- ci/collect_artifacts.py | 0 ci/run_scheduled_sanitizer_test.py | 0 2 files changed, 0 insertions(+), 0 deletions(-) mode change 100644 => 100755 ci/collect_artifacts.py mode change 100644 => 100755 ci/run_scheduled_sanitizer_test.py diff --git a/ci/collect_artifacts.py b/ci/collect_artifacts.py old mode 100644 new mode 100755 diff --git a/ci/run_scheduled_sanitizer_test.py b/ci/run_scheduled_sanitizer_test.py old mode 100644 new mode 100755 -- GitLab From 71c7d64e210c29b001b92eadee4f9700800d50b9 Mon Sep 17 00:00:00 2001 From: knj Date: Tue, 8 Nov 2022 15:17:19 +0100 Subject: [PATCH 10/33] add shebang to file for correct interpreter --- ci/collect_artifacts.py | 1 + 1 file changed, 1 insertion(+) diff --git a/ci/collect_artifacts.py b/ci/collect_artifacts.py index 616c4c5c6b..498eab86c0 100755 --- a/ci/collect_artifacts.py +++ b/ci/collect_artifacts.py @@ -1,3 +1,4 @@ +#! /usr/bin/env python3 import pathlib import argparse import re -- GitLab From 7da0af73c6efe255d230ac00bbe91e9c6647f969 Mon Sep 17 00:00:00 2001 From: knj Date: Tue, 8 Nov 2022 15:52:08 +0100 Subject: [PATCH 11/33] intentionally add address error for testing --- lib_enc/hq_core_enc.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib_enc/hq_core_enc.c b/lib_enc/hq_core_enc.c index bb66182d3d..aeb83d3068 100644 --- a/lib_enc/hq_core_enc.c +++ b/lib_enc/hq_core_enc.c @@ -74,7 +74,7 @@ void hq_core_enc( wmops_sub_start( "hq_core_enc" ); - set_f( t_audio, 0, L_FRAME48k ); + set_f( t_audio, 0, L_FRAME48k * 2 ); st->Nb_ACELP_frames = 0; /* set input_frame length */ -- GitLab From bede8a40d52fe3456d293a9df15df5851cb251b3 Mon Sep 17 00:00:00 2001 From: knj Date: Tue, 8 Nov 2022 16:13:44 +0100 Subject: [PATCH 12/33] add error handling --- ci/collect_artifacts.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/ci/collect_artifacts.py b/ci/collect_artifacts.py index 498eab86c0..72a8446d5c 100755 --- a/ci/collect_artifacts.py +++ b/ci/collect_artifacts.py @@ -32,6 +32,9 @@ def collect_for_sanitizer_test(file): _, test, filename = m_line.groups() filename = pathlib.Path(filename).name m_file = re.match(pattern_file, filename) + if m_file is None: + print(f"Unexpected: no match on {filename} with {pattern_file} - skip") + continue filename_start = m_file.groups() files = [ -- GitLab From 59bd28808d6a16605301ddc367065d313e6d3ac6 Mon Sep 17 00:00:00 2001 From: knj Date: Tue, 8 Nov 2022 17:04:55 +0100 Subject: [PATCH 13/33] also archive noPLC log files --- .gitlab-ci.yml | 1 + ci/collect_artifacts.py | 14 ++++++++++++++ 2 files changed, 15 insertions(+) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 062b0d3272..8cc57ccb30 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -591,6 +591,7 @@ codec-comparison-on-main-push: paths: - ep_015.g192 - ./LOGS + - ./LOGS_noPLC sanitizer-test-mono: extends: .sanitizer-test-template diff --git a/ci/collect_artifacts.py b/ci/collect_artifacts.py index 72a8446d5c..93f0259f1f 100755 --- a/ci/collect_artifacts.py +++ b/ci/collect_artifacts.py @@ -21,6 +21,7 @@ def collect_for_sanitizer_test(file): console_log = f.readlines() files_to_archive = list() + files_to_archive_noPLC = list() pattern_line = "(Encoding|Decoding) failed .*for \/.*(CLANG.|VALGRIND)\/(.*)" pattern_file = "(.*_b[1-9]*_.*_rs|.*_b[1-9]*_.*_cbr).*" @@ -43,6 +44,12 @@ def collect_for_sanitizer_test(file): if f.name.startswith(filename_start) ] files_to_archive.extend(files) + files_noPlc = [ + f + for f in pathlib.Path(f"{test}/logs_noPLC/").iterdir() + if f.name.startswith(filename_start) + ] + files_to_archive_noPLC.extend(files_noPlc) log_folder = pathlib.Path("./LOGS") log_folder.mkdir() @@ -51,6 +58,13 @@ def collect_for_sanitizer_test(file): target = log_folder.joinpath(source.name) source.rename(target) + log_folder_noPLC = pathlib.Path("./LOGS_noPLC") + log_folder_noPLC.mkdir() + for p in files_to_archive_noPLC: + source = pathlib.Path(p) + target = log_folder_noPLC.joinpath(source.name) + source.rename(target) + if __name__ == "__main__": parser = argparse.ArgumentParser() -- GitLab From 9598005e5b85f8b81eaab1147673acce360fbc93 Mon Sep 17 00:00:00 2001 From: knj Date: Tue, 8 Nov 2022 17:14:19 +0100 Subject: [PATCH 14/33] Revert "intentionally add address error for testing" This reverts commit 7da0af73c6efe255d230ac00bbe91e9c6647f969. --- lib_enc/hq_core_enc.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib_enc/hq_core_enc.c b/lib_enc/hq_core_enc.c index aeb83d3068..bb66182d3d 100644 --- a/lib_enc/hq_core_enc.c +++ b/lib_enc/hq_core_enc.c @@ -74,7 +74,7 @@ void hq_core_enc( wmops_sub_start( "hq_core_enc" ); - set_f( t_audio, 0, L_FRAME48k * 2 ); + set_f( t_audio, 0, L_FRAME48k ); st->Nb_ACELP_frames = 0; /* set input_frame length */ -- GitLab From b900dd12ee81b31ce9849dcb6d50adb234635fa5 Mon Sep 17 00:00:00 2001 From: knj Date: Tue, 8 Nov 2022 17:15:15 +0100 Subject: [PATCH 15/33] account for bitrates that start with "0" --- ci/collect_artifacts.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/collect_artifacts.py b/ci/collect_artifacts.py index 93f0259f1f..13ba73c31d 100755 --- a/ci/collect_artifacts.py +++ b/ci/collect_artifacts.py @@ -24,7 +24,7 @@ def collect_for_sanitizer_test(file): files_to_archive_noPLC = list() pattern_line = "(Encoding|Decoding) failed .*for \/.*(CLANG.|VALGRIND)\/(.*)" - pattern_file = "(.*_b[1-9]*_.*_rs|.*_b[1-9]*_.*_cbr).*" + pattern_file = "(.*_b[0-9]*_.*_rs|.*_b[0-9]*_.*_cbr).*" for line in console_log: m_line = re.match(pattern_line, line) -- GitLab From 1cd396cd05b0272988a475c91cbd9005445c606b Mon Sep 17 00:00:00 2001 From: knj Date: Tue, 8 Nov 2022 17:25:34 +0100 Subject: [PATCH 16/33] separate logs between the tests --- ci/collect_artifacts.py | 39 +++++++++++++++++++++++++++------------ 1 file changed, 27 insertions(+), 12 deletions(-) diff --git a/ci/collect_artifacts.py b/ci/collect_artifacts.py index 13ba73c31d..dc15c1a6c9 100755 --- a/ci/collect_artifacts.py +++ b/ci/collect_artifacts.py @@ -20,8 +20,8 @@ def collect_for_sanitizer_test(file): with open(file) as f: console_log = f.readlines() - files_to_archive = list() - files_to_archive_noPLC = list() + files_to_archive = dict() + files_to_archive_noPLC = dict() pattern_line = "(Encoding|Decoding) failed .*for \/.*(CLANG.|VALGRIND)\/(.*)" pattern_file = "(.*_b[0-9]*_.*_rs|.*_b[0-9]*_.*_cbr).*" @@ -43,27 +43,42 @@ def collect_for_sanitizer_test(file): for f in pathlib.Path(f"{test}/logs/").iterdir() if f.name.startswith(filename_start) ] - files_to_archive.extend(files) + if test in files_to_archive: + files_to_archive[test].extend(files) + else: + files_to_archive[test] = files + files_noPlc = [ f for f in pathlib.Path(f"{test}/logs_noPLC/").iterdir() if f.name.startswith(filename_start) ] - files_to_archive_noPLC.extend(files_noPlc) + if test in files_to_archive_noPLC: + files_to_archive_noPLC[test].extend(files_noPlc) + else: + files_to_archive_noPLC[test] = files_noPlc log_folder = pathlib.Path("./LOGS") log_folder.mkdir() - for p in files_to_archive: - source = pathlib.Path(p) - target = log_folder.joinpath(source.name) - source.rename(target) + for test in files_to_archive.keys(): + log_folder.joinpath(test).mkdir() + for test, files in files_to_archive.items(): + folder = log_folder.joinpath(test) + for p in files: + source = pathlib.Path(p) + target = folder.joinpath(source.name) + source.rename(target) log_folder_noPLC = pathlib.Path("./LOGS_noPLC") log_folder_noPLC.mkdir() - for p in files_to_archive_noPLC: - source = pathlib.Path(p) - target = log_folder_noPLC.joinpath(source.name) - source.rename(target) + for test in files_to_archive_noPLC.keys(): + log_folder_noPLC.joinpath(test).mkdir() + for test, files in files_to_archive_noPLC.items(): + folder = log_folder_noPLC.joinpath(test) + for p in files: + source = pathlib.Path(p) + target = folder.joinpath(source.name) + source.rename(target) if __name__ == "__main__": -- GitLab From 3e1ceefc4606b65e98606eec5e23354d060b102d Mon Sep 17 00:00:00 2001 From: knj Date: Tue, 8 Nov 2022 15:52:08 +0100 Subject: [PATCH 17/33] intentionally add address error for testing --- lib_enc/hq_core_enc.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib_enc/hq_core_enc.c b/lib_enc/hq_core_enc.c index bb66182d3d..aeb83d3068 100644 --- a/lib_enc/hq_core_enc.c +++ b/lib_enc/hq_core_enc.c @@ -74,7 +74,7 @@ void hq_core_enc( wmops_sub_start( "hq_core_enc" ); - set_f( t_audio, 0, L_FRAME48k ); + set_f( t_audio, 0, L_FRAME48k * 2 ); st->Nb_ACELP_frames = 0; /* set input_frame length */ -- GitLab From 4c2d91b7aa1587a8379678dae52ac37c1df79e9c Mon Sep 17 00:00:00 2001 From: knj Date: Tue, 8 Nov 2022 17:36:22 +0100 Subject: [PATCH 18/33] Revert "intentionally add address error for testing" This reverts commit 3e1ceefc4606b65e98606eec5e23354d060b102d. --- lib_enc/hq_core_enc.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib_enc/hq_core_enc.c b/lib_enc/hq_core_enc.c index aeb83d3068..bb66182d3d 100644 --- a/lib_enc/hq_core_enc.c +++ b/lib_enc/hq_core_enc.c @@ -74,7 +74,7 @@ void hq_core_enc( wmops_sub_start( "hq_core_enc" ); - set_f( t_audio, 0, L_FRAME48k * 2 ); + set_f( t_audio, 0, L_FRAME48k ); st->Nb_ACELP_frames = 0; /* set input_frame length */ -- GitLab From 92715bf335ff410c6940fb99fa4c5d63810f09ac Mon Sep 17 00:00:00 2001 From: knj Date: Tue, 8 Nov 2022 17:37:32 +0100 Subject: [PATCH 19/33] rename LOG -> LOG_PLC for clarity --- .gitlab-ci.yml | 2 +- ci/collect_artifacts.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 8cc57ccb30..332bd4125e 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -590,7 +590,7 @@ codec-comparison-on-main-push: when: always paths: - ep_015.g192 - - ./LOGS + - ./LOGS_PLC - ./LOGS_noPLC sanitizer-test-mono: diff --git a/ci/collect_artifacts.py b/ci/collect_artifacts.py index dc15c1a6c9..43dd629622 100755 --- a/ci/collect_artifacts.py +++ b/ci/collect_artifacts.py @@ -58,7 +58,7 @@ def collect_for_sanitizer_test(file): else: files_to_archive_noPLC[test] = files_noPlc - log_folder = pathlib.Path("./LOGS") + log_folder = pathlib.Path("./LOGS_PLC") log_folder.mkdir() for test in files_to_archive.keys(): log_folder.joinpath(test).mkdir() -- GitLab From 4375f0870648bcf41b9f5df40fd16aef59ee762c Mon Sep 17 00:00:00 2001 From: knj Date: Tue, 8 Nov 2022 15:52:08 +0100 Subject: [PATCH 20/33] intentionally add address error for testing --- lib_enc/hq_core_enc.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib_enc/hq_core_enc.c b/lib_enc/hq_core_enc.c index bb66182d3d..aeb83d3068 100644 --- a/lib_enc/hq_core_enc.c +++ b/lib_enc/hq_core_enc.c @@ -74,7 +74,7 @@ void hq_core_enc( wmops_sub_start( "hq_core_enc" ); - set_f( t_audio, 0, L_FRAME48k ); + set_f( t_audio, 0, L_FRAME48k * 2 ); st->Nb_ACELP_frames = 0; /* set input_frame length */ -- GitLab From 1c25ef1314c26ad3b9d891576b0b8da1acddebf4 Mon Sep 17 00:00:00 2001 From: knj Date: Thu, 10 Nov 2022 12:51:43 +0100 Subject: [PATCH 21/33] factor out files collection in function --- ci/collect_artifacts.py | 39 ++++++++++++++++++--------------------- 1 file changed, 18 insertions(+), 21 deletions(-) diff --git a/ci/collect_artifacts.py b/ci/collect_artifacts.py index 43dd629622..385ea87cd5 100755 --- a/ci/collect_artifacts.py +++ b/ci/collect_artifacts.py @@ -15,17 +15,12 @@ def main(args): collect_for_sanitizer_test(file) -def collect_for_sanitizer_test(file): - - with open(file) as f: - console_log = f.readlines() - - files_to_archive = dict() - files_to_archive_noPLC = dict() +def find_failed_files_for_sanitizer_test(console_log: str, subfolder:str) -> dict(): pattern_line = "(Encoding|Decoding) failed .*for \/.*(CLANG.|VALGRIND)\/(.*)" pattern_file = "(.*_b[0-9]*_.*_rs|.*_b[0-9]*_.*_cbr).*" + files_found = dict() for line in console_log: m_line = re.match(pattern_line, line) @@ -36,27 +31,29 @@ def collect_for_sanitizer_test(file): if m_file is None: print(f"Unexpected: no match on {filename} with {pattern_file} - skip") continue - filename_start = m_file.groups() + filename_start = m_file.groups()[0] + folder = pathlib.Path(f"{test}/{subfolder}/") files = [ f - for f in pathlib.Path(f"{test}/logs/").iterdir() + for f in folder.iterdir() if f.name.startswith(filename_start) ] - if test in files_to_archive: - files_to_archive[test].extend(files) + if test in files_found: + files_found[test].extend(files) else: - files_to_archive[test] = files + files_found[test] = files - files_noPlc = [ - f - for f in pathlib.Path(f"{test}/logs_noPLC/").iterdir() - if f.name.startswith(filename_start) - ] - if test in files_to_archive_noPLC: - files_to_archive_noPLC[test].extend(files_noPlc) - else: - files_to_archive_noPLC[test] = files_noPlc + return files_found + + +def collect_for_sanitizer_test(file): + + with open(file) as f: + console_log = f.readlines() + + files_to_archive_noPLC = find_failed_files_for_sanitizer_test(console_log, "logs_noPLC") + files_to_archive = find_failed_files_for_sanitizer_test(console_log, "logs") log_folder = pathlib.Path("./LOGS_PLC") log_folder.mkdir() -- GitLab From 72888f76857f706ca7687b98f352d73372b2684e Mon Sep 17 00:00:00 2001 From: knj Date: Thu, 10 Nov 2022 21:31:48 +0100 Subject: [PATCH 22/33] delete bitstreams of failed encs before PLC run --- ci/collect_artifacts.py | 23 +++++++++++++++-------- ci/run_scheduled_sanitizer_test.py | 22 +++++++++++++++++++++- 2 files changed, 36 insertions(+), 9 deletions(-) diff --git a/ci/collect_artifacts.py b/ci/collect_artifacts.py index 385ea87cd5..d3228ba2e8 100755 --- a/ci/collect_artifacts.py +++ b/ci/collect_artifacts.py @@ -15,7 +15,11 @@ def main(args): collect_for_sanitizer_test(file) -def find_failed_files_for_sanitizer_test(console_log: str, subfolder:str) -> dict(): +def find_failed_files_for_sanitizer_test( + console_log: list, subfolder: str, which="LOGS" +) -> dict(): + + assert which in ["LOGS", "FILE_BASENAMES"] pattern_line = "(Encoding|Decoding) failed .*for \/.*(CLANG.|VALGRIND)\/(.*)" pattern_file = "(.*_b[0-9]*_.*_rs|.*_b[0-9]*_.*_cbr).*" @@ -33,12 +37,13 @@ def find_failed_files_for_sanitizer_test(console_log: str, subfolder:str) -> dic continue filename_start = m_file.groups()[0] - folder = pathlib.Path(f"{test}/{subfolder}/") - files = [ - f - for f in folder.iterdir() - if f.name.startswith(filename_start) - ] + if which == "LOGS": + folder = pathlib.Path(f"{test}/{subfolder}/") + files = [ + f for f in folder.iterdir() if f.name.startswith(filename_start) + ] + elif which == "FILE_BASENAMES": + files = [filename_start] if test in files_found: files_found[test].extend(files) else: @@ -52,7 +57,9 @@ def collect_for_sanitizer_test(file): with open(file) as f: console_log = f.readlines() - files_to_archive_noPLC = find_failed_files_for_sanitizer_test(console_log, "logs_noPLC") + files_to_archive_noPLC = find_failed_files_for_sanitizer_test( + console_log, "logs_noPLC" + ) files_to_archive = find_failed_files_for_sanitizer_test(console_log, "logs") log_folder = pathlib.Path("./LOGS_PLC") diff --git a/ci/run_scheduled_sanitizer_test.py b/ci/run_scheduled_sanitizer_test.py index 58f480d13e..008247078f 100755 --- a/ci/run_scheduled_sanitizer_test.py +++ b/ci/run_scheduled_sanitizer_test.py @@ -5,6 +5,10 @@ import sys import subprocess import pathlib +CI_SCRIPT_DIR = "./ci" +sys.path.append(CI_SCRIPT_DIR) +from collect_artifacts import find_failed_files_for_sanitizer_test + DURATION = "30" CFG = "ci_linux_ltv.json" @@ -82,7 +86,9 @@ def run_check(modes: list, out_formats: list, tests: list, run_fec: bool = True) ) with open(CONSOLE_OUT_FILE, "a") as f: - proc = subprocess.Popen(cmd_no_fec, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + proc = subprocess.Popen( + cmd_no_fec, stdout=subprocess.PIPE, stderr=subprocess.PIPE + ) for c in iter(lambda: proc.stdout.read(1), b""): sys.stdout.buffer.write(c) f.write(c.decode("utf8")) @@ -96,6 +102,20 @@ def run_check(modes: list, out_formats: list, tests: list, run_fec: bool = True) if not run_fec: return returncode_no_fec + # delete bitstream files for all failed modes to prevent follow-up errors in decoder-only run + with open(CONSOLE_OUT_FILE) as f: + console_log = f.readlines() + failed_files = find_failed_files_for_sanitizer_test( + console_log, "logs", "FILE_BASENAMES" + ) + for t in tests: + bs_folder = pathlib.Path(f"{t}/enc") + file_starts = failed_files[t] + for f in bs_folder.iterdir(): + for fs in file_starts: + if f.name.startswith(fs): + bs_folder.joinpath(f).unlink() + ### second run: decoder only with disturbed bitstream # generate error pattern -- GitLab From 45fe557b1d49b8eb8b8b744b4b496bbc86b6ece6 Mon Sep 17 00:00:00 2001 From: knj Date: Thu, 10 Nov 2022 21:50:33 +0100 Subject: [PATCH 23/33] iterate over tests that had failures, not all --- ci/run_scheduled_sanitizer_test.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/run_scheduled_sanitizer_test.py b/ci/run_scheduled_sanitizer_test.py index 008247078f..ed7beba02b 100755 --- a/ci/run_scheduled_sanitizer_test.py +++ b/ci/run_scheduled_sanitizer_test.py @@ -108,7 +108,7 @@ def run_check(modes: list, out_formats: list, tests: list, run_fec: bool = True) failed_files = find_failed_files_for_sanitizer_test( console_log, "logs", "FILE_BASENAMES" ) - for t in tests: + for t in failed_files.keys(): bs_folder = pathlib.Path(f"{t}/enc") file_starts = failed_files[t] for f in bs_folder.iterdir(): -- GitLab From b179a47735f4eaa4c1b14653437fb479da2d0f5f Mon Sep 17 00:00:00 2001 From: knj Date: Thu, 10 Nov 2022 22:07:18 +0100 Subject: [PATCH 24/33] fix path --- ci/run_scheduled_sanitizer_test.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/run_scheduled_sanitizer_test.py b/ci/run_scheduled_sanitizer_test.py index ed7beba02b..b9505bb54a 100755 --- a/ci/run_scheduled_sanitizer_test.py +++ b/ci/run_scheduled_sanitizer_test.py @@ -114,7 +114,7 @@ def run_check(modes: list, out_formats: list, tests: list, run_fec: bool = True) for f in bs_folder.iterdir(): for fs in file_starts: if f.name.startswith(fs): - bs_folder.joinpath(f).unlink() + f.unlink() ### second run: decoder only with disturbed bitstream -- GitLab From c65a182f1952ba18201ef524a3dd7eb07ed37574 Mon Sep 17 00:00:00 2001 From: knj Date: Thu, 10 Nov 2022 22:50:55 +0100 Subject: [PATCH 25/33] add debug printout --- ci/run_scheduled_sanitizer_test.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/ci/run_scheduled_sanitizer_test.py b/ci/run_scheduled_sanitizer_test.py index b9505bb54a..2d6a86f31d 100755 --- a/ci/run_scheduled_sanitizer_test.py +++ b/ci/run_scheduled_sanitizer_test.py @@ -99,6 +99,8 @@ def run_check(modes: list, out_formats: list, tests: list, run_fec: bool = True) returncode_no_fec = proc.returncode + print("returncode_no_fec:", returncode_no_fec) + if not run_fec: return returncode_no_fec @@ -150,6 +152,7 @@ def run_check(modes: list, out_formats: list, tests: list, run_fec: bool = True) proc.wait() returncode_fec = proc.returncode + print("returncode_fec:", returncode_fec) if returncode_fec not in [0, 101]: raise IvasBuildAndRunFailed("failed at second run (PLC)") -- GitLab From a721911fd69d266207e26a9ae763f355933ca466 Mon Sep 17 00:00:00 2001 From: knj Date: Fri, 11 Nov 2022 09:05:48 +0100 Subject: [PATCH 26/33] Revert "intentionally add address error for testing" This reverts commit 4375f0870648bcf41b9f5df40fd16aef59ee762c. --- lib_enc/hq_core_enc.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib_enc/hq_core_enc.c b/lib_enc/hq_core_enc.c index aeb83d3068..bb66182d3d 100644 --- a/lib_enc/hq_core_enc.c +++ b/lib_enc/hq_core_enc.c @@ -74,7 +74,7 @@ void hq_core_enc( wmops_sub_start( "hq_core_enc" ); - set_f( t_audio, 0, L_FRAME48k * 2 ); + set_f( t_audio, 0, L_FRAME48k ); st->Nb_ACELP_frames = 0; /* set input_frame length */ -- GitLab From 9e284e6bb9ac2d1c189b4151b8e143416d2bb1ab Mon Sep 17 00:00:00 2001 From: knj Date: Fri, 11 Nov 2022 09:29:02 +0100 Subject: [PATCH 27/33] Revert "Revert "intentionally add address error for testing"" This reverts commit a721911fd69d266207e26a9ae763f355933ca466. --- lib_enc/hq_core_enc.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib_enc/hq_core_enc.c b/lib_enc/hq_core_enc.c index bb66182d3d..aeb83d3068 100644 --- a/lib_enc/hq_core_enc.c +++ b/lib_enc/hq_core_enc.c @@ -74,7 +74,7 @@ void hq_core_enc( wmops_sub_start( "hq_core_enc" ); - set_f( t_audio, 0, L_FRAME48k ); + set_f( t_audio, 0, L_FRAME48k * 2 ); st->Nb_ACELP_frames = 0; /* set input_frame length */ -- GitLab From f82a35c9cc876a1e663286e3098fc0d1bb426d6e Mon Sep 17 00:00:00 2001 From: kiene Date: Fri, 11 Nov 2022 10:33:05 +0100 Subject: [PATCH 28/33] add debug printouts --- ci/run_scheduled_sanitizer_test.py | 6 ++---- scripts/IvasBuildAndRunChecks.py | 2 ++ 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/ci/run_scheduled_sanitizer_test.py b/ci/run_scheduled_sanitizer_test.py index 7ddae8072d..04cef8240e 100755 --- a/ci/run_scheduled_sanitizer_test.py +++ b/ci/run_scheduled_sanitizer_test.py @@ -94,12 +94,10 @@ def run_check(modes: list, out_formats: list, tests: list, run_fec: bool = True) f.write(c.decode("utf8")) proc.wait() - if proc.returncode not in [0, 101]: - raise IvasBuildAndRunFailed("Failed at first run (no PLC)") - returncode_no_fec = proc.returncode - print("returncode_no_fec:", returncode_no_fec) + if returncode_no_fec not in [0, 101]: + raise IvasBuildAndRunFailed("Failed at first run (no PLC)") if not run_fec: return returncode_no_fec diff --git a/scripts/IvasBuildAndRunChecks.py b/scripts/IvasBuildAndRunChecks.py index 1cf4a868db..58dd353e87 100755 --- a/scripts/IvasBuildAndRunChecks.py +++ b/scripts/IvasBuildAndRunChecks.py @@ -197,6 +197,8 @@ class IvasBuildAndRunChecks(IvasScriptsCommon.IvasScript): runner = br.build_and_run_dict[check]["runner"] failed_encs = runner.failed_modes["enc"] failed_decs = runner.failed_modes["dec"] + print(failed_encs) + print(failed_decs) if len(failed_encs) > 0 or len(failed_decs) > 0: return RET_CODE_FAILURE else: -- GitLab From baec03822f0d44542b0b5d0c0fc76ff097450069 Mon Sep 17 00:00:00 2001 From: knj Date: Fri, 11 Nov 2022 11:05:10 +0100 Subject: [PATCH 29/33] check for errors in all checks before returning --- scripts/IvasBuildAndRunChecks.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/scripts/IvasBuildAndRunChecks.py b/scripts/IvasBuildAndRunChecks.py index 58dd353e87..937ea2c273 100755 --- a/scripts/IvasBuildAndRunChecks.py +++ b/scripts/IvasBuildAndRunChecks.py @@ -193,6 +193,7 @@ class IvasBuildAndRunChecks(IvasScriptsCommon.IvasScript): self.args["create_complexity_tables"] ) + returncode = 0 for check in checks: runner = br.build_and_run_dict[check]["runner"] failed_encs = runner.failed_modes["enc"] @@ -200,10 +201,9 @@ class IvasBuildAndRunChecks(IvasScriptsCommon.IvasScript): print(failed_encs) print(failed_decs) if len(failed_encs) > 0 or len(failed_decs) > 0: - return RET_CODE_FAILURE - else: - return 0 + returncode = RET_CODE_FAILURE + return returncode if __name__ == "__main__": script = IvasBuildAndRunChecks() -- GitLab From 32eec63734d96711590eeffe28c204d6ab5e7d71 Mon Sep 17 00:00:00 2001 From: kiene Date: Fri, 11 Nov 2022 11:14:49 +0100 Subject: [PATCH 30/33] remove printouts --- scripts/IvasBuildAndRunChecks.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/scripts/IvasBuildAndRunChecks.py b/scripts/IvasBuildAndRunChecks.py index 937ea2c273..0b2a2932c0 100755 --- a/scripts/IvasBuildAndRunChecks.py +++ b/scripts/IvasBuildAndRunChecks.py @@ -198,8 +198,6 @@ class IvasBuildAndRunChecks(IvasScriptsCommon.IvasScript): runner = br.build_and_run_dict[check]["runner"] failed_encs = runner.failed_modes["enc"] failed_decs = runner.failed_modes["dec"] - print(failed_encs) - print(failed_decs) if len(failed_encs) > 0 or len(failed_decs) > 0: returncode = RET_CODE_FAILURE -- GitLab From 14fc44e8d4b18034fcdd6c44e29178038f69faa3 Mon Sep 17 00:00:00 2001 From: knj Date: Fri, 11 Nov 2022 11:32:42 +0100 Subject: [PATCH 31/33] use imported function rather then subprocessing the whole script --- ci/run_scheduled_sanitizer_test.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/ci/run_scheduled_sanitizer_test.py b/ci/run_scheduled_sanitizer_test.py index 04cef8240e..4209e438bd 100755 --- a/ci/run_scheduled_sanitizer_test.py +++ b/ci/run_scheduled_sanitizer_test.py @@ -7,7 +7,10 @@ import pathlib CI_SCRIPT_DIR = "./ci" sys.path.append(CI_SCRIPT_DIR) -from collect_artifacts import find_failed_files_for_sanitizer_test +from collect_artifacts import ( + find_failed_files_for_sanitizer_test, + collect_for_sanitizer_test, +) DURATION = "120" @@ -35,8 +38,7 @@ def main(args): modes = get_modes(in_format) returncode = run_check(modes, out_formats, tests, run_fec=run_fec) - cmd_collect_artifacts = [COLLECT_ARTIFACTS_SCRIPT, "sanitizers", CONSOLE_OUT_FILE] - subprocess.call(cmd_collect_artifacts) + collect_for_sanitizer_test(CONSOLE_OUT_FILE) sys.exit(returncode) -- GitLab From 2c3bb66e168e0cbd97ba587fb97de64bfe8a196e Mon Sep 17 00:00:00 2001 From: knj Date: Fri, 11 Nov 2022 11:55:37 +0100 Subject: [PATCH 32/33] Revert "Revert "Revert "intentionally add address error for testing""" This reverts commit 9e284e6bb9ac2d1c189b4151b8e143416d2bb1ab. Revertception! --- lib_enc/hq_core_enc.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib_enc/hq_core_enc.c b/lib_enc/hq_core_enc.c index aeb83d3068..bb66182d3d 100644 --- a/lib_enc/hq_core_enc.c +++ b/lib_enc/hq_core_enc.c @@ -74,7 +74,7 @@ void hq_core_enc( wmops_sub_start( "hq_core_enc" ); - set_f( t_audio, 0, L_FRAME48k * 2 ); + set_f( t_audio, 0, L_FRAME48k ); st->Nb_ACELP_frames = 0; /* set input_frame length */ -- GitLab From 31fac1477e125b9444b288f03c5234eb5fca286c Mon Sep 17 00:00:00 2001 From: knj Date: Mon, 14 Nov 2022 09:56:39 +0100 Subject: [PATCH 33/33] remove unused variable --- ci/run_scheduled_sanitizer_test.py | 1 - 1 file changed, 1 deletion(-) diff --git a/ci/run_scheduled_sanitizer_test.py b/ci/run_scheduled_sanitizer_test.py index 4209e438bd..a63c175d56 100755 --- a/ci/run_scheduled_sanitizer_test.py +++ b/ci/run_scheduled_sanitizer_test.py @@ -23,7 +23,6 @@ MC_MODES = ["5_1", "5_1_2", "5_1_4", "7_1", "7_1_4"] SCRIPT_DIR = pathlib.Path("./scripts").resolve() -COLLECT_ARTIFACTS_SCRIPT = "ci/collect_artifacts.py" CONSOLE_OUT_FILE = "output_san.txt" -- GitLab