From 9701cfbdcae7b7528db3e0facd76e4e5391e7473 Mon Sep 17 00:00:00 2001 From: knj Date: Mon, 8 Jan 2024 16:31:19 +0100 Subject: [PATCH 1/9] enable usan inthe ltv sanitizer tests --- .gitlab-ci.yml | 2 +- ci/run_scheduled_sanitizer_test.py | 2 +- scripts/IvasBuildAndRunChecks.py | 5 +++++ scripts/pyivastest/IvasSvnBuilder.py | 9 ++++++++- 4 files changed, 15 insertions(+), 3 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 0051d0cdd9..27212c8646 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -4,7 +4,7 @@ variables: BUILD_OUTPUT: "build_output.txt" EVS_BE_TEST_DIR: "/usr/local/be_2_evs_test" EVS_BE_WIN_TEST_DIR: "C:/Users/gitlab-runner/testvec" - SANITIZER_TESTS: "CLANG1 CLANG2" + SANITIZER_TESTS: "CLANG1 CLANG2 CLANG3" OUT_FORMATS_CHANNEL_BASED: "stereo mono 5_1 5_1_2 5_1_4 7_1 7_1_4" OUT_FORMATS_SCENE_BASED: "FOA HOA2 HOA3" OUT_FORMATS_BINAURAL: "BINAURAL BINAURAL_ROOM_IR BINAURAL_ROOM_REVERB" diff --git a/ci/run_scheduled_sanitizer_test.py b/ci/run_scheduled_sanitizer_test.py index c6d3d78d45..6f59165847 100755 --- a/ci/run_scheduled_sanitizer_test.py +++ b/ci/run_scheduled_sanitizer_test.py @@ -46,7 +46,7 @@ from combine_genpatt_and_jbm_profile import combine_error_profiles SCRIPT_DIR = pathlib.Path("./scripts").resolve() DURATION = "120" CFG = "ci_linux_ltv.json" -SUPPORTED_TESTS = ["CLANG1", "CLANG2", "CLANG3", "VALGRIND"] +SUPPORTED_TESTS = ["CLANG1", "CLANG2", "CLANG3"] EP_FILE = "ep_015.g192" DLY_PROFILE_IN = SCRIPT_DIR.joinpath("dly_error_profiles/dly_error_profile_5.dat") DLY_PROFILE_OUT = "dly_profile.dat" diff --git a/scripts/IvasBuildAndRunChecks.py b/scripts/IvasBuildAndRunChecks.py index f32ade7773..523eee3a98 100755 --- a/scripts/IvasBuildAndRunChecks.py +++ b/scripts/IvasBuildAndRunChecks.py @@ -91,6 +91,10 @@ class IvasBuildAndRunChecks(IvasScriptsCommon.IvasScript): self.parser.add_argument( "--rebuild", help="force a rebuild of the binaries", action="store_true" ) + self.parser.add_argument( + "--usan_supp_file", help="suppression file for undef behaviour sanitizer", + default=None, + ) def run(self): @@ -157,6 +161,7 @@ class IvasBuildAndRunChecks(IvasScriptsCommon.IvasScript): format_select_list=modes, formats_fname=self.args["format_file"], max_workers=self.args["max_workers"], + usan_supp_file=self.args["usan_supp_file"], ) IvasScriptsCommon.runner_setup( br.build_and_run_dict[check]["runner"], self.args diff --git a/scripts/pyivastest/IvasSvnBuilder.py b/scripts/pyivastest/IvasSvnBuilder.py index 0d7cc9c847..dff7616818 100644 --- a/scripts/pyivastest/IvasSvnBuilder.py +++ b/scripts/pyivastest/IvasSvnBuilder.py @@ -1175,6 +1175,7 @@ class IvasBuilderAndRunner(IvasBaseClass): defines_to_disable=None, formats_fname="", max_workers=1, + usan_supp_file=None, ): n_cpus = cpu_count() @@ -1190,8 +1191,14 @@ class IvasBuilderAndRunner(IvasBaseClass): else: defines_to_disable_check = defines_to_disable.copy() if check.startswith("CLANG"): - make_options.append("CLANG=" + check[-1]) + clang_n = check[-1] + make_options.append("CLANG=" + clang_n) defines_to_disable_check.extend(["RAM_COUNTING_TOOL"]) + + # for undefined behaviou sanitizer, pass suppression file if given + if clang_n == "3" and usan_supp_file is not None: + run_tool = [f"UBSAN_OPTIONS=suppressions={usan_supp_file},report_error_type=1"] + elif check == "VALGRIND": defines_to_disable_check.extend(["RAM_COUNTING_TOOL"]) run_tool = [ -- GitLab From 2c80863b7d31ea02c5330096d9ba9270722d520e Mon Sep 17 00:00:00 2001 From: knj Date: Mon, 8 Jan 2024 16:35:05 +0100 Subject: [PATCH 2/9] pass suppression file to IvasBuildAndRunChecks script --- ci/run_scheduled_sanitizer_test.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/ci/run_scheduled_sanitizer_test.py b/ci/run_scheduled_sanitizer_test.py index 6f59165847..22428fa7f7 100755 --- a/ci/run_scheduled_sanitizer_test.py +++ b/ci/run_scheduled_sanitizer_test.py @@ -47,6 +47,7 @@ SCRIPT_DIR = pathlib.Path("./scripts").resolve() DURATION = "120" CFG = "ci_linux_ltv.json" SUPPORTED_TESTS = ["CLANG1", "CLANG2", "CLANG3"] +USAN_SUPP_FILE = SCRIPT_DIR.joinpath("ubsan.supp") EP_FILE = "ep_015.g192" DLY_PROFILE_IN = SCRIPT_DIR.joinpath("dly_error_profiles/dly_error_profile_5.dat") DLY_PROFILE_OUT = "dly_profile.dat" @@ -145,6 +146,8 @@ def run_check(in_format: str, out_formats: list, tests: list, run_fec: bool = Tr "--oc", *out_formats, *md_file_command, + "--usan_supp_file", + USAN_SUPP_FILE, ] print( -- GitLab From edff91e40f6b26496125e54c27a71d8f89e3a387 Mon Sep 17 00:00:00 2001 From: knj Date: Mon, 8 Jan 2024 16:36:32 +0100 Subject: [PATCH 3/9] turn Path into string --- ci/run_scheduled_sanitizer_test.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/run_scheduled_sanitizer_test.py b/ci/run_scheduled_sanitizer_test.py index 22428fa7f7..5007914b95 100755 --- a/ci/run_scheduled_sanitizer_test.py +++ b/ci/run_scheduled_sanitizer_test.py @@ -47,7 +47,7 @@ SCRIPT_DIR = pathlib.Path("./scripts").resolve() DURATION = "120" CFG = "ci_linux_ltv.json" SUPPORTED_TESTS = ["CLANG1", "CLANG2", "CLANG3"] -USAN_SUPP_FILE = SCRIPT_DIR.joinpath("ubsan.supp") +USAN_SUPP_FILE = str(SCRIPT_DIR.joinpath("ubsan.supp")) EP_FILE = "ep_015.g192" DLY_PROFILE_IN = SCRIPT_DIR.joinpath("dly_error_profiles/dly_error_profile_5.dat") DLY_PROFILE_OUT = "dly_profile.dat" -- GitLab From 83c896dceef149297e1bd36b1c5e533e49ca0fb1 Mon Sep 17 00:00:00 2001 From: knj Date: Mon, 8 Jan 2024 16:46:06 +0100 Subject: [PATCH 4/9] pass UBSAN_OPTIONS env var to the subprocesses --- scripts/pyivastest/IvasModeRunner.py | 8 +++++--- scripts/pyivastest/IvasSvnBuilder.py | 4 +++- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/scripts/pyivastest/IvasModeRunner.py b/scripts/pyivastest/IvasModeRunner.py index 2ab5f16243..cb4b9e9225 100644 --- a/scripts/pyivastest/IvasModeRunner.py +++ b/scripts/pyivastest/IvasModeRunner.py @@ -68,6 +68,7 @@ class IvasModeRunner(IvasModeCollector.IvasModeCollector): dir_name=".", bin_suffix="", test_tool="", + run_env={}, silent=False, site_config="", sample_rate_enc_in=None, @@ -115,6 +116,7 @@ class IvasModeRunner(IvasModeCollector.IvasModeCollector): self.decoder = "" self.bin_suffix = bin_suffix self.test_tool = test_tool + self.run_env = os.environ.copy().update(run_env) self.silent = silent self.failed_threads = None self.max_workers = max_workers @@ -314,7 +316,7 @@ class IvasModeRunner(IvasModeCollector.IvasModeCollector): + [enc_file_name, dec_file_name] ) - cur_dec_result = subprocess.run(dec_cmd, capture_output=True, text=True) + cur_dec_result = subprocess.run(dec_cmd, capture_output=True, text=True, env=self.run_env) dec_log.write(" ".join(dec_cmd)) dec_log.write(cur_dec_result.stderr) dec_log.write(cur_dec_result.stdout) @@ -703,7 +705,7 @@ class IvasModeRunner(IvasModeCollector.IvasModeCollector): enc_log = open(enc_log_name, "w") enc_log.write(" ".join(enc_cmd)) - enc_result = subprocess.run(enc_cmd, capture_output=True, text=True) + enc_result = subprocess.run(enc_cmd, capture_output=True, text=True, env=self.run_env) error = enc_result.returncode enc_log.write(enc_result.stderr) enc_log.write(enc_result.stdout) @@ -729,7 +731,7 @@ class IvasModeRunner(IvasModeCollector.IvasModeCollector): enc_log = open(enc_log_name, "a") enc_log.write(" ".join(proc_cmd)) proc_result = subprocess.run( - proc_cmd, capture_output=True, text=True, encoding="utf8" + proc_cmd, capture_output=True, text=True, encoding="utf8", ) enc_log.write(proc_result.stderr) enc_log.write(proc_result.stdout) diff --git a/scripts/pyivastest/IvasSvnBuilder.py b/scripts/pyivastest/IvasSvnBuilder.py index dff7616818..df1c7e064e 100644 --- a/scripts/pyivastest/IvasSvnBuilder.py +++ b/scripts/pyivastest/IvasSvnBuilder.py @@ -1182,6 +1182,7 @@ class IvasBuilderAndRunner(IvasBaseClass): # do not use all cores to avoid weird getting-stuck issues observed on Mac... make_options = ["-j" ,f"{n_cpus - 2}"] run_tool = "" + run_env = dict() if defines_to_enable is None: defines_to_enable_check = [] else: @@ -1197,7 +1198,7 @@ class IvasBuilderAndRunner(IvasBaseClass): # for undefined behaviou sanitizer, pass suppression file if given if clang_n == "3" and usan_supp_file is not None: - run_tool = [f"UBSAN_OPTIONS=suppressions={usan_supp_file},report_error_type=1"] + run_env["UBSAN_OPTIONS": f"suppressions={usan_supp_file},report_error_type=1"] elif check == "VALGRIND": defines_to_disable_check.extend(["RAM_COUNTING_TOOL"]) @@ -1213,6 +1214,7 @@ class IvasBuilderAndRunner(IvasBaseClass): self.add_build_and_run_config( check, run_tool=run_tool, + run_env=run_env, make_options=make_options, format_select_list=format_select_list, mode_select_list=mode_select_list, -- GitLab From 53eb9fd3d8ac9b8237771d6d50dc2b12bd1d93bc Mon Sep 17 00:00:00 2001 From: knj Date: Mon, 8 Jan 2024 16:52:31 +0100 Subject: [PATCH 5/9] fix bugs in python scripts --- scripts/pyivastest/IvasModeRunner.py | 2 +- scripts/pyivastest/IvasSvnBuilder.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/pyivastest/IvasModeRunner.py b/scripts/pyivastest/IvasModeRunner.py index cb4b9e9225..0fc0f2970b 100644 --- a/scripts/pyivastest/IvasModeRunner.py +++ b/scripts/pyivastest/IvasModeRunner.py @@ -731,7 +731,7 @@ class IvasModeRunner(IvasModeCollector.IvasModeCollector): enc_log = open(enc_log_name, "a") enc_log.write(" ".join(proc_cmd)) proc_result = subprocess.run( - proc_cmd, capture_output=True, text=True, encoding="utf8", + proc_cmd, capture_output=True, text=True, encoding="utf8" ) enc_log.write(proc_result.stderr) enc_log.write(proc_result.stdout) diff --git a/scripts/pyivastest/IvasSvnBuilder.py b/scripts/pyivastest/IvasSvnBuilder.py index df1c7e064e..7daca6ee27 100644 --- a/scripts/pyivastest/IvasSvnBuilder.py +++ b/scripts/pyivastest/IvasSvnBuilder.py @@ -1198,7 +1198,7 @@ class IvasBuilderAndRunner(IvasBaseClass): # for undefined behaviou sanitizer, pass suppression file if given if clang_n == "3" and usan_supp_file is not None: - run_env["UBSAN_OPTIONS": f"suppressions={usan_supp_file},report_error_type=1"] + run_env["UBSAN_OPTIONS"] = f"suppressions={usan_supp_file},report_error_type=1" elif check == "VALGRIND": defines_to_disable_check.extend(["RAM_COUNTING_TOOL"]) -- GitLab From 0042d05ad6ab328d7bdc16439aff187d6c5fd99a Mon Sep 17 00:00:00 2001 From: knj Date: Mon, 8 Jan 2024 16:57:31 +0100 Subject: [PATCH 6/9] add missing argument to function --- scripts/pyivastest/IvasSvnBuilder.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/scripts/pyivastest/IvasSvnBuilder.py b/scripts/pyivastest/IvasSvnBuilder.py index 7daca6ee27..a6c7825ebe 100644 --- a/scripts/pyivastest/IvasSvnBuilder.py +++ b/scripts/pyivastest/IvasSvnBuilder.py @@ -763,6 +763,7 @@ class IvasBuilderAndRunner(IvasBaseClass): self, cfg_name, run_tool="", + run_env={}, make_options=[], defines_to_enable=[], defines_to_disable=[], @@ -829,6 +830,7 @@ class IvasBuilderAndRunner(IvasBaseClass): dir_name=run_dir, bin_suffix=self.builder.binary_ext, test_tool=run_tool, + run_env=run_env, site_config=self.site_config, sample_rate_enc_in=sample_rate_enc_in, formats_fname=formats_fname, -- GitLab From b50230683b9fa429fcc3e95b8e90f3e74b590b57 Mon Sep 17 00:00:00 2001 From: knj Date: Tue, 9 Jan 2024 13:16:25 +0100 Subject: [PATCH 7/9] supress error printout of suppressed USAN errors --- scripts/pyivastest/IvasModeAnalyzer.py | 20 ++++++++++++++++++++ scripts/pyivastest/IvasSvnBuilder.py | 3 +++ 2 files changed, 23 insertions(+) diff --git a/scripts/pyivastest/IvasModeAnalyzer.py b/scripts/pyivastest/IvasModeAnalyzer.py index 7e6d40b2e5..47ca7cd96f 100644 --- a/scripts/pyivastest/IvasModeAnalyzer.py +++ b/scripts/pyivastest/IvasModeAnalyzer.py @@ -151,6 +151,7 @@ class IvasModeAnalyzer(IvasModeCollector): site_config="", log_dir="logs", formats_fname=None, + supp_file_provided=False, formats_dict={}, enable_logging=False, console_logger_level="", @@ -174,6 +175,7 @@ class IvasModeAnalyzer(IvasModeCollector): self.log_select_list = {} self.selected_logs = {} self.check = "" + self.supp_file_provided = supp_file_provided self.analyze_encoder = True self.analyze_decoder = True self.html_table = None @@ -692,8 +694,11 @@ class IvasModeAnalyzer(IvasModeCollector): def get_run_errors(self, failed_modes): + is_clang_usan = False if self.check.lower().startswith("clang"): num = int(self.check[-1]) + is_clang_usan = num == 3 + parse_func = functools.partial(self.parse_clang_log, clang_num=num) print_func = functools.partial(self.print_clang_errors, clang_num=num) elif self.check.lower().startswith("valgrind"): @@ -719,6 +724,13 @@ class IvasModeAnalyzer(IvasModeCollector): is_failed_mode = mode in failed_modes["enc"] + # for clang ubsan with suppression file: non-suppressed errors cause crashes + # at runtime. So, if the mode did not fail, all the possibly present errors + # in the log files should be part of the suppression file, so do not + # print about them or ad them to the html file + if is_clang_usan and self.supp_file_provided and not is_failed_mode: + continue + # add error to html report if it is an actual sanitizer error # or if the en-/decoder crashed if ( @@ -747,6 +759,14 @@ class IvasModeAnalyzer(IvasModeCollector): mode_failed_for_oc = False is_failed_mode = oc_failed_at_least_once and mode_failed_for_oc + + # for clang ubsan with suppression file: non-suppressed errors cause crashes + # at runtime. So, if the mode did not fail, all the possibly present errors + # in the log files should be part of the suppression file, so do not + # print about them or ad them to the html file + if is_clang_usan and self.supp_file_provided and not is_failed_mode: + continue + # add error to html report if it is an actual sanitizer error # or if the en-/decoder crashed if ( diff --git a/scripts/pyivastest/IvasSvnBuilder.py b/scripts/pyivastest/IvasSvnBuilder.py index a6c7825ebe..9e6296ea59 100644 --- a/scripts/pyivastest/IvasSvnBuilder.py +++ b/scripts/pyivastest/IvasSvnBuilder.py @@ -764,6 +764,7 @@ class IvasBuilderAndRunner(IvasBaseClass): cfg_name, run_tool="", run_env={}, + supp_file_provided=False, make_options=[], defines_to_enable=[], defines_to_disable=[], @@ -853,6 +854,7 @@ class IvasBuilderAndRunner(IvasBaseClass): enable_logging=True, logger_name="{}.{}analyzer".format(self.logger.name, cfg_name), log_level=self.logger.level, + supp_file_provided=supp_file_provided, ) cfg_dict = { cfg_name: { @@ -1224,6 +1226,7 @@ class IvasBuilderAndRunner(IvasBaseClass): defines_to_disable=defines_to_disable_check, formats_fname=formats_fname, max_workers=max_workers, + supp_file_provided=usan_supp_file is not None, ) self.build_and_run_dict[check]["analyzer"].check = check -- GitLab From c5c3f29cbaa42c29ad563eea97b8bbbdb8c28750 Mon Sep 17 00:00:00 2001 From: knj Date: Tue, 9 Jan 2024 13:31:25 +0100 Subject: [PATCH 8/9] Revert "supress error printout of suppressed USAN errors" This reverts commit b50230683b9fa429fcc3e95b8e90f3e74b590b57. --- scripts/pyivastest/IvasModeAnalyzer.py | 20 -------------------- scripts/pyivastest/IvasSvnBuilder.py | 3 --- 2 files changed, 23 deletions(-) diff --git a/scripts/pyivastest/IvasModeAnalyzer.py b/scripts/pyivastest/IvasModeAnalyzer.py index 47ca7cd96f..7e6d40b2e5 100644 --- a/scripts/pyivastest/IvasModeAnalyzer.py +++ b/scripts/pyivastest/IvasModeAnalyzer.py @@ -151,7 +151,6 @@ class IvasModeAnalyzer(IvasModeCollector): site_config="", log_dir="logs", formats_fname=None, - supp_file_provided=False, formats_dict={}, enable_logging=False, console_logger_level="", @@ -175,7 +174,6 @@ class IvasModeAnalyzer(IvasModeCollector): self.log_select_list = {} self.selected_logs = {} self.check = "" - self.supp_file_provided = supp_file_provided self.analyze_encoder = True self.analyze_decoder = True self.html_table = None @@ -694,11 +692,8 @@ class IvasModeAnalyzer(IvasModeCollector): def get_run_errors(self, failed_modes): - is_clang_usan = False if self.check.lower().startswith("clang"): num = int(self.check[-1]) - is_clang_usan = num == 3 - parse_func = functools.partial(self.parse_clang_log, clang_num=num) print_func = functools.partial(self.print_clang_errors, clang_num=num) elif self.check.lower().startswith("valgrind"): @@ -724,13 +719,6 @@ class IvasModeAnalyzer(IvasModeCollector): is_failed_mode = mode in failed_modes["enc"] - # for clang ubsan with suppression file: non-suppressed errors cause crashes - # at runtime. So, if the mode did not fail, all the possibly present errors - # in the log files should be part of the suppression file, so do not - # print about them or ad them to the html file - if is_clang_usan and self.supp_file_provided and not is_failed_mode: - continue - # add error to html report if it is an actual sanitizer error # or if the en-/decoder crashed if ( @@ -759,14 +747,6 @@ class IvasModeAnalyzer(IvasModeCollector): mode_failed_for_oc = False is_failed_mode = oc_failed_at_least_once and mode_failed_for_oc - - # for clang ubsan with suppression file: non-suppressed errors cause crashes - # at runtime. So, if the mode did not fail, all the possibly present errors - # in the log files should be part of the suppression file, so do not - # print about them or ad them to the html file - if is_clang_usan and self.supp_file_provided and not is_failed_mode: - continue - # add error to html report if it is an actual sanitizer error # or if the en-/decoder crashed if ( diff --git a/scripts/pyivastest/IvasSvnBuilder.py b/scripts/pyivastest/IvasSvnBuilder.py index 9e6296ea59..a6c7825ebe 100644 --- a/scripts/pyivastest/IvasSvnBuilder.py +++ b/scripts/pyivastest/IvasSvnBuilder.py @@ -764,7 +764,6 @@ class IvasBuilderAndRunner(IvasBaseClass): cfg_name, run_tool="", run_env={}, - supp_file_provided=False, make_options=[], defines_to_enable=[], defines_to_disable=[], @@ -854,7 +853,6 @@ class IvasBuilderAndRunner(IvasBaseClass): enable_logging=True, logger_name="{}.{}analyzer".format(self.logger.name, cfg_name), log_level=self.logger.level, - supp_file_provided=supp_file_provided, ) cfg_dict = { cfg_name: { @@ -1226,7 +1224,6 @@ class IvasBuilderAndRunner(IvasBaseClass): defines_to_disable=defines_to_disable_check, formats_fname=formats_fname, max_workers=max_workers, - supp_file_provided=usan_supp_file is not None, ) self.build_and_run_dict[check]["analyzer"].check = check -- GitLab From a2e8a7eae4537f2c96b4fa8b353fbd26eeea2ee1 Mon Sep 17 00:00:00 2001 From: knj Date: Tue, 9 Jan 2024 13:33:53 +0100 Subject: [PATCH 9/9] do update of env dictionary correctly --- scripts/pyivastest/IvasModeRunner.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/scripts/pyivastest/IvasModeRunner.py b/scripts/pyivastest/IvasModeRunner.py index 0fc0f2970b..c67f252e1d 100644 --- a/scripts/pyivastest/IvasModeRunner.py +++ b/scripts/pyivastest/IvasModeRunner.py @@ -116,7 +116,8 @@ class IvasModeRunner(IvasModeCollector.IvasModeCollector): self.decoder = "" self.bin_suffix = bin_suffix self.test_tool = test_tool - self.run_env = os.environ.copy().update(run_env) + self.run_env = os.environ.copy() + self.run_env.update(run_env) self.silent = silent self.failed_threads = None self.max_workers = max_workers -- GitLab