From 76924b5bb3f1ff54685eae436744244347d16950 Mon Sep 17 00:00:00 2001 From: knj Date: Fri, 21 Jul 2023 16:37:33 +0200 Subject: [PATCH 1/2] Add MR pipeline jobs for USAN + suppression file --- .gitlab-ci.yml | 49 +++++++++++++++++++++++++++++ CMakeLists.txt | 16 ++++++++-- Makefile | 8 +++-- scripts/ubsan.supp | 56 +++++++++++++++++++++++++++++++++ tests/renderer/test_renderer.py | 32 ++++++++++++------- tests/renderer/utils.py | 23 +++++++++----- 6 files changed, 161 insertions(+), 23 deletions(-) create mode 100644 scripts/ubsan.supp diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 342a6c4dc8..cc24ab729e 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -388,6 +388,28 @@ codec-asan: - test_output.txt expose_as: "asan selftest results" +# code selftest testvectors with address-sanitizer binaries +codec-usan: + extends: + - .test-job-linux + - .rules-merge-request + stage: test + needs: ["build-codec-sanitizers-linux"] + script: + - *print-common-info + - make clean + - make -j CLANG=3 + - UBSAN_OPTIONS=suppressions=scripts/ubsan.supp,report_error_type=1 python3 scripts/self_test.py --create + - grep_exit_code=0 + - grep UndefinedBehaviorSanitizer scripts/ref/logs/* || grep_exit_code=$? + - if [ $grep_exit_code != 1 ] ; then echo "Run errors in self_test.py with Clang undefined-behavior-sanitizer"; exit 1; fi + artifacts: + name: "mr-$CI_MERGE_REQUEST_IID--sha-$CI_COMMIT_SHORT_SHA--stage-$CI_JOB_STAGE--results" + expire_in: 1 week + paths: + - scripts/ref/logs/ + expose_as: "usan selftest results" + # test renderer executable renderer-smoke-test: extends: @@ -455,6 +477,33 @@ renderer-msan: junit: - report-junit.xml +# test renderer executable with cmake + usan +renderer-usan: + extends: + - .test-job-linux + - .rules-merge-request + needs: ["build-codec-linux-cmake"] + stage: test + script: + - cmake -B cmake-build -G "Unix Makefiles" -DCLANG=usan -DCOPY_EXECUTABLES_FROM_BUILD_DIR=true + - cmake --build cmake-build -- -j + - UBSAN_OPTIONS=suppressions=scripts/ubsan.supp,report_error_type=1,log_path=usan_log_catchall python3 -m pytest -q -n auto -rA --junit-xml=report-junit.xml tests/renderer/test_renderer.py + - grep_exit_code=0 + - touch usan_log_empty # Creates an empty file, this is to avoid "grep: usan_log_*: No such file or directory" in case no USAN failures are reported from pytest + - grep UndefinedBehaviorSanitizer usan_log_* || grep_exit_code=$? + - if [ $grep_exit_code != 1 ] ; then echo "Run errors in test_renderer.py with Clang undefined-behavior-sanitizer"; exit 1; fi + + artifacts: + name: "mr-$CI_MERGE_REQUEST_IID--sha-$CI_COMMIT_SHORT_SHA--job-$CI_JOB_NAME--results" + expire_in: 1 week + when: always + paths: + - report-junit.xml + expose_as: "renderer usan pytest results" + reports: + junit: + - report-junit.xml + # compare renderer bitexactness between target and source branch renderer-pytest-on-merge-request: extends: diff --git a/CMakeLists.txt b/CMakeLists.txt index 270636eb5e..d08135676c 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -70,8 +70,20 @@ if(UNIX) set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -fsanitize=address") set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -fsanitize=address") elseif("${CLANG}" MATCHES "3" OR "${CLANG}" MATCHES "usan") - set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -fsanitize=undefined") - set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -fsanitize=undefined") + # NOTE: keep in sync with list in Makefile + set(USAN_CHECKS_ENABLE + undefined # Default checks + # Extra checks + float-divide-by-zero + implicit-conversion + local-bounds + ) + list(JOIN USAN_CHECKS_ENABLE "," USAN_CHECKS_ENABLE) + + set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -fsanitize=${USAN_CHECKS_ENABLE} -fsanitize-recover=${USAN_CHECKS_ENABLE}") + set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -fsanitize=${USAN_CHECKS_ENABLE} -fsanitize-recover=${USAN_CHECKS_ENABLE}") + else() + message(FATAL_ERROR "Unknown CLANG setting: ${CLANG}") endif() endif() # GCOV diff --git a/Makefile b/Makefile index d14ddfbd1b..138a4cdaed 100644 --- a/Makefile +++ b/Makefile @@ -77,8 +77,12 @@ LDFLAGS += -fsanitize=address endif ifeq "$(CLANG)" "3" CC = $(CCCLANG) -CFLAGS += -fsanitize=undefined -LDFLAGS += -fsanitize=undefined +# NOTE: keep in sync with list in CMakeLists.txt +usan_checks = undefined,float-divide-by-zero,implicit-conversion,local-bounds +CFLAGS += -fsanitize=$(usan_checks) +CFLAGS += -fsanitize-recover=$(usan_checks) +LDFLAGS += -fsanitize=$(usan_checks) +LDFLAGS += -fsanitize-recover=$(usan_checks) endif ifeq "$(RELEASE)" "1" diff --git a/scripts/ubsan.supp b/scripts/ubsan.supp new file mode 100644 index 0000000000..54461a31dd --- /dev/null +++ b/scripts/ubsan.supp @@ -0,0 +1,56 @@ +# From self_test.py +alignment:hrtf_file_reader.c +bounds:dec_acelp.c +bounds:enc_acelp.c +bounds:enc_gain.c +bounds:ivas_spar_decoder.c +bounds:trans_direct.c +bounds:trans_inv.c +float-cast-overflow:ivas_dirac_com.c +float-cast-overflow:ivas_mct_core_enc.c +float-divide-by-zero:ivas_mct_core_enc.c +float-divide-by-zero:ivas_stereo_cng_dec.c +float-divide-by-zero:swb_tbe_enc.c +implicit-integer-sign-change:ACcontextMapping.c +implicit-integer-sign-change:avq_dec.c +implicit-integer-sign-change:bitstream.c +implicit-integer-sign-change:cod4t64.c +implicit-integer-sign-change:dec_prm.c +implicit-integer-sign-change:dec4t64.c +implicit-integer-sign-change:enc_acelp.c +implicit-integer-sign-change:enc_prm.c +implicit-integer-sign-change:enh40.c +implicit-integer-sign-change:inov_dec.c +implicit-integer-sign-change:inov_enc.c +implicit-integer-sign-change:ivas_mdct_core_dec.c +implicit-integer-sign-change:jbm_jb4sb.c +implicit-integer-sign-change:jbm_pcmdsp_apa.c +implicit-integer-sign-change:cod4t64_fast.c +implicit-integer-sign-change:range_enc.c +implicit-integer-sign-change:tcq_position_arith.c +implicit-integer-sign-change:tonalMDCTconcealment.c +implicit-signed-integer-truncation:ACcontextMapping_dec.c +implicit-signed-integer-truncation:ACcontextMapping_enc.c +implicit-signed-integer-truncation:cng_dec.c +implicit-signed-integer-truncation:cod_tcx.c +implicit-signed-integer-truncation:dec_tcx.c +implicit-signed-integer-truncation:hdecnrm.c +implicit-signed-integer-truncation:ivas_qmetadata_enc.c +implicit-signed-integer-truncation:lib_dec.c +implicit-signed-integer-truncation:longarith.c +implicit-signed-integer-truncation:pvq_core_dec.c +implicit-signed-integer-truncation:pvq_core_enc.c +implicit-signed-integer-truncation:tcq_position_arith.c +implicit-signed-integer-truncation:tools.c +null:ivas_dirac_com.c +pointer-overflow:ivas_dirac_dec.c +pointer-overflow:ivas_dirac_output_synthesis_dec.c +shift-base:basop32.c +shift-base:enh40.c +shift-base:enh40.h +shift-base:enh1632.c +shift-base:hq_lr_dec.c +signed-integer-overflow:basop32.c + +# renderer test +null:lib_rend.c diff --git a/tests/renderer/test_renderer.py b/tests/renderer/test_renderer.py index 22f439c8b1..c057ff9a02 100644 --- a/tests/renderer/test_renderer.py +++ b/tests/renderer/test_renderer.py @@ -36,13 +36,13 @@ from .utils import * @pytest.mark.parametrize("out_fmt", OUTPUT_FORMATS[2:]) @pytest.mark.parametrize("in_fmt", INPUT_FORMATS_AMBI) def test_ambisonics(test_info, in_fmt, out_fmt): - run_renderer(in_fmt, out_fmt) + run_renderer(in_fmt, out_fmt, test_case_name=test_info.node.name) @pytest.mark.parametrize("out_fmt", OUTPUT_FORMATS_BINAURAL) @pytest.mark.parametrize("in_fmt", INPUT_FORMATS_AMBI) def test_ambisonics_binaural_static(test_info, in_fmt, out_fmt): - run_renderer(in_fmt, out_fmt) + run_renderer(in_fmt, out_fmt, test_case_name=test_info.node.name) @pytest.mark.parametrize("trj_file", HR_TRAJECTORIES_TO_TEST) @@ -52,6 +52,7 @@ def test_ambisonics_binaural_headrotation(test_info, in_fmt, out_fmt, trj_file): run_renderer( in_fmt, out_fmt, + test_case_name=test_info.node.name, trj_file=HR_TRAJECTORY_DIR.joinpath(f"{trj_file}.csv"), ) @@ -251,7 +252,7 @@ def test_ambisonics_binaural_headrotation_refveclev_vs_refvec( @pytest.mark.parametrize("out_fmt", OUTPUT_FORMATS[2:]) @pytest.mark.parametrize("in_fmt", INPUT_FORMATS_MC) def test_multichannel(test_info, in_fmt, out_fmt): - run_renderer(in_fmt, out_fmt) + run_renderer(in_fmt, out_fmt, test_case_name=test_info.node.name) @pytest.mark.parametrize("out_fmt", OUTPUT_FORMATS_BINAURAL) @@ -260,7 +261,7 @@ def test_multichannel_binaural_static(test_info, in_fmt, out_fmt): if in_fmt in ["MONO", "STEREO"]: pytest.skip("MONO or STEREO to Binaural rendering unsupported") - run_renderer(in_fmt, out_fmt) + run_renderer(in_fmt, out_fmt, test_case_name=test_info.node.name) @pytest.mark.parametrize("trj_file", HR_TRAJECTORIES_TO_TEST) @@ -274,12 +275,14 @@ def test_multichannel_binaural_headrotation(test_info, in_fmt, out_fmt, trj_file run_renderer( in_fmt, out_fmt, + test_case_name=test_info.node.name, trj_file=HR_TRAJECTORY_DIR.joinpath(f"{trj_file}.csv"), ) else: run_renderer( in_fmt, out_fmt, + test_case_name=test_info.node.name, trj_file=HR_TRAJECTORY_DIR.joinpath(f"{trj_file}.csv"), ) @@ -319,7 +322,7 @@ def test_multichannel_binaural_headrotation_refvec_rotating(test_info, in_fmt, o @pytest.mark.parametrize("out_fmt", OUTPUT_FORMATS[2:]) @pytest.mark.parametrize("in_fmt", INPUT_FORMATS_ISM) def test_ism(test_info, in_fmt, out_fmt): - run_renderer(in_fmt, out_fmt, in_meta_files=FORMAT_TO_METADATA_FILES[in_fmt]) + run_renderer(in_fmt, out_fmt, test_case_name=test_info.node.name, in_meta_files=FORMAT_TO_METADATA_FILES[in_fmt]) @pytest.mark.parametrize("out_fmt", OUTPUT_FORMATS_BINAURAL) @@ -331,9 +334,9 @@ def test_ism_binaural_static(test_info, in_fmt, out_fmt): in_meta_files = None if out_fmt == "BINAURAL": - run_renderer(in_fmt, out_fmt, in_meta_files=in_meta_files) + run_renderer(in_fmt, out_fmt, test_case_name=test_info.node.name, in_meta_files=in_meta_files) else: - run_renderer(in_fmt, out_fmt, in_meta_files=in_meta_files) + run_renderer(in_fmt, out_fmt, test_case_name=test_info.node.name, in_meta_files=in_meta_files) @pytest.mark.parametrize("trj_file", HR_TRAJECTORIES_TO_TEST) @@ -349,6 +352,7 @@ def test_ism_binaural_headrotation(test_info, in_fmt, out_fmt, trj_file): run_renderer( in_fmt, out_fmt, + test_case_name=test_info.node.name, trj_file=HR_TRAJECTORY_DIR.joinpath(f"{trj_file}.csv"), in_meta_files=in_meta_files, ) @@ -356,6 +360,7 @@ def test_ism_binaural_headrotation(test_info, in_fmt, out_fmt, trj_file): run_renderer( in_fmt, out_fmt, + test_case_name=test_info.node.name, trj_file=HR_TRAJECTORY_DIR.joinpath(f"{trj_file}.csv"), in_meta_files=in_meta_files, ) @@ -400,7 +405,7 @@ def test_ism_binaural_headrotation_refvec_rotating(test_info, in_fmt, out_fmt): @pytest.mark.parametrize("out_fmt", OUTPUT_FORMATS) @pytest.mark.parametrize("in_fmt", INPUT_FORMATS_MASA) def test_masa(test_info, in_fmt, out_fmt): - run_renderer(in_fmt, out_fmt, in_meta_files=FORMAT_TO_METADATA_FILES[in_fmt]) + run_renderer(in_fmt, out_fmt, test_case_name=test_info.node.name, in_meta_files=FORMAT_TO_METADATA_FILES[in_fmt]) """ Custom loudspeaker layouts """ @@ -409,7 +414,7 @@ def test_masa(test_info, in_fmt, out_fmt): @pytest.mark.parametrize("out_fmt", OUTPUT_FORMATS[2:]) @pytest.mark.parametrize("in_layout", CUSTOM_LS_TO_TEST) def test_custom_ls_input(test_info, in_layout, out_fmt): - run_renderer(CUSTOM_LAYOUT_DIR.joinpath(f"{in_layout}.txt"), out_fmt) + run_renderer(CUSTOM_LAYOUT_DIR.joinpath(f"{in_layout}.txt"), out_fmt, test_case_name=test_info.node.name) @pytest.mark.parametrize("out_fmt", CUSTOM_LS_TO_TEST) @@ -418,6 +423,7 @@ def test_custom_ls_output(test_info, in_fmt, out_fmt): run_renderer( in_fmt, CUSTOM_LAYOUT_DIR.joinpath(f"{out_fmt}.txt"), + test_case_name=test_info.node.name, ) @@ -427,6 +433,7 @@ def test_custom_ls_input_output(test_info, in_fmt, out_fmt): run_renderer( CUSTOM_LAYOUT_DIR.joinpath(f"{in_fmt}.txt"), CUSTOM_LAYOUT_DIR.joinpath(f"{out_fmt}.txt"), + test_case_name=test_info.node.name, ) @@ -436,6 +443,7 @@ def test_custom_ls_input_binaural(test_info, in_layout, out_fmt): run_renderer( CUSTOM_LAYOUT_DIR.joinpath(f"{in_layout}.txt"), out_fmt, + test_case_name=test_info.node.name, ) @@ -446,6 +454,7 @@ def test_custom_ls_input_binaural_headrotation(test_info, in_layout, out_fmt, tr run_renderer( CUSTOM_LAYOUT_DIR.joinpath(f"{in_layout}.txt"), out_fmt, + test_case_name=test_info.node.name, trj_file=HR_TRAJECTORY_DIR.joinpath(f"{trj_file}.csv"), ) @@ -459,6 +468,7 @@ def test_metadata(test_info, in_fmt, out_fmt): run_renderer( "META", out_fmt, + test_case_name=test_info.node.name, metadata_input=TEST_VECTOR_DIR.joinpath(f"{in_fmt}.txt"), ) @@ -470,11 +480,11 @@ def test_metadata(test_info, in_fmt, out_fmt): @pytest.mark.parametrize("in_fmt", ["MONO"]) @pytest.mark.parametrize("non_diegetic_pan", ["0", "-30", "45", "90", "-90"]) def test_non_diegetic_pan_static(test_info, in_fmt, out_fmt, non_diegetic_pan): - run_renderer(in_fmt, out_fmt, non_diegetic_pan=non_diegetic_pan) + run_renderer(in_fmt, out_fmt, test_case_name=test_info.node.name, non_diegetic_pan=non_diegetic_pan) @pytest.mark.parametrize("out_fmt", ["STEREO"]) @pytest.mark.parametrize("in_fmt", ["ISM1"]) @pytest.mark.parametrize("non_diegetic_pan", ["0", "-30", "45", "90", "-90"]) def test_non_diegetic_pan_ism_static(test_info, in_fmt, out_fmt, non_diegetic_pan): - run_renderer(in_fmt, out_fmt, non_diegetic_pan=non_diegetic_pan) + run_renderer(in_fmt, out_fmt, test_case_name=test_info.node.name, non_diegetic_pan=non_diegetic_pan) diff --git a/tests/renderer/utils.py b/tests/renderer/utils.py index 356902ff47..cf4bbd1bdf 100644 --- a/tests/renderer/utils.py +++ b/tests/renderer/utils.py @@ -29,6 +29,7 @@ import logging import subprocess as sp import sys +import os from pathlib import Path from tempfile import TemporaryDirectory from typing import Optional, Tuple, Dict @@ -47,11 +48,10 @@ import pyaudio3dtools def test_info(request): return request - -def run_cmd(cmd): +def run_cmd(cmd, env=None): logging.info(f"\nRunning command\n{' '.join(cmd)}\n") try: - sp.run(cmd, check=True, capture_output=True, text=True) + sp.run(cmd, check=True, capture_output=True, text=True, env=env) except sp.CalledProcessError as e: raise SystemError( f"Command returned non-zero exit status ({e.returncode}): {' '.join(e.cmd)}\n{e.stderr}\n{e.stdout}" @@ -114,6 +114,7 @@ def run_renderer( output_path_base: str = OUTPUT_PATH_CUT, binary_suffix: str = "", is_comparetest: Optional[bool] = False, + test_case_name: Optional[str] = None, ) -> Tuple[np.ndarray, int]: """CuT creation with standalone renderer""" if trj_file is not None: @@ -200,7 +201,12 @@ def run_renderer( if config_file is not None: cmd.extend(["-rc", str(config_file)]) - run_cmd(cmd) + # Set env variables for UBSAN + env = os.environ.copy() + if test_case_name and "UBSAN_OPTIONS" in env.keys(): + env["UBSAN_OPTIONS"] = env["UBSAN_OPTIONS"] + f",log_path=usan_log_{test_case_name}" + + run_cmd(cmd, env) return pyaudio3dtools.audiofile.readfile(out_file) @@ -208,20 +214,21 @@ def compare_renderer_vs_mergetarget(test_info, in_fmt, out_fmt, **kwargs): ref, ref_fs = run_renderer( in_fmt, out_fmt, + test_case_name=test_info.node.name, binary_suffix=BIN_SUFFIX_MERGETARGET, output_path_base=OUTPUT_PATH_REF, **kwargs, ) - cut, cut_fs = run_renderer(in_fmt, out_fmt, **kwargs) + cut, cut_fs = run_renderer(in_fmt, out_fmt, test_case_name=test_info.node.name, **kwargs) check_BE(test_info, ref, ref_fs, cut, cut_fs) def compare_renderer_vs_pyscripts(test_info, in_fmt, out_fmt, **kwargs): ref, ref_fs = run_pyscripts(in_fmt, out_fmt, **kwargs) - cut, cut_fs = run_renderer(in_fmt, out_fmt, **kwargs) + cut, cut_fs = run_renderer(in_fmt, out_fmt, test_case_name=test_info.node.name, **kwargs) check_BE(test_info, ref, ref_fs, cut, cut_fs) def compare_renderer_args(test_info, in_fmt, out_fmt, ref_kwargs: Dict, cut_kwargs: Dict): - ref, ref_fs = run_renderer(in_fmt, out_fmt, **ref_kwargs) - cut, cut_fs = run_renderer(in_fmt, out_fmt, **cut_kwargs) + ref, ref_fs = run_renderer(in_fmt, out_fmt, test_case_name=test_info.node.name, **ref_kwargs) + cut, cut_fs = run_renderer(in_fmt, out_fmt, test_case_name=test_info.node.name, **cut_kwargs) check_BE(test_info, ref, ref_fs, cut, cut_fs) \ No newline at end of file -- GitLab From 3984d9afd8ef1b7318e143ebcccfa261bf50d441 Mon Sep 17 00:00:00 2001 From: knj Date: Tue, 25 Jul 2023 12:25:05 +0200 Subject: [PATCH 2/2] update suppression file --- scripts/ubsan.supp | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/scripts/ubsan.supp b/scripts/ubsan.supp index 54461a31dd..8f312aa15a 100644 --- a/scripts/ubsan.supp +++ b/scripts/ubsan.supp @@ -7,8 +7,6 @@ bounds:ivas_spar_decoder.c bounds:trans_direct.c bounds:trans_inv.c float-cast-overflow:ivas_dirac_com.c -float-cast-overflow:ivas_mct_core_enc.c -float-divide-by-zero:ivas_mct_core_enc.c float-divide-by-zero:ivas_stereo_cng_dec.c float-divide-by-zero:swb_tbe_enc.c implicit-integer-sign-change:ACcontextMapping.c @@ -45,12 +43,10 @@ implicit-signed-integer-truncation:tools.c null:ivas_dirac_com.c pointer-overflow:ivas_dirac_dec.c pointer-overflow:ivas_dirac_output_synthesis_dec.c +pointer-overflow:ivas_dirac_rend.c shift-base:basop32.c shift-base:enh40.c shift-base:enh40.h shift-base:enh1632.c shift-base:hq_lr_dec.c signed-integer-overflow:basop32.c - -# renderer test -null:lib_rend.c -- GitLab