From 8b3fbac3a997ea9ef337cb86a426e95f6a430308 Mon Sep 17 00:00:00 2001 From: knj Date: Mon, 29 Jan 2024 13:55:26 +0100 Subject: [PATCH 1/5] add timeout for runIvasCodec.py --- scripts/pyivastest/IvasModeRunner.py | 46 +++++++++++++++++-------- scripts/pyivastest/IvasScriptsCommon.py | 14 ++++++++ scripts/runIvasCodec.py | 1 + 3 files changed, 47 insertions(+), 14 deletions(-) diff --git a/scripts/pyivastest/IvasModeRunner.py b/scripts/pyivastest/IvasModeRunner.py index c67f252e1d..57ef3ba31d 100644 --- a/scripts/pyivastest/IvasModeRunner.py +++ b/scripts/pyivastest/IvasModeRunner.py @@ -50,6 +50,8 @@ import pyaudio3dtools.audiofile as af import pyaudio3dtools.audioarray as ar BW_TO_SR = {"nb": 8, "wb": 16, "swb": 32, "fb": 48} +RET_CODE_TIMEOUT_EXP = 9999 + class Error(Exception): pass @@ -85,6 +87,7 @@ class IvasModeRunner(IvasModeCollector.IvasModeCollector): console_logger_level="", logger_name="IvasBaseClass", log_level=logging.DEBUG, + timeout=None, ): super().__init__( @@ -97,6 +100,7 @@ class IvasModeRunner(IvasModeCollector.IvasModeCollector): logger_name=logger_name, log_level=log_level, ) + self.timeout = timeout self.lock = threading.Lock() self.dec_queue = None self.enc_queue = None @@ -317,13 +321,22 @@ class IvasModeRunner(IvasModeCollector.IvasModeCollector): + [enc_file_name, dec_file_name] ) - cur_dec_result = subprocess.run(dec_cmd, capture_output=True, text=True, env=self.run_env) dec_log.write(" ".join(dec_cmd)) - dec_log.write(cur_dec_result.stderr) - dec_log.write(cur_dec_result.stdout) - if cur_dec_result.returncode != 0: + try: + cur_dec_result = subprocess.run(dec_cmd, capture_output=True, text=True, env=self.run_env, timeout=self.timeout) + returncode = cur_dec_result.returncode + dec_log.write(cur_dec_result.stderr) + dec_log.write(cur_dec_result.stdout) + except subprocess.TimeoutExpired: + returncode = RET_CODE_TIMEOUT_EXP + + if returncode != 0: + fail_string = "Decoding {} to {} failed!" + if returncode == RET_CODE_TIMEOUT_EXP: + fail_string = fail_string[:-1] + f" due to timeout after {self.timeout} seconds!" + self.logger.error( - "Decoding {} to {} failed!".format(enc_file_name, dec_file_name) + fail_string.format(enc_file_name, dec_file_name) ) self.lock.acquire() if self.stats: @@ -331,9 +344,7 @@ class IvasModeRunner(IvasModeCollector.IvasModeCollector): self.show_progress() self.results.append( [ - "Decoding failed to {} for {}".format( - output_config, enc_file_name - ), + fail_string.format(output_config, enc_file_name), config["ivas_format"], enc_dec_cmd["table_name"], dec_log_name, @@ -706,10 +717,13 @@ class IvasModeRunner(IvasModeCollector.IvasModeCollector): enc_log = open(enc_log_name, "w") enc_log.write(" ".join(enc_cmd)) - enc_result = subprocess.run(enc_cmd, capture_output=True, text=True, env=self.run_env) - error = enc_result.returncode - enc_log.write(enc_result.stderr) - enc_log.write(enc_result.stdout) + try: + enc_result = subprocess.run(enc_cmd, capture_output=True, text=True, env=self.run_env, timeout=self.timeout) + error = enc_result.returncode + enc_log.write(enc_result.stderr) + enc_log.write(enc_result.stdout) + except subprocess.TimeoutExpired: + error = RET_CODE_TIMEOUT_EXP if error == 0 and "bitstream_processing" in enc_dec_cmd: bs_in_file = enc_file_name @@ -797,13 +811,17 @@ class IvasModeRunner(IvasModeCollector.IvasModeCollector): self.lock.release() if error != 0: + fail_string = "Encoding failed for {}" + if error == RET_CODE_TIMEOUT_EXP: + fail_string = fail_string + f" due to timeout after {self.timeout} seconds" + self.lock.acquire() if self.stats: self.stats["num_enc_errors"] += 1 self.show_progress() self.results.append( [ - "Encoding failed for {}".format(enc_file_name), + fail_string.format(enc_file_name), mode, enc_dec_cmd["table_name"], enc_log_name, @@ -811,7 +829,7 @@ class IvasModeRunner(IvasModeCollector.IvasModeCollector): ) self.failed_modes["enc"].append(mode) self.lock.release() - self.logger.error("Encoding failed for {}".format(enc_file_name)) + self.logger.error(fail_string.format(enc_file_name)) else: self.logger.info("Encoding successful for {}".format(enc_file_name)) diff --git a/scripts/pyivastest/IvasScriptsCommon.py b/scripts/pyivastest/IvasScriptsCommon.py index 0d1558c8ec..0d5cc990b6 100644 --- a/scripts/pyivastest/IvasScriptsCommon.py +++ b/scripts/pyivastest/IvasScriptsCommon.py @@ -141,6 +141,11 @@ class IvasScriptArgParser(argparse.ArgumentParser): default=multiprocessing.cpu_count(), type=int, ) + self.add_argument( + "--timeout", + help="Timeout duration for an individual encoder/decoder run", + default=None, + ) if not minimal: self.add_argument( "-C", @@ -339,6 +344,15 @@ class IvasScriptArgParser(argparse.ArgumentParser): args["error"] = 1 args["exit"] = 1 + if args["timeout"] is not None: + try: + timeout = float(args["timeout"]) + args["timeout"] = timeout + except ValueError: + print(f"Given timeout value {args['timeout']} is not a valid number.") + args["error"] = 1 + args["exit"] = 1 + if not self.minimal: format_dict = self.get_format_dict(args["format_file"]) if format_dict == -1: diff --git a/scripts/runIvasCodec.py b/scripts/runIvasCodec.py index 7cf1d64a6b..f8486ac5f3 100755 --- a/scripts/runIvasCodec.py +++ b/scripts/runIvasCodec.py @@ -133,6 +133,7 @@ class RunIvasCodec(IvasScriptsCommon.IvasScript): sample_rate_dec_out=self.args["srout"], enable_logging=True, logger_name="{}.runner".format(self.logger.name), + timeout=self.args["timeout"], ) IvasScriptsCommon.runner_setup(runner, self.args) -- GitLab From 82b2fc255032914b718fa02084f64686b5bb5c6f Mon Sep 17 00:00:00 2001 From: knj Date: Mon, 29 Jan 2024 14:05:11 +0100 Subject: [PATCH 2/5] add timeout for IvasBuildAndRunChecks.py too --- scripts/pyivastest/IvasScriptsCommon.py | 3 +++ scripts/pyivastest/IvasSvnBuilder.py | 2 ++ 2 files changed, 5 insertions(+) diff --git a/scripts/pyivastest/IvasScriptsCommon.py b/scripts/pyivastest/IvasScriptsCommon.py index 0d5cc990b6..a6158b9908 100644 --- a/scripts/pyivastest/IvasScriptsCommon.py +++ b/scripts/pyivastest/IvasScriptsCommon.py @@ -704,6 +704,9 @@ def runner_setup(runner, args): if args["filter_regex"]: runner.filter = args["filter_regex"] + if args["timeout"] is not None: + runner.timeout = args["timeout"] + def analyzer_setup(analyzer, args): bs_proc_chain = {} diff --git a/scripts/pyivastest/IvasSvnBuilder.py b/scripts/pyivastest/IvasSvnBuilder.py index a935ae4e8f..99b5ac57b6 100644 --- a/scripts/pyivastest/IvasSvnBuilder.py +++ b/scripts/pyivastest/IvasSvnBuilder.py @@ -775,6 +775,7 @@ class IvasBuilderAndRunner(IvasBaseClass): formats_dict={}, formats_fname="", max_workers=1, + timeout=None, ): """ @@ -840,6 +841,7 @@ class IvasBuilderAndRunner(IvasBaseClass): enable_logging=True, logger_name="{}.{}runner".format(self.logger.name, cfg_name), log_level=self.logger.level, + timeout=timeout, ) if format_select_list: newRunner.set_format_select_list(format_select_list) -- GitLab From 387f6098a4a55ea3bda1a2ec2181251c916ee859 Mon Sep 17 00:00:00 2001 From: knj Date: Mon, 29 Jan 2024 14:23:31 +0100 Subject: [PATCH 3/5] use timeout in ci scripts --- .gitlab-ci.yml | 6 +++--- ci/run_scheduled_sanitizer_test.py | 3 +++ ci/smoke_test.sh | 17 +++++++++-------- 3 files changed, 15 insertions(+), 11 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index c306a2537b..1092d94066 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -989,11 +989,11 @@ check-first-frame-is-sid: # run all modes and cut bitstream to start with an SID. Use stereo output to limit runtime, test is only about decoding - ism_md_cmd="--ism_metadata_files /usr/local/ltv/ltvISM1.csv /usr/local/ltv/ltvISM2.csv /usr/local/ltv/ltvISM3.csv /usr/local/ltv/ltvISM4.csv" - modes=$(scripts/runIvasCodec.py -l | grep dtx | grep -vE "FOA|HOA" ) - - scripts/runIvasCodec.py -z console -p scripts/config/ci_linux_sidstart_test.json -m $modes -s --bs_length 200 -U 0:20 --oc stereo $ism_md_cmd || exit_code_no_sba=$? + - scripts/runIvasCodec.py -z console -p scripts/config/ci_linux_sidstart_test.json -m $modes -s --bs_length 200 -U 0:20 --oc stereo $ism_md_cmd --timeout 10 || exit_code_no_sba=$? - modes=$(scripts/runIvasCodec.py -l | grep dtx | grep -E "HOA") - - scripts/runIvasCodec.py -z console -p scripts/config/ci_linux_sidstart_test.json -m $modes -s --bs_length 100 -U 70:80 --oc stereo || exit_code_hoa=$? + - scripts/runIvasCodec.py -z console -p scripts/config/ci_linux_sidstart_test.json -m $modes -s --bs_length 100 -U 70:80 --oc stereo --timeout 10 || exit_code_hoa=$? - modes=$(scripts/runIvasCodec.py -l | grep dtx | grep "FOA") - - scripts/runIvasCodec.py -z console -p scripts/config/ci_linux_sidstart_test.json -m $modes -s --bs_length 100 -U 75:110 --oc stereo || exit_code_foa=$? + - scripts/runIvasCodec.py -z console -p scripts/config/ci_linux_sidstart_test.json -m $modes -s --bs_length 100 -U 75:110 --oc stereo timeout 10 || exit_code_foa=$? - if [ $exit_code_no_sba -ne 0 ] || [ $exit_code_hoa -ne 0 ] || [ $exit_code_hoa -ne 0 ]; then exit 1; fi artifacts: diff --git a/ci/run_scheduled_sanitizer_test.py b/ci/run_scheduled_sanitizer_test.py index 5007914b95..30d7bbeb7d 100755 --- a/ci/run_scheduled_sanitizer_test.py +++ b/ci/run_scheduled_sanitizer_test.py @@ -148,6 +148,9 @@ def run_check(in_format: str, out_formats: list, tests: list, run_fec: bool = Tr *md_file_command, "--usan_supp_file", USAN_SUPP_FILE, + # timeout of 5 minutes per en/decoding to safeguard against endless loops + "--timeout", + 60 * 5, ] print( diff --git a/ci/smoke_test.sh b/ci/smoke_test.sh index a2f3ffb82b..e38c3d7dd3 100755 --- a/ci/smoke_test.sh +++ b/ci/smoke_test.sh @@ -56,6 +56,7 @@ dly_profile=./scripts/dly_error_profiles/dly_error_profile_10.dat ism_md_cmd="--ism_metadata_files /usr/local/ltv/ltvISM1.csv /usr/local/ltv/ltvISM2.csv /usr/local/ltv/ltvISM3.csv /usr/local/ltv/ltvISM4.csv" duration_arg="-U 1:2" verbosity_cmd="-z console" +timeout_cmd="--timeout 5" ep_file="ci/complexity_measurements/ep_10pct_fer.g192" if [ $BUILD -eq 1 ];then @@ -84,21 +85,21 @@ echo "\n======================= 0. preparing combined format test inputs ======= ism_modes=$(./scripts/runIvasCodec.py -l | grep ^ISM) non_ism_modes=$(./scripts/runIvasCodec.py -l | grep -v ^ISM) echo "\n======================= 1. non-ism modes no FEC =======================\n\n" -./scripts/runIvasCodec.py $verbosity_cmd -m $non_ism_modes -p $cfg $duration_arg | tee smoke_test_output.txt +./scripts/runIvasCodec.py $verbosity_cmd -m $non_ism_modes -p $cfg $duration_arg $timeout_cmd | tee smoke_test_output.txt echo "\n======================= 2. ism modes no FEC =======================\n\n" -./scripts/runIvasCodec.py $verbosity_cmd -m $ism_modes -p $cfg $duration_arg $ism_md_cmd | tee smoke_test_output.txt +./scripts/runIvasCodec.py $verbosity_cmd -m $ism_modes -p $cfg $duration_arg $ism_md_cmd $timeout_cmd | tee smoke_test_output.txt # run the decoding again, but with 15% frame loss echo "\n======================= 3. all modes with FEC =======================\n\n" -./scripts/runIvasCodec.py $verbosity_cmd -p $cfg $duration_arg -f="$ep_file" --decoder_only | tee smoke_test_output_plc.txt +./scripts/runIvasCodec.py $verbosity_cmd -p $cfg $duration_arg -f="$ep_file" --decoder_only $timeout_cmd | tee smoke_test_output_plc.txt # run JBM modes - EXT is excluded as not supported yet # OMASA disabled for now modes_with_no_ext_out=$(./scripts/runIvasCodec.py -l | grep -v ^MASA | grep -v ^ISM | grep -v OMASA) modes_with_ext_out=$(./scripts/runIvasCodec.py -l | grep 'MASA\|ISM' | grep -v ^ISM+ | grep -v OMASA) echo "\n======================= 4. JBM, modes with no EXT =======================\n\n" -./scripts/runIvasCodec.py $verbosity_cmd -m $modes_with_no_ext_out -p $cfg $duration_arg --decoder_only --jbm_file $dly_profile | tee smoke_test_output_jbm_noEXT.txt +./scripts/runIvasCodec.py $verbosity_cmd -m $modes_with_no_ext_out -p $cfg $duration_arg --decoder_only --jbm_file $dly_profile $timeout_cmd | tee smoke_test_output_jbm_noEXT.txt echo "\n======================= 5. JBM, modes with EXT =======================\n\n" -./scripts/runIvasCodec.py $verbosity_cmd -m $modes_with_ext_out -p $cfg $duration_arg --decoder_only --jbm_file $dly_profile --oc BINAURAL BINAURAL_ROOM_IR mono stereo FOA HOA3 5_1 7_1_4 | tee -a smoke_test_output_jbm_noEXT.txt +./scripts/runIvasCodec.py $verbosity_cmd -m $modes_with_ext_out -p $cfg $duration_arg --decoder_only --jbm_file $dly_profile --oc BINAURAL BINAURAL_ROOM_IR mono stereo FOA HOA3 5_1 7_1_4 $timeout_cmd | tee -a smoke_test_output_jbm_noEXT.txt # run all modes with binaural output using external files modes_with_bin_out="FOA HOA2 HOA3 PlanarFOA PlanarHOA2 PlanarHOA3 MASA MC ISM1 ISM2 ISM3 ISM4" @@ -107,14 +108,14 @@ bin_out_modes="BINAURAL BINAURAL_ROOM_IR" echo "\n======================= 6. binaural out with HRTF files - WB =======================\n\n" wb_modes=$(./scripts/runIvasCodec.py -l -C $modes_with_bin_out | grep _wb_) hrtf_wb="../scripts/binauralRenderer_interface/binaural_renderers_hrtf_data/ivas_binaural_16kHz.bin" -./scripts/runIvasCodec.py $verbosity_cmd -p $cfg -m $wb_modes $duration_arg -D="-hrtf ${hrtf_wb}" --decoder_only --oc $bin_out_modes | tee -a smoke_test_output_hrtf.txt +./scripts/runIvasCodec.py $verbosity_cmd -p $cfg -m $wb_modes $duration_arg -D="-hrtf ${hrtf_wb}" --decoder_only --oc $bin_out_modes $timeout_cmd | tee -a smoke_test_output_hrtf.txt echo "\n======================= 7. binaural out with HRTF files - SWB =======================\n\n" swb_modes=$(./scripts/runIvasCodec.py -l -C $modes_with_bin_out | grep _swb_) hrtf_swb="../scripts/binauralRenderer_interface/binaural_renderers_hrtf_data/ivas_binaural_32kHz.bin" -./scripts/runIvasCodec.py $verbosity_cmd -p $cfg -m $swb_modes $duration_arg -D="-hrtf ${hrtf_swb}" --decoder_only --oc $bin_out_modes | tee -a smoke_test_output_hrtf.txt +./scripts/runIvasCodec.py $verbosity_cmd -p $cfg -m $swb_modes $duration_arg -D="-hrtf ${hrtf_swb}" --decoder_only --oc $bin_out_modes $timeout_cmd | tee -a smoke_test_output_hrtf.txt echo "\n======================= 8. binaural out with HRTF files - FB =======================\n\n" fb_modes=$(./scripts/runIvasCodec.py -l -C $modes_with_bin_out | grep _fb_) hrtf_fb="../scripts/binauralRenderer_interface/binaural_renderers_hrtf_data/ivas_binaural_48kHz.bin" -./scripts/runIvasCodec.py $verbosity_cmd -p $cfg -m $fb_modes $duration_arg -D="-hrtf ${hrtf_fb}" --decoder_only --oc $bin_out_modes | tee -a smoke_test_output_hrtf.txt +./scripts/runIvasCodec.py $verbosity_cmd -p $cfg -m $fb_modes $duration_arg -D="-hrtf ${hrtf_fb}" --decoder_only --oc $bin_out_modes $timeout_cmd | tee -a smoke_test_output_hrtf.txt -- GitLab From 3796465f65f4fe43e11208af8cc14126b6ebf5be Mon Sep 17 00:00:00 2001 From: knj Date: Mon, 29 Jan 2024 14:24:36 +0100 Subject: [PATCH 4/5] increase timeout to be safe for good runs --- ci/run_scheduled_sanitizer_test.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ci/run_scheduled_sanitizer_test.py b/ci/run_scheduled_sanitizer_test.py index 30d7bbeb7d..33148897ca 100755 --- a/ci/run_scheduled_sanitizer_test.py +++ b/ci/run_scheduled_sanitizer_test.py @@ -148,9 +148,9 @@ def run_check(in_format: str, out_formats: list, tests: list, run_fec: bool = Tr *md_file_command, "--usan_supp_file", USAN_SUPP_FILE, - # timeout of 5 minutes per en/decoding to safeguard against endless loops + # timeout of 15 minutes per en/decoding to safeguard against endless loops "--timeout", - 60 * 5, + 60 * 15, ] print( -- GitLab From dd694c6cae2e3c30e773b86f0fcd657545227052 Mon Sep 17 00:00:00 2001 From: knj Date: Mon, 29 Jan 2024 14:54:53 +0100 Subject: [PATCH 5/5] set timeout values for pytest jobs too --- .gitlab-ci.yml | 45 ++++++++++++++++++++++++++++----------------- 1 file changed, 28 insertions(+), 17 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 1092d94066..e45d569347 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -515,7 +515,8 @@ renderer-smoke-test: stage: test script: - make -j IVAS_rend - - python3 -m pytest -q -n auto -rA --junit-xml=report-junit.xml tests/renderer/test_renderer.py + - testcase_timeout=60 + - python3 -m pytest -q -n auto -rA --junit-xml=report-junit.xml tests/renderer/test_renderer.py --testcase_timeout=$testcase_timeout artifacts: name: "mr-$CI_MERGE_REQUEST_IID--sha-$CI_COMMIT_SHORT_SHA--job-$CI_JOB_NAME--results" expire_in: 1 week @@ -537,7 +538,8 @@ renderer-asan: script: - cmake -B cmake-build -G "Unix Makefiles" -DCLANG=asan -DCOPY_EXECUTABLES_FROM_BUILD_DIR=true - cmake --build cmake-build -- -j - - python3 -m pytest -q -n auto -rA --junit-xml=report-junit.xml tests/renderer/test_renderer.py + - testcase_timeout=180 + - python3 -m pytest -q -n auto -rA --junit-xml=report-junit.xml tests/renderer/test_renderer.py --testcase_timeout=$testcase_timeout artifacts: name: "mr-$CI_MERGE_REQUEST_IID--sha-$CI_COMMIT_SHORT_SHA--job-$CI_JOB_NAME--results" @@ -560,7 +562,8 @@ renderer-msan: script: - cmake -B cmake-build -G "Unix Makefiles" -DCLANG=msan -DCOPY_EXECUTABLES_FROM_BUILD_DIR=true - cmake --build cmake-build -- -j - - python3 -m pytest -q -n auto -rA --junit-xml=report-junit.xml tests/renderer/test_renderer.py + - testcase_timeout=180 + - python3 -m pytest -q -n auto -rA --junit-xml=report-junit.xml tests/renderer/test_renderer.py --testcase_timeout=$testcase_timeout artifacts: name: "mr-$CI_MERGE_REQUEST_IID--sha-$CI_COMMIT_SHORT_SHA--job-$CI_JOB_NAME--results" @@ -583,7 +586,8 @@ renderer-usan: script: - cmake -B cmake-build -G "Unix Makefiles" -DCLANG=usan -DCOPY_EXECUTABLES_FROM_BUILD_DIR=true - cmake --build cmake-build -- -j - - UBSAN_OPTIONS=suppressions=scripts/ubsan.supp,report_error_type=1,log_path=usan_log_catchall python3 -m pytest -q -n auto -rA --junit-xml=report-junit.xml tests/renderer/test_renderer.py + - testcase_timeout=180 + - UBSAN_OPTIONS=suppressions=scripts/ubsan.supp,report_error_type=1,log_path=usan_log_catchall python3 -m pytest -q -n auto -rA --junit-xml=report-junit.xml tests/renderer/test_renderer.py --testcase_timeout=$testcase_timeout - grep_exit_code=0 - touch usan_log_empty # Creates an empty file, this is to avoid "grep: usan_log_*: No such file or directory" in case no USAN failures are reported from pytest - grep UndefinedBehaviorSanitizer usan_log_* || grep_exit_code=$? @@ -625,13 +629,14 @@ renderer-pytest-on-merge-request: - if [ $ref_using_main == 0 ]; then git checkout $source_branch_commit_sha; fi - exit_code=0 - - python3 -m pytest -q --log-level ERROR -n auto -rA --html=report.html --self-contained-html --junit-xml=report-junit.xml tests/renderer/test_renderer.py --create_ref || exit_code=$? + - testcase_timeout=60 + - python3 -m pytest -q --log-level ERROR -n auto -rA --html=report.html --self-contained-html --junit-xml=report-junit.xml tests/renderer/test_renderer.py --create_ref --testcase_timeout=$testcase_timeout || exit_code=$? ### Run test using branch scripts and input - if [ $ref_using_main == 1 ]; then git checkout $source_branch_commit_sha; fi # run test - - python3 -m pytest -q --log-level ERROR -n auto -rA --html=report.html --self-contained-html --junit-xml=report-junit.xml tests/renderer/test_renderer_.py --create_cut || exit_code=$? + - python3 -m pytest -q --log-level ERROR -n auto -rA --html=report.html --self-contained-html --junit-xml=report-junit.xml tests/renderer/test_renderer_.py --create_cut --testcase_timeout=$testcase_timeout || exit_code=$? - zero_errors=$(cat report-junit.xml | grep -c 'errors="0"') || true - *merge-request-comparison-check @@ -661,7 +666,8 @@ split-rendering-smoke-test: script: - *enable-split-rendering - make -j INCLUDE_SPLIT=1 - - python3 -m pytest -q -n auto -rA --junit-xml=report-junit.xml tests/split_rendering/test_split_rendering.py + - testcase_timeout=10 + - python3 -m pytest -q -n auto -rA --junit-xml=report-junit.xml tests/split_rendering/test_split_rendering.py --testcase_timeout=$testcase_timeout artifacts: name: "mr-$CI_MERGE_REQUEST_IID--sha-$CI_COMMIT_SHORT_SHA--job-$CI_JOB_NAME--results" expire_in: 1 week @@ -711,7 +717,8 @@ split-rendering-pytest-on-merge-request: - if [ $ref_using_main == 0 ]; then git restore lib_com/options.h; fi # Revert changes back before checking out another branch to avoid conflicts - if [ $ref_using_main == 0 ]; then git checkout $source_branch_commit_sha; fi - exit_code=0 - - python3 -m pytest -q --log-level ERROR -n auto -rA --html=report.html --self-contained-html --junit-xml=report-junit.xml tests/split_rendering/test_split_rendering.py --create_ref || exit_code=$? + - testcase_timeout=60 + - python3 -m pytest -q --log-level ERROR -n auto -rA --html=report.html --self-contained-html --junit-xml=report-junit.xml tests/split_rendering/test_split_rendering.py --create_ref --testcase_timeout=$testcase_timeout || exit_code=$? # back to source branch - git restore lib_com/options.h # Revert changes back before checking out another branch to avoid conflicts @@ -725,7 +732,7 @@ split-rendering-pytest-on-merge-request: - if [ $ref_using_main == 1 ]; then git checkout $source_branch_commit_sha; fi # run test - - python3 -m pytest -q --log-level ERROR -n auto -rA --html=report.html --self-contained-html --junit-xml=report-junit.xml tests/split_rendering/test_split_rendering.py --create_cut || exit_code=$? + - python3 -m pytest -q --log-level ERROR -n auto -rA --html=report.html --self-contained-html --junit-xml=report-junit.xml tests/split_rendering/test_split_rendering.py --create_cut --testcase_timeout=$testcase_timeout || exit_code=$? - zero_errors=$(cat report-junit.xml | grep -c 'errors="0"') || true - *merge-request-comparison-check @@ -780,7 +787,8 @@ ivas-pytest-on-merge-request: ### run pytest - exit_code=0 - - python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR -v --html=report.html --self-contained-html --junit-xml=report-junit.xml || exit_code=$? + - testcase_timeout=60 + - python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR -v --html=report.html --self-contained-html --junit-xml=report-junit.xml --testcase_timeout=$testcase_timeout || exit_code=$? - zero_errors=$(cat report-junit.xml | grep -c 'errors="0"') || true - *merge-request-comparison-check @@ -880,14 +888,15 @@ evs-pytest-on-merge-request: ### prepare pytest # create references - - python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR/test_param_file.py -v --update_ref 1 -m create_ref --param_file scripts/config/self_test_evs.prm + - testcase_timeout=60 + - python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR/test_param_file.py -v --update_ref 1 -m create_ref --param_file scripts/config/self_test_evs.prm --testcase_timeout=$testcase_timeout ### Run test using branch scripts and input - if [ $ref_using_main == 1 ]; then git checkout $source_branch_commit_sha; fi ### run pytest for EVS cases - exit_code=0 - - python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR/test_param_file.py -v --param_file scripts/config/self_test_evs.prm --html=report.html --self-contained-html --junit-xml=report-junit-evs.xml || exit_code=$? + - python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR/test_param_file.py -v --param_file scripts/config/self_test_evs.prm --html=report.html --self-contained-html --junit-xml=report-junit-evs.xml --testcase_timeout=$testcase_timeout || exit_code=$? - zero_errors=$(cat report-junit-evs.xml | grep -c 'errors="0"') || true - *merge-request-comparison-check @@ -1129,15 +1138,16 @@ codec-comparison-on-main-push: - mv IVAS_cod_test IVAS_cod - mv IVAS_dec_test IVAS_dec # create references - - python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR -v --update_ref 1 -m create_ref - - python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR -v --update_ref 1 -m create_ref_part2 + - testcase_timeout=60 + - python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR -v --update_ref 1 -m create_ref --testcase_timeout=$testcase_timeout + - python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR -v --update_ref 1 -m create_ref_part2 --testcase_timeout=$testcase_timeout ### re-checkout the latest commit here, if ref_using_main is set - if [ $ref_using_main -eq 1 ]; then git checkout $latest_commit;fi ### run pytest - exit_code=0 - - python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR -v --html=report.html --self-contained-html --junit-xml=report-junit.xml || exit_code=$? + - python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR -v --html=report.html --self-contained-html --junit-xml=report-junit.xml --testcase_timeout=$testcase_timeout || exit_code=$? - if [ $exit_code -eq 1 ] && [ $non_be_flag == 0 ]; then echo "pytest run had failures and non-BE flag not present"; exit $EXIT_CODE_FAIL; fi - zero_errors=$(cat report-junit.xml | grep -c 'errors="0"') || true - if [ $exit_code -eq 1 ] && [ $zero_errors == 1 ]; then echo "pytest run had failures, but no errors and non-BE flag present"; exit $EXIT_CODE_NON_BE; fi @@ -1231,11 +1241,12 @@ test-long-self-test: # create references - exit_code_ref=0 - - python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR/test_param_file.py -v --update_ref 1 -m create_ref --param_file scripts/config/self_test_ltv.prm || exit_code_ref=$? + - testcase_timeout=300 + - python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR/test_param_file.py -v --update_ref 1 -m create_ref --param_file scripts/config/self_test_ltv.prm --testcase_timeout=$testcase_timeout || exit_code_ref=$? ### run pytest self-test using long test vectors - exit_code=0 - - python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR/test_param_file.py -v --param_file scripts/config/self_test_ltv.prm --html=report-ltv.html --self-contained-html --junit-xml=report-junit-ltv.xml || exit_code=$? + - python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR/test_param_file.py -v --param_file scripts/config/self_test_ltv.prm --html=report-ltv.html --self-contained-html --junit-xml=report-junit-ltv.xml --testcase_timeout=$testcase_timeout || exit_code=$? - zero_errors=$(cat report-junit-ltv.xml | grep -c 'errors="0"') || true - if [ $zero_errors != 1 ]; then echo "Run errors encountered!"; exit $EXIT_CODE_FAIL; fi -- GitLab