diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 083ef2adf840d4cfffdd7a38fc07869019d4128a..9e36a3580ddaf6a61703efcada91dac5bd144455 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -144,15 +144,42 @@ self-test-on-merge-request: - python3 ./scripts/self_test.py --encref IVAS_cod_ref --decref IVAS_dec_ref --enctest IVAS_cod_test --dectest IVAS_dec_test | tee test_output.txt ### analyse test output + + # some helper variables - "|| true" to prevent failures from grep not finding anything + - non_be_flag=$(echo $CI_MERGE_REQUEST_TITLE | grep -c --ignore-case "\[non[ -]*be\]") || true + - run_errors=$(cat test_output.txt | grep -c "test conditions had run errors") || true + - bitexact=$(cat test_output.txt | grep -c "All [0-9]* tests are bitexact") || true + - EXIT_CODE_NON_BE=123 + - EXIT_CODE_FAIL=1 + + - selftest_exit_code=0 + # check for crashes during the test, if any happened, fail the test - - if cat test_output.txt | grep -c "Run errors were encountered for the following conditions:"; then echo "Codec had run errors"; exit 1; fi - # check for non bitexact output and fail test if the merge request does not have a non-BE tag - - if ! cat test_output.txt | grep -c "All [0-9]* tests are bitexact" && ! echo $CI_MERGE_REQUEST_TITLE | grep -c --ignore-case "\[non[ -]*be\]"; then echo "Non-bitexact cases without non-BE tag encountered"; exit 1; fi + - if [ $run_errors != 0 ] ; then echo "Run errors in self_test.py"; exit $EXIT_CODE_FAIL; fi + + # check for non bitexact output and store exit code to also always run the SBA pytest + - if [ $bitexact == 0 ] && [ $non_be_flag == 0 ] ; then echo "Non-bitexact cases without non-BE tag encountered"; selftest_exit_code=$EXIT_CODE_FAIL; fi + - if [ $bitexact == 0 ] && [ $non_be_flag != 0 ]; then echo "Non-bitexact cases with non-BE tag encountered"; selftest_exit_code=$EXIT_CODE_NON_BE; fi + + ### run SBA pytest + - exit_code=0 + - python3 ./scripts/ivas_pytests/self_test_b.py --encref IVAS_cod_ref --decref IVAS_dec_ref --encdut IVAS_cod_test --decdut IVAS_dec_test || exit_code=$? + - if [ $exit_code -eq 1 ] && [ $non_be_flag == 0 ]; then echo "pytest run had failures and non-BE flag not present"; exit $EXIT_CODE_FAIL; fi + - zero_errors=$(cat report-junit.xml | grep -c 'testsuite errors="0"') || true + - if [ $exit_code -eq 1 ] && [ $zero_errors == 1]; then echo "pytest run had failures, but no errors and non-BE flag present"; exit $EXIT_CODE_NON_BE; fi + - if [ $exit_code -ne 0 ]; then echo "pytest run had errors"; exit $EXIT_CODE_FAIL; fi; + # return exit code from selftest if everything went well with the pytest run + - exit $selftest_exit_code + allow_failure: + exit_codes: + - 123 artifacts: paths: - test_output.txt - scripts/test/logs - scripts/ref/logs + reports: + junit: report-junit.xml # Pull state of a branch on 3GPP repo, push to a mirror repo. diff --git a/pytest.ini b/pytest.ini new file mode 100644 index 0000000000000000000000000000000000000000..9d9a3bbf8e4c108aa406026e9df3175c7aae17ad --- /dev/null +++ b/pytest.ini @@ -0,0 +1,8 @@ +# pytest.ini +# note: per convention, this file is placed in the root directory of the repository +[pytest] +addopts = -ra --tb=short --basetemp=./tmp -v +junit_family=xunit1 +log_file_level = DEBUG +log_format = %(asctime)s %(levelname)s %(message)s +log_date_format = %Y-%m-%d %H:%M:%S diff --git a/scripts/ivas_pytests/conftest.py b/scripts/ivas_pytests/conftest.py index fc0eff1087dd53cf845e80eff80af0b1f11c2bb0..a2b1e9d60b5d963a733b66d3abceb997d0e5bd6b 100644 --- a/scripts/ivas_pytests/conftest.py +++ b/scripts/ivas_pytests/conftest.py @@ -206,11 +206,13 @@ class EncoderFrontend: if self.stderr: stderr_str = textwrap.indent(self.stderr, prefix="\t") log_dbg_msg(f"{self._type} encoder stderr:\n{stderr_str}") - assert self.returncode == 0, self._type + " encoder terminated with a non-0 return code" + if self.returncode: + pytest.fail(f"{self._type} encoder terminated with a non-0 return code: {self.returncode}") def _check_run(self): - if self.returncode: - assert self.returncode == 0, self._type + " encoder terminated with a non-0 return code" + if self.returncode is not None: + if self.returncode: + pytest.fail(f"{self._type} encoder terminated with a non-0 return code: {self.returncode}") else: logger.warning("%s encoder was set-up, but not run", self._type) # next assert is not OK since stderr contains messages even when encoding was successful @@ -223,7 +225,7 @@ def dut_encoder_frontend(dut_encoder_path) -> EncoderFrontend: yield encoder # Fixture teardown -# encoder._check_run() + encoder._check_run() @pytest.fixture(scope="session") @@ -322,11 +324,13 @@ class DecoderFrontend: if self.stderr: stderr_str = textwrap.indent(self.stderr, prefix="\t") log_dbg_msg(f"{self._type} decoder stderr:\n{stderr_str}") - assert self.returncode == 0, self._type + " decoder terminated with a non-0 return code" + if self.returncode: + pytest.fail(f"{self._type} decoder terminated with a non-0 return code: {self.returncode}") def _check_run(self): - if self.returncode: - assert self.returncode == 0, self._type + " decoder terminated with a non-0 return code" + if self.returncode is not None: + if self.returncode: + pytest.fail(f"{self._type} decoder terminated with a non-0 return code: {self.returncode}") else: logger.warning("%s decoder was set-up, but not run", self._type) # next assert is not OK since stderr contains messages even when decoding was successful @@ -339,7 +343,7 @@ def dut_decoder_frontend(dut_decoder_path) -> DecoderFrontend: yield decoder # Fixture teardown -# decoder._check_run() + decoder._check_run() @pytest.fixture(scope="session") diff --git a/scripts/ivas_pytests/self_test_b.py b/scripts/ivas_pytests/self_test_b.py index 60d4126d04f7909bead2638305e07728049c497f..09aab53fbeb5f56d6adda2d640668db4fbc46762 100755 --- a/scripts/ivas_pytests/self_test_b.py +++ b/scripts/ivas_pytests/self_test_b.py @@ -42,15 +42,9 @@ import os import sys import argparse import subprocess -import shutil import platform from pathlib import Path -TRUNK_SVN_PATH = "https://svnext02.iis.fraunhofer.de/ivas_dev/trunk" - -SVNUSER = os.getenv("SVNUSER") -SVNPASSWD = os.getenv("SVNPASSWD") - BIN_EXT = ".exe" if platform.system() == "Windows" else "" HERE = Path(__file__).parent.resolve() DEFAULT_ENCODER_DUT = str(HERE.joinpath(f"../../IVAS_cod{BIN_EXT}").resolve()) @@ -58,54 +52,11 @@ DEFAULT_DECODER_DUT = str(HERE.joinpath(f"../../IVAS_dec{BIN_EXT}").resolve()) DEFAULT_ENCODER_REF = str(HERE.joinpath(f"../../IVAS_cod_ref{BIN_EXT}").resolve()) DEFAULT_DECODER_REF = str(HERE.joinpath(f"../../IVAS_dec_ref{BIN_EXT}").resolve()) CREND_UNITTEST_REF = str(HERE.joinpath(f"tests/unit_tests/crend/IVAS_crend_unit_test_ref{BIN_EXT}").resolve()) -DATA_DIR = str(HERE.joinpath("../../../data").resolve()) -TEST_VECTOR_DIR = str(HERE.joinpath("testv").resolve()) +TEST_VECTOR_DIR = str(HERE.joinpath("../testv").resolve()) REFERENCE_DIR = str(HERE.joinpath("ref").resolve()) DUT_BASE_DIR = str(HERE.joinpath("dut").resolve()) -def get_svn_trunk_revision(): - """ - Get the SVN HEAD revision number of the trunk folder. - """ - command = ["svn"] - if SVNUSER: - command.extend(["--username", SVNUSER]) - if SVNPASSWD: - command.extend(["--password", SVNPASSWD]) - command.extend(["info", TRUNK_SVN_PATH]) - result = subprocess.run(command, capture_output=True, check=True) - for line in result.stdout.decode("ascii").splitlines(): - if line.startswith("Revision:"): - return int(line.split()[1]) - assert 0, "unable to determine SVN trunk revision" - - -def get_svn_local_revision(): - """ - Get the SVN revision number of the local (trunk) folder. - """ - command = ["svn", "info", "."] - result = subprocess.run(command, capture_output=True, check=True) - for line in result.stdout.decode("ascii").splitlines(): - if line.startswith("Revision:"): - return int(line.split()[1]) - return 0 # unable to determine local SVN revision - - -def svn_export_trunk(revision, out_folder): - """ - Export the SVN trunk folder in a specific revision. - """ - command = ["svn"] - if SVNUSER: - command.extend(["--username", SVNUSER]) - if SVNPASSWD: - command.extend(["--password", SVNPASSWD]) - command.extend(["export", "-q", "-r", str(revision), TRUNK_SVN_PATH, out_folder]) - subprocess.run(command, check=True) - - def build_enc_and_dec(src_dir): """ Build the encoder and decoder binaries. @@ -146,40 +97,6 @@ def build_crend_unittest(src_dir): subprocess.run(command, check=True) -def build_ref_binaries(ref_decoder_path, ref_encoder_path, ref_version): - """ - Build the REF binaries. - """ - print("Building the REF binaries") - if ref_version is None: - ref_version = get_svn_local_revision() - if ref_version == 0: - ref_version = get_svn_trunk_revision() - - if ref_version == 0: - # special REF binaries generation: copy DUT binaries - print("- copy DUT binaries -> REF binaries") - dut_src_dir = str(HERE.joinpath("../..").resolve()) - shutil.copy(f"{dut_src_dir}/IVAS_cod{BIN_EXT}", ref_encoder_path) - shutil.copy(f"{dut_src_dir}/IVAS_dec{BIN_EXT}", ref_decoder_path) - shutil.copy(f"{dut_src_dir}/scripts/ivas_pytests/tests/unit_tests/crend/IVAS_crend_unit_test{BIN_EXT}", CREND_UNITTEST_REF) - return - - # is a local directory with the reference version available? - ref_src_dir = str(HERE.joinpath(f"../../../ref_src/{ref_version}").resolve()) - if not os.path.exists(ref_src_dir): - print(f"ref src dir {ref_src_dir} does not exist") - svn_export_trunk(ref_version, ref_src_dir) - build_enc_and_dec(ref_src_dir) - print(f"copy: {ref_src_dir}/IVAS_cod{BIN_EXT} -> {ref_encoder_path}") - shutil.copy(f"{ref_src_dir}/IVAS_cod{BIN_EXT}", ref_encoder_path) - print(f"copy: {ref_src_dir}/IVAS_dec{BIN_EXT} -> {ref_decoder_path}") - shutil.copy(f"{ref_src_dir}/IVAS_dec{BIN_EXT}", ref_decoder_path) - build_crend_unittest(ref_src_dir) - print(f"copy: {ref_src_dir}/scripts/ivas_pytests/tests/unit_tests/crend/IVAS_crend_unit_test{BIN_EXT} -> {CREND_UNITTEST_REF}") - shutil.copy(f"{ref_src_dir}/scripts/ivas_pytests/tests/unit_tests/crend/IVAS_crend_unit_test{BIN_EXT}", CREND_UNITTEST_REF) - - def build_dut_binaries(): """ Build the DUT binaries. @@ -190,60 +107,12 @@ def build_dut_binaries(): build_crend_unittest(dut_src_dir) -def setup_test_vector_directory(): - """ - Setup the test vector directory. - - Copy test vectors from data directory. - Current assumption: data directory is available and parallel to trunk. - Later, after some cleanup in the data folder, we can consider to do - "svn export" of the test vectors folder. - """ - os.makedirs(TEST_VECTOR_DIR) - os.makedirs(f"{TEST_VECTOR_DIR}/spar_foa_bs/enc") - os.makedirs(f"{TEST_VECTOR_DIR}/spar_foa_bs/dec") - os.makedirs(f"{TEST_VECTOR_DIR}/crend/tests/crend") - os.makedirs(f"{TEST_VECTOR_DIR}/crend/tests/reverb") - os.makedirs(f"{TEST_VECTOR_DIR}/crend/tests/proximity") - os.makedirs(f"{TEST_VECTOR_DIR}/crend/tests/ingest") - os.makedirs(f"{TEST_VECTOR_DIR}/crend/tests/otr") - systest_subfolder = ["spar_foa_bs/enc/in", "spar_foa_bs/dec/plc_file"] - for folder in systest_subfolder: - shutil.copytree(f"{DATA_DIR}/ivas_pytests/system_tests/{folder}", f"{TEST_VECTOR_DIR}/{folder}") - unittest_subfolder = [ - "crend/tests/crend/in_wav", - "crend/tests/crend/in_new_wav", - "crend/tests/crend/in_csv_wav", - "crend/tests/reverb/in_wav", - "crend/tests/proximity/in_wav", - "crend/tests/proximity/bitstream", - "crend/tests/ingest/in_wav", - "crend/tests/custom/in_wav", - "crend/tests/hr/in_wav", - "crend/tests/SOFA", - "crend/tests/default-SOFA", - "crend/tests/custom_SOFA", - "crend/tests/hrtf_ref", - "crend/tests/csv", - "crend/tests/render_config", - ] - for folder in unittest_subfolder: - shutil.copytree(f"{DATA_DIR}/ivas_pytests/unit_tests/{folder}", f"{TEST_VECTOR_DIR}/{folder}") - - def main(argv): # check for python >= 3.7 if sys.version_info[0] < 3 or sys.version_info[1] < 7: sys.exit("This script is written for Python >= 3.7. Found: " + platform.python_version()) parser = argparse.ArgumentParser(formatter_class=argparse.RawTextHelpFormatter) - parser.add_argument( - "--rref", - action="store", - type=int, - default=None, - help="SVN revision for the reference (default: HEAD)", - ) parser.add_argument( "--create_only", action="store_true", @@ -294,14 +163,11 @@ def main(argv): else: decref_path = DEFAULT_DECODER_REF if not os.path.exists(encref_path) or not os.path.exists(decref_path): - build_ref_binaries(decref_path, encref_path, args.rref) + sys.exit("Reference binaries do not exist.") # check for test vectors - if os.path.exists(TEST_VECTOR_DIR): - print(f"Using existing test vector directory {TEST_VECTOR_DIR}") - else: - print(f"Creating test vector directory {TEST_VECTOR_DIR}") - setup_test_vector_directory() + if not os.path.exists(TEST_VECTOR_DIR): + sys.exit(f"Test vector directory {TEST_VECTOR_DIR} does not exist.") # check for references if os.path.exists(REFERENCE_DIR): @@ -328,6 +194,10 @@ def main(argv): encref_path, "--ref_decoder_path", decref_path, + "--dut_encoder_path", + encdut_path, + "--dut_decoder_path", + decdut_path, ] # work-around in unit tests via environment variable # TESTVECTOR_PATH_REL_GROUPB: to specify the test vector directory relative to ivas_pytests folder @@ -336,10 +206,10 @@ def main(argv): my_env["TESTVECTOR_PATH_REL_GROUPB"] = "testv/" my_env["TESTVECTOR_PATH_REL_TRUNK"] = "/scripts/ivas_pytests/testv/" # leading "/" is important my_env["CREND_UNIT_TEST_BIN"] = CREND_UNITTEST_REF - print("pytest command line to be executed from trunk folder:") + print("pytest command line to be executed from project root folder:") print(" ".join(base_cmd + ["-m", "create_ref", "-n", args.numprocesses])) subprocess.run(base_cmd + ["-m", "create_ref", "-n", args.numprocesses], check=False, env=my_env) - print("pytest command line to be executed from trunk folder:") + print("pytest command line to be executed from project root folder:") print(" ".join(base_cmd + ["-m", "create_ref_serial"])) subprocess.run(base_cmd + ["-m", "create_ref_serial"], check=False, env=my_env) @@ -366,9 +236,10 @@ def main(argv): encdut_path, "--dut_decoder_path", decdut_path, + "--junit-xml=report-junit.xml", ] # print pytest commandline - print("pytest command line to be executed from trunk folder:") + print("pytest command line to be executed from project root folder:") print(" ".join(cmd)) result = subprocess.run(cmd, check=False) return result.returncode diff --git a/scripts/ivas_pytests/tests/cut_pcm.py b/scripts/ivas_pytests/tests/cut_pcm.py new file mode 100755 index 0000000000000000000000000000000000000000..62af257a7351424850b5aa3185fe22d2b8ccaccc --- /dev/null +++ b/scripts/ivas_pytests/tests/cut_pcm.py @@ -0,0 +1,113 @@ +#!/usr/bin/env python3 + +__license__ = """ +(C) 2022 IVAS codec Public Collaboration with portions copyright Dolby International AB, Ericsson AB, +Fraunhofer-Gesellschaft zur Foerderung der angewandten Forschung e.V., Huawei Technologies Co. LTD., +Koninklijke Philips N.V., Nippon Telegraph and Telephone Corporation, Nokia Technologies Oy, Orange, +Panasonic Holdings Corporation, Qualcomm Technologies, Inc., VoiceAge Corporation, and other +contributors to this repository. All Rights Reserved. + +This software is protected by copyright law and by international treaties. +The IVAS codec Public Collaboration consisting of Dolby International AB, Ericsson AB, +Fraunhofer-Gesellschaft zur Foerderung der angewandten Forschung e.V., Huawei Technologies Co. LTD., +Koninklijke Philips N.V., Nippon Telegraph and Telephone Corporation, Nokia Technologies Oy, Orange, +Panasonic Holdings Corporation, Qualcomm Technologies, Inc., VoiceAge Corporation, and other +contributors to this repository retain full ownership rights in their respective contributions in +the software. This notice grants no license of any kind, including but not limited to patent +license, nor is any license granted by implication, estoppel or otherwise. + +Contributors are required to enter into the IVAS codec Public Collaboration agreement before making +contributions. + +This software is provided "AS IS", without any express or implied warranties. The software is in the +development stage. It is intended exclusively for experts who have experience with such software and +solely for the purpose of inspection. All implied warranties of non-infringement, merchantability +and fitness for a particular purpose are hereby disclaimed and excluded. + +Any dispute, controversy or claim arising under or in relation to providing this software shall be +submitted to and settled by the final, binding jurisdiction of the courts of Munich, Germany in +accordance with the laws of the Federal Republic of Germany excluding its conflict of law rules and +the United Nations Convention on Contracts on the International Sales of Goods. +""" + +__doc__ = """ +Script to cut samples from a 16-bit PCM file. + +USAGE : cut_pcm.py in_file_pcm out_file_pcm num_channels sample_rate start duration [gain] +in_file_pcm: input PCM file +out_file_pcm: output PCM file +num_channels: number of channels +sample_rate: sample rate in Hz +start: first sample from input file in seconds +duration: duration in seconds +gain: optional gain value to apply to the copied samples +""" + +import sys +import platform + + +def usage(): + print(__doc__) + return 1 + + +def cut_samples(in_file, out_file, num_channels, sample_rate, start, duration, gain="1.0"): + """ + Function to cut samples from a 16-bit PCM file. + """ + + # check for python >= 3.7 + if sys.version_info[0] < 3 or sys.version_info[1] < 7: + sys.exit("This script is written for Python >= 3.7. Found: " + platform.python_version()) + + # all input parameters are strings - convert some + num_ch = int(num_channels) + fs = int(sample_rate) + start_sec = float(start) + dur_sec = float(duration) + gain_f = float(gain) + + with open(in_file, "rb") as fid_in: + fid_in.seek(0, 2) + num_in_samples = fid_in.tell() / 2 + num_in_samples_ch = num_in_samples / num_ch + num_samples_to_skip = int(start_sec * fs) + dur_samples = dur_sec * fs + if num_samples_to_skip + dur_samples > num_in_samples_ch: + sys.exit( + f"requested too many samples ({num_samples_to_skip}+{dur_samples})" + + f" - input is too short ({num_in_samples_ch})" + ) + num_bytes_to_skip = num_samples_to_skip * num_ch * 2 + num_bytes_to_copy = dur_samples * num_ch * 2 + fid_in.seek(num_bytes_to_skip, 0) + num_clip = 0 + with open(out_file, "wb") as fid_out: + bytes_written = 0 + while bytes_written < num_bytes_to_copy: + data = fid_in.read(2) + val = int.from_bytes(data, byteorder="little", signed=True) + val = int(val * gain_f) + if val > 32767: + val = 32767 + num_clip += 1 + if val < -32768: + val = -32768 + num_clip += 1 + data = val.to_bytes(2, byteorder="little", signed=True) + written = fid_out.write(data) + assert written == 2, f"Error writing data: {written} != {2}" + bytes_written += 2 + if num_clip: + print(f"{num_clip} output samples have been clipped.") + + +def main(argv): + if len(argv) < 7: + return usage() + return cut_samples(*argv[1:]) + + +if __name__ == "__main__": + sys.exit(main(sys.argv)) diff --git a/scripts/ivas_pytests/tests/system_tests/test_spar_foa_bs_dec.py b/scripts/ivas_pytests/tests/system_tests/test_spar_foa_bs_dec.py deleted file mode 100644 index 48380b3077d7fa6fc2baf3ece1334533be228bbc..0000000000000000000000000000000000000000 --- a/scripts/ivas_pytests/tests/system_tests/test_spar_foa_bs_dec.py +++ /dev/null @@ -1,186 +0,0 @@ -""" - (C) 2022 IVAS codec Public Collaboration with portions copyright Dolby International AB, Ericsson AB, - Fraunhofer-Gesellschaft zur Foerderung der angewandten Forschung e.V., Huawei Technologies Co. LTD., - Koninklijke Philips N.V., Nippon Telegraph and Telephone Corporation, Nokia Technologies Oy, Orange, - Panasonic Holdings Corporation, Qualcomm Technologies, Inc., VoiceAge Corporation, and other - contributors to this repository. All Rights Reserved. - - This software is protected by copyright law and by international treaties. - The IVAS codec Public Collaboration consisting of Dolby International AB, Ericsson AB, - Fraunhofer-Gesellschaft zur Foerderung der angewandten Forschung e.V., Huawei Technologies Co. LTD., - Koninklijke Philips N.V., Nippon Telegraph and Telephone Corporation, Nokia Technologies Oy, Orange, - Panasonic Holdings Corporation, Qualcomm Technologies, Inc., VoiceAge Corporation, and other - contributors to this repository retain full ownership rights in their respective contributions in - the software. This notice grants no license of any kind, including but not limited to patent - license, nor is any license granted by implication, estoppel or otherwise. - - Contributors are required to enter into the IVAS codec Public Collaboration agreement before making - contributions. - - This software is provided "AS IS", without any express or implied warranties. The software is in the - development stage. It is intended exclusively for experts who have experience with such software and - solely for the purpose of inspection. All implied warranties of non-infringement, merchantability - and fitness for a particular purpose are hereby disclaimed and excluded. - - Any dispute, controversy or claim arising under or in relation to providing this software shall be - submitted to and settled by the final, binding jurisdiction of the courts of Munich, Germany in - accordance with the laws of the Federal Republic of Germany excluding its conflict of law rules and - the United Nations Convention on Contracts on the International Sales of Goods. -""" - -""" -Test file to run C decoder code. -The outputs are compared with C generated references. -""" - -import os -import pytest -import errno -import shutil -import sys - -sys.path.append('scripts/ivas_pytests/') -sys.path.append('scripts/ivas_pytests/tests/') -from il2mm import il2mm -from cmp_custom import cmp_custom -from conftest import DecoderFrontend - -#params -tag_list = ['stv_FOA', 'WYZX_Talker_Moving_15s_22s'] -tag_list_bw_force = ['stv_FOA'] -dtx_set = ['0', '1'] -dict_fsample_bw = {'48':'3', '32':'2', '16':'1'} -dict_bw_idx = {'FB':'3', 'SWB':'2', 'WB':'1'} -dict_bw_tag = {'SWB':'_ForceSWB', 'WB':'_ForceWB'} -ivas_br_list = ['32000', '64000', '96000', '160000', '256000'] -sample_rate_list = ['48', '32', '16'] -sample_rate_bw_idx_list = [('48', 'SWB'), ('48', 'WB'), ('32', 'WB')] -AbsTol = '3' - -ch_count_foa = 4 - - -def check_and_makedir(dir_path): - if not os.path.exists(dir_path): - try: - os.makedirs(dir_path) - except OSError as e: - if e.errno != errno.EEXIST: - raise # raises the error again - - -@pytest.mark.parametrize("ivas_br", ivas_br_list) -@pytest.mark.parametrize("dtx", dtx_set) -@pytest.mark.parametrize("tag", tag_list) -@pytest.mark.parametrize("fs", sample_rate_list) -def test_spar_foa_dec_system( - dut_decoder_frontend: DecoderFrontend, - reference_path, - dut_base_path, - ivas_br, - dtx, - tag, - fs -): - tag = tag + '_' + fs + 'k' - bw = "FB" - - #dec - spar_foa_dec(dut_decoder_frontend, reference_path, dut_base_path, tag, ch_count_foa, fs, ivas_br, dtx, bw) - - -@pytest.mark.parametrize("ivas_br", ivas_br_list) -@pytest.mark.parametrize("dtx", dtx_set) -@pytest.mark.parametrize("tag", tag_list_bw_force) -@pytest.mark.parametrize("sample_rate_bw_idx", sample_rate_bw_idx_list) -def test_spar_foa_dec_BWforce_system( - dut_decoder_frontend: DecoderFrontend, - reference_path, - dut_base_path, - ivas_br, - dtx, - tag, - sample_rate_bw_idx -): - - fs = sample_rate_bw_idx[0] - bw = sample_rate_bw_idx[1] - tag = tag + '_' + fs + 'k' - - #dec - spar_foa_dec(dut_decoder_frontend, reference_path, dut_base_path, tag, ch_count_foa, fs, ivas_br, dtx, bw) - - -######################################################### -############ test function ############################## - -def spar_foa_dec( - decoder_frontend, - reference_path, - dut_base_path, - tag, - ch_count, - sampling_rate, - ivas_br, - dtx, - ivas_max_bw -): - - ######### run cmd ##################################### - #sampling rate to BW mapping - bw_idx = dict_fsample_bw[sampling_rate] - if int(dict_bw_idx[ivas_max_bw]) < int(bw_idx): - bw_tag = dict_bw_tag[ivas_max_bw] - tag = tag + bw_tag - - tag_out = f"{tag}_ivasbr{ivas_br[:-3]}k_DTX{dtx}" - - dut_out_dir = f"{dut_base_path}/spar_foa_bs/raw/{tag_out}" - ref_out_dir = f"{reference_path}/spar_foa_bs/raw/{tag_out}" - check_and_makedir(dut_out_dir) - - ref_in_pkt = f"{reference_path}/spar_foa_bs/pkt/{tag_out}.pkt" - - decoder_frontend.run( - "FOA", - sampling_rate, - ref_in_pkt, - f"{dut_out_dir}/out.raw", - ) - - il2mm(f"{dut_out_dir}/out.raw", ch_count) - - ######### compare cmd ##################################### - - end_skip_samples = '0' - - test_fail = False - for count in range(ch_count): - ch_id = str(count + 1) - - if cmp_custom( - f"{dut_out_dir}/out{ch_id}ch.raw", - f"{ref_out_dir}/out{ch_id}ch.raw", - "2", - AbsTol, - end_skip_samples - ) != 0: - test_fail = True - - ##File removal## - # Unclear whether this clean-up is still needed - # TODO: check temp file generation - for count in range(ch_count, 16): - ch_id = str(count + 1) - temp_raw_file = f"{dut_out_dir}/out{ch_id}ch.raw" - if os.path.isfile(temp_raw_file): - os.remove(temp_raw_file) - - temp_md_file = f"{dut_out_dir}/MD_OUT_DUMMY.BIN" - if os.path.isfile(temp_md_file): - os.remove(temp_md_file) - - shutil.rmtree(dut_out_dir, ignore_errors=True) - - ##report failure - assert not test_fail diff --git a/scripts/ivas_pytests/tests/system_tests/test_spar_foa_bs_dec_plc.py b/scripts/ivas_pytests/tests/system_tests/test_spar_foa_bs_dec_plc.py index 490e94364079c5bc5f00ada63fea450f64459474..d364ca3b01bbb64ed0fc0e1c85aa071cbfc57a2c 100644 --- a/scripts/ivas_pytests/tests/system_tests/test_spar_foa_bs_dec_plc.py +++ b/scripts/ivas_pytests/tests/system_tests/test_spar_foa_bs_dec_plc.py @@ -41,13 +41,16 @@ from cmp_custom import cmp_custom from conftest import EncoderFrontend, DecoderFrontend #params -tag_list = ['stv_FOA', 'WYZX_Talker_Moving_7_12s'] +tag_list = ['stvFOA'] plc_patterns = ['PLperc12mblen5', 'PLperc40mblen50', 'PLperc42mblen2'] dtx_set = ['0', '1'] ivas_br_list = ['32000', '64000', '96000', '256000'] sampling_rate_list = ['48', '32', '16'] -agc_tag_list = ['stv_FOA_24dB'] +# we need signals amplified by 24dB - until those are available, use existing 'stvFOA' +agc_tag_list = ['stvFOA'] +# TODO: create and use new 24dB signals +# agc_tag_list = ['stvFOA_24dB_'] ch_count_foa = 4 AbsTol = '3' @@ -82,7 +85,7 @@ def test_spar_foa_plc_system( plc_pattern, fs ): - tag = tag + '_' + fs + 'k' + tag = tag + fs + 'c' agc = 0 if update_ref == 1 and plc_pattern == plc_patterns[0] and ref_encoder_path: @@ -113,7 +116,7 @@ def test_spar_foa_agc_plc_system( plc_pattern, fs ): - tag = tag + '_' + fs + 'k' + tag = tag + fs + 'c' agc = 1 if update_ref == 1 and plc_pattern == plc_patterns[0] and ref_encoder_path: @@ -128,7 +131,7 @@ def test_spar_foa_agc_plc_system( ############ test function ############################## def spar_foa_ref_enc( ref_encoder_path, - data_system_tests_path, + test_vector_path, reference_path, tag, sampling_rate, @@ -145,7 +148,11 @@ def spar_foa_ref_enc( if agc == 1: tag_out += '_AGC1' - input_path = f"{data_system_tests_path}/spar_foa_bs/enc/in/{tag}/in.raw" + input_path = f"{test_vector_path}/{tag}.pcm" + cut_file = f"{test_vector_path}/{tag}_cut.pcm" + # we expect that the reference generation for create_ref did already run and the cut PCM files are available + assert os.path.exists(cut_file) + input_path = cut_file ref_pkt_file = f"{ref_out_dir}/{tag_out}.pkt" @@ -167,7 +174,7 @@ def spar_foa_ref_enc( def spar_foa_dec_plc( decoder_frontend, - data_system_tests_path, + test_vector_path, reference_path, dut_base_path, ref_decoder_path, @@ -194,7 +201,7 @@ def spar_foa_dec_plc( check_and_makedir(dut_out_dir) check_and_makedir(ref_out_dir) - plc_file = f"{data_system_tests_path}/spar_foa_bs/dec/plc_file/{plc_pattern}.g192" + plc_file = f"{test_vector_path}/{plc_pattern}.g192" ref_in_pkt = f"{reference_path}/spar_foa_bs/pkt/{tag_out}.pkt" if ref_decoder_path: diff --git a/scripts/ivas_pytests/tests/system_tests/test_spar_foa_bs_enc.py b/scripts/ivas_pytests/tests/system_tests/test_spar_foa_bs_enc.py index 796b3eb5d369bd0ab22eb8d5b59cf840882dbd3a..97bd6cdd4f3e0d5d19048da0828c7784c1829d8f 100644 --- a/scripts/ivas_pytests/tests/system_tests/test_spar_foa_bs_enc.py +++ b/scripts/ivas_pytests/tests/system_tests/test_spar_foa_bs_enc.py @@ -42,17 +42,16 @@ import sys sys.path.append('scripts/ivas_pytests/') sys.path.append('scripts/ivas_pytests/tests/') from il2mm import il2mm -from cmp_custom import cmp_custom +from cmp_custom import cmp_custom +from cut_pcm import cut_samples from conftest import EncoderFrontend, DecoderFrontend -# disable SPAR HOA tests until SPAR_HOA macro is enabled -SPAR_HOA_TEST = 1 #params -tag_list = ['stv_FOA', 'WYZX_Talker_Moving_15s_22s'] -tag_list_HOA2 = ['stv_HOA2'] -tag_list_HOA3 = ['stv_HOA3'] +tag_list = ['stvFOA'] +tag_list_HOA2 = ['test_HOA2_v3'] +tag_list_HOA3 = ['test_HOA3_v3'] -tag_list_bw_force = ['stv_FOA'] +tag_list_bw_force = ['stvFOA'] dtx_set = ['0', '1'] dict_fsample_bw = {'48':'3', '32':'2', '16':'1'} dict_bw_idx = {'FB':'3', 'SWB':'2', 'WB':'1'} @@ -64,8 +63,10 @@ ivas_br_HOA3 = ['256000', '384000', '512000'] sample_rate_list = ['48', '32', '16'] bypass_list = [1, 2] -agc_tag_list = ['stv_FOA_24dB'] -agc_list = [-1, 0, 1] +# we need signals amplified by 24dB - until those are available, use existing 'stvFOA' +agc_tag_list = ['stvFOA'] +# TODO: create and use new 24dB signals +# agc_tag_list = ['stvFOA_24dB_'] sample_rate_bw_idx_list = [('48', 'SWB'), ('48', 'WB'), ('32', 'WB')] @@ -102,7 +103,7 @@ def test_bypass_enc( if update_ref == 1 and bypass == 1: pytest.skip() - tag = tag + '_' + fs + 'k' + tag = tag + fs + 'c' ivas_br = '256000' dtx = '0' max_bw = "FB" @@ -155,7 +156,7 @@ def test_bypass_enc( @pytest.mark.parametrize("dtx", dtx_set) @pytest.mark.parametrize("tag", tag_list) @pytest.mark.parametrize("fs", sample_rate_list) -@pytest.mark.parametrize("agc", agc_list) +#@pytest.mark.parametrize("agc", agc_list) def test_spar_foa_enc_system( dut_encoder_frontend: EncoderFrontend, dut_decoder_frontend: DecoderFrontend, @@ -169,14 +170,12 @@ def test_spar_foa_enc_system( dtx, tag, fs, - agc +# agc ): - if update_ref == 1 and agc >= 0: - pytest.skip() - - tag = tag + '_' + fs + 'k' + tag = tag + fs + 'c' max_bw = "FB" bypass = -1 + agc = 0 use_agc_ref = 0 sba_order = "+1" output_config = "FOA" @@ -234,15 +233,11 @@ def test_spar_hoa2_enc_system( ivas_br, tag ): - # until SPAR HOA is enabled - if not SPAR_HOA_TEST: - pytest.skip() - fs = '48' dtx = '0' agc = -1 - tag = tag + '_' + fs + 'k' +# tag = tag + '_' + fs + 'k' max_bw = "FB" bypass = -1 use_agc_ref = 0 @@ -265,7 +260,8 @@ def test_spar_hoa2_enc_system( agc, use_agc_ref, sba_order, - update_ref + update_ref, + '.wav' ) # dec @@ -302,15 +298,11 @@ def test_spar_hoa3_enc_system( ivas_br, tag ): - # until SPAR HOA is enabled - if not SPAR_HOA_TEST: - pytest.skip() - fs = '48' dtx = '0' agc = -1 - tag = tag + '_' + fs + 'k' +# tag = tag + '_' + fs + 'k' max_bw = "FB" bypass = -1 use_agc_ref = 0 @@ -333,7 +325,8 @@ def test_spar_hoa3_enc_system( agc, use_agc_ref, sba_order, - update_ref + update_ref, + '.wav' ) # dec @@ -376,7 +369,7 @@ def test_spar_foa_enc_BWforce_system( ): fs = sample_rate_bw_idx[0] bw = sample_rate_bw_idx[1] - tag = tag + '_' + fs + 'k' + tag = tag + fs + 'c' bypass = -1 agc = 0 use_agc_ref = 0 @@ -441,7 +434,7 @@ def test_spar_foa_enc_agc_system( tag, fs ): - tag = tag + '_' + fs + 'k' + tag = tag + fs + 'c' max_bw = "FB" bypass = -1 agc = 1 @@ -465,7 +458,8 @@ def test_spar_foa_enc_agc_system( agc, use_agc_ref, sba_order, - update_ref + update_ref, + cut_gain="16.0" ) # dec @@ -492,7 +486,7 @@ def test_spar_foa_enc_agc_system( ############ test function ############################## def spar_foa_enc( encoder_frontend, - data_system_tests_path, + test_vector_path, ref_encoder_path, reference_path, dut_base_path, @@ -505,7 +499,9 @@ def spar_foa_enc( agc, use_agc_ref, sba_order, - update_ref + update_ref, + in_extension = '.pcm', + cut_gain = '1.0' ): ######### run cmd ##################################### @@ -534,11 +530,24 @@ def spar_foa_enc( dut_pkt_file = f"{dut_out_dir}/{tag_out}{long_tag_ext}.pkt" ref_pkt_file = f"{ref_out_dir}/{tag_out}{short_tag_ext}.pkt" - input_path = f"{data_system_tests_path}/spar_foa_bs/enc/in/{tag_in}/in.raw" + input_path = f"{test_vector_path}/{tag_in}{in_extension}" agc_op = agc if agc >= 0 else None bypass_mode = bypass if bypass >= 0 else None dtx_mode = dtx == '1' + if in_extension == '.pcm': + # cut input PCM file - currently with fixed (i.e. not test dependant) values + num_channels = "4" # currently only FOA inputs end up, here + cut_from = "0.0" + cut_len = "5.0" + if cut_gain == "1.0": + cut_file = f"{test_vector_path}/{tag_in}_cut{in_extension}" + else: + cut_file = f"{test_vector_path}/{tag_in}_cut_{cut_gain}{in_extension}" + if not os.path.exists(cut_file): + cut_samples(input_path, cut_file, num_channels, sampling_rate + "000", cut_from, cut_len, cut_gain) + input_path = cut_file + if ref_encoder_path: ref_encoder = EncoderFrontend(ref_encoder_path, "REF") # call REF encoder diff --git a/scripts/ivas_pytests/tests/unit_tests/test_crend_unittest.py b/scripts/ivas_pytests/tests/unit_tests/disabled_test_crend_unittest.py similarity index 100% rename from scripts/ivas_pytests/tests/unit_tests/test_crend_unittest.py rename to scripts/ivas_pytests/tests/unit_tests/disabled_test_crend_unittest.py diff --git a/scripts/testv/PLperc12mblen5.g192 b/scripts/testv/PLperc12mblen5.g192 new file mode 100644 index 0000000000000000000000000000000000000000..0ebac7685fdfe422c720ee9943efeceecd9d3413 --- /dev/null +++ b/scripts/testv/PLperc12mblen5.g192 @@ -0,0 +1 @@ +!k!k!k!k!k!k!k!k!k!k!k!k!k!k!k!k k!k!k!k!k!k!k!k!k!k!k!k!k!k k k!k!k!k!k!k!k!k!k!k!k!k!k!k!k k!k!k!k!k!k!k!k!k!k!k k k!k!k!k!k!k!k!k!k!k!k!k!k!k!k!k!k!k!k!k!k!k!k!k!k!k!k!k!k!k!k k k k k k!k!k!k!k!k!k k k!k!k!k!k!k!k!k k!k!k!k!k!k!k!k!k!k!k!k!k!k!k!k!k!k!k k!k k k!k!k!k!k!k!k!k!k k k k!k!k!k!k!k!k!k!k!k!k!k!k k!k!k!k!k!k!k!k!k!k!k!k!k!k k k k!k!k!k!k!k!k!k!k!k!k!k!k k!k!k!k!k!k!k!k!k!k!k!k!k!k!k!k!k!k!k!k!k!k!k!k!k!k!k k k k!k!k!k k k!k!k!k!k!k!k k!k!k!k!k!k!k!k!k!k!k!k!k!k!k!k!k!k!k!k!k!k!k!k!k \ No newline at end of file diff --git a/scripts/testv/PLperc40mblen50.g192 b/scripts/testv/PLperc40mblen50.g192 new file mode 100644 index 0000000000000000000000000000000000000000..28299ab2f6b40d50e9a921b513d5a13b4f1a7226 --- /dev/null +++ b/scripts/testv/PLperc40mblen50.g192 @@ -0,0 +1 @@ +!k!k!k!k!k!k!k!k!k!k!k!k!k!k!k!k!k!k!k!k!k!k!k!k!k k k k k k k k k k k k k k k k k k k k k k k k k k k k k k k k k k k k k k k k k k k k k k k k k k k!k!k!k!k!k!k!k!k!k!k!k!k!k!k!k!k!k!k!k!k!k!k!k!k!k!k!k!k!k!k!k!k!k!k!k!k!k!k!k!k!k!k!k!k!k!k!k!k!k!k k k k k k k k k k k k k k k k k k k k k k k k k k k k k k k k k k k k k k k k k k k k k k k k k k k!k!k!k!k!k!k!k!k!k!k!k!k!k!k!k!k!k!k!k!k!k!k!k!k!k!k!k!k!k!k!k!k!k!k!k!k!k!k!k!k!k!k!k!k!k!k!k!k!k!k!k!k!k!k!k!k!k!k!k!k!k!k!k!k!k!k!k!k!k!k!k!k!k!k!k \ No newline at end of file diff --git a/scripts/testv/PLperc42mblen2.g192 b/scripts/testv/PLperc42mblen2.g192 new file mode 100644 index 0000000000000000000000000000000000000000..2129aa901d298d5cd39eef379ae76c40b162e307 --- /dev/null +++ b/scripts/testv/PLperc42mblen2.g192 @@ -0,0 +1 @@ +!k!k!k k!k k!k!k!k k!k!k!k!k k!k!k k!k!k!k k!k k!k k k!k!k!k!k k!k k!k k!k k!k!k k!k k!k k!k!k!k!k!k!k k!k!k!k!k!k k k!k k k!k!k!k k k!k k k!k!k!k!k k k!k!k!k!k!k!k!k!k k!k!k k!k!k k!k!k!k k k!k k k!k k!k k k!k k k!k k k!k k!k!k k!k k k!k!k k!k!k!k!k k k!k k k!k k k!k k!k k!k!k!k k!k!k k k!k!k!k k!k!k k k!k k!k k k!k k!k k k!k!k!k k k!k k k!k!k k!k!k!k k!k k k!k k k!k k!k!k!k!k k!k k!k k k!k k k!k k k!k k!k!k k!k!k k!k!k k!k k!k!k k k!k k k!k k!k k!k k!k k!k!k!k!k k!k!k k!k k!k k!k!k k k!k k k!k \ No newline at end of file