Skip to content
Commits on Source (34)
Expand all Show whitespace changes
Inline Side-by-side
.gitlab-ci.yml
View file @ 47b3142b
......@@ -398,7 +398,7 @@ build-codec-windows-msbuild:
- *print-common-info-windows
- *activate-WX-windows
- py .\scripts\strip_split_rendering.py
- MSBuild.exe .\Workspace_msvc\Workspace_msvc.sln /property:Configuration=Debug
- MSBuild.exe -maxcpucount .\Workspace_msvc\Workspace_msvc.sln /property:Configuration=Debug
# ---------------------------------------------------------------
# Test jobs for merge requests
......@@ -449,17 +449,21 @@ codec-msan:
- *print-common-info
- make clean
- make -j CLANG=1
- python3 scripts/self_test.py -z console --create | tee test_output.txt
- run_errors=$(cat test_output.txt | grep -ic "run errors") || true
- if [ $run_errors != 0 ] ; then echo "Run errors in self_test.py with Clang memory-sanitizer"; exit 1; fi
- testcase_timeout=180
- exit_code=0
- python3 -m pytest tests/codec_be_on_mr_nonselection/test_param_file.py -v --update_ref 1 -m create_ref --html=report.html --self-contained-html --junit-xml=report-junit.xml --testcase_timeout=$testcase_timeout --ref_encoder_path ./IVAS_cod --ref_decoder_path ./IVAS_dec || exit_code=$?
- if [ $exit_code != 0 ] ; then echo "Run errors found by Clang memory-sanitizer"; exit 1; fi
artifacts:
name: "mr-$CI_MERGE_REQUEST_IID--sha-$CI_COMMIT_SHORT_SHA--stage-$CI_JOB_STAGE--results"
expire_in: 1 week
when: always
paths:
- scripts/ref/logs/
- test_output.txt
- report-junit.xml
- report.html
expose_as: "msan selftest results"
reports:
junit:
- report-junit.xml
# code selftest testvectors with address-sanitizer binaries
codec-asan:
......@@ -472,17 +476,21 @@ codec-asan:
- *print-common-info
- make clean
- make -j CLANG=2
- python3 scripts/self_test.py -z console --create | tee test_output.txt
- run_errors=$(cat test_output.txt | grep -ic "run errors") || true
- if [ $run_errors != 0 ] ; then echo "Run errors in self_test.py with Clang address-sanitizer"; exit 1; fi
- testcase_timeout=180
- exit_code=0
- python3 -m pytest tests/codec_be_on_mr_nonselection/test_param_file.py -v --update_ref 1 -m create_ref --html=report.html --self-contained-html --junit-xml=report-junit.xml --testcase_timeout=$testcase_timeout --ref_encoder_path ./IVAS_cod --ref_decoder_path ./IVAS_dec || exit_code=$?
- if [ $exit_code != 0 ] ; then echo "Run errors found by Clang address-sanitizer"; exit 1; fi
artifacts:
name: "mr-$CI_MERGE_REQUEST_IID--sha-$CI_COMMIT_SHORT_SHA--stage-$CI_JOB_STAGE--results"
expire_in: 1 week
when: always
paths:
- scripts/ref/logs/
- test_output.txt
- report-junit.xml
- report.html
expose_as: "asan selftest results"
reports:
junit:
- report-junit.xml
# code selftest testvectors with address-sanitizer binaries
codec-usan:
......@@ -495,17 +503,21 @@ codec-usan:
- *print-common-info
- make clean
- make -j CLANG=3
- UBSAN_OPTIONS=suppressions=scripts/ubsan.supp,report_error_type=1 python3 scripts/self_test.py -z console --create
- grep_exit_code=0
- grep UndefinedBehaviorSanitizer scripts/ref/logs/* || grep_exit_code=$?
- if [ $grep_exit_code != 1 ] ; then echo "Run errors in self_test.py with Clang undefined-behavior-sanitizer"; exit 1; fi
- testcase_timeout=180
- exit_code=0
- UBSAN_OPTIONS=suppressions=scripts/ubsan.supp,report_error_type=1 python3 -m pytest tests/codec_be_on_mr_nonselection/test_param_file.py -v --update_ref 1 -m create_ref --html=report.html --self-contained-html --junit-xml=report-junit.xml --testcase_timeout=$testcase_timeout --ref_encoder_path ./IVAS_cod --ref_decoder_path ./IVAS_dec || exit_code=$?
- if [ $exit_code != 0 ] ; then echo "Run errors found by Clang undefined-behavior-sanitizer"; exit 1; fi
artifacts:
name: "mr-$CI_MERGE_REQUEST_IID--sha-$CI_COMMIT_SHORT_SHA--stage-$CI_JOB_STAGE--results"
expire_in: 1 week
when: always
paths:
- scripts/ref/logs/
- report-junit.xml
- report.html
expose_as: "usan selftest results"
reports:
junit:
- report-junit.xml
# test renderer executable
renderer-smoke-test:
......@@ -1046,7 +1058,7 @@ be-2-evs-windows:
- *print-common-info-windows
- $winoutdata = $null
- MSBuild.exe .\Workspace_msvc\Workspace_msvc.sln /property:Configuration=Release | tee -variable winoutdata
- MSBuild.exe -maxcpucount .\Workspace_msvc\Workspace_msvc.sln /property:Configuration=Release | tee -variable winoutdata
- $winoutdata | Out-File $BUILD_OUTPUT -Encoding Utf8
# copy over to never change the testvector dir
......@@ -1183,7 +1195,7 @@ test-be-to-release:
script:
- echo "$CI_COMMIT_BRANCH"
- MSBuild.exe .\Workspace_msvc\Workspace_msvc.sln /property:Configuration=Debug /p:Platform=win32 /m | tee -variable winoutdata
- MSBuild.exe -maxcpucount .\Workspace_msvc\Workspace_msvc.sln /property:Configuration=Debug /p:Platform=win32 /m | tee -variable winoutdata
# path to release candidate refs defined in config.toml
......
ci/smoke_test.sh
View file @ 47b3142b
......@@ -56,7 +56,7 @@ dly_profile=./scripts/dly_error_profiles/dly_error_profile_10.dat
ism_md_cmd="--ism_metadata_files /usr/local/ltv/ltvISM1.csv /usr/local/ltv/ltvISM2.csv /usr/local/ltv/ltvISM3.csv /usr/local/ltv/ltvISM4.csv"
duration_arg="-U 1:2"
verbosity_cmd="-z console"
timeout_cmd="--timeout 5"
timeout_cmd="--timeout 20"
ep_file="ci/complexity_measurements/ep_10pct_fer.g192"
if [ $BUILD -eq 1 ];then
......
scripts/parse_mld_xml.py
View file @ 47b3142b
......@@ -25,11 +25,13 @@ if __name__ == '__main__':
with open(csv_file,'w') as outfile:
for testcase in testcases:
if testcase.find('.//skipped') == None:
if testcase.get('file') == None:
fulltestname = testcase.get('classname').replace('.','/') + ".py::" + testcase.get('name')
else:
fulltestname = testcase.get('file') + "::" + testcase.get('name')
system_out = testcase.find(".//system-out")
if testcase.find('.//property') == None:
mld_val = None
if system_out is not None:
for line in system_out.text.split('\n'):
if line.startswith('MLD:'):
mld_val = float(line.split()[1])
else:
mld_val = testcase.find('.//property').get('value') # Currently MLD is the only set property. If more are added updates are needed here.
outfile.write(fulltestname + ';' + str(mld_val)+'\n')
tests/cmp_pcm.py
View file @ 47b3142b
......@@ -12,7 +12,7 @@ import pyivastest
import numpy as np
def cmp_pcm(file1, file2, out_config, fs, get_mld = False) -> (int, str):
def cmp_pcm(file1, file2, out_config, fs, get_mld = False, mld_lim = 0) -> (int, str):
"""
Compare 2 PCM files for bitexactness
"""
......@@ -55,7 +55,11 @@ def cmp_pcm(file1, file2, out_config, fs, get_mld = False) -> (int, str):
if get_mld:
mld_msg = f"MLD: {cmp_result['MLD']}"
print(mld_msg)
return 1, "FAIL: Files have different content"
if cmp_result['MLD'] <= mld_lim:
return 0, f"MLD: {cmp_result['MLD']} <= {mld_lim}"
else:
return 1, f"MLD: {cmp_result['MLD']} > {mld_lim}"
return 1, "Non-BE"
if __name__ == "__main__":
......@@ -71,6 +75,7 @@ if __name__ == "__main__":
)
parser.add_argument("-s", "--sampling_rate", type=int, default=48000, dest="fs")
parser.add_argument("--get_mld", action="store_true")
parser.add_argument("--mld_lim", type=float, default=0, dest="mld_lim")
args = parser.parse_args()
result, msg = cmp_pcm(**vars(args))
......
tests/codec_be_on_mr_nonselection/test_masa_enc_dec.py
View file @ 47b3142b
......@@ -93,6 +93,7 @@ def check_and_makedir(dir_path):
masa_metadata_audio_ndir_ntransportch_dtx_list,
)
def test_masa_enc_dec(
record_property,
dut_encoder_frontend: EncoderFrontend,
dut_decoder_frontend: DecoderFrontend,
ref_encoder_frontend: EncoderFrontend,
......@@ -106,6 +107,7 @@ def test_masa_enc_dec(
test_vector_path,
output_mode,
get_mld,
get_mld_lim,
):
# Input parameters
in_fs = 48
......@@ -201,8 +203,13 @@ def test_masa_enc_dec(
# Compare audio outputs
pcmcmp_res, reason = cmp_pcm(
dec_output_dut, dec_output_ref, output_mode, int(out_fs * 1000), get_mld=get_mld
dec_output_dut, dec_output_ref, output_mode, int(out_fs * 1000), get_mld=get_mld, mld_lim=get_mld_lim
)
if get_mld:
mld = 0
if "MLD" in reason:
mld = float(reason.split(':')[1].split()[0])
record_property("MLD",mld)
# Fail if compare fails compare result
if metacmp_res == False and pcmcmp_res != 0:
......@@ -219,11 +226,19 @@ def test_masa_enc_dec(
filecmp_res = cmp(dec_output_ref, dec_output_dut)
if filecmp_res == False:
cmp_result, reason = cmp_pcm(
dec_output_dut, dec_output_ref, output_mode, int(out_fs * 1000), get_mld=get_mld
dec_output_dut, dec_output_ref, output_mode, int(out_fs * 1000), get_mld=get_mld, mld_lim=get_mld_lim
)
if get_mld:
mld = 0
if "MLD" in reason:
mld = float(reason.split(':')[1].split()[0])
record_property("MLD",mld)
# Report compare result
assert cmp_result == 0, reason
if cmp_result != 0:
pytest.fail(reason)
else:
if get_mld:
record_property("MLD","0")
print("Comparison bit exact")
# remove_output(
......
tests/codec_be_on_mr_nonselection/test_param_file.py
View file @ 47b3142b
......@@ -125,6 +125,7 @@ def convert_test_string_to_tag(test_string):
@pytest.mark.create_ref
@pytest.mark.parametrize("test_tag", list(param_file_test_dict.keys()))
def test_param_file_tests(
record_property,
dut_encoder_frontend: EncoderFrontend,
dut_decoder_frontend: DecoderFrontend,
ref_encoder_frontend: EncoderFrontend,
......@@ -137,6 +138,7 @@ def test_param_file_tests(
keep_files,
test_tag,
get_mld,
get_mld_lim,
):
enc_opts, dec_opts, sim_opts, eid_opts = param_file_test_dict[test_tag]
......@@ -325,10 +327,16 @@ def test_param_file_tests(
ref_output_file = f"{reference_path}/param_file/dec/{output_file}"
fs = int(sampling_rate) * 1000
output_differs, reason = cmp_pcm(
dut_output_file, ref_output_file, output_config, fs, get_mld=get_mld
dut_output_file, ref_output_file, output_config, fs, get_mld=get_mld, mld_lim=get_mld_lim
)
md_out_files = get_expected_md_files(ref_output_file, enc_split, output_config)
if get_mld:
mld = 0
if "MLD" in reason:
mld = float(reason.split(':')[1].split()[0])
record_property("MLD",mld)
metadata_differs = False
for md_file in md_out_files:
dut_metadata_file = Path(f"{dut_base_path}/param_file/dec/{md_file}")
......@@ -347,13 +355,13 @@ def test_param_file_tests(
if output_differs or metadata_differs:
msg = "Difference between ref and dut in "
if output_differs and metadata_differs:
msg += "output and metadata"
msg += f"output ({reason}) and metadata"
elif output_differs:
msg += "output only"
msg += f"output only ({reason})"
elif metadata_differs:
msg += "metadata only"
assert False, msg
pytest.fail(msg)
# remove DUT output files when test result is OK (to save disk space)
if not keep_files:
......
tests/codec_be_on_mr_nonselection/test_sba_bs_dec_plc.py
View file @ 47b3142b
......@@ -75,6 +75,7 @@ def check_and_makedir(dir_path):
@pytest.mark.parametrize("fs", sampling_rate_list)
@pytest.mark.parametrize("gain_flag", gain_list)
def test_sba_plc_system(
record_property,
dut_decoder_frontend: DecoderFrontend,
test_vector_path,
reference_path,
......@@ -89,6 +90,7 @@ def test_sba_plc_system(
fs,
gain_flag,
get_mld,
get_mld_lim,
):
SID = 0
if dtx == '1' and ivas_br not in ['13200','16400','24400','32000', '64000']:
......@@ -105,6 +107,7 @@ def test_sba_plc_system(
# dec
sba_dec_plc(
record_property,
dut_decoder_frontend,
test_vector_path,
reference_path,
......@@ -120,12 +123,14 @@ def test_sba_plc_system(
gain_flag,
keep_files,
get_mld=get_mld,
get_mld_lim=get_mld_lim,
)
#########################################################
# -------------------- test function --------------------
def sba_dec_plc(
record_property,
decoder_frontend,
test_vector_path,
reference_path,
......@@ -141,6 +146,7 @@ def sba_dec_plc(
gain_flag,
keep_files,
get_mld=False,
get_mld_lim=0,
):
# ------------ run cmd ------------
......@@ -191,10 +197,16 @@ def sba_dec_plc(
# -------------- compare cmd --------------
fs = int(sampling_rate) * 1000
cmp_result, reason = cmp_pcm(dut_out_raw, ref_out_raw, output_config, fs, get_mld=get_mld)
cmp_result, reason = cmp_pcm(dut_out_raw, ref_out_raw, output_config, fs, get_mld=get_mld, mld_lim=get_mld_lim)
if get_mld:
mld = 0
if "MLD" in reason:
mld = float(reason.split(':')[1].split()[0])
record_property("MLD",mld)
# report compare result
assert cmp_result == 0, reason
if cmp_result != 0:
pytest.fail(reason)
# remove DUT output files when test result is OK (to save disk space)
if not keep_files:
......
tests/codec_be_on_mr_nonselection/test_sba_bs_enc.py
View file @ 47b3142b
......@@ -89,6 +89,7 @@ def check_and_makedir(dir_path):
@pytest.mark.parametrize("tag", tag_list)
@pytest.mark.parametrize("fs", sample_rate_list)
def test_pca_enc(
record_property,
dut_encoder_frontend: EncoderFrontend,
dut_decoder_frontend: DecoderFrontend,
test_vector_path,
......@@ -101,6 +102,7 @@ def test_pca_enc(
tag,
fs,
get_mld,
get_mld_lim,
):
pca = True
tag = tag + fs + "c"
......@@ -134,6 +136,7 @@ def test_pca_enc(
# dec
sba_dec(
record_property,
dut_decoder_frontend,
ref_decoder_frontend,
reference_path,
......@@ -149,6 +152,7 @@ def test_pca_enc(
gain_flag,
keep_files,
get_mld=get_mld,
get_mld_lim=get_mld_lim,
pca=pca,
)
......@@ -161,6 +165,7 @@ def test_pca_enc(
@pytest.mark.parametrize("gain_flag", gain_list)
@pytest.mark.parametrize("SID", SID_list)
def test_sba_enc_system(
record_property,
dut_encoder_frontend: EncoderFrontend,
dut_decoder_frontend: DecoderFrontend,
test_vector_path,
......@@ -178,6 +183,7 @@ def test_sba_enc_system(
gain_flag,
SID,
get_mld,
get_mld_lim,
):
if dtx == "1" and ivas_br not in ["13200", "16400", "24400", "32000", "64000"]:
......@@ -227,6 +233,7 @@ def test_sba_enc_system(
# dec
sba_dec(
record_property,
dut_decoder_frontend,
ref_decoder_frontend,
reference_path,
......@@ -241,7 +248,8 @@ def test_sba_enc_system(
update_ref,
gain_flag,
keep_files,
get_mld=get_mld
get_mld=get_mld,
get_mld_lim=get_mld_lim
)
......@@ -249,6 +257,7 @@ def test_sba_enc_system(
@pytest.mark.parametrize("ivas_br", ivas_br_HOA2)
@pytest.mark.parametrize("tag", tag_list_HOA2)
def test_spar_hoa2_enc_system(
record_property,
dut_encoder_frontend: EncoderFrontend,
dut_decoder_frontend: DecoderFrontend,
test_vector_path,
......@@ -261,6 +270,7 @@ def test_spar_hoa2_enc_system(
ivas_br,
tag,
get_mld,
get_mld_lim,
):
fs = "48"
dtx = "0"
......@@ -292,6 +302,7 @@ def test_spar_hoa2_enc_system(
# dec
sba_dec(
record_property,
dut_decoder_frontend,
ref_decoder_frontend,
reference_path,
......@@ -307,6 +318,7 @@ def test_spar_hoa2_enc_system(
gain_flag,
keep_files,
get_mld=get_mld,
get_mld_lim=get_mld_lim,
)
......@@ -314,6 +326,7 @@ def test_spar_hoa2_enc_system(
@pytest.mark.parametrize("ivas_br", ivas_br_HOA3)
@pytest.mark.parametrize("tag", tag_list_HOA3)
def test_spar_hoa3_enc_system(
record_property,
dut_encoder_frontend: EncoderFrontend,
dut_decoder_frontend: DecoderFrontend,
test_vector_path,
......@@ -326,6 +339,7 @@ def test_spar_hoa3_enc_system(
ivas_br,
tag,
get_mld,
get_mld_lim,
):
fs = "48"
dtx = "0"
......@@ -357,6 +371,7 @@ def test_spar_hoa3_enc_system(
# dec
sba_dec(
record_property,
dut_decoder_frontend,
ref_decoder_frontend,
reference_path,
......@@ -371,7 +386,8 @@ def test_spar_hoa3_enc_system(
update_ref,
gain_flag,
keep_files,
get_mld=get_mld
get_mld=get_mld,
get_mld_lim=get_mld_lim,
)
......@@ -381,6 +397,7 @@ def test_spar_hoa3_enc_system(
@pytest.mark.parametrize("tag", tag_list_bw_force)
@pytest.mark.parametrize("sample_rate_bw_idx", sample_rate_bw_idx_list)
def test_sba_enc_BWforce_system(
record_property,
dut_encoder_frontend: EncoderFrontend,
dut_decoder_frontend: DecoderFrontend,
test_vector_path,
......@@ -395,6 +412,7 @@ def test_sba_enc_BWforce_system(
tag,
sample_rate_bw_idx,
get_mld,
get_mld_lim,
):
if dtx == "1" and ivas_br not in ["32000", "64000"]:
# skip high bitrates for DTX until DTX issue is resolved
......@@ -432,6 +450,7 @@ def test_sba_enc_BWforce_system(
# dec
sba_dec(
record_property,
dut_decoder_frontend,
ref_decoder_frontend,
reference_path,
......@@ -447,6 +466,7 @@ def test_sba_enc_BWforce_system(
gain_flag,
keep_files,
get_mld=get_mld,
get_mld_lim=get_mld_lim,
)
......@@ -594,6 +614,7 @@ def sba_enc(
def sba_dec(
record_property,
decoder_frontend,
ref_decoder_frontend,
reference_path,
......@@ -609,6 +630,7 @@ def sba_dec(
gain_flag,
keep_files,
get_mld=False,
get_mld_lim=0,
pca=False,
):
# -------- run cmd ------------
......@@ -663,10 +685,16 @@ def sba_dec(
)
fs = int(sampling_rate) * 1000
cmp_result, reason = cmp_pcm(dut_out_raw, ref_out_raw, output_config, fs, get_mld=get_mld)
cmp_result, reason = cmp_pcm(dut_out_raw, ref_out_raw, output_config, fs, get_mld=get_mld, mld_lim=get_mld_lim )
if get_mld:
mld = 0
if "MLD" in reason:
mld = float(reason.split(':')[1].split()[0])
record_property("MLD",mld)
# report compare result
assert cmp_result == 0, reason
if cmp_result != 0:
pytest.fail(reason)
# remove DUT output files when test result is OK (to save disk space)
if not keep_files:
......
tests/conftest.py
View file @ 47b3142b
......@@ -158,6 +158,12 @@ def pytest_addoption(parser):
help="Run the MLD tool instead of just comparing for bitexactness",
)
parser.addoption(
"--mld-lim",
action="store",
help="MLD limit for comparison (default: 0)",
default="0",
)
@pytest.fixture(scope="session", autouse=True)
def update_ref(request):
......@@ -177,6 +183,12 @@ def get_mld(request):
"""
return request.config.option.mld
@pytest.fixture(scope="session", autouse=True)
def get_mld_lim(request):
"""
Return MLD limit for MLD comparison
"""
return float(request.config.getoption("--mld-lim"))
@pytest.fixture(scope="session")
def keep_files(request) -> bool:
......@@ -292,6 +304,8 @@ class EncoderFrontend:
pytest.fail(
f"{self._type} encoder terminated with a non-0 return code: {self.returncode}"
)
if self.stderr and "UndefinedBehaviorSanitizer" in self.stderr:
pytest.fail("Undefined Behaviour runtime error encountered")
def _check_run(self):
if self.returncode is not None:
......@@ -485,6 +499,8 @@ class DecoderFrontend:
pytest.fail(
f"{self._type} decoder terminated with a non-0 return code: {self.returncode}"
)
if self.stderr and "UndefinedBehaviorSanitizer" in self.stderr:
pytest.fail("Undefined Behaviour runtime error encountered")
def _check_run(self):
if self.returncode is not None:
......
tests/renderer/test_renderer.py
View file @ 47b3142b
This diff is collapsed.
tests/renderer/utils.py
View file @ 47b3142b
......@@ -94,6 +94,7 @@ def check_BE(
def run_renderer(
record_property,
test_info,
in_fmt: str,
out_fmt: str,
......@@ -109,6 +110,8 @@ def run_renderer(
binary_suffix: str = "",
frame_size: Optional[str] = "20ms",
hrtf_file: Optional[str] = None,
get_mld = False,
mld_lim = 0,
) -> str:
# prepare arguments and filepaths
if trj_file is not None:
......@@ -226,6 +229,15 @@ def run_renderer(
# CUT creation mode will run a comparison with REF
out_file_ref = str(OUTPUT_PATH_REF.joinpath(out_file_stem))
if get_mld:
output_differs, reason = cmp_pcm(out_file, out_file_ref, out_fmt, ref_fs, get_mld=get_mld, mld_lim=get_mld_lim)
mld = 0
if "MLD" in reason:
mld = float(reason.split(':')[1].split()[0])
record_property("MLD",mld)
if output_differs:
pytest.fail(f"Output differs: ({reason})")
else:
try:
ref, ref_fs = readfile(out_file_ref)
except FileNotFoundError:
......@@ -248,9 +260,10 @@ def run_renderer(
def compare_renderer_args(
test_info, in_fmt, out_fmt, ref_kwargs: Dict, cut_kwargs: Dict
record_property, test_info, in_fmt, out_fmt, ref_kwargs: Dict, cut_kwargs: Dict
):
out_file_ref = run_renderer(
record_property,
test_info,
in_fmt,
out_fmt,
......@@ -258,6 +271,7 @@ def compare_renderer_args(
)
ref, ref_fs = readfile(out_file_ref)
out_file_cut = run_renderer(
record_property,
test_info,
in_fmt,
out_fmt,
......
tests/test_26444.py
View file @ 47b3142b
......@@ -88,7 +88,7 @@ def test_evs_26444(test_tag):
result1 = filecmp.cmp(cmd1[0], cmd1[1])
result2 = True
if not (result1 and result2):
assert False, "Output differs"
pytest.fail("Output differs")
\ No newline at end of file
tests/test_param_file_ltv.py
View file @ 47b3142b
......@@ -126,6 +126,7 @@ def convert_test_string_to_tag(test_string):
@pytest.mark.create_ref
@pytest.mark.parametrize("test_tag", list(param_file_test_dict.keys()))
def test_param_file_tests(
record_property,
dut_encoder_frontend: EncoderFrontend,
dut_decoder_frontend: DecoderFrontend,
ref_encoder_frontend: EncoderFrontend,
......@@ -138,6 +139,7 @@ def test_param_file_tests(
keep_files,
test_tag,
get_mld,
get_mld_lim,
):
enc_opts, dec_opts, sim_opts, eid_opts = param_file_test_dict[test_tag]
......@@ -326,10 +328,16 @@ def test_param_file_tests(
ref_output_file = f"{reference_path}/param_file/dec/{output_file}"
fs = int(sampling_rate) * 1000
output_differs, reason = cmp_pcm(
dut_output_file, ref_output_file, output_config, fs, get_mld=get_mld
dut_output_file, ref_output_file, output_config, fs, get_mld=get_mld, mld_lim=get_mld_lim
)
md_out_files = get_expected_md_files(ref_output_file, enc_split, output_config)
if get_mld:
mld = 0
if "MLD" in reason:
mld = float(reason.split(':')[1].split()[0])
record_property("MLD",mld)
metadata_differs = False
for md_file in md_out_files:
dut_metadata_file = Path(f"{dut_base_path}/param_file/dec/{md_file}")
......@@ -348,13 +356,12 @@ def test_param_file_tests(
if output_differs or metadata_differs:
msg = "Difference between ref and dut in "
if output_differs and metadata_differs:
msg += "output and metadata"
msg += f"output ({reason}) and metadata"
elif output_differs:
msg += "output only"
msg += f"output only ({reason})"
elif metadata_differs:
msg += "metadata only"
assert False, msg
pytest.fail(msg)
# remove DUT output files when test result is OK (to save disk space)
if not keep_files:
......