Skip to content
Commits on Source (34)
Expand all Show whitespace changes
Inline Side-by-side
.gitlab-ci.yml
View file @ 47b3142b
...@@ -398,7 +398,7 @@ build-codec-windows-msbuild: ...@@ -398,7 +398,7 @@ build-codec-windows-msbuild:
- *print-common-info-windows - *print-common-info-windows
- *activate-WX-windows - *activate-WX-windows
- py .\scripts\strip_split_rendering.py - py .\scripts\strip_split_rendering.py
- MSBuild.exe .\Workspace_msvc\Workspace_msvc.sln /property:Configuration=Debug - MSBuild.exe -maxcpucount .\Workspace_msvc\Workspace_msvc.sln /property:Configuration=Debug
# --------------------------------------------------------------- # ---------------------------------------------------------------
# Test jobs for merge requests # Test jobs for merge requests
...@@ -449,17 +449,21 @@ codec-msan: ...@@ -449,17 +449,21 @@ codec-msan:
- *print-common-info - *print-common-info
- make clean - make clean
- make -j CLANG=1 - make -j CLANG=1
- python3 scripts/self_test.py -z console --create | tee test_output.txt - testcase_timeout=180
- run_errors=$(cat test_output.txt | grep -ic "run errors") || true - exit_code=0
- if [ $run_errors != 0 ] ; then echo "Run errors in self_test.py with Clang memory-sanitizer"; exit 1; fi - python3 -m pytest tests/codec_be_on_mr_nonselection/test_param_file.py -v --update_ref 1 -m create_ref --html=report.html --self-contained-html --junit-xml=report-junit.xml --testcase_timeout=$testcase_timeout --ref_encoder_path ./IVAS_cod --ref_decoder_path ./IVAS_dec || exit_code=$?
- if [ $exit_code != 0 ] ; then echo "Run errors found by Clang memory-sanitizer"; exit 1; fi
artifacts: artifacts:
name: "mr-$CI_MERGE_REQUEST_IID--sha-$CI_COMMIT_SHORT_SHA--stage-$CI_JOB_STAGE--results" name: "mr-$CI_MERGE_REQUEST_IID--sha-$CI_COMMIT_SHORT_SHA--stage-$CI_JOB_STAGE--results"
expire_in: 1 week expire_in: 1 week
when: always when: always
paths: paths:
- scripts/ref/logs/ - report-junit.xml
- test_output.txt - report.html
expose_as: "msan selftest results" expose_as: "msan selftest results"
reports:
junit:
- report-junit.xml
# code selftest testvectors with address-sanitizer binaries # code selftest testvectors with address-sanitizer binaries
codec-asan: codec-asan:
...@@ -472,17 +476,21 @@ codec-asan: ...@@ -472,17 +476,21 @@ codec-asan:
- *print-common-info - *print-common-info
- make clean - make clean
- make -j CLANG=2 - make -j CLANG=2
- python3 scripts/self_test.py -z console --create | tee test_output.txt - testcase_timeout=180
- run_errors=$(cat test_output.txt | grep -ic "run errors") || true - exit_code=0
- if [ $run_errors != 0 ] ; then echo "Run errors in self_test.py with Clang address-sanitizer"; exit 1; fi - python3 -m pytest tests/codec_be_on_mr_nonselection/test_param_file.py -v --update_ref 1 -m create_ref --html=report.html --self-contained-html --junit-xml=report-junit.xml --testcase_timeout=$testcase_timeout --ref_encoder_path ./IVAS_cod --ref_decoder_path ./IVAS_dec || exit_code=$?
- if [ $exit_code != 0 ] ; then echo "Run errors found by Clang address-sanitizer"; exit 1; fi
artifacts: artifacts:
name: "mr-$CI_MERGE_REQUEST_IID--sha-$CI_COMMIT_SHORT_SHA--stage-$CI_JOB_STAGE--results" name: "mr-$CI_MERGE_REQUEST_IID--sha-$CI_COMMIT_SHORT_SHA--stage-$CI_JOB_STAGE--results"
expire_in: 1 week expire_in: 1 week
when: always when: always
paths: paths:
- scripts/ref/logs/ - report-junit.xml
- test_output.txt - report.html
expose_as: "asan selftest results" expose_as: "asan selftest results"
reports:
junit:
- report-junit.xml
# code selftest testvectors with address-sanitizer binaries # code selftest testvectors with address-sanitizer binaries
codec-usan: codec-usan:
...@@ -495,17 +503,21 @@ codec-usan: ...@@ -495,17 +503,21 @@ codec-usan:
- *print-common-info - *print-common-info
- make clean - make clean
- make -j CLANG=3 - make -j CLANG=3
- UBSAN_OPTIONS=suppressions=scripts/ubsan.supp,report_error_type=1 python3 scripts/self_test.py -z console --create - testcase_timeout=180
- grep_exit_code=0 - exit_code=0
- grep UndefinedBehaviorSanitizer scripts/ref/logs/* || grep_exit_code=$? - UBSAN_OPTIONS=suppressions=scripts/ubsan.supp,report_error_type=1 python3 -m pytest tests/codec_be_on_mr_nonselection/test_param_file.py -v --update_ref 1 -m create_ref --html=report.html --self-contained-html --junit-xml=report-junit.xml --testcase_timeout=$testcase_timeout --ref_encoder_path ./IVAS_cod --ref_decoder_path ./IVAS_dec || exit_code=$?
- if [ $grep_exit_code != 1 ] ; then echo "Run errors in self_test.py with Clang undefined-behavior-sanitizer"; exit 1; fi - if [ $exit_code != 0 ] ; then echo "Run errors found by Clang undefined-behavior-sanitizer"; exit 1; fi
artifacts: artifacts:
name: "mr-$CI_MERGE_REQUEST_IID--sha-$CI_COMMIT_SHORT_SHA--stage-$CI_JOB_STAGE--results" name: "mr-$CI_MERGE_REQUEST_IID--sha-$CI_COMMIT_SHORT_SHA--stage-$CI_JOB_STAGE--results"
expire_in: 1 week expire_in: 1 week
when: always when: always
paths: paths:
- scripts/ref/logs/ - report-junit.xml
- report.html
expose_as: "usan selftest results" expose_as: "usan selftest results"
reports:
junit:
- report-junit.xml
# test renderer executable # test renderer executable
renderer-smoke-test: renderer-smoke-test:
...@@ -1046,7 +1058,7 @@ be-2-evs-windows: ...@@ -1046,7 +1058,7 @@ be-2-evs-windows:
- *print-common-info-windows - *print-common-info-windows
- $winoutdata = $null - $winoutdata = $null
- MSBuild.exe .\Workspace_msvc\Workspace_msvc.sln /property:Configuration=Release | tee -variable winoutdata - MSBuild.exe -maxcpucount .\Workspace_msvc\Workspace_msvc.sln /property:Configuration=Release | tee -variable winoutdata
- $winoutdata | Out-File $BUILD_OUTPUT -Encoding Utf8 - $winoutdata | Out-File $BUILD_OUTPUT -Encoding Utf8
# copy over to never change the testvector dir # copy over to never change the testvector dir
...@@ -1183,7 +1195,7 @@ test-be-to-release: ...@@ -1183,7 +1195,7 @@ test-be-to-release:
script: script:
- echo "$CI_COMMIT_BRANCH" - echo "$CI_COMMIT_BRANCH"
- MSBuild.exe .\Workspace_msvc\Workspace_msvc.sln /property:Configuration=Debug /p:Platform=win32 /m | tee -variable winoutdata - MSBuild.exe -maxcpucount .\Workspace_msvc\Workspace_msvc.sln /property:Configuration=Debug /p:Platform=win32 /m | tee -variable winoutdata
# path to release candidate refs defined in config.toml # path to release candidate refs defined in config.toml
......
ci/smoke_test.sh
View file @ 47b3142b
...@@ -56,7 +56,7 @@ dly_profile=./scripts/dly_error_profiles/dly_error_profile_10.dat ...@@ -56,7 +56,7 @@ dly_profile=./scripts/dly_error_profiles/dly_error_profile_10.dat
ism_md_cmd="--ism_metadata_files /usr/local/ltv/ltvISM1.csv /usr/local/ltv/ltvISM2.csv /usr/local/ltv/ltvISM3.csv /usr/local/ltv/ltvISM4.csv" ism_md_cmd="--ism_metadata_files /usr/local/ltv/ltvISM1.csv /usr/local/ltv/ltvISM2.csv /usr/local/ltv/ltvISM3.csv /usr/local/ltv/ltvISM4.csv"
duration_arg="-U 1:2" duration_arg="-U 1:2"
verbosity_cmd="-z console" verbosity_cmd="-z console"
timeout_cmd="--timeout 5" timeout_cmd="--timeout 20"
ep_file="ci/complexity_measurements/ep_10pct_fer.g192" ep_file="ci/complexity_measurements/ep_10pct_fer.g192"
if [ $BUILD -eq 1 ];then if [ $BUILD -eq 1 ];then
......
scripts/parse_mld_xml.py
View file @ 47b3142b
...@@ -25,11 +25,13 @@ if __name__ == '__main__': ...@@ -25,11 +25,13 @@ if __name__ == '__main__':
with open(csv_file,'w') as outfile: with open(csv_file,'w') as outfile:
for testcase in testcases: for testcase in testcases:
if testcase.find('.//skipped') == None:
if testcase.get('file') == None:
fulltestname = testcase.get('classname').replace('.','/') + ".py::" + testcase.get('name')
else:
fulltestname = testcase.get('file') + "::" + testcase.get('name') fulltestname = testcase.get('file') + "::" + testcase.get('name')
system_out = testcase.find(".//system-out") if testcase.find('.//property') == None:
mld_val = None mld_val = None
if system_out is not None: else:
for line in system_out.text.split('\n'): mld_val = testcase.find('.//property').get('value') # Currently MLD is the only set property. If more are added updates are needed here.
if line.startswith('MLD:'):
mld_val = float(line.split()[1])
outfile.write(fulltestname + ';' + str(mld_val)+'\n') outfile.write(fulltestname + ';' + str(mld_val)+'\n')
tests/cmp_pcm.py
View file @ 47b3142b
...@@ -12,7 +12,7 @@ import pyivastest ...@@ -12,7 +12,7 @@ import pyivastest
import numpy as np import numpy as np
def cmp_pcm(file1, file2, out_config, fs, get_mld = False) -> (int, str): def cmp_pcm(file1, file2, out_config, fs, get_mld = False, mld_lim = 0) -> (int, str):
""" """
Compare 2 PCM files for bitexactness Compare 2 PCM files for bitexactness
""" """
...@@ -55,7 +55,11 @@ def cmp_pcm(file1, file2, out_config, fs, get_mld = False) -> (int, str): ...@@ -55,7 +55,11 @@ def cmp_pcm(file1, file2, out_config, fs, get_mld = False) -> (int, str):
if get_mld: if get_mld:
mld_msg = f"MLD: {cmp_result['MLD']}" mld_msg = f"MLD: {cmp_result['MLD']}"
print(mld_msg) print(mld_msg)
return 1, "FAIL: Files have different content" if cmp_result['MLD'] <= mld_lim:
return 0, f"MLD: {cmp_result['MLD']} <= {mld_lim}"
else:
return 1, f"MLD: {cmp_result['MLD']} > {mld_lim}"
return 1, "Non-BE"
if __name__ == "__main__": if __name__ == "__main__":
...@@ -71,6 +75,7 @@ if __name__ == "__main__": ...@@ -71,6 +75,7 @@ if __name__ == "__main__":
) )
parser.add_argument("-s", "--sampling_rate", type=int, default=48000, dest="fs") parser.add_argument("-s", "--sampling_rate", type=int, default=48000, dest="fs")
parser.add_argument("--get_mld", action="store_true") parser.add_argument("--get_mld", action="store_true")
parser.add_argument("--mld_lim", type=float, default=0, dest="mld_lim")
args = parser.parse_args() args = parser.parse_args()
result, msg = cmp_pcm(**vars(args)) result, msg = cmp_pcm(**vars(args))
......
tests/codec_be_on_mr_nonselection/test_masa_enc_dec.py
View file @ 47b3142b
...@@ -93,6 +93,7 @@ def check_and_makedir(dir_path): ...@@ -93,6 +93,7 @@ def check_and_makedir(dir_path):
masa_metadata_audio_ndir_ntransportch_dtx_list, masa_metadata_audio_ndir_ntransportch_dtx_list,
) )
def test_masa_enc_dec( def test_masa_enc_dec(
record_property,
dut_encoder_frontend: EncoderFrontend, dut_encoder_frontend: EncoderFrontend,
dut_decoder_frontend: DecoderFrontend, dut_decoder_frontend: DecoderFrontend,
ref_encoder_frontend: EncoderFrontend, ref_encoder_frontend: EncoderFrontend,
...@@ -106,6 +107,7 @@ def test_masa_enc_dec( ...@@ -106,6 +107,7 @@ def test_masa_enc_dec(
test_vector_path, test_vector_path,
output_mode, output_mode,
get_mld, get_mld,
get_mld_lim,
): ):
# Input parameters # Input parameters
in_fs = 48 in_fs = 48
...@@ -201,8 +203,13 @@ def test_masa_enc_dec( ...@@ -201,8 +203,13 @@ def test_masa_enc_dec(
# Compare audio outputs # Compare audio outputs
pcmcmp_res, reason = cmp_pcm( pcmcmp_res, reason = cmp_pcm(
dec_output_dut, dec_output_ref, output_mode, int(out_fs * 1000), get_mld=get_mld dec_output_dut, dec_output_ref, output_mode, int(out_fs * 1000), get_mld=get_mld, mld_lim=get_mld_lim
) )
if get_mld:
mld = 0
if "MLD" in reason:
mld = float(reason.split(':')[1].split()[0])
record_property("MLD",mld)
# Fail if compare fails compare result # Fail if compare fails compare result
if metacmp_res == False and pcmcmp_res != 0: if metacmp_res == False and pcmcmp_res != 0:
...@@ -219,11 +226,19 @@ def test_masa_enc_dec( ...@@ -219,11 +226,19 @@ def test_masa_enc_dec(
filecmp_res = cmp(dec_output_ref, dec_output_dut) filecmp_res = cmp(dec_output_ref, dec_output_dut)
if filecmp_res == False: if filecmp_res == False:
cmp_result, reason = cmp_pcm( cmp_result, reason = cmp_pcm(
dec_output_dut, dec_output_ref, output_mode, int(out_fs * 1000), get_mld=get_mld dec_output_dut, dec_output_ref, output_mode, int(out_fs * 1000), get_mld=get_mld, mld_lim=get_mld_lim
) )
if get_mld:
mld = 0
if "MLD" in reason:
mld = float(reason.split(':')[1].split()[0])
record_property("MLD",mld)
# Report compare result # Report compare result
assert cmp_result == 0, reason if cmp_result != 0:
pytest.fail(reason)
else: else:
if get_mld:
record_property("MLD","0")
print("Comparison bit exact") print("Comparison bit exact")
# remove_output( # remove_output(
......
tests/codec_be_on_mr_nonselection/test_param_file.py
View file @ 47b3142b
...@@ -125,6 +125,7 @@ def convert_test_string_to_tag(test_string): ...@@ -125,6 +125,7 @@ def convert_test_string_to_tag(test_string):
@pytest.mark.create_ref @pytest.mark.create_ref
@pytest.mark.parametrize("test_tag", list(param_file_test_dict.keys())) @pytest.mark.parametrize("test_tag", list(param_file_test_dict.keys()))
def test_param_file_tests( def test_param_file_tests(
record_property,
dut_encoder_frontend: EncoderFrontend, dut_encoder_frontend: EncoderFrontend,
dut_decoder_frontend: DecoderFrontend, dut_decoder_frontend: DecoderFrontend,
ref_encoder_frontend: EncoderFrontend, ref_encoder_frontend: EncoderFrontend,
...@@ -137,6 +138,7 @@ def test_param_file_tests( ...@@ -137,6 +138,7 @@ def test_param_file_tests(
keep_files, keep_files,
test_tag, test_tag,
get_mld, get_mld,
get_mld_lim,
): ):
enc_opts, dec_opts, sim_opts, eid_opts = param_file_test_dict[test_tag] enc_opts, dec_opts, sim_opts, eid_opts = param_file_test_dict[test_tag]
...@@ -325,10 +327,16 @@ def test_param_file_tests( ...@@ -325,10 +327,16 @@ def test_param_file_tests(
ref_output_file = f"{reference_path}/param_file/dec/{output_file}" ref_output_file = f"{reference_path}/param_file/dec/{output_file}"
fs = int(sampling_rate) * 1000 fs = int(sampling_rate) * 1000
output_differs, reason = cmp_pcm( output_differs, reason = cmp_pcm(
dut_output_file, ref_output_file, output_config, fs, get_mld=get_mld dut_output_file, ref_output_file, output_config, fs, get_mld=get_mld, mld_lim=get_mld_lim
) )
md_out_files = get_expected_md_files(ref_output_file, enc_split, output_config) md_out_files = get_expected_md_files(ref_output_file, enc_split, output_config)
if get_mld:
mld = 0
if "MLD" in reason:
mld = float(reason.split(':')[1].split()[0])
record_property("MLD",mld)
metadata_differs = False metadata_differs = False
for md_file in md_out_files: for md_file in md_out_files:
dut_metadata_file = Path(f"{dut_base_path}/param_file/dec/{md_file}") dut_metadata_file = Path(f"{dut_base_path}/param_file/dec/{md_file}")
...@@ -347,13 +355,13 @@ def test_param_file_tests( ...@@ -347,13 +355,13 @@ def test_param_file_tests(
if output_differs or metadata_differs: if output_differs or metadata_differs:
msg = "Difference between ref and dut in " msg = "Difference between ref and dut in "
if output_differs and metadata_differs: if output_differs and metadata_differs:
msg += "output and metadata" msg += f"output ({reason}) and metadata"
elif output_differs: elif output_differs:
msg += "output only" msg += f"output only ({reason})"
elif metadata_differs: elif metadata_differs:
msg += "metadata only" msg += "metadata only"
assert False, msg pytest.fail(msg)
# remove DUT output files when test result is OK (to save disk space) # remove DUT output files when test result is OK (to save disk space)
if not keep_files: if not keep_files:
......
tests/codec_be_on_mr_nonselection/test_sba_bs_dec_plc.py
View file @ 47b3142b
...@@ -75,6 +75,7 @@ def check_and_makedir(dir_path): ...@@ -75,6 +75,7 @@ def check_and_makedir(dir_path):
@pytest.mark.parametrize("fs", sampling_rate_list) @pytest.mark.parametrize("fs", sampling_rate_list)
@pytest.mark.parametrize("gain_flag", gain_list) @pytest.mark.parametrize("gain_flag", gain_list)
def test_sba_plc_system( def test_sba_plc_system(
record_property,
dut_decoder_frontend: DecoderFrontend, dut_decoder_frontend: DecoderFrontend,
test_vector_path, test_vector_path,
reference_path, reference_path,
...@@ -89,6 +90,7 @@ def test_sba_plc_system( ...@@ -89,6 +90,7 @@ def test_sba_plc_system(
fs, fs,
gain_flag, gain_flag,
get_mld, get_mld,
get_mld_lim,
): ):
SID = 0 SID = 0
if dtx == '1' and ivas_br not in ['13200','16400','24400','32000', '64000']: if dtx == '1' and ivas_br not in ['13200','16400','24400','32000', '64000']:
...@@ -105,6 +107,7 @@ def test_sba_plc_system( ...@@ -105,6 +107,7 @@ def test_sba_plc_system(
# dec # dec
sba_dec_plc( sba_dec_plc(
record_property,
dut_decoder_frontend, dut_decoder_frontend,
test_vector_path, test_vector_path,
reference_path, reference_path,
...@@ -120,12 +123,14 @@ def test_sba_plc_system( ...@@ -120,12 +123,14 @@ def test_sba_plc_system(
gain_flag, gain_flag,
keep_files, keep_files,
get_mld=get_mld, get_mld=get_mld,
get_mld_lim=get_mld_lim,
) )
######################################################### #########################################################
# -------------------- test function -------------------- # -------------------- test function --------------------
def sba_dec_plc( def sba_dec_plc(
record_property,
decoder_frontend, decoder_frontend,
test_vector_path, test_vector_path,
reference_path, reference_path,
...@@ -141,6 +146,7 @@ def sba_dec_plc( ...@@ -141,6 +146,7 @@ def sba_dec_plc(
gain_flag, gain_flag,
keep_files, keep_files,
get_mld=False, get_mld=False,
get_mld_lim=0,
): ):
# ------------ run cmd ------------ # ------------ run cmd ------------
...@@ -191,10 +197,16 @@ def sba_dec_plc( ...@@ -191,10 +197,16 @@ def sba_dec_plc(
# -------------- compare cmd -------------- # -------------- compare cmd --------------
fs = int(sampling_rate) * 1000 fs = int(sampling_rate) * 1000
cmp_result, reason = cmp_pcm(dut_out_raw, ref_out_raw, output_config, fs, get_mld=get_mld) cmp_result, reason = cmp_pcm(dut_out_raw, ref_out_raw, output_config, fs, get_mld=get_mld, mld_lim=get_mld_lim)
if get_mld:
mld = 0
if "MLD" in reason:
mld = float(reason.split(':')[1].split()[0])
record_property("MLD",mld)
# report compare result # report compare result
assert cmp_result == 0, reason if cmp_result != 0:
pytest.fail(reason)
# remove DUT output files when test result is OK (to save disk space) # remove DUT output files when test result is OK (to save disk space)
if not keep_files: if not keep_files:
......
tests/codec_be_on_mr_nonselection/test_sba_bs_enc.py
View file @ 47b3142b
...@@ -89,6 +89,7 @@ def check_and_makedir(dir_path): ...@@ -89,6 +89,7 @@ def check_and_makedir(dir_path):
@pytest.mark.parametrize("tag", tag_list) @pytest.mark.parametrize("tag", tag_list)
@pytest.mark.parametrize("fs", sample_rate_list) @pytest.mark.parametrize("fs", sample_rate_list)
def test_pca_enc( def test_pca_enc(
record_property,
dut_encoder_frontend: EncoderFrontend, dut_encoder_frontend: EncoderFrontend,
dut_decoder_frontend: DecoderFrontend, dut_decoder_frontend: DecoderFrontend,
test_vector_path, test_vector_path,
...@@ -101,6 +102,7 @@ def test_pca_enc( ...@@ -101,6 +102,7 @@ def test_pca_enc(
tag, tag,
fs, fs,
get_mld, get_mld,
get_mld_lim,
): ):
pca = True pca = True
tag = tag + fs + "c" tag = tag + fs + "c"
...@@ -134,6 +136,7 @@ def test_pca_enc( ...@@ -134,6 +136,7 @@ def test_pca_enc(
# dec # dec
sba_dec( sba_dec(
record_property,
dut_decoder_frontend, dut_decoder_frontend,
ref_decoder_frontend, ref_decoder_frontend,
reference_path, reference_path,
...@@ -149,6 +152,7 @@ def test_pca_enc( ...@@ -149,6 +152,7 @@ def test_pca_enc(
gain_flag, gain_flag,
keep_files, keep_files,
get_mld=get_mld, get_mld=get_mld,
get_mld_lim=get_mld_lim,
pca=pca, pca=pca,
) )
...@@ -161,6 +165,7 @@ def test_pca_enc( ...@@ -161,6 +165,7 @@ def test_pca_enc(
@pytest.mark.parametrize("gain_flag", gain_list) @pytest.mark.parametrize("gain_flag", gain_list)
@pytest.mark.parametrize("SID", SID_list) @pytest.mark.parametrize("SID", SID_list)
def test_sba_enc_system( def test_sba_enc_system(
record_property,
dut_encoder_frontend: EncoderFrontend, dut_encoder_frontend: EncoderFrontend,
dut_decoder_frontend: DecoderFrontend, dut_decoder_frontend: DecoderFrontend,
test_vector_path, test_vector_path,
...@@ -178,6 +183,7 @@ def test_sba_enc_system( ...@@ -178,6 +183,7 @@ def test_sba_enc_system(
gain_flag, gain_flag,
SID, SID,
get_mld, get_mld,
get_mld_lim,
): ):
if dtx == "1" and ivas_br not in ["13200", "16400", "24400", "32000", "64000"]: if dtx == "1" and ivas_br not in ["13200", "16400", "24400", "32000", "64000"]:
...@@ -227,6 +233,7 @@ def test_sba_enc_system( ...@@ -227,6 +233,7 @@ def test_sba_enc_system(
# dec # dec
sba_dec( sba_dec(
record_property,
dut_decoder_frontend, dut_decoder_frontend,
ref_decoder_frontend, ref_decoder_frontend,
reference_path, reference_path,
...@@ -241,7 +248,8 @@ def test_sba_enc_system( ...@@ -241,7 +248,8 @@ def test_sba_enc_system(
update_ref, update_ref,
gain_flag, gain_flag,
keep_files, keep_files,
get_mld=get_mld get_mld=get_mld,
get_mld_lim=get_mld_lim
) )
...@@ -249,6 +257,7 @@ def test_sba_enc_system( ...@@ -249,6 +257,7 @@ def test_sba_enc_system(
@pytest.mark.parametrize("ivas_br", ivas_br_HOA2) @pytest.mark.parametrize("ivas_br", ivas_br_HOA2)
@pytest.mark.parametrize("tag", tag_list_HOA2) @pytest.mark.parametrize("tag", tag_list_HOA2)
def test_spar_hoa2_enc_system( def test_spar_hoa2_enc_system(
record_property,
dut_encoder_frontend: EncoderFrontend, dut_encoder_frontend: EncoderFrontend,
dut_decoder_frontend: DecoderFrontend, dut_decoder_frontend: DecoderFrontend,
test_vector_path, test_vector_path,
...@@ -261,6 +270,7 @@ def test_spar_hoa2_enc_system( ...@@ -261,6 +270,7 @@ def test_spar_hoa2_enc_system(
ivas_br, ivas_br,
tag, tag,
get_mld, get_mld,
get_mld_lim,
): ):
fs = "48" fs = "48"
dtx = "0" dtx = "0"
...@@ -292,6 +302,7 @@ def test_spar_hoa2_enc_system( ...@@ -292,6 +302,7 @@ def test_spar_hoa2_enc_system(
# dec # dec
sba_dec( sba_dec(
record_property,
dut_decoder_frontend, dut_decoder_frontend,
ref_decoder_frontend, ref_decoder_frontend,
reference_path, reference_path,
...@@ -307,6 +318,7 @@ def test_spar_hoa2_enc_system( ...@@ -307,6 +318,7 @@ def test_spar_hoa2_enc_system(
gain_flag, gain_flag,
keep_files, keep_files,
get_mld=get_mld, get_mld=get_mld,
get_mld_lim=get_mld_lim,
) )
...@@ -314,6 +326,7 @@ def test_spar_hoa2_enc_system( ...@@ -314,6 +326,7 @@ def test_spar_hoa2_enc_system(
@pytest.mark.parametrize("ivas_br", ivas_br_HOA3) @pytest.mark.parametrize("ivas_br", ivas_br_HOA3)
@pytest.mark.parametrize("tag", tag_list_HOA3) @pytest.mark.parametrize("tag", tag_list_HOA3)
def test_spar_hoa3_enc_system( def test_spar_hoa3_enc_system(
record_property,
dut_encoder_frontend: EncoderFrontend, dut_encoder_frontend: EncoderFrontend,
dut_decoder_frontend: DecoderFrontend, dut_decoder_frontend: DecoderFrontend,
test_vector_path, test_vector_path,
...@@ -326,6 +339,7 @@ def test_spar_hoa3_enc_system( ...@@ -326,6 +339,7 @@ def test_spar_hoa3_enc_system(
ivas_br, ivas_br,
tag, tag,
get_mld, get_mld,
get_mld_lim,
): ):
fs = "48" fs = "48"
dtx = "0" dtx = "0"
...@@ -357,6 +371,7 @@ def test_spar_hoa3_enc_system( ...@@ -357,6 +371,7 @@ def test_spar_hoa3_enc_system(
# dec # dec
sba_dec( sba_dec(
record_property,
dut_decoder_frontend, dut_decoder_frontend,
ref_decoder_frontend, ref_decoder_frontend,
reference_path, reference_path,
...@@ -371,7 +386,8 @@ def test_spar_hoa3_enc_system( ...@@ -371,7 +386,8 @@ def test_spar_hoa3_enc_system(
update_ref, update_ref,
gain_flag, gain_flag,
keep_files, keep_files,
get_mld=get_mld get_mld=get_mld,
get_mld_lim=get_mld_lim,
) )
...@@ -381,6 +397,7 @@ def test_spar_hoa3_enc_system( ...@@ -381,6 +397,7 @@ def test_spar_hoa3_enc_system(
@pytest.mark.parametrize("tag", tag_list_bw_force) @pytest.mark.parametrize("tag", tag_list_bw_force)
@pytest.mark.parametrize("sample_rate_bw_idx", sample_rate_bw_idx_list) @pytest.mark.parametrize("sample_rate_bw_idx", sample_rate_bw_idx_list)
def test_sba_enc_BWforce_system( def test_sba_enc_BWforce_system(
record_property,
dut_encoder_frontend: EncoderFrontend, dut_encoder_frontend: EncoderFrontend,
dut_decoder_frontend: DecoderFrontend, dut_decoder_frontend: DecoderFrontend,
test_vector_path, test_vector_path,
...@@ -395,6 +412,7 @@ def test_sba_enc_BWforce_system( ...@@ -395,6 +412,7 @@ def test_sba_enc_BWforce_system(
tag, tag,
sample_rate_bw_idx, sample_rate_bw_idx,
get_mld, get_mld,
get_mld_lim,
): ):
if dtx == "1" and ivas_br not in ["32000", "64000"]: if dtx == "1" and ivas_br not in ["32000", "64000"]:
# skip high bitrates for DTX until DTX issue is resolved # skip high bitrates for DTX until DTX issue is resolved
...@@ -432,6 +450,7 @@ def test_sba_enc_BWforce_system( ...@@ -432,6 +450,7 @@ def test_sba_enc_BWforce_system(
# dec # dec
sba_dec( sba_dec(
record_property,
dut_decoder_frontend, dut_decoder_frontend,
ref_decoder_frontend, ref_decoder_frontend,
reference_path, reference_path,
...@@ -447,6 +466,7 @@ def test_sba_enc_BWforce_system( ...@@ -447,6 +466,7 @@ def test_sba_enc_BWforce_system(
gain_flag, gain_flag,
keep_files, keep_files,
get_mld=get_mld, get_mld=get_mld,
get_mld_lim=get_mld_lim,
) )
...@@ -594,6 +614,7 @@ def sba_enc( ...@@ -594,6 +614,7 @@ def sba_enc(
def sba_dec( def sba_dec(
record_property,
decoder_frontend, decoder_frontend,
ref_decoder_frontend, ref_decoder_frontend,
reference_path, reference_path,
...@@ -609,6 +630,7 @@ def sba_dec( ...@@ -609,6 +630,7 @@ def sba_dec(
gain_flag, gain_flag,
keep_files, keep_files,
get_mld=False, get_mld=False,
get_mld_lim=0,
pca=False, pca=False,
): ):
# -------- run cmd ------------ # -------- run cmd ------------
...@@ -663,10 +685,16 @@ def sba_dec( ...@@ -663,10 +685,16 @@ def sba_dec(
) )
fs = int(sampling_rate) * 1000 fs = int(sampling_rate) * 1000
cmp_result, reason = cmp_pcm(dut_out_raw, ref_out_raw, output_config, fs, get_mld=get_mld) cmp_result, reason = cmp_pcm(dut_out_raw, ref_out_raw, output_config, fs, get_mld=get_mld, mld_lim=get_mld_lim )
if get_mld:
mld = 0
if "MLD" in reason:
mld = float(reason.split(':')[1].split()[0])
record_property("MLD",mld)
# report compare result # report compare result
assert cmp_result == 0, reason if cmp_result != 0:
pytest.fail(reason)
# remove DUT output files when test result is OK (to save disk space) # remove DUT output files when test result is OK (to save disk space)
if not keep_files: if not keep_files:
......
tests/conftest.py
View file @ 47b3142b
...@@ -158,6 +158,12 @@ def pytest_addoption(parser): ...@@ -158,6 +158,12 @@ def pytest_addoption(parser):
help="Run the MLD tool instead of just comparing for bitexactness", help="Run the MLD tool instead of just comparing for bitexactness",
) )
parser.addoption(
"--mld-lim",
action="store",
help="MLD limit for comparison (default: 0)",
default="0",
)
@pytest.fixture(scope="session", autouse=True) @pytest.fixture(scope="session", autouse=True)
def update_ref(request): def update_ref(request):
...@@ -177,6 +183,12 @@ def get_mld(request): ...@@ -177,6 +183,12 @@ def get_mld(request):
""" """
return request.config.option.mld return request.config.option.mld
@pytest.fixture(scope="session", autouse=True)
def get_mld_lim(request):
"""
Return MLD limit for MLD comparison
"""
return float(request.config.getoption("--mld-lim"))
@pytest.fixture(scope="session") @pytest.fixture(scope="session")
def keep_files(request) -> bool: def keep_files(request) -> bool:
...@@ -292,6 +304,8 @@ class EncoderFrontend: ...@@ -292,6 +304,8 @@ class EncoderFrontend:
pytest.fail( pytest.fail(
f"{self._type} encoder terminated with a non-0 return code: {self.returncode}" f"{self._type} encoder terminated with a non-0 return code: {self.returncode}"
) )
if self.stderr and "UndefinedBehaviorSanitizer" in self.stderr:
pytest.fail("Undefined Behaviour runtime error encountered")
def _check_run(self): def _check_run(self):
if self.returncode is not None: if self.returncode is not None:
...@@ -485,6 +499,8 @@ class DecoderFrontend: ...@@ -485,6 +499,8 @@ class DecoderFrontend:
pytest.fail( pytest.fail(
f"{self._type} decoder terminated with a non-0 return code: {self.returncode}" f"{self._type} decoder terminated with a non-0 return code: {self.returncode}"
) )
if self.stderr and "UndefinedBehaviorSanitizer" in self.stderr:
pytest.fail("Undefined Behaviour runtime error encountered")
def _check_run(self): def _check_run(self):
if self.returncode is not None: if self.returncode is not None:
......
tests/renderer/test_renderer.py
View file @ 47b3142b
This diff is collapsed.
tests/renderer/utils.py
View file @ 47b3142b
...@@ -94,6 +94,7 @@ def check_BE( ...@@ -94,6 +94,7 @@ def check_BE(
def run_renderer( def run_renderer(
record_property,
test_info, test_info,
in_fmt: str, in_fmt: str,
out_fmt: str, out_fmt: str,
...@@ -109,6 +110,8 @@ def run_renderer( ...@@ -109,6 +110,8 @@ def run_renderer(
binary_suffix: str = "", binary_suffix: str = "",
frame_size: Optional[str] = "20ms", frame_size: Optional[str] = "20ms",
hrtf_file: Optional[str] = None, hrtf_file: Optional[str] = None,
get_mld = False,
mld_lim = 0,
) -> str: ) -> str:
# prepare arguments and filepaths # prepare arguments and filepaths
if trj_file is not None: if trj_file is not None:
...@@ -226,6 +229,15 @@ def run_renderer( ...@@ -226,6 +229,15 @@ def run_renderer(
# CUT creation mode will run a comparison with REF # CUT creation mode will run a comparison with REF
out_file_ref = str(OUTPUT_PATH_REF.joinpath(out_file_stem)) out_file_ref = str(OUTPUT_PATH_REF.joinpath(out_file_stem))
if get_mld:
output_differs, reason = cmp_pcm(out_file, out_file_ref, out_fmt, ref_fs, get_mld=get_mld, mld_lim=get_mld_lim)
mld = 0
if "MLD" in reason:
mld = float(reason.split(':')[1].split()[0])
record_property("MLD",mld)
if output_differs:
pytest.fail(f"Output differs: ({reason})")
else:
try: try:
ref, ref_fs = readfile(out_file_ref) ref, ref_fs = readfile(out_file_ref)
except FileNotFoundError: except FileNotFoundError:
...@@ -248,9 +260,10 @@ def run_renderer( ...@@ -248,9 +260,10 @@ def run_renderer(
def compare_renderer_args( def compare_renderer_args(
test_info, in_fmt, out_fmt, ref_kwargs: Dict, cut_kwargs: Dict record_property, test_info, in_fmt, out_fmt, ref_kwargs: Dict, cut_kwargs: Dict
): ):
out_file_ref = run_renderer( out_file_ref = run_renderer(
record_property,
test_info, test_info,
in_fmt, in_fmt,
out_fmt, out_fmt,
...@@ -258,6 +271,7 @@ def compare_renderer_args( ...@@ -258,6 +271,7 @@ def compare_renderer_args(
) )
ref, ref_fs = readfile(out_file_ref) ref, ref_fs = readfile(out_file_ref)
out_file_cut = run_renderer( out_file_cut = run_renderer(
record_property,
test_info, test_info,
in_fmt, in_fmt,
out_fmt, out_fmt,
......
tests/test_26444.py
View file @ 47b3142b
...@@ -88,7 +88,7 @@ def test_evs_26444(test_tag): ...@@ -88,7 +88,7 @@ def test_evs_26444(test_tag):
result1 = filecmp.cmp(cmd1[0], cmd1[1]) result1 = filecmp.cmp(cmd1[0], cmd1[1])
result2 = True result2 = True
if not (result1 and result2): if not (result1 and result2):
assert False, "Output differs" pytest.fail("Output differs")
\ No newline at end of file
tests/test_param_file_ltv.py
View file @ 47b3142b
...@@ -126,6 +126,7 @@ def convert_test_string_to_tag(test_string): ...@@ -126,6 +126,7 @@ def convert_test_string_to_tag(test_string):
@pytest.mark.create_ref @pytest.mark.create_ref
@pytest.mark.parametrize("test_tag", list(param_file_test_dict.keys())) @pytest.mark.parametrize("test_tag", list(param_file_test_dict.keys()))
def test_param_file_tests( def test_param_file_tests(
record_property,
dut_encoder_frontend: EncoderFrontend, dut_encoder_frontend: EncoderFrontend,
dut_decoder_frontend: DecoderFrontend, dut_decoder_frontend: DecoderFrontend,
ref_encoder_frontend: EncoderFrontend, ref_encoder_frontend: EncoderFrontend,
...@@ -138,6 +139,7 @@ def test_param_file_tests( ...@@ -138,6 +139,7 @@ def test_param_file_tests(
keep_files, keep_files,
test_tag, test_tag,
get_mld, get_mld,
get_mld_lim,
): ):
enc_opts, dec_opts, sim_opts, eid_opts = param_file_test_dict[test_tag] enc_opts, dec_opts, sim_opts, eid_opts = param_file_test_dict[test_tag]
...@@ -326,10 +328,16 @@ def test_param_file_tests( ...@@ -326,10 +328,16 @@ def test_param_file_tests(
ref_output_file = f"{reference_path}/param_file/dec/{output_file}" ref_output_file = f"{reference_path}/param_file/dec/{output_file}"
fs = int(sampling_rate) * 1000 fs = int(sampling_rate) * 1000
output_differs, reason = cmp_pcm( output_differs, reason = cmp_pcm(
dut_output_file, ref_output_file, output_config, fs, get_mld=get_mld dut_output_file, ref_output_file, output_config, fs, get_mld=get_mld, mld_lim=get_mld_lim
) )
md_out_files = get_expected_md_files(ref_output_file, enc_split, output_config) md_out_files = get_expected_md_files(ref_output_file, enc_split, output_config)
if get_mld:
mld = 0
if "MLD" in reason:
mld = float(reason.split(':')[1].split()[0])
record_property("MLD",mld)
metadata_differs = False metadata_differs = False
for md_file in md_out_files: for md_file in md_out_files:
dut_metadata_file = Path(f"{dut_base_path}/param_file/dec/{md_file}") dut_metadata_file = Path(f"{dut_base_path}/param_file/dec/{md_file}")
...@@ -348,13 +356,12 @@ def test_param_file_tests( ...@@ -348,13 +356,12 @@ def test_param_file_tests(
if output_differs or metadata_differs: if output_differs or metadata_differs:
msg = "Difference between ref and dut in " msg = "Difference between ref and dut in "
if output_differs and metadata_differs: if output_differs and metadata_differs:
msg += "output and metadata" msg += f"output ({reason}) and metadata"
elif output_differs: elif output_differs:
msg += "output only" msg += f"output only ({reason})"
elif metadata_differs: elif metadata_differs:
msg += "metadata only" msg += "metadata only"
pytest.fail(msg)
assert False, msg
# remove DUT output files when test result is OK (to save disk space) # remove DUT output files when test result is OK (to save disk space)
if not keep_files: if not keep_files:
......