From 5c4823b9756af6579f67ea19ef79fb16b35f34e1 Mon Sep 17 00:00:00 2001 From: Tapani Pihlajakuja Date: Fri, 9 Jun 2023 14:50:12 +0300 Subject: [PATCH 1/4] Adds pytest framework tests for MASA including EXT output. --- tests/test_masa_enc_dec.py | 276 +++++++++++++++++++++++++++++++++++++ 1 file changed, 276 insertions(+) create mode 100644 tests/test_masa_enc_dec.py diff --git a/tests/test_masa_enc_dec.py b/tests/test_masa_enc_dec.py new file mode 100644 index 0000000000..41b8381378 --- /dev/null +++ b/tests/test_masa_enc_dec.py @@ -0,0 +1,276 @@ +__copyright__ = \ + """ + (C) 2022 IVAS codec Public Collaboration with portions copyright Dolby International AB, Ericsson AB, + Fraunhofer-Gesellschaft zur Foerderung der angewandten Forschung e.V., Huawei Technologies Co. LTD., + Koninklijke Philips N.V., Nippon Telegraph and Telephone Corporation, Nokia Technologies Oy, Orange, + Panasonic Holdings Corporation, Qualcomm Technologies, Inc., VoiceAge Corporation, and other + contributors to this repository. All Rights Reserved. + + This software is protected by copyright law and by international treaties. + The IVAS codec Public Collaboration consisting of Dolby International AB, Ericsson AB, + Fraunhofer-Gesellschaft zur Foerderung der angewandten Forschung e.V., Huawei Technologies Co. LTD., + Koninklijke Philips N.V., Nippon Telegraph and Telephone Corporation, Nokia Technologies Oy, Orange, + Panasonic Holdings Corporation, Qualcomm Technologies, Inc., VoiceAge Corporation, and other + contributors to this repository retain full ownership rights in their respective contributions in + the software. This notice grants no license of any kind, including but not limited to patent + license, nor is any license granted by implication, estoppel or otherwise. + + Contributors are required to enter into the IVAS codec Public Collaboration agreement before making + contributions. + + This software is provided "AS IS", without any express or implied warranties. The software is in the + development stage. It is intended exclusively for experts who have experience with such software and + solely for the purpose of inspection. All implied warranties of non-infringement, merchantability + and fitness for a particular purpose are hereby disclaimed and excluded. + + Any dispute, controversy or claim arising under or in relation to providing this software shall be + submitted to and settled by the final, binding jurisdiction of the courts of Munich, Germany in + accordance with the laws of the Federal Republic of Germany excluding its conflict of law rules and + the United Nations Convention on Contracts on the International Sales of Goods. + """ + +__doc__ = \ + """ + Test file to run C encoder and decoder code. + The outputs are compared with C generated references. + """ + +import os +import errno +import pytest +from typing import Optional +from filecmp import cmp + +from cmp_pcm import cmp_pcm +from conftest import EncoderFrontend, DecoderFrontend + +# params +#output_mode_list = ['MONO', 'STEREO', '5_1', '7_1', '5_1_2', '5_1_4', '7_1_4', 'FOA', 'HOA2', 'HOA3', 'BINAURAL', 'BINAURAL_ROOM', 'EXT'] +output_mode_list = ['BINAURAL', 'EXT'] +ivas_br_masa = [13200, 16400, 24400, 32000, 48000, 64000, 80000, 96000, 128000, 160000, 192000, 256000, 384000, 512000] + +# Write file-based parameter sets here (metafile, pcm/wave, numTC, numDir, DTX_toggle) +masa_metadata_audio_ndir_ntransportch_dtx_list = [#('stv1MASA1TC48c.met', 'stv1MASA1TC48c.wav', 1, 1, False), + #('stv1MASA2TC48c.met', 'stv1MASA2TC48c.wav', 1, 2, False), + ('stv2MASA1TC48c.met', 'stv2MASA1TC48c.wav', 2, 1, False), + #('stv2MASA2TC48c.met', 'stv2MASA2TC48c.wav', 2, 2, False), + #('stv1MASA1TC48n.met', 'stv1MASA1TC48n.wav', 1, 1, True), + ('stv1MASA2TC48n.met', 'stv1MASA2TC48n.wav', 1, 2, True)] + +# Used to not test every combination +test_split_br = [13200, 24400, 48000, 80000, 128000, 256000, 512000]; +AbsTol = '0' + + +def check_and_makedir(dir_path): + if not os.path.exists(dir_path): + try: + os.makedirs(dir_path) + except OSError as e: + if e.errno != errno.EEXIST: + raise # raises the error again + + +@pytest.mark.parametrize("output_mode", output_mode_list) +@pytest.mark.parametrize("ivas_br", ivas_br_masa) +@pytest.mark.parametrize("masa_metadata_audio_ndir_ntransportch_dtx", masa_metadata_audio_ndir_ntransportch_dtx_list) +def test_masa_enc_dec( + dut_encoder_frontend: EncoderFrontend, + dut_decoder_frontend: DecoderFrontend, + ref_encoder_path, + ref_decoder_path, + reference_path, + dut_base_path, + keep_files, + ivas_br, + masa_metadata_audio_ndir_ntransportch_dtx, + test_vector_path, + output_mode, +): + # Input parameters + in_fs = 48 + out_fs = 48 + metadata = masa_metadata_audio_ndir_ntransportch_dtx[0] + input_audio = masa_metadata_audio_ndir_ntransportch_dtx[1] + n_directions = masa_metadata_audio_ndir_ntransportch_dtx[2] + masa_channel_count = masa_metadata_audio_ndir_ntransportch_dtx[3] + dtx = masa_metadata_audio_ndir_ntransportch_dtx[4] + masa_path = f"{test_vector_path}/{metadata}" + input_audio_path = f"{test_vector_path}/{input_audio}" + + # Apply test skipping here + if dtx: + if output_mode == 'EXT' and ivas_br in test_split_br: + pytest.skip("Skipping some DTX bitrates for EXT") + else: + pytest.skip("Skipping some other DTX bitrates for other outputs") + + # Set reference encoder and decoder + ref_encoder_frontend = EncoderFrontend(ref_encoder_path, "REF") + ref_decoder_frontend = DecoderFrontend(ref_decoder_path, "REF") + + # Set output paths + out_dir_bs_ref = f"{reference_path}/masa_test/bitstreams" + out_dir_bs_dut = f"{dut_base_path}/masa_test/bitstreams" + out_dir_dec_output_ref = f"{reference_path}/masa_test/dec_output" + out_dir_dec_output_dut = f"{dut_base_path}/masa_test/dec_output" + check_and_makedir(out_dir_bs_ref) + check_and_makedir(out_dir_bs_dut) + check_and_makedir(out_dir_dec_output_ref) + check_and_makedir(out_dir_dec_output_dut) + + # To avoid conflicting names in case of parallel test execution, differentiate all cases + output_bitstream_ref = f"{out_dir_bs_ref}/masa{masa_channel_count}_ndirs{n_directions}_outputmode{output_mode}_ivasbr{ivas_br}k_DTX{dtx}.bts" + output_bitstream_dut = f"{out_dir_bs_dut}/masa{masa_channel_count}_ndirs{n_directions}_outputmode{output_mode}_ivasbr{ivas_br}k_DTX{dtx}.bts" + dec_output_ref = f"{out_dir_dec_output_ref}/masa{masa_channel_count}_ndirs{n_directions}_outputmode{output_mode}_ivasbr{ivas_br}k_DTX{dtx}.wav" + dec_output_dut = f"{out_dir_dec_output_dut}/masa{masa_channel_count}_ndirs{n_directions}_outputmode{output_mode}_ivasbr{ivas_br}k_DTX{dtx}.wav" + if output_mode == 'EXT': + dec_met_output_ref = f"{dec_output_ref}.met" + dec_met_output_dut = f"{dec_output_dut}.met" + else: + dec_met_output_ref = None + dec_met_output_dut = None + + # Encode REF + ivas_enc( + ref_encoder_frontend, + masa_channel_count, + masa_path, + ivas_br, + in_fs, + input_audio_path, + output_bitstream_ref, + dtx, + ) + + # Encode DUT + ivas_enc( + dut_encoder_frontend, + masa_channel_count, + masa_path, + ivas_br, + in_fs, + input_audio_path, + output_bitstream_dut, + dtx, + ) + + # Decode REF + ivas_dec( + ref_decoder_frontend, + output_mode, + out_fs, + output_bitstream_ref, + dec_output_ref, + ) + + # Decode DUT + ivas_dec( + dut_decoder_frontend, + output_mode, + out_fs, + output_bitstream_dut, + dec_output_dut, + ) + + # Compare outputs. For EXT output, also compare metadata. + if output_mode == 'EXT': + # Compare metadata as binary blob + metacmp_res = cmp(dec_met_output_ref, dec_met_output_dut) + + # Compare audio outputs + pcmcmp_res, reason = cmp_pcm(dec_output_dut, dec_output_ref, output_mode, int(out_fs*1000)) + + # Fail if compare fails compare result + if metacmp_res == False and pcmcmp_res != 0: + pytest.fail("Metadata and transport output difference detected") + elif metacmp_res == False: + pytest.fail("Metadata output difference detected") + elif pcmcmp_res != 0: + pytest.fail("Transport output difference detected") + + else: + # Compare audio outputs + filecmp_res = cmp(dec_output_ref, dec_output_dut) + if filecmp_res == False: + cmp_result, reason = cmp_pcm(dec_output_dut, dec_output_ref, output_mode, int(out_fs*1000)) + # Report compare result + assert cmp_result == 0, reason + + + + remove_output( + keep_files, + output_bitstream_ref, + output_bitstream_dut, + dec_output_ref, + dec_output_dut, + dec_met_output_ref, + dec_met_output_dut, + ) + + +######################################################### +# -------------------- test function -------------------- +def ivas_enc( + encoder_frontend, + masa_channel_count, + masa_path, + ivas_br, + sampling_rate, + input_audio_path, + output_bitstream, + dtx: Optional[bool] = False, +): + + # ------------ run cmd ------------ + options = ['-masa', f"{masa_channel_count}", f"{masa_path}"] + + # call encoder + encoder_frontend.run( + ivas_br, + sampling_rate, + input_audio_path, + output_bitstream, + dtx_mode=dtx, + add_option_list=options, + ) + + +def ivas_dec( + decoder_frontend, + output_mode, + sampling_rate, + input_bitstream, + output_path, +): + + # -------- run cmd ------------ + + # call decoder + decoder_frontend.run( + output_mode, + sampling_rate, + input_bitstream, + output_path, + ) + + +def remove_output( + keep_files, + output_bitstream_ref, + output_bitstream_dut, + dec_output_ref, + dec_output_dut, + dec_met_output_ref: Optional[str] = None, + dec_met_output_dut: Optional[str] = None, +): + if not keep_files: + os.remove(output_bitstream_ref) + os.remove(output_bitstream_dut) + os.remove(dec_output_ref) + os.remove(dec_output_dut) + if dec_met_output_ref is not None: + os.remove(dec_met_output_ref) + if dec_met_output_dut is not None: + os.remove(dec_met_output_dut) -- GitLab From e16f8a88a291455eb9de2f5e1e74d942139e0f70 Mon Sep 17 00:00:00 2001 From: Tapani Pihlajakuja Date: Fri, 9 Jun 2023 16:10:28 +0300 Subject: [PATCH 2/4] Add support for update_ref in MASA tests. --- tests/test_masa_enc_dec.py | 158 +++++++++++++++++++------------------ 1 file changed, 82 insertions(+), 76 deletions(-) diff --git a/tests/test_masa_enc_dec.py b/tests/test_masa_enc_dec.py index 41b8381378..4308947783 100644 --- a/tests/test_masa_enc_dec.py +++ b/tests/test_masa_enc_dec.py @@ -71,6 +71,7 @@ def check_and_makedir(dir_path): raise # raises the error again +@pytest.mark.create_ref @pytest.mark.parametrize("output_mode", output_mode_list) @pytest.mark.parametrize("ivas_br", ivas_br_masa) @pytest.mark.parametrize("masa_metadata_audio_ndir_ntransportch_dtx", masa_metadata_audio_ndir_ntransportch_dtx_list) @@ -81,6 +82,7 @@ def test_masa_enc_dec( ref_decoder_path, reference_path, dut_base_path, + update_ref, keep_files, ivas_br, masa_metadata_audio_ndir_ntransportch_dtx, @@ -131,83 +133,87 @@ def test_masa_enc_dec( dec_met_output_ref = None dec_met_output_dut = None - # Encode REF - ivas_enc( - ref_encoder_frontend, - masa_channel_count, - masa_path, - ivas_br, - in_fs, - input_audio_path, - output_bitstream_ref, - dtx, - ) - - # Encode DUT - ivas_enc( - dut_encoder_frontend, - masa_channel_count, - masa_path, - ivas_br, - in_fs, - input_audio_path, - output_bitstream_dut, - dtx, - ) - - # Decode REF - ivas_dec( - ref_decoder_frontend, - output_mode, - out_fs, - output_bitstream_ref, - dec_output_ref, - ) + if update_ref == 2 or update_ref == 1: + # Encode REF + ivas_enc( + ref_encoder_frontend, + masa_channel_count, + masa_path, + ivas_br, + in_fs, + input_audio_path, + output_bitstream_ref, + dtx, + ) + + # Decode REF + ivas_dec( + ref_decoder_frontend, + output_mode, + out_fs, + output_bitstream_ref, + dec_output_ref, + ) + + if update_ref == 2 or update_ref == 0: + # Encode DUT + ivas_enc( + dut_encoder_frontend, + masa_channel_count, + masa_path, + ivas_br, + in_fs, + input_audio_path, + output_bitstream_dut, + dtx, + ) + + # Decode DUT + ivas_dec( + dut_decoder_frontend, + output_mode, + out_fs, + output_bitstream_dut, + dec_output_dut, + ) + + # Compare outputs. For EXT output, also compare metadata. + if output_mode == 'EXT': + # Compare metadata as binary blob + metacmp_res = cmp(dec_met_output_ref, dec_met_output_dut) + + # Compare audio outputs + pcmcmp_res, reason = cmp_pcm(dec_output_dut, dec_output_ref, output_mode, int(out_fs*1000)) + + # Fail if compare fails compare result + if metacmp_res == False and pcmcmp_res != 0: + pytest.fail("Metadata and transport output difference detected") + elif metacmp_res == False: + pytest.fail("Metadata output difference detected") + elif pcmcmp_res != 0: + pytest.fail("Transport output difference detected") + else: + print("Comparison bit exact") - # Decode DUT - ivas_dec( - dut_decoder_frontend, - output_mode, - out_fs, - output_bitstream_dut, - dec_output_dut, - ) - - # Compare outputs. For EXT output, also compare metadata. - if output_mode == 'EXT': - # Compare metadata as binary blob - metacmp_res = cmp(dec_met_output_ref, dec_met_output_dut) - - # Compare audio outputs - pcmcmp_res, reason = cmp_pcm(dec_output_dut, dec_output_ref, output_mode, int(out_fs*1000)) - - # Fail if compare fails compare result - if metacmp_res == False and pcmcmp_res != 0: - pytest.fail("Metadata and transport output difference detected") - elif metacmp_res == False: - pytest.fail("Metadata output difference detected") - elif pcmcmp_res != 0: - pytest.fail("Transport output difference detected") - - else: - # Compare audio outputs - filecmp_res = cmp(dec_output_ref, dec_output_dut) - if filecmp_res == False: - cmp_result, reason = cmp_pcm(dec_output_dut, dec_output_ref, output_mode, int(out_fs*1000)) - # Report compare result - assert cmp_result == 0, reason - - - - remove_output( - keep_files, - output_bitstream_ref, - output_bitstream_dut, - dec_output_ref, - dec_output_dut, - dec_met_output_ref, - dec_met_output_dut, - ) + else: + # Compare audio outputs + filecmp_res = cmp(dec_output_ref, dec_output_dut) + if filecmp_res == False: + cmp_result, reason = cmp_pcm(dec_output_dut, dec_output_ref, output_mode, int(out_fs*1000)) + # Report compare result + assert cmp_result == 0, reason + else: + print("Comparison bit exact") + + # remove_output( + # keep_files, + # output_bitstream_ref, + # output_bitstream_dut, + # dec_output_ref, + # dec_output_dut, + # dec_met_output_ref, + # dec_met_output_dut, + # ) ######################################################### -- GitLab From 111332a15531f01f74324e28ef1926d16d59da9b Mon Sep 17 00:00:00 2001 From: Tapani Pihlajakuja Date: Fri, 9 Jun 2023 16:50:49 +0300 Subject: [PATCH 3/4] Fix order in comment --- tests/test_masa_enc_dec.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_masa_enc_dec.py b/tests/test_masa_enc_dec.py index 4308947783..9950889598 100644 --- a/tests/test_masa_enc_dec.py +++ b/tests/test_masa_enc_dec.py @@ -49,7 +49,7 @@ from conftest import EncoderFrontend, DecoderFrontend output_mode_list = ['BINAURAL', 'EXT'] ivas_br_masa = [13200, 16400, 24400, 32000, 48000, 64000, 80000, 96000, 128000, 160000, 192000, 256000, 384000, 512000] -# Write file-based parameter sets here (metafile, pcm/wave, numTC, numDir, DTX_toggle) +# Write file-based parameter sets here (metafile, pcm/wave, numDir, numTC, DTX_toggle) masa_metadata_audio_ndir_ntransportch_dtx_list = [#('stv1MASA1TC48c.met', 'stv1MASA1TC48c.wav', 1, 1, False), #('stv1MASA2TC48c.met', 'stv1MASA2TC48c.wav', 1, 2, False), ('stv2MASA1TC48c.met', 'stv2MASA1TC48c.wav', 2, 1, False), -- GitLab From 910654a49fe11dc935e0bf0939ed9ea6c77d2ece Mon Sep 17 00:00:00 2001 From: Tapani Pihlajakuja Date: Mon, 12 Jun 2023 09:00:05 +0300 Subject: [PATCH 4/4] Update copyright year. --- tests/test_masa_enc_dec.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_masa_enc_dec.py b/tests/test_masa_enc_dec.py index 9950889598..c09da3c0d2 100644 --- a/tests/test_masa_enc_dec.py +++ b/tests/test_masa_enc_dec.py @@ -1,6 +1,6 @@ __copyright__ = \ """ - (C) 2022 IVAS codec Public Collaboration with portions copyright Dolby International AB, Ericsson AB, + (C) 2022-2023 IVAS codec Public Collaboration with portions copyright Dolby International AB, Ericsson AB, Fraunhofer-Gesellschaft zur Foerderung der angewandten Forschung e.V., Huawei Technologies Co. LTD., Koninklijke Philips N.V., Nippon Telegraph and Telephone Corporation, Nokia Technologies Oy, Orange, Panasonic Holdings Corporation, Qualcomm Technologies, Inc., VoiceAge Corporation, and other -- GitLab