Commit 5e97e17a authored by Tapani Pihlajakuja's avatar Tapani Pihlajakuja
Browse files

Merge branch 'ci/remove_separate_masa_test' into 'main' 

[CI] Remove separate MASA test system and add the coverage to param file tests.

See merge request !1651
parents 9ad6ef61 2cb801e0

scripts/config/self_test.prm

+56 −0
Original line number Diff line number Diff line
@@ -1079,6 +1079,62 @@ eid-xor -fer -vbr -bs g192 -ep g192 bit ../scripts/dly_error_profiles/ep_5pct.g1 
../IVAS_cod -masa 2 testv/stv2MASA2TC48c.met 512000 48 testv/stv2MASA2TC48c.wav bit

../IVAS_dec 5_1 48 bit testv/stv2MASA2TC48c.wav_512000_48-48_5_1.tst



// MASA 1dir 1TC at 13.2 kbps, 48kHz in, 48kHz out, EXT out

../IVAS_cod -masa 1 testv/stv1MASA1TC48c.met 13200 48 testv/stv1MASA1TC48c.wav bit

../IVAS_dec EXT 48 bit testv/stv1MASA1TC48c.wav_13200_48-48_EXT.tst



// MASA 1dir 2TC at 16.4 kbps, 48kHz in, 48kHz out, EXT out

../IVAS_cod -masa 2 testv/stv1MASA2TC48c.met 16400 48 testv/stv1MASA2TC48c.wav bit

../IVAS_dec EXT 48 bit testv/stv1MASA2TC48c.wav_16400_48-48_EXT.tst



// MASA 2dir 1TC at 24.4 kbps, 48kHz in, 48kHz out, EXT out

../IVAS_cod -masa 1 testv/stv2MASA1TC48c.met 24400 48 testv/stv2MASA1TC48c.wav bit

../IVAS_dec EXT 48 bit testv/stv2MASA1TC48c.wav_24400_48-48_EXT.tst



// MASA 2dir 2TC at 32 kbps, 48kHz in, 48kHz out, EXT out

../IVAS_cod -masa 2 testv/stv2MASA2TC48c.met 32000 48 testv/stv2MASA2TC48c.wav bit

../IVAS_dec EXT 48 bit testv/stv2MASA2TC48c.wav_32000_48-48_EXT.tst



// MASA 1dir 1TC at 48 kbps, 48kHz in, 48kHz out, EXT out

../IVAS_cod -masa 1 testv/stv1MASA1TC48c.met 48000 48 testv/stv1MASA1TC48c.wav bit

../IVAS_dec EXT 48 bit testv/stv1MASA1TC48c.wav_48000_48-48_EXT.tst



// MASA 1dir 2TC at 64 kbps, 48kHz in, 48kHz out, EXT out

../IVAS_cod -masa 2 testv/stv1MASA2TC48c.met 64000 48 testv/stv1MASA2TC48c.wav bit

../IVAS_dec EXT 48 bit testv/stv1MASA2TC48c.wav_64000_48-48_EXT.tst



// MASA 2dir 1TC at 80 kbps, 48kHz in, 48kHz out, EXT out

../IVAS_cod -masa 1 testv/stv2MASA1TC48c.met 80000 48 testv/stv2MASA1TC48c.wav bit

../IVAS_dec EXT 48 bit testv/stv2MASA1TC48c.wav_80000_48-48_EXT.tst



// MASA 2dir 2TC at 96 kbps, 48kHz in, 48kHz out, EXT out

../IVAS_cod -masa 2 testv/stv2MASA2TC48c.met 96000 48 testv/stv2MASA2TC48c.wav bit

../IVAS_dec EXT 48 bit testv/stv2MASA2TC48c.wav_96000_48-48_EXT.tst



// MASA 2dir 2TC at 128 kbps, 48kHz in, 48kHz out, EXT out

../IVAS_cod -masa 2 testv/stv2MASA2TC48c.met 128000 48 testv/stv2MASA2TC48c.wav bit

../IVAS_dec EXT 48 bit testv/stv2MASA2TC48c.wav_128000_48-48_EXT.tst



// MASA 2dir 2TC at 160 kbps, 48kHz in, 48kHz out, EXT out

../IVAS_cod -masa 2 testv/stv2MASA2TC48c.met 160000 48 testv/stv2MASA2TC48c.wav bit

../IVAS_dec EXT 48 bit testv/stv2MASA2TC48c.wav_160000_48-48_EXT.tst



// MASA 2dir 2TC at 192 kbps, 48kHz in, 48kHz out, EXT out

../IVAS_cod -masa 2 testv/stv2MASA2TC48c.met 192000 48 testv/stv2MASA2TC48c.wav bit

../IVAS_dec EXT 48 bit testv/stv2MASA2TC48c.wav_192000_48-48_EXT.tst



// MASA 2dir 2TC at 256 kbps, 48kHz in, 48kHz out, EXT out

../IVAS_cod -masa 2 testv/stv2MASA2TC48c.met 256000 48 testv/stv2MASA2TC48c.wav bit

../IVAS_dec EXT 48 bit testv/stv2MASA2TC48c.wav_256000_48-48_EXT.tst



// MASA 2dir 2TC at 384 kbps, 48kHz in, 48kHz out, EXT out

../IVAS_cod -masa 2 testv/stv2MASA2TC48c.met 384000 48 testv/stv2MASA2TC48c.wav bit

../IVAS_dec EXT 48 bit testv/stv2MASA2TC48c.wav_384000_48-48_EXT.tst



// MASA 2dir 2TC at 512 kbps, 48kHz in, 48kHz out, EXT out

../IVAS_cod -masa 2 testv/stv2MASA2TC48c.met 512000 48 testv/stv2MASA2TC48c.wav bit

../IVAS_dec EXT 48 bit testv/stv2MASA2TC48c.wav_512000_48-48_EXT.tst



// MASA 1dir 1TC at 13.2 kbps, 48kHz in, 48kHz out, DTX on, 7_1_4 out

../IVAS_cod -dtx -masa 1 testv/stv1MASA1TC48n.met 13200 48 testv/stv1MASA1TC48n.wav bit

../IVAS_dec 7_1_4 48 bit testv/stv1MASA1TC48n.wav_13200_48-48_DTX_7_1_4.tst

scripts/config/self_test_ltv.prm

+56 −0
Original line number Diff line number Diff line
@@ -1078,6 +1078,62 @@ eid-xor -fer -vbr -bs g192 -ep g192 bit ../scripts/dly_error_profiles/ep_5pct.g1 
../IVAS_cod -masa 2 testv/ltv48_MASA2TC.met 512000 48 testv/ltv48_MASA2TC.wav bit

../IVAS_dec 5_1 48 bit testv/ltv48_MASA2TC.wav_512000_48-48_5_1.tst



// MASA 1TC at 13.2 kbps, 48kHz in, 48kHz out, EXT out

../IVAS_cod -masa 1 testv/ltv48_MASA1TC.met 13200 48 testv/ltv48_MASA1TC.wav bit

../IVAS_dec EXT 48 bit testv/ltv48_MASA1TC.wav_13200_48-48_EXT.tst



// MASA 2TC at 16.4 kbps, 48kHz in, 48kHz out, EXT out

../IVAS_cod -masa 2 testv/ltv48_MASA2TC.met 16400 48 testv/ltv48_MASA2TC.wav bit

../IVAS_dec EXT 48 bit testv/ltv48_MASA2TC.wav_16400_48-48_EXT.tst



// MASA 1TC at 24.4 kbps, 48kHz in, 48kHz out, EXT out

../IVAS_cod -masa 1 testv/ltv48_MASA1TC.met 24400 48 testv/ltv48_MASA1TC.wav bit

../IVAS_dec EXT 48 bit testv/ltv48_MASA1TC.wav_24400_48-48_EXT.tst



// MASA 2TC at 32 kbps, 48kHz in, 48kHz out, EXT out

../IVAS_cod -masa 2 testv/ltv48_MASA2TC.met 32000 48 testv/ltv48_MASA2TC.wav bit

../IVAS_dec EXT 48 bit testv/ltv48_MASA2TC.wav_32000_48-48_EXT.tst



// MASA 1TC at 48 kbps, 48kHz in, 48kHz out, EXT out

../IVAS_cod -masa 1 testv/ltv48_MASA1TC.met 48000 48 testv/ltv48_MASA1TC.wav bit

../IVAS_dec EXT 48 bit testv/ltv48_MASA1TC.wav_48000_48-48_EXT.tst



// MASA 2TC at 64 kbps, 48kHz in, 48kHz out, EXT out

../IVAS_cod -masa 2 testv/ltv48_MASA2TC.met 64000 48 testv/ltv48_MASA2TC.wav bit

../IVAS_dec EXT 48 bit testv/ltv48_MASA2TC.wav_64000_48-48_EXT.tst



// MASA 1TC at 80 kbps, 48kHz in, 48kHz out, EXT out

../IVAS_cod -masa 1 testv/ltv48_MASA1TC.met 80000 48 testv/ltv48_MASA1TC.wav bit

../IVAS_dec EXT 48 bit testv/ltv48_MASA1TC.wav_80000_48-48_EXT.tst



// MASA 2TC at 96 kbps, 48kHz in, 48kHz out, EXT out

../IVAS_cod -masa 2 testv/ltv48_MASA2TC.met 96000 48 testv/ltv48_MASA2TC.wav bit

../IVAS_dec EXT 48 bit testv/ltv48_MASA2TC.wav_96000_48-48_EXT.tst



// MASA 2TC at 128 kbps, 48kHz in, 48kHz out, EXT out

../IVAS_cod -masa 2 testv/ltv48_MASA2TC.met 128000 48 testv/ltv48_MASA2TC.wav bit

../IVAS_dec EXT 48 bit testv/ltv48_MASA2TC.wav_128000_48-48_EXT.tst



// MASA 2TC at 160 kbps, 48kHz in, 48kHz out, EXT out

../IVAS_cod -masa 2 testv/ltv48_MASA2TC.met 160000 48 testv/ltv48_MASA2TC.wav bit

../IVAS_dec EXT 48 bit testv/ltv48_MASA2TC.wav_160000_48-48_EXT.tst



// MASA 2TC at 192 kbps, 48kHz in, 48kHz out, EXT out

../IVAS_cod -masa 2 testv/ltv48_MASA2TC.met 192000 48 testv/ltv48_MASA2TC.wav bit

../IVAS_dec EXT 48 bit testv/ltv48_MASA2TC.wav_192000_48-48_EXT.tst



// MASA 2TC at 256 kbps, 48kHz in, 48kHz out, EXT out

../IVAS_cod -masa 2 testv/ltv48_MASA2TC.met 256000 48 testv/ltv48_MASA2TC.wav bit

../IVAS_dec EXT 48 bit testv/ltv48_MASA2TC.wav_256000_48-48_EXT.tst



// MASA 2TC at 384 kbps, 48kHz in, 48kHz out, EXT out

../IVAS_cod -masa 2 testv/ltv48_MASA2TC.met 384000 48 testv/ltv48_MASA2TC.wav bit

../IVAS_dec EXT 48 bit testv/ltv48_MASA2TC.wav_384000_48-48_EXT.tst



// MASA 2TC at 512 kbps, 48kHz in, 48kHz out, EXT out

../IVAS_cod -masa 2 testv/ltv48_MASA2TC.met 512000 48 testv/ltv48_MASA2TC.wav bit

../IVAS_dec EXT 48 bit testv/ltv48_MASA2TC.wav_512000_48-48_EXT.tst



// MASA 1dir 1TC at 13.2 kbps, 48kHz in, 48kHz out, DTX on, 7_1_4 out

../IVAS_cod -dtx -masa 1 testv/ltv48_MASA1TC.met 13200 48 testv/ltv48_MASA1TC.wav bit

../IVAS_dec 7_1_4 48 bit testv/ltv48_MASA1TC.wav_13200_48-48_DTX_7_1_4.tst

tests/codec_be_on_mr_nonselection/test_masa_enc_dec.py

deleted100644 → 0
+0 −305
Original line number Diff line number Diff line 
__copyright__ = """

    (C) 2022-2024 IVAS codec Public Collaboration with portions copyright Dolby International AB, Ericsson AB,

    Fraunhofer-Gesellschaft zur Foerderung der angewandten Forschung e.V., Huawei Technologies Co. LTD.,

    Koninklijke Philips N.V., Nippon Telegraph and Telephone Corporation, Nokia Technologies Oy, Orange,

    Panasonic Holdings Corporation, Qualcomm Technologies, Inc., VoiceAge Corporation, and other

    contributors to this repository. All Rights Reserved.



    This software is protected by copyright law and by international treaties.

    The IVAS codec Public Collaboration consisting of Dolby International AB, Ericsson AB,

    Fraunhofer-Gesellschaft zur Foerderung der angewandten Forschung e.V., Huawei Technologies Co. LTD.,

    Koninklijke Philips N.V., Nippon Telegraph and Telephone Corporation, Nokia Technologies Oy, Orange,

    Panasonic Holdings Corporation, Qualcomm Technologies, Inc., VoiceAge Corporation, and other

    contributors to this repository retain full ownership rights in their respective contributions in

    the software. This notice grants no license of any kind, including but not limited to patent

    license, nor is any license granted by implication, estoppel or otherwise.



    Contributors are required to enter into the IVAS codec Public Collaboration agreement before making

    contributions.



    This software is provided "AS IS", without any express or implied warranties. The software is in the

    development stage. It is intended exclusively for experts who have experience with such software and

    solely for the purpose of inspection. All implied warranties of non-infringement, merchantability

    and fitness for a particular purpose are hereby disclaimed and excluded.



    Any dispute, controversy or claim arising under or in relation to providing this software shall be

    submitted to and settled by the final, binding jurisdiction of the courts of Munich, Germany in

    accordance with the laws of the Federal Republic of Germany excluding its conflict of law rules and

    the United Nations Convention on Contracts on the International Sales of Goods.

    """



__doc__ = """

    Test file to run C encoder and decoder code.

    The outputs are compared with C generated references.

    """



import errno

import os

from filecmp import cmp

from typing import Optional



import pytest



from tests.cmp_pcm import cmp_pcm

from tests.conftest import DecoderFrontend, EncoderFrontend, parse_properties



# params

# output_mode_list = ['MONO', 'STEREO', '5_1', '7_1', '5_1_2', '5_1_4', '7_1_4', 'FOA', 'HOA2', 'HOA3', 'BINAURAL', 'BINAURAL_ROOM', 'EXT']

output_mode_list = ["BINAURAL", "EXT"]

ivas_br_masa = [

    13200,

    16400,

    24400,

    32000,

    48000,

    64000,

    80000,

    96000,

    128000,

    160000,

    192000,

    256000,

    384000,

    512000,

]



# Write file-based parameter sets here (metafile, pcm/wave, numDir, numTC, DTX_toggle)

masa_metadata_audio_ndir_ntransportch_dtx_list = [  # ('stv1MASA1TC48c.met', 'stv1MASA1TC48c.wav', 1, 1, False),

    # ('stv1MASA2TC48c.met', 'stv1MASA2TC48c.wav', 1, 2, False),

    ("stv2MASA1TC48c.met", "stv2MASA1TC48c.wav", 2, 1, False),

    # ('stv2MASA2TC48c.met', 'stv2MASA2TC48c.wav', 2, 2, False),

    # ('stv1MASA1TC48n.met', 'stv1MASA1TC48n.wav', 1, 1, True),

    ("stv1MASA2TC48n.met", "stv1MASA2TC48n.wav", 1, 2, True),

]



# Used to not test every combination

test_split_br = [13200, 24400, 48000, 80000, 128000, 256000, 512000]

AbsTol = "0"





def check_and_makedir(dir_path):

    if not os.path.exists(dir_path):

        try:

            os.makedirs(dir_path)

        except OSError as e:

            if e.errno != errno.EEXIST:

                raise  # raises the error again





@pytest.mark.create_ref

@pytest.mark.parametrize("output_mode", output_mode_list)

@pytest.mark.parametrize("ivas_br", ivas_br_masa)

@pytest.mark.parametrize(

    "masa_metadata_audio_ndir_ntransportch_dtx",

    masa_metadata_audio_ndir_ntransportch_dtx_list,

)

def test_masa_enc_dec(

    record_property,

    props_to_record,

    dut_encoder_frontend: EncoderFrontend,

    dut_decoder_frontend: DecoderFrontend,

    ref_encoder_frontend: EncoderFrontend,

    ref_decoder_frontend: DecoderFrontend,

    reference_path,

    dut_base_path,

    update_ref,

    keep_files,

    ivas_br,

    masa_metadata_audio_ndir_ntransportch_dtx,

    test_vector_path,

    output_mode,

    get_mld,

    get_mld_lim,

    decoder_only,

    abs_tol,

    get_ssnr,

):

    # Input parameters

    in_fs = 48

    out_fs = 48

    metadata = masa_metadata_audio_ndir_ntransportch_dtx[0]

    input_audio = masa_metadata_audio_ndir_ntransportch_dtx[1]

    n_directions = masa_metadata_audio_ndir_ntransportch_dtx[2]

    masa_channel_count = masa_metadata_audio_ndir_ntransportch_dtx[3]

    dtx = masa_metadata_audio_ndir_ntransportch_dtx[4]

    masa_path = f"{test_vector_path}/{metadata}"

    input_audio_path = f"{test_vector_path}/{input_audio}"



    # Apply test skipping here

    if dtx:

        if output_mode != "EXT":

            if ivas_br not in test_split_br:

                pytest.skip(

                    "Skipping some DTX bitrates for other than EXT output to save time"

                )



    # Set reference encoder and decoder



    # Set output paths

    out_dir_bs_ref = f"{reference_path}/masa_test/bitstreams"

    out_dir_bs_dut = f"{dut_base_path}/masa_test/bitstreams"

    out_dir_dec_output_ref = f"{reference_path}/masa_test/dec_output"

    out_dir_dec_output_dut = f"{dut_base_path}/masa_test/dec_output"

    check_and_makedir(out_dir_bs_ref)

    check_and_makedir(out_dir_bs_dut)

    check_and_makedir(out_dir_dec_output_ref)

    check_and_makedir(out_dir_dec_output_dut)



    # To avoid conflicting names in case of parallel test execution, differentiate all cases

    output_bitstream_ref = f"{out_dir_bs_ref}/masa{masa_channel_count}_ndirs{n_directions}_outputmode{output_mode}_ivasbr{ivas_br}k_DTX{dtx}.192"

    output_bitstream_dut = f"{out_dir_bs_dut}/masa{masa_channel_count}_ndirs{n_directions}_outputmode{output_mode}_ivasbr{ivas_br}k_DTX{dtx}.192"

    dec_output_ref = f"{out_dir_dec_output_ref}/masa{masa_channel_count}_ndirs{n_directions}_outputmode{output_mode}_ivasbr{ivas_br}k_DTX{dtx}.wav"

    dec_output_dut = f"{out_dir_dec_output_dut}/masa{masa_channel_count}_ndirs{n_directions}_outputmode{output_mode}_ivasbr{ivas_br}k_DTX{dtx}.wav"

    if output_mode == "EXT":

        dec_met_output_ref = f"{dec_output_ref}.met"

        dec_met_output_dut = f"{dec_output_dut}.met"

    else:

        dec_met_output_ref = None

        dec_met_output_dut = None



    if update_ref == 2 or update_ref == 1:

        # Encode REF

        ivas_enc(

            ref_encoder_frontend,

            masa_channel_count,

            masa_path,

            ivas_br,

            in_fs,

            input_audio_path,

            output_bitstream_ref,

            dtx,

        )



        # Decode REF

        ivas_dec(

            ref_decoder_frontend,

            output_mode,

            out_fs,

            output_bitstream_ref,

            dec_output_ref,

        )



    if update_ref == 2 or update_ref == 0:

        if decoder_only:

            output_bitstream_dut = output_bitstream_ref

        else:

            # Encode DUT

            ivas_enc(

                dut_encoder_frontend,

                masa_channel_count,

                masa_path,

                ivas_br,

                in_fs,

                input_audio_path,

                output_bitstream_dut,

                dtx,

            )



        # Decode DUT

        ivas_dec(

            dut_decoder_frontend,

            output_mode,

            out_fs,

            output_bitstream_dut,

            dec_output_dut,

        )



        # Compare outputs. For EXT output, also compare metadata.

        metacmp_res = True

        if output_mode == "EXT":

            # Compare metadata as binary blob

            metacmp_res = cmp(dec_met_output_ref, dec_met_output_dut)



        # Compare audio outputs

        pcmcmp_res, reason = cmp_pcm(

            dec_output_dut,

            dec_output_ref,

            output_mode,

            int(out_fs * 1000),

            get_mld=get_mld,

            mld_lim=get_mld_lim,

            abs_tol=abs_tol,

            get_ssnr=get_ssnr

        )



        props = parse_properties(reason, pcmcmp_res != 0, props_to_record)

        for k, v in props.items():

            record_property(k, v)



        if get_mld and get_mld_lim > 0:

            if pcmcmp_res != 0:

                pytest.fail(reason)

        else:

            # Fail if compare fails compare result

            if not metacmp_res and pcmcmp_res != 0:

                pytest.fail("Metadata and transport output difference detected")

            elif not metacmp_res:

                pytest.fail("Metadata output difference detected")

            elif pcmcmp_res != 0:

                pytest.fail("Transport output difference detected")

            else:

                print("Comparison bit exact")





#########################################################

# -------------------- test function --------------------

def ivas_enc(

    encoder_frontend,

    masa_channel_count,

    masa_path,

    ivas_br,

    sampling_rate,

    input_audio_path,

    output_bitstream,

    dtx: Optional[bool] = False,

):

    # ------------  run cmd  ------------

    options = ["-masa", f"{masa_channel_count}", f"{masa_path}"]



    # call encoder

    encoder_frontend.run(

        ivas_br,

        sampling_rate,

        input_audio_path,

        output_bitstream,

        dtx_mode=dtx,

        add_option_list=options,

    )





def ivas_dec(

    decoder_frontend,

    output_mode,

    sampling_rate,

    input_bitstream,

    output_path,

):

    # --------  run cmd  ------------



    # call decoder

    decoder_frontend.run(

        output_mode,

        sampling_rate,

        input_bitstream,

        output_path,

    )





def remove_output(

    keep_files,

    output_bitstream_ref,

    output_bitstream_dut,

    dec_output_ref,

    dec_output_dut,

    dec_met_output_ref: Optional[str] = None,

    dec_met_output_dut: Optional[str] = None,

):

    if not keep_files:

        os.remove(output_bitstream_ref)

        os.remove(output_bitstream_dut)

        os.remove(dec_output_ref)

        os.remove(dec_output_dut)

        if dec_met_output_ref is not None:

            os.remove(dec_met_output_ref)

        if dec_met_output_dut is not None:

            os.remove(dec_met_output_dut)