Commit fdd77170 authored by norvell's avatar norvell
Browse files

Merge branch 'ci/mld-jobs-with-tolerance-limit' into 'main' 

[BASOP CI] Add MLD limit to give pass for tests below limit

See merge request !1326
parents c00edd2b 74f3cf68

scripts/parse_mld_xml.py

+10 −8
Original line number Diff line number Diff line
@@ -25,11 +25,13 @@ if __name__ == '__main__': 


    with open(csv_file,'w') as outfile:

        for testcase in testcases:

            if testcase.find('.//skipped') == None:

                if testcase.get('file') == None:

                    fulltestname = testcase.get('classname').replace('.','/') + ".py::" + testcase.get('name')

                else:

                    fulltestname = testcase.get('file') + "::" + testcase.get('name')

            system_out = testcase.find(".//system-out")

                if testcase.find('.//property') == None:

                    mld_val = None

            if system_out is not None:

                for line in system_out.text.split('\n'):

                    if line.startswith('MLD:'):

                        mld_val = float(line.split()[1])

                else:

                    mld_val = testcase.find('.//property').get('value') # Currently MLD is the only set property. If more are added updates are needed here.

                outfile.write(fulltestname + ';' + str(mld_val)+'\n')

tests/cmp_pcm.py

+8 −3
Original line number Diff line number Diff line
@@ -12,7 +12,7 @@ import pyivastest 
import numpy as np





def cmp_pcm(file1, file2, out_config, fs, get_mld = False) -> (int, str):

def cmp_pcm(file1, file2, out_config, fs, get_mld = False, mld_lim = 0) -> (int, str):

    """

    Compare 2 PCM files for bitexactness

    """
@@ -55,7 +55,11 @@ def cmp_pcm(file1, file2, out_config, fs, get_mld = False) -> (int, str): 
        if get_mld:

            mld_msg = f"MLD: {cmp_result['MLD']}"

            print(mld_msg)

        return 1, "FAIL: Files have different content"

            if cmp_result['MLD'] <= mld_lim:

                return 0, f"MLD: {cmp_result['MLD']} <= {mld_lim}"

            else:    

                return 1, f"MLD: {cmp_result['MLD']} > {mld_lim}"

        return 1, "Non-BE"





if __name__ == "__main__":
@@ -71,6 +75,7 @@ if __name__ == "__main__": 
    )

    parser.add_argument("-s", "--sampling_rate", type=int, default=48000, dest="fs")

    parser.add_argument("--get_mld", action="store_true")

    parser.add_argument("--mld_lim", type=float, default=0, dest="mld_lim")

    args = parser.parse_args()



    result, msg = cmp_pcm(**vars(args))

tests/codec_be_on_mr_nonselection/test_masa_enc_dec.py

+18 −3
Original line number Diff line number Diff line
@@ -93,6 +93,7 @@ def check_and_makedir(dir_path): 
    masa_metadata_audio_ndir_ntransportch_dtx_list,

)

def test_masa_enc_dec(

    record_property,

    dut_encoder_frontend: EncoderFrontend,

    dut_decoder_frontend: DecoderFrontend,

    ref_encoder_frontend: EncoderFrontend,
@@ -106,6 +107,7 @@ def test_masa_enc_dec( 
    test_vector_path,

    output_mode,

    get_mld,

    get_mld_lim,

):

    # Input parameters

    in_fs = 48
@@ -201,8 +203,13 @@ def test_masa_enc_dec( 


            # Compare audio outputs

            pcmcmp_res, reason = cmp_pcm(

                dec_output_dut, dec_output_ref, output_mode, int(out_fs * 1000), get_mld=get_mld

                dec_output_dut, dec_output_ref, output_mode, int(out_fs * 1000), get_mld=get_mld, mld_lim=get_mld_lim

            )

            if get_mld:

                mld = 0

                if "MLD" in reason:

                    mld = float(reason.split(':')[1].split()[0])

                record_property("MLD",mld)



            # Fail if compare fails compare result

            if metacmp_res == False and pcmcmp_res != 0:
@@ -219,11 +226,19 @@ def test_masa_enc_dec( 
            filecmp_res = cmp(dec_output_ref, dec_output_dut)

            if filecmp_res == False:

                cmp_result, reason = cmp_pcm(

                    dec_output_dut, dec_output_ref, output_mode, int(out_fs * 1000), get_mld=get_mld

                    dec_output_dut, dec_output_ref, output_mode, int(out_fs * 1000), get_mld=get_mld, mld_lim=get_mld_lim

                )

                if get_mld:

                    mld = 0

                    if "MLD" in reason:

                        mld = float(reason.split(':')[1].split()[0])

                    record_property("MLD",mld)

                # Report compare result

                assert cmp_result == 0, reason

                if cmp_result != 0:

                    pytest.fail(reason)                

            else:

                if get_mld:

                    record_property("MLD","0")

                print("Comparison bit exact")



        # remove_output(

tests/codec_be_on_mr_nonselection/test_param_file.py

+12 −4
Original line number Diff line number Diff line
@@ -125,6 +125,7 @@ def convert_test_string_to_tag(test_string): 
@pytest.mark.create_ref

@pytest.mark.parametrize("test_tag", list(param_file_test_dict.keys()))

def test_param_file_tests(

    record_property,

    dut_encoder_frontend: EncoderFrontend,

    dut_decoder_frontend: DecoderFrontend,

    ref_encoder_frontend: EncoderFrontend,
@@ -137,6 +138,7 @@ def test_param_file_tests( 
    keep_files,

    test_tag,

    get_mld,

    get_mld_lim,

):

    enc_opts, dec_opts, sim_opts, eid_opts = param_file_test_dict[test_tag]
@@ -325,10 +327,16 @@ def test_param_file_tests( 
        ref_output_file = f"{reference_path}/param_file/dec/{output_file}"

        fs = int(sampling_rate) * 1000

        output_differs, reason = cmp_pcm(

            dut_output_file, ref_output_file, output_config, fs, get_mld=get_mld

            dut_output_file, ref_output_file, output_config, fs, get_mld=get_mld, mld_lim=get_mld_lim

        )

        md_out_files = get_expected_md_files(ref_output_file, enc_split, output_config)



        if get_mld:

            mld = 0

            if "MLD" in reason:

                mld = float(reason.split(':')[1].split()[0])

            record_property("MLD",mld)



        metadata_differs = False

        for md_file in md_out_files:

            dut_metadata_file = Path(f"{dut_base_path}/param_file/dec/{md_file}")
@@ -347,13 +355,13 @@ def test_param_file_tests( 
        if output_differs or metadata_differs:

            msg = "Difference between ref and dut in "

            if output_differs and metadata_differs:

                msg += "output and metadata"

                msg += f"output ({reason}) and metadata"

            elif output_differs:

                msg += "output only"

                msg += f"output only ({reason})"

            elif metadata_differs:

                msg += "metadata only"



            assert False, msg

            pytest.fail(msg)



        # remove DUT output files when test result is OK (to save disk space)

        if not keep_files:

tests/codec_be_on_mr_nonselection/test_sba_bs_dec_plc.py

+14 −2
Original line number Diff line number Diff line
@@ -75,6 +75,7 @@ def check_and_makedir(dir_path): 
@pytest.mark.parametrize("fs", sampling_rate_list)

@pytest.mark.parametrize("gain_flag", gain_list)

def test_sba_plc_system(

    record_property,

    dut_decoder_frontend: DecoderFrontend,

    test_vector_path,

    reference_path,
@@ -89,6 +90,7 @@ def test_sba_plc_system( 
    fs,

    gain_flag,

    get_mld,

    get_mld_lim,

):

    SID = 0

    if dtx == '1' and ivas_br not in ['13200','16400','24400','32000', '64000']:
@@ -105,6 +107,7 @@ def test_sba_plc_system( 


    # dec

    sba_dec_plc(

        record_property,

        dut_decoder_frontend,

        test_vector_path,

        reference_path,
@@ -120,12 +123,14 @@ def test_sba_plc_system( 
        gain_flag,

        keep_files,

        get_mld=get_mld,

        get_mld_lim=get_mld_lim,

    )





#########################################################

# -------------------- test function --------------------

def sba_dec_plc(

    record_property,

    decoder_frontend,

    test_vector_path,

    reference_path,
@@ -141,6 +146,7 @@ def sba_dec_plc( 
    gain_flag,

    keep_files,

    get_mld=False,

    get_mld_lim=0,

):



    # ------------  run cmd  ------------
@@ -191,10 +197,16 @@ def sba_dec_plc( 


        # --------------  compare cmd  --------------

        fs = int(sampling_rate) * 1000

        cmp_result, reason = cmp_pcm(dut_out_raw, ref_out_raw, output_config, fs, get_mld=get_mld)

        cmp_result, reason = cmp_pcm(dut_out_raw, ref_out_raw, output_config, fs, get_mld=get_mld, mld_lim=get_mld_lim)

        if get_mld:

            mld = 0

            if "MLD" in reason:

                mld = float(reason.split(':')[1].split()[0])

            record_property("MLD",mld)



        # report compare result

        assert cmp_result == 0, reason

        if cmp_result != 0:

            pytest.fail(reason)



        # remove DUT output files when test result is OK (to save disk space)

        if not keep_files: