Commit 017cb80f authored by BOHMRR's avatar BOHMRR
Browse files

pytest reporting improvements 

- cmp_custom() returns reason (str) in addition to retval (int)
- reason is part of the assert and will show up in the short test summary
- SBA tests now do a single comparison on the interleaved PCM; il2mm.py is removed
parent bb2e79c3

tests/cmp_custom.py

+18 −18
Original line number Diff line number Diff line
@@ -89,7 +89,7 @@ class CompareSamples: 
        self.file_1.seek(0)

        self.file_2.seek(0)



    def print_summary(self):

    def print_summary(self) -> (int, str):

        """

        Print the summary of the comparison.

        """
@@ -105,19 +105,18 @@ class CompareSamples: 
        if not self.diff_present:

            print("Comparison success")

            print("")

            return 0

        else:

            return 0, "Comparison success"



        # comparison failed

        print(

            f"First unmatched diff ==> {self.first_diff}",

            f"at sample num {self.first_diff_sample_num}",

        )

            print(

                f"MAXIMUM ABS DIFF ==> {self.max_diff} at sample num {self.max_diff_sample_num}"

            )

        diff_msg = f"MAXIMUM ABS DIFF ==> {self.max_diff} at sample num {self.max_diff_sample_num}"

        print(diff_msg)

        print("Comparison failed")

        print("")

            return 1

        return 1

        return 1, f"Comparison failed, {diff_msg}"



    def compare_next_sample(self):

        """
@@ -148,7 +147,7 @@ class CompareSamples: 


def usage():

    print(__doc__)

    return 1

    return 1, ""





def cmp_custom(
@@ -157,7 +156,7 @@ def cmp_custom( 
    sample_size_in_bytes_str,

    tolerance_str,

    end_samples_to_skip_str="0",

):

) -> (int, str):

    """

    Function to compare the samples in 2 PCM files.

    """
@@ -191,10 +190,11 @@ def cmp_custom( 
    return cmp_samples.print_summary()





def main(argv):

def main(argv) -> int:

    if len(argv) < 5:

        return usage()

    return cmp_custom(*argv[1:])

    retval, reason = cmp_custom(*argv[1:])

    return retval





if __name__ == "__main__":

tests/il2mm.py

deleted100644 → 0
+0 −61
Original line number Diff line number Diff line 
"""

   (C) 2022 IVAS codec Public Collaboration with portions copyright Dolby International AB, Ericsson AB,

   Fraunhofer-Gesellschaft zur Foerderung der angewandten Forschung e.V., Huawei Technologies Co. LTD.,

   Koninklijke Philips N.V., Nippon Telegraph and Telephone Corporation, Nokia Technologies Oy, Orange,

   Panasonic Holdings Corporation, Qualcomm Technologies, Inc., VoiceAge Corporation, and other

   contributors to this repository. All Rights Reserved.



   This software is protected by copyright law and by international treaties.

   The IVAS codec Public Collaboration consisting of Dolby International AB, Ericsson AB,

   Fraunhofer-Gesellschaft zur Foerderung der angewandten Forschung e.V., Huawei Technologies Co. LTD.,

   Koninklijke Philips N.V., Nippon Telegraph and Telephone Corporation, Nokia Technologies Oy, Orange,

   Panasonic Holdings Corporation, Qualcomm Technologies, Inc., VoiceAge Corporation, and other

   contributors to this repository retain full ownership rights in their respective contributions in

   the software. This notice grants no license of any kind, including but not limited to patent

   license, nor is any license granted by implication, estoppel or otherwise.



   Contributors are required to enter into the IVAS codec Public Collaboration agreement before making

   contributions.



   This software is provided "AS IS", without any express or implied warranties. The software is in the

   development stage. It is intended exclusively for experts who have experience with such software and

   solely for the purpose of inspection. All implied warranties of non-infringement, merchantability

   and fitness for a particular purpose are hereby disclaimed and excluded.



   Any dispute, controversy or claim arising under or in relation to providing this software shall be

   submitted to and settled by the final, binding jurisdiction of the courts of Munich, Germany in

   accordance with the laws of the Federal Republic of Germany excluding its conflict of law rules and

   the United Nations Convention on Contracts on the International Sales of Goods.

"""



import os





def il2mm(file_in, num_ch, b_delete=True):

    """

    Convert interleaved input file to multiple mono output files.

    """

    num_bytes_per_sample = 2

    num_bytes_per_frame = num_bytes_per_sample * num_ch

    num_bytes_per_channel = os.path.getsize(file_in) / num_ch



    with open(file_in, "rb") as fid_in:

        out_path = os.path.splitext(file_in)[0]

        for chan in range(num_ch):

            file_out = out_path + str(chan + 1) + "ch.raw"

            with open(file_out, "wb") as fid_out:

                bytes_written = 0

                offset = chan * num_bytes_per_sample

                fid_in.seek(offset, 0)

                while bytes_written < num_bytes_per_channel:

                    data = fid_in.read(num_bytes_per_sample)

                    fid_in.seek(num_bytes_per_frame - num_bytes_per_sample, 1)

                    written = fid_out.write(bytes(data))

                    assert (

                        written == num_bytes_per_sample

                    ), f"Error writing data: {written} != {num_bytes_per_sample}"

                    bytes_written += num_bytes_per_sample



    # delete interleaved input file

    if b_delete:

        os.remove(file_in)

tests/test_param_file.py

+2 −2
Original line number Diff line number Diff line
@@ -317,5 +317,5 @@ def compare( 
    """

    sample_size = "2"  # 16-bit samples

    tolerance = "0"  # zero tolerance for BE testing

    cmp_result = cmp_custom(pcm_file_1, pcm_file_2, sample_size, tolerance)

    assert cmp_result == 0
 
    cmp_result, reason = cmp_custom(pcm_file_1, pcm_file_2, sample_size, tolerance)

    assert cmp_result == 0, reason

tests/test_sba_bs_dec_plc.py

+17 −28
Original line number Diff line number Diff line
@@ -30,10 +30,8 @@ 


import os

import pytest

import shutil

import errno



from il2mm import il2mm

from cmp_custom import cmp_custom 

from conftest import DecoderFrontend
@@ -45,7 +43,6 @@ ivas_br_list = ['32000', '64000', '96000', '256000'] 
sampling_rate_list = ['48', '32', '16']

agc_list = [0, 1]



ch_count_foa = 4

AbsTol = '3'
@@ -88,7 +85,7 @@ def test_sba_plc_system( 
    tag = tag + fs + 'c'



    #dec

    sba_dec_plc(dut_decoder_frontend, test_vector_path, reference_path, dut_base_path, ref_decoder_path, tag, ch_count_foa, fs, ivas_br, dtx, plc_pattern, update_ref, agc)

    sba_dec_plc(dut_decoder_frontend, test_vector_path, reference_path, dut_base_path, ref_decoder_path, tag, fs, ivas_br, dtx, plc_pattern, update_ref, agc)





#########################################################
@@ -100,7 +97,6 @@ def sba_dec_plc( 
    dut_base_path,

    ref_decoder_path,

    tag,

    ch_count,

    sampling_rate,

    ivas_br,

    dtx,
@@ -116,8 +112,8 @@ def sba_dec_plc( 
        tag_out += '_AGC1'

    plc_tag_out = f"{tag_out}_{plc_pattern}"



    dut_out_dir = f"{dut_base_path}/sba_bs/raw/{plc_tag_out}"

    ref_out_dir = f"{reference_path}/sba_bs/raw/{plc_tag_out}"

    dut_out_dir = f"{dut_base_path}/sba_bs/raw"

    ref_out_dir = f"{reference_path}/sba_bs/raw"



    check_and_makedir(dut_out_dir)

    check_and_makedir(ref_out_dir)
@@ -126,6 +122,9 @@ def sba_dec_plc( 
    ref_in_pkt = f"{reference_path}/sba_bs/pkt/{tag_out}.pkt"

    ref_in_pkt_dutenc = f"{reference_path}/sba_bs/pkt/{tag_out}_dutenc.pkt"



    dut_out_raw = f"{dut_out_dir}/{plc_tag_out}.raw"

    ref_out_raw = f"{ref_out_dir}/{plc_tag_out}.raw"



    if ref_decoder_path:

        ref_decoder = DecoderFrontend(ref_decoder_path, "REF")
@@ -134,44 +133,34 @@ def sba_dec_plc( 
            "FOA",

            sampling_rate,

            ref_in_pkt,

            f"{ref_out_dir}/out.raw",

            ref_out_raw,

            plc_file=plc_file,

        )



        # convert REF interleaved to multi-mono

        il2mm(f"{ref_out_dir}/out.raw", ch_count)



    if update_ref == 0:

        # call DUT decoder

        decoder_frontend.run(

            "FOA",

            sampling_rate,

            ref_in_pkt_dutenc,

            f"{dut_out_dir}/out.raw",

            dut_out_raw,

            plc_file=plc_file,

        )



        il2mm(f"{dut_out_dir}/out.raw", ch_count)



        #########  compare cmd   #####################################



        end_skip_samples = '0'



        test_fail = False

        for count in range(ch_count):

            ch_id = str(count + 1)



            if cmp_custom(

                f"{dut_out_dir}/out{ch_id}ch.raw",

                f"{ref_out_dir}/out{ch_id}ch.raw",

        cmp_result, reason = cmp_custom(

            dut_out_raw,

            ref_out_raw,

            "2",

            AbsTol,

            end_skip_samples

            ) != 0:

                test_fail = True

        )



        ##File removal##

        shutil.rmtree(dut_out_dir, ignore_errors=True)

        os.remove(dut_out_raw)



        ##report failure

        assert not test_fail
 
        assert cmp_result == 0, reason

tests/test_sba_bs_enc.py

+16 −36
Original line number Diff line number Diff line
@@ -36,9 +36,7 @@ The outputs are compared with C generated references. 
import os

import pytest

import errno

import shutil



from il2mm import il2mm

from cmp_custom import cmp_custom

from cut_pcm import cut_samples

from conftest import EncoderFrontend, DecoderFrontend
@@ -64,7 +62,6 @@ agc_list = [0, 1] 
sample_rate_bw_idx_list = [('48', 'SWB'), ('48', 'WB'), ('32', 'WB')]



AbsTol = '4'

ch_count_foa = 4





def check_and_makedir(dir_path):
@@ -129,7 +126,6 @@ def test_bypass_enc( 
        reference_path,

        dut_base_path,

        tag,

        ch_count_foa,

        fs,

        ivas_br,

        dtx,
@@ -199,7 +195,6 @@ def test_sba_enc_system( 
        reference_path,

        dut_base_path,

        tag,

        ch_count_foa,

        fs,

        ivas_br,

        dtx,
@@ -261,7 +256,6 @@ def test_spar_hoa2_enc_system( 
        reference_path,

        dut_base_path,

        tag,

        ch_count_foa,

        fs,

        ivas_br,

        dtx,
@@ -323,7 +317,6 @@ def test_spar_hoa3_enc_system( 
        reference_path,

        dut_base_path,

        tag,

        ch_count_foa,

        fs,

        ivas_br,

        dtx,
@@ -386,7 +379,6 @@ def test_sba_enc_BWforce_system( 
        reference_path,

        dut_base_path,

        tag,

        ch_count_foa,

        fs,

        ivas_br,

        dtx,
@@ -515,7 +507,6 @@ def sba_dec( 
    reference_path,

    dut_base_path,

    tag,

    ch_count,

    sampling_rate,

    ivas_br,

    dtx,
@@ -546,12 +537,15 @@ def sba_dec( 
    # to avoid conflicting names in case of parallel test execution, differentiate all cases

    long_tag_ext = f"_AGC{agc}_pca{bypass}"



    dut_out_dir = f"{dut_base_path}/sba_bs/raw/{tag_out}{long_tag_ext}"

    ref_out_dir = f"{reference_path}/sba_bs/raw/{tag_out}{short_tag_ext}"

    dut_out_dir = f"{dut_base_path}/sba_bs/raw"

    ref_out_dir = f"{reference_path}/sba_bs/raw"



    dut_in_pkt = f"{dut_base_path}/sba_bs/pkt/{tag_out}{long_tag_ext}.pkt"

    ref_in_pkt = f"{reference_path}/sba_bs/pkt/{tag_out}{short_tag_ext}.pkt"



    dut_out_raw = f"{dut_out_dir}/{tag_out}{long_tag_ext}.raw"

    ref_out_raw = f"{ref_out_dir}/{tag_out}{short_tag_ext}.raw"



    check_and_makedir(dut_out_dir)

    check_and_makedir(ref_out_dir)
@@ -563,48 +557,34 @@ def sba_dec( 
            output_config,

            sampling_rate,

            ref_in_pkt,

            f"{ref_out_dir}/out.raw",

            ref_out_raw,

        )



        # convert REF interleaved to multi-mono

        il2mm(f"{ref_out_dir}/out.raw", ch_count, b_delete=not keep_files)



    if update_ref == 0:

        # call DUT decoder

        decoder_frontend.run(

            output_config,

            sampling_rate,

            dut_in_pkt,

            f"{dut_out_dir}/out.raw",

            dut_out_raw,

        )



        il2mm(f"{dut_out_dir}/out.raw", ch_count, b_delete=not keep_files)



        #########  compare cmd   #####################################



        end_skip_samples = '0'



        test_fail = False

        for count in range(ch_count):

            ch_id = str(count + 1)



            # TEST

            fsize1 = os.path.getsize(f"{dut_out_dir}/out{ch_id}ch.raw")

            fsize2 = os.path.getsize(f"{ref_out_dir}/out{ch_id}ch.raw")

            print(f"Want to compare {dut_out_dir}/out{ch_id}ch.raw ({fsize1} bytes) with {ref_out_dir}/out{ch_id}ch.raw ({fsize2} bytes)")

            if cmp_custom(

                f"{dut_out_dir}/out{ch_id}ch.raw",

                f"{ref_out_dir}/out{ch_id}ch.raw",

        cmp_result, reason = cmp_custom(

            dut_out_raw,

            ref_out_raw,

            "2",

            AbsTol,

            end_skip_samples

            ) != 0:

                test_fail = True

        )



        ##File removal##

        if not keep_files:

            os.remove(dut_in_pkt)

            shutil.rmtree(dut_out_dir, ignore_errors=True)

            os.remove(dut_out_raw)



        ##report failure

        assert not test_fail
 
        assert cmp_result == 0, reason