Commit 844f99d4 authored by Jan Kiene's avatar Jan Kiene
Browse files

add dmx comparison + add to report

parent 2c0e6187

tests/cmp_pcm.py

+51 −25
Original line number Diff line number Diff line
@@ -15,6 +15,7 @@ sys.path.append(os.path.join(os.path.dirname(THIS_PATH), "../scripts")) 
import numpy as np

import pyaudio3dtools

import pyivastest



# Hack to resolve import when using from command line or from within scripts.

try:

    from .constants import ODG_PATTERN_PQEVALAUDIO
@@ -25,7 +26,7 @@ except ImportError: 
def cmp_pcm(

    ref_file,

    cmp_file,

    out_config,

    nchannels: int,

    fs,

    get_mld=False,

    allow_differing_lengths=False,
@@ -38,23 +39,18 @@ def cmp_pcm( 
    odg_ref=None,

    ref_jbm_tf: Optional[Path] = None,

    cut_jbm_tf: Optional[Path] = None,

    quiet: Optional[bool] = False,

) -> (int, str):

    """

    Compare 2 PCM files for bitexactness

    """

    print("Cmp PCM Report")

    print("=====================")



    out_config = "MONO" if out_config == "" else out_config

    # out_config may be a string or a Path. Wrap in str() to avoid error in case it is a Path.

    if str(out_config).upper() not in pyivastest.constants.OC_TO_NCHANNELS:

        nchannels = (

            pyivastest.IvasScriptsCommon.IvasScript.get_n_channels_from_ls_layout(

                out_config

            )

        )

    else:

        nchannels = pyivastest.constants.OC_TO_NCHANNELS[out_config.upper()]

    output_target = sys.stdout

    if quiet:

        output_target = open(os.devnull, "w")



    print("Cmp PCM Report", file=output_target)

    print("=====================", file=output_target)



    s1, fs1 = pyaudio3dtools.audiofile.readfile(

        ref_file, nchannels, fs, outdtype=np.int16
@@ -80,12 +76,17 @@ def cmp_pcm( 
    if allow_differing_lengths:

        # to allow for MLD comparison, pad shorter file

        max_len = max(s1.shape[0], s2.shape[0])

        s1 = np.pad(s1,((0,max_len - s1.shape[0]),(0,0)),mode='constant',constant_values=0)

        s2 = np.pad(s2,((0,max_len - s2.shape[0]),(0,0)),mode='constant',constant_values=0)

        s1 = np.pad(

            s1, ((0, max_len - s1.shape[0]), (0, 0)), mode="constant", constant_values=0

        )

        s2 = np.pad(

            s2, ((0, max_len - s2.shape[0]), (0, 0)), mode="constant", constant_values=0

        )

    elif s1.shape != s2.shape:

        print(

            f"file size in samples: file 1 = {s1.shape[0]},",

            f"file 2 = {s2.shape[0]}",

            file=output_target,

        )

        reason = "FAIL: File lengths differ. MAXIMUM ABS DIFF: None"

        if get_mld:
@@ -96,7 +97,7 @@ def cmp_pcm( 
    cmp_result = pyaudio3dtools.audioarray.compare(

        s1,

        s2,

        fs,

        fs * 1000,

        per_frame=False,

        get_mld=get_mld,

        get_ssnr=get_ssnr,
@@ -113,8 +114,8 @@ def cmp_pcm( 
    elif not cmp_result["bitexact"]:

        diff_msg = f"MAXIMUM ABS DIFF ==> {cmp_result['max_abs_diff']} at sample num {cmp_result['max_abs_diff_pos_sample']} (assuming {nchannels} channels)"

        first_msg = f"First diff found at sample num {cmp_result['first_diff_pos_sample']} in channel {cmp_result['first_diff_pos_channel']}, frame {cmp_result['first_diff_pos_frame']} (assuming {nchannels} channels, {fs} sampling rate)"

        print(diff_msg)

        print(first_msg)

        print(diff_msg, file=output_target)

        print(first_msg, file=output_target)



        reason = f"Non-BE - MAXIMUM ABS DIFF: {cmp_result['max_abs_diff']}"

        output_differs = 1
@@ -122,7 +123,7 @@ def cmp_pcm( 
    if get_mld:

        mld_msg = f"MLD: {cmp_result['MLD']}"

        reason += " - " + mld_msg

        print(mld_msg)

        print(mld_msg, file=output_target)



        if cmp_result["MLD"] <= mld_lim:

            output_differs = 0
@@ -149,11 +150,15 @@ def cmp_pcm( 
                32767,

            ).astype(np.int16)



        pqeval_output = pqevalaudio_wrapper(odg_files[odg_input], odg_files[odg_ref], 48000)

        pqeval_output = pqevalaudio_wrapper(

            odg_files[odg_input], odg_files[odg_ref], 48000

        )

        match_odg = re.search(ODG_PATTERN_PQEVALAUDIO, pqeval_output)

        odg_ref = float(match_odg.groups()[0])



        pqeval_output = pqevalaudio_wrapper(odg_files[odg_input], odg_files[odg_test], 48000)

        pqeval_output = pqevalaudio_wrapper(

            odg_files[odg_input], odg_files[odg_test], 48000

        )

        match_odg = re.search(ODG_PATTERN_PQEVALAUDIO, pqeval_output)

        odg_test = float(match_odg.groups()[0])
@@ -161,7 +166,10 @@ def cmp_pcm( 


        msg = f"ODG: {odg}"

        reason += " - " + msg

        print(msg)

        print(msg, file=output_target)



    if quiet:

        output_target.close()



    return output_differs, reason
@@ -200,6 +208,21 @@ def pqevalaudio_wrapper( 
        return result.stdout.decode("utf8")





def out_config_2_nchannels(out_config):

    out_config = "MONO" if out_config == "" else out_config

    # out_config may be a string or a Path. Wrap in str() to avoid error in case it is a Path.

    if str(out_config).upper() not in pyivastest.constants.OC_TO_NCHANNELS:

        nchannels = (

            pyivastest.IvasScriptsCommon.IvasScript.get_n_channels_from_ls_layout(

                out_config

            )

        )

    else:

        nchannels = pyivastest.constants.OC_TO_NCHANNELS[out_config.upper()]



    return nchannels





if __name__ == "__main__":

    parser = argparse.ArgumentParser()

    parser.add_argument("ref_file", type=str)
@@ -217,8 +240,11 @@ if __name__ == "__main__": 
    parser.add_argument("--get_odg", action="store_true")

    parser.add_argument("--get_ssnr", action="store_true")

    parser.add_argument("--allow_differing_lengths", action="store_true")

    args = parser.parse_args()

    parser.add_argument("--quiet", action="store_true")

    args = vars(parser.parse_args())



    args["nchannels"] = out_config_2_nchannels(args.pop("out_config"))



    result, msg = cmp_pcm(**vars(args))

    result, msg = cmp_pcm(**args)

    print(msg)

    sys.exit(result)

tests/codec_be_on_mr_nonselection/test_param_file.py

+29 −2
Original line number Diff line number Diff line
@@ -39,9 +39,15 @@ from pathlib import Path 
from subprocess import run

import pytest

import re

import sys

import numpy as np



from tests.cmp_pcm import cmp_pcm

THIS_PATH = os.path.join(os.getcwd(), __file__)

sys.path.append(os.path.join(os.path.dirname(THIS_PATH), "../../scripts"))



import pyaudio3dtools



from tests.cmp_pcm import cmp_pcm, out_config_2_nchannels

from tests.cmp_stats_files import cmp_stats_files

from tests.conftest import DecoderFrontend, EncoderFrontend, parse_properties

from tests.testconfig import PARAM_FILE
@@ -50,6 +56,7 @@ from tests.constants import ( 
    MAX_ENC_STATS_DIFF,

    SCRIPTS_DIR,

    MAX_ENC_DIFF,

    DMX_DIFF,

)

from tests.renderer.utils import check_and_makedir, binauralize_input_and_output
@@ -491,6 +498,26 @@ def run_test( 
    )



    if update_ref in [0, 2]:

        if compare_enc_dmx:

            dut_dmx_files = Path(

                f"{dut_base_path}/param_file/enc/{bitstream_file}"

            ).parent.glob(f"{Path(bitstream_file).stem}.dmx.ch*.pcm")



            for dut_dmx_file in dut_dmx_files:

                ref_dmx_file = str(dut_dmx_file).replace(dut_base_path, reference_path)

                testv_sig, _ = pyaudio3dtools.audiofile.readfile(testv_file)

                nchannels = testv_sig.shape[1]

                dmx_differs, reason = cmp_pcm(

                    ref_dmx_file,

                    dut_dmx_file,

                    nchannels,

                    in_sr,

                    quiet=True,

                )



                prop = parse_properties(reason, dmx_differs, [DMX_DIFF])

                dut_decoder_frontend.record_property(DMX_DIFF, prop[DMX_DIFF])



        # Output file names for comparison

        dut_output_file = f"{dut_base_path}/param_file/dec/{output_file}"

        ref_output_file = f"{reference_path}/param_file/dec/{output_file}"
@@ -560,7 +587,7 @@ def run_test( 
        output_differs, reason = cmp_pcm(

            ref_file,

            dut_output_file,

            output_config,

            out_config_2_nchannels(output_config),

            fs,

            get_mld=get_mld,

            mld_lim=get_mld_lim,

tests/conftest.py

+2 −1
Original line number Diff line number Diff line
@@ -47,6 +47,7 @@ from typing import Optional, Union 
import numpy as np

from .constants import (

    # MAX_ENC_DIFF_NAME_PATTERN,

    DMX_DIFF,

    MAX_ENC_DIFF_PARAM_NAME,

    MLD_PATTERN,

    MAX_DIFF_PATTERN,
@@ -1131,7 +1132,7 @@ def parse_properties(text_to_parse: str, output_differs: bool, props_to_record: 
        if prop == MLD:

            mld = float(re.search(MLD_PATTERN, text_to_parse).groups(1)[0])

            props[prop] = mld

        elif prop == MAX_ABS_DIFF:

        elif prop == MAX_ABS_DIFF or prop == DMX_DIFF:

            max_diff = 0

            if output_differs:

                if (match := re.search(MAX_DIFF_PATTERN, text_to_parse)) is not None:

tests/constants.py

+1 −0
Original line number Diff line number Diff line
@@ -16,6 +16,7 @@ MAX_ENC_DIFF_PARAM_NAME = "MAXIMUM ENC DIFF PARAM" 
ENC_CORE_OVL = "ENC_CORE_OVL"

MAX_OVL = "MAX_OVL"

MIN_OVL = "MIN_OVL"

DMX_DIFF = "DMX MAXIMUM ABS DIFF"



# regex patterns for parsing the output from comparisons -> mainly for BASOP ci

MLD_PATTERN = r"MLD: ([\d\.]*)"