Commit 654ff3c4 authored by Jan Kiene's avatar Jan Kiene
Browse files

Merge branch 'ci/compare-dmx-pytest' into 'main' 

[BASOP-CI] Ci add comparison of encoder dmx dump to pytest

See merge request !1945
parents a126dbfe 5fbb7bf6

lib_dec/lib_dec.c

+1 −0
Original line number Diff line number Diff line
@@ -45,6 +45,7 @@ 
#include <assert.h>

#ifdef DEBUGGING

#include "debug.h"

#include <string.h>

#endif

#include "wmc_auto.h"

lib_enc/ivas_core_pre_proc_front.c

+10 −0
Original line number Diff line number Diff line
@@ -34,6 +34,7 @@ 
#include "options.h"

#ifdef DEBUGGING

#include "debug.h"

#include <string.h>

#endif

#include "cnst.h"

#include "ivas_cnst.h"
@@ -220,6 +221,15 @@ ivas_error pre_proc_front_ivas( 
        }

    }



#ifdef DEBUG_MODE_INFO

    if ( !( hCPE->hStereoTD != NULL && n > 0 ) )

    {

        int16_t tmp_dmx_in[L_FRAME48k];

        mvr2s( signal_in - NS2SA( st->input_Fs, ACELP_LOOK_NS ), tmp_dmx_in, input_frame );

        dbgwrite( tmp_dmx_in, sizeof( int16_t ), input_frame, 1, strcat( fname( debug_dir, "ivas_input_dmx", 0, n + 1, ENC ), ".pcm" ) );

    }

#endif



    lMemRecalc_12k8 = 0;

    lMemRecalc = 0;

    if ( element_mode == IVAS_CPE_TD || element_mode == IVAS_CPE_MDCT )

scripts/pyaudio3dtools/audioarray.py

+2 −1
Original line number Diff line number Diff line
@@ -426,8 +426,9 @@ def compare( 
                raise NotImplementedError(

                    f"wav-diff tool not available for {curr_platform}"

                )

            

            search_path = toolsdir.joinpath(curr_platform.replace("Windows", "Win32"))

            wdiff = search_path.joinpath("wav-diff")

            wdiff = search_path.joinpath("wav-diff").with_suffix(".exe" if curr_platform == "Windows" else "")



            if not wdiff.exists():

                wdiff = shutil.which("wav-diff")

tests/cmp_pcm.py

+37 −20
Original line number Diff line number Diff line
@@ -26,7 +26,7 @@ except ImportError: 
def cmp_pcm(

    ref_file,

    cmp_file,

    out_config,

    nchannels: int,

    fs,

    get_mld=False,

    allow_differing_lengths=False,
@@ -39,23 +39,18 @@ def cmp_pcm( 
    odg_ref=None,

    ref_jbm_tf: Optional[Path] = None,

    cut_jbm_tf: Optional[Path] = None,

    quiet: Optional[bool] = False,

) -> tuple[int, str]:

    """

    Compare 2 PCM files for bitexactness

    """

    print("Cmp PCM Report")

    print("=====================")



    out_config = "MONO" if out_config == "" else out_config

    # out_config may be a string or a Path. Wrap in str() to avoid error in case it is a Path.

    if str(out_config).upper() not in pyivastest.constants.OC_TO_NCHANNELS:

        nchannels = (

            pyivastest.IvasScriptsCommon.IvasScript.get_n_channels_from_ls_layout(

                out_config

            )

        )

    else:

        nchannels = pyivastest.constants.OC_TO_NCHANNELS[out_config.upper()]

    output_target = sys.stdout

    if quiet:

        output_target = open(os.devnull, "w")



    print("Cmp PCM Report", file=output_target)

    print("=====================", file=output_target)



    s1, fs1 = pyaudio3dtools.audiofile.readfile(

        ref_file, nchannels, fs, outdtype=np.int16
@@ -85,6 +80,7 @@ def cmp_pcm( 
        print(

            f"file size in samples: file 1 = {s1.shape[0]},",

            f"file 2 = {s2.shape[0]}",

            file=output_target,

        )

        reason = "FAIL: File lengths differ. MAXIMUM ABS DIFF: None"

        if get_mld:
@@ -95,7 +91,7 @@ def cmp_pcm( 
    cmp_result = pyaudio3dtools.audioarray.compare(

        s1,

        s2,

        fs,

        fs * 1000,

        per_frame=False,

        get_mld=get_mld,

        get_ssnr=get_ssnr,
@@ -113,8 +109,8 @@ def cmp_pcm( 
    elif not cmp_result["bitexact"]:

        diff_msg = f"MAXIMUM ABS DIFF ==> {cmp_result['max_abs_diff']} at sample num {cmp_result['max_abs_diff_pos_sample']} (assuming {nchannels} channels)"

        first_msg = f"First diff found at sample num {cmp_result['first_diff_pos_sample']} in channel {cmp_result['first_diff_pos_channel']}, frame {cmp_result['first_diff_pos_frame']} (assuming {nchannels} channels, {fs} sampling rate)"

        print(diff_msg)

        print(first_msg)

        print(diff_msg, file=output_target)

        print(first_msg, file=output_target)



        reason = f"Non-BE - MAXIMUM ABS DIFF: {cmp_result['max_abs_diff']}"

        output_differs = 1
@@ -122,7 +118,7 @@ def cmp_pcm( 
    if get_mld:

        mld_msg = f"MLD: {cmp_result['MLD']}"

        reason += " - " + mld_msg

        print(mld_msg)

        print(mld_msg, file=output_target)



        if cmp_result["MLD"] <= mld_lim:

            output_differs = 0
@@ -171,7 +167,10 @@ def cmp_pcm( 


        msg = f"ODG: {odg}"

        reason += " - " + msg

        print(msg)

        print(msg, file=output_target)



    if quiet:

        output_target.close()



    return output_differs, reason
@@ -214,6 +213,21 @@ def pqevalaudio_wrapper( 
        return result.stdout.decode("utf8")





def out_config_2_nchannels(out_config):

    out_config = "MONO" if out_config == "" else out_config

    # out_config may be a string or a Path. Wrap in str() to avoid error in case it is a Path.

    if str(out_config).upper() not in pyivastest.constants.OC_TO_NCHANNELS:

        nchannels = (

            pyivastest.IvasScriptsCommon.IvasScript.get_n_channels_from_ls_layout(

                out_config

            )

        )

    else:

        nchannels = pyivastest.constants.OC_TO_NCHANNELS[out_config.upper()]



    return nchannels





if __name__ == "__main__":

    parser = argparse.ArgumentParser()

    parser.add_argument("ref_file", type=str)
@@ -231,8 +245,11 @@ if __name__ == "__main__": 
    parser.add_argument("--get_odg", action="store_true")

    parser.add_argument("--get_ssnr", action="store_true")

    parser.add_argument("--allow_differing_lengths", action="store_true")

    args = parser.parse_args()

    parser.add_argument("--quiet", action="store_true")

    args = vars(parser.parse_args())



    args["nchannels"] = out_config_2_nchannels(args.pop("out_config"))



    result, msg = cmp_pcm(**vars(args))

    result, msg = cmp_pcm(**args)

    print(msg)

    sys.exit(result)

tests/codec_be_on_mr_nonselection/test_param_file.py

+31 −2
Original line number Diff line number Diff line
@@ -39,14 +39,19 @@ from pathlib import Path 
from subprocess import run

import pytest

import re

import sys

import numpy as np



from tests.cmp_pcm import cmp_pcm

THIS_PATH = os.path.join(os.getcwd(), __file__)

sys.path.append(os.path.join(os.path.dirname(THIS_PATH), "../../scripts"))



from tests.cmp_pcm import cmp_pcm, out_config_2_nchannels

from tests.cmp_stats_files import cmp_stats_files

from tests.conftest import (

    DecoderFrontend,

    EncoderFrontend,

    parse_properties,

    compare_dmx_signals,

    log_dbg_msg,

)

from tests.testconfig import PARAM_FILE
@@ -55,6 +60,9 @@ from tests.constants import ( 
    MAX_ENC_STATS_DIFF,

    SCRIPTS_DIR,

    MAX_ENC_DIFF,

    DMX_DIFF,

    DMX_MLD,

    DMX_SSNR,

)

from tests.renderer.utils import check_and_makedir, binauralize_input_and_output
@@ -210,6 +218,7 @@ def test_param_file_tests( 
    get_enc_stats,

    get_odg,

    compare_to_input,

    compare_enc_dmx,

):

    enc_opts, dec_opts, sim_opts, eid_opts = param_file_test_dict[test_tag]
@@ -240,6 +249,7 @@ def test_param_file_tests( 
        get_enc_stats,

        get_odg,

        compare_to_input,

        compare_enc_dmx,

    )
@@ -270,6 +280,7 @@ def run_test( 
    get_enc_stats,

    get_odg,

    compare_to_input,

    compare_enc_dmx,

):

    # If compare_to_input is set, only run pass-through test cases

    if compare_to_input:
@@ -331,6 +342,7 @@ def run_test( 
            enc_split,

            update_ref,

            get_enc_stats,

            compare_enc_dmx,

        )



        # compare binary files extracted from the encoder
@@ -492,6 +504,20 @@ def run_test( 
    )



    if update_ref in [0, 2]:

        if compare_enc_dmx:

            dut_dmx_files = sorted(

                Path(f"{dut_base_path}/param_file/enc/{bitstream_file}").parent.glob(

                    f"{Path(bitstream_file).stem}.dmx.ch*.pcm"

                )

            )

            ref_dmx_files = [

                str(f).replace(dut_base_path, reference_path) for f in dut_dmx_files

            ]



            prop_results = compare_dmx_signals(ref_dmx_files, dut_dmx_files, in_sr)

            for prop, val in prop_results.items():

                dut_decoder_frontend.record_property(prop, val)



        # Output file names for comparison

        dut_output_file = f"{dut_base_path}/param_file/dec/{output_file}"

        ref_output_file = f"{reference_path}/param_file/dec/{output_file}"
@@ -561,7 +587,7 @@ def run_test( 
        output_differs, reason = cmp_pcm(

            ref_file,

            dut_output_file,

            output_config,

            out_config_2_nchannels(output_config),

            fs,

            get_mld=get_mld,

            mld_lim=get_mld_lim,
@@ -662,6 +688,7 @@ def encode( 
    enc_opts_list,

    update_ref,

    get_enc_stats=False,

    compare_enc_dmx=False,

):

    """

    Call REF and/or DUT encoder.
@@ -692,6 +719,7 @@ def encode( 
            ref_out_file,

            add_option_list=enc_opts_list,

            stats_file=ref_stats_file,

            compare_enc_dmx=compare_enc_dmx,

        )



    if update_ref in [0, 2]:
@@ -705,6 +733,7 @@ def encode( 
            dut_out_file,

            add_option_list=enc_opts_list,

            stats_file=dut_stats_file,

            compare_enc_dmx=compare_enc_dmx,

        )