Commit fba82c09 authored by Archit Tamarapu's avatar Archit Tamarapu
Browse files

Merge branch 'main' of ssh://forge.3gpp.org:29419/ivas-codec-pc/ivas-codec... 

Merge branch 'main' of ssh://forge.3gpp.org:29419/ivas-codec-pc/ivas-codec into 1254-ci-renderer-pytest-on-merge-request-broken
parents 0471f7c3 2dbc5206

scripts/create_histogram_summary.py

+24 −28
Original line number Diff line number Diff line
@@ -71,17 +71,17 @@ if __name__ == "__main__": 
        CATEGORIES = IVAS_CATEGORIES

    if args.diff:

        limits_per_measure = {

            "MLD": ("MLD", [-math.inf, 0, 0.1, 0.2, 0.3, 0.4, math.inf]),

            "DIFF": ("MAXIMUM ABS DIFF", [-32768, 0, 128, 256, 512, 32767]),

            "SSNR": ("MIN_SSNR", [-math.inf, -0.4, -0.3, -0.2, -0.1, 0, math.inf]),

            "ODG": ("MIN_ODG", [-5.0, -0.3, -0.2, -0.1, 0, math.inf]),

            "MLD": ("MLD", None),

            "DIFF": ("MAXIMUM ABS DIFF", None),

            "SSNR": ("MIN_SSNR", None),

            "ODG": ("MIN_ODG", None),

        }

    else:

        limits_per_measure = {

            "MLD": ("MLD", [0, 5, 10, math.inf]),

            "DIFF": ("MAXIMUM ABS DIFF", [0, 1024, 16384, 32769]),

            "SSNR": ("MIN_SSNR", [-math.inf, 0, 20, 40, 60, 100]),

            "ODG": ("MIN_ODG", [-5, -4, -3, -2, -1, 0]),

            "MLD": ("MLD", [0, 1, 2, 3, 4, 5, 10, 20, math.inf]),

            "DIFF": ("MAXIMUM ABS DIFF", [0, 16, 256, 1024, 2048, 4096, 8192, 16384, 32769]),

            "SSNR": ("MIN_SSNR", [-math.inf, 0, 10, 20, 30, 40, 40, 50, 60, 100]),

            "ODG": ("MIN_ODG", [-5, -4, -3, -2, -1, 0, 1, 2, 3, 4, 5]),

        }

    (measure_label, limits) = limits_per_measure[measure]
@@ -97,14 +97,19 @@ if __name__ == "__main__": 
            for k, val in zip(keys, row[1:]):

                results_sorted[testcase][k] = val



    if limits is None:

        vals = [float(x) for x in [m[measure_label] for m in results_sorted.values() if m[measure_label] != "None" and m[measure_label] != ""]]

        start = min(vals)

        f = 10 ** (2 - int(np.floor(np.log10(abs(start)))) - 1)

        start = np.floor(start*f)/f

        step = (max(vals) - start)/10

        f = 10 ** (2 - int(np.floor(np.log10(abs(step)))) - 1)

        step = np.ceil(step*f)/f

        limits = np.arange(start, 10*step, step)



    # Output CSV file

    with open(csv_summary, "w") as fp:

        limits_labels = [

            f"{str(a)} -- {str(b)}" for (a, b) in zip(limits[0:-1], limits[1:])

        ] + ["None"]

        # Zero difference is treated as a special category for MLD and MAXIMUM ABS DIFF

        if (measure_label == "MLD" or measure_label == "MAXIMUM ABS DIFF") and not args.diff:

            limits_labels = ["0"] + limits_labels

        limits_labels = [f"{a:g}" for a in limits] + ["","None"] # Put None cases in separate bin

        headerline = f"Format;Category;" + ";".join(limits_labels) + "\n"

        fp.write(headerline)
@@ -120,27 +125,18 @@ if __name__ == "__main__": 
                        if m["Format"] == fmt and m["Category"] == cat

                    ]

                ]

                # Zero difference is treated as a special category for MLD and MAXIMUM ABS DIFF

                if (measure_label == "MLD" or measure_label == "MAXIMUM ABS DIFF") and not args.diff:

                    val = [

                        float(x) for x in values if x != "None" and x != "0" and x != ""

                    ]

                    zero = [sum([1 for x in values if x == "0"])]

                    none = [sum([1 for x in values if x == "None" or x == ""])]

                else:

                # Create separate bin for None (errors)

                val = [float(x) for x in values if x != "None" and x != ""]

                    zero = []

                none = [sum([1 for x in values if x == "None" or x == ""])]

                hist, _ = np.histogram(val, limits)

                data = np.array(zero + list(hist) + none)

                data = np.array(list(hist) + [0] + none + [0])



                # CSV output

                line = f"{fmt};{cat};{'; '.join(map(str,data))}\n"

                fp.write(line)



                # Matplotlib histogram

                labels = [x if len(x) < 10 else x.replace('--','--\n') for x in limits_labels]

                ax.bar(labels, data, 0.5, label=cat, bottom=bottom)

                ax.bar(limits_labels, data, 1, align='edge', edgecolor='black', linewidth=0.5, label=cat, bottom=bottom)

                bottom += data



            # Histogram layout

tests/codec_be_on_mr_nonselection/test_param_file.py

+13 −6
Original line number Diff line number Diff line
@@ -43,7 +43,12 @@ import numpy as np 


from tests.cmp_pcm import cmp_pcm

from tests.cmp_stats_files import cmp_stats_files

from tests.conftest import DecoderFrontend, EncoderFrontend, parse_properties

from tests.conftest import (

    DecoderFrontend,

    EncoderFrontend,

    parse_properties,

    log_dbg_msg,

)

from tests.testconfig import PARAM_FILE

from tests.constants import (

    MAX_ENC_FILE_LENGTH_DIFF,
@@ -206,7 +211,6 @@ def test_param_file_tests( 
    get_odg,

    compare_to_input,

):



    enc_opts, dec_opts, sim_opts, eid_opts = param_file_test_dict[test_tag]



    run_test(
@@ -267,7 +271,6 @@ def run_test( 
    get_odg,

    compare_to_input,

):



    # If compare_to_input is set, only run pass-through test cases

    if compare_to_input:

        passthrough = [
@@ -489,7 +492,6 @@ def run_test( 
    )



    if update_ref in [0, 2]:



        # Output file names for comparison

        dut_output_file = f"{dut_base_path}/param_file/dec/{output_file}"

        ref_output_file = f"{reference_path}/param_file/dec/{output_file}"
@@ -499,7 +501,6 @@ def run_test( 
        odg_test = None

        odg_ref = None

        if get_odg:



            # Find input format

            in_fmt = [(a, b) for (a, b) in INPUT_FMT if re.search(a, enc_opts)][0][1]
@@ -763,7 +764,10 @@ def simulate( 
        cmd_opts[3] = f"{dut_out_dir}/{netsim_tracefile}"

        cmd_opts[2] = f"{dut_out_dir}/{netsim_outfile}"  # dut_out_file



    run(netsim + cmd_opts, check=False)

    netsim_cmd = netsim + cmd_opts

    netsim_cmd_str = " ".join(netsim_cmd)

    log_dbg_msg(f"netsim command:\n{netsim_cmd_str}")

    run(netsim_cmd, check=False)





def error_insertion(
@@ -813,6 +817,9 @@ def error_insertion( 
    elif update_ref in [0, 2]:

        cmd_opts[-1] = f"{dut_out_dir}/{eid_xor_outfile}"  # ref_out_file



    eid_cmd = eid_xor + cmd_opts

    eid_cmd_str = " ".join(eid_cmd)

    log_dbg_msg(f"eid-xor command:\n{eid_cmd_str}")

    run(eid_xor + cmd_opts, check=False)

tests/conftest.py

+5 −4
Original line number Diff line number Diff line
@@ -387,11 +387,13 @@ def dut_encoder_path(request) -> str: 


    return path





# fixture returns test information, enabling per-testcase SNR

@pytest.fixture

def test_info(request):

    return request





class EncoderFrontend:

    def __init__(self, path, enc_type, record_property, timeout=None) -> None:

        self._path = Path(path).absolute()
@@ -1052,7 +1054,6 @@ def compare_to_input(request) -> bool: 
    return request.config.getoption("--compare_to_input")







def pytest_configure(config):

    config.addinivalue_line("markers", "serial: mark test to run only in serial")

    if config.option.param_file: