Commit 53becf4b authored by Jan Kiene's avatar Jan Kiene
Browse files

report percentages for MAX_ENC_DIFF and the name of the parameter

parent 15804e07

tests/cmp_stats_files.py

+3 −1
Original line number Diff line number Diff line
@@ -41,6 +41,7 @@ def cmp_stats_files( 
    enc_test_result_msg = ""

    max_total_num_diff = 0

    max_total_num_diff_ratio = 0

    max_diff_name = ""

    for name in ref_stats_names:

        if name in dut_stats_names:

            # retrieve the dictionaries
@@ -108,6 +109,7 @@ def cmp_stats_files( 
            if total_num_diff_ratio > max_total_num_diff_ratio:

                max_total_num_diff = total_num_diff

                max_total_num_diff_ratio = total_num_diff_ratio

                max_diff_name = name



            # update test result

            if result_len_check or result_diff_check:
@@ -117,7 +119,7 @@ def cmp_stats_files( 
            print(msg)



    if enc_test_result and max_total_num_diff > 0:

        msg = f"MAXIMUM ENC DIFF: {max_total_num_diff} ({(max_total_num_diff_ratio*100):.2f}%) "

        msg = f"MAXIMUM ENC DIFF: {max_total_num_diff} ({(max_total_num_diff_ratio*100):.2f}%) MAX_ENC_DIFF_NAME: '{max_diff_name}'"

        enc_test_result_msg += msg

        print(msg)

tests/conftest.py

+65 −24
Original line number Diff line number Diff line
@@ -46,6 +46,8 @@ import tempfile 
from typing import Optional, Union

import numpy as np

from .constants import (

    MAX_ENC_DIFF_NAME_PATTERN,

    MAX_ENC_DIFF_PARAM_NAME,

    MLD_PATTERN,

    MAX_DIFF_PATTERN,

    SSNR_PATTERN,
@@ -273,6 +275,7 @@ def pytest_addoption(parser): 
        default=False,

    )





@pytest.fixture(scope="session", autouse=True)

def update_ref(request):

    """
@@ -332,6 +335,7 @@ def get_odg(request): 
    """

    return request.config.option.odg





@pytest.fixture(scope="session")

def abs_tol(request) -> int:

    """
@@ -408,7 +412,9 @@ class EncoderFrontend: 
        hist_dicts = []



        if not os.path.exists(dbg_tweak_folder):

            print(f"No statistics have been extracted from the res/ folder to the {stats_file} file!")

            print(

                f"No statistics have been extracted from the res/ folder to the {stats_file} file!"

            )

        else:

            for f in ENC_AUX_FILES:

                filename = f[0]
@@ -543,7 +549,11 @@ class EncoderFrontend: 
        self.stdout = result.stdout.decode("ascii")



        # Record core encoder clipping

        for (prop, pattern) in [(ENC_CORE_OVL,ENC_CORE_OVL_PATTERN),(MAX_OVL,MAX_OVL_PATTERN),(MIN_OVL,MIN_OVL_PATTERN)]:

        for prop, pattern in [

            (ENC_CORE_OVL, ENC_CORE_OVL_PATTERN),

            (MAX_OVL, MAX_OVL_PATTERN),

            (MIN_OVL, MIN_OVL_PATTERN),

        ]:

            val = 0

            search_result = re.search(pattern, self.stdout)

            if search_result:
@@ -576,7 +586,9 @@ class EncoderFrontend: 




@pytest.fixture(scope="function")

def dut_encoder_frontend(dut_encoder_path, request, record_property) -> Union[None, EncoderFrontend]:

def dut_encoder_frontend(

    dut_encoder_path, request, record_property

) -> Union[None, EncoderFrontend]:

    """

    Return a :class:`conftest.EncoderFrontend` instance as DUT for the test session.

    """
@@ -584,7 +596,9 @@ def dut_encoder_frontend(dut_encoder_path, request, record_property) -> Union[No 


    if dut_encoder_path:

        timeout = request.config.getoption("--testcase_timeout")

        encoder = EncoderFrontend(dut_encoder_path, "DUT", record_property, timeout=timeout)

        encoder = EncoderFrontend(

            dut_encoder_path, "DUT", record_property, timeout=timeout

        )

    yield encoder



    if encoder is not None:
@@ -623,7 +637,9 @@ def ref_encoder_path(request) -> str: 




@pytest.fixture(scope="function")

def ref_encoder_frontend(ref_encoder_path, request, record_property) -> Union[None, EncoderFrontend]:

def ref_encoder_frontend(

    ref_encoder_path, request, record_property

) -> Union[None, EncoderFrontend]:

    """

    Return a :class:`conftest.EncoderFrontend` instance as REF for the test session.

    """
@@ -631,7 +647,9 @@ def ref_encoder_frontend(ref_encoder_path, request, record_property) -> Union[No 


    if ref_encoder_path:

        timeout = request.config.getoption("--testcase_timeout")

        encoder = EncoderFrontend(ref_encoder_path, "REF", record_property, timeout=timeout)

        encoder = EncoderFrontend(

            ref_encoder_path, "REF", record_property, timeout=timeout

        )



    yield encoder
@@ -846,7 +864,9 @@ class DecoderFrontend: 




@pytest.fixture(scope="function")

def dut_decoder_frontend(dut_decoder_path, request, record_property) -> Union[None, DecoderFrontend]:

def dut_decoder_frontend(

    dut_decoder_path, request, record_property

) -> Union[None, DecoderFrontend]:

    """

    Return a :class:`conftest.DecoderFrontend` instance as DUT for the test session.

    """
@@ -854,7 +874,13 @@ def dut_decoder_frontend(dut_decoder_path, request, record_property) -> Union[No 


    if dut_decoder_path:

        timeout = request.config.getoption("--testcase_timeout")

        decoder = DecoderFrontend(dut_decoder_path, "DUT", record_property, timeout=timeout, fr=request.config.option.dut_fr)

        decoder = DecoderFrontend(

            dut_decoder_path,

            "DUT",

            record_property,

            timeout=timeout,

            fr=request.config.option.dut_fr,

        )



    yield decoder
@@ -894,7 +920,9 @@ def ref_decoder_path(request) -> str: 




@pytest.fixture(scope="function")

def ref_decoder_frontend(ref_decoder_path, request, record_property) -> Union[None, DecoderFrontend]:

def ref_decoder_frontend(

    ref_decoder_path, request, record_property

) -> Union[None, DecoderFrontend]:

    """

    Return a :class:`conftest.DecoderFrontend` instance as DUT for the test session.

    """
@@ -902,7 +930,9 @@ def ref_decoder_frontend(ref_decoder_path, request, record_property) -> Union[No 


    if ref_decoder_path:

        timeout = request.config.getoption("--testcase_timeout")

        decoder = DecoderFrontend(ref_decoder_path, "REF", record_property, timeout=timeout)

        decoder = DecoderFrontend(

            ref_decoder_path, "REF", record_property, timeout=timeout

        )



    yield decoder
@@ -959,7 +989,10 @@ def reference_path(request) -> str: 


    path = str(path.resolve())



    if request.config.option.update_ref == "0" and not request.config.option.compare_to_input:

    if (

        request.config.option.update_ref == "0"

        and not request.config.option.compare_to_input

    ):

        if not os.path.isdir(path):

            raise FileNotFoundError(

                f"REF path {path} not found!\nPlease generate the references, first!\n!"
@@ -1000,6 +1033,7 @@ def encoder_only(request) -> bool: 
    """

    return request.config.getoption("--encoder_only")





@pytest.fixture(scope="session", autouse=True)

def compare_to_input(request) -> bool:

    """
@@ -1007,6 +1041,7 @@ def compare_to_input(request) -> bool: 
    """

    return request.config.getoption("--compare_to_input")





def pytest_configure(config):

    config.addinivalue_line("markers", "serial: mark test to run only in serial")

    if config.option.param_file:
@@ -1079,12 +1114,18 @@ def parse_properties(text_to_parse: str, output_differs: bool, props_to_record: 
            props["MIN_ODG_CHANNEL"] = min_odg_channel

        elif prop == MAX_ENC_DIFF:

            search_result = re.search(MAX_ENC_DIFF_PATTERN, text_to_parse)

            max_enc_diff = 0

            max_enc_diff_ratio = 0.0

            name_result = ""

            if search_result:

                max_enc_diff, max_enc_diff_ratio = search_result.groups(0)

                if max_enc_diff:

                    max_enc_diff = float(max_enc_diff)

            props[MAX_ENC_DIFF] = max_enc_diff

                _, max_enc_diff_ratio = search_result.groups(0)

                if max_enc_diff_ratio:

                    max_enc_diff_ratio = float(max_enc_diff_ratio)



                name_result = re.search(

                    MAX_ENC_DIFF_NAME_PATTERN, text_to_parse

                ).groups(0)[0]

            props[MAX_ENC_DIFF] = max_enc_diff_ratio

            props[MAX_ENC_DIFF_PARAM_NAME] = name_result

    return props

tests/constants.py

+2 −0
Original line number Diff line number Diff line
@@ -12,6 +12,7 @@ MAX_ABS_DIFF = "MAXIMUM ABS DIFF" 
SSNR = "SSNR"

ODG = "ODG"

MAX_ENC_DIFF = "MAXIMUM ENC DIFF"

MAX_ENC_DIFF_PARAM_NAME = "MAXIMUM ENC DIFF PARAM"

ENC_CORE_OVL = "ENC_CORE_OVL"

MAX_OVL = "MAX_OVL"

MIN_OVL = "MIN_OVL"
@@ -23,6 +24,7 @@ ODG_PATTERN_PQEVALAUDIO = r"Objective Difference Grade: (-*\d*\.\d*)" 
ODG_PATTERN = r"ODG: (-*\d*\.\d*)"

SSNR_PATTERN = r"Channel \d* SSNR: (nan|[+-]*inf|[-*\d\.]*)"

MAX_ENC_DIFF_PATTERN = r"MAXIMUM ENC DIFF: (\d+) \((\d+\.\d+)%\)"

MAX_ENC_DIFF_NAME_PATTERN = r"MAX_ENC_DIFF_NAME: '(.*)'"

ENC_CORE_OVL_PATTERN = r"Core input overload detected: (\d+)"

MAX_OVL_PATTERN = r"Max overload value: (\d+\.\d+)"

MIN_OVL_PATTERN = r"Min overload value: (-*\d+\.\d+)"