Commit 70d7bce9 authored by Jan Kiene's avatar Jan Kiene
Browse files

Merge branch 'ci/encoder-stats-changes' into 'main' 

[CI] report percentages for MAX_ENC_DIFF

See merge request !1879
parents 8b0cdbe1 329c22ff

tests/cmp_stats_files.py

+10 −8
Original line number Diff line number Diff line
@@ -18,7 +18,7 @@ def str2num(s): 




def cmp_stats_files(

    ref_stats_file, dut_stats_file, min_enc_file_length_diff=0.1, min_enc_stats_diff=0.1

    ref_stats_file, dut_stats_file, max_enc_file_length_diff, max_enc_stats_diff

) -> (int, str):

    """

    Compare two .stats files containing encoder statistics (extracted from binary files)
@@ -41,6 +41,7 @@ def cmp_stats_files( 
    enc_test_result_msg = ""

    max_total_num_diff = 0

    max_total_num_diff_ratio = 0

    max_diff_name = ""

    for name in ref_stats_names:

        if name in dut_stats_names:

            # retrieve the dictionaries
@@ -59,7 +60,7 @@ def cmp_stats_files( 
                if (

                    abs(ref_stats_dict["length"] - dut_stats_dict["length"])

                    / file_length

                    > min_enc_file_length_diff

                    > max_enc_file_length_diff

                ):

                    result_len_check = 1
@@ -97,7 +98,7 @@ def cmp_stats_files( 
            )



            msg += f"the total number of differences is {total_num_diff} ({(total_num_diff_ratio*100):.2f}%)"

            if total_num_diff_ratio > min_enc_stats_diff:

            if total_num_diff_ratio > max_enc_stats_diff:

                result_diff_check = 1

                msg += "! "

            else:
@@ -108,6 +109,7 @@ def cmp_stats_files( 
            if total_num_diff_ratio > max_total_num_diff_ratio:

                max_total_num_diff = total_num_diff

                max_total_num_diff_ratio = total_num_diff_ratio

                max_diff_name = name



            # update test result

            if result_len_check or result_diff_check:
@@ -116,8 +118,8 @@ def cmp_stats_files( 


            print(msg)



    if enc_test_result and max_total_num_diff > 0:

        msg = f"MAXIMUM ENC DIFF: {max_total_num_diff} ({(max_total_num_diff_ratio*100):.2f}%) "

    if enc_test_result or max_total_num_diff > 0:

        msg = f"MAXIMUM ENC DIFF: {max_diff_name} {max_total_num_diff} ({(max_total_num_diff_ratio*100):.2f}%)"

        enc_test_result_msg += msg

        print(msg)
@@ -130,13 +132,13 @@ if __name__ == "__main__": 
    parser.add_argument("ref_stats_file", type=str)

    parser.add_argument("dut_stats_file", type=str)

    parser.add_argument(

        "--min_enc_file_length_diff",

        "--max_enc_file_length_diff",

        type=float,

        default=0.1,

        dest="min_enc_file_length_diff",

        dest="max_enc_file_length_diff",

    )

    parser.add_argument(

        "--min_enc_stats_diff", type=float, default=0.1, dest="min_enc_stats_diff"

        "--max_enc_stats_diff", type=float, default=0.1, dest="max_enc_stats_diff"

    )

    args = parser.parse_args()

tests/codec_be_on_mr_nonselection/test_param_file.py

+9 −7
Original line number Diff line number Diff line
@@ -47,8 +47,8 @@ from tests.cmp_stats_files import cmp_stats_files 
from tests.conftest import DecoderFrontend, EncoderFrontend, parse_properties

from tests.testconfig import PARAM_FILE

from tests.constants import (

    MIN_ENC_FILE_LENGTH_DIFF,

    MIN_ENC_STATS_DIFF,

    MAX_ENC_FILE_LENGTH_DIFF,

    MAX_ENC_STATS_DIFF,

    SCRIPTS_DIR,

    MAX_ENC_DIFF,

)
@@ -251,13 +251,15 @@ def test_param_file_tests( 
            enc_test_result, enc_test_result_msg = cmp_stats_files(

                ref_stats_file,

                dut_stats_file,

                min_enc_file_length_diff=MIN_ENC_FILE_LENGTH_DIFF,

                min_enc_stats_diff=MIN_ENC_STATS_DIFF,

                max_enc_file_length_diff=MAX_ENC_FILE_LENGTH_DIFF,

                max_enc_stats_diff=MAX_ENC_STATS_DIFF,

            )



            print("")

            cmp_result_msg += enc_test_result_msg



            # avoid double recording of the encoder diff

            if encoder_only:

                props = parse_properties(cmp_result_msg, False, [MAX_ENC_DIFF])

                for k, v in props.items():

                    dut_encoder_frontend.record_property(k, v)

tests/codec_be_on_mr_nonselection/test_sba.py

+3 −3
Original line number Diff line number Diff line
@@ -42,7 +42,7 @@ from tests.cmp_pcm import cmp_pcm 
from tests.conftest import DecoderFrontend, EncoderFrontend

from ..conftest import parse_properties

from ..cmp_stats_files import cmp_stats_files

from ..constants import TESTV_DIR, MIN_ENC_FILE_LENGTH_DIFF, MIN_ENC_STATS_DIFF

from ..constants import TESTV_DIR, MAX_ENC_FILE_LENGTH_DIFF, MAX_ENC_STATS_DIFF

from tests.testconfig import use_ltv
@@ -309,8 +309,8 @@ def test_sba_enc_system( 
            enc_test_result, enc_test_result_msg = cmp_stats_files(

                ref_stats_file,

                dut_stats_file,

                min_enc_file_length_diff=MIN_ENC_FILE_LENGTH_DIFF,

                min_enc_stats_diff=MIN_ENC_STATS_DIFF,

                max_enc_file_length_diff=MAX_ENC_FILE_LENGTH_DIFF,

                max_enc_stats_diff=MAX_ENC_STATS_DIFF,

            )



            print("")

tests/conftest.py

+59 −24
Original line number Diff line number Diff line
@@ -46,6 +46,8 @@ import tempfile 
from typing import Optional, Union

import numpy as np

from .constants import (

    # MAX_ENC_DIFF_NAME_PATTERN,

    MAX_ENC_DIFF_PARAM_NAME,

    MLD_PATTERN,

    MAX_DIFF_PATTERN,

    SSNR_PATTERN,
@@ -273,6 +275,7 @@ def pytest_addoption(parser): 
        default=False,

    )





@pytest.fixture(scope="session", autouse=True)

def update_ref(request):

    """
@@ -332,6 +335,7 @@ def get_odg(request): 
    """

    return request.config.option.odg





@pytest.fixture(scope="session")

def abs_tol(request) -> int:

    """
@@ -408,7 +412,9 @@ class EncoderFrontend: 
        hist_dicts = []



        if not os.path.exists(dbg_tweak_folder):

            print(f"No statistics have been extracted from the res/ folder to the {stats_file} file!")

            print(

                f"No statistics have been extracted from the res/ folder to the {stats_file} file!"

            )

        else:

            for f in ENC_AUX_FILES:

                filename = f[0]
@@ -543,7 +549,11 @@ class EncoderFrontend: 
        self.stdout = result.stdout.decode("ascii")



        # Record core encoder clipping

        for (prop, pattern) in [(ENC_CORE_OVL,ENC_CORE_OVL_PATTERN),(MAX_OVL,MAX_OVL_PATTERN),(MIN_OVL,MIN_OVL_PATTERN)]:

        for prop, pattern in [

            (ENC_CORE_OVL, ENC_CORE_OVL_PATTERN),

            (MAX_OVL, MAX_OVL_PATTERN),

            (MIN_OVL, MIN_OVL_PATTERN),

        ]:

            val = 0

            search_result = re.search(pattern, self.stdout)

            if search_result:
@@ -576,7 +586,9 @@ class EncoderFrontend: 




@pytest.fixture(scope="function")

def dut_encoder_frontend(dut_encoder_path, request, record_property) -> Union[None, EncoderFrontend]:

def dut_encoder_frontend(

    dut_encoder_path, request, record_property

) -> Union[None, EncoderFrontend]:

    """

    Return a :class:`conftest.EncoderFrontend` instance as DUT for the test session.

    """
@@ -584,7 +596,9 @@ def dut_encoder_frontend(dut_encoder_path, request, record_property) -> Union[No 


    if dut_encoder_path:

        timeout = request.config.getoption("--testcase_timeout")

        encoder = EncoderFrontend(dut_encoder_path, "DUT", record_property, timeout=timeout)

        encoder = EncoderFrontend(

            dut_encoder_path, "DUT", record_property, timeout=timeout

        )

    yield encoder



    if encoder is not None:
@@ -623,7 +637,9 @@ def ref_encoder_path(request) -> str: 




@pytest.fixture(scope="function")

def ref_encoder_frontend(ref_encoder_path, request, record_property) -> Union[None, EncoderFrontend]:

def ref_encoder_frontend(

    ref_encoder_path, request, record_property

) -> Union[None, EncoderFrontend]:

    """

    Return a :class:`conftest.EncoderFrontend` instance as REF for the test session.

    """
@@ -631,7 +647,9 @@ def ref_encoder_frontend(ref_encoder_path, request, record_property) -> Union[No 


    if ref_encoder_path:

        timeout = request.config.getoption("--testcase_timeout")

        encoder = EncoderFrontend(ref_encoder_path, "REF", record_property, timeout=timeout)

        encoder = EncoderFrontend(

            ref_encoder_path, "REF", record_property, timeout=timeout

        )



    yield encoder
@@ -846,7 +864,9 @@ class DecoderFrontend: 




@pytest.fixture(scope="function")

def dut_decoder_frontend(dut_decoder_path, request, record_property) -> Union[None, DecoderFrontend]:

def dut_decoder_frontend(

    dut_decoder_path, request, record_property

) -> Union[None, DecoderFrontend]:

    """

    Return a :class:`conftest.DecoderFrontend` instance as DUT for the test session.

    """
@@ -854,7 +874,13 @@ def dut_decoder_frontend(dut_decoder_path, request, record_property) -> Union[No 


    if dut_decoder_path:

        timeout = request.config.getoption("--testcase_timeout")

        decoder = DecoderFrontend(dut_decoder_path, "DUT", record_property, timeout=timeout, fr=request.config.option.dut_fr)

        decoder = DecoderFrontend(

            dut_decoder_path,

            "DUT",

            record_property,

            timeout=timeout,

            fr=request.config.option.dut_fr,

        )



    yield decoder
@@ -894,7 +920,9 @@ def ref_decoder_path(request) -> str: 




@pytest.fixture(scope="function")

def ref_decoder_frontend(ref_decoder_path, request, record_property) -> Union[None, DecoderFrontend]:

def ref_decoder_frontend(

    ref_decoder_path, request, record_property

) -> Union[None, DecoderFrontend]:

    """

    Return a :class:`conftest.DecoderFrontend` instance as DUT for the test session.

    """
@@ -902,7 +930,9 @@ def ref_decoder_frontend(ref_decoder_path, request, record_property) -> Union[No 


    if ref_decoder_path:

        timeout = request.config.getoption("--testcase_timeout")

        decoder = DecoderFrontend(ref_decoder_path, "REF", record_property, timeout=timeout)

        decoder = DecoderFrontend(

            ref_decoder_path, "REF", record_property, timeout=timeout

        )



    yield decoder
@@ -959,7 +989,10 @@ def reference_path(request) -> str: 


    path = str(path.resolve())



    if request.config.option.update_ref == "0" and not request.config.option.compare_to_input:

    if (

        request.config.option.update_ref == "0"

        and not request.config.option.compare_to_input

    ):

        if not os.path.isdir(path):

            raise FileNotFoundError(

                f"REF path {path} not found!\nPlease generate the references, first!\n!"
@@ -1000,6 +1033,7 @@ def encoder_only(request) -> bool: 
    """

    return request.config.getoption("--encoder_only")





@pytest.fixture(scope="session", autouse=True)

def compare_to_input(request) -> bool:

    """
@@ -1007,6 +1041,7 @@ def compare_to_input(request) -> bool: 
    """

    return request.config.getoption("--compare_to_input")





def pytest_configure(config):

    config.addinivalue_line("markers", "serial: mark test to run only in serial")

    if config.option.param_file:
@@ -1079,12 +1114,12 @@ def parse_properties(text_to_parse: str, output_differs: bool, props_to_record: 
            props["MIN_ODG_CHANNEL"] = min_odg_channel

        elif prop == MAX_ENC_DIFF:

            search_result = re.search(MAX_ENC_DIFF_PATTERN, text_to_parse)

            max_enc_diff = 0

            max_enc_diff_ratio = 0.0

            max_enc_diff_param_name = ""

            if search_result:

                max_enc_diff, max_enc_diff_ratio = search_result.groups(0)

                if max_enc_diff:

                    max_enc_diff = float(max_enc_diff)

            props[MAX_ENC_DIFF] = max_enc_diff

                max_enc_diff_param_name, _, max_enc_diff_ratio = search_result.groups(0)

            props[MAX_ENC_DIFF] = float(max_enc_diff_ratio)

            props[MAX_ENC_DIFF_PARAM_NAME] = max_enc_diff_param_name

    return props

tests/constants.py

+6 −6
Original line number Diff line number Diff line
@@ -12,6 +12,7 @@ MAX_ABS_DIFF = "MAXIMUM ABS DIFF" 
SSNR = "SSNR"

ODG = "ODG"

MAX_ENC_DIFF = "MAXIMUM ENC DIFF"

MAX_ENC_DIFF_PARAM_NAME = "MAXIMUM ENC DIFF PARAM"

ENC_CORE_OVL = "ENC_CORE_OVL"

MAX_OVL = "MAX_OVL"

MIN_OVL = "MIN_OVL"
@@ -22,17 +23,16 @@ MAX_DIFF_PATTERN = r"MAXIMUM ABS DIFF: (\d*)" 
ODG_PATTERN_PQEVALAUDIO = r"Objective Difference Grade: (-*\d*\.\d*)"

ODG_PATTERN = r"ODG: (-*\d*\.\d*)"

SSNR_PATTERN = r"Channel \d* SSNR: (nan|[+-]*inf|[-*\d\.]*)"

MAX_ENC_DIFF_PATTERN = r"MAXIMUM ENC DIFF: (\d+) \((\d+\.\d+)%\)"

MAX_ENC_DIFF_PATTERN = r"MAXIMUM ENC DIFF: ([^ ]*) (\d+) \((\d+\.\d+)%\)"

ENC_CORE_OVL_PATTERN = r"Core input overload detected: (\d+)"

MAX_OVL_PATTERN = r"Max overload value: (\d+\.\d+)"

MIN_OVL_PATTERN = r"Min overload value: (-*\d+\.\d+)"



# maximum "relative" tolerable difference between ref and dut encoder file lengths

MAX_ENC_FILE_LENGTH_DIFF = 0.1



# minimum difference between ref and dut encoder file lengths

MIN_ENC_FILE_LENGTH_DIFF = 0.1

# minimum difference between the statistics of ref and dut encoder files

MIN_ENC_STATS_DIFF = 0.1



# maximum "relative" tolerable difference between the statistics of ref and dut encoder files

MAX_ENC_STATS_DIFF = 0.5



# list of encoder filename patterns with their data type and number of samples per frame

# note: instead of specifying the number of samples per frame, you can use a formula incl. 'fs', e.g. 'fs/50'