Commit d4574f97 authored by Jan Kiene's avatar Jan Kiene
Browse files

Merge branch 'basop-ci/synch-main-and-basop-ci-branch' into 'main' 

[BASOP-CI]synch main and basop ci branch

See merge request !1621
parents 4b040b1f 54607495

ci/basop-pages/basop_index.html

+0 −6
Original line number Diff line number Diff line
@@ -13,10 +13,4 @@ 
    <li><a href="ivas-pytest-mld-long-dec-lev-10-index.html">ivas-pytest-mld-long-dec-lev-10</a></li>

  </ul>



  <h2>Test Coverage</h2>



  <br>

  tbd...

  </br>



</body>

ci/basop-pages/create_report_pages.py

+99 −32
Original line number Diff line number Diff line 
import csv

import pathlib

import argparse

from functools import partial





CSV_DELIM = ";"
@@ -11,11 +12,14 @@ SUBPAGE_TMPL_CSS = """ 
  overflow:hidden;padding:10px 5px;word-break:normal;}

.tbase th{border-color:black;border-style:solid;border-width:1px;font-family:sans-serif;font-size:14px;

  font-weight:normal;overflow:hidden;padding:10px 5px;word-break:normal;}

.tbase .tunder{font-weight:bold;text-align:center;text-decoration:underline;vertical-align:top}

.tbase .tcenter{font-weight:bold;text-align:center;vertical-align:top}

.tbase .tleft{text-align:left;vertical-align:top}

.tbase .tincrease{text-align:left;vertical-align:top;background-color:#ff0000;border-color:inherit;font-weight:bold;}

.tbase .treduce{text-align:left;vertical-align:top;background-color:#00ff00;border-color:inherit;font-weight:bold;}

.tbase .tunder{font-weight:bold;text-align:center;text-decoration:underline;}

.tbase .tcenter{font-weight:bold;text-align:center;}

.tbase .tleft{text-align:left;horizontal-align:bottom}

.tbase .tincrease{text-align:left;background-color:#ff5500;border-color:inherit;font-weight:bold;}

.tbase .treduce{text-align:left;background-color:#acff00;border-color:inherit;font-weight:bold;}



.arrowup {font-weight:bold;font-size:200%;}

.arrowdown {font-weight:bold;font-size:200%;}

</style>

"""
@@ -27,12 +31,41 @@ Comparing: 
<ul>

    <li>Current run - id: <a href="https://forge.3gpp.org/rep/sa4/audio/ivas-basop/-/jobs/{id_current}">{id_current}</a></li>

    <li>Previous run - id: <a href="https://forge.3gpp.org/rep/sa4/audio/ivas-basop/-/jobs/{id_previous}">{id_previous}</a></li>

    <li><a href="{job_name}--merged_csv--{id_current}--{id_previous}.csv">Merged csv data</a></li>

    <li><a href="{job_name}--merged_csv--{id_current}.csv">Merged csv data</a></li>

</ul>



<br>

<b>Table is sorted by Difference in MLD with ERRORs or missing values ("None", "") being on top additionally.</b>

<b>How is the table sorted?</b>

<ul>

    <li>Cases with result ERROR or invalid/missing values for the numerical measures are given first</li>

    <li>Next are all cases with MLD(current) != MLD(previous), sorted in descending order (biggest MLD increase comes first)</li>

    <li>All cases with no difference in MLD are at the bottom</li>

</ul>

<br>

<b>What do the colours indicate</b>

<ul>

    <li>

        <span style="background-color:#ff5500;">Red background</span>:

        <ul>

            <li>This testcases either triggered an ERROR in the pytest run (probably a crash in the codec) or did not give valid values for the numerical measures (probably an error in the scripts).</li>

            <li>For this testcase the MLD increased wrt the previous run - MLD(current) > MLD(previous)</li>

        </ul>

    </li>

    <li>

        <span style="background-color:#acff00;">Green background</span>:

        <ul>

            <li>For this testcase the MLD decreased wrt the previous run - MLD(current) < MLD(previous)</li>

        </ul>

    </li>

</ul>

<br>

<b>How to interpret the Result column?</b>

<ul>

    <li>ERROR: An error occured during test run. It should be checked if a crash in the codec occured.</li>

    <li>FAIL: An MLD value > 0 was reported. This is to be expected, the test just reports a failure due to how the tests are currently implemented.</li>

    <li>PASS: MLD value of 0 was reported. This should only be the case for some special operating points and could indicate that parts of the codec are not (fully) converted yet.</li>

</ul>

<b>



<table class="tbase"><thead>

  <tr>
@@ -54,12 +87,15 @@ TH_TMPL_GLOBAL = '<th class="tunder" rowspan="2">{}</th>' 
TH_TMPL_DIFFERENTIAL = '<th class="tunder" colspan="2">{}</th>'

TH_TMPL_SECOND_ROW = '<th class="tcenter">{}</th>'



ARROW_UP = '<span class="arrowup">&#11008;</span>'

ARROW_DOWN = '<span class="arrowdown">&#11010;</span>'



# expected columns. actual columns are filtered from the incoming data later, this

# is mainly for controlling the order in the output table

COLUMNS = ["testcase", "Result", "MLD", "MAXIMUM ABS DIFF"]

COLUMNS_GLOBAL = COLUMNS[:1]

COLUMNS_DIFFERENTIAL = COLUMNS[1:]



COLUMNS_DIFFERENTIAL_NOT_MLD = COLUMNS_DIFFERENTIAL[2:]



def create_subpage(

    html_out,
@@ -78,8 +114,8 @@ def create_subpage( 
    table_header_a = "".join([TH_TMPL_GLOBAL.format(c) for c in COLUMNS_GLOBAL] + [TH_TMPL_DIFFERENTIAL.format(c) for c in COLUMNS_DIFFERENTIAL])

    table_header_b = list()

    for c in COLUMNS_DIFFERENTIAL:

        table_header_b.append(TH_TMPL_SECOND_ROW.format(id_previous))

        table_header_b.append(TH_TMPL_SECOND_ROW.format(id_current))

        table_header_b.append(TH_TMPL_SECOND_ROW.format(f"Previous Run<br>ID: {id_previous}"))

        table_header_b.append(TH_TMPL_SECOND_ROW.format(f"Current Run<br>ID: {id_current}"))

    table_header_b = "".join(table_header_b)

    table_body = "\n".join(

        tr_from_row(row, id_current, id_previous) for row in merged_reports
@@ -127,20 +163,30 @@ def tr_from_row(row, id_current, id_previous): 
        prev = row[f"{c}-{id_previous}"]

        curr = row[f"{c}-{id_current}"]



        # use red background if increase, green if decrease, white if same

        td_tmpl = TD_TMPL_NORMAL

        if c == "Result":

            # print errors in bold red font

            td_tmpl = TD_TMPL_INCREASE if prev == "ERROR" else TD_TMPL_NORMAL

            tr.append(td_tmpl.format(prev))

            td_tmpl = TD_TMPL_INCREASE if curr == "ERROR" else TD_TMPL_NORMAL

            tr.append(td_tmpl.format(curr))

        else:

            td_tmpl_curr = TD_TMPL_NORMAL

            td_tmpl_prev = TD_TMPL_NORMAL

            try:

                if float(curr) > float(prev):

                td_tmpl = TD_TMPL_INCREASE

            if float(curr) < float(prev):

                td_tmpl = TD_TMPL_REDUCE

                    curr += f" {ARROW_UP}"

                    td_tmpl_curr = TD_TMPL_INCREASE

                elif float(curr) < float(prev):

                    curr += f" {ARROW_DOWN}"

                    td_tmpl_curr = TD_TMPL_REDUCE

            except ValueError:

                # if we land here, one of the cells is not a number, this indicates a crash

                # or some error in the scripts, so mark with red as well

            td_tmpl = TD_TMPL_INCREASE

                td_tmpl_curr = TD_TMPL_INCREASE

                td_tmpl_prev = TD_TMPL_INCREASE



        tr.append(td_tmpl.format(row[f"{c}-{id_previous}"]))

        tr.append(td_tmpl.format(row[f"{c}-{id_current}"]))

            tr.append(td_tmpl_prev.format(prev))

            tr.append(td_tmpl_curr.format(curr))



    return TR_TMPL.format("\n".join(tr))
@@ -165,17 +211,38 @@ def merge_and_cleanup_mld_reports( 
    mld_col_curr = f"MLD-{id_current}"

    mld_col_prev = f"MLD-{id_previous}"



    # sort based on difference in MLD between current and previous run

    # put cases with "None" at the top of the list

    def sort_func(x):

        vals_missing = ["None", ""]



        if x[mld_col_curr] in vals_missing or x[mld_col_prev] in vals_missing:

    def sort_func(x, other_col_pairs):

        """

        Sort function for the rows. Puts missing or invalid values on top as those usually

        indicate crashes. Then sorts by MLD difference in descending order. MLD diffs of zero

        are uninteresting and are put last.

        """

        try:

            float(x[mld_col_curr])

            float(x[mld_col_prev])

        except ValueError:

            # Value is no valid floating point value

            return float("inf")



        return float(x[mld_col_curr]) - float(x[mld_col_prev])

        diff = float(x[mld_col_curr]) - float(x[mld_col_prev])



        # if no diff in mld col found, check if there is a diff in any other measure

        if diff == 0:

            diff = float("-inf")



            diff_other = 0

            for col_pair in other_col_pairs:

                col_prev = col_pair[0]

                col_curr = col_pair[1]

                diff_other += abs(float(x[col_curr]) - float(x[col_prev]))



            if diff_other > 0:

                diff = -1000000



        return diff



    merged = sorted(merged, key=sort_func, reverse=True)

    other_col_pairs = [(f"{col}-{id_previous}", f"{col}-{id_current}") for col in COLUMNS_DIFFERENTIAL_NOT_MLD]

    merged = sorted(merged, key=partial(sort_func, other_col_pairs=other_col_pairs), reverse=True)



    # remove the unecessary whole path from the testcase names

    for row in merged:

ci/setup_pages.py

+2 −0
Original line number Diff line number Diff line
@@ -23,6 +23,8 @@ JOBS_FLOAT_REPO = [ 
]

JOBS_BASOP_REPO = [

    "ivas-pytest-mld-long-dec",

    "ivas-pytest-mld-long-dec-lev+10",

    "ivas-pytest-mld-long-dec-lev-10",

]



JOBS_FOR_PROJECT_ID = {