Commit bd10823c authored by malenov's avatar malenov
Browse files

update of /scripts and inclusion of /ci

parent 926433d7

ci/__init__.py

0 → 100644
+0 −0

Empty file added.

ci/basop-pages/basop_index.html

0 → 100644
+23 −0
Original line number Diff line number Diff line 
<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.01 Transitional//EN" "http://www.w3.org/TR/html4/loose.dtd">

<html>

  <head>

  </head>

<body>

  <h1>Ivas BASOP code Development</h1>



  <h2>Daily long testvector tests</h2>



  <ul>

    <li><a href="ivas-pytest-compare_ref-long-dec-index.html">ivas-pytest-compare_ref-long-dec</a></li>

    <li><a href="ivas-pytest-compare_ref-long-dec-lev+10-index.html">ivas-pytest-compare_ref-long-dec-lev+10</a></li>

    <li><a href="ivas-pytest-compare_ref-long-dec-lev-10-index.html">ivas-pytest-compare_ref-long-dec-lev-10</a></li>

    <li><a href="ivas-pytest-compare_ref-long-enc-index.html">ivas-pytest-compare_ref-long-enc</a></li>

    <li><a href="ivas-pytest-compare_ref-long-enc-lev+10-index.html">ivas-pytest-compare_ref-long-enc-lev+10</a></li>

    <li><a href="ivas-pytest-compare_ref-long-enc-lev-10-index.html">ivas-pytest-compare_ref-long-enc-lev-10</a></li>

  </ul>



  <h3>Complexity Reports</h3>



  {}



</body>

ci/basop-pages/create_report_pages.py

0 → 100644
+340 −0
Original line number Diff line number Diff line 
import csv

import pathlib

import argparse

from functools import partial



FORMATS = ["Stereo", "ISM", "Multichannel", "MASA", "SBA", "OSBA", "OMASA", "Renderer"]



CSV_DELIM = ";"

SUBPAGE_TMPL_CSS = """

<style type="text/css">

.tbase  {border-collapse:collapse;border-spacing:0;}

.tbase td{border-color:black;border-style:solid;border-width:1px;font-family:sans-serif;font-size:14px;

  overflow:hidden;padding:10px 5px;word-break:normal;}

.tbase th{border-color:black;border-style:solid;border-width:1px;font-family:sans-serif;font-size:14px;

  font-weight:normal;overflow:hidden;padding:10px 5px;word-break:normal;}

.tbase .tunder{font-weight:bold;text-align:center;text-decoration:underline;}

.tbase .tcenter{font-weight:bold;text-align:center;}

.tbase .tleft{text-align:left;horizontal-align:bottom}

.tbase .tincrease{text-align:left;background-color:#ff5500;border-color:inherit;font-weight:bold;}

.tbase .treduce{text-align:left;background-color:#acff00;border-color:inherit;font-weight:bold;}



.arrowup {font-weight:bold;font-size:200%;}

.arrowdown {font-weight:bold;font-size:200%;}

</style>

"""



SUBPAGE_TMPL_HTML = """



<h1>Report for job {job_name}</h1>



Comparing:

<ul>

    <li>Current run - id: <a href="https://forge.3gpp.org/rep/sa4/audio/ivas-basop/-/jobs/{id_current}">{id_current}</a></li>

    <li>Previous run - id: <a href="https://forge.3gpp.org/rep/sa4/audio/ivas-basop/-/jobs/{id_previous}">{id_previous}</a></li>

    <li><a href="{job_name}--merged_csv--{id_current}.csv">Merged csv data</a></li>

</ul>



<br>

<h2><a href="summary_{job_name}.html">Summary page</a></h2>

<br>

<br>



<b>How is the table sorted?</b>

<ul>

    <li>Cases with result ERROR or invalid/missing values for the numerical measures are given first</li>

    <li>Next are all cases with MLD(current) != MLD(previous), sorted in descending order (biggest MLD increase comes first)</li>

    <li>All cases with no difference in MLD are at the bottom</li>

</ul>

<br>

<b>What do the colours indicate</b>

<ul>

    <li>

        <span style="background-color:#ff5500;">Red background</span>:

        <ul>

            <li>This testcases either triggered an ERROR in the pytest run (probably a crash in the codec) or did not give valid values for the numerical measures (probably an error in the scripts).</li>

            <li>For this testcase the MLD increased wrt the previous run - MLD(current) > MLD(previous)</li>

        </ul>

    </li>

    <li>

        <span style="background-color:#acff00;">Green background</span>:

        <ul>

            <li>For this testcase the MLD decreased wrt the previous run - MLD(current) < MLD(previous)</li>

        </ul>

    </li>

</ul>

<br>

<b>How to interpret the Result column?</b>

<ul>

    <li>ERROR: An error occured during test run. It should be checked if a crash in the codec occured.</li>

    <li>FAIL: An MLD value > 0 was reported. This is to be expected, the test just reports a failure due to how the tests are currently implemented.</li>

    <li>PASS: MLD value of 0 was reported. This should only be the case for some special operating points and could indicate that parts of the codec are not (fully) converted yet.</li>

</ul>

<b>



<table class="tbase"><thead>

  <tr>

    {table_header_a}

  </tr>

  <tr>

    {table_header_b}

  </tr></thead>

<tbody>

{table_body}

</tbody>

</table>

"""

TD_TMPL_NORMAL = "<td class='tleft'>{}</td>"

TD_TMPL_INCREASE = "<td class='tincrease'>{}</td>"

TD_TMPL_REDUCE = "<td class='treduce'>{}</td>"

TR_TMPL = "<tr>{}</tr>"

TH_TMPL_GLOBAL = '<th class="tunder" rowspan="2">{}</th>'

TH_TMPL_DIFFERENTIAL = '<th class="tunder" colspan="2">{}</th>'

TH_TMPL_SECOND_ROW = '<th class="tcenter">{}</th>'



ARROW_UP = '<span class="arrowup">&#11008;</span>'

ARROW_DOWN = '<span class="arrowdown">&#11010;</span>'



# expected columns. actual columns are filtered from the incoming data later, this

# is mainly for controlling the order in the output table

COLUMNS = [

    "testcase",

    "Format",

    "Category",

    "Result",

    "MLD",

    "MAXIMUM ABS DIFF",

    "MIN_SSNR",

    "MIN_ODG",

]

COLUMNS_GLOBAL = COLUMNS[:1]

COLUMNS_DIFFERENTIAL = COLUMNS[3:]

COLUMNS_DIFFERENTIAL_NOT_MLD = COLUMNS_DIFFERENTIAL[2:]





def create_subpage(

    html_out,

    csv_out,

    csv_current: str,

    csv_previous: str,

    id_current: int,

    id_previous: int,

    job_name: str,

):

    merged_reports = merge_and_cleanup_mld_reports(

        csv_current, csv_previous, id_current, id_previous

    )

    write_out_csv(merged_reports, merged_reports[0].keys(), csv_out)



    table_header_a = "".join(

        [TH_TMPL_GLOBAL.format(c) for c in COLUMNS_GLOBAL]

        + [TH_TMPL_DIFFERENTIAL.format(c) for c in COLUMNS_DIFFERENTIAL]

    )

    table_header_b = list()

    for c in COLUMNS_DIFFERENTIAL:

        table_header_b.append(

            TH_TMPL_SECOND_ROW.format(f"Previous Run<br>ID: {id_previous}")

        )

        table_header_b.append(

            TH_TMPL_SECOND_ROW.format(f"Current Run<br>ID: {id_current}")

        )

    table_header_b = "".join(table_header_b)

    table_body = "\n".join(

        tr_from_row(row, id_current, id_previous) for row in merged_reports

    )



    new_subpage = SUBPAGE_TMPL_CSS + SUBPAGE_TMPL_HTML.format(

        id_current=id_current,

        id_previous=id_previous,

        table_body=table_body,

        job_name=job_name,

        table_header_a=table_header_a,

        table_header_b=table_header_b,

    )

    with open(html_out, "w") as f:

        f.write(new_subpage)





def write_out_csv(data, col_names, outfile):

    with open(outfile, "w") as f:

        writer = csv.DictWriter(f, col_names, delimiter=";")

        writer.writeheader()

        for row in data:

            writer.writerow(row)





def tr_from_row(row, id_current, id_previous):

    tr = list()



    # pre-filter columns to handle case where new columns are added

    # only include columns that are there in both data

    columns_global = [c for c in COLUMNS_GLOBAL if c in row]

    diff_col_tmpl = "{}-{}"

    incoming_cols = row.keys()

    columns_differential = [

        c

        for c in COLUMNS_DIFFERENTIAL

        if diff_col_tmpl.format(c, id_current) in incoming_cols

        and diff_col_tmpl.format(c, id_previous) in incoming_cols

    ]



    for c in columns_global:

        # this is currently for the "testcase" column - here we don't compare, just one value is used

        tr.append(TD_TMPL_NORMAL.format(row[c]))

    for c in columns_differential:

        # this is for all columns where we compare between current and previous run

        prev = row[f"{c}-{id_previous}"]

        curr = row[f"{c}-{id_current}"]



        if c == "Result":

            # print errors in bold red font

            td_tmpl = TD_TMPL_INCREASE if prev == "ERROR" else TD_TMPL_NORMAL

            tr.append(td_tmpl.format(prev))

            td_tmpl = TD_TMPL_INCREASE if curr == "ERROR" else TD_TMPL_NORMAL

            tr.append(td_tmpl.format(curr))

        else:

            td_tmpl_curr = TD_TMPL_NORMAL

            td_tmpl_prev = TD_TMPL_NORMAL

            try:

                if float(curr) > float(prev):

                    curr += f" {ARROW_UP}"

                    # increase is bad -> mark in red, execpt for SNR for which it is good -> mark in green

                    td_tmpl_curr = (

                        TD_TMPL_REDUCE if c == "MIN_SSNR" else TD_TMPL_INCREASE

                    )

                elif float(curr) < float(prev):

                    curr += f" {ARROW_DOWN}"

                    # reduce is good -> mark in green, execpt for SNR for which it is bad -> mark in red

                    td_tmpl_curr = (

                        TD_TMPL_INCREASE if c == "MIN_SSNR" else TD_TMPL_REDUCE

                    )

            except ValueError:

                # if we land here, one of the cells is not a number, this indicates a crash

                # or some error in the scripts, so mark with red as well

                td_tmpl_curr = TD_TMPL_INCREASE

                td_tmpl_prev = TD_TMPL_INCREASE



            tr.append(td_tmpl_prev.format(prev))

            tr.append(td_tmpl_curr.format(curr))



    return TR_TMPL.format("\n".join(tr))





def merge_and_cleanup_mld_reports(

    csv_current: str, csv_previous: str, id_current: int, id_previous: int

):

    with open(csv_current) as f:

        current_reader = csv.DictReader(f, delimiter=CSV_DELIM)

        current = list(current_reader)

    with open(csv_previous) as f:

        previous = list(csv.DictReader(f, delimiter=CSV_DELIM))



    # TODO: handle newly added testcases - for now assume that both have the same columns

    merge_key = "testcase"

    other_keys = [k for k in current_reader.fieldnames if k != merge_key]

    merged = merge_tables(

        current, previous, id_current, id_previous, merge_key, other_keys

    )



    # TODO: sort on result as well

    mld_col_curr = f"MLD-{id_current}"

    mld_col_prev = f"MLD-{id_previous}"



    def sort_func(x, other_col_pairs):

        """

        Sort function for the rows. Puts missing or invalid values on top as those usually

        indicate crashes. Then sorts by MLD difference in descending order. MLD diffs of zero

        are uninteresting and are put last.

        """

        try:

            cols = [mld_col_curr, mld_col_prev] + [p[1] for p in other_col_pairs]

            for c in cols:

                float(x[c])

        except ValueError:

            # Value is no valid floating point value

            return float("inf")



        diff = float(x[mld_col_curr]) - float(x[mld_col_prev])



        # if no diff in mld col found, check if there is a diff in any other measure

        if diff == 0:

            diff = float("-inf")



            diff_other = 0

            for col_pair in other_col_pairs:

                col_prev = col_pair[0]

                col_curr = col_pair[1]



                try:

                    diff_other += abs(float(x[col_curr]) - float(x[col_prev]))

                except ValueError:

                    # can't make float from the column contents, probably NONE -> put to top

                    diff_other += float("inf")



            if diff_other > 0:

                diff = -1000000



        return diff



    other_col_pairs = [

        (f"{col}-{id_previous}", f"{col}-{id_current}")

        for col in COLUMNS_DIFFERENTIAL_NOT_MLD

    ]

    merged = sorted(

        merged, key=partial(sort_func, other_col_pairs=other_col_pairs), reverse=True

    )



    # remove the unecessary whole path from the testcase names

    for row in merged:

        row["testcase"] = pathlib.Path(row["testcase"]).name



    return merged





def merge_tables(tbl1, tbl2, suffix1, suffix2, merge_key, other_keys):

    merged = list()



    for row1 in tbl1:

        new_row = dict()

        for key in other_keys:

            new_row[f"{key}-{suffix1}"] = row1[key]



        found_merge_key_in_both_tbls = False

        for row2 in tbl2:

            if row1[merge_key] == row2[merge_key]:

                new_row[merge_key] = row1[merge_key]

                for key in other_keys:

                    if key in row2:  # In case key is missing, just insert a blank

                        new_row[f"{key}-{suffix2}"] = row2[key]

                    else:

                        new_row[f"{key}-{suffix2}"] = ""



                found_merge_key_in_both_tbls = True

                break



        if found_merge_key_in_both_tbls:

            merged.append(new_row)



    return merged





if __name__ == "__main__":

    parser = argparse.ArgumentParser()

    parser.add_argument("html_out")

    parser.add_argument("csv_out")

    parser.add_argument("csv_current")

    parser.add_argument("csv_previous")

    parser.add_argument("id_current", type=int)

    parser.add_argument("id_previous", type=int)

    parser.add_argument("job_name")

    args = parser.parse_args()



    create_subpage(

        args.html_out,

        args.csv_out,

        args.csv_current,

        args.csv_previous,

        args.id_current,

        args.id_previous,

        args.job_name,

    )

ci/basop-pages/create_summary_page.py

0 → 100644
+55 −0
Original line number Diff line number Diff line 
import argparse



from create_report_pages import SUBPAGE_TMPL_CSS, FORMATS



MEASURES = ["MLD","DIFF","SSNR","ODG"]



SUMMARY_PAGE_TMPL_HTML = """



<h1>Summary for job {job_name}, ID: {id_current}</h1>



{images}



"""



def create_summary_page(

    html_out,

    id_current: int,

    job_name: str,

):

    images = histogram_summary(job_name)



    new_summary_page = SUBPAGE_TMPL_CSS + SUMMARY_PAGE_TMPL_HTML.format(

        id_current=id_current,

        job_name=job_name,

        images=images,

    )

    with open(html_out, "w") as f:

        f.write(new_summary_page)



def histogram_summary(

    job_name:str,

    ):

    images = "<hr>"

    for m in MEASURES:

        images += (

            f"<h2>{m} summary {job_name}</h2>\n"

            + " ".join(

                [f"<img src=images_{job_name}/summary_{m}_{x}.png>" for x in FORMATS]

            )

            + f'\n<br><a href="images_{job_name}/summary_{m}.csv">summary_{m}.csv</a><hr>\n\n'

        )

    return images



if __name__ == "__main__":

    parser = argparse.ArgumentParser()

    parser.add_argument("html_out")

    parser.add_argument("id_current", type=int)

    parser.add_argument("job_name")

    args = parser.parse_args()



    create_summary_page(

        args.html_out,

        args.id_current,

        args.job_name,

    )

ci/build_all_linux.sh

0 → 100644
+42 −0
Original line number Diff line number Diff line 
#! /usr/bin/bash



# (C) 2022-2024 IVAS codec Public Collaboration with portions copyright Dolby International AB, Ericsson AB,

# Fraunhofer-Gesellschaft zur Foerderung der angewandten Forschung e.V., Huawei Technologies Co. LTD.,

# Koninklijke Philips N.V., Nippon Telegraph and Telephone Corporation, Nokia Technologies Oy, Orange,

# Panasonic Holdings Corporation, Qualcomm Technologies, Inc., VoiceAge Corporation, and other

# contributors to this repository. All Rights Reserved.



# This software is protected by copyright law and by international treaties.

# The IVAS codec Public Collaboration consisting of Dolby International AB, Ericsson AB,

# Fraunhofer-Gesellschaft zur Foerderung der angewandten Forschung e.V., Huawei Technologies Co. LTD.,

# Koninklijke Philips N.V., Nippon Telegraph and Telephone Corporation, Nokia Technologies Oy, Orange,

# Panasonic Holdings Corporation, Qualcomm Technologies, Inc., VoiceAge Corporation, and other

# contributors to this repository retain full ownership rights in their respective contributions in

# the software. This notice grants no license of any kind, including but not limited to patent

# license, nor is any license granted by implication, estoppel or otherwise.



# Contributors are required to enter into the IVAS codec Public Collaboration agreement before making

# contributions.



# This software is provided "AS IS", without any express or implied warranties. The software is in the

# development stage. It is intended exclusively for experts who have experience with such software and

# solely for the purpose of inspection. All implied warranties of non-infringement, merchantability

# and fitness for a particular purpose are hereby disclaimed and excluded.



# Any dispute, controversy or claim arising under or in relation to providing this software shall be

# submitted to and settled by the final, binding jurisdiction of the courts of Munich, Germany in

# accordance with the laws of the Federal Republic of Germany excluding its conflict of law rules and

# the United Nations Convention on Contracts on the International Sales of Goods.



if [ ! -d "lib_com" ]; then

    echo "not in root directory! - please run in IVAS root"

    exit 1

fi



# first build codec, everything else needs this anyway

make clean && make all -j

# build unittests

make unittests -j

# build standalone TD object renderer

make -C scripts/td_object_renderer/object_renderer_standalone -j