Add output of scripts to reproduce critical cases

#!/usr/bin/env python3

import argparse

from pathlib import Path

import pandas as pd

REPRODUCE_REGRESSION_SCRIPT_TMPL = """

#!/bin/bash -x

SCRIPTS_DIR=/usr/local/scripts

LTV_DIR=/usr/local/ltv

MIN_DATE={min_date}

MIN_SHA={min_sha}

LEVEL_SCALING={level_scaling}

TESTCASE="{testcase}"

REF_ENC1={REF_ENC1}

REF_DEC1={REF_DEC1}

DUT_ENC1={DUT_ENC1}

DUT_DEC1={DUT_DEC1}

REF_ENC2={REF_ENC2}

REF_DEC2={REF_DEC2}

DUT_ENC2={DUT_ENC2}

DUT_DEC2={DUT_DEC2}

# Obtain executables from past reference

git checkout 'ivas-float-update@{$MIN_DATE 22:00:00}'

make clean

make -j

mv IVAS_cod IVAS_cod_ref_1

mv IVAS_dec IVAS_dec_ref_1

mv IVAS_rend IVAS_rend_ref_1

git checkout $MIN_SHA

make clean

make -j

mv IVAS_cod IVAS_cod_1

mv IVAS_dec IVAS_dec_1

mv IVAS_rend IVAS_rend_1

# Obtain latest executables

git checkout ivas-float-update

git pull

make clean

make -j

mv IVAS_cod IVAS_cod_ref_2

mv IVAS_dec IVAS_dec_ref_2

mv IVAS_rend IVAS_rend_ref_2

git checkout main

git pull

make clean

make -j

mv IVAS_cod IVAS_cod_2

mv IVAS_dec IVAS_dec_2

mv IVAS_rend IVAS_rend_2

# Get fresh copy of scripts, tests and ci

cp -r $SCRIPTS_DIR/{scripts,tests,ci,pytest.ini} .

python3 ci/remove_unsupported_testcases.py scripts/config/self_test.prm scripts/config/self_test_ltv.prm # Should not be needed since only supported testcases should be input

# Get LTVs

cp $LTV_DIR/* scripts/testv

# Apply level scaling

tests/scale_pcm.py ./scripts/testv/ "$LEVEL_SCALING"

# Run tests

cp IVAS_rend_ref_1 IVAS_rend_ref

cp IVAS_rend_1 IVAS_rend

python3 -m pytest "$TESTCASE" -n 1 --update_ref 1 --create_ref --param_file scripts/config/self_test_ltv.prm --ref_encoder_path $REF_ENC1 --ref_decoder_path $REF_DEC1 

python3 -m pytest "$TESTCASE" -n 1                --create_cut --param_file scripts/config/self_test_ltv.prm --dut_encoder_path $DUT_ENC1 --dut_decoder_path $DUT_DEC1 --mld --ssnr --odg

# Store results from first run

mkdir tests1

cp -r tests/ref tests/dut tests1

cp -r tests/renderer/ref tests/renderer/cut tests1

cp IVAS_rend_ref_2 IVAS_rend_ref

cp IVAS_rend_2 IVAS_rend

python3 -m pytest "$TESTCASE" -n 1 --update_ref 1 --create_ref --param_file scripts/config/self_test_ltv.prm --ref_encoder_path $REF_ENC2 --ref_decoder_path $REF_DEC2

python3 -m pytest "$TESTCASE" -n 1                --create_cut --param_file scripts/config/self_test_ltv.prm --dut_encoder_path $DUT_ENC2 --dut_decoder_path $DUT_DEC2 --mld --ssnr --odg

"""

def main(logs_dir, output_filename, measure):

    input_path = Path(logs_dir)

    logs = [f for f in input_path.iterdir() if f.is_dir()]

    # Build dict of scores

    formatdict = {}

    sha = {}

    logdict = {}

    for log in logs:

        date = log.name

        logdict[date] = {}

        formatdict[date] = {}

        for logfile in log.glob("*.csv"):

            tmp = logfile.name.split("-")

            job = "-".join(tmp[3:-4])

            sha[date] = tmp[-1].split(".")[0]

            data = pd.read_csv(logfile, usecols=["testcase", measure, "format"])

            logdict[date][job] = {}

            formatdict[date][job] = {}

            for testcase, value, format in zip(

                data["testcase"], data[measure], data["format"]

            ):

                formatdict[date][job][testcase] = format

                logdict[date][job][testcase] = value

    # Restructure dict

    csv_rows = []

    formats = []

    for date, jobs in logdict.items():

        for job, testcases in jobs.items():

            for testcase, value in testcases.items():

                csv_rows.append((job, testcase, date, value))

                formats.append((job, testcase, date, formatdict[date][job][testcase]))

    result = pd.DataFrame(csv_rows, columns=["job", "testcase", "date", "value"])

    result = result.pivot(

        index=["job", "testcase"], columns="date", values="value"

    ).reset_index()

    f = pd.DataFrame(formats, columns=["job", "testcase", "date", "format"])

    f = f.pivot(

        index=["job", "testcase"], columns="date", values="format"

    ).reset_index()

    values = result.iloc[:, 2:]

    last_date = values.columns[-1]

    result.insert(2, "format", f[last_date])

    result.insert(3, "min_date", values.idxmin(axis=1))

    result.insert(4, "min_sha", result["min_date"].map(sha))

    result.insert(5, "curr_value", values[last_date])

    result.insert(6, "min_value", values.min(axis=1))

    result.insert(7, "diff", result["curr_value"] - result["min_value"])

    result.insert(8, "ratio", result["curr_value"] / result["min_value"])

    result.loc[result["min_value"] == 0, "ratio"] = (

        1  # Set ratio to 1 for denominator 0

    )

    result["min_sha"] = "'" + result["min_sha"]

    result.to_csv(output_filename, sep=";", index=False)

    critical = result.iloc[:, 0:9]

    formats = list(set(critical["format"]))

    critical3 = pd.DataFrame()

    for format in formats:

        top3 = (

            critical[critical["format"] == format]

            .sort_values(by="ratio", ascending=False)

            .head(3)

        )

        critical3 = pd.concat([critical3, top3], ignore_index=True)

    critical3.to_csv("critical3.csv", sep=";", index=False)

    row_counter = 1

    for row in critical3.row():

        # Find level

        level_scaling = 1.0

        if "lev+10" in row["job"]:

            level_scaling = 3.162

        if "lev-10" in row["job"]:

            level_scaling = 0.3162

        # Find executables setup

        REF_ENC1 = "IVAS_cod_ref_1"

        REF_DEC1 = "IVAS_dec_ref_1"

        DUT_ENC1 = "IVAS_ref_1"

        DUT_DEC1 = "IVAS_dec_1"

        REF_ENC2 = "IVAS_cod_ref_2"

        REF_DEC2 = "IVAS_dec_ref_2"

        DUT_ENC2 = "IVAS_ref_2"

        DUT_DEC2 = "IVAS_dec_2"

        if "dec" in row["job"]:

            DUT_ENC1 = "IVAS_cod_ref_1"

            DUT_ENC2 = "IVAS_cod_ref_2"

        if "enc" in row["job"]:

            DUT_DEC1 = "IVAS_dec_ref_1"

            DUT_DEC2 = "IVAS_dec_ref_2"

        script_content = REPRODUCE_REGRESSION_SCRIPT_TMPL.format(

            min_date=row["min_date"],

            min_sha=row["min_sha"][1:],

            LEVEL_SCALING={level_scaling},

            TESTCASE=row["testcase"][1:],

            REF_ENC1 = REF_ENC1, 

            REF_DEC1 = REF_DEC1, 

            DUT_ENC1 = DUT_ENC1, 

            DUT_DEC1 = DUT_DEC1,

            REF_ENC2 = REF_ENC2,

            REF_DEC2 = REF_DEC2,

            DUT_ENC2 = DUT_ENC2,

            DUT_DEC2 = DUT_DEC2,

        )

        script_filename = f"regression_{row_counter:02d}.bash"

        with open(script_filename, "w") as f:

            f.write(script_content)

        row_counter = row_counter + 1

if __name__ == "__main__":

    parser = argparse.ArgumentParser(description="logs dir")

    parser.add_argument(

        "logs_dir",

        type=str,

        help="Logs dir, e.g. logs",

    )

    parser.add_argument(

        "output_filename",

        type=str,

        help="Filename of the combined csv file. e.g mld.csv",

    )

    parser.add_argument(

        "--measure",

        type=str,

        help="Measure for summary, one of MLD MIN_SSNR MAX_ABS_DIFF MIN_ODG, (default: MLD)",

        default="MLD",

    )

    args = parser.parse_args()

    main(args.logs_dir, args.output_filename, args.measure)

#!/usr/bin/env python3

import argparse

from pathlib import Path

import pandas as pd

def main(logs_dir, output_filename, measure):

    input_path = Path(logs_dir)

    logs = [f for f in input_path.iterdir() if f.is_dir()]

    # Build dict of scores

    logdict = {}

    for log in logs:

        date = log.name

        logdict[date] = {}

        for logfile in log.glob('*.csv'):

            tmp = logfile.name.split('-')

            job = '-'.join(tmp[3:-4])

            #sha = tmp[-1].split('.')[0]

            #logdict[date]["sha"] = sha # Maybe we want to store the SHA too somehow.

            data = pd.read_csv(logfile, usecols=["testcase", measure])

            logdict[date][job] = {}

            for testcase, value in zip(data["testcase"], data[measure]):

                logdict[date][job][testcase] = value

    # Restructure dict

    csv_rows = []

    for date, jobs in logdict.items():

        for job, testcases in jobs.items():

            for testcase, value in testcases.items():

                csv_rows.append((job, testcase, date, value))

    result = pd.DataFrame(csv_rows, columns=["job","testcase","date","value"])

    result = result.pivot(index=['job', 'testcase'], columns='date', values="value").reset_index()

    result.to_csv(output_filename, sep=';', index=False)

if __name__ == "__main__":

    parser = argparse.ArgumentParser(description="logs dir")

    parser.add_argument(

        "logs_dir",

        type=str,

        help="Logs dir, e.g. logs",

    )

    parser.add_argument(

        "output_filename",

        type=str,

        help="Filename of the combined csv file. e.g mld.csv",

    )

    parser.add_argument(

        "--measure",

        type=str,

        help="Measure for summary, one of MLD MIN_SSNR MAX_ABS_DIFF MIN_ODG, (default: MLD)",

        default="MLD",

    )

    args = parser.parse_args()

    main(args.logs_dir, args.output_filename, args.measure)

 No newline at end of file