Commit df01165b authored by norvell's avatar norvell
Browse files

Split regressions script in two. Add thresholds to find_regressions_from_logs.py

parent 2cf3845f

scripts/find_regressions_from_logs.py

+48 −154
Original line number Diff line number Diff line
@@ -4,99 +4,7 @@ import argparse 
from pathlib import Path

import pandas as pd



REPRODUCE_REGRESSION_SCRIPT_TMPL = """

#!/bin/bash -x



SCRIPTS_DIR=/usr/local/scripts

LTV_DIR=/usr/local/ltv



MIN_DATE={min_date}

MIN_SHA={min_sha}

LEVEL_SCALING={level_scaling}

TESTCASE="{testcase}"



REF_ENC1={REF_ENC1}

REF_DEC1={REF_DEC1}

DUT_ENC1={DUT_ENC1}

DUT_DEC1={DUT_DEC1}



REF_ENC2={REF_ENC2}

REF_DEC2={REF_DEC2}

DUT_ENC2={DUT_ENC2}

DUT_DEC2={DUT_DEC2}



INV_LEVEL_SCALING=$(awk "BEGIN {{print 1.0 / $LEVEL_SCALING}}")



# Obtain executables from past reference

git checkout `git rev-list -1 --before="$MIN_DATE 22:00:00" ivas-float-update`

echo "ivas_float_update, min version: `git rev-parse HEAD`" > versions.txt

make clean

make -j

mv IVAS_cod IVAS_cod_ref_1

mv IVAS_dec IVAS_dec_ref_1

mv IVAS_rend IVAS_rend_ref_1



git checkout $MIN_SHA

echo "main, min version: `git rev-parse HEAD`" >> versions.txt

make clean

make -j

mv IVAS_cod IVAS_cod_1

mv IVAS_dec IVAS_dec_1

mv IVAS_rend IVAS_rend_1



# Obtain latest executables

git checkout ivas-float-update

git pull

echo "ivas-float-update, current version: `git rev-parse HEAD`" >> versions.txt

make clean

make -j

mv IVAS_cod IVAS_cod_ref_2

mv IVAS_dec IVAS_dec_ref_2

mv IVAS_rend IVAS_rend_ref_2



git checkout main

git pull

echo "main, current version: `git rev-parse HEAD`" >> versions.txt

make clean

make -j

mv IVAS_cod IVAS_cod_2

mv IVAS_dec IVAS_dec_2

mv IVAS_rend IVAS_rend_2



# Get fresh copy of scripts, tests and ci

cp -r $SCRIPTS_DIR/{{scripts,tests,ci,pytest.ini}} .

rm -rf tests/ref tests/dut tests/renderer/ref tests/renderer/cut

python3 ci/remove_unsupported_testcases.py scripts/config/self_test.prm scripts/config/self_test_ltv.prm



# Get LTVs

cp $LTV_DIR/* scripts/testv



# Apply level scaling

tests/scale_pcm.py ./scripts/testv/ "$LEVEL_SCALING"



# Run tests

cp IVAS_rend_ref_1 IVAS_rend_ref

cp IVAS_rend_1 IVAS_rend

python3 -m pytest "$TESTCASE" -n 1 --update_ref 1 --create_ref --param_file scripts/config/self_test_ltv.prm --use_ltv --ref_encoder_path $REF_ENC1 --ref_decoder_path $REF_DEC1 

python3 -m pytest "$TESTCASE" -n 1                --create_cut --param_file scripts/config/self_test_ltv.prm --use_ltv --dut_encoder_path $DUT_ENC1 --dut_decoder_path $DUT_DEC1 --mld --ssnr --odg --scalefac $INV_LEVEL_SCALING --junit-xml=report1.xml --html=report1.html --self-contained-html

python3 scripts/parse_xml_report.py report1.xml report1.csv



# Store results from first run

mkdir -p tests1/renderer

cp -r tests/ref tests/dut tests1

cp -r tests/renderer/ref tests1/renderer

cp -r tests/renderer/cut tests1/renderer



cp IVAS_rend_ref_2 IVAS_rend_ref

cp IVAS_rend_2 IVAS_rend

python3 -m pytest "$TESTCASE" -n 1 --update_ref 1 --create_ref --param_file scripts/config/self_test_ltv.prm --use_ltv --ref_encoder_path $REF_ENC2 --ref_decoder_path $REF_DEC2

python3 -m pytest "$TESTCASE" -n 1                --create_cut --param_file scripts/config/self_test_ltv.prm --use_ltv --dut_encoder_path $DUT_ENC2 --dut_decoder_path $DUT_DEC2 --mld --ssnr --odg --scalefac $INV_LEVEL_SCALING --junit-xml=report2.xml --html=report2.html --self-contained-html

python3 scripts/parse_xml_report.py report2.xml report2.csv



"""





def main(logs_dir, output_filename, measure):

def main(logs_dir, output_filename, measure, days, all_results, diff_thr, ratio_thr, curr_value_thr):



    input_path = Path(logs_dir)

    logs = [f for f in input_path.iterdir() if f.is_dir()]
@@ -142,6 +50,14 @@ def main(logs_dir, output_filename, measure): 
        index=["job", "testcase"], columns="date", values="format"

    ).reset_index()



    if days != -1:

        len_history = len(result.columns) - 2

        if len_history < days:

            print(f"*** Warning: Requested number of days exceeds the number of days in the logs. Will use all {len_history} available days.")

        if len_history > days:

            remove_days = len_history - days

            result = result.drop(result.columns[2:2+remove_days], axis=1)



    values = result.iloc[:, 2:]

    last_date = values.columns[-1]
@@ -156,69 +72,16 @@ def main(logs_dir, output_filename, measure): 
        1  # Set ratio to 1 for denominator 0

    )



    result["min_sha"] = "'" + result["min_sha"]



    result.to_csv(output_filename, sep=";", index=False)

    if not all_results:

        # Filter out test cases above thresholds

        result = result[result["diff"] > diff_thr]

        result = result[result["ratio"] > ratio_thr]

        result = result[result["curr_value"] > curr_value_thr]



    critical = result.iloc[:, 0:9]

    formats = list(set(critical["format"]))

    formats.sort()

    critical3 = pd.DataFrame()

    result["min_sha"] = "'" + result["min_sha"] # Add apostrophy to prevent Excel reading this as a number



    for format in formats:

        top3 = (

            critical[critical["format"] == format]

            .sort_values(by="ratio", ascending=False)

            .head(3)

        )

        critical3 = pd.concat([critical3, top3], ignore_index=True)



    critical3.to_csv("critical3.csv", sep=";", index=False)



    for row_counter, row in critical3.iterrows():



        # Find level

        level_scaling = 1.0

        if "lev+10" in row["job"]:

            level_scaling = 3.162

        if "lev-10" in row["job"]:

            level_scaling = 0.3162



        # Find executables setup

        REF_ENC1 = "IVAS_cod_ref_1"

        REF_DEC1 = "IVAS_dec_ref_1"

        DUT_ENC1 = "IVAS_cod_1"

        DUT_DEC1 = "IVAS_dec_1"

        REF_ENC2 = "IVAS_cod_ref_2"

        REF_DEC2 = "IVAS_dec_ref_2"

        DUT_ENC2 = "IVAS_cod_2"

        DUT_DEC2 = "IVAS_dec_2"



        if "dec" in row["job"]:

            DUT_ENC1 = "IVAS_cod_ref_1"

            DUT_ENC2 = "IVAS_cod_ref_2"

        if "enc" in row["job"]:

            DUT_DEC1 = "IVAS_dec_ref_1"

            DUT_DEC2 = "IVAS_dec_ref_2"



        script_content = REPRODUCE_REGRESSION_SCRIPT_TMPL.format(

            min_date=row["min_date"],

            min_sha=row["min_sha"][1:],

            level_scaling=level_scaling,

            testcase=row["testcase"],

            REF_ENC1=REF_ENC1,

            REF_DEC1=REF_DEC1,

            DUT_ENC1=DUT_ENC1,

            DUT_DEC1=DUT_DEC1,

            REF_ENC2=REF_ENC2,

            REF_DEC2=REF_DEC2,

            DUT_ENC2=DUT_ENC2,

            DUT_DEC2=DUT_DEC2,

        )

    result.to_csv(output_filename, sep=";", index=False)



        script_filename = f"regression_{row_counter+2:03d}.bash"

        with open(script_filename, "w") as f:

            f.write(script_content)





if __name__ == "__main__":
@@ -239,7 +102,38 @@ if __name__ == "__main__": 
        help="Measure for summary, one of MLD MIN_SSNR MAX_ABS_DIFF MIN_ODG, (default: MLD)",

        default="MLD",

    )

    parser.add_argument(

        "--days",

        type=int,

        help="Number of days in history, (default: whole history)",

        default=-1,

    )

    parser.add_argument(

        "--all_results", 

        action="store_true",

        help="Output all results, including cases without regression (default: off)",

        default=False,        

    )

    parser.add_argument(

        "--diff_thr",

        type=float,

        help="Include test cases with diff above diff_thr, (default: 0.0)",

        default=0.0,

    )

    parser.add_argument(

        "--ratio_thr",

        type=float,

        help="Include test cases with ratio above ratio_thr, (default: 1.0)",

        default=1.0,

    )

    parser.add_argument(

        "--curr_value_thr",

        type=float,

        help="Include test cases with curr_value above curr_value_thr, (default: 0.0)",

        default=0.0,

    )



    args = parser.parse_args()



    main(args.logs_dir, args.output_filename, args.measure)
 
    main(args.logs_dir, args.output_filename, args.measure, args.days, args.all_results, args.diff_thr, args.ratio_thr, args.curr_value_thr)

scripts/generate_scripts_from_regressions.py

0 → 100644
+155 −0
Original line number Diff line number Diff line 
#!/usr/bin/env python3



import argparse

import pandas as pd



REPRODUCE_REGRESSION_SCRIPT_TMPL = """

#!/bin/bash -x



SCRIPTS_DIR=/usr/local/scripts

LTV_DIR=/usr/local/ltv



MIN_DATE={min_date}

MIN_SHA={min_sha}

LEVEL_SCALING={level_scaling}

TESTCASE="{testcase}"



REF_ENC1={REF_ENC1}

REF_DEC1={REF_DEC1}

DUT_ENC1={DUT_ENC1}

DUT_DEC1={DUT_DEC1}



REF_ENC2={REF_ENC2}

REF_DEC2={REF_DEC2}

DUT_ENC2={DUT_ENC2}

DUT_DEC2={DUT_DEC2}



INV_LEVEL_SCALING=$(awk "BEGIN {{print 1.0 / $LEVEL_SCALING}}")



# Obtain executables from past reference

git checkout `git rev-list -1 --before="$MIN_DATE 22:00:00" ivas-float-update`

echo "ivas_float_update, min version: `git rev-parse HEAD`" > versions.txt

make clean

make -j

mv IVAS_cod IVAS_cod_ref_1

mv IVAS_dec IVAS_dec_ref_1

mv IVAS_rend IVAS_rend_ref_1



git checkout $MIN_SHA

echo "main, min version: `git rev-parse HEAD`" >> versions.txt

make clean

make -j

mv IVAS_cod IVAS_cod_1

mv IVAS_dec IVAS_dec_1

mv IVAS_rend IVAS_rend_1



# Obtain latest executables

git checkout ivas-float-update

git pull

echo "ivas-float-update, current version: `git rev-parse HEAD`" >> versions.txt

make clean

make -j

mv IVAS_cod IVAS_cod_ref_2

mv IVAS_dec IVAS_dec_ref_2

mv IVAS_rend IVAS_rend_ref_2



git checkout main

git pull

echo "main, current version: `git rev-parse HEAD`" >> versions.txt

make clean

make -j

mv IVAS_cod IVAS_cod_2

mv IVAS_dec IVAS_dec_2

mv IVAS_rend IVAS_rend_2



# Get fresh copy of scripts, tests and ci

cp -r $SCRIPTS_DIR/{{scripts,tests,ci,pytest.ini}} .

rm -rf tests/ref tests/dut tests/renderer/ref tests/renderer/cut

python3 ci/remove_unsupported_testcases.py scripts/config/self_test.prm scripts/config/self_test_ltv.prm



# Get LTVs

cp $LTV_DIR/* scripts/testv



# Apply level scaling

tests/scale_pcm.py ./scripts/testv/ "$LEVEL_SCALING"



# Run tests

cp IVAS_rend_ref_1 IVAS_rend_ref

cp IVAS_rend_1 IVAS_rend

python3 -m pytest "$TESTCASE" -n 1 --update_ref 1 --create_ref --param_file scripts/config/self_test_ltv.prm --use_ltv --ref_encoder_path $REF_ENC1 --ref_decoder_path $REF_DEC1 

python3 -m pytest "$TESTCASE" -n 1                --create_cut --param_file scripts/config/self_test_ltv.prm --use_ltv --dut_encoder_path $DUT_ENC1 --dut_decoder_path $DUT_DEC1 --mld --ssnr --odg --scalefac $INV_LEVEL_SCALING --junit-xml=report1.xml --html=report1.html --self-contained-html

python3 scripts/parse_xml_report.py report1.xml report1.csv



# Store results from first run

mkdir -p tests1/renderer

cp -r tests/ref tests/dut tests1

cp -r tests/renderer/ref tests1/renderer

cp -r tests/renderer/cut tests1/renderer



cp IVAS_rend_ref_2 IVAS_rend_ref

cp IVAS_rend_2 IVAS_rend

python3 -m pytest "$TESTCASE" -n 1 --update_ref 1 --create_ref --param_file scripts/config/self_test_ltv.prm --use_ltv --ref_encoder_path $REF_ENC2 --ref_decoder_path $REF_DEC2

python3 -m pytest "$TESTCASE" -n 1                --create_cut --param_file scripts/config/self_test_ltv.prm --use_ltv --dut_encoder_path $DUT_ENC2 --dut_decoder_path $DUT_DEC2 --mld --ssnr --odg --scalefac $INV_LEVEL_SCALING --junit-xml=report2.xml --html=report2.html --self-contained-html

python3 scripts/parse_xml_report.py report2.xml report2.csv



"""





def main(critical_list):



    critical = pd.read_csv(critical_list, sep=";")

                       

    for row_counter, row in critical.iterrows():



        # Find level

        level_scaling = 1.0

        if "lev+10" in row["job"]:

            level_scaling = 3.162

        if "lev-10" in row["job"]:

            level_scaling = 0.3162



        # Find executables setup

        REF_ENC1 = "IVAS_cod_ref_1"

        REF_DEC1 = "IVAS_dec_ref_1"

        DUT_ENC1 = "IVAS_cod_1"

        DUT_DEC1 = "IVAS_dec_1"

        REF_ENC2 = "IVAS_cod_ref_2"

        REF_DEC2 = "IVAS_dec_ref_2"

        DUT_ENC2 = "IVAS_cod_2"

        DUT_DEC2 = "IVAS_dec_2"



        if "dec" in row["job"]:

            DUT_ENC1 = "IVAS_cod_ref_1"

            DUT_ENC2 = "IVAS_cod_ref_2"

        if "enc" in row["job"]:

            DUT_DEC1 = "IVAS_dec_ref_1"

            DUT_DEC2 = "IVAS_dec_ref_2"



        script_content = REPRODUCE_REGRESSION_SCRIPT_TMPL.format(

            min_date=row["min_date"],

            min_sha=row["min_sha"][1:],

            level_scaling=level_scaling,

            testcase=row["testcase"],

            REF_ENC1=REF_ENC1,

            REF_DEC1=REF_DEC1,

            DUT_ENC1=DUT_ENC1,

            DUT_DEC1=DUT_DEC1,

            REF_ENC2=REF_ENC2,

            REF_DEC2=REF_DEC2,

            DUT_ENC2=DUT_ENC2,

            DUT_DEC2=DUT_DEC2,

        )



        script_filename = f"regression_{row_counter+2:03d}.bash"

        with open(script_filename, "w") as f:

            f.write(script_content)



if __name__ == "__main__":

    parser = argparse.ArgumentParser(description="Generate bash scripts to reproduce test cases from critical testcase csv list")

    parser.add_argument(

        "critical_list",

        type=str,

        help="critical file list, e.g. critical.csv",

    )

    args = parser.parse_args()

    main(args.critical_list)