diff --git a/scripts/find_regressions_from_logs.py b/scripts/find_regressions_from_logs.py new file mode 100644 index 0000000000000000000000000000000000000000..068fe77871fe2b16a3e11d950c183d0de6787dd7 --- /dev/null +++ b/scripts/find_regressions_from_logs.py @@ -0,0 +1,245 @@ +#!/usr/bin/env python3 + +import argparse +from pathlib import Path +import pandas as pd + +REPRODUCE_REGRESSION_SCRIPT_TMPL = """ +#!/bin/bash -x + +SCRIPTS_DIR=/usr/local/scripts +LTV_DIR=/usr/local/ltv + +MIN_DATE={min_date} +MIN_SHA={min_sha} +LEVEL_SCALING={level_scaling} +TESTCASE="{testcase}" + +REF_ENC1={REF_ENC1} +REF_DEC1={REF_DEC1} +DUT_ENC1={DUT_ENC1} +DUT_DEC1={DUT_DEC1} + +REF_ENC2={REF_ENC2} +REF_DEC2={REF_DEC2} +DUT_ENC2={DUT_ENC2} +DUT_DEC2={DUT_DEC2} + +INV_LEVEL_SCALING=$(awk "BEGIN {{print 1.0 / $LEVEL_SCALING}}") + +# Obtain executables from past reference +git checkout `git rev-list -1 --before="$MIN_DATE 22:00:00" ivas-float-update` +echo "ivas_float_update, min version: `git rev-parse HEAD`" > versions.txt +make clean +make -j +mv IVAS_cod IVAS_cod_ref_1 +mv IVAS_dec IVAS_dec_ref_1 +mv IVAS_rend IVAS_rend_ref_1 + +git checkout $MIN_SHA +echo "main, min version: `git rev-parse HEAD`" >> versions.txt +make clean +make -j +mv IVAS_cod IVAS_cod_1 +mv IVAS_dec IVAS_dec_1 +mv IVAS_rend IVAS_rend_1 + +# Obtain latest executables +git checkout ivas-float-update +git pull +echo "ivas-float-update, current version: `git rev-parse HEAD`" >> versions.txt +make clean +make -j +mv IVAS_cod IVAS_cod_ref_2 +mv IVAS_dec IVAS_dec_ref_2 +mv IVAS_rend IVAS_rend_ref_2 + +git checkout main +git pull +echo "main, current version: `git rev-parse HEAD`" >> versions.txt +make clean +make -j +mv IVAS_cod IVAS_cod_2 +mv IVAS_dec IVAS_dec_2 +mv IVAS_rend IVAS_rend_2 + +# Get fresh copy of scripts, tests and ci +cp -r $SCRIPTS_DIR/{{scripts,tests,ci,pytest.ini}} . +rm -rf tests/ref tests/dut tests/renderer/ref tests/renderer/cut +python3 ci/remove_unsupported_testcases.py scripts/config/self_test.prm scripts/config/self_test_ltv.prm + +# Get LTVs +cp $LTV_DIR/* scripts/testv + +# Apply level scaling +tests/scale_pcm.py ./scripts/testv/ "$LEVEL_SCALING" + +# Run tests +cp IVAS_rend_ref_1 IVAS_rend_ref +cp IVAS_rend_1 IVAS_rend +python3 -m pytest "$TESTCASE" -n 1 --update_ref 1 --create_ref --param_file scripts/config/self_test_ltv.prm --use_ltv --ref_encoder_path $REF_ENC1 --ref_decoder_path $REF_DEC1 +python3 -m pytest "$TESTCASE" -n 1 --create_cut --param_file scripts/config/self_test_ltv.prm --use_ltv --dut_encoder_path $DUT_ENC1 --dut_decoder_path $DUT_DEC1 --mld --ssnr --odg --scalefac $INV_LEVEL_SCALING --junit-xml=report1.xml --html=report1.html --self-contained-html +python3 scripts/parse_xml_report.py report1.xml report1.csv + +# Store results from first run +mkdir -p tests1/renderer +cp -r tests/ref tests/dut tests1 +cp -r tests/renderer/ref tests1/renderer +cp -r tests/renderer/cut tests1/renderer + +cp IVAS_rend_ref_2 IVAS_rend_ref +cp IVAS_rend_2 IVAS_rend +python3 -m pytest "$TESTCASE" -n 1 --update_ref 1 --create_ref --param_file scripts/config/self_test_ltv.prm --use_ltv --ref_encoder_path $REF_ENC2 --ref_decoder_path $REF_DEC2 +python3 -m pytest "$TESTCASE" -n 1 --create_cut --param_file scripts/config/self_test_ltv.prm --use_ltv --dut_encoder_path $DUT_ENC2 --dut_decoder_path $DUT_DEC2 --mld --ssnr --odg --scalefac $INV_LEVEL_SCALING --junit-xml=report2.xml --html=report2.html --self-contained-html +python3 scripts/parse_xml_report.py report2.xml report2.csv + +""" + + +def main(logs_dir, output_filename, measure): + + input_path = Path(logs_dir) + logs = [f for f in input_path.iterdir() if f.is_dir()] + + # Build dict of scores + formatdict = {} + sha = {} + logdict = {} + for log in logs: + date = log.name + logdict[date] = {} + formatdict[date] = {} + for logfile in log.glob("*.csv"): + tmp = logfile.name.split("-") + job = "-".join(tmp[3:-4]) + sha[date] = tmp[-1].split(".")[0] + data = pd.read_csv(logfile, usecols=["testcase", measure, "format"]) + logdict[date][job] = {} + formatdict[date][job] = {} + + for testcase, value, format in zip( + data["testcase"], data[measure], data["format"] + ): + formatdict[date][job][testcase] = format + logdict[date][job][testcase] = value + + # Restructure dict + csv_rows = [] + formats = [] + for date, jobs in logdict.items(): + for job, testcases in jobs.items(): + for testcase, value in testcases.items(): + csv_rows.append((job, testcase, date, value)) + formats.append((job, testcase, date, formatdict[date][job][testcase])) + + result = pd.DataFrame(csv_rows, columns=["job", "testcase", "date", "value"]) + result = result.pivot( + index=["job", "testcase"], columns="date", values="value" + ).reset_index() + + f = pd.DataFrame(formats, columns=["job", "testcase", "date", "format"]) + f = f.pivot( + index=["job", "testcase"], columns="date", values="format" + ).reset_index() + + values = result.iloc[:, 2:] + last_date = values.columns[-1] + + result.insert(2, "format", f[last_date]) + result.insert(3, "min_date", values.idxmin(axis=1)) + result.insert(4, "min_sha", result["min_date"].map(sha)) + result.insert(5, "curr_value", values[last_date]) + result.insert(6, "min_value", values.min(axis=1)) + result.insert(7, "diff", result["curr_value"] - result["min_value"]) + result.insert(8, "ratio", result["curr_value"] / result["min_value"]) + result.loc[result["min_value"] == 0, "ratio"] = ( + 1 # Set ratio to 1 for denominator 0 + ) + + result["min_sha"] = "'" + result["min_sha"] + + result.to_csv(output_filename, sep=";", index=False) + + critical = result.iloc[:, 0:9] + formats = list(set(critical["format"])) + formats.sort() + critical3 = pd.DataFrame() + + for format in formats: + top3 = ( + critical[critical["format"] == format] + .sort_values(by="ratio", ascending=False) + .head(3) + ) + critical3 = pd.concat([critical3, top3], ignore_index=True) + + critical3.to_csv("critical3.csv", sep=";", index=False) + + for row_counter, row in critical3.iterrows(): + + # Find level + level_scaling = 1.0 + if "lev+10" in row["job"]: + level_scaling = 3.162 + if "lev-10" in row["job"]: + level_scaling = 0.3162 + + # Find executables setup + REF_ENC1 = "IVAS_cod_ref_1" + REF_DEC1 = "IVAS_dec_ref_1" + DUT_ENC1 = "IVAS_cod_1" + DUT_DEC1 = "IVAS_dec_1" + REF_ENC2 = "IVAS_cod_ref_2" + REF_DEC2 = "IVAS_dec_ref_2" + DUT_ENC2 = "IVAS_cod_2" + DUT_DEC2 = "IVAS_dec_2" + + if "dec" in row["job"]: + DUT_ENC1 = "IVAS_cod_ref_1" + DUT_ENC2 = "IVAS_cod_ref_2" + if "enc" in row["job"]: + DUT_DEC1 = "IVAS_dec_ref_1" + DUT_DEC2 = "IVAS_dec_ref_2" + + script_content = REPRODUCE_REGRESSION_SCRIPT_TMPL.format( + min_date=row["min_date"], + min_sha=row["min_sha"][1:], + level_scaling=level_scaling, + testcase=row["testcase"], + REF_ENC1=REF_ENC1, + REF_DEC1=REF_DEC1, + DUT_ENC1=DUT_ENC1, + DUT_DEC1=DUT_DEC1, + REF_ENC2=REF_ENC2, + REF_DEC2=REF_DEC2, + DUT_ENC2=DUT_ENC2, + DUT_DEC2=DUT_DEC2, + ) + + script_filename = f"regression_{row_counter+2:03d}.bash" + with open(script_filename, "w") as f: + f.write(script_content) + + +if __name__ == "__main__": + parser = argparse.ArgumentParser(description="logs dir") + parser.add_argument( + "logs_dir", + type=str, + help="Logs dir, e.g. logs", + ) + parser.add_argument( + "output_filename", + type=str, + help="Filename of the combined csv file. e.g mld.csv", + ) + parser.add_argument( + "--measure", + type=str, + help="Measure for summary, one of MLD MIN_SSNR MAX_ABS_DIFF MIN_ODG, (default: MLD)", + default="MLD", + ) + + args = parser.parse_args() + + main(args.logs_dir, args.output_filename, args.measure)