From df01165bf7dc24b6b0e3ac50a35d83a11167b0d9 Mon Sep 17 00:00:00 2001 From: Erik Norvell Date: Fri, 5 Sep 2025 07:16:35 +0200 Subject: [PATCH] Split regressions script in two. Add thresholds to find_regressions_from_logs.py --- scripts/find_regressions_from_logs.py | 202 +++++-------------- scripts/generate_scripts_from_regressions.py | 155 ++++++++++++++ 2 files changed, 203 insertions(+), 154 deletions(-) create mode 100644 scripts/generate_scripts_from_regressions.py diff --git a/scripts/find_regressions_from_logs.py b/scripts/find_regressions_from_logs.py index 068fe77871..67a7447ddf 100644 --- a/scripts/find_regressions_from_logs.py +++ b/scripts/find_regressions_from_logs.py @@ -4,99 +4,7 @@ import argparse from pathlib import Path import pandas as pd -REPRODUCE_REGRESSION_SCRIPT_TMPL = """ -#!/bin/bash -x - -SCRIPTS_DIR=/usr/local/scripts -LTV_DIR=/usr/local/ltv - -MIN_DATE={min_date} -MIN_SHA={min_sha} -LEVEL_SCALING={level_scaling} -TESTCASE="{testcase}" - -REF_ENC1={REF_ENC1} -REF_DEC1={REF_DEC1} -DUT_ENC1={DUT_ENC1} -DUT_DEC1={DUT_DEC1} - -REF_ENC2={REF_ENC2} -REF_DEC2={REF_DEC2} -DUT_ENC2={DUT_ENC2} -DUT_DEC2={DUT_DEC2} - -INV_LEVEL_SCALING=$(awk "BEGIN {{print 1.0 / $LEVEL_SCALING}}") - -# Obtain executables from past reference -git checkout `git rev-list -1 --before="$MIN_DATE 22:00:00" ivas-float-update` -echo "ivas_float_update, min version: `git rev-parse HEAD`" > versions.txt -make clean -make -j -mv IVAS_cod IVAS_cod_ref_1 -mv IVAS_dec IVAS_dec_ref_1 -mv IVAS_rend IVAS_rend_ref_1 - -git checkout $MIN_SHA -echo "main, min version: `git rev-parse HEAD`" >> versions.txt -make clean -make -j -mv IVAS_cod IVAS_cod_1 -mv IVAS_dec IVAS_dec_1 -mv IVAS_rend IVAS_rend_1 - -# Obtain latest executables -git checkout ivas-float-update -git pull -echo "ivas-float-update, current version: `git rev-parse HEAD`" >> versions.txt -make clean -make -j -mv IVAS_cod IVAS_cod_ref_2 -mv IVAS_dec IVAS_dec_ref_2 -mv IVAS_rend IVAS_rend_ref_2 - -git checkout main -git pull -echo "main, current version: `git rev-parse HEAD`" >> versions.txt -make clean -make -j -mv IVAS_cod IVAS_cod_2 -mv IVAS_dec IVAS_dec_2 -mv IVAS_rend IVAS_rend_2 - -# Get fresh copy of scripts, tests and ci -cp -r $SCRIPTS_DIR/{{scripts,tests,ci,pytest.ini}} . -rm -rf tests/ref tests/dut tests/renderer/ref tests/renderer/cut -python3 ci/remove_unsupported_testcases.py scripts/config/self_test.prm scripts/config/self_test_ltv.prm - -# Get LTVs -cp $LTV_DIR/* scripts/testv - -# Apply level scaling -tests/scale_pcm.py ./scripts/testv/ "$LEVEL_SCALING" - -# Run tests -cp IVAS_rend_ref_1 IVAS_rend_ref -cp IVAS_rend_1 IVAS_rend -python3 -m pytest "$TESTCASE" -n 1 --update_ref 1 --create_ref --param_file scripts/config/self_test_ltv.prm --use_ltv --ref_encoder_path $REF_ENC1 --ref_decoder_path $REF_DEC1 -python3 -m pytest "$TESTCASE" -n 1 --create_cut --param_file scripts/config/self_test_ltv.prm --use_ltv --dut_encoder_path $DUT_ENC1 --dut_decoder_path $DUT_DEC1 --mld --ssnr --odg --scalefac $INV_LEVEL_SCALING --junit-xml=report1.xml --html=report1.html --self-contained-html -python3 scripts/parse_xml_report.py report1.xml report1.csv - -# Store results from first run -mkdir -p tests1/renderer -cp -r tests/ref tests/dut tests1 -cp -r tests/renderer/ref tests1/renderer -cp -r tests/renderer/cut tests1/renderer - -cp IVAS_rend_ref_2 IVAS_rend_ref -cp IVAS_rend_2 IVAS_rend -python3 -m pytest "$TESTCASE" -n 1 --update_ref 1 --create_ref --param_file scripts/config/self_test_ltv.prm --use_ltv --ref_encoder_path $REF_ENC2 --ref_decoder_path $REF_DEC2 -python3 -m pytest "$TESTCASE" -n 1 --create_cut --param_file scripts/config/self_test_ltv.prm --use_ltv --dut_encoder_path $DUT_ENC2 --dut_decoder_path $DUT_DEC2 --mld --ssnr --odg --scalefac $INV_LEVEL_SCALING --junit-xml=report2.xml --html=report2.html --self-contained-html -python3 scripts/parse_xml_report.py report2.xml report2.csv - -""" - - -def main(logs_dir, output_filename, measure): +def main(logs_dir, output_filename, measure, days, all_results, diff_thr, ratio_thr, curr_value_thr): input_path = Path(logs_dir) logs = [f for f in input_path.iterdir() if f.is_dir()] @@ -142,6 +50,14 @@ def main(logs_dir, output_filename, measure): index=["job", "testcase"], columns="date", values="format" ).reset_index() + if days != -1: + len_history = len(result.columns) - 2 + if len_history < days: + print(f"*** Warning: Requested number of days exceeds the number of days in the logs. Will use all {len_history} available days.") + if len_history > days: + remove_days = len_history - days + result = result.drop(result.columns[2:2+remove_days], axis=1) + values = result.iloc[:, 2:] last_date = values.columns[-1] @@ -156,69 +72,16 @@ def main(logs_dir, output_filename, measure): 1 # Set ratio to 1 for denominator 0 ) - result["min_sha"] = "'" + result["min_sha"] + if not all_results: + # Filter out test cases above thresholds + result = result[result["diff"] > diff_thr] + result = result[result["ratio"] > ratio_thr] + result = result[result["curr_value"] > curr_value_thr] + + result["min_sha"] = "'" + result["min_sha"] # Add apostrophy to prevent Excel reading this as a number result.to_csv(output_filename, sep=";", index=False) - critical = result.iloc[:, 0:9] - formats = list(set(critical["format"])) - formats.sort() - critical3 = pd.DataFrame() - - for format in formats: - top3 = ( - critical[critical["format"] == format] - .sort_values(by="ratio", ascending=False) - .head(3) - ) - critical3 = pd.concat([critical3, top3], ignore_index=True) - - critical3.to_csv("critical3.csv", sep=";", index=False) - - for row_counter, row in critical3.iterrows(): - - # Find level - level_scaling = 1.0 - if "lev+10" in row["job"]: - level_scaling = 3.162 - if "lev-10" in row["job"]: - level_scaling = 0.3162 - - # Find executables setup - REF_ENC1 = "IVAS_cod_ref_1" - REF_DEC1 = "IVAS_dec_ref_1" - DUT_ENC1 = "IVAS_cod_1" - DUT_DEC1 = "IVAS_dec_1" - REF_ENC2 = "IVAS_cod_ref_2" - REF_DEC2 = "IVAS_dec_ref_2" - DUT_ENC2 = "IVAS_cod_2" - DUT_DEC2 = "IVAS_dec_2" - - if "dec" in row["job"]: - DUT_ENC1 = "IVAS_cod_ref_1" - DUT_ENC2 = "IVAS_cod_ref_2" - if "enc" in row["job"]: - DUT_DEC1 = "IVAS_dec_ref_1" - DUT_DEC2 = "IVAS_dec_ref_2" - - script_content = REPRODUCE_REGRESSION_SCRIPT_TMPL.format( - min_date=row["min_date"], - min_sha=row["min_sha"][1:], - level_scaling=level_scaling, - testcase=row["testcase"], - REF_ENC1=REF_ENC1, - REF_DEC1=REF_DEC1, - DUT_ENC1=DUT_ENC1, - DUT_DEC1=DUT_DEC1, - REF_ENC2=REF_ENC2, - REF_DEC2=REF_DEC2, - DUT_ENC2=DUT_ENC2, - DUT_DEC2=DUT_DEC2, - ) - - script_filename = f"regression_{row_counter+2:03d}.bash" - with open(script_filename, "w") as f: - f.write(script_content) if __name__ == "__main__": @@ -239,7 +102,38 @@ if __name__ == "__main__": help="Measure for summary, one of MLD MIN_SSNR MAX_ABS_DIFF MIN_ODG, (default: MLD)", default="MLD", ) + parser.add_argument( + "--days", + type=int, + help="Number of days in history, (default: whole history)", + default=-1, + ) + parser.add_argument( + "--all_results", + action="store_true", + help="Output all results, including cases without regression (default: off)", + default=False, + ) + parser.add_argument( + "--diff_thr", + type=float, + help="Include test cases with diff above diff_thr, (default: 0.0)", + default=0.0, + ) + parser.add_argument( + "--ratio_thr", + type=float, + help="Include test cases with ratio above ratio_thr, (default: 1.0)", + default=1.0, + ) + parser.add_argument( + "--curr_value_thr", + type=float, + help="Include test cases with curr_value above curr_value_thr, (default: 0.0)", + default=0.0, + ) args = parser.parse_args() - main(args.logs_dir, args.output_filename, args.measure) + main(args.logs_dir, args.output_filename, args.measure, args.days, args.all_results, args.diff_thr, args.ratio_thr, args.curr_value_thr) + diff --git a/scripts/generate_scripts_from_regressions.py b/scripts/generate_scripts_from_regressions.py new file mode 100644 index 0000000000..4487f84480 --- /dev/null +++ b/scripts/generate_scripts_from_regressions.py @@ -0,0 +1,155 @@ +#!/usr/bin/env python3 + +import argparse +import pandas as pd + +REPRODUCE_REGRESSION_SCRIPT_TMPL = """ +#!/bin/bash -x + +SCRIPTS_DIR=/usr/local/scripts +LTV_DIR=/usr/local/ltv + +MIN_DATE={min_date} +MIN_SHA={min_sha} +LEVEL_SCALING={level_scaling} +TESTCASE="{testcase}" + +REF_ENC1={REF_ENC1} +REF_DEC1={REF_DEC1} +DUT_ENC1={DUT_ENC1} +DUT_DEC1={DUT_DEC1} + +REF_ENC2={REF_ENC2} +REF_DEC2={REF_DEC2} +DUT_ENC2={DUT_ENC2} +DUT_DEC2={DUT_DEC2} + +INV_LEVEL_SCALING=$(awk "BEGIN {{print 1.0 / $LEVEL_SCALING}}") + +# Obtain executables from past reference +git checkout `git rev-list -1 --before="$MIN_DATE 22:00:00" ivas-float-update` +echo "ivas_float_update, min version: `git rev-parse HEAD`" > versions.txt +make clean +make -j +mv IVAS_cod IVAS_cod_ref_1 +mv IVAS_dec IVAS_dec_ref_1 +mv IVAS_rend IVAS_rend_ref_1 + +git checkout $MIN_SHA +echo "main, min version: `git rev-parse HEAD`" >> versions.txt +make clean +make -j +mv IVAS_cod IVAS_cod_1 +mv IVAS_dec IVAS_dec_1 +mv IVAS_rend IVAS_rend_1 + +# Obtain latest executables +git checkout ivas-float-update +git pull +echo "ivas-float-update, current version: `git rev-parse HEAD`" >> versions.txt +make clean +make -j +mv IVAS_cod IVAS_cod_ref_2 +mv IVAS_dec IVAS_dec_ref_2 +mv IVAS_rend IVAS_rend_ref_2 + +git checkout main +git pull +echo "main, current version: `git rev-parse HEAD`" >> versions.txt +make clean +make -j +mv IVAS_cod IVAS_cod_2 +mv IVAS_dec IVAS_dec_2 +mv IVAS_rend IVAS_rend_2 + +# Get fresh copy of scripts, tests and ci +cp -r $SCRIPTS_DIR/{{scripts,tests,ci,pytest.ini}} . +rm -rf tests/ref tests/dut tests/renderer/ref tests/renderer/cut +python3 ci/remove_unsupported_testcases.py scripts/config/self_test.prm scripts/config/self_test_ltv.prm + +# Get LTVs +cp $LTV_DIR/* scripts/testv + +# Apply level scaling +tests/scale_pcm.py ./scripts/testv/ "$LEVEL_SCALING" + +# Run tests +cp IVAS_rend_ref_1 IVAS_rend_ref +cp IVAS_rend_1 IVAS_rend +python3 -m pytest "$TESTCASE" -n 1 --update_ref 1 --create_ref --param_file scripts/config/self_test_ltv.prm --use_ltv --ref_encoder_path $REF_ENC1 --ref_decoder_path $REF_DEC1 +python3 -m pytest "$TESTCASE" -n 1 --create_cut --param_file scripts/config/self_test_ltv.prm --use_ltv --dut_encoder_path $DUT_ENC1 --dut_decoder_path $DUT_DEC1 --mld --ssnr --odg --scalefac $INV_LEVEL_SCALING --junit-xml=report1.xml --html=report1.html --self-contained-html +python3 scripts/parse_xml_report.py report1.xml report1.csv + +# Store results from first run +mkdir -p tests1/renderer +cp -r tests/ref tests/dut tests1 +cp -r tests/renderer/ref tests1/renderer +cp -r tests/renderer/cut tests1/renderer + +cp IVAS_rend_ref_2 IVAS_rend_ref +cp IVAS_rend_2 IVAS_rend +python3 -m pytest "$TESTCASE" -n 1 --update_ref 1 --create_ref --param_file scripts/config/self_test_ltv.prm --use_ltv --ref_encoder_path $REF_ENC2 --ref_decoder_path $REF_DEC2 +python3 -m pytest "$TESTCASE" -n 1 --create_cut --param_file scripts/config/self_test_ltv.prm --use_ltv --dut_encoder_path $DUT_ENC2 --dut_decoder_path $DUT_DEC2 --mld --ssnr --odg --scalefac $INV_LEVEL_SCALING --junit-xml=report2.xml --html=report2.html --self-contained-html +python3 scripts/parse_xml_report.py report2.xml report2.csv + +""" + + +def main(critical_list): + + critical = pd.read_csv(critical_list, sep=";") + + for row_counter, row in critical.iterrows(): + + # Find level + level_scaling = 1.0 + if "lev+10" in row["job"]: + level_scaling = 3.162 + if "lev-10" in row["job"]: + level_scaling = 0.3162 + + # Find executables setup + REF_ENC1 = "IVAS_cod_ref_1" + REF_DEC1 = "IVAS_dec_ref_1" + DUT_ENC1 = "IVAS_cod_1" + DUT_DEC1 = "IVAS_dec_1" + REF_ENC2 = "IVAS_cod_ref_2" + REF_DEC2 = "IVAS_dec_ref_2" + DUT_ENC2 = "IVAS_cod_2" + DUT_DEC2 = "IVAS_dec_2" + + if "dec" in row["job"]: + DUT_ENC1 = "IVAS_cod_ref_1" + DUT_ENC2 = "IVAS_cod_ref_2" + if "enc" in row["job"]: + DUT_DEC1 = "IVAS_dec_ref_1" + DUT_DEC2 = "IVAS_dec_ref_2" + + script_content = REPRODUCE_REGRESSION_SCRIPT_TMPL.format( + min_date=row["min_date"], + min_sha=row["min_sha"][1:], + level_scaling=level_scaling, + testcase=row["testcase"], + REF_ENC1=REF_ENC1, + REF_DEC1=REF_DEC1, + DUT_ENC1=DUT_ENC1, + DUT_DEC1=DUT_DEC1, + REF_ENC2=REF_ENC2, + REF_DEC2=REF_DEC2, + DUT_ENC2=DUT_ENC2, + DUT_DEC2=DUT_DEC2, + ) + + script_filename = f"regression_{row_counter+2:03d}.bash" + with open(script_filename, "w") as f: + f.write(script_content) + +if __name__ == "__main__": + parser = argparse.ArgumentParser(description="Generate bash scripts to reproduce test cases from critical testcase csv list") + parser.add_argument( + "critical_list", + type=str, + help="critical file list, e.g. critical.csv", + ) + args = parser.parse_args() + main(args.critical_list) -- GitLab