From aa9ca52d0667711efbbd1cc3d269fdea0d94e12c Mon Sep 17 00:00:00 2001 From: Erik Norvell Date: Fri, 28 Jun 2024 13:15:01 +0200 Subject: [PATCH 01/15] Updated detailed report. --- scripts/parse_xml_report.py | 57 +++++++++++++++++++++++-------------- 1 file changed, 35 insertions(+), 22 deletions(-) diff --git a/scripts/parse_xml_report.py b/scripts/parse_xml_report.py index 4b61dc1a94..f48b5d9e19 100644 --- a/scripts/parse_xml_report.py +++ b/scripts/parse_xml_report.py @@ -40,6 +40,12 @@ if __name__ == "__main__": results_unsorted = {} count = {'PASS':0,'FAIL':0,'ERROR':0} + # Formats, categories and MLD histogram limits + formats = {'**Unidentified format**':r'.*','Stereo':r'stereo', 'ISM':r'ISM', 'Multichannel':r'Multi-channel', + 'MASA':r'MASA','SBA':r'SBA', 'OSBA':r'OSBA', 'OMASA':r'OMASA','Renderer':r'renderer'} + categories = {'Normal operation':r'.*', 'DTX':r'DTX', 'PLC':r'%', 'Bitrate switching':r'br sw|bitrate switching', 'JBM':r'JBM' } + limits = [0,5,10,20,math.inf] + for testcase in testcases: if testcase.find(".//skipped") is None: if testcase.get("file") is None: @@ -64,43 +70,50 @@ if __name__ == "__main__": testresult = 'PASS' properties_values = [str(properties_found.get(p)) for p in PROPERTIES] + + # Identify format and category (mode of operation) + fmt = [f for f in formats if re.search(formats[f], fulltestname, re.IGNORECASE)][-1] + cat = [c for c in categories if re.search(categories[c], fulltestname, re.IGNORECASE)][-1] + # For ERROR cases, both a FAIL and an ERROR result is generated. # Here, a FAIL would be overwritten with an ERROR result since it has the same name. - results_unsorted[fulltestname] = (testresult, properties_values) + # Todo: Change to fielded dict for all entries to eliminate confusing literal indices + results_unsorted[fulltestname] = (fmt, cat, testresult, properties_values) results_sorted = dict(sorted(results_unsorted.items())) with open(csv_file, "w") as outfile: - headerline = ";".join(["testcase","Result"] + PROPERTIES) + "\n" + headerline = ";".join(["testcase","Format","Category","Result"] + PROPERTIES) + "\n" outfile.write(headerline) for test in results_sorted: - count[results_sorted[test][0]] += 1 - line = ";".join([test,results_sorted[test][0]] + results_sorted[test][1]) + "\n" + count[results_sorted[test][2]] += 1 + line = ";".join([test] + list(results_sorted[test])[0:-1] + results_sorted[test][-1]) + "\n" outfile.write(line) - categories = {'Normal operation':r'.*', 'JBM':r'JBM', 'PLC':r'%', 'Bitrate switching':r'br sw|bitrate switching'} - limits = [0,5,10,20,math.inf] tmp = {} + for fmt in formats: + tmp[fmt] = {} + for cat in categories: + tmp[fmt][cat] = [] if summary_file is not None: with open(summary_file, "w") as fp: for testcase in results_sorted: - cat = [c for c in categories if re.search(categories[c],testcase)][-1] - if cat in tmp: - tmp[cat].append(results_sorted[testcase][1][0]) # Add MLD score to list - else: - tmp[cat] = [results_sorted[testcase][1][0]] - - headerline = "Category;0;" + ";".join([f"{str(a)} -- {str(b)}" for (a,b) in zip(limits[0:-1],limits[1:])]) + ";None\n" - fp.write(headerline) - for cat in tmp: - # Separate 0 and None as special cases - mld = [float(x) for x in tmp[cat] if x != 'None' and x != '0'] - zero = sum([1 for x in tmp[cat] if x == '0']) - none = sum([1 for x in tmp[cat] if x == 'None']) - hist, _ = np.histogram(mld,limits) - line = f"{cat}; {str(zero)}; {'; '.join(map(str,hist))}; {str(none)}\n" - fp.write(line) + (fmt, cat, _, _) = results_sorted[testcase] + tmp[fmt][cat].append(results_sorted[testcase][-1][0]) # Add MLD score to list + + for fmt in tmp: + headerline = f"{fmt};\nCategory;0;" + ";".join([f"{str(a)} -- {str(b)}" for (a,b) in zip(limits[0:-1],limits[1:])]) + ";None\n" + fp.write(headerline) + for cat in categories: + # Separate 0 and None as special cases + mld = [float(x) for x in tmp[fmt][cat] if x != 'None' and x != '0'] + zero = sum([1 for x in tmp[fmt][cat] if x == '0']) + none = sum([1 for x in tmp[fmt][cat] if x == 'None']) + hist, _ = np.histogram(mld,limits) + line = f"{cat}; {str(zero)}; {'; '.join(map(str,hist))}; {str(none)}\n" + fp.write(line) + fp.write("\n") print( f"Parsed testsuite with {count['PASS']+count['FAIL']+count['ERROR']} tests: {count['PASS']} passes, {count['FAIL']} failures and {count['ERROR']} errors." -- GitLab From 884f6f26cc22f0b455c9eab7a78927a06e6026ec Mon Sep 17 00:00:00 2001 From: Erik Norvell Date: Mon, 1 Jul 2024 07:37:29 +0200 Subject: [PATCH 02/15] Fix for format identification to pick the earliest match in the test case --- scripts/parse_xml_report.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/scripts/parse_xml_report.py b/scripts/parse_xml_report.py index f48b5d9e19..5b9492d958 100644 --- a/scripts/parse_xml_report.py +++ b/scripts/parse_xml_report.py @@ -41,8 +41,8 @@ if __name__ == "__main__": count = {'PASS':0,'FAIL':0,'ERROR':0} # Formats, categories and MLD histogram limits - formats = {'**Unidentified format**':r'.*','Stereo':r'stereo', 'ISM':r'ISM', 'Multichannel':r'Multi-channel', - 'MASA':r'MASA','SBA':r'SBA', 'OSBA':r'OSBA', 'OMASA':r'OMASA','Renderer':r'renderer'} + formats = {'Stereo':r'stereo', 'ISM':r'ISM', 'Multichannel':r'Multi-channel', + 'MASA':r'MASA','SBA':r'SBA', 'OSBA':r'OSBA', 'OMASA':r'OMASA','Renderer':r'renderer','**Unidentified format**':r'.*'} categories = {'Normal operation':r'.*', 'DTX':r'DTX', 'PLC':r'%', 'Bitrate switching':r'br sw|bitrate switching', 'JBM':r'JBM' } limits = [0,5,10,20,math.inf] @@ -72,7 +72,8 @@ if __name__ == "__main__": properties_values = [str(properties_found.get(p)) for p in PROPERTIES] # Identify format and category (mode of operation) - fmt = [f for f in formats if re.search(formats[f], fulltestname, re.IGNORECASE)][-1] + # For the format, favor the earliest match in the test case name + fmt = min([(f, re.search(formats[f], fulltestname, re.IGNORECASE).end()) for f in formats if re.search(formats[f], fulltestname, re.IGNORECASE)] , key=lambda x: x[1])[0] cat = [c for c in categories if re.search(categories[c], fulltestname, re.IGNORECASE)][-1] # For ERROR cases, both a FAIL and an ERROR result is generated. -- GitLab From 56e874cc602c78993c04be80f1f2c2c3a41e19ad Mon Sep 17 00:00:00 2001 From: Erik Norvell Date: Mon, 1 Jul 2024 08:22:03 +0200 Subject: [PATCH 03/15] Harmonize results into a dict --- scripts/parse_xml_report.py | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/scripts/parse_xml_report.py b/scripts/parse_xml_report.py index 5b9492d958..ef3c5823a6 100644 --- a/scripts/parse_xml_report.py +++ b/scripts/parse_xml_report.py @@ -79,16 +79,21 @@ if __name__ == "__main__": # For ERROR cases, both a FAIL and an ERROR result is generated. # Here, a FAIL would be overwritten with an ERROR result since it has the same name. # Todo: Change to fielded dict for all entries to eliminate confusing literal indices - results_unsorted[fulltestname] = (fmt, cat, testresult, properties_values) + #results_unsorted[fulltestname] = (fmt, cat, testresult, properties_values) + results_unsorted[fulltestname] = {'Format':fmt, 'Category':cat, 'Result':testresult} + for propertyname, propertyvalue in zip(PROPERTIES, properties_values): + results_unsorted[fulltestname][propertyname] = propertyvalue results_sorted = dict(sorted(results_unsorted.items())) + keys = list(results_sorted[list(results_unsorted.keys())[0]].keys()) + with open(csv_file, "w") as outfile: - headerline = ";".join(["testcase","Format","Category","Result"] + PROPERTIES) + "\n" + headerline = ";".join(["testcase"] + keys) + "\n" outfile.write(headerline) for test in results_sorted: - count[results_sorted[test][2]] += 1 - line = ";".join([test] + list(results_sorted[test])[0:-1] + results_sorted[test][-1]) + "\n" + count[results_sorted[test]["Result"]] += 1 + line = ";".join([test] + list(results_sorted[test].values())) + "\n" outfile.write(line) tmp = {} @@ -100,8 +105,9 @@ if __name__ == "__main__": if summary_file is not None: with open(summary_file, "w") as fp: for testcase in results_sorted: - (fmt, cat, _, _) = results_sorted[testcase] - tmp[fmt][cat].append(results_sorted[testcase][-1][0]) # Add MLD score to list + fmt = results_sorted[testcase]["Format"] + cat = results_sorted[testcase]["Category"] + tmp[fmt][cat].append(results_sorted[testcase]["MLD"]) # Add MLD score to list for fmt in tmp: headerline = f"{fmt};\nCategory;0;" + ";".join([f"{str(a)} -- {str(b)}" for (a,b) in zip(limits[0:-1],limits[1:])]) + ";None\n" -- GitLab From dd7428e18a4571eeba7b0dbf55edc9a5b2764ba6 Mon Sep 17 00:00:00 2001 From: Erik Norvell Date: Mon, 1 Jul 2024 11:05:01 +0200 Subject: [PATCH 04/15] Change layout of matplotlib histogram --- scripts/parse_xml_report.py | 34 +++++++++++++++++++++++++++++----- 1 file changed, 29 insertions(+), 5 deletions(-) diff --git a/scripts/parse_xml_report.py b/scripts/parse_xml_report.py index ef3c5823a6..addddafda4 100644 --- a/scripts/parse_xml_report.py +++ b/scripts/parse_xml_report.py @@ -4,6 +4,7 @@ import argparse import re import math import numpy as np +import matplotlib.pyplot as plt from xml.etree import ElementTree """ @@ -41,8 +42,10 @@ if __name__ == "__main__": count = {'PASS':0,'FAIL':0,'ERROR':0} # Formats, categories and MLD histogram limits + # formats = {'Stereo':r'stereo', 'ISM':r'ISM', 'Multichannel':r'Multi-channel', + # 'MASA':r'MASA','SBA':r'SBA', 'OSBA':r'OSBA', 'OMASA':r'OMASA','Renderer':r'renderer'} formats = {'Stereo':r'stereo', 'ISM':r'ISM', 'Multichannel':r'Multi-channel', - 'MASA':r'MASA','SBA':r'SBA', 'OSBA':r'OSBA', 'OMASA':r'OMASA','Renderer':r'renderer','**Unidentified format**':r'.*'} + 'OSBA':r'OSBA', 'OMASA':r'OMASA','MASA':r'MASA','SBA':r'SBA', 'Renderer':r'renderer'} categories = {'Normal operation':r'.*', 'DTX':r'DTX', 'PLC':r'%', 'Bitrate switching':r'br sw|bitrate switching', 'JBM':r'JBM' } limits = [0,5,10,20,math.inf] @@ -108,20 +111,41 @@ if __name__ == "__main__": fmt = results_sorted[testcase]["Format"] cat = results_sorted[testcase]["Category"] tmp[fmt][cat].append(results_sorted[testcase]["MLD"]) # Add MLD score to list - + + subplot = 0 + fig, ax = plt.subplots(len(formats)//2, 2) for fmt in tmp: - headerline = f"{fmt};\nCategory;0;" + ";".join([f"{str(a)} -- {str(b)}" for (a,b) in zip(limits[0:-1],limits[1:])]) + ";None\n" + limits_labels = ["0"] + [f"{str(a)} -- {str(b)}" for (a,b) in zip(limits[0:-1],limits[1:])] + ["None"] + headerline = f"{fmt};\nCategory;" + ";".join(limits_labels) + "\n" fp.write(headerline) + bottom = np.zeros(len(limits)+1) for cat in categories: + # CSV output # Separate 0 and None as special cases mld = [float(x) for x in tmp[fmt][cat] if x != 'None' and x != '0'] zero = sum([1 for x in tmp[fmt][cat] if x == '0']) - none = sum([1 for x in tmp[fmt][cat] if x == 'None']) + none = sum([1 for x in tmp[fmt][cat] if x == 'None']) hist, _ = np.histogram(mld,limits) - line = f"{cat}; {str(zero)}; {'; '.join(map(str,hist))}; {str(none)}\n" + data = np.array([zero] + list(hist) + [none]) + line = f"{cat}; {'; '.join(map(str,data))}\n" fp.write(line) + + # Matplotlib histogram + a = subplot % 4 + b = subplot // 4 + ax[a][b].bar(limits_labels, data, 0.5, label=cat, bottom=bottom) + bottom += data fp.write("\n") + ax[a][b].set_title(fmt) + ax[a][b].legend(loc="best") + ax[a][b].set_xlabel("MLD") + ax[a][b].set_ylabel("Number of test cases") + subplot += 1 + fig.set_figheight(20) + fig.set_figwidth(16) + plt.savefig(summary_file +".png") + print( f"Parsed testsuite with {count['PASS']+count['FAIL']+count['ERROR']} tests: {count['PASS']} passes, {count['FAIL']} failures and {count['ERROR']} errors." ) -- GitLab From 407fed135b2bdfb3a691945562606bdd2e9750e5 Mon Sep 17 00:00:00 2001 From: Erik Norvell Date: Mon, 1 Jul 2024 16:07:26 +0200 Subject: [PATCH 05/15] Put report summary into separate script scripts/create_histogram_summary.py --- scripts/create_histogram_summary.py | 93 +++++++++++++++++++++++++++++ scripts/parse_xml_report.py | 81 +++++-------------------- 2 files changed, 109 insertions(+), 65 deletions(-) create mode 100644 scripts/create_histogram_summary.py diff --git a/scripts/create_histogram_summary.py b/scripts/create_histogram_summary.py new file mode 100644 index 0000000000..4116a97e21 --- /dev/null +++ b/scripts/create_histogram_summary.py @@ -0,0 +1,93 @@ +#!/usr/bin/env python3 + +import argparse +import math +import numpy as np +import matplotlib.pyplot as plt +import csv +from parse_xml_report import FORMATS, CATEGORIES + +""" +Parses a CSV report and creates a summary report. +""" + + + +# Main routine +if __name__ == "__main__": + parser = argparse.ArgumentParser( + description="Parses a CSV report and creates a summary report." + ) + parser.add_argument( + "csv_report", + type=str, + help="CSV report file of test cases, e.g. report.csv", + ) + parser.add_argument( + "csv_summary", type=str, help="Output CSV file, e.g. summary.csv" + ) + parser.add_argument( + "csv_image", + type=str, + nargs='?', + help="Summary image file, e.g. summary.png", + default = None + ) + args = parser.parse_args() + csv_report = args.csv_report + csv_summary = args.csv_summary + csv_image = args.csv_image + + limits = [0, 5, 10, math.inf] + + # Load CSV report + results_sorted = {} + with open(csv_report,'r') as fp: + reader = csv.reader(fp, delimiter=';') + header = next(reader) + keys = header[1:-1] + for row in reader: + testcase = row[0] + results_sorted[testcase] = {} + for k, val in zip(keys, row[1:-1]): + results_sorted[testcase][k] = val + + # Output CSV file + with open(csv_summary, "w") as fp: + limits_labels = ["0"] + [f"{str(a)} -- {str(b)}" for (a,b) in zip(limits[0:-1],limits[1:])] + ["None"] + headerline = f"Format;Category;" + ";".join(limits_labels) + "\n" + fp.write(headerline) + + subplot = 0 + fig, ax = plt.subplots(len(FORMATS)//2, 2) + for fmt in FORMATS: + bottom = np.zeros(len(limits)+1) + for cat in CATEGORIES: + mld_vals = [x for x in [m["MLD"] for m in results_sorted.values() if m["Format"] == fmt and m["Category"] == cat]] + mld = [float(x) for x in mld_vals if x != 'None' and x != '0'] + zero = sum([1 for x in mld_vals if x == '0']) + none = sum([1 for x in mld_vals if x == 'None']) + hist, _ = np.histogram(mld,limits) + data = np.array([zero] + list(hist) + [none]) + + # CSV output + line = f"{fmt};{cat};{'; '.join(map(str,data))}\n" + fp.write(line) + + # Matplotlib histogram + a = subplot % 4 + b = subplot // 4 + ax[a][b].bar(limits_labels, data, 0.5, label=cat, bottom=bottom) + bottom += data + + # Histogram layout + ax[a][b].set_title(fmt) + ax[a][b].legend(loc="best") + ax[a][b].set_xlabel("MLD") + ax[a][b].set_ylabel("Number of test cases") + subplot += 1 + + fig.set_figheight(20) + fig.set_figwidth(16) + if csv_image: + plt.savefig(csv_image) diff --git a/scripts/parse_xml_report.py b/scripts/parse_xml_report.py index addddafda4..87c660b8c3 100644 --- a/scripts/parse_xml_report.py +++ b/scripts/parse_xml_report.py @@ -13,41 +13,39 @@ Parse a junit report and create a summary report. PROPERTIES = ["MLD", "MAXIMUM ABS DIFF", "MIN_SSNR"] +FORMATS = {'Stereo':r'stereo', 'ISM':r'ISM', 'Multichannel':r'Multi-channel', + 'OSBA':r'OSBA', 'OMASA':r'OMASA','MASA':r'MASA','SBA':r'SBA', 'Renderer':r'renderer'} + +CATEGORIES = {'Normal operation':r'.*', 'DTX':r'DTX', 'PLC':r'%', 'Bitrate switching':r'br sw|bitrate switching', 'JBM':r'JBM' } # Main routine if __name__ == "__main__": parser = argparse.ArgumentParser( - description="Parse a junit report and create a MLD summary report." + description="Parse a junit report and create an MLD summary report." ) parser.add_argument( "xml_report", type=str, help="XML junit report input file, e.g. report-junit.xml", ) - parser.add_argument("csv_file", type=str, help="Output CSV file, e.g. mld.csv") - parser.add_argument("summary_file", type=str, nargs='?', help="Summary CSV file, e.g. summary.csv", default = None) + parser.add_argument( + "csv_file", + type=str, + help="Output CSV file, e.g. report.csv" + ) args = parser.parse_args() xml_report = args.xml_report csv_file = args.csv_file - summary_file = args.summary_file tree = ElementTree.parse(xml_report) testsuite = tree.find(".//testsuite") - - testcases = tree.findall(".//testcase") results_unsorted = {} count = {'PASS':0,'FAIL':0,'ERROR':0} - # Formats, categories and MLD histogram limits - # formats = {'Stereo':r'stereo', 'ISM':r'ISM', 'Multichannel':r'Multi-channel', - # 'MASA':r'MASA','SBA':r'SBA', 'OSBA':r'OSBA', 'OMASA':r'OMASA','Renderer':r'renderer'} - formats = {'Stereo':r'stereo', 'ISM':r'ISM', 'Multichannel':r'Multi-channel', - 'OSBA':r'OSBA', 'OMASA':r'OMASA','MASA':r'MASA','SBA':r'SBA', 'Renderer':r'renderer'} - categories = {'Normal operation':r'.*', 'DTX':r'DTX', 'PLC':r'%', 'Bitrate switching':r'br sw|bitrate switching', 'JBM':r'JBM' } - limits = [0,5,10,20,math.inf] + for testcase in testcases: if testcase.find(".//skipped") is None: @@ -76,21 +74,21 @@ if __name__ == "__main__": # Identify format and category (mode of operation) # For the format, favor the earliest match in the test case name - fmt = min([(f, re.search(formats[f], fulltestname, re.IGNORECASE).end()) for f in formats if re.search(formats[f], fulltestname, re.IGNORECASE)] , key=lambda x: x[1])[0] - cat = [c for c in categories if re.search(categories[c], fulltestname, re.IGNORECASE)][-1] + fmt = min([(f, re.search(FORMATS[f], fulltestname, re.IGNORECASE).end()) for f in FORMATS if re.search(FORMATS[f], fulltestname, re.IGNORECASE)] , key=lambda x: x[1])[0] + + # Note that only one category is selected, even though several may match, e.g. bitrate switching + JBM. Here the last match is picked. + cat = [c for c in CATEGORIES if re.search(CATEGORIES[c], fulltestname, re.IGNORECASE)][-1] # For ERROR cases, both a FAIL and an ERROR result is generated. # Here, a FAIL would be overwritten with an ERROR result since it has the same name. - # Todo: Change to fielded dict for all entries to eliminate confusing literal indices - #results_unsorted[fulltestname] = (fmt, cat, testresult, properties_values) results_unsorted[fulltestname] = {'Format':fmt, 'Category':cat, 'Result':testresult} for propertyname, propertyvalue in zip(PROPERTIES, properties_values): results_unsorted[fulltestname][propertyname] = propertyvalue results_sorted = dict(sorted(results_unsorted.items())) - keys = list(results_sorted[list(results_unsorted.keys())[0]].keys()) + # Write CSV file with open(csv_file, "w") as outfile: headerline = ";".join(["testcase"] + keys) + "\n" outfile.write(headerline) @@ -99,53 +97,6 @@ if __name__ == "__main__": line = ";".join([test] + list(results_sorted[test].values())) + "\n" outfile.write(line) - tmp = {} - for fmt in formats: - tmp[fmt] = {} - for cat in categories: - tmp[fmt][cat] = [] - - if summary_file is not None: - with open(summary_file, "w") as fp: - for testcase in results_sorted: - fmt = results_sorted[testcase]["Format"] - cat = results_sorted[testcase]["Category"] - tmp[fmt][cat].append(results_sorted[testcase]["MLD"]) # Add MLD score to list - - subplot = 0 - fig, ax = plt.subplots(len(formats)//2, 2) - for fmt in tmp: - limits_labels = ["0"] + [f"{str(a)} -- {str(b)}" for (a,b) in zip(limits[0:-1],limits[1:])] + ["None"] - headerline = f"{fmt};\nCategory;" + ";".join(limits_labels) + "\n" - fp.write(headerline) - bottom = np.zeros(len(limits)+1) - for cat in categories: - # CSV output - # Separate 0 and None as special cases - mld = [float(x) for x in tmp[fmt][cat] if x != 'None' and x != '0'] - zero = sum([1 for x in tmp[fmt][cat] if x == '0']) - none = sum([1 for x in tmp[fmt][cat] if x == 'None']) - hist, _ = np.histogram(mld,limits) - data = np.array([zero] + list(hist) + [none]) - line = f"{cat}; {'; '.join(map(str,data))}\n" - fp.write(line) - - # Matplotlib histogram - a = subplot % 4 - b = subplot // 4 - ax[a][b].bar(limits_labels, data, 0.5, label=cat, bottom=bottom) - bottom += data - fp.write("\n") - - ax[a][b].set_title(fmt) - ax[a][b].legend(loc="best") - ax[a][b].set_xlabel("MLD") - ax[a][b].set_ylabel("Number of test cases") - subplot += 1 - fig.set_figheight(20) - fig.set_figwidth(16) - plt.savefig(summary_file +".png") - print( f"Parsed testsuite with {count['PASS']+count['FAIL']+count['ERROR']} tests: {count['PASS']} passes, {count['FAIL']} failures and {count['ERROR']} errors." ) -- GitLab From 64f890dfa898946e70dc7912dc53977a20c0663e Mon Sep 17 00:00:00 2001 From: Erik Norvell Date: Tue, 2 Jul 2024 13:46:59 +0200 Subject: [PATCH 06/15] Add option to select SSNR and MAX ABS DIFF from create_histogram_summary.py --- scripts/create_histogram_summary.py | 60 +++++++++++++++++++---------- scripts/parse_xml_report.py | 1 - 2 files changed, 39 insertions(+), 22 deletions(-) diff --git a/scripts/create_histogram_summary.py b/scripts/create_histogram_summary.py index 4116a97e21..61cec7eef2 100644 --- a/scripts/create_histogram_summary.py +++ b/scripts/create_histogram_summary.py @@ -33,61 +33,79 @@ if __name__ == "__main__": help="Summary image file, e.g. summary.png", default = None ) + parser.add_argument( + "--measure", + type=str, + nargs=1, + help="Measure, any of: MLD, DIFF, SSNR, default: MLD", + default = ['MLD'] + ) args = parser.parse_args() csv_report = args.csv_report csv_summary = args.csv_summary csv_image = args.csv_image + measure = args.measure[0] - limits = [0, 5, 10, math.inf] + limits_per_measure = {'MLD':('MLD',[0, 5, 10, math.inf]), 'DIFF':('MAXIMUM ABS DIFF',[0, 1024, 16384, 32769]), 'SSNR':('MIN_SSNR',[-math.inf, 0, 20, 40, 60, 100])} + (measure_label, limits) = limits_per_measure[measure] # Load CSV report results_sorted = {} with open(csv_report,'r') as fp: reader = csv.reader(fp, delimiter=';') header = next(reader) - keys = header[1:-1] + keys = header[1:] for row in reader: testcase = row[0] results_sorted[testcase] = {} - for k, val in zip(keys, row[1:-1]): + for k, val in zip(keys, row[1:]): results_sorted[testcase][k] = val # Output CSV file with open(csv_summary, "w") as fp: - limits_labels = ["0"] + [f"{str(a)} -- {str(b)}" for (a,b) in zip(limits[0:-1],limits[1:])] + ["None"] + limits_labels = [f"{str(a)} -- {str(b)}" for (a,b) in zip(limits[0:-1],limits[1:])] + ["None"] + # Zero difference is treated as a special category for MLD and MAXIMUM ABS DIFF + if measure_label == "MLD" or measure_label == "MAXIMUM ABS DIFF": + limits_labels = ["0"] + limits_labels headerline = f"Format;Category;" + ";".join(limits_labels) + "\n" fp.write(headerline) subplot = 0 - fig, ax = plt.subplots(len(FORMATS)//2, 2) + fig, ax = plt.subplots(len(FORMATS)//4, 4) for fmt in FORMATS: - bottom = np.zeros(len(limits)+1) + bottom = np.zeros(len(limits_labels)) + row = subplot // 4 + col = subplot % 4 for cat in CATEGORIES: - mld_vals = [x for x in [m["MLD"] for m in results_sorted.values() if m["Format"] == fmt and m["Category"] == cat]] - mld = [float(x) for x in mld_vals if x != 'None' and x != '0'] - zero = sum([1 for x in mld_vals if x == '0']) - none = sum([1 for x in mld_vals if x == 'None']) - hist, _ = np.histogram(mld,limits) - data = np.array([zero] + list(hist) + [none]) + values = [x for x in [m[measure_label] for m in results_sorted.values() if m["Format"] == fmt and m["Category"] == cat]] + # Zero difference is treated as a special category for MLD and MAXIMUM ABS DIFF + if measure_label == "MLD" or measure_label == "MAXIMUM ABS DIFF": + val = [float(x) for x in values if x != 'None' and x != '0' and x != ''] + zero = [sum([1 for x in values if x == '0'])] + none = [sum([1 for x in values if x == 'None' or x == ''])] + else: + val = [float(x) for x in values if x != 'None' and x != ''] + zero = [] + none = [sum([1 for x in values if x == 'None' or x == ''])] + hist, _ = np.histogram(val,limits) + data = np.array(zero + list(hist) + none) # CSV output line = f"{fmt};{cat};{'; '.join(map(str,data))}\n" fp.write(line) # Matplotlib histogram - a = subplot % 4 - b = subplot // 4 - ax[a][b].bar(limits_labels, data, 0.5, label=cat, bottom=bottom) + ax[row][col].bar(limits_labels, data, 0.5, label=cat, bottom=bottom) bottom += data # Histogram layout - ax[a][b].set_title(fmt) - ax[a][b].legend(loc="best") - ax[a][b].set_xlabel("MLD") - ax[a][b].set_ylabel("Number of test cases") + ax[row][col].set_title(fmt) + ax[row][col].legend(loc="best") + ax[row][col].set_xlabel(measure_label) + ax[row][col].set_ylabel("Number of test cases") subplot += 1 - fig.set_figheight(20) - fig.set_figwidth(16) + fig.set_figheight(16) + fig.set_figwidth(32) if csv_image: plt.savefig(csv_image) diff --git a/scripts/parse_xml_report.py b/scripts/parse_xml_report.py index 87c660b8c3..e466304393 100644 --- a/scripts/parse_xml_report.py +++ b/scripts/parse_xml_report.py @@ -4,7 +4,6 @@ import argparse import re import math import numpy as np -import matplotlib.pyplot as plt from xml.etree import ElementTree """ -- GitLab From a9c36213f010cd6a26ff180662f3254fb273e0c8 Mon Sep 17 00:00:00 2001 From: Erik Norvell Date: Wed, 3 Jul 2024 09:07:20 +0200 Subject: [PATCH 07/15] Sort results in full report by FORMATS and CATEGORIES --- scripts/parse_xml_report.py | 31 ++++++++++++++++++------------- 1 file changed, 18 insertions(+), 13 deletions(-) diff --git a/scripts/parse_xml_report.py b/scripts/parse_xml_report.py index e466304393..cdd504c977 100644 --- a/scripts/parse_xml_report.py +++ b/scripts/parse_xml_report.py @@ -13,7 +13,7 @@ Parse a junit report and create a summary report. PROPERTIES = ["MLD", "MAXIMUM ABS DIFF", "MIN_SSNR"] FORMATS = {'Stereo':r'stereo', 'ISM':r'ISM', 'Multichannel':r'Multi-channel', - 'OSBA':r'OSBA', 'OMASA':r'OMASA','MASA':r'MASA','SBA':r'SBA', 'Renderer':r'renderer'} + 'MASA':r'(? Date: Wed, 3 Jul 2024 14:11:06 +0200 Subject: [PATCH 08/15] Add fix which may be needed for using matplotlib on computer without DISPLAY --- scripts/create_histogram_summary.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/scripts/create_histogram_summary.py b/scripts/create_histogram_summary.py index 61cec7eef2..3bfdd207df 100644 --- a/scripts/create_histogram_summary.py +++ b/scripts/create_histogram_summary.py @@ -3,6 +3,8 @@ import argparse import math import numpy as np +import matplotlib # These steps are added as a precaution in case the gitlab runner +matplotlib.use('Agg') # needs DISPLAY to render the plots, even if they are written to file. import matplotlib.pyplot as plt import csv from parse_xml_report import FORMATS, CATEGORIES -- GitLab From 2e98f7fd15cccb6e77638af8e0e08eebc4965c47 Mon Sep 17 00:00:00 2001 From: Erik Norvell Date: Wed, 3 Jul 2024 15:11:06 +0200 Subject: [PATCH 09/15] Fix in create_report_pages.py for added columns -- to be verified --- ci/basop-pages/create_report_pages.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ci/basop-pages/create_report_pages.py b/ci/basop-pages/create_report_pages.py index c849b160be..58d0e1ecef 100644 --- a/ci/basop-pages/create_report_pages.py +++ b/ci/basop-pages/create_report_pages.py @@ -92,9 +92,9 @@ ARROW_DOWN = '⬂' # expected columns. actual columns are filtered from the incoming data later, this # is mainly for controlling the order in the output table -COLUMNS = ["testcase", "Result", "MLD", "MAXIMUM ABS DIFF", "MIN_SSNR"] +COLUMNS = ["testcase", "Format", "Category", "Result", "MLD", "MAXIMUM ABS DIFF", "MIN_SSNR"] COLUMNS_GLOBAL = COLUMNS[:1] -COLUMNS_DIFFERENTIAL = COLUMNS[1:] +COLUMNS_DIFFERENTIAL = COLUMNS[3:] COLUMNS_DIFFERENTIAL_NOT_MLD = COLUMNS_DIFFERENTIAL[2:] -- GitLab From 10c395a94bced4eefb4ea6c6c37c6370e5c18f9d Mon Sep 17 00:00:00 2001 From: Erik Norvell Date: Wed, 3 Jul 2024 15:52:09 +0200 Subject: [PATCH 10/15] Fix for empty cells in csv report and optional argument for histogram png --- ci/basop-pages/create_report_pages.py | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/ci/basop-pages/create_report_pages.py b/ci/basop-pages/create_report_pages.py index 58d0e1ecef..5fda318b33 100644 --- a/ci/basop-pages/create_report_pages.py +++ b/ci/basop-pages/create_report_pages.py @@ -34,6 +34,8 @@ Comparing:
  • Merged csv data
    • + +
    How is the table sorted?
      @@ -106,6 +108,7 @@ def create_subpage( id_current: int, id_previous: int, job_name: str, + histogram: str, ): merged_reports = merge_and_cleanup_mld_reports( csv_current, csv_previous, id_current, id_previous @@ -135,6 +138,7 @@ def create_subpage( job_name=job_name, table_header_a=table_header_a, table_header_b=table_header_b, + histogram=histogram ) with open(html_out, "w") as f: f.write(new_subpage) @@ -242,7 +246,8 @@ def merge_and_cleanup_mld_reports( for col_pair in other_col_pairs: col_prev = col_pair[0] col_curr = col_pair[1] - diff_other += abs(float(x[col_curr]) - float(x[col_prev])) + if x[col_curr] != '' and x[col_prev] != '': + diff_other += abs(float(x[col_curr]) - float(x[col_prev])) if diff_other > 0: diff = -1000000 @@ -293,8 +298,14 @@ if __name__ == "__main__": parser.add_argument("id_current", type=int) parser.add_argument("id_previous", type=int) parser.add_argument("job_name") + parser.add_argument("--histogram", type=str, nargs=1, help="histogram picture", default = None) args = parser.parse_args() + if args.histogram != None: + histogram = args.histogram[0] + else: + histogram = "None" + create_subpage( args.html_out, args.csv_out, @@ -303,4 +314,5 @@ if __name__ == "__main__": args.id_current, args.id_previous, args.job_name, + histogram, ) -- GitLab From c4ffb1befadb4c8ec5a96b3b1ade6ca7f8d80755 Mon Sep 17 00:00:00 2001 From: Erik Norvell Date: Wed, 3 Jul 2024 16:59:15 +0200 Subject: [PATCH 11/15] Separate images per format --- ci/basop-pages/create_report_pages.py | 16 +++++++--------- scripts/create_histogram_summary.py | 26 ++++++++++++-------------- 2 files changed, 19 insertions(+), 23 deletions(-) diff --git a/ci/basop-pages/create_report_pages.py b/ci/basop-pages/create_report_pages.py index 5fda318b33..40f0832e21 100644 --- a/ci/basop-pages/create_report_pages.py +++ b/ci/basop-pages/create_report_pages.py @@ -34,7 +34,7 @@ Comparing:
    • Merged csv data
    - +{images}
    How is the table sorted? @@ -108,7 +108,7 @@ def create_subpage( id_current: int, id_previous: int, job_name: str, - histogram: str, + histogram, ): merged_reports = merge_and_cleanup_mld_reports( csv_current, csv_previous, id_current, id_previous @@ -131,6 +131,7 @@ def create_subpage( table_body = "\n".join( tr_from_row(row, id_current, id_previous) for row in merged_reports ) + images = " ".join([f"" for x in histogram]) new_subpage = SUBPAGE_TMPL_CSS + SUBPAGE_TMPL_HTML.format( id_current=id_current, id_previous=id_previous, @@ -138,7 +139,7 @@ def create_subpage( job_name=job_name, table_header_a=table_header_a, table_header_b=table_header_b, - histogram=histogram + images=images ) with open(html_out, "w") as f: f.write(new_subpage) @@ -298,13 +299,10 @@ if __name__ == "__main__": parser.add_argument("id_current", type=int) parser.add_argument("id_previous", type=int) parser.add_argument("job_name") - parser.add_argument("--histogram", type=str, nargs=1, help="histogram picture", default = None) + parser.add_argument("--histogram", type=str, nargs='+', help="histogram picture(s)", default = []) args = parser.parse_args() - if args.histogram != None: - histogram = args.histogram[0] - else: - histogram = "None" + create_subpage( args.html_out, @@ -314,5 +312,5 @@ if __name__ == "__main__": args.id_current, args.id_previous, args.job_name, - histogram, + args.histogram, ) diff --git a/scripts/create_histogram_summary.py b/scripts/create_histogram_summary.py index 3bfdd207df..caabd4b3c7 100644 --- a/scripts/create_histogram_summary.py +++ b/scripts/create_histogram_summary.py @@ -7,6 +7,7 @@ import matplotlib # These steps are added as a precaution in case the gitlab matplotlib.use('Agg') # needs DISPLAY to render the plots, even if they are written to file. import matplotlib.pyplot as plt import csv +import os from parse_xml_report import FORMATS, CATEGORIES """ @@ -72,12 +73,9 @@ if __name__ == "__main__": headerline = f"Format;Category;" + ";".join(limits_labels) + "\n" fp.write(headerline) - subplot = 0 - fig, ax = plt.subplots(len(FORMATS)//4, 4) for fmt in FORMATS: + fig, ax = plt.subplots() bottom = np.zeros(len(limits_labels)) - row = subplot // 4 - col = subplot % 4 for cat in CATEGORIES: values = [x for x in [m[measure_label] for m in results_sorted.values() if m["Format"] == fmt and m["Category"] == cat]] # Zero difference is treated as a special category for MLD and MAXIMUM ABS DIFF @@ -97,17 +95,17 @@ if __name__ == "__main__": fp.write(line) # Matplotlib histogram - ax[row][col].bar(limits_labels, data, 0.5, label=cat, bottom=bottom) + ax.bar(limits_labels, data, 0.5, label=cat, bottom=bottom) bottom += data # Histogram layout - ax[row][col].set_title(fmt) - ax[row][col].legend(loc="best") - ax[row][col].set_xlabel(measure_label) - ax[row][col].set_ylabel("Number of test cases") - subplot += 1 + ax.set_title(fmt) + ax.legend(loc="best") + ax.set_xlabel(measure_label) + ax.set_ylabel("Number of test cases") - fig.set_figheight(16) - fig.set_figwidth(32) - if csv_image: - plt.savefig(csv_image) + fig.set_figheight(4) + fig.set_figwidth(6) + if csv_image: + base, ext = os.path.splitext(csv_image) + plt.savefig(f"{base}_{fmt}{ext}") -- GitLab From 2d7f4c0b7f246547f8fef4c96d46c9a70a32b8c1 Mon Sep 17 00:00:00 2001 From: Erik Norvell Date: Thu, 4 Jul 2024 12:50:14 +0200 Subject: [PATCH 12/15] Add images in separate subfolder images --- ci/basop-pages/create_report_pages.py | 26 +++++++++++++++++++++----- scripts/create_histogram_summary.py | 2 +- 2 files changed, 22 insertions(+), 6 deletions(-) diff --git a/ci/basop-pages/create_report_pages.py b/ci/basop-pages/create_report_pages.py index 40f0832e21..830032c6f2 100644 --- a/ci/basop-pages/create_report_pages.py +++ b/ci/basop-pages/create_report_pages.py @@ -3,6 +3,7 @@ import pathlib import argparse from functools import partial +FORMATS = ['Stereo', 'ISM', 'Multichannel','MASA','SBA', 'OSBA', 'OMASA', 'Renderer'] CSV_DELIM = ";" SUBPAGE_TMPL_CSS = """ @@ -25,7 +26,7 @@ SUBPAGE_TMPL_CSS = """ SUBPAGE_TMPL_HTML = """ -

    Report for job {job_name}

    Report for job {job_name} Comparing: -{images} +{images_mld} + +{images_diff} + +{images_ssnr} +
    How is the table sorted? @@ -131,7 +137,15 @@ def create_subpage( table_body = "\n".join( tr_from_row(row, id_current, id_previous) for row in merged_reports ) - images = " ".join([f"" for x in histogram]) + if histogram: + images_mld = f"

    MLD summary {job_name}

    \n" + " ".join([f"" for x in FORMATS]) + images_ssnr = f"

    MIN_SSNR summary {job_name}

    \n" + " ".join([f"" for x in FORMATS]) + images_diff = f"

    MAX ABS DIFFERENCE summary {job_name}

    \n" + " ".join([f"" for x in FORMATS]) + else: + images_mld = "" + images_ssnr = "" + images_diff = "" + new_subpage = SUBPAGE_TMPL_CSS + SUBPAGE_TMPL_HTML.format( id_current=id_current, id_previous=id_previous, @@ -139,7 +153,9 @@ def create_subpage( job_name=job_name, table_header_a=table_header_a, table_header_b=table_header_b, - images=images + images_mld=images_mld, + images_ssnr=images_ssnr, + images_diff=images_diff, ) with open(html_out, "w") as f: f.write(new_subpage) @@ -299,7 +315,7 @@ if __name__ == "__main__": parser.add_argument("id_current", type=int) parser.add_argument("id_previous", type=int) parser.add_argument("job_name") - parser.add_argument("--histogram", type=str, nargs='+', help="histogram picture(s)", default = []) + parser.add_argument("--histogram", action='store_true') args = parser.parse_args() diff --git a/scripts/create_histogram_summary.py b/scripts/create_histogram_summary.py index caabd4b3c7..eab71dacf8 100644 --- a/scripts/create_histogram_summary.py +++ b/scripts/create_histogram_summary.py @@ -66,7 +66,7 @@ if __name__ == "__main__": # Output CSV file with open(csv_summary, "w") as fp: - limits_labels = [f"{str(a)} -- {str(b)}" for (a,b) in zip(limits[0:-1],limits[1:])] + ["None"] + limits_labels = [f"{str(a)} --\n {str(b)}" for (a,b) in zip(limits[0:-1],limits[1:])] + ["None"] # Zero difference is treated as a special category for MLD and MAXIMUM ABS DIFF if measure_label == "MLD" or measure_label == "MAXIMUM ABS DIFF": limits_labels = ["0"] + limits_labels -- GitLab From 57cb11bb93d826cc8285025e8b3a20867f641308 Mon Sep 17 00:00:00 2001 From: Erik Norvell Date: Thu, 4 Jul 2024 12:58:21 +0200 Subject: [PATCH 13/15] Fix for empty cells in report --- ci/basop-pages/create_report_pages.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/ci/basop-pages/create_report_pages.py b/ci/basop-pages/create_report_pages.py index 830032c6f2..4177930597 100644 --- a/ci/basop-pages/create_report_pages.py +++ b/ci/basop-pages/create_report_pages.py @@ -247,8 +247,9 @@ def merge_and_cleanup_mld_reports( are uninteresting and are put last. """ try: - float(x[mld_col_curr]) - float(x[mld_col_prev]) + cols = [mld_col_curr, mld_col_prev] + [p[1] for p in other_col_pairs] + for c in cols: + float(x[c]) except ValueError: # Value is no valid floating point value return float("inf") @@ -263,8 +264,7 @@ def merge_and_cleanup_mld_reports( for col_pair in other_col_pairs: col_prev = col_pair[0] col_curr = col_pair[1] - if x[col_curr] != '' and x[col_prev] != '': - diff_other += abs(float(x[col_curr]) - float(x[col_prev])) + diff_other += abs(float(x[col_curr]) - float(x[col_prev])) if diff_other > 0: diff = -1000000 -- GitLab From 389bb7cfa72376466c4118b08d53bddb948a9624 Mon Sep 17 00:00:00 2001 From: Erik Norvell Date: Thu, 4 Jul 2024 14:24:16 +0200 Subject: [PATCH 14/15] Added links to summary csv files --- ci/basop-pages/create_report_pages.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/ci/basop-pages/create_report_pages.py b/ci/basop-pages/create_report_pages.py index 4177930597..38a5106326 100644 --- a/ci/basop-pages/create_report_pages.py +++ b/ci/basop-pages/create_report_pages.py @@ -138,9 +138,9 @@ def create_subpage( tr_from_row(row, id_current, id_previous) for row in merged_reports ) if histogram: - images_mld = f"

    MLD summary {job_name}

    \n" + " ".join([f"" for x in FORMATS]) - images_ssnr = f"

    MIN_SSNR summary {job_name}

    \n" + " ".join([f"" for x in FORMATS]) - images_diff = f"

    MAX ABS DIFFERENCE summary {job_name}

    \n" + " ".join([f"" for x in FORMATS]) + images_mld = f"

    MLD summary {job_name}

    \n" + " ".join([f"" for x in FORMATS]) + f"\n
    summary_{id_current}_MLD.csv" + images_ssnr = f"

    MIN_SSNR summary {job_name}

    \n" + " ".join([f"" for x in FORMATS]) + f"\n
    summary_{id_current}_SSNR.csv" + images_diff = f"

    MAX ABS DIFFERENCE summary {job_name}

    \n" + " ".join([f"" for x in FORMATS]) + f"\n
    summary_{id_current}_DIFF.csv" else: images_mld = "" images_ssnr = "" -- GitLab From c8d0817f11d2ce072932c670ec68b9c94756335a Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Thu, 4 Jul 2024 15:06:59 +0200 Subject: [PATCH 15/15] run formatter on new files --- ci/basop-pages/create_report_pages.py | 42 +++++++++++---- scripts/create_histogram_summary.py | 58 +++++++++++++-------- scripts/parse_xml_report.py | 73 ++++++++++++++++++--------- 3 files changed, 119 insertions(+), 54 deletions(-) diff --git a/ci/basop-pages/create_report_pages.py b/ci/basop-pages/create_report_pages.py index 38a5106326..25fde10409 100644 --- a/ci/basop-pages/create_report_pages.py +++ b/ci/basop-pages/create_report_pages.py @@ -3,7 +3,7 @@ import pathlib import argparse from functools import partial -FORMATS = ['Stereo', 'ISM', 'Multichannel','MASA','SBA', 'OSBA', 'OMASA', 'Renderer'] +FORMATS = ["Stereo", "ISM", "Multichannel", "MASA", "SBA", "OSBA", "OMASA", "Renderer"] CSV_DELIM = ";" SUBPAGE_TMPL_CSS = """ @@ -100,7 +100,15 @@ ARROW_DOWN = '⬂' # expected columns. actual columns are filtered from the incoming data later, this # is mainly for controlling the order in the output table -COLUMNS = ["testcase", "Format", "Category", "Result", "MLD", "MAXIMUM ABS DIFF", "MIN_SSNR"] +COLUMNS = [ + "testcase", + "Format", + "Category", + "Result", + "MLD", + "MAXIMUM ABS DIFF", + "MIN_SSNR", +] COLUMNS_GLOBAL = COLUMNS[:1] COLUMNS_DIFFERENTIAL = COLUMNS[3:] COLUMNS_DIFFERENTIAL_NOT_MLD = COLUMNS_DIFFERENTIAL[2:] @@ -138,9 +146,27 @@ def create_subpage( tr_from_row(row, id_current, id_previous) for row in merged_reports ) if histogram: - images_mld = f"

    MLD summary {job_name}

    \n" + " ".join([f"" for x in FORMATS]) + f"\n
    summary_{id_current}_MLD.csv" - images_ssnr = f"

    MIN_SSNR summary {job_name}

    \n" + " ".join([f"" for x in FORMATS]) + f"\n
    summary_{id_current}_SSNR.csv" - images_diff = f"

    MAX ABS DIFFERENCE summary {job_name}

    \n" + " ".join([f"" for x in FORMATS]) + f"\n
    summary_{id_current}_DIFF.csv" + images_mld = ( + f"

    MLD summary {job_name}

    \n" + + " ".join( + [f"" for x in FORMATS] + ) + + f'\n
    summary_{id_current}_MLD.csv' + ) + images_ssnr = ( + f"

    MIN_SSNR summary {job_name}

    \n" + + " ".join( + [f"" for x in FORMATS] + ) + + f'\n
    summary_{id_current}_SSNR.csv' + ) + images_diff = ( + f"

    MAX ABS DIFFERENCE summary {job_name}

    \n" + + " ".join( + [f"" for x in FORMATS] + ) + + f'\n
    summary_{id_current}_DIFF.csv' + ) else: images_mld = "" images_ssnr = "" @@ -155,7 +181,7 @@ def create_subpage( table_header_b=table_header_b, images_mld=images_mld, images_ssnr=images_ssnr, - images_diff=images_diff, + images_diff=images_diff, ) with open(html_out, "w") as f: f.write(new_subpage) @@ -315,11 +341,9 @@ if __name__ == "__main__": parser.add_argument("id_current", type=int) parser.add_argument("id_previous", type=int) parser.add_argument("job_name") - parser.add_argument("--histogram", action='store_true') + parser.add_argument("--histogram", action="store_true") args = parser.parse_args() - - create_subpage( args.html_out, args.csv_out, diff --git a/scripts/create_histogram_summary.py b/scripts/create_histogram_summary.py index eab71dacf8..1ab951ac72 100644 --- a/scripts/create_histogram_summary.py +++ b/scripts/create_histogram_summary.py @@ -3,8 +3,10 @@ import argparse import math import numpy as np -import matplotlib # These steps are added as a precaution in case the gitlab runner -matplotlib.use('Agg') # needs DISPLAY to render the plots, even if they are written to file. +# These next three lines are added as a precaution in case the gitlab runner +# needs DISPLAY to render the plots, even if they are written to file. +import matplotlib +matplotlib.use("Agg") import matplotlib.pyplot as plt import csv import os @@ -15,7 +17,6 @@ Parses a CSV report and creates a summary report. """ - # Main routine if __name__ == "__main__": parser = argparse.ArgumentParser( @@ -30,18 +31,18 @@ if __name__ == "__main__": "csv_summary", type=str, help="Output CSV file, e.g. summary.csv" ) parser.add_argument( - "csv_image", - type=str, - nargs='?', - help="Summary image file, e.g. summary.png", - default = None + "csv_image", + type=str, + nargs="?", + help="Summary image file, e.g. summary.png", + default=None, ) parser.add_argument( "--measure", type=str, nargs=1, help="Measure, any of: MLD, DIFF, SSNR, default: MLD", - default = ['MLD'] + default=["MLD"], ) args = parser.parse_args() csv_report = args.csv_report @@ -49,13 +50,17 @@ if __name__ == "__main__": csv_image = args.csv_image measure = args.measure[0] - limits_per_measure = {'MLD':('MLD',[0, 5, 10, math.inf]), 'DIFF':('MAXIMUM ABS DIFF',[0, 1024, 16384, 32769]), 'SSNR':('MIN_SSNR',[-math.inf, 0, 20, 40, 60, 100])} + limits_per_measure = { + "MLD": ("MLD", [0, 5, 10, math.inf]), + "DIFF": ("MAXIMUM ABS DIFF", [0, 1024, 16384, 32769]), + "SSNR": ("MIN_SSNR", [-math.inf, 0, 20, 40, 60, 100]), + } (measure_label, limits) = limits_per_measure[measure] # Load CSV report results_sorted = {} - with open(csv_report,'r') as fp: - reader = csv.reader(fp, delimiter=';') + with open(csv_report, "r") as fp: + reader = csv.reader(fp, delimiter=";") header = next(reader) keys = header[1:] for row in reader: @@ -66,7 +71,9 @@ if __name__ == "__main__": # Output CSV file with open(csv_summary, "w") as fp: - limits_labels = [f"{str(a)} --\n {str(b)}" for (a,b) in zip(limits[0:-1],limits[1:])] + ["None"] + limits_labels = [ + f"{str(a)} --\n {str(b)}" for (a, b) in zip(limits[0:-1], limits[1:]) + ] + ["None"] # Zero difference is treated as a special category for MLD and MAXIMUM ABS DIFF if measure_label == "MLD" or measure_label == "MAXIMUM ABS DIFF": limits_labels = ["0"] + limits_labels @@ -77,23 +84,32 @@ if __name__ == "__main__": fig, ax = plt.subplots() bottom = np.zeros(len(limits_labels)) for cat in CATEGORIES: - values = [x for x in [m[measure_label] for m in results_sorted.values() if m["Format"] == fmt and m["Category"] == cat]] + values = [ + x + for x in [ + m[measure_label] + for m in results_sorted.values() + if m["Format"] == fmt and m["Category"] == cat + ] + ] # Zero difference is treated as a special category for MLD and MAXIMUM ABS DIFF if measure_label == "MLD" or measure_label == "MAXIMUM ABS DIFF": - val = [float(x) for x in values if x != 'None' and x != '0' and x != ''] - zero = [sum([1 for x in values if x == '0'])] - none = [sum([1 for x in values if x == 'None' or x == ''])] + val = [ + float(x) for x in values if x != "None" and x != "0" and x != "" + ] + zero = [sum([1 for x in values if x == "0"])] + none = [sum([1 for x in values if x == "None" or x == ""])] else: - val = [float(x) for x in values if x != 'None' and x != ''] + val = [float(x) for x in values if x != "None" and x != ""] zero = [] - none = [sum([1 for x in values if x == 'None' or x == ''])] - hist, _ = np.histogram(val,limits) + none = [sum([1 for x in values if x == "None" or x == ""])] + hist, _ = np.histogram(val, limits) data = np.array(zero + list(hist) + none) # CSV output line = f"{fmt};{cat};{'; '.join(map(str,data))}\n" fp.write(line) - + # Matplotlib histogram ax.bar(limits_labels, data, 0.5, label=cat, bottom=bottom) bottom += data diff --git a/scripts/parse_xml_report.py b/scripts/parse_xml_report.py index cdd504c977..3bc0b619ab 100644 --- a/scripts/parse_xml_report.py +++ b/scripts/parse_xml_report.py @@ -12,10 +12,24 @@ Parse a junit report and create a summary report. PROPERTIES = ["MLD", "MAXIMUM ABS DIFF", "MIN_SSNR"] -FORMATS = {'Stereo':r'stereo', 'ISM':r'ISM', 'Multichannel':r'Multi-channel', - 'MASA':r'(?