diff --git a/ci/basop-pages/create_report_pages.py b/ci/basop-pages/create_report_pages.py
index c849b160be964c75935ffa4e79e858249545e479..25fde10409bba73e20a58dbc7c0e5f7f101a9a2e 100644
--- a/ci/basop-pages/create_report_pages.py
+++ b/ci/basop-pages/create_report_pages.py
@@ -3,6 +3,7 @@ import pathlib
import argparse
from functools import partial
+FORMATS = ["Stereo", "ISM", "Multichannel", "MASA", "SBA", "OSBA", "OMASA", "Renderer"]
CSV_DELIM = ";"
SUBPAGE_TMPL_CSS = """
@@ -25,7 +26,7 @@ SUBPAGE_TMPL_CSS = """
SUBPAGE_TMPL_HTML = """
-
Report for job {job_name}
Report for job {job_name}
Comparing:
+{images_mld}
+
+{images_diff}
+
+{images_ssnr}
+
+
How is the table sorted?
@@ -92,9 +100,17 @@ ARROW_DOWN = '⬂'
# expected columns. actual columns are filtered from the incoming data later, this
# is mainly for controlling the order in the output table
-COLUMNS = ["testcase", "Result", "MLD", "MAXIMUM ABS DIFF", "MIN_SSNR"]
+COLUMNS = [
+ "testcase",
+ "Format",
+ "Category",
+ "Result",
+ "MLD",
+ "MAXIMUM ABS DIFF",
+ "MIN_SSNR",
+]
COLUMNS_GLOBAL = COLUMNS[:1]
-COLUMNS_DIFFERENTIAL = COLUMNS[1:]
+COLUMNS_DIFFERENTIAL = COLUMNS[3:]
COLUMNS_DIFFERENTIAL_NOT_MLD = COLUMNS_DIFFERENTIAL[2:]
@@ -106,6 +122,7 @@ def create_subpage(
id_current: int,
id_previous: int,
job_name: str,
+ histogram,
):
merged_reports = merge_and_cleanup_mld_reports(
csv_current, csv_previous, id_current, id_previous
@@ -128,6 +145,33 @@ def create_subpage(
table_body = "\n".join(
tr_from_row(row, id_current, id_previous) for row in merged_reports
)
+ if histogram:
+ images_mld = (
+ f"MLD summary {job_name}
\n"
+ + " ".join(
+ [f"
" for x in FORMATS]
+ )
+ + f'\n
summary_{id_current}_MLD.csv'
+ )
+ images_ssnr = (
+ f"MIN_SSNR summary {job_name}
\n"
+ + " ".join(
+ [f"
" for x in FORMATS]
+ )
+ + f'\n
summary_{id_current}_SSNR.csv'
+ )
+ images_diff = (
+ f"MAX ABS DIFFERENCE summary {job_name}
\n"
+ + " ".join(
+ [f"
" for x in FORMATS]
+ )
+ + f'\n
summary_{id_current}_DIFF.csv'
+ )
+ else:
+ images_mld = ""
+ images_ssnr = ""
+ images_diff = ""
+
new_subpage = SUBPAGE_TMPL_CSS + SUBPAGE_TMPL_HTML.format(
id_current=id_current,
id_previous=id_previous,
@@ -135,6 +179,9 @@ def create_subpage(
job_name=job_name,
table_header_a=table_header_a,
table_header_b=table_header_b,
+ images_mld=images_mld,
+ images_ssnr=images_ssnr,
+ images_diff=images_diff,
)
with open(html_out, "w") as f:
f.write(new_subpage)
@@ -226,8 +273,9 @@ def merge_and_cleanup_mld_reports(
are uninteresting and are put last.
"""
try:
- float(x[mld_col_curr])
- float(x[mld_col_prev])
+ cols = [mld_col_curr, mld_col_prev] + [p[1] for p in other_col_pairs]
+ for c in cols:
+ float(x[c])
except ValueError:
# Value is no valid floating point value
return float("inf")
@@ -293,6 +341,7 @@ if __name__ == "__main__":
parser.add_argument("id_current", type=int)
parser.add_argument("id_previous", type=int)
parser.add_argument("job_name")
+ parser.add_argument("--histogram", action="store_true")
args = parser.parse_args()
create_subpage(
@@ -303,4 +352,5 @@ if __name__ == "__main__":
args.id_current,
args.id_previous,
args.job_name,
+ args.histogram,
)
diff --git a/scripts/create_histogram_summary.py b/scripts/create_histogram_summary.py
new file mode 100644
index 0000000000000000000000000000000000000000..1ab951ac72af1f926a3a0bc7d5ba08bda781a9ef
--- /dev/null
+++ b/scripts/create_histogram_summary.py
@@ -0,0 +1,127 @@
+#!/usr/bin/env python3
+
+import argparse
+import math
+import numpy as np
+# These next three lines are added as a precaution in case the gitlab runner
+# needs DISPLAY to render the plots, even if they are written to file.
+import matplotlib
+matplotlib.use("Agg")
+import matplotlib.pyplot as plt
+import csv
+import os
+from parse_xml_report import FORMATS, CATEGORIES
+
+"""
+Parses a CSV report and creates a summary report.
+"""
+
+
+# Main routine
+if __name__ == "__main__":
+ parser = argparse.ArgumentParser(
+ description="Parses a CSV report and creates a summary report."
+ )
+ parser.add_argument(
+ "csv_report",
+ type=str,
+ help="CSV report file of test cases, e.g. report.csv",
+ )
+ parser.add_argument(
+ "csv_summary", type=str, help="Output CSV file, e.g. summary.csv"
+ )
+ parser.add_argument(
+ "csv_image",
+ type=str,
+ nargs="?",
+ help="Summary image file, e.g. summary.png",
+ default=None,
+ )
+ parser.add_argument(
+ "--measure",
+ type=str,
+ nargs=1,
+ help="Measure, any of: MLD, DIFF, SSNR, default: MLD",
+ default=["MLD"],
+ )
+ args = parser.parse_args()
+ csv_report = args.csv_report
+ csv_summary = args.csv_summary
+ csv_image = args.csv_image
+ measure = args.measure[0]
+
+ limits_per_measure = {
+ "MLD": ("MLD", [0, 5, 10, math.inf]),
+ "DIFF": ("MAXIMUM ABS DIFF", [0, 1024, 16384, 32769]),
+ "SSNR": ("MIN_SSNR", [-math.inf, 0, 20, 40, 60, 100]),
+ }
+ (measure_label, limits) = limits_per_measure[measure]
+
+ # Load CSV report
+ results_sorted = {}
+ with open(csv_report, "r") as fp:
+ reader = csv.reader(fp, delimiter=";")
+ header = next(reader)
+ keys = header[1:]
+ for row in reader:
+ testcase = row[0]
+ results_sorted[testcase] = {}
+ for k, val in zip(keys, row[1:]):
+ results_sorted[testcase][k] = val
+
+ # Output CSV file
+ with open(csv_summary, "w") as fp:
+ limits_labels = [
+ f"{str(a)} --\n {str(b)}" for (a, b) in zip(limits[0:-1], limits[1:])
+ ] + ["None"]
+ # Zero difference is treated as a special category for MLD and MAXIMUM ABS DIFF
+ if measure_label == "MLD" or measure_label == "MAXIMUM ABS DIFF":
+ limits_labels = ["0"] + limits_labels
+ headerline = f"Format;Category;" + ";".join(limits_labels) + "\n"
+ fp.write(headerline)
+
+ for fmt in FORMATS:
+ fig, ax = plt.subplots()
+ bottom = np.zeros(len(limits_labels))
+ for cat in CATEGORIES:
+ values = [
+ x
+ for x in [
+ m[measure_label]
+ for m in results_sorted.values()
+ if m["Format"] == fmt and m["Category"] == cat
+ ]
+ ]
+ # Zero difference is treated as a special category for MLD and MAXIMUM ABS DIFF
+ if measure_label == "MLD" or measure_label == "MAXIMUM ABS DIFF":
+ val = [
+ float(x) for x in values if x != "None" and x != "0" and x != ""
+ ]
+ zero = [sum([1 for x in values if x == "0"])]
+ none = [sum([1 for x in values if x == "None" or x == ""])]
+ else:
+ val = [float(x) for x in values if x != "None" and x != ""]
+ zero = []
+ none = [sum([1 for x in values if x == "None" or x == ""])]
+ hist, _ = np.histogram(val, limits)
+ data = np.array(zero + list(hist) + none)
+
+ # CSV output
+ line = f"{fmt};{cat};{'; '.join(map(str,data))}\n"
+ fp.write(line)
+
+ # Matplotlib histogram
+ ax.bar(limits_labels, data, 0.5, label=cat, bottom=bottom)
+ bottom += data
+
+ # Histogram layout
+ ax.set_title(fmt)
+ ax.legend(loc="best")
+ ax.set_xlabel(measure_label)
+ ax.set_ylabel("Number of test cases")
+
+ fig.set_figheight(4)
+ fig.set_figwidth(6)
+ if csv_image:
+ base, ext = os.path.splitext(csv_image)
+ plt.savefig(f"{base}_{fmt}{ext}")
diff --git a/scripts/parse_xml_report.py b/scripts/parse_xml_report.py
index 4b61dc1a9499aa83fd72eb47b74e2e1ebde65566..3bc0b619ab19b9f9e97be55f0289797035c22ab5 100644
--- a/scripts/parse_xml_report.py
+++ b/scripts/parse_xml_report.py
@@ -12,33 +12,52 @@ Parse a junit report and create a summary report.
PROPERTIES = ["MLD", "MAXIMUM ABS DIFF", "MIN_SSNR"]
+FORMATS = {
+ "Stereo": r"stereo",
+ "ISM": r"ISM",
+ "Multichannel": r"Multi-channel",
+ "MASA": r"(?