diff --git a/ci/basop-pages/create_report_pages.py b/ci/basop-pages/create_report_pages.py index 2f01db7bdec36df2a5748b76b88035792254a091..41ee81af1bb51ff41f63f80e79f72d84c6f77d57 100644 --- a/ci/basop-pages/create_report_pages.py +++ b/ci/basop-pages/create_report_pages.py @@ -37,6 +37,7 @@ Comparing:

Summary page

+

Split comparison summary page



diff --git a/ci/basop-pages/create_summary_page.py b/ci/basop-pages/create_summary_page.py index 1c2b46480213f24913e7288aa2659a828f1ec1e6..32855fce251b418a24394210f77050827acdb6b7 100644 --- a/ci/basop-pages/create_summary_page.py +++ b/ci/basop-pages/create_summary_page.py @@ -4,11 +4,11 @@ from typing import List from create_report_pages import SUBPAGE_TMPL_CSS, FORMATS -title = { +TITLE_4_MEASURE = { "MLD": "Maximum MLD across channels", - "DIFF": "Maximim absolute difference across channels", - "SSNR": "Minimum SSNR across channels", - "ODG": "Minimum PEAQ ODG across channels", + "MAX_ABS_DIFF": "Maximum absolute difference across channels", + "MIN_SSNR": "Minimum SSNR across channels", + "MIN_ODG": "Minimum PEAQ ODG across channels", "DELTA_ODG": "PEAQ ODG using binauralized input and output", } @@ -16,9 +16,13 @@ SUMMARY_PAGE_TMPL_HTML = """

Summary for job {job_name}, ID: {id_current}

+
+ {images} """ +IMAGE_HTML_TMPL = "" +SUBHEADING_HTML_TMP = "

{subtitle}

\n" def create_summary_page( @@ -26,53 +30,50 @@ def create_summary_page( id_current: int, job_name: str, measures: List[str], + image_dir: str, ): - images = histogram_summary(job_name, measures) + html = "\n
\n".join( + [ + SUBHEADING_HTML_TMP.format(subtitle=TITLE_4_MEASURE[m]) + + " ".join( + [ + IMAGE_HTML_TMPL.format(measure=m, format=f, image_dir=image_dir) + for f in FORMATS + ] + ) + for m in measures + ] + ) new_summary_page = SUBPAGE_TMPL_CSS + SUMMARY_PAGE_TMPL_HTML.format( id_current=id_current, job_name=job_name, - images=images, + images=html, ) with open(html_out, "w") as f: f.write(new_summary_page) -def histogram_summary( - job_name: str, - measures: List[str], -): - images = "
" - for m in measures: - images += ( - f"

{title[m]}

\n" - + " ".join( - [f"" for x in FORMATS] - ) - + f'\n
summary_{m}.csv
\n\n' - ) - return images - - if __name__ == "__main__": parser = argparse.ArgumentParser() parser.add_argument("html_out") parser.add_argument("id_current", type=int) parser.add_argument("job_name") + parser.add_argument("image_dir") parser.add_argument( "--measures", nargs="+", - help=f"List of measures to include in summary. Allowed values: {' '.join(title.keys())}", - default=["MLD", "DIFF", "SSNR", "ODG"], + help=f"List of measures to include in summary. Allowed values: {' '.join(TITLE_4_MEASURE.keys())}", + # exclude DELTA_ODG here + default=list(TITLE_4_MEASURE.keys())[:-1], ) args = parser.parse_args() - if not all([m in title for m in args.measures]): - raise ValueError(f"Invalid list of measures: {args.measures}, expected one of {' '.join(title.keys())}") + if not all([m in TITLE_4_MEASURE for m in args.measures]): + raise ValueError( + f"Invalid list of measures: {args.measures}, expected one of {' '.join(TITLE_4_MEASURE.keys())}" + ) create_summary_page( - args.html_out, - args.id_current, - args.job_name, - args.measures, + args.html_out, args.id_current, args.job_name, args.measures, args.image_dir ) diff --git a/scripts/batch_comp_audio.py b/scripts/batch_comp_audio.py index 7372857edb9ac5e4e964228adef6eaed5aa8f136..32ab7bebb04ee181c530fd7603e82b91e80a7a21 100755 --- a/scripts/batch_comp_audio.py +++ b/scripts/batch_comp_audio.py @@ -1,33 +1,33 @@ #!/usr/bin/env python3 """ - (C) 2022-2025 IVAS codec Public Collaboration with portions copyright Dolby International AB, Ericsson AB, - Fraunhofer-Gesellschaft zur Foerderung der angewandten Forschung e.V., Huawei Technologies Co. LTD., - Koninklijke Philips N.V., Nippon Telegraph and Telephone Corporation, Nokia Technologies Oy, Orange, - Panasonic Holdings Corporation, Qualcomm Technologies, Inc., VoiceAge Corporation, and other - contributors to this repository. All Rights Reserved. - - This software is protected by copyright law and by international treaties. - The IVAS codec Public Collaboration consisting of Dolby International AB, Ericsson AB, - Fraunhofer-Gesellschaft zur Foerderung der angewandten Forschung e.V., Huawei Technologies Co. LTD., - Koninklijke Philips N.V., Nippon Telegraph and Telephone Corporation, Nokia Technologies Oy, Orange, - Panasonic Holdings Corporation, Qualcomm Technologies, Inc., VoiceAge Corporation, and other - contributors to this repository retain full ownership rights in their respective contributions in - the software. This notice grants no license of any kind, including but not limited to patent - license, nor is any license granted by implication, estoppel or otherwise. - - Contributors are required to enter into the IVAS codec Public Collaboration agreement before making - contributions. - - This software is provided "AS IS", without any express or implied warranties. The software is in the - development stage. It is intended exclusively for experts who have experience with such software and - solely for the purpose of inspection. All implied warranties of non-infringement, merchantability - and fitness for a particular purpose are hereby disclaimed and excluded. - - Any dispute, controversy or claim arising under or in relation to providing this software shall be - submitted to and settled by the final, binding jurisdiction of the courts of Munich, Germany in - accordance with the laws of the Federal Republic of Germany excluding its conflict of law rules and - the United Nations Convention on Contracts on the International Sales of Goods. +(C) 2022-2025 IVAS codec Public Collaboration with portions copyright Dolby International AB, Ericsson AB, +Fraunhofer-Gesellschaft zur Foerderung der angewandten Forschung e.V., Huawei Technologies Co. LTD., +Koninklijke Philips N.V., Nippon Telegraph and Telephone Corporation, Nokia Technologies Oy, Orange, +Panasonic Holdings Corporation, Qualcomm Technologies, Inc., VoiceAge Corporation, and other +contributors to this repository. All Rights Reserved. + +This software is protected by copyright law and by international treaties. +The IVAS codec Public Collaboration consisting of Dolby International AB, Ericsson AB, +Fraunhofer-Gesellschaft zur Foerderung der angewandten Forschung e.V., Huawei Technologies Co. LTD., +Koninklijke Philips N.V., Nippon Telegraph and Telephone Corporation, Nokia Technologies Oy, Orange, +Panasonic Holdings Corporation, Qualcomm Technologies, Inc., VoiceAge Corporation, and other +contributors to this repository retain full ownership rights in their respective contributions in +the software. This notice grants no license of any kind, including but not limited to patent +license, nor is any license granted by implication, estoppel or otherwise. + +Contributors are required to enter into the IVAS codec Public Collaboration agreement before making +contributions. + +This software is provided "AS IS", without any express or implied warranties. The software is in the +development stage. It is intended exclusively for experts who have experience with such software and +solely for the purpose of inspection. All implied warranties of non-infringement, merchantability +and fitness for a particular purpose are hereby disclaimed and excluded. + +Any dispute, controversy or claim arising under or in relation to providing this software shall be +submitted to and settled by the final, binding jurisdiction of the courts of Munich, Germany in +accordance with the laws of the Federal Republic of Germany excluding its conflict of law rules and +the United Nations Convention on Contracts on the International Sales of Goods. """ import argparse @@ -95,7 +95,7 @@ def main(args): repeat(fol2), repeat(outputs), repeat(tool), - repeat(test_offset_ms) + repeat(test_offset_ms), ) if args.sort: @@ -164,10 +164,17 @@ def compare_files(f, fol1, fol2, outputs_dict, tool, test_offset_ms): s2, fs2 = readfile(f2, outdtype="int16") cmp_result = compare(s1, s2, fs1, per_frame=False, get_mld=True) tool_output = cmp_result["MLD"] + elif tool == "ssnr": + s1, fs1 = readfile(f1, outdtype="int16") + s2, fs2 = readfile(f2, outdtype="int16") + cmp_result = compare(s1, s2, fs1, per_frame=False, get_ssnr=True) + tool_output = cmp_result["SSNR"] elif tool == "pyaudio3dtools": s1, fs1 = readfile(f1, outdtype="int16") s2, fs2 = readfile(f2, outdtype="int16") - cmp_result = compare(s1, s2, fs1, per_frame=False, test_start_offset_ms=test_offset_ms) + cmp_result = compare( + s1, s2, fs1, per_frame=False, test_start_offset_ms=test_offset_ms + ) tool_output = cmp_result["max_abs_diff"] with threading.Lock(): @@ -298,16 +305,16 @@ if __name__ == "__main__": ) parser.add_argument( "--tool", - choices=["mld", "CompAudio", "pyaudio3dtools"], + choices=["mld", "CompAudio", "pyaudio3dtools", "ssnr"], default="CompAudio", help="Compare tool to run", ) parser.add_argument( - "--test_offset_ms", - type=int, - default=0, - help="Offset in miliseconds that is ignored at the start of the files in folder2 (only used if tool=pyaudio3dtools)" - ) + "--test_offset_ms", + type=int, + default=0, + help="Offset in miliseconds that is ignored at the start of the files in folder2 (only used if tool=pyaudio3dtools)", + ) args = parser.parse_args() sys.exit(main(args)) diff --git a/scripts/create_histogram_summary.py b/scripts/create_histogram_summary.py deleted file mode 100644 index af9a11de2ab48e10f897301da566b826ca06c0c7..0000000000000000000000000000000000000000 --- a/scripts/create_histogram_summary.py +++ /dev/null @@ -1,156 +0,0 @@ -#!/usr/bin/env python3 - -import argparse -import math -import numpy as np - -# These next three lines are added as a precaution in case the gitlab runner -# needs DISPLAY to render the plots, even if they are written to file. -import matplotlib - -matplotlib.use("Agg") -import matplotlib.pyplot as plt -import csv -import os -from parse_xml_report import IVAS_FORMATS, EVS_FORMATS, IVAS_CATEGORIES, EVS_CATEGORIES - -""" -Parses a CSV report and creates a summary report. -""" - - -# Main routine -if __name__ == "__main__": - parser = argparse.ArgumentParser( - description="Parses a CSV report and creates a summary report." - ) - parser.add_argument( - "csv_report", - type=str, - help="CSV report file of test cases, e.g. report.csv", - ) - parser.add_argument( - "csv_summary", type=str, help="Output CSV file, e.g. summary.csv" - ) - parser.add_argument( - "csv_image", - type=str, - nargs="?", - help="Summary image file, e.g. summary.png", - default=None, - ) - parser.add_argument( - "--measure", - type=str, - nargs=1, - help="Measure, any of: MLD, DIFF, SSNR, ODG, default: MLD", - default=["MLD"], - ) - parser.add_argument( - "--evs", - action="store_true", - help="Parse using EVS 26.444 formats", - default=False, - ) - parser.add_argument( - "--diff", - action="store_true", - help="Use limits for diff scores", - default=False, - ) - args = parser.parse_args() - csv_report = args.csv_report - csv_summary = args.csv_summary - csv_image = args.csv_image - measure = args.measure[0] - if args.evs: - FORMATS = EVS_FORMATS - CATEGORIES = EVS_CATEGORIES - else: - FORMATS = IVAS_FORMATS - CATEGORIES = IVAS_CATEGORIES - if args.diff: - limits_per_measure = { - "MLD": ("MLD", None), - "DIFF": ("MAXIMUM ABS DIFF", None), - "SSNR": ("MIN_SSNR", None), - "ODG": ("MIN_ODG", None), - "DELTA_ODG": ("DELTA_ODG", None), - } - else: - limits_per_measure = { - "MLD": ("MLD", [0, 1, 2, 3, 4, 5, 10, 20, math.inf]), - "DIFF": ("MAXIMUM ABS DIFF", [0, 16, 256, 1024, 2048, 4096, 8192, 16384, 32769]), - "SSNR": ("MIN_SSNR", [-math.inf, 0, 10, 20, 30, 40, 40, 50, 60, 100]), - "ODG": ("MIN_ODG", [-5, -2, -1, -0.5, -0.4, -0.3, -0.2, -0.1, 0, 0.1, 0.5]), - "DELTA_ODG": ("DELTA_ODG", [-5, -2, -1, -0.5, -0.4, -0.3, -0.2, -0.1, 0, 0.1, 0.5]), - } - (measure_label, limits) = limits_per_measure[measure] - - # Load CSV report - results_sorted = {} - with open(csv_report, "r") as fp: - reader = csv.reader(fp, delimiter=";") - header = next(reader) - keys = header[1:] - for row in reader: - testcase = row[0] - results_sorted[testcase] = {} - for k, val in zip(keys, row[1:]): - results_sorted[testcase][k] = val - - if limits is None: - vals = [float(x) for x in [m[measure_label] for m in results_sorted.values() if m[measure_label] != "None" and m[measure_label] != ""]] - start = min(vals) - f = 10 ** (2 - int(np.floor(np.log10(abs(start)))) - 1) - start = np.floor(start*f)/f - step = (max(vals) - start)/10 - f = 10 ** (2 - int(np.floor(np.log10(abs(step)))) - 1) - step = np.ceil(step*f)/f - limits = np.arange(start, 10*step, step) - - # Output CSV file - with open(csv_summary, "w") as fp: - limits_labels = [f"{a:g}" for a in limits] + ["","None"] # Put None cases in separate bin - headerline = f"Format;Category;" + ";".join(limits_labels) + "\n" - fp.write(headerline) - - for fmt in FORMATS: - fig, ax = plt.subplots() - bottom = np.zeros(len(limits_labels)) - for cat in CATEGORIES: - values = [ - x - for x in [ - m[measure_label] - for m in results_sorted.values() - if m["Format"] == fmt and m["Category"] == cat - ] - ] - # Create separate bin for None (errors) - val = [float(x) for x in values if x != "None" and x != ""] - none = [sum([1 for x in values if x == "None" or x == ""])] - hist, _ = np.histogram(val, limits) - data = np.array(list(hist) + [0] + none + [0]) - - # CSV output - line = f"{fmt};{cat};{'; '.join(map(str,data))}\n" - fp.write(line) - - # Matplotlib histogram - ax.bar(limits_labels, data, 1, align='edge', edgecolor='black', linewidth=0.5, label=cat, bottom=bottom) - bottom += data - - # Histogram layout - ax.set_title(fmt) - ax.legend(loc="best") - ax.set_xlabel(measure_label) - if "DIFF" in measure_label: - ax.set_xticks(range(len(limits_labels)), limits_labels, rotation=35) - ax.set_ylabel("Number of test cases") - - fig.set_figheight(4) - fig.set_figwidth(6) - if csv_image: - base, ext = os.path.splitext(csv_image) - plt.savefig(f"{base}_{fmt}{ext}") diff --git a/scripts/create_histograms.py b/scripts/create_histograms.py new file mode 100644 index 0000000000000000000000000000000000000000..b2a9f0ec1c810de755e05e95433e3897320fff40 --- /dev/null +++ b/scripts/create_histograms.py @@ -0,0 +1,200 @@ +#!/usr/bin/env python3 + +import argparse +import math +import pathlib +import sys +import pandas as pd +import numpy as np +from typing import List + + +# hack for avoiding missing DISPLAY variable in headless CI runners +import matplotlib + +matplotlib.use("Agg") +import matplotlib.pyplot as plt + + +BINS_FOR_MEASURES = { + "MLD": [0, 1, 2, 3, 4, 5, 10, 20, math.inf], + "MAX_ABS_DIFF": [0, 16, 256, 1024, 2048, 4096, 8192, 16384, 32769], + "MIN_SSNR": [-math.inf, 0, 10, 20, 30, 40, 40, 50, 60, 100, math.inf], + "MIN_ODG": [-5, -4, -3, -2, -1, -0.5, -0.4, -0.3, -0.2, -0.1, 0, 0.1, 0.5], + "DELTA_ODG": [-5, -4, -3, -2, -1, -0.5, -0.4, -0.3, -0.2, -0.1, 0, 0.1, 0.5], +} + +DEFAULT_MEASURES = ["MAX_ABS_DIFF", "MLD", "MIN_SSNR", "MIN_ODG"] + +### !!! Note: this is duplicated in tests/constatns.py. If you change this here, ALSO ADAPT IT THERE!!! +### (importing from there failed for unknown reasons in some jobs on some runners and I don't have time to properly investigate this...) +### below lines are the original solution, kept here for reference + +# HERE = pathlib.Path(__file__).parent +# ROOT_DIR = HERE.parent +# sys.path.append(str(ROOT_DIR)) +# from tests.constants import CAT_NORMAL, CAT_BITRATE_SWITCHING, CAT_DTX, CAT_JBM, CAT_PLC + +CAT_NORMAL = "normal operation" +CAT_DTX = "DTX" +CAT_PLC = "PLC" +CAT_BITRATE_SWITCHING = "bitrate switching" +CAT_JBM = "JBM" + +COLORS_FOR_CATEGORIES = { + CAT_DTX: "tab:blue", + CAT_PLC: "tab:orange", + CAT_NORMAL: "tab:green", + CAT_JBM: "tab:red", + CAT_BITRATE_SWITCHING: "tab:purple", +} + + +def get_bins_for_diff(data: pd.Series): + return np.round(np.linspace(data.min(), data.max(), num=10), decimals=2) + + +def create_histograms( + df: pd.DataFrame, + measures: List[str], + output_folder: pathlib.Path, + display_only: bool, + bins_for_measures=BINS_FOR_MEASURES, + prefix="", + write_out_histograms=False, +): + formats = df["format"].unique() + categories = df["category"].unique() + + if not display_only or write_out_histograms: + output_folder.mkdir(exist_ok=True, parents=True) + + for measure in measures: + measure_in_df = prefix + measure + bins = bins_for_measures.get(measure, get_bins_for_diff(df[measure_in_df])) + x = [f"{x}" for x in bins] + ["", "ERROR"] + + df_hist = pd.DataFrame(columns=["format", "category"] + x) + hist_row_count = 0 + + for fmt in formats: + fig, ax = plt.subplots() + ax.xaxis.set_major_formatter("{x:.1f}") + bottom = np.zeros(len(x)) + for cat in categories: + data_mask = np.logical_and(df["format"] == fmt, df["category"] == cat) + df_slice = df[data_mask] + error_mask = df_slice["result"] == "ERROR" + n_errors = np.sum(error_mask) + df_slice = df_slice[np.logical_not(error_mask)] + + counts, _ = np.histogram(df_slice[measure_in_df], bins) + + data = np.concatenate([counts, [0], [n_errors], [0]]) + ax.bar( + x, + data, + 1, + align="edge", + edgecolor="black", + linewidth=0.5, + label=cat, + bottom=bottom, + color=COLORS_FOR_CATEGORIES[cat], + ) + bottom += data + + hist_row = [fmt, cat] + list(counts) + [0] + [0, n_errors] + df_hist.loc[hist_row_count] = hist_row + hist_row_count += 1 + + # Histogram layout + ax.set_title(fmt) + ax.legend(loc="best") + ax.set_xlabel(measure) + if "DIFF" in measure or len(bins_for_measures) == 0: + ax.set_xticks(range(len(x)), x, rotation=35) + else: + ax.set_xticks(range(len(x)), x) + ax.set_ylabel("Number of test cases") + + fig.set_figheight(4) + fig.set_figwidth(6) + plt.tight_layout() + + if not display_only: + image_file = f"histogram_{measure}_{fmt}.png" + image_path = output_folder.joinpath(image_file) + plt.savefig(image_path) + plt.close(fig) + + if write_out_histograms: + df_hist.to_csv( + output_folder.joinpath(f"histogram_{measure}.csv"), index=False + ) + + if display_only: + plt.show() + + +if __name__ == "__main__": + parser = argparse.ArgumentParser( + description="Parses a csv file generated by parse_xml_report and creates histograms for the given measures." + ) + parser.add_argument( + "csv_report", + type=str, + help="CSV report file as generated by parse_xml_report.py", + ) + parser.add_argument( + "output_folder", type=pathlib.Path, help="Output folder for writing the " + ) + parser.add_argument( + "--display-only", + action="store_true", + help="Do not write the output files, but display the graphs instead.", + ) + parser.add_argument( + "--no-bins", + action="store_true", + help="""Do not use the hardcoded bins for creating the spectrograms. +Use this for visualising diff scores.""", + ) + allowed_measures = " ".join(BINS_FOR_MEASURES.keys()) + parser.add_argument( + "--measures", + nargs="+", + default=DEFAULT_MEASURES, + help=f"Measures to plot from the csv file. One of {allowed_measures}", + ) + parser.add_argument( + "--prefix", + default="", + help="Common suffix to use when collecting measures from the input csv file", + ) + parser.add_argument( + "--write-out-histograms", + action="store_true", + help="Write out the histogram values to csv", + ) + args = parser.parse_args() + df = pd.read_csv(args.csv_report) + + # filter out missing format/category values + mask_format_missing = df["format"].isna() + mask_category_missing = df["category"].isna() + df = df[~mask_format_missing | ~mask_category_missing] + + bins_for_measures = BINS_FOR_MEASURES + if args.no_bins: + bins_for_measures = {} + + create_histograms( + df, + args.measures, + args.output_folder, + args.display_only, + bins_for_measures, + args.prefix, + args.write_out_histograms, + ) diff --git a/scripts/diff_report.py b/scripts/diff_report.py index 5ab64e956fc65bedeefb39137ebc474a837dd860..0aefeaccdcb251eb1426c63d967dda23e5dec378 100644 --- a/scripts/diff_report.py +++ b/scripts/diff_report.py @@ -33,25 +33,29 @@ the United Nations Convention on Contracts on the International Sales of Goods. import pandas as pd import argparse import sys -import os -import pathlib COLUMNS_TO_COMPARE = [ "MLD", - "MAXIMUM ABS DIFF", + "MAX_ABS_DIFF", "MIN_SSNR", "MIN_ODG", ] + def main(args): - df_ref = pd.read_csv(args.csv_ref, sep=";") - df_test = pd.read_csv(args.csv_test, sep=";") + df_ref = pd.read_csv(args.csv_ref).sort_values( + by=["testcase", "format", "category"] + ) + df_test = pd.read_csv(args.csv_test).sort_values( + by=["testcase", "format", "category"] + ) for col in COLUMNS_TO_COMPARE: df_ref[col] = df_test[col] - df_ref[col] - df_ref.to_csv(args.csv_diff, index=False, sep=";") + df_ref.to_csv(args.csv_diff, index=False) return 0 + if __name__ == "__main__": parser = argparse.ArgumentParser() parser.add_argument("csv_ref") diff --git a/scripts/parse_xml_report.py b/scripts/parse_xml_report.py index 1f2157ca8f5b6dc30cdd53e9bf080b9a0e8afba3..948bda43086f1145b34babfbbefcf4182d2aaae6 100644 --- a/scripts/parse_xml_report.py +++ b/scripts/parse_xml_report.py @@ -1,186 +1,180 @@ #!/usr/bin/env python3 import argparse -import re -import math -import numpy as np +import pandas as pd from xml.etree import ElementTree +from collections import Counter +from typing import Optional +from enum import Enum + + +SPLIT_STRING = "_split" +WHOLE_STRING = "_whole" + + +class Result(str, Enum): + ERROR = "ERROR" + FAIL = "FAIL" + PASS = "PASS" + + +class TestcaseParser(dict): + def __init__(self, testcases: list): + super().__init__() + + for tc in testcases: + self.parse_testcase(tc) + + def parse_testcase(self, testcase): + """ + Get all properties + name for a testcase + """ + + filename = testcase.get( + "file", testcase.get("classname").replace(".", "/") + ".py" + ) + fulltestname = filename + "::" + testcase.get("name") + + result = get_result_from_testcase(testcase) + # for ERRORS, two testcases are recorded, one with FAIL and one with ERROR + # if we already have this testcase, do a sanity check and set result to ERROR + if fulltestname in self: + results = [self[fulltestname]["result"], result] + assert any(r == Result.ERROR for r in results) + self[fulltestname]["result"] = Result.ERROR + return + + ret = {} + ret["testcase"] = fulltestname + ret["result"] = result + properties = { + p.get("name"): p.get("value") for p in testcase.findall(".//property") + } + + ### handle split comparison results + split_props = {k: v for k, v in properties.items() if SPLIT_STRING in k} + whole_props = {k: v for k, v in properties.items() if WHOLE_STRING in k} + other_props = { + k: v + for k, v in properties.items() + if WHOLE_STRING not in k and SPLIT_STRING not in k + } + + if len(split_props) > 0 and len(whole_props) > 0: + measures_from_split = set( + [m.split(SPLIT_STRING)[0] for m in split_props.keys()] + ) + measures_from_whole = set( + [m.split(WHOLE_STRING)[0] for m in whole_props.keys()] + ) + assert measures_from_split == measures_from_whole + measures = measures_from_whole + + # collect existing split suffixes by evaluating one of the measures only + # get one measure from set and add it back immediately + m_tmp = measures.pop() + measures.add(m_tmp) + + splits = sorted( + [ + k.split(SPLIT_STRING)[-1] + for k in split_props.keys() + if k.startswith(m_tmp) + ] + ) + + # record each split under a separate key + # the dict per key has the same fulltestname and an additional key "split" + # this way, the resulting DataFrame in the end can be split by testnames + for s in splits: + split_key = f"{fulltestname} - {s}" + ret_split = {"testcase": fulltestname, "split": s} + for m in measures: + ret_split.update({m: split_props[m + SPLIT_STRING + f"{s}"]}) + + ret_split.update(other_props) + self[split_key] = ret_split + + # it can be the case that there are no splits defined in the pytest suite, e.g. for the renderer + # then, there are only "_whole" values recorded where we only need to remove the suffix + # this if also handles the split case - if there are splits, there was also a "_whole" comparison done + if len(whole_props) > 0: + properties = { + k.replace(WHOLE_STRING, ""): v for k, v in whole_props.items() + } + properties["split"] = "whole" + properties.update(other_props) + + ret.update(properties) + self[fulltestname] = ret + + def to_df(self) -> pd.DataFrame: + testcases = list(self.values()) + df = pd.DataFrame(testcases) + return df + + +def xml_to_dataframe(xml_report: str) -> pd.DataFrame: + tree = ElementTree.parse(xml_report) + root = tree.getroot() + + testcases = root[0].findall("testcase") + testcases = [tc for tc in testcases if tc.find("skipped") is None] + + testcase_parser = TestcaseParser(testcases) + testcase_df = testcase_parser.to_df() + + return testcase_df + + +def get_result_from_testcase(testcase: ElementTree.Element) -> str: + if testcase.find("failure") is not None: + testresult = Result.FAIL + elif testcase.find("error") is not None: + testresult = Result.ERROR + else: + testresult = Result.PASS + + return testresult + + +def main(xml_report: str, csv_file: str, split_csv_file: Optional[str]): + df = xml_to_dataframe(xml_report) + n_testcases = len(df) + count = Counter(df["result"]) + + if split_csv_file is not None: + mask_errors = df["result"] == Result.ERROR + mask_whole = df["split"] == "whole" + mask_single = mask_errors | mask_whole + df_split = df[~mask_single] + df_split.to_csv(split_csv_file, index=False) + + df = df[mask_single] + + df.to_csv(csv_file, index=False) + + print( + f"Parsed testsuite with {n_testcases} tests: {count[Result.PASS]} passes, {count[Result.FAIL]} failures and {count[Result.ERROR]} errors." + ) + -""" -Parse a junit report and create a summary report. -""" - -PROPERTIES = ["MLD", "MAXIMUM ABS DIFF", "MIN_SSNR", "MIN_ODG"] - -IVAS_FORMATS = { - "Stereo": r"stereo", - "ISM": r"ISM", - "Multichannel": r"Multi-channel|MC", - "MASA": r"(? tuple[int, str]: + scalefac: int = 1, + split_idx: np.ndarray = np.empty(0), +) -> tuple[List[int], List[str]]: """ Compare 2 PCM files for bitexactness """ @@ -67,13 +68,13 @@ def cmp_pcm( if fs1 != fs2: reason = "FAIL: Sampling rate differs." - return 1, reason + return [1], [reason] # In case number of channels do not match, fail already now. Could happen in case of # comparison to input with for a non-passthrough mode. if s1.shape[1] != s2.shape[1]: reason = "FAIL: Number of channels differ." - return 1, reason + return [1], [reason] handle_differing_lengths = "fail" if allow_differing_lengths: @@ -88,108 +89,115 @@ def cmp_pcm( if get_mld: reason += " - MLD: None" - return 1, reason - - # Apply scalefac if specified. Useful in case scaling has been applied on the input, and the inverse is scaling is supplied in scalefac. - if scalefac != 1: - s1 = np.round(s1*scalefac, 0) # Need rounding for max abs diff search - s2 = np.round(s2*scalefac, 0) - - cmp_result = pyaudio3dtools.audioarray.compare( - s1, - s2, - fs, - per_frame=False, - get_mld=get_mld, - get_ssnr=get_ssnr, - ssnr_thresh_low=-50, - ref_jbm_tf=ref_jbm_tf, - test_jbm_tf=cut_jbm_tf, - handle_differing_lengths=handle_differing_lengths, - ) + return [1], [reason] + + output_differs_parts = [] + reason_parts = [] + + for s1, s2 in zip(np.split(s1, split_idx), np.split(s2, split_idx)): + # Apply scalefac if specified. Useful in case scaling has been applied on the input, and the inverse is scaling is supplied in scalefac. + if scalefac != 1: + s1 = np.round(s1 * scalefac, 0) # Need rounding for max abs diff search + s2 = np.round(s2 * scalefac, 0) + + cmp_result = pyaudio3dtools.audioarray.compare( + s1, + s2, + fs, + per_frame=False, + get_mld=get_mld, + get_ssnr=get_ssnr, + ssnr_thresh_low=-50, + ref_jbm_tf=ref_jbm_tf, + test_jbm_tf=cut_jbm_tf, + handle_differing_lengths=handle_differing_lengths, + ) + + output_differs = 0 + reason = "SUCCESS: Files are bitexact" + + if not cmp_result["bitexact"] and cmp_result["max_abs_diff"] <= abs_tol: + reason = "SUCCESS: Maximum absolute diff below threshold" + elif not cmp_result["bitexact"]: + diff_msg = f"MAXIMUM ABS DIFF ==> {cmp_result['max_abs_diff']} at sample num {cmp_result['max_abs_diff_pos_sample']} (assuming {nchannels} channels)" + first_msg = f"First diff found at sample num {cmp_result['first_diff_pos_sample']} in channel {cmp_result['first_diff_pos_channel']}, frame {cmp_result['first_diff_pos_frame']} (assuming {nchannels} channels, {fs} sampling rate)" + print(diff_msg, file=output_target) + print(first_msg, file=output_target) - output_differs = 0 - reason = "SUCCESS: Files are bitexact" - - if not cmp_result["bitexact"] and cmp_result["max_abs_diff"] <= abs_tol: - reason = "SUCCESS: Maximum absolute diff below threshold" - elif not cmp_result["bitexact"]: - diff_msg = f"MAXIMUM ABS DIFF ==> {cmp_result['max_abs_diff']} at sample num {cmp_result['max_abs_diff_pos_sample']} (assuming {nchannels} channels)" - first_msg = f"First diff found at sample num {cmp_result['first_diff_pos_sample']} in channel {cmp_result['first_diff_pos_channel']}, frame {cmp_result['first_diff_pos_frame']} (assuming {nchannels} channels, {fs} sampling rate)" - print(diff_msg, file=output_target) - print(first_msg, file=output_target) - - reason = f"Non-BE - MAXIMUM ABS DIFF: {cmp_result['max_abs_diff']}" - output_differs = 1 - - if get_mld: - mld_msg = f"MLD: {cmp_result['MLD']}" - reason += " - " + mld_msg - print(mld_msg, file=output_target) - - if cmp_result["MLD"] <= mld_lim: - output_differs = 0 - reason += f" <= {mld_lim}" - else: - reason += f" > {mld_lim}" - - if get_ssnr: - reason += " - " - for i, s in enumerate(cmp_result["SSNR"], start=1): - msg = f"Channel {i} SSNR: {s}" - reason += msg + " - " - - if get_odg: - for n in range(nchannels): - pqeval_output = pqevalaudio_wrapper(s1[:, n], s2[:, n], fs) + reason = f"Non-BE - MAXIMUM ABS DIFF: {cmp_result['max_abs_diff']}" + output_differs = 1 + if get_mld: + mld_msg = f"MLD: {cmp_result['MLD']}" + reason += " - " + mld_msg + print(mld_msg, file=output_target) + + if cmp_result["MLD"] <= mld_lim: + output_differs = 0 + reason += f" <= {mld_lim}" + else: + reason += f" > {mld_lim}" + + if get_ssnr: + reason += " - " + for i, s in enumerate(cmp_result["SSNR"], start=1): + msg = f"Channel {i} SSNR: {s}" + reason += msg + " - " + + if get_odg: + for n in range(nchannels): + pqeval_output = pqevalaudio_wrapper(s1[:, n], s2[:, n], fs) + + match_odg = re.search(ODG_PATTERN_PQEVALAUDIO, pqeval_output) + odg = float(match_odg.groups()[0]) + msg = f"Channel {n} ODG: {odg}" + reason += " - " + msg + print(msg) + + if get_odg_bin: + odg_files = {} + for f in [odg_input, odg_test, odg_ref]: + # Load PEAQ test files and ensure 48 kHz sampling rate + s, fs = pyaudio3dtools.audiofile.readfile( + f, nchannels, fs, outdtype=np.int16 + ) + odg_files[f] = np.clip( + pyaudio3dtools.audioarray.resample(s.astype(float), fs, 48000), + -32768, + 32767, + ).astype(np.int16) + + pqeval_output = pqevalaudio_wrapper( + odg_files[odg_input], odg_files[odg_ref], 48000 + ) match_odg = re.search(ODG_PATTERN_PQEVALAUDIO, pqeval_output) - odg = float(match_odg.groups()[0]) - msg = f"Channel {n} ODG: {odg}" - reason += " - " + msg - print(msg) - - if get_odg_bin: - odg_files = {} - for f in [odg_input, odg_test, odg_ref]: - # Load PEAQ test files and ensure 48 kHz sampling rate - s, fs = pyaudio3dtools.audiofile.readfile( - f, nchannels, fs, outdtype=np.int16 + try: + odg_ref = float(match_odg.groups()[0]) + except AttributeError: + raise OdgParsingFailed("Could not get Odg for ref signal") + + pqeval_output = pqevalaudio_wrapper( + odg_files[odg_input], odg_files[odg_test], 48000 ) - odg_files[f] = np.clip( - pyaudio3dtools.audioarray.resample(s.astype(float), fs, 48000), - -32768, - 32767, - ).astype(np.int16) - - pqeval_output = pqevalaudio_wrapper( - odg_files[odg_input], odg_files[odg_ref], 48000 - ) - match_odg = re.search(ODG_PATTERN_PQEVALAUDIO, pqeval_output) - try: - odg_ref = float(match_odg.groups()[0]) - except AttributeError: - raise OdgParsingFailed("Could not get Odg for ref signal") - - pqeval_output = pqevalaudio_wrapper( - odg_files[odg_input], odg_files[odg_test], 48000 - ) - match_odg = re.search(ODG_PATTERN_PQEVALAUDIO, pqeval_output) - try: - odg_test = float(match_odg.groups()[0]) - except AttributeError: - raise OdgParsingFailed("Could not get Odg for test signal") + match_odg = re.search(ODG_PATTERN_PQEVALAUDIO, pqeval_output) + try: + odg_test = float(match_odg.groups()[0]) + except AttributeError: + raise OdgParsingFailed("Could not get Odg for test signal") + + odg = odg_test - odg_ref # Todo: store both rather than difference? - odg = odg_test - odg_ref # Todo: store both rather than difference? + msg = f"Delta-ODG: {odg}" + reason += " - " + msg + print(msg, file=output_target) - msg = f"Delta-ODG: {odg}" - reason += " - " + msg - print(msg, file=output_target) + if quiet: + output_target.close() - if quiet: - output_target.close() + output_differs_parts.append(output_differs) + reason_parts.append(reason) - return output_differs, reason + return output_differs_parts, reason_parts class OdgParsingFailed(Exception): @@ -262,12 +270,18 @@ if __name__ == "__main__": parser.add_argument("--get_odg", action="store_true") parser.add_argument("--get_ssnr", action="store_true") parser.add_argument("--allow_differing_lengths", action="store_true") - parser.add_argument("--scalefac", type=float, default=1, dest="scalefac", help="Scale factor to be applied before comparing the output. Useful when input scaling has been applied.") + parser.add_argument( + "--scalefac", + type=float, + default=1, + dest="scalefac", + help="Scale factor to be applied before comparing the output. Useful when input scaling has been applied.", + ) parser.add_argument("--quiet", action="store_true") args = vars(parser.parse_args()) args["nchannels"] = out_config_2_nchannels(args.pop("out_config")) result, msg = cmp_pcm(**args) - print(msg) - sys.exit(result) + print(msg[0]) + sys.exit(result[0]) diff --git a/tests/codec_be_on_mr_nonselection/test_param_file.py b/tests/codec_be_on_mr_nonselection/test_param_file.py index ac2d84d1e7af645500434e259bb2337f341c7f85..d2acacd39eacbb6513f0e01b5804f1ac6990c659 100644 --- a/tests/codec_be_on_mr_nonselection/test_param_file.py +++ b/tests/codec_be_on_mr_nonselection/test_param_file.py @@ -53,6 +53,8 @@ from tests.conftest import ( parse_properties, compare_dmx_signals, log_dbg_msg, + get_split_idx, + get_format_from_enc_opts, ) from tests.testconfig import PARAM_FILE from tests.constants import ( @@ -60,6 +62,11 @@ from tests.constants import ( MAX_ENC_STATS_DIFF, SCRIPTS_DIR, MAX_ENC_DIFF, + CAT_NORMAL, + CAT_DTX, + CAT_BITRATE_SWITCHING, + CAT_JBM, + CAT_PLC, ) from tests.renderer.utils import check_and_makedir, binauralize_input_and_output @@ -217,6 +224,7 @@ def test_param_file_tests( get_odg_bin, compare_to_input, compare_enc_dmx, + split_comparison, ): enc_opts, dec_opts, sim_opts, eid_opts = param_file_test_dict[test_tag] @@ -249,6 +257,7 @@ def test_param_file_tests( get_odg_bin, compare_to_input, compare_enc_dmx, + split_comparison, ) @@ -281,6 +290,7 @@ def run_test( get_odg_bin, compare_to_input, compare_enc_dmx, + split_comparison, ): # If compare_to_input is set, only run pass-through test cases if compare_to_input: @@ -294,6 +304,10 @@ def run_test( "All non-passthrough modes are skipped when --compare-to-input is set" ) + testcase_props = {} + testcase_props["format"] = get_format_from_enc_opts(enc_opts) + testcase_props["category"] = CAT_NORMAL + tag_str = convert_test_string_to_tag(test_tag) # evaluate encoder options @@ -313,6 +327,9 @@ def run_test( bitrate = enc_split.pop() in_sr = sampling_rate + if "-dtx" in enc_opts: + testcase_props["category"] = CAT_DTX + # bitrate can be a filename: change it to an absolute path if not bitrate.isdigit(): if compare_enc_dmx: @@ -320,11 +337,21 @@ def run_test( "Rate switching + --compare_enc_dmx currently skipped due to DEBUGGING code limitations with varying number of transport channels" ) bitrate = Path(bitrate[3:]).absolute() + testcase_props["category"] = CAT_BITRATE_SWITCHING testv_base = testv_file.split("/")[-1] if testv_base.endswith(".pcm"): testv_base = testv_base[:-4] + if sim_opts != "": + testcase_props["category"] = CAT_JBM + if eid_opts != "": + testcase_props["category"] = CAT_PLC + + if update_ref != 1: + for k, v in testcase_props.items(): + dut_encoder_frontend.record_property(k, v) + assert bitstream_file == "bit" # in the parameter file, only "bit" is used as bitstream file name # -> construct bitstream filename @@ -371,8 +398,8 @@ def run_test( # avoid double recording of the encoder diff if encoder_only: - props = parse_properties(cmp_result_msg, False, [MAX_ENC_DIFF]) - for k, v in props.items(): + result_props = parse_properties(cmp_result_msg, False, [MAX_ENC_DIFF]) + for k, v in result_props.items(): dut_encoder_frontend.record_property(k, v) if encoder_only: @@ -413,7 +440,6 @@ def run_test( ) # check for eid-xor command line - if eid_opts != "": eid_split = eid_opts.split() assert len(eid_split) >= 3, "eid-xor expects at least 3 parameters" @@ -588,7 +614,14 @@ def run_test( ref_file = ref_output_file fs = int(sampling_rate) * 1000 - output_differs, reason = cmp_pcm( + + ### run the comparison tools + split_idx = np.empty(0) + prop_suffix = [""] + + # 1. run comparison on whole files - this is done always, regardless of the presence of --split_comparison + + output_differs_parts, reason_parts = cmp_pcm( ref_file, dut_output_file, out_config_2_nchannels(output_config), @@ -606,13 +639,55 @@ def run_test( ref_jbm_tf=ref_tracefile_dec, cut_jbm_tf=dut_tracefile_dec, scalefac=test_info.config.option.scalefac, + split_idx=split_idx, ) - cmp_result_msg += reason + # 2. run comparison on split files if --split_comparison is given + # for JBM cases, comparison will fail because of length mismatch beetween split wav files and tracefiles + # -> skip split comparison for these cases + if split_comparison and not sim_opts: + split_idx = get_split_idx(str(Path(testv_file).stem), int(sampling_rate)) + + # this extra if takes care of cases where no splits are found, e.g. the "NOOP" case in the self_test_ltv prm file + # if this would not be there, then the comparison of the whole file would run twice + if len(split_idx) > 0: + output_differs_splits, reason_splits = cmp_pcm( + ref_file, + dut_output_file, + out_config_2_nchannels(output_config), + fs, + get_mld=get_mld, + mld_lim=get_mld_lim, + abs_tol=abs_tol, + allow_differing_lengths=allow_differing_lengths, + get_ssnr=get_ssnr, + get_odg=get_odg, + get_odg_bin=get_odg_bin, + odg_input=odg_input, + odg_test=odg_test, + odg_ref=odg_ref, + ref_jbm_tf=ref_tracefile_dec, + cut_jbm_tf=dut_tracefile_dec, + scalefac=test_info.config.option.scalefac, + split_idx=split_idx, + ) + output_differs_parts += output_differs_splits + reason_parts += reason_splits - props = parse_properties(cmp_result_msg, output_differs, props_to_record) - for k, v in props.items(): - dut_decoder_frontend.record_property(k, v) + # separate if to also record the whole-file comparison for JBM cases with "_whole" + if split_comparison: + prop_suffix = ["_whole"] + [ + f"_split{i:03d}" for i in range(1, len(split_idx) + 1) + ] + + for output_differs, reason, suffix in zip( + output_differs_parts, reason_parts, prop_suffix + ): + result_props = parse_properties( + reason, output_differs, props_to_record, suffix + ) + for k, v in result_props.items(): + dut_decoder_frontend.record_property(k, v) metadata_differs = False @@ -647,19 +722,22 @@ def run_test( if enc_test_result: pytest.fail("Too high difference in encoder statistics found.") + at_least_one_output_differs = any(output_differs_parts) + # TODO + reason = reason_parts[0] if tracefile_last_rtp_numbers_differ: pytest.fail( "Last RTP sequence num in tracefiles differ for JBM decoding - Not all frames were decoded in both ref and dut." ) elif get_mld and get_mld_lim > 0: - if output_differs: + if at_least_one_output_differs: pytest.fail(reason) else: - if output_differs or metadata_differs: + if at_least_one_output_differs or metadata_differs: msg = "Difference between ref and dut in " - if output_differs and metadata_differs: + if at_least_one_output_differs and metadata_differs: msg += f"output ({reason}) and metadata" - elif output_differs: + elif at_least_one_output_differs: msg += f"output only ({reason})" elif metadata_differs: msg += "metadata only" diff --git a/tests/codec_be_on_mr_nonselection/test_sba.py b/tests/codec_be_on_mr_nonselection/test_sba.py index f0e81e82d2dc71dcb689c066714de02437020f4e..6b1746f858a749059de4160906da9e3ddf97c7cf 100644 --- a/tests/codec_be_on_mr_nonselection/test_sba.py +++ b/tests/codec_be_on_mr_nonselection/test_sba.py @@ -35,6 +35,7 @@ __doc__ = """ import os import pytest +import numpy as np from cut_bs import cut_from_start from pathlib import Path @@ -44,9 +45,18 @@ from tests.conftest import ( EncoderFrontend, compare_dmx_signals, parse_properties, + get_split_idx, ) from ..cmp_stats_files import cmp_stats_files -from ..constants import TESTV_DIR, MAX_ENC_FILE_LENGTH_DIFF, MAX_ENC_STATS_DIFF +from ..constants import ( + CAT_BITRATE_SWITCHING, + CAT_DTX, + CAT_NORMAL, + CAT_PLC, + TESTV_DIR, + MAX_ENC_FILE_LENGTH_DIFF, + MAX_ENC_STATS_DIFF, +) from tests.testconfig import use_ltv from tests.renderer.utils import check_and_makedir, binauralize_input_and_output @@ -116,6 +126,7 @@ def test_pca_enc( get_enc_stats, compare_to_input, compare_enc_dmx, + split_comparison, ): pca = True bitrate = "256000" @@ -128,6 +139,7 @@ def test_pca_enc( cut_testv = True cut_gain = "1.0" plc_pattern = None + testcase_props = {"format": "SBA", "category": CAT_NORMAL} if "ltv" in tag: tag = f"ltv{sampling_rate}_FOA" @@ -140,6 +152,10 @@ def test_pca_enc( input_config = SBA_FORMAT[abs(int(sba_order))] + if update_ref != 1: + for k, v in testcase_props.items(): + dut_encoder_frontend.record_property(k, v) + if not decoder_only: sba_enc( dut_encoder_frontend, @@ -198,6 +214,7 @@ def test_pca_enc( get_odg=get_odg, get_odg_bin=get_odg_bin, compare_to_input=compare_to_input, + split_comparison=split_comparison, ) @@ -237,6 +254,7 @@ def test_sba_enc_system( get_enc_stats, compare_to_input, compare_enc_dmx, + split_comparison, ): plc_pattern = None pca = False @@ -246,6 +264,7 @@ def test_sba_enc_system( cut_gain = "1.0" plc_pattern = None cut_testv = True + testcase_props = {"format": "SBA", "category": CAT_NORMAL} if dtx == "1" and bitrate not in ["13200", "16400", "24400", "32000", "64000"]: # skip high bitrates for DTX until DTX issue is resolved @@ -283,6 +302,16 @@ def test_sba_enc_system( cut_gain = "1.0" input_config = SBA_FORMAT[abs(int(sba_order))] + if dtx: + testcase_props["category"] = CAT_DTX + try: + int(bitrate) + except ValueError: + testcase_props["category"] = CAT_BITRATE_SWITCHING + + if update_ref != 1: + for k, v in testcase_props.items(): + dut_encoder_frontend.record_property(k, v) if not decoder_only: ref_stats_file, dut_stats_file = sba_enc( @@ -338,6 +367,7 @@ def test_sba_enc_system( props = parse_properties( enc_test_result_msg, enc_test_result != 0, props_to_record ) + props.update(testcase_props) for k, v in props.items(): dut_encoder_frontend.record_property(k, v) @@ -377,6 +407,7 @@ def test_sba_enc_system( get_odg=get_odg, get_odg_bin=get_odg_bin, compare_to_input=compare_to_input, + split_comparison=split_comparison, ) @@ -408,6 +439,7 @@ def test_spar_hoa2_enc_system( get_enc_stats, compare_to_input, compare_enc_dmx, + split_comparison, ): sampling_rate = "48" pca = False @@ -420,6 +452,7 @@ def test_spar_hoa2_enc_system( cut_gain = "1.0" plc_pattern = None cut_testv = False + testcase_props = {"format": "SBA", "category": CAT_NORMAL} if "ltv" in tag: tag = f"ltv{sampling_rate}_HOA2" @@ -429,6 +462,9 @@ def test_spar_hoa2_enc_system( assert 0 input_config = SBA_FORMAT[abs(int(sba_order))] + if update_ref != 1: + for k, v in testcase_props.items(): + dut_encoder_frontend.record_property(k, v) if not decoder_only: ref_stats_file, dut_stats_file = sba_enc( @@ -484,6 +520,7 @@ def test_spar_hoa2_enc_system( props = parse_properties( enc_test_result_msg, enc_test_result != 0, props_to_record ) + props.update(testcase_props) for k, v in props.items(): dut_encoder_frontend.record_property(k, v) @@ -523,6 +560,7 @@ def test_spar_hoa2_enc_system( get_odg=get_odg, get_odg_bin=get_odg_bin, compare_to_input=compare_to_input, + split_comparison=split_comparison, ) @@ -554,6 +592,7 @@ def test_spar_hoa3_enc_system( get_enc_stats, compare_to_input, compare_enc_dmx, + split_comparison, ): sampling_rate = "48" pca = False @@ -566,6 +605,7 @@ def test_spar_hoa3_enc_system( cut_gain = "1.0" plc_pattern = None cut_testv = False + testcase_props = {"format": "SBA", "category": CAT_NORMAL} if "ltv" in tag: tag = f"ltv{sampling_rate}_HOA3" @@ -575,6 +615,9 @@ def test_spar_hoa3_enc_system( assert 0 input_config = SBA_FORMAT[abs(int(sba_order))] + if update_ref != 1: + for k, v in testcase_props.items(): + dut_encoder_frontend.record_property(k, v) if not decoder_only: ref_stats_file, dut_stats_file = sba_enc( @@ -624,6 +667,7 @@ def test_spar_hoa3_enc_system( props = parse_properties( enc_test_result_msg, enc_test_result != 0, props_to_record ) + props.update(testcase_props) for k, v in props.items(): dut_encoder_frontend.record_property(k, v) @@ -663,6 +707,7 @@ def test_spar_hoa3_enc_system( get_odg=get_odg, get_odg_bin=get_odg_bin, compare_to_input=compare_to_input, + split_comparison=split_comparison, ) @@ -698,6 +743,7 @@ def test_sba_enc_BWforce_system( get_enc_stats, compare_to_input, compare_enc_dmx, + split_comparison, ): sid = 0 plc_pattern = None @@ -709,6 +755,7 @@ def test_sba_enc_BWforce_system( cut_testv = False sampling_rate = sample_rate_bw_idx[0] max_bw = sample_rate_bw_idx[1] + testcase_props = {"format": "SBA", "category": CAT_NORMAL} if dtx == "1" and bitrate not in ["32000", "64000"]: # skip high bitrates for DTX until DTX issue is resolved @@ -729,6 +776,16 @@ def test_sba_enc_BWforce_system( assert 0 input_config = SBA_FORMAT[abs(int(sba_order))] + if dtx: + testcase_props["category"] = CAT_DTX + try: + int(bitrate) + except ValueError: + testcase_props["category"] = CAT_BITRATE_SWITCHING + + if update_ref != 1: + for k, v in testcase_props.items(): + dut_encoder_frontend.record_property(k, v) if not decoder_only: ref_stats_file, dut_stats_file = sba_enc( @@ -784,6 +841,7 @@ def test_sba_enc_BWforce_system( props = parse_properties( enc_test_result_msg, enc_test_result != 0, props_to_record ) + props.update(testcase_props) for k, v in props.items(): dut_encoder_frontend.record_property(k, v) @@ -823,6 +881,7 @@ def test_sba_enc_BWforce_system( get_odg=get_odg, get_odg_bin=get_odg_bin, compare_to_input=compare_to_input, + split_comparison=split_comparison, ) @@ -862,6 +921,7 @@ def test_sba_plc_system( get_enc_stats, compare_to_input, compare_enc_dmx, + split_comparison, ): sid = 0 pca = False @@ -869,6 +929,7 @@ def test_sba_plc_system( sba_order = "+1" cut_testv = True output_config = "FOA" + testcase_props = {"format": "SBA", "category": CAT_NORMAL} if dtx == "1" and bitrate not in ["13200", "16400", "24400", "32000", "64000"]: # skip high bitrates for DTX until DTX issue is resolved @@ -905,6 +966,18 @@ def test_sba_plc_system( cut_gain = "1.0" input_config = SBA_FORMAT[abs(int(sba_order))] + if dtx: + testcase_props["category"] = CAT_DTX + try: + int(bitrate) + except ValueError: + testcase_props["category"] = CAT_BITRATE_SWITCHING + if plc_pattern is not None: + testcase_props["category"] = CAT_PLC + + if update_ref != 1: + for k, v in testcase_props.items(): + dut_encoder_frontend.record_property(k, v) if not decoder_only: sba_enc( @@ -964,6 +1037,7 @@ def test_sba_plc_system( get_odg=get_odg, get_odg_bin=get_odg_bin, compare_to_input=compare_to_input, + split_comparison=split_comparison, ) @@ -1140,6 +1214,7 @@ def sba_dec( get_odg=False, get_odg_bin=False, compare_to_input=False, + split_comparison=False, ): dut_pkt_dir = f"{dut_base_path}/sba_bs/pkt" ref_pkt_dir = f"{reference_path}/sba_bs/pkt" @@ -1251,7 +1326,14 @@ def sba_dec( allow_differing_lengths = True sampling_rate_Hz = int(sampling_rate) * 1000 - cmp_result, reason = cmp_pcm( + + ### run the comparison tools + split_idx = np.empty(0) + prop_suffix = [""] + + # 1. run comparison on whole files - this is done always, regardless of the presence of --split_comparison + + output_differs_parts, reason_parts = cmp_pcm( ref_out_file, dut_out_file, output_config, @@ -1267,13 +1349,51 @@ def sba_dec( odg_test=odg_test, odg_ref=odg_ref, scalefac=test_info.config.option.scalefac, + split_idx=split_idx, ) - text_to_parse = reason - props = parse_properties(text_to_parse, cmp_result != 0, props_to_record) - for k, v in props.items(): - dut_decoder_frontend.record_property(k, v) + # 2. run comparison on split files if --split_comparison is given + # in the "cut until bitstream starts with SID" case, we can't do the split comp as the indices would not match anymore + if split_comparison and sid != 1: + input_file = f"{test_vector_path}/{tag}.wav" + split_idx = get_split_idx(str(Path(input_file).stem), int(sampling_rate)) + + # this extra if takes care of cases where no splits are found, e.g. the "NOOP" case in the self_test_ltv prm file + # if this would not be there, then the comparison of the whole file would run twice + if len(split_idx) > 0: + output_differs_splits, reason_splits = cmp_pcm( + ref_out_file, + dut_out_file, + output_config, + sampling_rate_Hz, + get_mld=get_mld, + mld_lim=get_mld_lim, + abs_tol=abs_tol, + allow_differing_lengths=allow_differing_lengths, + get_ssnr=get_ssnr, + get_odg=get_odg, + get_odg_bin=get_odg_bin, + odg_input=odg_input, + odg_test=odg_test, + odg_ref=odg_ref, + scalefac=test_info.config.option.scalefac, + split_idx=split_idx, + ) + output_differs_parts += output_differs_splits + reason_parts += reason_splits + + if split_comparison: + prop_suffix = ["_whole"] + [ + f"_split{i:03d}" for i in range(1, len(split_idx) + 1) + ] + + for output_differs, reason, suffix in zip( + output_differs_parts, reason_parts, prop_suffix + ): + props = parse_properties(reason, output_differs, props_to_record, suffix) + for k, v in props.items(): + dut_decoder_frontend.record_property(k, v) # report compare result - if cmp_result != 0: - pytest.fail(text_to_parse) + if output_differs_parts[0] != 0: + pytest.fail(reason_parts[0]) diff --git a/tests/conftest.py b/tests/conftest.py index 3b75cbc5570f91cd98259ef7a31fe7464e9b8d8c..5d3632a762c2818b21363114c83a964b5b5ece90 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -36,7 +36,6 @@ import logging import os import re import json -import shutil from tests import testconfig import pytest import platform @@ -46,16 +45,16 @@ from subprocess import TimeoutExpired, run from tempfile import NamedTemporaryFile from shutil import move import tempfile -from typing import Optional, Union +from typing import Optional, Union, List import numpy as np from .constants import ( - # MAX_ENC_DIFF_NAME_PATTERN, DMX_DIFF, DMX_MLD, DMX_SSNR, MAX_ENC_DIFF_PARAM_NAME, MLD_PATTERN, MAX_DIFF_PATTERN, + SPLIT_IDX, SSNR_PATTERN, ENC_AUX_FILES, ODG_PATTERN, @@ -306,6 +305,13 @@ def pytest_addoption(parser): default=1, ) + parser.addoption( + "--split-comparison", + action="store_true", + help="If given, split the output files by the provided indices", + ) + + @pytest.fixture(scope="session", autouse=True) def update_ref(request): """ @@ -365,6 +371,7 @@ def get_odg(request): """ return request.config.option.odg + @pytest.fixture(scope="session", autouse=True) def get_odg_bin(request): """ @@ -440,6 +447,11 @@ def test_info(request): pytest.fail(request.error) +@pytest.fixture(scope="session") +def split_comparison(request): + return request.config.option.split_comparison + + class EncoderFrontend: def __init__(self, path, enc_type, record_property, timeout=None) -> None: self._path = Path(path).absolute() @@ -1157,7 +1169,9 @@ def props_to_record( return props -def parse_properties(text_to_parse: str, output_differs: bool, props_to_record: list): +def parse_properties( + text_to_parse: str, output_differs: bool, props_to_record: list, suffix: str = "" +): """ Record the given properties in the report by parsing their values from the text. """ @@ -1167,7 +1181,7 @@ def parse_properties(text_to_parse: str, output_differs: bool, props_to_record: for prop in props_to_record: if prop == MLD or prop == DMX_MLD: mld = float(re.search(MLD_PATTERN, text_to_parse).groups(1)[0]) - props[prop] = mld + props[prop + suffix] = mld elif prop == MAX_ABS_DIFF or prop == DMX_DIFF: max_diff = 0 if output_differs: @@ -1175,33 +1189,35 @@ def parse_properties(text_to_parse: str, output_differs: bool, props_to_record: max_diff = match.groups(1)[0] else: raise MaxDiffPatternNotFound() - props[prop] = max_diff + props[prop + suffix] = max_diff elif prop == SSNR or prop == DMX_SSNR: ssnrs = re.findall(SSNR_PATTERN, text_to_parse) min_ssnr = min(ssnrs) min_ssnr_channel = ssnrs.index(min_ssnr) - prefix = "MIN" if prop == SSNR else "DMX" - props[f"{prefix}_SSNR"] = min_ssnr - props[f"{prefix}_SSNR_CHANNEL"] = min_ssnr_channel + propname = "MIN_SSNR" + if prop == DMX_SSNR: + propname = "DMX_MIN_SSNR" + props[propname + suffix] = min_ssnr + props[f"{propname}_CHANNEL" + suffix] = min_ssnr_channel elif prop == ODG: odgs = re.findall(ODG_PATTERN, text_to_parse) min_odg = min(odgs) min_odg_channel = odgs.index(min_odg) - props["MIN_ODG"] = min_odg - props["MIN_ODG_CHANNEL"] = min_odg_channel + props["MIN_ODG" + suffix] = min_odg + props["MIN_ODG_CHANNEL" + suffix] = min_odg_channel elif prop == MAX_ENC_DIFF: search_result = re.search(MAX_ENC_DIFF_PATTERN, text_to_parse) max_enc_diff_ratio = 0.0 max_enc_diff_param_name = "" if search_result: max_enc_diff_param_name, _, max_enc_diff_ratio = search_result.groups(0) - props[MAX_ENC_DIFF] = float(max_enc_diff_ratio) - props[MAX_ENC_DIFF_PARAM_NAME] = max_enc_diff_param_name + props[MAX_ENC_DIFF + suffix] = float(max_enc_diff_ratio) + props[MAX_ENC_DIFF_PARAM_NAME + suffix] = max_enc_diff_param_name elif prop == DELTA_ODG: delta_odg = re.search(DELTA_ODG_PATTERN, text_to_parse) if delta_odg: - props["DELTA_ODG"] = delta_odg.groups(1)[0] + props["DELTA_ODG" + suffix] = delta_odg.groups(1)[0] return props @@ -1217,26 +1233,86 @@ def compare_dmx_signals(ref_dmx_files, dut_dmx_files, fs) -> dict: ref_dmx_files, dmx_file_ref_tmp.name, out_nchans=nchannels, - in_fs=fs*1000, + in_fs=fs * 1000, ) pyaudio3dtools.audiofile.combinefiles( dut_dmx_files, dmx_file_dut_tmp.name, out_nchans=nchannels, - in_fs=fs*1000, + in_fs=fs * 1000, ) dmx_differs, reason = cmp_pcm( dmx_file_ref_tmp.name, dmx_file_dut_tmp.name, nchannels, - fs*1000, + fs * 1000, get_mld=True, get_ssnr=True, quiet=True, ) + dmx_differs = dmx_differs[0] + reason = reason[0] dmx_props = [DMX_DIFF, DMX_MLD, DMX_SSNR] prop_results = parse_properties(reason, dmx_differs, dmx_props) return prop_results + + +def get_split_idx(input_file: str, sampling_rate_khz: int) -> Optional[np.ndarray]: + """ + Return array for splitting the output file before doing the comparison. + + If no list of indices is available for the given input file, an empty array is returned. + """ + assert sampling_rate_khz in [16, 32, 48] + + input_file = input_file.lower() + if "omasa" in input_file: + format = "_".join(input_file.split("_")[1:-1]) + elif "osba" in input_file: + if "foa" in input_file: + format = "osba_foa" + else: + format = "osba_hoa" + else: + format = input_file.split("_")[-1].lower() + idx = SPLIT_IDX.get(format, np.empty(0)) + + # copy is important because we modify the array below for fs != 16 + # without copy, the constant would be modified and future split values would be wrong + idx = idx.copy() + + if len(idx) > 0 and sampling_rate_khz != 16: + idx *= sampling_rate_khz // 16 + + return idx + + +IVAS_ENC_FORMATS = { + "sba": "SBA", + "masa": "MASA", + "ism_sba": "OSBA", + "ism_masa": "OMASA", + "ism": "ISM", + "mc": "Multichannel", + "stereo_dmx_evs": "Stereo DMX EVS", + "stereo": "Stereo", +} +# NOTE: the blank at the end is important to prevent e.g. "-ism_masa" matching on "-ism" only +PATTERN_IVAS_ENC_FORMAT = re.compile(r"-(" + r"|".join(IVAS_ENC_FORMATS.keys()) + ") ") + + +def get_format_from_enc_opts(enc_opts: str) -> str: + """ + Parse the encoder format from the encoder options by searching for any of the + '-' arguments. If none of them is given, encoder will run in EVS mode. + """ + format = "Mono" + m = re.search(PATTERN_IVAS_ENC_FORMAT, enc_opts) + if m is not None: + enc_format_str = m.groups()[0] + format = IVAS_ENC_FORMATS[enc_format_str] + + return format diff --git a/tests/constants.py b/tests/constants.py index 9990db37cc690d2a52eb983df927740eacf5b46b..d70d1251943f6b675613f0210b6f1f8675bf4fea 100644 --- a/tests/constants.py +++ b/tests/constants.py @@ -8,17 +8,17 @@ TESTV_DIR = SCRIPTS_DIR.joinpath("testv") # Properties to record MLD = "MLD" -MAX_ABS_DIFF = "MAXIMUM ABS DIFF" +MAX_ABS_DIFF = "MAX_ABS_DIFF" SSNR = "SSNR" ODG = "ODG" -DELTA_ODG = "Delta-ODG" -MAX_ENC_DIFF = "MAXIMUM ENC DIFF" -MAX_ENC_DIFF_PARAM_NAME = "MAXIMUM ENC DIFF PARAM" +DELTA_ODG = "DELTA_ODG" +MAX_ENC_DIFF = "MAXIMUM_ENC_DIFF" +MAX_ENC_DIFF_PARAM_NAME = "MAXIMUM_ENC_DIFF_PARAM" ENC_CORE_OVL = "ENC_CORE_OVL" MAX_OVL = "MAX_OVL" MIN_OVL = "MIN_OVL" -DMX_DIFF = "DMX MAXIMUM ABS DIFF" -DMX_MLD = "DMX MLD" +DMX_DIFF = "DMX_MAX_ABS_DIFF" +DMX_MLD = "DMX_MLD" DMX_SSNR = "DMX_SSNR" # regex patterns for parsing the output from comparisons -> mainly for BASOP ci @@ -65,3 +65,403 @@ ENC_AUX_FILES = [ ["total_brate", np.float32, "fs/50"], ["vad_flag", np.int16, "fs/50"], ] + +### !!! Note: this is duplicated in scripts/create_histogram.py. If you change this here, ALSO ADAPT IT THERE!!! +### (importing from here failed for unknown reasons in some jobs on some runners and I don't have time to properly investigate this...) +CAT_NORMAL = "normal operation" +CAT_DTX = "DTX" +CAT_PLC = "PLC" +CAT_BITRATE_SWITCHING = "bitrate switching" +CAT_JBM = "JBM" + +# lists of indices for splitting of output files +# values are in SAMPLES for 16kHz (!). For higher rates, need to multiply +SPLIT_IDX_LTV_STEREO_16KHZ = np.asarray( + [ + 302096, + 519280, + 613072, + 690176, + 740992, + 749024, + 782096, + 950064, + 1045408, + 1141408, + 1237424, + 1333392, + 1397712, + 1472912, + 1755840, + 1781680, + 1789712, + 1957664, + 2158112, + 2294704, + 2454688, + 2525872, + 2573888, + 2645504, + 2746096, + 2842096, + 2970080, + 3219792, + ] +) +SPLIT_IDX_LTV_ISM1_16KHZ = np.asarray( + [ + 72000, + 164800, + 233600, + 422400, + 492800, + 592000, + 720000, + 792000, + 873600, + 947200, + 1001600, + 1060805, + 1120000, + 1284809, + 1356800, + 1448000, + 1513600, + 1731200, + 1828480, + 1910400, + 1966409, + 2150400, + 2235088, + 2299552, + 2460800, + ] +) +SPLIT_IDX_LTV_ISM234_16KHZ = np.asarray( + [ + 72000, + 164800, + 233600, + 432000, + 592000, + 720000, + 792000, + 873600, + 937600, + 1216000, + 1448000, + 1513600, + 1731200, + 1828480, + 1931200, + 2156800, + 2235088, + 2299552, + 2460800, + ] +) +SPLIT_IDX_LTV_FOA_16KHZ = np.asarray( + [ + 111840, + 176320, + 231200, + 327280, + 359280, + 404384, + 452800, + 465600, + 496272, + 565632, + 735712, + 846624, + 1007184, + 1166720, + 1316640, + 1391840, + 1699200, + 1716800, + 1812640, + 1908640, + 2036304, + 2196320, + 2364592, + 2524624, + ] +) +SPLIT_IDX_LTV_HOA3_16KHZ = np.asarray( + [ + 69328, + 169120, + 339200, + 450144, + 610128, + 685120, + 759600, + 845120, + 930144, + 1090144, + 1240128, + 1313600, + 1619200, + 1640000, + 1704416, + 1759360, + 1855360, + 1887360, + 1932480, + 1982400, + 1993600, + 2024640, + 2136128, + 2232400, + 2328480, + 2424640, + ] +) +SPLIT_IDX_LTV_MASA_1TC_16KHZ = np.asarray( + [ + 112000, + 192012, + 288000, + 376008, + 528000, + 659200, + 716803, + 822406, + 892800, + 1024002, + 1120000, + 1220789, + 1316831, + 1444183, + 1588641, + 1719739, + ] +) +SPLIT_IDX_LTV_MASA_2TC_16KHZ = np.asarray( + [ + 119984, + 255588, + 286417, + 480008, + 560002, + 625505, + 716938, + 944138, + 1037348, + 1076332, + 1131937, + 1295806, + 1479906, + 1616003, + 1743895, + 1792355, + 1920075, + 1968066, + 2112000, + 2207637, + 2317218, + 2436529, + 2613994, + ] +) +SPLIT_IDX_LTV_MC_16KHZ = np.asarray( + [ + 69328, + 169120, + 339200, + 450144, + 610128, + 685120, + 759600, + 845120, + 930144, + 1090144, + 1240128, + 1313600, + 1619200, + 1640000, + 1704416, + 1759360, + 1855360, + 1887360, + 1932480, + 1982400, + 1993600, + 2024640, + 2136128, + 2232400, + 2328480, + 2424640, + 2552400, + ] +) +SPLIT_IDX_LTV_OMASA_1ISM_16KHZ = np.asarray( + [ + 182561, + 250497, + 321192, + 398875, + 462969, + 589104, + 634642, + 704772, + 761550, + 841565, + 905594, + 1077445, + 1147515, + 1204145, + 1284198, + 1348213, + 1474397, + 1519986, + 1590029, + 1686404, + 1726821, + 1790877, + 1872012, + ] +) +SPLIT_IDX_LTV_OMASA_2ISM_16KHZ = np.asarray( + [ + 104025, + 188810, + 248000, + 600000, + 727048, + 832000, + 952951, + 970604, + 1037526, + 1104909, + 1233053, + 1295925, + 1361517, + 1414356, + 1476020, + 1549127, + 1605498, + 1727087, + 1876418, + ] +) +SPLIT_IDX_LTV_OMASA_3ISM_16KHZ = np.asarray( + [ + 155662, + 238144, + 340776, + 415986, + 575167, + 707032, + 832054, + 975997, + 1104017, + 1197434, + 1247993, + 1344330, + 1482134, + 1549575, + 1688766, + 1745715, + ] +) +SPLIT_IDX_LTV_OMASA_4ISM_16KHZ = np.asarray( + [ + 63985, + 192015, + 316768, + 391082, + 500584, + 597477, + 719995, + 838699, + 952531, + 1040041, + 1143121, + 1271008, + 1503986, + 1626407, + 1687966, + 1796807, + 1878304, + ] +) +SPLIT_IDX_LTV_OSBA_FOA_16KHZ = np.asarray( + [ + 110243, + 176353, + 231257, + 327228, + 406430, + 453959, + 465000, + 496215, + 565603, + 735726, + 846637, + 1166658, + 1316644, + 1391826, + 1697613, + 1716813, + 1812764, + 1908681, + 2036312, + 2193583, + 2361556, + ] +) +SPLIT_IDX_LTV_OSBA_HOA_16KHZ = np.asarray( + [ + 69324, + 169120, + 339217, + 450133, + 739726, + 930163, + 1090182, + 1240385, + 1316803, + 1617775, + 1640120, + 1704436, + 1759348, + 1854498, + 1887201, + 1932676, + 1983352, + 1991890, + 2024762, + 2136276, + 2231825, + 2328364, + 2424105, + ] +) + +SPLIT_IDX = { + "stereo": SPLIT_IDX_LTV_STEREO_16KHZ, + "mono": SPLIT_IDX_LTV_STEREO_16KHZ, + "1ism": SPLIT_IDX_LTV_ISM1_16KHZ, + "2ism": SPLIT_IDX_LTV_ISM234_16KHZ, + "3ism": SPLIT_IDX_LTV_ISM234_16KHZ, + "4ism": SPLIT_IDX_LTV_ISM234_16KHZ, + "foa": SPLIT_IDX_LTV_FOA_16KHZ, + "hoa2": SPLIT_IDX_LTV_HOA3_16KHZ, + "hoa3": SPLIT_IDX_LTV_HOA3_16KHZ, + "masa1tc": SPLIT_IDX_LTV_MASA_1TC_16KHZ, + "masa2tc": SPLIT_IDX_LTV_MASA_2TC_16KHZ, + "mc51": SPLIT_IDX_LTV_MC_16KHZ, + # there is always one signal that does something different... + "mc512": SPLIT_IDX_LTV_MC_16KHZ[:-1], + "mc514": SPLIT_IDX_LTV_MC_16KHZ, + "mc71": SPLIT_IDX_LTV_MC_16KHZ, + "mc714": SPLIT_IDX_LTV_MC_16KHZ, + "omasa_1ism": SPLIT_IDX_LTV_OMASA_1ISM_16KHZ, + "omasa_2ism": SPLIT_IDX_LTV_OMASA_2ISM_16KHZ, + "omasa_3ism": SPLIT_IDX_LTV_OMASA_3ISM_16KHZ, + "omasa_4ism": SPLIT_IDX_LTV_OMASA_4ISM_16KHZ, + "osba_foa": SPLIT_IDX_LTV_OSBA_FOA_16KHZ, + "osba_hoa": SPLIT_IDX_LTV_OSBA_HOA_16KHZ, +} diff --git a/tests/renderer/constants.py b/tests/renderer/constants.py index 39f57a6d892893f0d1405326205b0c646758ad5a..c61800a1a28829ef70bb9e7b7a0c6e1be3a4087e 100644 --- a/tests/renderer/constants.py +++ b/tests/renderer/constants.py @@ -1,33 +1,33 @@ #!/usr/bin/env python3 """ - (C) 2022-2025 IVAS codec Public Collaboration with portions copyright Dolby International AB, Ericsson AB, - Fraunhofer-Gesellschaft zur Foerderung der angewandten Forschung e.V., Huawei Technologies Co. LTD., - Koninklijke Philips N.V., Nippon Telegraph and Telephone Corporation, Nokia Technologies Oy, Orange, - Panasonic Holdings Corporation, Qualcomm Technologies, Inc., VoiceAge Corporation, and other - contributors to this repository. All Rights Reserved. +(C) 2022-2025 IVAS codec Public Collaboration with portions copyright Dolby International AB, Ericsson AB, +Fraunhofer-Gesellschaft zur Foerderung der angewandten Forschung e.V., Huawei Technologies Co. LTD., +Koninklijke Philips N.V., Nippon Telegraph and Telephone Corporation, Nokia Technologies Oy, Orange, +Panasonic Holdings Corporation, Qualcomm Technologies, Inc., VoiceAge Corporation, and other +contributors to this repository. All Rights Reserved. - This software is protected by copyright law and by international treaties. - The IVAS codec Public Collaboration consisting of Dolby International AB, Ericsson AB, - Fraunhofer-Gesellschaft zur Foerderung der angewandten Forschung e.V., Huawei Technologies Co. LTD., - Koninklijke Philips N.V., Nippon Telegraph and Telephone Corporation, Nokia Technologies Oy, Orange, - Panasonic Holdings Corporation, Qualcomm Technologies, Inc., VoiceAge Corporation, and other - contributors to this repository retain full ownership rights in their respective contributions in - the software. This notice grants no license of any kind, including but not limited to patent - license, nor is any license granted by implication, estoppel or otherwise. +This software is protected by copyright law and by international treaties. +The IVAS codec Public Collaboration consisting of Dolby International AB, Ericsson AB, +Fraunhofer-Gesellschaft zur Foerderung der angewandten Forschung e.V., Huawei Technologies Co. LTD., +Koninklijke Philips N.V., Nippon Telegraph and Telephone Corporation, Nokia Technologies Oy, Orange, +Panasonic Holdings Corporation, Qualcomm Technologies, Inc., VoiceAge Corporation, and other +contributors to this repository retain full ownership rights in their respective contributions in +the software. This notice grants no license of any kind, including but not limited to patent +license, nor is any license granted by implication, estoppel or otherwise. - Contributors are required to enter into the IVAS codec Public Collaboration agreement before making - contributions. +Contributors are required to enter into the IVAS codec Public Collaboration agreement before making +contributions. - This software is provided "AS IS", without any express or implied warranties. The software is in the - development stage. It is intended exclusively for experts who have experience with such software and - solely for the purpose of inspection. All implied warranties of non-infringement, merchantability - and fitness for a particular purpose are hereby disclaimed and excluded. +This software is provided "AS IS", without any express or implied warranties. The software is in the +development stage. It is intended exclusively for experts who have experience with such software and +solely for the purpose of inspection. All implied warranties of non-infringement, merchantability +and fitness for a particular purpose are hereby disclaimed and excluded. - Any dispute, controversy or claim arising under or in relation to providing this software shall be - submitted to and settled by the final, binding jurisdiction of the courts of Munich, Germany in - accordance with the laws of the Federal Republic of Germany excluding its conflict of law rules and - the United Nations Convention on Contracts on the International Sales of Goods. +Any dispute, controversy or claim arising under or in relation to providing this software shall be +submitted to and settled by the final, binding jurisdiction of the courts of Munich, Germany in +accordance with the laws of the Federal Republic of Germany excluding its conflict of law rules and +the United Nations Convention on Contracts on the International Sales of Goods. """ from pathlib import Path @@ -298,6 +298,77 @@ FORMAT_TO_METADATA_FILES = { ], } +FORMAT_TO_METADATA_FILES_LTV = { + "ISM1": [str(TESTV_DIR.joinpath("ltvISM1.csv"))], + "ISM2": [ + str(TESTV_DIR.joinpath("ltvISM1.csv")), + str(TESTV_DIR.joinpath("ltvISM2.csv")), + ], + "ISM3": [ + str(TESTV_DIR.joinpath("ltvISM1.csv")), + str(TESTV_DIR.joinpath("ltvISM2.csv")), + str(TESTV_DIR.joinpath("ltvISM3.csv")), + ], + "ISM4": [ + str(TESTV_DIR.joinpath("ltvISM1.csv")), + str(TESTV_DIR.joinpath("ltvISM2.csv")), + str(TESTV_DIR.joinpath("ltvISM3.csv")), + str(TESTV_DIR.joinpath("ltvISM4.csv")), + ], + "NDP_ISM4": [ + str(TESTV_DIR.joinpath("ltvISM1.csv")), + str(TESTV_DIR.joinpath("stvISM2_non-diegetic-pan.csv")), + str(TESTV_DIR.joinpath("ltvISM3.csv")), + str(TESTV_DIR.joinpath("ltvISM4.csv")), + ], + "MASA1": [str(TESTV_DIR.joinpath("ltv48_MASA1TC.met"))], + "MASA2": [str(TESTV_DIR.joinpath("ltv48_MASA2TC.met"))], + "OMASA_1_1": [ + str(TESTV_DIR.joinpath("ltvISM1.csv")), + str(TESTV_DIR.joinpath("ltv48_OMASA_1ISM_1TC.met")), + ], + "OMASA_1_2": [ + str(TESTV_DIR.joinpath("ltvISM1.csv")), + str(TESTV_DIR.joinpath("ltvISM2.csv")), + str(TESTV_DIR.joinpath("ltv48_OMASA_2ISM_1TC.met")), + ], + "OMASA_1_3": [ + str(TESTV_DIR.joinpath("ltvISM1.csv")), + str(TESTV_DIR.joinpath("ltvISM2.csv")), + str(TESTV_DIR.joinpath("ltvISM3.csv")), + str(TESTV_DIR.joinpath("ltv48_OMASA_3ISM_1TC.met")), + ], + "OMASA_1_4": [ + str(TESTV_DIR.joinpath("ltvISM1.csv")), + str(TESTV_DIR.joinpath("ltvISM2.csv")), + str(TESTV_DIR.joinpath("ltvISM3.csv")), + str(TESTV_DIR.joinpath("ltvISM4.csv")), + str(TESTV_DIR.joinpath("ltv48_OMASA_4ISM_1TC.met")), + ], + "OMASA_2_1": [ + str(TESTV_DIR.joinpath("ltvISM1.csv")), + str(TESTV_DIR.joinpath("ltv48_OMASA_1ISM_2TC.met")), + ], + "OMASA_2_2": [ + str(TESTV_DIR.joinpath("ltvISM1.csv")), + str(TESTV_DIR.joinpath("ltvISM2.csv")), + str(TESTV_DIR.joinpath("ltv48_OMASA_2ISM_2TC.met")), + ], + "OMASA_2_3": [ + str(TESTV_DIR.joinpath("ltvISM1.csv")), + str(TESTV_DIR.joinpath("ltvISM2.csv")), + str(TESTV_DIR.joinpath("ltvISM3.csv")), + str(TESTV_DIR.joinpath("ltv48_OMASA_3ISM_2TC.met")), + ], + "OMASA_2_4": [ + str(TESTV_DIR.joinpath("ltvISM1.csv")), + str(TESTV_DIR.joinpath("ltvISM2.csv")), + str(TESTV_DIR.joinpath("ltvISM3.csv")), + str(TESTV_DIR.joinpath("ltvISM4.csv")), + str(TESTV_DIR.joinpath("ltv48_OMASA_4ISM_2TC.met")), + ], +} + """ Input formats """ INPUT_FORMATS_AMBI = ["FOA", "HOA2", "HOA3"] @@ -347,4 +418,5 @@ PEAQ_SUPPORTED_FMT = [ "BINAURAL", "BINAURAL_ROOM_IR", "BINAURAL_ROOM_REVERB", -] \ No newline at end of file +] + diff --git a/tests/renderer/test_renderer.py b/tests/renderer/test_renderer.py index b5758d6f06edff56a2c3fbb64ec8be7e37314df9..60ec37a69c07daf64609076cf69b590250fee051 100644 --- a/tests/renderer/test_renderer.py +++ b/tests/renderer/test_renderer.py @@ -33,6 +33,7 @@ the United Nations Convention on Contracts on the International Sales of Goods. import pytest from .constants import ( + FORMAT_TO_METADATA_FILES_LTV, OUTPUT_FORMATS, INPUT_FORMATS_AMBI, FRAMING_TO_TEST, @@ -62,7 +63,6 @@ from ..conftest import props_to_record """ Ambisonics """ - @pytest.mark.parametrize("out_fmt", OUTPUT_FORMATS) @pytest.mark.parametrize("in_fmt", INPUT_FORMATS_AMBI) @pytest.mark.parametrize("frame_size", FRAMING_TO_TEST) @@ -78,6 +78,7 @@ def test_ambisonics( get_ssnr, get_odg, get_odg_bin, + split_comparison, ): run_renderer( record_property, @@ -92,10 +93,10 @@ def test_ambisonics( get_ssnr=get_ssnr, get_odg=get_odg, get_odg_bin=get_odg_bin, + split_comparison=split_comparison, ) - @pytest.mark.parametrize("out_fmt", OUTPUT_FORMATS_BINAURAL) @pytest.mark.parametrize("in_fmt", INPUT_FORMATS_AMBI) @pytest.mark.parametrize("frame_size", FRAMING_TO_TEST) @@ -111,6 +112,7 @@ def test_ambisonics_binaural_static( get_ssnr, get_odg, get_odg_bin, + split_comparison, ): run_renderer( record_property, @@ -125,10 +127,10 @@ def test_ambisonics_binaural_static( get_ssnr=get_ssnr, get_odg=get_odg, get_odg_bin=get_odg_bin, + split_comparison=split_comparison, ) - @pytest.mark.parametrize("trj_file", HR_TRAJECTORIES_TO_TEST) @pytest.mark.parametrize("out_fmt", OUTPUT_FORMATS_BINAURAL) @pytest.mark.parametrize("in_fmt", INPUT_FORMATS_AMBI) @@ -146,6 +148,7 @@ def test_ambisonics_binaural_headrotation( get_ssnr, get_odg, get_odg_bin, + split_comparison, ): run_renderer( record_property, @@ -161,11 +164,11 @@ def test_ambisonics_binaural_headrotation( get_ssnr=get_ssnr, get_odg=get_odg, get_odg_bin=get_odg_bin, + split_comparison=split_comparison, ) @pytest.mark.skip(reason="Not supported for BASOP code currently") - @pytest.mark.parametrize("out_fmt", OUTPUT_FORMATS_BINAURAL[2:]) @pytest.mark.parametrize("in_fmt", INPUT_FORMATS_AMBI) @pytest.mark.parametrize("frame_size", FRAMING_TO_TEST) @@ -183,6 +186,7 @@ def test_dynamic_acoustic_environment( get_odg, get_odg_bin, aeid, + split_comparison, ): rend_config_path = TEST_VECTOR_DIR.joinpath(f"rend_config_combined.cfg") rend_config_path.with_stem(f"rend_config") @@ -202,11 +206,11 @@ def test_dynamic_acoustic_environment( get_odg_bin=get_odg_bin, config_file=rend_config_path, aeid=aeid, + split_comparison=split_comparison, ) @pytest.mark.skip(reason="Not supported for BASOP code currently") - @pytest.mark.parametrize("out_fmt", OUTPUT_FORMATS_BINAURAL[2:]) @pytest.mark.parametrize("in_fmt", INPUT_FORMATS_AMBI) @pytest.mark.parametrize("frame_size", FRAMING_TO_TEST) @@ -222,6 +226,7 @@ def test_dynamic_acoustic_environment_file( get_ssnr, get_odg, get_odg_bin, + split_comparison, ): rend_config_path = TEST_VECTOR_DIR.joinpath(f"rend_config_combined.cfg") rend_config_path.with_stem(f"rend_config") @@ -243,13 +248,13 @@ def test_dynamic_acoustic_environment_file( get_odg_bin=get_odg_bin, config_file=rend_config_path, aeid=aeid, + split_comparison=split_comparison, ) """ Multichannel """ - @pytest.mark.parametrize("out_fmt", OUTPUT_FORMATS) @pytest.mark.parametrize("in_fmt", INPUT_FORMATS_MC) @pytest.mark.parametrize("frame_size", FRAMING_TO_TEST) @@ -265,6 +270,7 @@ def test_multichannel( get_ssnr, get_odg, get_odg_bin, + split_comparison, ): run_renderer( record_property, @@ -279,10 +285,10 @@ def test_multichannel( get_ssnr=get_ssnr, get_odg=get_odg, get_odg_bin=get_odg_bin, + split_comparison=split_comparison, ) - @pytest.mark.parametrize("out_fmt", OUTPUT_FORMATS_BINAURAL) @pytest.mark.parametrize("in_fmt", INPUT_FORMATS_MC) @pytest.mark.parametrize("frame_size", FRAMING_TO_TEST) @@ -298,6 +304,7 @@ def test_multichannel_binaural_static( get_ssnr, get_odg, get_odg_bin, + split_comparison, ): if in_fmt in ["MONO", "STEREO"]: pytest.skip("MONO or STEREO to Binaural rendering unsupported") @@ -315,10 +322,10 @@ def test_multichannel_binaural_static( get_ssnr=get_ssnr, get_odg=get_odg, get_odg_bin=get_odg_bin, + split_comparison=split_comparison, ) - @pytest.mark.parametrize("trj_file", HR_TRAJECTORIES_TO_TEST) @pytest.mark.parametrize("out_fmt", OUTPUT_FORMATS_BINAURAL) @pytest.mark.parametrize("in_fmt", INPUT_FORMATS_MC) @@ -336,6 +343,7 @@ def test_multichannel_binaural_headrotation( get_ssnr, get_odg, get_odg_bin, + split_comparison, ): if in_fmt in ["MONO", "STEREO"]: pytest.skip("MONO or STEREO to Binaural rendering unsupported") @@ -354,13 +362,13 @@ def test_multichannel_binaural_headrotation( get_ssnr=get_ssnr, get_odg=get_odg, get_odg_bin=get_odg_bin, + split_comparison=split_comparison, ) """ ISM """ - @pytest.mark.parametrize("out_fmt", OUTPUT_FORMATS) @pytest.mark.parametrize("in_fmt", INPUT_FORMATS_ISM) @pytest.mark.parametrize("frame_size", FRAMING_TO_TEST) @@ -376,14 +384,20 @@ def test_ism( get_ssnr, get_odg, get_odg_bin, + split_comparison, ): + md_files = ( + FORMAT_TO_METADATA_FILES_LTV[in_fmt] + if test_info.config.option.use_ltv + else FORMAT_TO_METADATA_FILES[in_fmt] + ) run_renderer( record_property, props_to_record, test_info, in_fmt, out_fmt, - in_meta_files=FORMAT_TO_METADATA_FILES[in_fmt], + in_meta_files=md_files, binary_suffix=EXE_SUFFIX, frame_size=frame_size, get_mld=get_mld, @@ -391,10 +405,10 @@ def test_ism( get_ssnr=get_ssnr, get_odg=get_odg, get_odg_bin=get_odg_bin, + split_comparison=split_comparison, ) - @pytest.mark.parametrize("out_fmt", OUTPUT_FORMATS_BINAURAL) @pytest.mark.parametrize("in_fmt", INPUT_FORMATS_ISM) @pytest.mark.parametrize("frame_size", FRAMING_TO_TEST) @@ -410,9 +424,14 @@ def test_ism_binaural_static( get_ssnr, get_odg, get_odg_bin, + split_comparison, ): try: - in_meta_files = FORMAT_TO_METADATA_FILES[in_fmt] + in_meta_files = ( + FORMAT_TO_METADATA_FILES_LTV[in_fmt] + if test_info.config.option.use_ltv + else FORMAT_TO_METADATA_FILES[in_fmt] + ) except KeyError: in_meta_files = None @@ -430,10 +449,10 @@ def test_ism_binaural_static( get_ssnr=get_ssnr, get_odg=get_odg, get_odg_bin=get_odg_bin, + split_comparison=split_comparison, ) - @pytest.mark.parametrize("trj_file", HR_TRAJECTORIES_TO_TEST) @pytest.mark.parametrize("out_fmt", OUTPUT_FORMATS_BINAURAL) @pytest.mark.parametrize("in_fmt", INPUT_FORMATS_ISM) @@ -451,9 +470,14 @@ def test_ism_binaural_headrotation( get_ssnr, get_odg, get_odg_bin, + split_comparison, ): try: - in_meta_files = FORMAT_TO_METADATA_FILES[in_fmt] + in_meta_files = ( + FORMAT_TO_METADATA_FILES_LTV[in_fmt] + if test_info.config.option.use_ltv + else FORMAT_TO_METADATA_FILES[in_fmt] + ) except KeyError: in_meta_files = None @@ -472,13 +496,13 @@ def test_ism_binaural_headrotation( get_ssnr=get_ssnr, get_odg=get_odg, get_odg_bin=get_odg_bin, + split_comparison=split_comparison, ) """ MASA """ - @pytest.mark.parametrize("out_fmt", OUTPUT_FORMATS) @pytest.mark.parametrize("in_fmt", INPUT_FORMATS_MASA) @pytest.mark.parametrize("frame_size", FRAMING_TO_TEST) @@ -494,14 +518,21 @@ def test_masa( get_ssnr, get_odg, get_odg_bin, + split_comparison, ): + md_files = ( + FORMAT_TO_METADATA_FILES_LTV[in_fmt] + if test_info.config.option.use_ltv + else FORMAT_TO_METADATA_FILES[in_fmt] + ) + run_renderer( record_property, props_to_record, test_info, in_fmt, out_fmt, - in_meta_files=FORMAT_TO_METADATA_FILES[in_fmt], + in_meta_files=md_files, binary_suffix=EXE_SUFFIX, frame_size=frame_size, get_mld=get_mld, @@ -509,10 +540,10 @@ def test_masa( get_ssnr=get_ssnr, get_odg=get_odg, get_odg_bin=get_odg_bin, + split_comparison=split_comparison, ) - @pytest.mark.parametrize("out_fmt", OUTPUT_FORMATS_BINAURAL) @pytest.mark.parametrize("in_fmt", INPUT_FORMATS_MASA) @pytest.mark.parametrize("frame_size", FRAMING_TO_TEST) @@ -528,17 +559,23 @@ def test_masa_binaural_static( get_ssnr, get_odg, get_odg_bin, + split_comparison, ): if out_fmt in ["BINAURAL_ROOM_IR", "BINAURAL_ROOM_REVERB"]: pytest.skip("Skipping binaural room outputs for MASA as unimplemented.") + md_files = ( + FORMAT_TO_METADATA_FILES_LTV[in_fmt] + if test_info.config.option.use_ltv + else FORMAT_TO_METADATA_FILES[in_fmt] + ) run_renderer( record_property, props_to_record, test_info, in_fmt, out_fmt, - in_meta_files=FORMAT_TO_METADATA_FILES[in_fmt], + in_meta_files=md_files, binary_suffix=EXE_SUFFIX, frame_size=frame_size, get_mld=get_mld, @@ -546,10 +583,10 @@ def test_masa_binaural_static( get_ssnr=get_ssnr, get_odg=get_odg, get_odg_bin=get_odg_bin, + split_comparison=split_comparison, ) - @pytest.mark.parametrize("trj_file", HR_TRAJECTORIES_TO_TEST) @pytest.mark.parametrize("out_fmt", OUTPUT_FORMATS_BINAURAL) @pytest.mark.parametrize("in_fmt", INPUT_FORMATS_MASA) @@ -567,10 +604,17 @@ def test_masa_binaural_headrotation( get_ssnr, get_odg, get_odg_bin, + split_comparison, ): if out_fmt in ["BINAURAL_ROOM_IR", "BINAURAL_ROOM_REVERB"]: pytest.skip("Skipping binaural room outputs for MASA as unimplemented.") + md_files = ( + FORMAT_TO_METADATA_FILES_LTV[in_fmt] + if test_info.config.option.use_ltv + else FORMAT_TO_METADATA_FILES[in_fmt] + ) + run_renderer( record_property, props_to_record, @@ -578,7 +622,7 @@ def test_masa_binaural_headrotation( in_fmt, out_fmt, trj_file=HR_TRAJECTORY_DIR.joinpath(f"{trj_file}.csv"), - in_meta_files=FORMAT_TO_METADATA_FILES[in_fmt], + in_meta_files=md_files, binary_suffix=EXE_SUFFIX, frame_size=frame_size, get_mld=get_mld, @@ -586,10 +630,10 @@ def test_masa_binaural_headrotation( get_ssnr=get_ssnr, get_odg=get_odg, get_odg_bin=get_odg_bin, + split_comparison=split_comparison, ) - @pytest.mark.parametrize("in_fmt", METADATA_SCENES_TO_TEST_MASA_PREREND) def test_masa_prerend( record_property, @@ -601,6 +645,7 @@ def test_masa_prerend( get_ssnr, get_odg, get_odg_bin, + split_comparison, ): run_renderer( record_property, @@ -615,13 +660,13 @@ def test_masa_prerend( get_ssnr=get_ssnr, get_odg=get_odg, get_odg_bin=get_odg_bin, + split_comparison=split_comparison, ) """ Custom loudspeaker layouts """ - @pytest.mark.parametrize("out_fmt", OUTPUT_FORMATS) @pytest.mark.parametrize("in_layout", CUSTOM_LS_TO_TEST) @pytest.mark.parametrize("frame_size", FRAMING_TO_TEST) @@ -637,6 +682,7 @@ def test_custom_ls_input( get_ssnr, get_odg, get_odg_bin, + split_comparison, ): run_renderer( record_property, @@ -651,10 +697,10 @@ def test_custom_ls_input( get_ssnr=get_ssnr, get_odg=get_odg, get_odg_bin=get_odg_bin, + split_comparison=split_comparison, ) - @pytest.mark.parametrize("out_fmt", CUSTOM_LS_TO_TEST) @pytest.mark.parametrize("in_fmt", OUTPUT_FORMATS) def test_custom_ls_output( @@ -668,6 +714,7 @@ def test_custom_ls_output( get_ssnr, get_odg, get_odg_bin, + split_comparison, ): run_renderer( record_property, @@ -681,10 +728,10 @@ def test_custom_ls_output( get_ssnr=get_ssnr, get_odg=get_odg, get_odg_bin=get_odg_bin, + split_comparison=split_comparison, ) - @pytest.mark.parametrize("out_fmt", CUSTOM_LS_TO_TEST) @pytest.mark.parametrize("in_fmt", CUSTOM_LS_TO_TEST) def test_custom_ls_input_output( @@ -698,6 +745,7 @@ def test_custom_ls_input_output( get_ssnr, get_odg, get_odg_bin, + split_comparison, ): run_renderer( record_property, @@ -711,10 +759,10 @@ def test_custom_ls_input_output( get_ssnr=get_ssnr, get_odg=get_odg, get_odg_bin=get_odg_bin, + split_comparison=split_comparison, ) - @pytest.mark.parametrize("out_fmt", OUTPUT_FORMATS_BINAURAL) @pytest.mark.parametrize("in_layout", CUSTOM_LS_TO_TEST) @pytest.mark.parametrize("frame_size", FRAMING_TO_TEST) @@ -730,6 +778,7 @@ def test_custom_ls_input_binaural( get_ssnr, get_odg, get_odg_bin, + split_comparison, ): run_renderer( record_property, @@ -744,10 +793,10 @@ def test_custom_ls_input_binaural( get_ssnr=get_ssnr, get_odg=get_odg, get_odg_bin=get_odg_bin, + split_comparison=split_comparison, ) - @pytest.mark.parametrize("trj_file", HR_TRAJECTORIES_TO_TEST) @pytest.mark.parametrize("out_fmt", OUTPUT_FORMATS_BINAURAL) @pytest.mark.parametrize("in_layout", CUSTOM_LS_TO_TEST) @@ -765,6 +814,7 @@ def test_custom_ls_input_binaural_headrotation( get_ssnr, get_odg, get_odg_bin, + split_comparison, ): run_renderer( record_property, @@ -780,13 +830,13 @@ def test_custom_ls_input_binaural_headrotation( get_ssnr=get_ssnr, get_odg=get_odg, get_odg_bin=get_odg_bin, + split_comparison=split_comparison, ) """ Metadata / scene description input """ - @pytest.mark.parametrize("out_fmt", OUTPUT_FORMATS) @pytest.mark.parametrize("in_fmt", METADATA_SCENES_TO_TEST) @pytest.mark.parametrize("frame_size", FRAMING_TO_TEST) @@ -802,6 +852,7 @@ def test_metadata( get_ssnr, get_odg, get_odg_bin, + split_comparison, ): run_renderer( record_property, @@ -817,13 +868,13 @@ def test_metadata( get_ssnr=get_ssnr, get_odg=get_odg, get_odg_bin=get_odg_bin, + split_comparison=split_comparison, ) """ non diegetic pan """ - @pytest.mark.parametrize("out_fmt", ["STEREO"]) @pytest.mark.parametrize("in_fmt", ["MONO"]) @pytest.mark.parametrize("non_diegetic_pan", ["0", "-30", "45", "90", "-90"]) @@ -839,6 +890,7 @@ def test_non_diegetic_pan_static( get_ssnr, get_odg, get_odg_bin, + split_comparison, ): run_renderer( record_property, @@ -853,10 +905,10 @@ def test_non_diegetic_pan_static( get_ssnr=get_ssnr, get_odg=get_odg, get_odg_bin=get_odg_bin, + split_comparison=split_comparison, ) - @pytest.mark.parametrize("out_fmt", ["STEREO"]) @pytest.mark.parametrize("in_fmt", ["ISM1"]) @pytest.mark.parametrize("non_diegetic_pan", ["0", "-30", "45", "90", "-90"]) @@ -872,6 +924,7 @@ def test_non_diegetic_pan_ism_static( get_ssnr, get_odg, get_odg_bin, + split_comparison, ): run_renderer( record_property, @@ -886,6 +939,7 @@ def test_non_diegetic_pan_ism_static( get_ssnr=get_ssnr, get_odg=get_odg, get_odg_bin=get_odg_bin, + split_comparison=split_comparison, ) @@ -909,9 +963,7 @@ def test_ambisonics_binaural_headrotation_refrotzero( in_fmt, out_fmt, trj_file, - get_mld, - get_mld_lim, - get_ssnr, + split_comparison, ): if test_info.config.option.create_ref or test_info.config.option.create_cut: pytest.skip("OTR tests only run for smoke test") @@ -932,6 +984,7 @@ def test_ambisonics_binaural_headrotation_refrotzero( "refrot_file": HR_TRAJECTORY_DIR.joinpath("const000.csv"), "frame_size": "5", }, + split_comparison=split_comparison, ) @@ -941,7 +994,12 @@ def test_ambisonics_binaural_headrotation_refrotzero( @pytest.mark.parametrize("out_fmt", OUTPUT_FORMATS_BINAURAL) @pytest.mark.parametrize("in_fmt", INPUT_FORMATS_AMBI) def test_ambisonics_binaural_headrotation_refrotequal( - record_property, props_to_record, test_info, in_fmt, out_fmt, get_mld, get_mld_lim + record_property, + props_to_record, + test_info, + in_fmt, + out_fmt, + split_comparison, ): if test_info.config.option.create_ref or test_info.config.option.create_cut: pytest.skip("OTR tests only run for smoke test") @@ -965,6 +1023,7 @@ def test_ambisonics_binaural_headrotation_refrotequal( ), "frame_size": "5", }, + split_comparison=split_comparison, ) @@ -982,8 +1041,7 @@ def test_ambisonics_binaural_headrotation_refveczero( in_fmt, out_fmt, trj_file, - get_mld, - get_mld_lim, + split_comparison, ): if test_info.config.option.create_ref or test_info.config.option.create_cut: pytest.skip("OTR tests only run for smoke test") @@ -1004,6 +1062,7 @@ def test_ambisonics_binaural_headrotation_refveczero( "refvec_file": HR_TRAJECTORY_DIR.joinpath("const000-Vector3.csv"), "frame_size": "5", }, + split_comparison=split_comparison, ) @@ -1014,7 +1073,12 @@ def test_ambisonics_binaural_headrotation_refveczero( @pytest.mark.parametrize("out_fmt", OUTPUT_FORMATS_BINAURAL) @pytest.mark.parametrize("in_fmt", INPUT_FORMATS_AMBI) def test_ambisonics_binaural_headrotation_refvecequal( - record_property, props_to_record, test_info, in_fmt, out_fmt, get_mld, get_mld_lim + record_property, + props_to_record, + test_info, + in_fmt, + out_fmt, + split_comparison, ): if test_info.config.option.create_ref or test_info.config.option.create_cut: pytest.skip("OTR tests only run for smoke test") @@ -1042,6 +1106,7 @@ def test_ambisonics_binaural_headrotation_refvecequal( ), "frame_size": "5", }, + split_comparison=split_comparison, ) @@ -1052,7 +1117,12 @@ def test_ambisonics_binaural_headrotation_refvecequal( @pytest.mark.parametrize("out_fmt", OUTPUT_FORMATS_BINAURAL) @pytest.mark.parametrize("in_fmt", INPUT_FORMATS_AMBI) def test_ambisonics_binaural_headrotation_refvec_rotating( - record_property, props_to_record, test_info, in_fmt, out_fmt, get_mld, get_mld_lim + record_property, + props_to_record, + test_info, + in_fmt, + out_fmt, + split_comparison, ): if test_info.config.option.create_ref or test_info.config.option.create_cut: pytest.skip("OTR tests only run for smoke test") @@ -1081,6 +1151,7 @@ def test_ambisonics_binaural_headrotation_refvec_rotating( ), "frame_size": "5", }, + split_comparison=split_comparison, ) @@ -1093,7 +1164,12 @@ def test_ambisonics_binaural_headrotation_refvec_rotating( @pytest.mark.parametrize("out_fmt", OUTPUT_FORMATS_BINAURAL) @pytest.mark.parametrize("in_fmt", INPUT_FORMATS_AMBI) def test_ambisonics_binaural_headrotation_refvec_rotating_fixed_pos_offset( - record_property, props_to_record, test_info, in_fmt, out_fmt, get_mld, get_mld_lim + record_property, + props_to_record, + test_info, + in_fmt, + out_fmt, + split_comparison, ): if test_info.config.option.create_ref or test_info.config.option.create_cut: pytest.skip("OTR tests only run for smoke test") @@ -1118,6 +1194,7 @@ def test_ambisonics_binaural_headrotation_refvec_rotating_fixed_pos_offset( ), "frame_size": "5", }, + split_comparison=split_comparison, ) @@ -1129,7 +1206,12 @@ def test_ambisonics_binaural_headrotation_refvec_rotating_fixed_pos_offset( @pytest.mark.parametrize("out_fmt", OUTPUT_FORMATS_BINAURAL) @pytest.mark.parametrize("in_fmt", INPUT_FORMATS_AMBI) def test_ambisonics_binaural_headrotation_refveclev_vs_refvec( - record_property, props_to_record, test_info, in_fmt, out_fmt, get_mld, get_mld_lim + record_property, + props_to_record, + test_info, + in_fmt, + out_fmt, + split_comparison, ): if test_info.config.option.create_ref or test_info.config.option.create_cut: pytest.skip("OTR tests only run for smoke test") @@ -1153,6 +1235,7 @@ def test_ambisonics_binaural_headrotation_refveclev_vs_refvec( "refvec_file": HR_TRAJECTORY_DIR.joinpath("full-circle-4s-Vector3.csv"), "frame_size": "5", }, + split_comparison=split_comparison, ) @@ -1163,7 +1246,12 @@ def test_ambisonics_binaural_headrotation_refveclev_vs_refvec( @pytest.mark.parametrize("out_fmt", OUTPUT_FORMATS_BINAURAL) @pytest.mark.parametrize("in_fmt", INPUT_FORMATS_MC) def test_multichannel_binaural_headrotation_refvec_rotating( - record_property, props_to_record, test_info, in_fmt, out_fmt, get_mld, get_mld_lim + record_property, + props_to_record, + test_info, + in_fmt, + out_fmt, + split_comparison, ): if test_info.config.option.create_ref or test_info.config.option.create_cut: pytest.skip("OTR tests only run for smoke test") @@ -1191,6 +1279,7 @@ def test_multichannel_binaural_headrotation_refvec_rotating( ), "frame_size": "5", }, + split_comparison=split_comparison, ) @@ -1201,13 +1290,22 @@ def test_multichannel_binaural_headrotation_refvec_rotating( @pytest.mark.parametrize("out_fmt", OUTPUT_FORMATS_BINAURAL) @pytest.mark.parametrize("in_fmt", INPUT_FORMATS_ISM) def test_ism_binaural_headrotation_refvec_rotating( - record_property, props_to_record, test_info, in_fmt, out_fmt, get_mld, get_mld_lim + record_property, + props_to_record, + test_info, + in_fmt, + out_fmt, + split_comparison, ): if test_info.config.option.create_ref or test_info.config.option.create_cut: pytest.skip("OTR tests only run for smoke test") try: - in_meta_files = FORMAT_TO_METADATA_FILES[in_fmt] + in_meta_files = ( + FORMAT_TO_METADATA_FILES_LTV[in_fmt] + if test_info.config.option.use_ltv + else FORMAT_TO_METADATA_FILES[in_fmt] + ) except KeyError: in_meta_files = None @@ -1233,4 +1331,5 @@ def test_ism_binaural_headrotation_refvec_rotating( "in_meta_files": in_meta_files, "frame_size": "5", }, + split_comparison=split_comparison, ) diff --git a/tests/renderer/utils.py b/tests/renderer/utils.py index 13b03edc0bb6288e7933575aa5dab5b49c34d469..0d4c2303d0b2d90c3a52a740966e15b4c32b5aa2 100644 --- a/tests/renderer/utils.py +++ b/tests/renderer/utils.py @@ -57,11 +57,12 @@ from .constants import ( BIN_SUFFIX_MERGETARGET, PEAQ_SUPPORTED_FMT, ) +from ..constants import CAT_NORMAL sys.path.append(SCRIPTS_DIR) from pyaudio3dtools.audiofile import readfile from ..cmp_pcm import cmp_pcm -from ..conftest import parse_properties +from ..conftest import parse_properties, get_split_idx def run_cmd(cmd, test_info, env=None): @@ -173,6 +174,7 @@ def run_renderer( out_file=None, sr=48, render_for_peaq=False, + split_comparison=False, ) -> str: # prepare arguments and filepaths if trj_file is not None: @@ -309,6 +311,15 @@ def run_renderer( env["UBSAN_OPTIONS"] + f",log_path=usan_log_{test_info.node.name}" ) + testcase_props = { + "format": "Renderer", + "category": CAT_NORMAL, + } + + if record_property is not None: + for k, v in testcase_props.items(): + record_property(k, v) + # run the renderer run_cmd(cmd, test_info, env) @@ -372,9 +383,13 @@ def run_renderer( else: odg_input = in_file - # see constants.py + ### run the comparison tools + split_idx = np.empty(0) + prop_suffix = [""] + + # 1. run comparison on whole files - this is done always, regardless of the presence of --split_comparison ref_fs = int(cmd[10]) * 1000 - output_differs, reason = cmp_pcm( + output_differs_parts, reason_parts = cmp_pcm( out_file_ref, out_file, out_fmt, @@ -389,14 +404,54 @@ def run_renderer( odg_test=odg_test, odg_ref=odg_ref, scalefac=test_info.config.option.scalefac, + split_idx=split_idx, ) - props = parse_properties(reason, output_differs, props_to_record) - for k, v in props.items(): - record_property(k, v) + # 2. run comparison on split files if --split_comparison is given + # for JBM cases, comparison will fail because of length mismatch beetween split wav files and tracefiles + # -> skip split comparison for these cases + if split_comparison: + split_idx = get_split_idx(str(Path(in_file).stem), ref_fs // 1000) + + # this extra if takes care of cases where no splits are found, e.g. the "NOOP" case in the self_test_ltv prm file + # if this would not be there, then the comparison of the whole file would run twice + if len(split_idx) > 0: + output_differs_splits, reason_splits = cmp_pcm( + out_file_ref, + out_file, + out_fmt, + ref_fs, + get_mld=get_mld, + mld_lim=get_mld_lim, + abs_tol=abs_tol, + get_ssnr=get_ssnr, + get_odg=get_odg, + get_odg_bin=get_odg_bin, + odg_input=odg_input, + odg_test=odg_test, + odg_ref=odg_ref, + scalefac=test_info.config.option.scalefac, + split_idx=split_idx, + ) + output_differs_parts += output_differs_splits + reason_parts += reason_splits + + if split_comparison: + prop_suffix = ["_whole"] + [ + f"_split{i:03d}" for i in range(1, len(split_idx) + 1) + ] + + for output_differs, reason, suffix in zip( + output_differs_parts, reason_parts, prop_suffix + ): + result_props = parse_properties( + reason, output_differs, props_to_record, suffix + ) + for k, v in result_props.items(): + record_property(k, v) - if output_differs: - pytest.fail(f"Output differs: ({reason})") + if output_differs_parts[0]: + pytest.fail(f"Output differs: ({reason_parts[0]})") # compare metadata files in case of MASA prerendering if "MASA" in str(out_fmt): @@ -416,6 +471,7 @@ def compare_renderer_args( out_fmt, ref_kwargs: Dict, cut_kwargs: Dict, + split_comparison=False, ): out_file_ref = run_renderer( record_property, @@ -424,6 +480,7 @@ def compare_renderer_args( in_fmt, out_fmt, **ref_kwargs, + split_comparison=split_comparison, ) ref, ref_fs = readfile(out_file_ref) out_file_cut = run_renderer( @@ -433,6 +490,7 @@ def compare_renderer_args( in_fmt, out_fmt, **cut_kwargs, + split_comparison=split_comparison, ) cut, cut_fs = readfile(out_file_cut) [diff_found, snr, gain_b, max_diff] = check_BE(test_info, ref, ref_fs, cut, cut_fs) diff --git a/tests/test_26444.py b/tests/test_26444.py index cdf065ef3e437fb740e61a34af4e065f8d4bc9e7..9420f15551033f7838b0df55d8744613496da509 100644 --- a/tests/test_26444.py +++ b/tests/test_26444.py @@ -40,11 +40,12 @@ import shutil from tests.cmp_pcm import cmp_pcm from tests.conftest import DecoderFrontend, EncoderFrontend, parse_properties +from tests.constants import CAT_BITRATE_SWITCHING, CAT_DTX, CAT_JBM, CAT_NORMAL, CAT_PLC test_dict = {} TEST_DIR = "evs_be_test" -scripts = [ +SCRIPTS = [ "Readme_AMRWB_IO_dec.txt", "Readme_AMRWB_IO_enc.txt", "Readme_EVS_dec.txt", @@ -52,7 +53,16 @@ scripts = [ "Readme_JBM_dec.txt", ] -for s in scripts: +FORMATS_4_SCRIPTS = dict( + zip( + [s.replace(".txt", "") for s in SCRIPTS], + ["AMRWBIO_dec", "AMRWBIO_enc", "EVS_dec", "EVS_enc", "EVS_JBM_dec"], + ) +) + +PATTERN_EVS_FORMAT = re.compile(r"(" + r"|".join(FORMATS_4_SCRIPTS.keys()) + ")") + +for s in SCRIPTS: with open(os.path.join(TEST_DIR, s), "r", encoding="UTF-8") as fp: tag = "" enc_opts = "" @@ -88,11 +98,33 @@ def test_evs_26444( abs_tol, get_ssnr, get_odg, + record_property, ): enc_opts, dec_opts, diff_opts = test_dict[test_tag] + testcase_props = {} + + # get format prop from test_tag + m = re.search(PATTERN_EVS_FORMAT, test_tag) + assert m is not None + testcase_props["format"] = FORMATS_4_SCRIPTS[m.groups()[0]] diff_opts = diff_opts.replace("./", TEST_DIR + "/") + category = CAT_NORMAL + if "JBM" in test_tag: + category = CAT_JBM + elif "br sw" in test_tag or "bitrate switching" in test_tag: + category = CAT_BITRATE_SWITCHING + elif "%" in test_tag: + category = CAT_PLC + elif "DTX" in test_tag: + category = CAT_DTX + + testcase_props["category"] = category + + for k, v in testcase_props.items(): + record_property(k, v) + if enc_opts: args = enc_opts.split()[1:] @@ -176,6 +208,8 @@ def test_evs_26444( get_ssnr=get_ssnr, get_odg=get_odg, ) + output_differs = output_differs[0] + reason = reason[0] props = parse_properties(reason, output_differs, props_to_record) for k, v in props.items():