From ba56efec2a6c24eda51b275fd11d564f8a9d9c08 Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Thu, 13 Nov 2025 15:49:54 +0100 Subject: [PATCH 1/9] conformance scripts - txt-based comp for csv files --- scripts/ivas_conformance/runConformance.py | 30 ++++++++++++++-------- tests/conformance-test/test_26252.py | 13 +++++++++- 2 files changed, 32 insertions(+), 11 deletions(-) diff --git a/scripts/ivas_conformance/runConformance.py b/scripts/ivas_conformance/runConformance.py index bb56bfacc6..f7a1e76507 100644 --- a/scripts/ivas_conformance/runConformance.py +++ b/scripts/ivas_conformance/runConformance.py @@ -835,17 +835,27 @@ class MLDConformance: f.write( f"{pytestTag}, {BE_flag}\n" ) - - for i in range(0,len(DUTmdFileList)): - if os.path.exists(DUTmdFileList[i]): - BE_flag = 0 - if not filecmp.cmp(REFmdFileList[i], DUTmdFileList[i]): - BE_flag = 1 - with open(self.BEcsv[tag], "a") as f: - f.write( - f"{DUTmdFileList[i]}, {BE_flag}\n" - ) + assert len(DUTmdFileList) == len(REFmdFileList) + for refMDfile, dutMDfile in zip(REFmdFileList, DUTmdFileList): + ref_exist = os.path.exists(refMDfile) + dut_exist = os.path.exists(dutMDfile) + + assert ref_exist == dut_exist + if not ref_exist: + continue + + BE_flag = 0 + with open(refMDfile, "r") as f_ref: + with open(dutMDfile, "r") as f_dut: + ref_content = f_ref.read() + dut_content = f_dut.read() + BE_flag = int(not (ref_content == dut_content)) + + with open(self.BEcsv[tag], "a") as f: + f.write( + f"{dutMDfile}, {BE_flag}\n" + ) def doBEanalysis(self, selectTag="all"): keys = IVAS_Bins.keys() if selectTag == "all" else [selectTag] diff --git a/tests/conformance-test/test_26252.py b/tests/conformance-test/test_26252.py index 395e258e12..82444d068d 100644 --- a/tests/conformance-test/test_26252.py +++ b/tests/conformance-test/test_26252.py @@ -130,7 +130,18 @@ def test_26252(test_tag, encoder_path, decoder_path, renderer_path, isar_post_re diff_opts = replace_paths(diff_opts, testv_path, ref_path, cut_path) result = True for cmd in diff_opts.split(';'): - result = result and filecmp.cmp(cmd.split()[1], cmd.split()[2]) + file_a = Path(cmd.split()[1]) + file_b = Path(cmd.split()[2]) + + assert file_a.suffix == file_b.suffix + if Path(file_a).suffix == ".csv": + with open(file_a, "r") as f_a: + with open(file_b, "r") as f_b: + a_content = f_a.read() + b_content = f_b.read() + result = result and a_content == b_content + else: + result = result and filecmp.cmp(file_a, file_b) if not result: assert False, "Output differs" -- GitLab From f6a4d522a39c0183f2ab1900751f633e314870cc Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Thu, 13 Nov 2025 16:18:15 +0100 Subject: [PATCH 2/9] add better reporting of reason for diff --- tests/conformance-test/test_26252.py | 18 ++++++++++++++++-- 1 file changed, 16 insertions(+), 2 deletions(-) diff --git a/tests/conformance-test/test_26252.py b/tests/conformance-test/test_26252.py index 82444d068d..fe994d3675 100644 --- a/tests/conformance-test/test_26252.py +++ b/tests/conformance-test/test_26252.py @@ -129,11 +129,14 @@ def test_26252(test_tag, encoder_path, decoder_path, renderer_path, isar_post_re diff_opts = replace_paths(diff_opts, testv_path, ref_path, cut_path) result = True + + suffix = "" for cmd in diff_opts.split(';'): file_a = Path(cmd.split()[1]) file_b = Path(cmd.split()[2]) - assert file_a.suffix == file_b.suffix + suffix = file_a.suffix + assert suffix == file_b.suffix if Path(file_a).suffix == ".csv": with open(file_a, "r") as f_a: with open(file_b, "r") as f_b: @@ -142,6 +145,17 @@ def test_26252(test_tag, encoder_path, decoder_path, renderer_path, isar_post_re result = result and a_content == b_content else: result = result and filecmp.cmp(file_a, file_b) + if not result: - assert False, "Output differs" + result_str = "Output differs in: " + if suffix == ".csv": + result_str += "object metadata" + elif suffix == ".met": + result_str += "MASA metadata" + elif suffix == ".wav": + result_str += "waveform" + else: + assert False, f"Either no diff commands were found or the output files had an incorrect suffix. \ncmd: {cmd}" + + pytest.fail(result_str) -- GitLab From 1fde59a25804a4fdabdf2bfaf97109ac4ef87115 Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Thu, 13 Nov 2025 16:20:15 +0100 Subject: [PATCH 3/9] fix printout for assertion --- tests/conformance-test/test_26252.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/conformance-test/test_26252.py b/tests/conformance-test/test_26252.py index fe994d3675..5616791fb5 100644 --- a/tests/conformance-test/test_26252.py +++ b/tests/conformance-test/test_26252.py @@ -155,7 +155,7 @@ def test_26252(test_tag, encoder_path, decoder_path, renderer_path, isar_post_re elif suffix == ".wav": result_str += "waveform" else: - assert False, f"Either no diff commands were found or the output files had an incorrect suffix. \ncmd: {cmd}" + assert False, f"Either no diff commands were found or the output files had an incorrect suffix. \ncmd: {diff_opts}" pytest.fail(result_str) -- GitLab From 82ff7683bb066a692919e44de6a8fb405240a786 Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Thu, 13 Nov 2025 17:21:08 +0100 Subject: [PATCH 4/9] fix comparison per file type and reporting --- tests/conformance-test/test_26252.py | 32 +++++++++++++++++----------- 1 file changed, 19 insertions(+), 13 deletions(-) diff --git a/tests/conformance-test/test_26252.py b/tests/conformance-test/test_26252.py index 5616791fb5..34f0b87315 100644 --- a/tests/conformance-test/test_26252.py +++ b/tests/conformance-test/test_26252.py @@ -125,37 +125,43 @@ def test_26252(test_tag, encoder_path, decoder_path, renderer_path, isar_post_re subprocess.run([renderer_path] + rend_opts.split()[1:], check = True) if isar_post_rend_opts: isar_post_rend_opts = replace_paths(isar_post_rend_opts, testv_path, ref_path, cut_path) - subprocess.run([isar_post_renderer_path] + isar_post_rend_opts.split()[1:], check = True) + subprocess.run([isar_post_renderer_path] + isar_post_rend_opts.split()[1:], check = True) diff_opts = replace_paths(diff_opts, testv_path, ref_path, cut_path) - result = True - suffix = "" + diff_in = {".wav": False, ".met": False, ".csv": False} + for cmd in diff_opts.split(';'): file_a = Path(cmd.split()[1]) file_b = Path(cmd.split()[2]) suffix = file_a.suffix assert suffix == file_b.suffix + assert suffix in diff_in + + # for .csv ISM metadata files, do text-based comparison to not take line endings into account + # everything else (.wav output files and MASA metadata files) is compared as binary files if Path(file_a).suffix == ".csv": with open(file_a, "r") as f_a: with open(file_b, "r") as f_b: a_content = f_a.read() b_content = f_b.read() - result = result and a_content == b_content + + files_equal = a_content == b_content else: - result = result and filecmp.cmp(file_a, file_b) + files_equal = filecmp.cmp(file_a, file_b) - if not result: + if not files_equal: + diff_in[suffix] = True + + if any(diff_in.values()): result_str = "Output differs in: " - if suffix == ".csv": - result_str += "object metadata" - elif suffix == ".met": - result_str += "MASA metadata" - elif suffix == ".wav": + if diff_in[".csv"]: + result_str += "object metadata " + if diff_in[".met"]: + result_str += "MASA metadata " + if diff_in[".wav"]: result_str += "waveform" - else: - assert False, f"Either no diff commands were found or the output files had an incorrect suffix. \ncmd: {diff_opts}" pytest.fail(result_str) -- GitLab From 1ebe0a0d99a1face11cdf71765a6846a10f581b6 Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Thu, 13 Nov 2025 18:22:05 +0100 Subject: [PATCH 5/9] handle .192 bitstream suffix --- tests/conformance-test/test_26252.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tests/conformance-test/test_26252.py b/tests/conformance-test/test_26252.py index 34f0b87315..63cddbd003 100644 --- a/tests/conformance-test/test_26252.py +++ b/tests/conformance-test/test_26252.py @@ -129,7 +129,7 @@ def test_26252(test_tag, encoder_path, decoder_path, renderer_path, isar_post_re diff_opts = replace_paths(diff_opts, testv_path, ref_path, cut_path) - diff_in = {".wav": False, ".met": False, ".csv": False} + diff_in = {".wav": False, ".met": False, ".csv": False, ".192": False} for cmd in diff_opts.split(';'): file_a = Path(cmd.split()[1]) @@ -162,6 +162,8 @@ def test_26252(test_tag, encoder_path, decoder_path, renderer_path, isar_post_re result_str += "MASA metadata " if diff_in[".wav"]: result_str += "waveform" + if diff_in[".192"]: + result_str += "bitstream" pytest.fail(result_str) -- GitLab From f96349c8192b7b7c020aa3fc749d0f3e4138d7a8 Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Thu, 13 Nov 2025 18:31:59 +0100 Subject: [PATCH 6/9] handle another bitstream suffix variant --- tests/conformance-test/test_26252.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/conformance-test/test_26252.py b/tests/conformance-test/test_26252.py index 63cddbd003..c16b42e62f 100644 --- a/tests/conformance-test/test_26252.py +++ b/tests/conformance-test/test_26252.py @@ -129,7 +129,7 @@ def test_26252(test_tag, encoder_path, decoder_path, renderer_path, isar_post_re diff_opts = replace_paths(diff_opts, testv_path, ref_path, cut_path) - diff_in = {".wav": False, ".met": False, ".csv": False, ".192": False} + diff_in = {".wav": False, ".met": False, ".csv": False, ".192": False, ".bit": False} for cmd in diff_opts.split(';'): file_a = Path(cmd.split()[1]) @@ -162,7 +162,7 @@ def test_26252(test_tag, encoder_path, decoder_path, renderer_path, isar_post_re result_str += "MASA metadata " if diff_in[".wav"]: result_str += "waveform" - if diff_in[".192"]: + if diff_in[".192"] or diff_in[".bit"]: result_str += "bitstream" pytest.fail(result_str) -- GitLab From dd14eb484ecb84850af61542dc06abcb0a69f0d1 Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Thu, 13 Nov 2025 18:33:47 +0100 Subject: [PATCH 7/9] small cleanup --- tests/conformance-test/test_26252.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/conformance-test/test_26252.py b/tests/conformance-test/test_26252.py index c16b42e62f..183f01e2ea 100644 --- a/tests/conformance-test/test_26252.py +++ b/tests/conformance-test/test_26252.py @@ -141,7 +141,7 @@ def test_26252(test_tag, encoder_path, decoder_path, renderer_path, isar_post_re # for .csv ISM metadata files, do text-based comparison to not take line endings into account # everything else (.wav output files and MASA metadata files) is compared as binary files - if Path(file_a).suffix == ".csv": + if suffix == ".csv": with open(file_a, "r") as f_a: with open(file_b, "r") as f_b: a_content = f_a.read() -- GitLab From 5b347f96c189c66ffc3b7b46ad38df7970b6d751 Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Fri, 14 Nov 2025 09:10:08 +0100 Subject: [PATCH 8/9] binary comparison for MASA metadata --- scripts/ivas_conformance/runConformance.py | 19 ++++++++++++++----- 1 file changed, 14 insertions(+), 5 deletions(-) diff --git a/scripts/ivas_conformance/runConformance.py b/scripts/ivas_conformance/runConformance.py index f7a1e76507..db75611d2e 100644 --- a/scripts/ivas_conformance/runConformance.py +++ b/scripts/ivas_conformance/runConformance.py @@ -845,12 +845,21 @@ class MLDConformance: if not ref_exist: continue + _, ref_suffix = os.path.splitext(refMDfile) + _, dut_suffix = os.path.splitext(dutMDfile) + assert ref_suffix == dut_suffix + BE_flag = 0 - with open(refMDfile, "r") as f_ref: - with open(dutMDfile, "r") as f_dut: - ref_content = f_ref.read() - dut_content = f_dut.read() - BE_flag = int(not (ref_content == dut_content)) + if ref_suffix == ".csv": + with open(refMDfile, "r") as f_ref: + with open(dutMDfile, "r") as f_dut: + ref_content = f_ref.read() + dut_content = f_dut.read() + BE_flag = int(not (ref_content == dut_content)) + elif ref_suffix == ".met": + BE_flag = int(not filecmp.cmp(refMDfile, dutMDfile)) + else: + assert False, f"MD file has unknown suffix {ref_suffix}" with open(self.BEcsv[tag], "a") as f: f.write( -- GitLab From da901c40e1e83719ded03cc10a7071dd9fb1b91f Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Fri, 14 Nov 2025 09:49:37 +0100 Subject: [PATCH 9/9] fix merge error --- scripts/ivas_conformance/runConformance.py | 65 ++++++++++------------ 1 file changed, 29 insertions(+), 36 deletions(-) diff --git a/scripts/ivas_conformance/runConformance.py b/scripts/ivas_conformance/runConformance.py index 6c301e08c8..4723b7a918 100644 --- a/scripts/ivas_conformance/runConformance.py +++ b/scripts/ivas_conformance/runConformance.py @@ -29,6 +29,7 @@ submitted to and settled by the final, binding jurisdiction of the courts of Mun accordance with the laws of the Federal Republic of Germany excluding its conflict of law rules and the United Nations Convention on Contracts on the International Sales of Goods. """ + import argparse import os import platform @@ -474,9 +475,9 @@ class MLDConformance: def genEncoderReferences(self, tag: str, encPytestTag: str): # RUN ENCODER'S OUTPUT DECODED WITH REF DECODER testDesc = self.TestDesc[tag][encPytestTag] - assert isinstance( - testDesc, BitstreamTestDescriptor - ), f"Expected bitstream test descriptor for {tag}" + assert isinstance(testDesc, BitstreamTestDescriptor), ( + f"Expected bitstream test descriptor for {tag}" + ) # Decode the encoded output with Reference IVAS decoder if tag == "ENC": @@ -510,9 +511,9 @@ class MLDConformance: def analyseWavOutputTest(self, tag: str, dutPytestTag: str): testDesc = self.TestDesc[tag][dutPytestTag] - assert isinstance( - testDesc, TestDescriptor - ), f"Expected pcm test descriptor for {tag}" + assert isinstance(testDesc, TestDescriptor), ( + f"Expected pcm test descriptor for {tag}" + ) if self.args.be_test: DUTmdFiles = self.getMDfileList(outFile=testDesc.dutOutput) @@ -537,9 +538,9 @@ class MLDConformance: def analyseOneEncoderTest(self, tag: str, encPytestTag: str): testDesc = self.TestDesc[tag][encPytestTag] - assert isinstance( - testDesc, BitstreamTestDescriptor - ), f"Expected bitstream test descriptor for {tag}" + assert isinstance(testDesc, BitstreamTestDescriptor), ( + f"Expected bitstream test descriptor for {tag}" + ) if self.args.be_test: self.beTest( @@ -566,9 +567,9 @@ class MLDConformance: def analyseOneIsarEncoderTest(self, tag: str, pytestTag: str): testDesc = self.TestDesc[tag][pytestTag] - assert isinstance( - testDesc, BitstreamTestDescriptor - ), f"Expected bitstream test descriptor for {tag}" + assert isinstance(testDesc, BitstreamTestDescriptor), ( + f"Expected bitstream test descriptor for {tag}" + ) if self.args.be_test: self.beTest( @@ -699,7 +700,7 @@ class MLDConformance: self.totalTests = len(selectedTests) print( - f"Executing tests for {tag} {'Filter='+self.filter if self.filter else ''} ({self.totalTests} tests)" + f"Executing tests for {tag} {'Filter=' + self.filter if self.filter else ''} ({self.totalTests} tests)" ) if not self.args.no_multi_processing: with Pool() as pool: @@ -728,7 +729,7 @@ class MLDConformance: self.totalTests = len(selectedTests) print( - f"Analysing tests for {tag} {'Filter='+self.filter if self.filter else ''} ({self.totalTests} tests)" + f"Analysing tests for {tag} {'Filter=' + self.filter if self.filter else ''} ({self.totalTests} tests)" ) if not self.args.no_multi_processing: with Pool() as pool: @@ -787,9 +788,9 @@ class MLDConformance: with tempfile.TemporaryDirectory() as tmpdir: refSamples, fsR = readfile(refFile, outdtype="float") dutSamples, fsD = readfile(dutFile, outdtype="float") - assert ( - refSamples.shape[1] == dutSamples.shape[1] - ), "No of channels mismatch if ref vs cut" + assert refSamples.shape[1] == dutSamples.shape[1], ( + "No of channels mismatch if ref vs cut" + ) maxDiff, rmsdB, beSamplesPercent = self.getSampleStats( refSamples, dutSamples ) @@ -863,19 +864,7 @@ class MLDConformance: if not filecmp.cmp(refFile, dutFile): BE_flag = 1 with open(self.BEcsv[tag], "a") as f: - f.write( - f"{pytestTag}, {BE_flag}\n" - ) - - for i in range(0,len(DUTmdFileList)): - if os.path.exists(DUTmdFileList[i]): - BE_flag = 0 - if not filecmp.cmp(REFmdFileList[i], DUTmdFileList[i]): - BE_flag = 1 - with open(self.BEcsv[tag], "a") as f: - f.write( - f"{DUTmdFileList[i]}, {BE_flag}\n" - ) + f.write(f"{pytestTag}, {BE_flag}\n") assert len(DUTmdFileList) == len(REFmdFileList) for refMDfile, dutMDfile in zip(REFmdFileList, DUTmdFileList): @@ -903,9 +892,7 @@ class MLDConformance: assert False, f"MD file has unknown suffix {ref_suffix}" with open(self.BEcsv[tag], "a") as f: - f.write( - f"{dutMDfile}, {BE_flag}\n" - ) + f.write(f"{dutMDfile}, {BE_flag}\n") def doBEanalysis(self, selectTag="all"): keys = IVAS_Bins.keys() if selectTag == "all" else [selectTag] @@ -936,9 +923,13 @@ class MLDConformance: dutMLD = mldCutWithTags["MLD"][indDut] diff = dutMLD - refMLD if diff.max() > threshold: - print(f"\033[91mMLD Corridor failed for {tag} with max MLD diff of {diff.max()} \033[00m") + print( + f"\033[91mMLD Corridor failed for {tag} with max MLD diff of {diff.max()} \033[00m" + ) else: - print(f"\033[92mMLD Corridor passed for {tag} with max MLD diff of {diff.max()} \033[00m") + print( + f"\033[92mMLD Corridor passed for {tag} with max MLD diff of {diff.max()} \033[00m" + ) def doAnalysis(self, selectTag="all", corridor=False): keys = IVAS_Bins.keys() if selectTag == "all" else [selectTag] @@ -1002,7 +993,9 @@ class MLDConformance: ) self.computeCorridor(mldRefWithTags, mdlCutWithTags, tag) else: - print(f"\033[91mMissing reference MLD file for {tag} : {refMldFile} \033[00m") + print( + f"\033[91mMissing reference MLD file for {tag} : {refMldFile} \033[00m" + ) if __name__ == "__main__": -- GitLab