From f5f085e3c597c2109ccb494bcc3b4ff1159fec50 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fabian=20M=C3=BCller?= Date: Wed, 5 Nov 2025 18:09:20 +0100 Subject: [PATCH 1/6] Make stats output visible after command exited --- scripts/ivas_conformance/runConformance.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/scripts/ivas_conformance/runConformance.py b/scripts/ivas_conformance/runConformance.py index 15f9fbcd53..2b3dd19387 100644 --- a/scripts/ivas_conformance/runConformance.py +++ b/scripts/ivas_conformance/runConformance.py @@ -628,3 +628,6 @@ if __name__ == "__main__": if not args.analyse_only: conformance.runTag(tag) conformance.doAnalysis(selectTag=tag) + + # final \n makes sure that the output from stat() is readable after the command terminates + print() -- GitLab From d6c508d9633a71e1f9a63c4e4ec05ed2b01674fc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fabian=20M=C3=BCller?= Date: Wed, 5 Nov 2025 18:10:11 +0100 Subject: [PATCH 2/6] Exit with error code when tests failed --- scripts/ivas_conformance/runConformance.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/scripts/ivas_conformance/runConformance.py b/scripts/ivas_conformance/runConformance.py index 2b3dd19387..9748fada69 100644 --- a/scripts/ivas_conformance/runConformance.py +++ b/scripts/ivas_conformance/runConformance.py @@ -631,3 +631,7 @@ if __name__ == "__main__": # final \n makes sure that the output from stat() is readable after the command terminates print() + + if conformance.failedTests.value != 0: + print(f"Error: {conformance.failedTests.value} tests failed") + sys.exit(1) -- GitLab From 27befb678cdebaa68e0f07bd54b26d91addcacaa Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Wed, 5 Nov 2025 18:39:29 +0100 Subject: [PATCH 3/6] switch to thread-based parallelism the previous process-based implementation can crash due to no pooling --- scripts/ivas_conformance/runConformance.py | 47 +++++++++------------- 1 file changed, 18 insertions(+), 29 deletions(-) diff --git a/scripts/ivas_conformance/runConformance.py b/scripts/ivas_conformance/runConformance.py index 9748fada69..621e5db6e3 100644 --- a/scripts/ivas_conformance/runConformance.py +++ b/scripts/ivas_conformance/runConformance.py @@ -29,6 +29,7 @@ submitted to and settled by the final, binding jurisdiction of the courts of Mun accordance with the laws of the Federal Republic of Germany excluding its conflict of law rules and the United Nations Convention on Contracts on the International Sales of Goods. """ + import argparse import os import platform @@ -37,8 +38,9 @@ import numpy as np import subprocess import tempfile import sys -from typing import Optional -from multiprocessing import Process, Value +from multiprocessing import Value +from concurrent.futures import ThreadPoolExecutor +from itertools import repeat import shutil sys.path.append(os.path.join(os.path.dirname(os.path.abspath(__file__)), "..")) @@ -178,9 +180,9 @@ class MLDConformance: print( f"Mapped decoder tests for {len(self.EncoderToDecoderCmdMap)} encoder tests out of {len(self.Commands['ENC'])} tests" ) - assert len(self.EncoderToDecoderCmdMap) == len( - self.Commands["ENC"] - ), "Failed to Map Encoder Commands to Decoder Commands" + assert len(self.EncoderToDecoderCmdMap) == len(self.Commands["ENC"]), ( + "Failed to Map Encoder Commands to Decoder Commands" + ) def genEncoderReferences(self, command: str, encCommandIdx: int): # RUN ENCODER COMMAND LINE WITH REFERENCE ENCODER @@ -209,18 +211,12 @@ class MLDConformance: self.stats() def runReferenceGeneration(self): - processes = list() # Multiprocess list commands = conformance.Commands["ENC"] self.totalTests = len(commands) if not self.args.no_multi_processing: - for commandIdx, command in enumerate(commands): - p = Process( - target=self.genEncoderReferences, args=(command, commandIdx) - ) - processes.append(p) - p.start() - for p in processes: - p.join() + command_ids = range(len(commands)) + with ThreadPoolExecutor(max_workers=os.cpu_count()) as executor: + list(executor.map(self.genEncoderReferences, commands, command_ids)) else: for commandIdx, command in enumerate(commands): conformance.genEncoderReferences(command, commandIdx) @@ -356,9 +352,9 @@ class MLDConformance: "$CUT_PATH/ref/sba_bs/", f"{self.testvDir}/ref/sba_bs/" ) else: - #command = command.replace( + # command = command.replace( # "$CUT_PATH/dut/sba_bs/pkt/", f"{self.outputDir}/dut/enc/" - #) + # ) command = command.replace( "$CUT_PATH/ref/param_file/enc/", f"{self.outputDir}/dut/enc/" ) @@ -412,18 +408,11 @@ class MLDConformance: self.totalTests = len(commands) print( - f"Executing tests for {tag} {'Filter='+self.filter if self.filter else ''} ({self.totalTests} tests)" + f"Executing tests for {tag} {'Filter=' + self.filter if self.filter else ''} ({self.totalTests} tests)" ) if not self.args.no_multi_processing: - for command in commands: - p = Process( - target=self.runOneCommand, - args=(tag, command), - ) - processes.append(p) - p.start() - for p in processes: - p.join() + with ThreadPoolExecutor(max_workers=os.cpu_count()) as executor: + list(executor.map(self.runOneCommand, repeat(tag), commands)) else: for command in commands: self.runOneCommand(tag, command) @@ -470,9 +459,9 @@ class MLDConformance: with tempfile.TemporaryDirectory() as tmpdir: refSamples, fsR = readfile(refFile, outdtype="float") dutSamples, fsD = readfile(dutFile, outdtype="float") - assert ( - refSamples.shape[1] == dutSamples.shape[1] - ), "No of channels mismatch if ref vs cut" + assert refSamples.shape[1] == dutSamples.shape[1], ( + "No of channels mismatch if ref vs cut" + ) maxDiff, rmsdB, beSamplesPercent = self.getSampleStats( refSamples, dutSamples ) -- GitLab From cdc6e6e7f8cf0c1157e8dd199e2284e61bd58147 Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Wed, 5 Nov 2025 18:49:06 +0100 Subject: [PATCH 4/6] remove now unused variable --- scripts/ivas_conformance/runConformance.py | 1 - 1 file changed, 1 deletion(-) diff --git a/scripts/ivas_conformance/runConformance.py b/scripts/ivas_conformance/runConformance.py index 621e5db6e3..c5efa177a8 100644 --- a/scripts/ivas_conformance/runConformance.py +++ b/scripts/ivas_conformance/runConformance.py @@ -397,7 +397,6 @@ class MLDConformance: with open(self.sampleStats[tag], "w") as f: f.write(f"PYTESTTAG, MAXDIFF, RMSdB, BEFRAMES_PERCENT, MAX_MLD\n") - processes = list() # Multiprocess list commands = list() if self.filter: for command in self.Commands[tag]: -- GitLab From 85499acc26aae52eb19caa7d7cd45a6a22f6ab76 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fabian=20M=C3=BCller?= Date: Wed, 5 Nov 2025 21:42:23 +0100 Subject: [PATCH 5/6] Fix output with --regenerate-enc-refs, too --- scripts/ivas_conformance/runConformance.py | 23 +++++++++++----------- 1 file changed, 11 insertions(+), 12 deletions(-) diff --git a/scripts/ivas_conformance/runConformance.py b/scripts/ivas_conformance/runConformance.py index c5efa177a8..09c4241620 100644 --- a/scripts/ivas_conformance/runConformance.py +++ b/scripts/ivas_conformance/runConformance.py @@ -604,18 +604,17 @@ if __name__ == "__main__": if args.regenerate_enc_refs: conformance.runReferenceGeneration() - sys.exit(0) - - testTags = ( - MLDConformance.IVAS_Bins.keys() if args.test_mode == "ALL" else [args.test_mode] - ) - for tag in testTags: - if tag == "ISAR": - # Not implemented yet - continue - if not args.analyse_only: - conformance.runTag(tag) - conformance.doAnalysis(selectTag=tag) + else: + testTags = ( + MLDConformance.IVAS_Bins.keys() if args.test_mode == "ALL" else [args.test_mode] + ) + for tag in testTags: + if tag == "ISAR": + # Not implemented yet + continue + if not args.analyse_only: + conformance.runTag(tag) + conformance.doAnalysis(selectTag=tag) # final \n makes sure that the output from stat() is readable after the command terminates print() -- GitLab From cedd28c2313db6175c976beae04a71a7a6079960 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fabian=20M=C3=BCller?= Date: Wed, 5 Nov 2025 22:30:13 +0100 Subject: [PATCH 6/6] Fix thread safety --- scripts/ivas_conformance/runConformance.py | 24 ++++++++++++++-------- 1 file changed, 15 insertions(+), 9 deletions(-) diff --git a/scripts/ivas_conformance/runConformance.py b/scripts/ivas_conformance/runConformance.py index 09c4241620..b9a246749e 100644 --- a/scripts/ivas_conformance/runConformance.py +++ b/scripts/ivas_conformance/runConformance.py @@ -37,8 +37,8 @@ import re import numpy as np import subprocess import tempfile +import threading import sys -from multiprocessing import Value from concurrent.futures import ThreadPoolExecutor from itertools import repeat import shutil @@ -115,10 +115,13 @@ class MLDConformance: self.testvecDir = args.testvecDir self.toolsdir = os.path.join(self.scriptsDir, "tools") self.testvDir = os.path.join(self.testvecDir, "testv") - self.executedTests = Value("i", 0) - self.failedTests = Value("i", 0) + self.executedTests = 0 + self.failedTests = 0 self.setup() + # synchronize all writes to state variables + self.lock = threading.Lock() + def accumulateCommands(self): for root, _, files in os.walk(self.testvecDir): for file_name in files: @@ -207,7 +210,8 @@ class MLDConformance: + [refEncOutput, refDecOutputFile] ) self.process(command=" ".join(refDecCmd)) - self.executedTests.value += 1 + with self.lock: + self.executedTests += 1 self.stats() def runReferenceGeneration(self): @@ -386,7 +390,8 @@ class MLDConformance: self.runOneRendererTest(tag, command) else: assert False, f"Un-implemented Tag {tag}" - self.executedTests.value += 1 + with self.lock: + self.executedTests += 1 self.stats() def runTag(self, tag: str): @@ -430,13 +435,14 @@ class MLDConformance: if c.returncode: with open(self.failedCmdsFile, "a") as f: f.write(command + "\n") - self.failedTests.value += 1 + with self.lock: + self.failedTests += 1 # c.check_returncode() return 0 def stats(self): print( - f"Executed: {self.executedTests.value} / {self.totalTests} Failed: {self.failedTests.value}", + f"Executed: {self.executedTests} / {self.totalTests} Failed: {self.failedTests}", end="\r", ) @@ -619,6 +625,6 @@ if __name__ == "__main__": # final \n makes sure that the output from stat() is readable after the command terminates print() - if conformance.failedTests.value != 0: - print(f"Error: {conformance.failedTests.value} tests failed") + if conformance.failedTests != 0: + print(f"Error: {conformance.failedTests} tests failed") sys.exit(1) -- GitLab