From 5a70b33a221fbd96ad611ee45a652fdb60dbb5bb Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Thu, 4 Sep 2025 15:59:40 +0200 Subject: [PATCH 1/4] add MD% hash generation script in python for cross-platform use - also adapted for characterization (new dir structure for P800) --- other/get_md5.ps1 | 42 ------------------- other/{get_md5.sh => get_md5.py} | 70 ++++++++++++++++++++------------ 2 files changed, 43 insertions(+), 69 deletions(-) delete mode 100644 other/get_md5.ps1 rename other/{get_md5.sh => get_md5.py} (61%) mode change 100755 => 100644 diff --git a/other/get_md5.ps1 b/other/get_md5.ps1 deleted file mode 100644 index c27d25f2..00000000 --- a/other/get_md5.ps1 +++ /dev/null @@ -1,42 +0,0 @@ -# (C) 2022-2025 IVAS codec Public Collaboration with portions copyright Dolby International AB, Ericsson AB, -# Fraunhofer-Gesellschaft zur Foerderung der angewandten Forschung e.V., Huawei Technologies Co. LTD., -# Koninklijke Philips N.V., Nippon Telegraph and Telephone Corporation, Nokia Technologies Oy, Orange, -# Panasonic Holdings Corporation, Qualcomm Technologies, Inc., VoiceAge Corporation, and other -# contributors to this repository. All Rights Reserved. - -# This software is protected by copyright law and by international treaties. -# The IVAS codec Public Collaboration consisting of Dolby International AB, Ericsson AB, -# Fraunhofer-Gesellschaft zur Foerderung der angewandten Forschung e.V., Huawei Technologies Co. LTD., -# Koninklijke Philips N.V., Nippon Telegraph and Telephone Corporation, Nokia Technologies Oy, Orange, -# Panasonic Holdings Corporation, Qualcomm Technologies, Inc., VoiceAge Corporation, and other -# contributors to this repository retain full ownership rights in their respective contributions in -# the software. This notice grants no license of any kind, including but not limited to patent -# license, nor is any license granted by implication, estoppel or otherwise. - -# Contributors are required to enter into the IVAS codec Public Collaboration agreement before making -# contributions. - -# This software is provided "AS IS", without any express or implied warranties. The software is in the -# development stage. It is intended exclusively for experts who have experience with such software and -# solely for the purpose of inspection. All implied warranties of non-infringement, merchantability -# and fitness for a particular purpose are hereby disclaimed and excluded. - -# Any dispute, controversy or claim arising under or in relation to providing this software shall be -# submitted to and settled by the final, binding jurisdiction of the courts of Munich, Germany in -# accordance with the laws of the Federal Republic of Germany excluding its conflict of law rules and -# the United Nations Convention on Contracts on the International Sales of Goods. - -param( - [Parameter(Mandatory)] - [String]$proc_output_dir, - [Parameter(Mandatory)] - [String]$out_file -) - -(Get-ChildItem $proc_output_dir -Recurse | Where-Object { $_.Name -like '*c??.wav' }) | -ForEach-Object { - $name = $_.Name - $hash = (Get-FileHash $_.FullName -Algorithm MD5).Hash - - "$name $hash" >> $out_file -} \ No newline at end of file diff --git a/other/get_md5.sh b/other/get_md5.py old mode 100755 new mode 100644 similarity index 61% rename from other/get_md5.sh rename to other/get_md5.py index 15caa467..557c8a50 --- a/other/get_md5.sh +++ b/other/get_md5.py @@ -1,6 +1,5 @@ -#!/bin/sh - -# (C) 2022-2025 IVAS codec Public Collaboration with portions copyright Dolby International AB, Ericsson AB, +#! /usr/bin/env python3 +# (C) 2022-2024 IVAS codec Public Collaboration with portions copyright Dolby International AB, Ericsson AB, # Fraunhofer-Gesellschaft zur Foerderung der angewandten Forschung e.V., Huawei Technologies Co. LTD., # Koninklijke Philips N.V., Nippon Telegraph and Telephone Corporation, Nokia Technologies Oy, Orange, # Panasonic Holdings Corporation, Qualcomm Technologies, Inc., VoiceAge Corporation, and other @@ -28,27 +27,44 @@ # accordance with the laws of the Federal Republic of Germany excluding its conflict of law rules and # the United Nations Convention on Contracts on the International Sales of Goods. -# simple script to create MD5 files as if produced by Microsoft Powershell -# run from within proc_output_x folder - -if [ $# -lt 2 ]; then - echo "Usage: $0 proc_output_dir testname.md5" - echo "Requires: md5sum (Linux) or md5 (macOS)" - exit 1 -fi - -system=$(uname -s) - -if [ "$system" = "Darwin" ]; then - md5="md5 -r" -else - md5=md5sum -fi - -tmpfile=`mktemp` -echo "" > $tmpfile -for f in $(find $1 -name "*c??.wav"); do - echo ${f##*/} $($md5 $f | awk '{ print toupper($1) }') >> $tmpfile -done -cat $tmpfile | sort > $2 -rm -f $tmpfile +import argparse +from pathlib import Path +from hashlib import md5 + + +def get_hash_line_for_file(file: Path, experiment_dir: Path): + with open(file, "rb") as f: + hash = md5(f.read()).hexdigest() + + filepath = file.relative_to(experiment_dir) + # always print forward slashes even on windows to be able to diff the result files + hashline = f"{str(filepath.as_posix())} {hash}\n" + + return hashline + + +def main(experiment_dir, out_file): + wav_files = sorted(experiment_dir.glob("proc_output*/**/*c??.wav")) + + hashlines = [get_hash_line_for_file(f, experiment_dir) for f in wav_files] + + with open(out_file, "w") as f: + f.writelines(hashlines) + + +if __name__ == "__main__": + parser = argparse.ArgumentParser( + description="Cross-platform script for generating MD5 hashes of output files for the characterization testing experiments." + ) + parser.add_argument( + "experiment_dir", + type=Path, + help="Directory of the respective experiment (e.g. experiments/characterization/P800-12", + ) + parser.add_argument( + "out_file", type=Path, help="Output text file with filenames and Hashes" + ) + + args = parser.parse_args() + + main(args.experiment_dir, args.out_file) -- GitLab From 71d8323d04c6267a0c8303d73019520c28811d05 Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Thu, 4 Sep 2025 17:01:42 +0200 Subject: [PATCH 2/4] add legacy MD5 scripts back in attic dir --- other/attic/get_md5.ps1 | 42 ++++++++++++++++++++++++++++++++ other/attic/get_md5.sh | 54 +++++++++++++++++++++++++++++++++++++++++ 2 files changed, 96 insertions(+) create mode 100644 other/attic/get_md5.ps1 create mode 100755 other/attic/get_md5.sh diff --git a/other/attic/get_md5.ps1 b/other/attic/get_md5.ps1 new file mode 100644 index 00000000..c27d25f2 --- /dev/null +++ b/other/attic/get_md5.ps1 @@ -0,0 +1,42 @@ +# (C) 2022-2025 IVAS codec Public Collaboration with portions copyright Dolby International AB, Ericsson AB, +# Fraunhofer-Gesellschaft zur Foerderung der angewandten Forschung e.V., Huawei Technologies Co. LTD., +# Koninklijke Philips N.V., Nippon Telegraph and Telephone Corporation, Nokia Technologies Oy, Orange, +# Panasonic Holdings Corporation, Qualcomm Technologies, Inc., VoiceAge Corporation, and other +# contributors to this repository. All Rights Reserved. + +# This software is protected by copyright law and by international treaties. +# The IVAS codec Public Collaboration consisting of Dolby International AB, Ericsson AB, +# Fraunhofer-Gesellschaft zur Foerderung der angewandten Forschung e.V., Huawei Technologies Co. LTD., +# Koninklijke Philips N.V., Nippon Telegraph and Telephone Corporation, Nokia Technologies Oy, Orange, +# Panasonic Holdings Corporation, Qualcomm Technologies, Inc., VoiceAge Corporation, and other +# contributors to this repository retain full ownership rights in their respective contributions in +# the software. This notice grants no license of any kind, including but not limited to patent +# license, nor is any license granted by implication, estoppel or otherwise. + +# Contributors are required to enter into the IVAS codec Public Collaboration agreement before making +# contributions. + +# This software is provided "AS IS", without any express or implied warranties. The software is in the +# development stage. It is intended exclusively for experts who have experience with such software and +# solely for the purpose of inspection. All implied warranties of non-infringement, merchantability +# and fitness for a particular purpose are hereby disclaimed and excluded. + +# Any dispute, controversy or claim arising under or in relation to providing this software shall be +# submitted to and settled by the final, binding jurisdiction of the courts of Munich, Germany in +# accordance with the laws of the Federal Republic of Germany excluding its conflict of law rules and +# the United Nations Convention on Contracts on the International Sales of Goods. + +param( + [Parameter(Mandatory)] + [String]$proc_output_dir, + [Parameter(Mandatory)] + [String]$out_file +) + +(Get-ChildItem $proc_output_dir -Recurse | Where-Object { $_.Name -like '*c??.wav' }) | +ForEach-Object { + $name = $_.Name + $hash = (Get-FileHash $_.FullName -Algorithm MD5).Hash + + "$name $hash" >> $out_file +} \ No newline at end of file diff --git a/other/attic/get_md5.sh b/other/attic/get_md5.sh new file mode 100755 index 00000000..15caa467 --- /dev/null +++ b/other/attic/get_md5.sh @@ -0,0 +1,54 @@ +#!/bin/sh + +# (C) 2022-2025 IVAS codec Public Collaboration with portions copyright Dolby International AB, Ericsson AB, +# Fraunhofer-Gesellschaft zur Foerderung der angewandten Forschung e.V., Huawei Technologies Co. LTD., +# Koninklijke Philips N.V., Nippon Telegraph and Telephone Corporation, Nokia Technologies Oy, Orange, +# Panasonic Holdings Corporation, Qualcomm Technologies, Inc., VoiceAge Corporation, and other +# contributors to this repository. All Rights Reserved. + +# This software is protected by copyright law and by international treaties. +# The IVAS codec Public Collaboration consisting of Dolby International AB, Ericsson AB, +# Fraunhofer-Gesellschaft zur Foerderung der angewandten Forschung e.V., Huawei Technologies Co. LTD., +# Koninklijke Philips N.V., Nippon Telegraph and Telephone Corporation, Nokia Technologies Oy, Orange, +# Panasonic Holdings Corporation, Qualcomm Technologies, Inc., VoiceAge Corporation, and other +# contributors to this repository retain full ownership rights in their respective contributions in +# the software. This notice grants no license of any kind, including but not limited to patent +# license, nor is any license granted by implication, estoppel or otherwise. + +# Contributors are required to enter into the IVAS codec Public Collaboration agreement before making +# contributions. + +# This software is provided "AS IS", without any express or implied warranties. The software is in the +# development stage. It is intended exclusively for experts who have experience with such software and +# solely for the purpose of inspection. All implied warranties of non-infringement, merchantability +# and fitness for a particular purpose are hereby disclaimed and excluded. + +# Any dispute, controversy or claim arising under or in relation to providing this software shall be +# submitted to and settled by the final, binding jurisdiction of the courts of Munich, Germany in +# accordance with the laws of the Federal Republic of Germany excluding its conflict of law rules and +# the United Nations Convention on Contracts on the International Sales of Goods. + +# simple script to create MD5 files as if produced by Microsoft Powershell +# run from within proc_output_x folder + +if [ $# -lt 2 ]; then + echo "Usage: $0 proc_output_dir testname.md5" + echo "Requires: md5sum (Linux) or md5 (macOS)" + exit 1 +fi + +system=$(uname -s) + +if [ "$system" = "Darwin" ]; then + md5="md5 -r" +else + md5=md5sum +fi + +tmpfile=`mktemp` +echo "" > $tmpfile +for f in $(find $1 -name "*c??.wav"); do + echo ${f##*/} $($md5 $f | awk '{ print toupper($1) }') >> $tmpfile +done +cat $tmpfile | sort > $2 +rm -f $tmpfile -- GitLab From 1b05e2cc84ac66432c746d3db4d2f7439e31c78a Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Thu, 4 Sep 2025 17:08:34 +0200 Subject: [PATCH 3/4] add check for duplicate hash values --- other/get_md5.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/other/get_md5.py b/other/get_md5.py index 557c8a50..1bf6b7ef 100644 --- a/other/get_md5.py +++ b/other/get_md5.py @@ -30,6 +30,7 @@ import argparse from pathlib import Path from hashlib import md5 +from collections import Counter def get_hash_line_for_file(file: Path, experiment_dir: Path): @@ -47,6 +48,13 @@ def main(experiment_dir, out_file): wav_files = sorted(experiment_dir.glob("proc_output*/**/*c??.wav")) hashlines = [get_hash_line_for_file(f, experiment_dir) for f in wav_files] + count = Counter([line.split()[-1] for line in hashlines]) + duplicates = [line for line in hashlines if count[line.split()[-1]] != 1] + + if len(duplicates) != 0: + print("Found duplicate hashes in these lines:") + for dup in duplicates: + print(dup) with open(out_file, "w") as f: f.writelines(hashlines) -- GitLab From 2a30de5c0d61febe3c38dfed51515dcaa12ed9c1 Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Fri, 5 Sep 2025 15:28:44 +0200 Subject: [PATCH 4/4] Make glob more explicit to only get actual output conditions. Credits to @tamarapu --- other/get_md5.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/other/get_md5.py b/other/get_md5.py index 1bf6b7ef..e482dd3b 100644 --- a/other/get_md5.py +++ b/other/get_md5.py @@ -45,7 +45,7 @@ def get_hash_line_for_file(file: Path, experiment_dir: Path): def main(experiment_dir, out_file): - wav_files = sorted(experiment_dir.glob("proc_output*/**/*c??.wav")) + wav_files = sorted(experiment_dir.glob("proc_output*/**/*c[0-9][0-9].wav")) hashlines = [get_hash_line_for_file(f, experiment_dir) for f in wav_files] count = Counter([line.split()[-1] for line in hashlines]) -- GitLab