From 53b032819c402ec9613ebedbe0bdd9ca8e48f90d Mon Sep 17 00:00:00 2001 From: Charles Kinuthia Date: Wed, 10 May 2023 14:49:37 +0200 Subject: [PATCH 1/2] Add EVS BE test for windows executables to ci pipeline Assumes official EVS BE test sequences are placed at "C:/Users/gitlab-runner/testvec" on the windows runners. --- .gitlab-ci.yml | 25 +++++++ ci/run_evs_be_win_test.py | 149 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 174 insertions(+) create mode 100644 ci/run_evs_be_win_test.py diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 7ce29dfd65..d61ad715f6 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -3,6 +3,7 @@ variables: LTV_DIR: "/usr/local/ltv" BUILD_OUTPUT: "build_output.txt" EVS_BE_TEST_DIR: "/usr/local/be_2_evs_test" + EVS_BE_WIN_TEST_DIR: "C:/Users/gitlab-runner/testvec" SANITIZER_TESTS: "CLANG1 CLANG2" OUT_FORMATS_CHANNEL_BASED: "stereo mono 5_1 5_1_2 5_1_4 7_1 7_1_4" OUT_FORMATS_SCENE_BASED: "FOA HOA2 HOA3" @@ -653,6 +654,30 @@ clang-format-check: # Test jobs for main branch # --------------------------------------------------------------- +# check bitexactness to EVS windows binaries +be-2-evs-windows: + extends: + - .rules-basis + tags: + - ericsson-windows-runner + stage: test + needs: ["build-codec-windows-msbuild"] + timeout: "20 minutes" # To be revisited + script: + - *print-common-info-windows + + - $winoutdata = $null + - MSBuild.exe .\Workspace_msvc\Workspace_msvc.sln /property:Configuration=Release | tee -variable winoutdata + - $winoutdata | Out-File $BUILD_OUTPUT -Encoding Utf8 + + # copy over to never change the testvector dir + - cp -r $EVS_BE_WIN_TEST_DIR ./evs_be_win_test + - cp IVAS_cod.exe ./evs_be_win_test/bin/IVAS_cod.exe + - cp IVAS_dec.exe ./evs_be_win_test/bin/IVAS_dec.exe + + - cd evs_be_win_test + - python ../ci/run_evs_be_win_test.py + # check bitexactness to EVS be-2-evs-linux: extends: diff --git a/ci/run_evs_be_win_test.py b/ci/run_evs_be_win_test.py new file mode 100644 index 0000000000..f1bcd7f3a4 --- /dev/null +++ b/ci/run_evs_be_win_test.py @@ -0,0 +1,149 @@ +""" + (C) 2022-2023 IVAS codec Public Collaboration with portions copyright Dolby International AB, Ericsson AB, + Fraunhofer-Gesellschaft zur Foerderung der angewandten Forschung e.V., Huawei Technologies Co. LTD., + Koninklijke Philips N.V., Nippon Telegraph and Telephone Corporation, Nokia Technologies Oy, Orange, + Panasonic Holdings Corporation, Qualcomm Technologies, Inc., VoiceAge Corporation, and other + contributors to this repository. All Rights Reserved. + + This software is protected by copyright law and by international treaties. + The IVAS codec Public Collaboration consisting of Dolby International AB, Ericsson AB, + Fraunhofer-Gesellschaft zur Foerderung der angewandten Forschung e.V., Huawei Technologies Co. LTD., + Koninklijke Philips N.V., Nippon Telegraph and Telephone Corporation, Nokia Technologies Oy, Orange, + Panasonic Holdings Corporation, Qualcomm Technologies, Inc., VoiceAge Corporation, and other + contributors to this repository retain full ownership rights in their respective contributions in + the software. This notice grants no license of any kind, including but not limited to patent + license, nor is any license granted by implication, estoppel or otherwise. + + Contributors are required to enter into the IVAS codec Public Collaboration agreement before making + contributions. + + This software is provided "AS IS", without any express or implied warranties. The software is in the + development stage. It is intended exclusively for experts who have experience with such software and + solely for the purpose of inspection. All implied warranties of non-infringement, merchantability + and fitness for a particular purpose are hereby disclaimed and excluded. + + Any dispute, controversy or claim arising under or in relation to providing this software shall be + submitted to and settled by the final, binding jurisdiction of the courts of Munich, Germany in + accordance with the laws of the Federal Republic of Germany excluding its conflict of law rules and + the United Nations Convention on Contracts on the International Sales of Goods. +""" + +from multiprocessing import Pool +import argparse +import subprocess +import os +import shutil +import sys + + +def run_condition(eval_cmd, diff_cmd, id_count): + """ Run ENC or DEC command string and compare output with EVS test vectors. """ + + cmd = subprocess.run(eval_cmd.split(), capture_output=True, text=True, check=True) + + # diff + diff_success = 1 + if ';' in diff_cmd: + diff_cmd1, diff_cmd2 = diff_cmd.split(';') + cmd1 = subprocess.run(diff_cmd1.split(), capture_output=True, text=True, check=True) + cmd2 = subprocess.run(diff_cmd2.split(), capture_output=True, text=True, check=True) + diff_success = cmd1.returncode + cmd2.returncode + else: + cmd = subprocess.run(diff_cmd.split(), capture_output=True, text=True, check=True) + diff_success = cmd.returncode + if diff_success == 0: + return None + else: + return f'[{str(id_count).rjust(3)}] FAIL: {" ".join(eval_cmd)}\n {diff_cmd}\n' + + +def environment_is_correct(paths): + """ + Check that the folder with the test resources is set up correctly: + - all Readme files there + - EVS binaries available in bin/ + - testv and switchPaths folder exist - Content is not checked, though + """ + ret = True + + for pth in paths: + if not os.path.exists(pth): + print(f"Environment setup is incorrect - {pth} not found.") + ret = False + + return ret + + +if __name__ == '__main__': + parser = argparse.ArgumentParser( + description='Run 26.444 test with parallel processes ') + parser.add_argument('-p', type=int, default=os.cpu_count(), + help='Number of processes (default cpu_count)') + parser.add_argument('-test_dir', type=str, default='./', + help='testvec directory from 26.444)') + parser.add_argument('-enc_bin', type=str, default='./bin/IVAS_cod.exe', + help='Encoder binary (default ./bin/IVAS_cod.exe)') + parser.add_argument('-dec_bin', type=str, default='./bin/IVAS_dec.exe', + help='Decoder binary (default ./bin/IVAS_dec.exe)') + + args = parser.parse_args() + test_vec_dir = args.test_dir + processes = args.p + enc_bin = args.enc_bin + dec_bin = args.dec_bin + + README_FILES = ['Readme_AMRWB_IO_dec.txt', 'Readme_AMRWB_IO_enc.txt', 'Readme_EVS_dec.txt', + 'Readme_EVS_enc.txt', 'Readme_JBM_dec.txt'] + + scripts = [os.path.join(test_vec_dir, script) for script in README_FILES] + + if not environment_is_correct([f'{test_vec_dir}/testv'] + scripts + [enc_bin, dec_bin]): + sys.exit(1) + + pool = Pool(processes) + results = [] + id_count = 0 + + for script in scripts: + with open(script) as file: + tmp_dir = None + eval_cmd = None + diff_cmd = '' + for line in file: + if line.startswith('TMP='): + assert tmp_dir is None + tmp_dir = line.split('"')[1] + if os.path.exists(tmp_dir): + shutil.rmtree(tmp_dir) + os.makedirs(tmp_dir) + line = line if tmp_dir is None else line.replace( + '$TMP/', f'{tmp_dir}/') + if '$CUT_DEC_BIN' in line: + eval_cmd = dec_bin + ' -q ' + ' '.join(line.split()[1:]) + if '$CUT_ENC_BIN' in line: + eval_cmd = enc_bin + ' -q ' + ' '.join(line.split()[1:]) + if '$DIFF_BIN' in line: + if 'Readme_JBM_dec.txt' in script: + if '-w' not in line.split()[1]: + diff_cmd = 'FC.exe ' + ' '.join(line.split()[1:]).replace('/', '\\').replace('\n', '/n') + continue + diff_cmd += '; FC.exe ' + ' '.join(line.split()[1:]).replace('/', '\\').replace('\n', '/n').replace('-w', '/W') + else: + diff_cmd = 'FC.exe ' + ' '.join(line.split()[1:]).replace('/', '\\').replace('\n', '/n') + results.append(pool.apply_async( + run_condition, args=(eval_cmd, diff_cmd, id_count))) + id_count = id_count + 1 + print('Total number of conditions for', '"' + + os.path.basename(script) + '": ' + str(id_count - 1)) + + results = [r.get() for r in results if r.get()] + if results: + print(f'\n{len(results)} test conditions failed:') + print('\n'.join(results)) + with open('failed.txt', 'w') as f: + print(f'\n{len(results)} test conditions failed:', file=f) + print('\n'.join(results), file=f) + sys.exit(1) + else: + print('\n *** All tests passed! ***') + sys.exit(0) -- GitLab From 26f6a0dd3035945303ac339eb58133147a686c6e Mon Sep 17 00:00:00 2001 From: Charles Kinuthia Date: Wed, 10 May 2023 15:45:13 +0200 Subject: [PATCH 2/2] use `ivas-windows` tag for `be-2-evs-windows job` --- .gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index d61ad715f6..f60b9bdb14 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -659,7 +659,7 @@ be-2-evs-windows: extends: - .rules-basis tags: - - ericsson-windows-runner + - ivas-windows stage: test needs: ["build-codec-windows-msbuild"] timeout: "20 minutes" # To be revisited -- GitLab