Merge branch '449-add-evs-be-test-for-windows-executables-to-ci-pipeline' into 'main'

  LTV_DIR: "/usr/local/ltv"

  BUILD_OUTPUT: "build_output.txt"

  EVS_BE_TEST_DIR: "/usr/local/be_2_evs_test"

  EVS_BE_WIN_TEST_DIR: "C:/Users/gitlab-runner/testvec"

  SANITIZER_TESTS: "CLANG1 CLANG2"

  OUT_FORMATS_CHANNEL_BASED: "stereo mono 5_1 5_1_2 5_1_4 7_1 7_1_4"

  OUT_FORMATS_SCENE_BASED: "FOA HOA2 HOA3"

# Test jobs for main branch

# ---------------------------------------------------------------

# check bitexactness to EVS windows binaries

be-2-evs-windows:

  extends:

    - .rules-basis

  tags:

    - ivas-windows

  stage: test

  needs: ["build-codec-windows-msbuild"]

  timeout: "20 minutes" # To be revisited

  script:

    - *print-common-info-windows

    - $winoutdata = $null

    - MSBuild.exe .\Workspace_msvc\Workspace_msvc.sln /property:Configuration=Release | tee -variable winoutdata

    - $winoutdata | Out-File $BUILD_OUTPUT -Encoding Utf8

    # copy over to never change the testvector dir

    - cp -r $EVS_BE_WIN_TEST_DIR ./evs_be_win_test

    - cp IVAS_cod.exe ./evs_be_win_test/bin/IVAS_cod.exe

    - cp IVAS_dec.exe ./evs_be_win_test/bin/IVAS_dec.exe

    - cd evs_be_win_test

    - python ../ci/run_evs_be_win_test.py

# check bitexactness to EVS

be-2-evs-linux:

  extends:

"""

   (C) 2022-2023 IVAS codec Public Collaboration with portions copyright Dolby International AB, Ericsson AB,

   Fraunhofer-Gesellschaft zur Foerderung der angewandten Forschung e.V., Huawei Technologies Co. LTD.,

   Koninklijke Philips N.V., Nippon Telegraph and Telephone Corporation, Nokia Technologies Oy, Orange,

   Panasonic Holdings Corporation, Qualcomm Technologies, Inc., VoiceAge Corporation, and other

   contributors to this repository. All Rights Reserved.

   This software is protected by copyright law and by international treaties.

   The IVAS codec Public Collaboration consisting of Dolby International AB, Ericsson AB,

   Fraunhofer-Gesellschaft zur Foerderung der angewandten Forschung e.V., Huawei Technologies Co. LTD.,

   Koninklijke Philips N.V., Nippon Telegraph and Telephone Corporation, Nokia Technologies Oy, Orange,

   Panasonic Holdings Corporation, Qualcomm Technologies, Inc., VoiceAge Corporation, and other

   contributors to this repository retain full ownership rights in their respective contributions in

   the software. This notice grants no license of any kind, including but not limited to patent

   license, nor is any license granted by implication, estoppel or otherwise.

   Contributors are required to enter into the IVAS codec Public Collaboration agreement before making

   contributions.

   This software is provided "AS IS", without any express or implied warranties. The software is in the

   development stage. It is intended exclusively for experts who have experience with such software and

   solely for the purpose of inspection. All implied warranties of non-infringement, merchantability

   and fitness for a particular purpose are hereby disclaimed and excluded.

   Any dispute, controversy or claim arising under or in relation to providing this software shall be

   submitted to and settled by the final, binding jurisdiction of the courts of Munich, Germany in

   accordance with the laws of the Federal Republic of Germany excluding its conflict of law rules and

   the United Nations Convention on Contracts on the International Sales of Goods.

"""

from multiprocessing import Pool

import argparse

import subprocess

import os

import shutil

import sys

def run_condition(eval_cmd, diff_cmd, id_count):

    """ Run ENC or DEC command string and compare output with EVS test vectors. """

    cmd = subprocess.run(eval_cmd.split(), capture_output=True, text=True, check=True)

    # diff

    diff_success = 1

    if ';' in diff_cmd:

        diff_cmd1, diff_cmd2 = diff_cmd.split(';')

        cmd1 = subprocess.run(diff_cmd1.split(), capture_output=True, text=True, check=True)

        cmd2 = subprocess.run(diff_cmd2.split(), capture_output=True, text=True, check=True)

        diff_success = cmd1.returncode + cmd2.returncode

    else:

        cmd = subprocess.run(diff_cmd.split(), capture_output=True, text=True, check=True)

        diff_success = cmd.returncode

    if diff_success == 0:

        return None

    else:

        return f'[{str(id_count).rjust(3)}] FAIL: {" ".join(eval_cmd)}\n            {diff_cmd}\n'

def environment_is_correct(paths):

    """

    Check that the folder with the test resources is set up correctly:

    - all Readme files there

    - EVS binaries available in bin/

    - testv and switchPaths folder exist - Content is not checked, though

    """

    ret = True

    for pth in paths:

        if not os.path.exists(pth):

            print(f"Environment setup is incorrect - {pth} not found.")

            ret = False

    return ret

if __name__ == '__main__':

    parser = argparse.ArgumentParser(

        description='Run 26.444 test with parallel processes ')

    parser.add_argument('-p', type=int, default=os.cpu_count(),

                        help='Number of processes (default cpu_count)')

    parser.add_argument('-test_dir', type=str, default='./',

                        help='testvec directory from 26.444)')

    parser.add_argument('-enc_bin', type=str, default='./bin/IVAS_cod.exe',

                        help='Encoder binary (default ./bin/IVAS_cod.exe)')

    parser.add_argument('-dec_bin', type=str, default='./bin/IVAS_dec.exe',

                        help='Decoder binary (default ./bin/IVAS_dec.exe)')

    args = parser.parse_args()

    test_vec_dir = args.test_dir

    processes = args.p

    enc_bin = args.enc_bin

    dec_bin = args.dec_bin

    README_FILES = ['Readme_AMRWB_IO_dec.txt', 'Readme_AMRWB_IO_enc.txt', 'Readme_EVS_dec.txt',

                    'Readme_EVS_enc.txt', 'Readme_JBM_dec.txt']

    scripts = [os.path.join(test_vec_dir, script) for script in README_FILES]

    if not environment_is_correct([f'{test_vec_dir}/testv'] + scripts + [enc_bin, dec_bin]):

        sys.exit(1)

    pool = Pool(processes)

    results = []

    id_count = 0

    for script in scripts:

        with open(script) as file:

            tmp_dir = None

            eval_cmd = None

            diff_cmd = ''

            for line in file:

                if line.startswith('TMP='):

                    assert tmp_dir is None

                    tmp_dir = line.split('"')[1]

                    if os.path.exists(tmp_dir):

                        shutil.rmtree(tmp_dir)

                    os.makedirs(tmp_dir)

                line = line if tmp_dir is None else line.replace(

                    '$TMP/', f'{tmp_dir}/')

                if '$CUT_DEC_BIN' in line:

                    eval_cmd = dec_bin + ' -q ' + ' '.join(line.split()[1:])

                if '$CUT_ENC_BIN' in line:

                    eval_cmd = enc_bin + ' -q ' + ' '.join(line.split()[1:])

                if '$DIFF_BIN' in line:

                    if 'Readme_JBM_dec.txt' in script:

                        if '-w' not in line.split()[1]:

                            diff_cmd = 'FC.exe ' + ' '.join(line.split()[1:]).replace('/', '\\').replace('\n', '/n')

                            continue

                        diff_cmd += '; FC.exe ' + ' '.join(line.split()[1:]).replace('/', '\\').replace('\n', '/n').replace('-w', '/W')

                    else:

                        diff_cmd = 'FC.exe ' + ' '.join(line.split()[1:]).replace('/', '\\').replace('\n', '/n')

                    results.append(pool.apply_async(

                        run_condition, args=(eval_cmd, diff_cmd, id_count)))

                    id_count = id_count + 1

            print('Total number of conditions for', '"' +

                  os.path.basename(script) + '": ' + str(id_count - 1))

    results = [r.get() for r in results if r.get()]

    if results:

        print(f'\n{len(results)} test conditions failed:')

        print('\n'.join(results))

        with open('failed.txt', 'w') as f:

            print(f'\n{len(results)} test conditions failed:', file=f)

            print('\n'.join(results), file=f)

        sys.exit(1)

    else:

        print('\n *** All tests passed! ***')

        sys.exit(0)