Commit 89beca73 authored by bayers's avatar bayers
Browse files

add possibility to compare encoder/decoder call runtimes from different run... 

add possibility to compare encoder/decoder call runtimes from different run logs, add OSBA modes to ivas modes and OSBA input formats, extend the script for preparing the combined formats to also produce files for OSBA
parent e6a32f2b

scripts/compare_runtimes_from_logs.py

0 → 100644
+97 −0
Original line number Diff line number Diff line 
#!/usr/bin/env python3



"""

   (C) 2022-2023 IVAS codec Public Collaboration with portions copyright Dolby International AB, Ericsson AB,

   Fraunhofer-Gesellschaft zur Foerderung der angewandten Forschung e.V., Huawei Technologies Co. LTD.,

   Koninklijke Philips N.V., Nippon Telegraph and Telephone Corporation, Nokia Technologies Oy, Orange,

   Panasonic Holdings Corporation, Qualcomm Technologies, Inc., VoiceAge Corporation, and other

   contributors to this repository. All Rights Reserved.



   This software is protected by copyright law and by international treaties.

   The IVAS codec Public Collaboration consisting of Dolby International AB, Ericsson AB,

   Fraunhofer-Gesellschaft zur Foerderung der angewandten Forschung e.V., Huawei Technologies Co. LTD.,

   Koninklijke Philips N.V., Nippon Telegraph and Telephone Corporation, Nokia Technologies Oy, Orange,

   Panasonic Holdings Corporation, Qualcomm Technologies, Inc., VoiceAge Corporation, and other

   contributors to this repository retain full ownership rights in their respective contributions in

   the software. This notice grants no license of any kind, including but not limited to patent

   license, nor is any license granted by implication, estoppel or otherwise.



   Contributors are required to enter into the IVAS codec Public Collaboration agreement before making

   contributions.



   This software is provided "AS IS", without any express or implied warranties. The software is in the

   development stage. It is intended exclusively for experts who have experience with such software and

   solely for the purpose of inspection. All implied warranties of non-infringement, merchantability

   and fitness for a particular purpose are hereby disclaimed and excluded.



   Any dispute, controversy or claim arising under or in relation to providing this software shall be

   submitted to and settled by the final, binding jurisdiction of the courts of Munich, Germany in

   accordance with the laws of the Federal Republic of Germany excluding its conflict of law rules and

   the United Nations Convention on Contracts on the International Sales of Goods.

"""



import os.path

import csv

import argparse



def parse_run_txt(filename):

    time_dict = {}

    with open(filename) as f:

        for line in f:

            if 'Encoding successful' in line:

                l = line.split()

                bin_file = os.path.basename(l[-4])

                execution_time = float(l[-2])

                time_dict[bin_file] = execution_time

            elif 'Decoding successful' in line:

                l = line.split()

                bin_file = os.path.basename(l[-4])

                execution_time = float(l[-2])

                time_dict[bin_file] = execution_time

    return time_dict



def main(args):



    td_ref = parse_run_txt(args.log1)

    td_test = parse_run_txt(args.log2)

    time_diff = {}

    for key in td_ref:

        if key in td_test:

            time_diff[key] = td_test[key] / td_ref[key] * 100



    sorted_time_diff = sorted(time_diff.items(), key=lambda item: item[1], reverse = True)



    if args.out_file is not None:

        with open(args.out_file, 'w', newline='') as f:

            writer = csv.writer(f)

            for entry in sorted_time_diff:

                writer.writerow([entry[0], entry[1]])



    if args.num_lines == -1:

        args.num_lines = len(sorted_time_diff)





    for idx in range(args.num_lines):

        print(f"{sorted_time_diff[idx][0]}: {sorted_time_diff[idx][1]}")





if __name__ == "__main__":



    parser = argparse.ArgumentParser(

        description="Compare execution times from two log file (Log file 1 as basis), output sorted by biggest increase"

    )

    parser.add_argument("log1", type=str, help="Log file 1")

    parser.add_argument("log2", type=str, help="Log file 2")

    parser.add_argument(

        "-n", "--num_lines", type=int, default=-1, help="Number of modes with the largest increases to output"

    )

    parser.add_argument(

        "-o",

        "--out_file",

        type=str,

        default=None,

        help="If given, write sorted changes to this fiel as comma separated values ",

    )

    args = parser.parse_args()



    main(args)
 
 No newline at end of file

scripts/config/ci_linux.json

+13 −1
Original line number Diff line number Diff line
@@ -36,6 +36,18 @@ 
        "OMASA_ISM4_1TC1DIR": "/usr/local/testv/stvOMASA_4ISM_1MASA1TC48c.wav",

        "OMASA_ISM4_1TC2DIR": "/usr/local/testv/stvOMASA_4ISM_2MASA1TC48c.wav",

        "OMASA_ISM4_2TC1DIR": "/usr/local/testv/stvOMASA_4ISM_1MASA2TC48c.wav",

        "OMASA_ISM4_2TC2DIR": "/usr/local/testv/stvOMASA_4ISM_2MASA2TC48c.wav"

        "OMASA_ISM4_2TC2DIR": "/usr/local/testv/stvOMASA_4ISM_2MASA2TC48c.wav",

        "OSBA_ISM1_FOA":      "/usr/local/testv/stvOSBA_1ISM_FOA48c.wav",

        "OSBA_ISM1_HOA2":     "/usr/local/testv/stvOSBA_1ISM_2OA48c.wav",

        "OSBA_ISM1_HOA3":     "/usr/local/testv/stvOSBA_1ISM_3OA48c.wav",

        "OSBA_ISM2_FOA":      "/usr/local/testv/stvOSBA_2ISM_FOA48c.wav",

        "OSBA_ISM2_HOA2":     "/usr/local/testv/stvOSBA_2ISM_2OA48c.wav",

        "OSBA_ISM2_HOA3":     "/usr/local/testv/stvOSBA_2ISM_3OA48c.wav",

        "OSBA_ISM3_FOA":      "/usr/local/testv/stvOSBA_3ISM_FOA48c.wav",

        "OSBA_ISM3_HOA2":     "/usr/local/testv/stvOSBA_3ISM_2OA48c.wav",

        "OSBA_ISM3_HOA3":     "/usr/local/testv/stvOSBA_3ISM_3OA48c.wav",

        "OSBA_ISM4_FOA":      "/usr/local/testv/stvOSBA_4ISM_FOA48c.wav",

        "OSBA_ISM4_HOA2":     "/usr/local/testv/stvOSBA_4ISM_2OA48c.wav",

        "OSBA_ISM4_HOA3":     "/usr/local/testv/stvOSBA_4ISM_3OA48c.wav"

    }

}

scripts/config/ivas_modes.json

+1383 −1

File changed.

Preview size limit exceeded, changes collapsed.

scripts/prepare_combined_format_inputs.py

+26 −0
Original line number Diff line number Diff line
@@ -65,6 +65,13 @@ masa_alts = ({'masa_meta_file': 'stv2MASA2TC48c.met', 
              'masa_audio_file': 'stv1MASA1TC48c.wav',

              'masa_tag': '1MASA1TC'})



sba_alts = ({'sba_audio_file': 'stvFOA48c.wav',

              'sba_tag': 'FOA'},

             {'sba_audio_file': 'stv2OA48c.wav',

              'sba_tag': '2OA'},

             {'sba_audio_file': 'stv3OA48c.wav',

              'sba_tag': '3OA'})



# files containing 1-4 ISMs as channels

ism_files = ('stv1ISM48s.wav', 'stv2ISM48s.wav', 'stv3ISM48s.wav', 'stv4ISM48s.wav')

# per-object metadata
@@ -100,6 +107,25 @@ for enum_idx, (ism_audio_file, ism_meta_file) in enumerate(zip(ism_files, ism_me 
        if not os.path.exists(masa_meta_name) or force_overwrite:

            shutil.copyfile(os.path.join(input_dir, masa_item['masa_meta_file']), masa_meta_name)

            wrote_files.append(masa_meta_name)

    for sba_item in sba_alts:

        sba_tag = sba_item['sba_tag']



        osba_file_body = f'stvOSBA_{n_isms}ISM_{sba_tag}48c'

        osba_file = os.path.join(output_dir, f'{osba_file_body}.wav')



        if not os.path.exists(osba_file) or force_overwrite:

            audiofile.combinefiles(in_filenames=[os.path.join(input_dir, ism_audio_file),

                                                os.path.join(input_dir, sba_item['sba_audio_file'])],

                                out_file=osba_file)

            wrote_files.append(osba_file)



        # copy ISM metadata files under names matching the combined file

        for ism_idx in range(n_isms):

            ism_file_name = os.path.join(output_dir, f'{osba_file_body}_ISM{ism_idx+1}{os.path.splitext(ism_meta_file)[1]}')

            if not os.path.exists(ism_file_name) or force_overwrite:

                shutil.copyfile(os.path.join(input_dir, ism_meta_file), ism_file_name)

                wrote_files.append(ism_file_name)





# info print. helps setting up .gitignore

if len(wrote_files) > 0:

scripts/pyivastest/IvasModeRunner.py

+8 −3
Original line number Diff line number Diff line
@@ -42,6 +42,7 @@ import traceback 
from copy import deepcopy

import re

import json

import time



from pyivastest import IvasModeCollector

from pyivastest import constants
@@ -376,7 +377,9 @@ class IvasModeRunner(IvasModeCollector.IvasModeCollector): 
                    + [enc_file_name, dec_file_name]

                )



            st = time.time()

            cur_dec_result = subprocess.run(dec_cmd, capture_output=True, text=True)

            et = time.time()

            dec_log.write(" ".join(dec_cmd))

            dec_log.write(cur_dec_result.stderr)

            dec_log.write(cur_dec_result.stdout)
@@ -412,8 +415,8 @@ class IvasModeRunner(IvasModeCollector.IvasModeCollector): 
                self.lock.release()

            else:

                self.logger.info(

                    "Decoding successful for {} to {}".format(

                        enc_file_name, dec_file_name

                    "Decoding successful for {} to {} in {} seconds".format(

                        enc_file_name, dec_file_name, et - st

                    )

                )

        else:
@@ -808,7 +811,9 @@ class IvasModeRunner(IvasModeCollector.IvasModeCollector): 


            enc_log = open(enc_log_name, "w")

            enc_log.write(" ".join(enc_cmd))

            st = time.time()

            enc_result = subprocess.run(enc_cmd, capture_output=True, text=True)

            et = time.time()

            error = enc_result.returncode

            enc_log.write(enc_result.stderr)

            enc_log.write(enc_result.stdout)
@@ -915,7 +920,7 @@ class IvasModeRunner(IvasModeCollector.IvasModeCollector): 
            self.lock.release()

            self.logger.error("Encoding failed for {}".format(enc_file_name))

        else:

            self.logger.info("Encoding successful for {}".format(enc_file_name))

            self.logger.info("Encoding successful for {} in {} seconds".format(enc_file_name, et-st))



        self.dec_queue["condition"].acquire()

        # create the entry in the decoder queue