Commit f5515e03 authored by Archit Tamarapu's avatar Archit Tamarapu
Browse files

[fix] logging improvements 

- avoid multiple logging handlers being added when the generate_test.py wrapper script is used
- add a busy indicator for CLI
- cleanup some leftover logging code
- formatting
parent 26090149

generate_test.py

+21 −6
Original line number Diff line number Diff line
@@ -35,15 +35,28 @@ from pathlib import Path 


from ivas_processing_scripts import config

from ivas_processing_scripts import main as generate_test

from ivas_processing_scripts.utils import apply_func_parallel



HERE = Path(__file__).parent.absolute()

EXPERIMENTS_P800 = [f"P800-{i}" for i in range(1, 10)]

EXPERIMENTS_BS1534 = [f"BS1534-{i}{x}" for i in range(1, 8) for x in ["a", "b"]]

LAB_IDS = ["a", "b", "c", "d"]

IN_FMT_FOR_MASA_EXPS = {

    "P800-8": {"cat1": "FOA", "cat2": "FOA", "cat3": "FOA", "cat4": "FOA", "cat5": "FOA", "cat6": "FOA"},

    "P800-9": {"cat1": "FOA", "cat2": "FOA", "cat3": "FOA", "cat4": "FOA", "cat5": "FOA", "cat6": "FOA"},

    "P800-8": {

        "cat1": "FOA",

        "cat2": "FOA",

        "cat3": "FOA",

        "cat4": "FOA",

        "cat5": "FOA",

        "cat6": "FOA",

    },

    "P800-9": {

        "cat1": "FOA",

        "cat2": "FOA",

        "cat3": "FOA",

        "cat4": "FOA",

        "cat5": "FOA",

        "cat6": "FOA",

    },

    "BS1534-7a": {"cat1": "FOA", "cat2": "HOA2"},

    "BS1534-7b": {"cat1": "FOA", "cat2": "HOA2"},

}
@@ -63,8 +76,8 @@ def generate_tests(exp_lab_pairs, create_cfg_only=False): 
        print("\n".join(["- " + cfg.name for cfg in cfgs]))

        return



    args = [Arguments(str(cfg)) for cfg in cfgs]

    apply_func_parallel(generate_test, zip(args), None)

    for cfg in cfgs:

        generate_test(Arguments(str(cfg)))





class Arguments:
@@ -124,7 +137,9 @@ def create_experiment_setup(experiment, lab) -> list[Path]: 
            if experiment in ["P800-5", "P800-9"] and cat_num >= 3:

                bg_noise_pre_proc_2["snr"] = 15

        if cfg.preprocessing_2.get("concatenate_input", None) is not None:

            cfg.preprocessing_2["concatenation_order"] = concatenation_order(lab, experiment, cat_num)

            cfg.preprocessing_2["concatenation_order"] = concatenation_order(

                lab, experiment, cat_num

            )



        # for MASA, the input format can differ between categories

        if (fmt_for_category := IN_FMT_FOR_MASA_EXPS.get(experiment, None)) is not None:

ivas_processing_scripts/__init__.py

+7 −1
Original line number Diff line number Diff line
@@ -54,12 +54,13 @@ from ivas_processing_scripts.utils import ( 
    DirManager,

    apply_func_parallel,

    progressbar_update,

    spinner,

)





def logging_init(args, cfg):

    """set up logging for a test file"""

    logger = logging.getLogger("__main__")

    logger = logging.getLogger("ivas_processing_scripts")

    logger.setLevel(logging.DEBUG)



    # console handler
@@ -155,6 +156,8 @@ def main(args): 
            preprocess_2(cfg, logger)



        # assemble a list of all item and condition combinations

        logger.info("  Generating remaining conditions with postprocessing")



        item_args = list()

        for (chain, tmp_dir, out_dir), (item, metadata) in product(

            zip(cfg.proc_chains, cfg.tmp_dirs, cfg.out_dirs),
@@ -180,6 +183,7 @@ def main(args): 
            progressbar_update(0, count, width)

            while not results.ready():

                progressbar_update(count - int(results._number_left), count, width)

                spinner()

                sleep(0.1)

            progressbar_update(count, count, width)

            print("\n", flush=True, file=sys.stdout)
@@ -192,3 +196,5 @@ def main(args): 


    # copy configuration to output directory

    cfg.to_file(cfg.output_path.joinpath(f"{cfg.name}.yml"))



    logger.handlers.clear()

ivas_processing_scripts/audiotools/audioarray.py

+0 −5
Original line number Diff line number Diff line
@@ -30,7 +30,6 @@ 
#  the United Nations Convention on Contracts on the International Sales of Goods.

#



import logging

from typing import Iterator, Optional, Tuple, Union

from warnings import warn
@@ -39,10 +38,6 @@ import scipy.signal as sig 


from .constants import DELAY_COMPENSATION_FOR_FILTERING, SEED_PADDING



logger = logging.getLogger("__main__")

logger.setLevel(logging.DEBUG)





"""Functions used in this module"""

ivas_processing_scripts/audiotools/audiofile.py

+3 −1
Original line number Diff line number Diff line
@@ -145,7 +145,9 @@ def write( 
        np.logical_or(x < np.iinfo(np.int16).min, x > np.iinfo(np.int16).max)

    )

    if clipped_samples > 0 and clipping_warning:

        logger.warning(f"  Warning: {clipped_samples} samples clipped while writing {filename}!")

        logger.warning(

            f"  Warning: {clipped_samples} samples clipped while writing {filename}!"

        )

        x = np.clip(x, np.iinfo(np.int16).min, np.iinfo(np.int16).max)



    if file_extension == ".wav":

ivas_processing_scripts/generation/__init__.py

+4 −2
Original line number Diff line number Diff line
@@ -51,7 +51,7 @@ from ivas_processing_scripts.utils import create_dir 


def logging_init(args, cfg):

    """set up logging for a test file"""

    logger = logging.getLogger("__main__")

    logger = logging.getLogger("ivas_processing_scripts.generation")

    logger.setLevel(logging.DEBUG)



    # console handler
@@ -68,7 +68,7 @@ def logging_init(args, cfg): 
    file_handler.setLevel(logging.DEBUG if args.debug else logging.INFO)

    logger.addHandler(file_handler)



    logger.info(f"Processing test configuration file {args.config}")

    logger.info(f"Processing item generation configuration file {args.config}")

    logger.info(f"Input path: {cfg.input_path.absolute()}")

    logger.info(f"Output path: {cfg.output_path.absolute()}")
@@ -107,3 +107,5 @@ def main(args): 
    # copy configuration to output directory

    with open(cfg.output_path.joinpath(f"{'_'.join(cfg.format)}.yml"), "w") as f:

        yaml.safe_dump(cfg._yaml_dump, f)



    logger.handlers.clear()