From f5515e0331313aecd4e652dbf64d61c8448f2096 Mon Sep 17 00:00:00 2001 From: Archit Tamarapu Date: Thu, 15 Jun 2023 14:13:38 +0200 Subject: [PATCH] [fix] logging improvements - avoid multiple logging handlers being added when the generate_test.py wrapper script is used - add a busy indicator for CLI - cleanup some leftover logging code - formatting --- generate_test.py | 27 ++++++++++++++----- ivas_processing_scripts/__init__.py | 8 +++++- .../audiotools/audioarray.py | 5 ---- .../audiotools/audiofile.py | 4 ++- .../generation/__init__.py | 6 +++-- .../processing/processing.py | 13 +++++++-- ivas_processing_scripts/utils.py | 12 ++++++--- 7 files changed, 55 insertions(+), 20 deletions(-) diff --git a/generate_test.py b/generate_test.py index 6233b421..d5648b41 100755 --- a/generate_test.py +++ b/generate_test.py @@ -35,15 +35,28 @@ from pathlib import Path from ivas_processing_scripts import config from ivas_processing_scripts import main as generate_test -from ivas_processing_scripts.utils import apply_func_parallel HERE = Path(__file__).parent.absolute() EXPERIMENTS_P800 = [f"P800-{i}" for i in range(1, 10)] EXPERIMENTS_BS1534 = [f"BS1534-{i}{x}" for i in range(1, 8) for x in ["a", "b"]] LAB_IDS = ["a", "b", "c", "d"] IN_FMT_FOR_MASA_EXPS = { - "P800-8": {"cat1": "FOA", "cat2": "FOA", "cat3": "FOA", "cat4": "FOA", "cat5": "FOA", "cat6": "FOA"}, - "P800-9": {"cat1": "FOA", "cat2": "FOA", "cat3": "FOA", "cat4": "FOA", "cat5": "FOA", "cat6": "FOA"}, + "P800-8": { + "cat1": "FOA", + "cat2": "FOA", + "cat3": "FOA", + "cat4": "FOA", + "cat5": "FOA", + "cat6": "FOA", + }, + "P800-9": { + "cat1": "FOA", + "cat2": "FOA", + "cat3": "FOA", + "cat4": "FOA", + "cat5": "FOA", + "cat6": "FOA", + }, "BS1534-7a": {"cat1": "FOA", "cat2": "HOA2"}, "BS1534-7b": {"cat1": "FOA", "cat2": "HOA2"}, } @@ -63,8 +76,8 @@ def generate_tests(exp_lab_pairs, create_cfg_only=False): print("\n".join(["- " + cfg.name for cfg in cfgs])) return - args = [Arguments(str(cfg)) for cfg in cfgs] - apply_func_parallel(generate_test, zip(args), None) + for cfg in cfgs: + generate_test(Arguments(str(cfg))) class Arguments: @@ -124,7 +137,9 @@ def create_experiment_setup(experiment, lab) -> list[Path]: if experiment in ["P800-5", "P800-9"] and cat_num >= 3: bg_noise_pre_proc_2["snr"] = 15 if cfg.preprocessing_2.get("concatenate_input", None) is not None: - cfg.preprocessing_2["concatenation_order"] = concatenation_order(lab, experiment, cat_num) + cfg.preprocessing_2["concatenation_order"] = concatenation_order( + lab, experiment, cat_num + ) # for MASA, the input format can differ between categories if (fmt_for_category := IN_FMT_FOR_MASA_EXPS.get(experiment, None)) is not None: diff --git a/ivas_processing_scripts/__init__.py b/ivas_processing_scripts/__init__.py index 4b508fc7..77c95905 100755 --- a/ivas_processing_scripts/__init__.py +++ b/ivas_processing_scripts/__init__.py @@ -54,12 +54,13 @@ from ivas_processing_scripts.utils import ( DirManager, apply_func_parallel, progressbar_update, + spinner, ) def logging_init(args, cfg): """set up logging for a test file""" - logger = logging.getLogger("__main__") + logger = logging.getLogger("ivas_processing_scripts") logger.setLevel(logging.DEBUG) # console handler @@ -155,6 +156,8 @@ def main(args): preprocess_2(cfg, logger) # assemble a list of all item and condition combinations + logger.info(" Generating remaining conditions with postprocessing") + item_args = list() for (chain, tmp_dir, out_dir), (item, metadata) in product( zip(cfg.proc_chains, cfg.tmp_dirs, cfg.out_dirs), @@ -180,6 +183,7 @@ def main(args): progressbar_update(0, count, width) while not results.ready(): progressbar_update(count - int(results._number_left), count, width) + spinner() sleep(0.1) progressbar_update(count, count, width) print("\n", flush=True, file=sys.stdout) @@ -192,3 +196,5 @@ def main(args): # copy configuration to output directory cfg.to_file(cfg.output_path.joinpath(f"{cfg.name}.yml")) + + logger.handlers.clear() diff --git a/ivas_processing_scripts/audiotools/audioarray.py b/ivas_processing_scripts/audiotools/audioarray.py index 50096ba1..090095b8 100755 --- a/ivas_processing_scripts/audiotools/audioarray.py +++ b/ivas_processing_scripts/audiotools/audioarray.py @@ -30,7 +30,6 @@ # the United Nations Convention on Contracts on the International Sales of Goods. # -import logging from typing import Iterator, Optional, Tuple, Union from warnings import warn @@ -39,10 +38,6 @@ import scipy.signal as sig from .constants import DELAY_COMPENSATION_FOR_FILTERING, SEED_PADDING -logger = logging.getLogger("__main__") -logger.setLevel(logging.DEBUG) - - """Functions used in this module""" diff --git a/ivas_processing_scripts/audiotools/audiofile.py b/ivas_processing_scripts/audiotools/audiofile.py index efdd2cab..b29ba670 100755 --- a/ivas_processing_scripts/audiotools/audiofile.py +++ b/ivas_processing_scripts/audiotools/audiofile.py @@ -145,7 +145,9 @@ def write( np.logical_or(x < np.iinfo(np.int16).min, x > np.iinfo(np.int16).max) ) if clipped_samples > 0 and clipping_warning: - logger.warning(f" Warning: {clipped_samples} samples clipped while writing {filename}!") + logger.warning( + f" Warning: {clipped_samples} samples clipped while writing {filename}!" + ) x = np.clip(x, np.iinfo(np.int16).min, np.iinfo(np.int16).max) if file_extension == ".wav": diff --git a/ivas_processing_scripts/generation/__init__.py b/ivas_processing_scripts/generation/__init__.py index 53892744..fdb83197 100755 --- a/ivas_processing_scripts/generation/__init__.py +++ b/ivas_processing_scripts/generation/__init__.py @@ -51,7 +51,7 @@ from ivas_processing_scripts.utils import create_dir def logging_init(args, cfg): """set up logging for a test file""" - logger = logging.getLogger("__main__") + logger = logging.getLogger("ivas_processing_scripts.generation") logger.setLevel(logging.DEBUG) # console handler @@ -68,7 +68,7 @@ def logging_init(args, cfg): file_handler.setLevel(logging.DEBUG if args.debug else logging.INFO) logger.addHandler(file_handler) - logger.info(f"Processing test configuration file {args.config}") + logger.info(f"Processing item generation configuration file {args.config}") logger.info(f"Input path: {cfg.input_path.absolute()}") logger.info(f"Output path: {cfg.output_path.absolute()}") @@ -107,3 +107,5 @@ def main(args): # copy configuration to output directory with open(cfg.output_path.joinpath(f"{'_'.join(cfg.format)}.yml"), "w") as f: yaml.safe_dump(cfg._yaml_dump, f) + + logger.handlers.clear() diff --git a/ivas_processing_scripts/processing/processing.py b/ivas_processing_scripts/processing/processing.py index 6fd7a963..71737753 100755 --- a/ivas_processing_scripts/processing/processing.py +++ b/ivas_processing_scripts/processing/processing.py @@ -52,7 +52,12 @@ from ivas_processing_scripts.audiotools.metadata import ( ) from ivas_processing_scripts.constants import LOGGER_DATEFMT, LOGGER_FORMAT from ivas_processing_scripts.processing.config import TestConfig -from ivas_processing_scripts.utils import list_audio, pairwise, progressbar_update +from ivas_processing_scripts.utils import ( + list_audio, + pairwise, + progressbar_update, + spinner, +) class Processing(ABC): @@ -85,7 +90,9 @@ def reorder_items_list(items_list: list, concatenation_order: list) -> list: name_to_full = {Path(full_file).name: full_file for full_file in items_list} if set(name_to_full.keys()) != set(concatenation_order): - raise ValueError(f"Items given in concatenation_order {concatenation_order} are not identical to what was found in the input folder {name_to_full.keys()}") + raise ValueError( + f"Items given in concatenation_order {concatenation_order} are not identical to what was found in the input folder {name_to_full.keys()}" + ) ordered_full_files = [ name_to_full[name] for name in concatenation_order if name in name_to_full @@ -259,6 +266,7 @@ def preprocess(cfg, logger): progressbar_update(0, count, width) while not results.ready(): progressbar_update(count - int(results._number_left), count, width) + spinner() sleep(0.1) progressbar_update(count, count, width) print("\n", flush=True, file=sys.stdout) @@ -329,6 +337,7 @@ def preprocess_2(cfg, logger): progressbar_update(0, count, width) while not results.ready(): progressbar_update(count - int(results._number_left), count, width) + spinner() sleep(0.1) progressbar_update(count, count, width) print("\n", flush=True, file=sys.stdout) diff --git a/ivas_processing_scripts/utils.py b/ivas_processing_scripts/utils.py index 6d29ba96..4680b224 100755 --- a/ivas_processing_scripts/utils.py +++ b/ivas_processing_scripts/utils.py @@ -35,7 +35,7 @@ import shutil import subprocess as sp import sys from concurrent.futures import ProcessPoolExecutor, ThreadPoolExecutor -from itertools import repeat, tee +from itertools import cycle, repeat, tee from os import devnull from pathlib import Path from shutil import which @@ -45,7 +45,7 @@ import yaml ALLOWED_INPUT_EXT = (".wav", ".pcm", ".txt", ".raw") BIN_DIR = Path(__file__).parent.joinpath("bin") - +BUSY_SPINNER = cycle(["|", "/", "-", "\\"]) """ Directory/path handling @@ -274,7 +274,7 @@ def pairwise(iter): def progressbar_update(progress, count, width): fill = int(width * progress / count) print( - f"{int(progress/count*100):3d}%{'|'}{'='*fill}{(' '*(width-fill))}{'|'}{progress:4d}/{count:4d}", + f"{int(progress/count*100):3d}%{'['}{'='*fill}{(' '*(width-fill))}{']'}{progress:4d}/{count:4d}", end="\r", file=sys.stdout, flush=True, @@ -292,6 +292,12 @@ def progressbar(iter: Iterable, width=80): print("\n", flush=True, file=sys.stdout) +def spinner(): + sys.stdout.write(f"{next(BUSY_SPINNER)}") + sys.stdout.flush() + sys.stdout.write("\b") + + def get_binary_paths(yaml_file_with_binary_paths): with open(yaml_file_with_binary_paths, "r") as f: data = yaml.safe_load(f) -- GitLab