Commit 303903e9 authored by Jan Kiene's avatar Jan Kiene
Browse files

make create_items_p800 script run categories in parallel

parent 185bc2a2
Loading
Loading
Loading
Loading
Loading
+5 −3
Original line number Diff line number Diff line
@@ -3,6 +3,7 @@ import argparse
from pathlib import Path

from ivas_processing_scripts import main as generate_test
from ivas_processing_scripts.utils import apply_func_parallel

P800_TESTS = [f"P800-{i}" for i in range(1, 8)]

@@ -11,15 +12,16 @@ class Arguments:
    def __init__(self, config):
        self.config = config
        self.debug = False
        # used to overwrite the multiprocessing key in the configs to rather parallelize on category level
        self.multiprocessing = False


def create_items(testname):
    p800_path = Path(f"experiments/selection/{testname}")
    p800_cfgs = p800_path.joinpath("config").glob("P800*cat*.yml")

    for cfg in p800_cfgs:
        args_dummy = Arguments(str(cfg))
        generate_test(args_dummy)
    args = [Arguments(str(cfg)) for cfg in p800_cfgs]
    apply_func_parallel(generate_test, zip( args ), type="mp")


# if is necessary here so that multiprocessing does not crash
+3 −0
Original line number Diff line number Diff line
@@ -87,6 +87,9 @@ def main(args):
    # parse configuration
    cfg = config.TestConfig(args.config)

    if hasattr(args, "multiprocessing"):
        cfg.multiprocessing = args.multiprocessing

    # set up processing chains
    chains.init_processing_chains(cfg)