Commit d700915d authored by Anika Treffehn's avatar Anika Treffehn
Browse files

tried to run conditions in parallel

parent 9f5f7bd5

ivas_processing_scripts/__init__.py

+54 −20
Original line number Diff line number Diff line
@@ -147,9 +147,50 @@ def main(args): 
            preprocess_2(cfg, logger)



        # run conditions

        for condition, out_dir, tmp_dir in zip(

            cfg.proc_chains, cfg.out_dirs, cfg.tmp_dirs

        ):

        # for condition, out_dir, tmp_dir in zip(

        #     cfg.proc_chains, cfg.out_dirs, cfg.tmp_dirs

        # ):

        #     chain = condition["processes"]

        #

        #     logger.info(f"  Generating condition: {condition['name']}")

        #

        #     apply_func_parallel(

        #         process_item,

        #         zip(

        #             cfg.items_list,

        #             repeat(tmp_dir),

        #             repeat(out_dir),

        #             repeat(chain),

        #             repeat(logger),

        #             cfg.metadata_path,

        #         ),

        #         None,

        #         "mp" if cfg.multiprocessing else None,

        #     )



        apply_func_parallel(

            process_item_parallel,

            zip(

                repeat(cfg),

                cfg.proc_chains,

                cfg.tmp_dirs,

                cfg.out_dirs,

                repeat(logger)

            ),

            None,

            "mp" if cfg.multiprocessing else None,

        )



        if hasattr(cfg, "preprocessing_2"):

            reverse_process_2(cfg, logger)



    # copy configuration to output directory

    with open(cfg.output_path.joinpath(f"{cfg.name}.yml"), "w") as f:

        yaml.safe_dump(cfg._yaml_dump, f)





def process_item_parallel(cfg, condition, tmp_dir, out_dir, logger):



    chain = condition["processes"]



    logger.info(f"  Generating condition: {condition['name']}")
@@ -167,10 +208,3 @@ def main(args): 
        None,

        "mp" if cfg.multiprocessing else None,

    )
 
 No newline at end of file
 


        if hasattr(cfg, "preprocessing_2"):

            reverse_process_2(cfg, logger)



    # copy configuration to output directory

    with open(cfg.output_path.joinpath(f"{cfg.name}.yml"), "w") as f:

        yaml.safe_dump(cfg._yaml_dump, f)