Commit 27876683 authored by Anika Treffehn's avatar Anika Treffehn
Browse files

added different multiprocessing keys

parent d700915d

examples/TEMPLATE.yml

+4 −1
Original line number Diff line number Diff line
@@ -9,8 +9,11 @@ 
### git commit SHA; default = git rev-parse HEAD

# git_sha: abc123



### Whether to use multiprocessing; default = true

### Whether to use multiprocessing in general; default = true

# multiprocessing: false

### Whether to use multiprocessing in main loop for items and/or conditions; default = true

# multiprocessing_items: false

# multiprocessing_conditions: false

### Deletion of temporary directories containing 

### intermediate processing files, bitstreams etc.; default = false

# delete_tmp: true

ivas_processing_scripts/__init__.py

+2 −24
Original line number Diff line number Diff line
@@ -146,28 +146,6 @@ def main(args): 
            # preprocess 2

            preprocess_2(cfg, logger)



        # run conditions

        # for condition, out_dir, tmp_dir in zip(

        #     cfg.proc_chains, cfg.out_dirs, cfg.tmp_dirs

        # ):

        #     chain = condition["processes"]

        #

        #     logger.info(f"  Generating condition: {condition['name']}")

        #

        #     apply_func_parallel(

        #         process_item,

        #         zip(

        #             cfg.items_list,

        #             repeat(tmp_dir),

        #             repeat(out_dir),

        #             repeat(chain),

        #             repeat(logger),

        #             cfg.metadata_path,

        #         ),

        #         None,

        #         "mp" if cfg.multiprocessing else None,

        #     )



        apply_func_parallel(

            process_item_parallel,

            zip(
@@ -178,7 +156,7 @@ def main(args): 
                repeat(logger)

            ),

            None,

            "mp" if cfg.multiprocessing else None,

            "mp" if cfg.multiprocessing_conditions else None,

        )



        if hasattr(cfg, "preprocessing_2"):
@@ -206,5 +184,5 @@ def process_item_parallel(cfg, condition, tmp_dir, out_dir, logger): 
            cfg.metadata_path,

        ),

        None,

        "mp" if cfg.multiprocessing else None,

        "mp" if cfg.multiprocessing_items else None,

    )
 
 No newline at end of file

ivas_processing_scripts/constants.py

+2 −0
Original line number Diff line number Diff line
@@ -58,6 +58,8 @@ DEFAULT_CONFIG = { 
    "date": f"{datetime.now().strftime('%Y%m%d_%H.%M.%S')}",

    "git_sha": f"{get_gitsha()}",

    "multiprocessing": True,

    "multiprocessing_items": True,

    "multiprocessing_conditions": True,

    "delete_tmp": False,

    "master_seed": 0,

    "metadata_path": None,