sporada/file-setup
1
0
Fork 0
mirror of https://gitlab.ub.uni-bielefeld.de/sfb1288inf/file-setup.git synced 2024-12-25 10:54:18 +00:00
Code Issues Projects Releases Wiki Activity

Codestyle enhancements

Browse Source
This commit is contained in:
Patrick Jentsch 2022-01-27 13:48:16 +01:00
parent 22a7217036
commit aacac4e0a0
1 changed files with 35 additions and 15 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

50
file-setup
View File

@ -7,7 +7,6 @@ __version__ = '0.1.0'
from argparse import ArgumentParser from argparse import ArgumentParser
from pyflow import WorkflowRunner from pyflow import WorkflowRunner
import json import json
import multiprocessing
import os import os
import sys import sys
@ -52,7 +51,9 @@ class CreatePDFWorkflow(WorkflowRunner):
os.path.join(self.job.output_dir, '{}.pdf'.format(self.job.name)) os.path.join(self.job.output_dir, '{}.pdf'.format(self.job.name))
) )
cmd += ' && ' cmd += ' && '
cmd += 'rm "{}"'.format(os.path.join(self.job.output_dir, 'inputs.txt')) # noqa cmd += 'rm "{}"'.format(
os.path.join(self.job.output_dir, 'inputs.txt')
)
self.addTask( self.addTask(
'convert', 'convert',
command=cmd, command=cmd,
@ -68,6 +69,7 @@ class MainWorkflow(WorkflowRunner):
self.jobs = [] self.jobs = []
def collect_jobs(self): def collect_jobs(self):
self.jobs = []
for dir in os.listdir(self.input_dir): for dir in os.listdir(self.input_dir):
if not os.path.isdir(os.path.join(self.input_dir, dir)): if not os.path.isdir(os.path.join(self.input_dir, dir)):
continue continue
@ -102,38 +104,53 @@ class MainWorkflow(WorkflowRunner):
create_pdf_tasks.append(task) create_pdf_tasks.append(task)
self.waitForTasks() self.waitForTasks()
output_files = [] outputs = []
for job in self.jobs: for job in self.jobs:
# Track output files # Track output files
relative_output_dir = os.path.relpath(job.output_dir, start=self.output_dir) # noqa relative_output_dir = os.path.relpath(
output_files.append( job.output_dir,
start=self.output_dir
)
outputs.append(
{ {
'description': 'PDF file without text layer.', 'description': 'PDF file without text layer.',
'file': os.path.join(relative_output_dir, '{}.pdf'.format(job.name)), # noqa 'file': os.path.join(
relative_output_dir,
'{}.pdf'.format(job.name)
),
'mimetype': 'application/pdf' 'mimetype': 'application/pdf'
} }
) )
with open(os.path.join(self.output_dir, 'output_records.json'), 'w') as f: # noqa with open(os.path.join(self.output_dir, 'outputs.json'), 'w') as f:
json.dump(output_files, f, indent=4) json.dump(outputs, f, indent=4)
def parse_args(): def parse_args():
parser = ArgumentParser(description='Pipeline for merging images') parser = ArgumentParser(description='Pipeline for merging images')
parser.add_argument( parser.add_argument(
'-i', '--input-dir', help='Input directory', required=True) '-i', '--input-dir',
help='Input directory',
required=True
)
parser.add_argument( parser.add_argument(
'-o', '--output-dir', help='Output directory', required=True) '-o', '--output-dir',
help='Output directory',
required=True
)
parser.add_argument( parser.add_argument(
'--log-dir', help='Logging directory (Default: --output-dir)') '--log-dir',
help='Logging directory (Default: --output-dir)'
)
parser.add_argument( parser.add_argument(
'--mem-mb', '--mem-mb',
help='Amount of system memory to be used (Default: min(--n-cores * 256, available system memory))', # noqa help='Amount of system memory to be used '
'(Default: min(--n-cores * 256, available system memory))',
type=int type=int
) )
parser.add_argument( parser.add_argument(
'--n-cores', '--n-cores',
default=min(2, multiprocessing.cpu_count()), default=1,
help='Number of CPU threads to be used (Default: min(2, CPU count))', help='Number of CPU threads to be used',
type=int type=int
) )
parser.add_argument( parser.add_argument(
@ -162,7 +179,10 @@ def main():
main_workflow = MainWorkflow(args.input_dir, args.output_dir) main_workflow = MainWorkflow(args.input_dir, args.output_dir)
main_workflow.collect_jobs() main_workflow.collect_jobs()
retval = main_workflow.run( retval = main_workflow.run(
dataDirRoot=args.log_dir, memMb=args.mem_mb, nCores=args.n_cores) # noqa dataDirRoot=args.log_dir,
memMb=args.mem_mb,
nCores=args.n_cores
)
sys.exit(retval) sys.exit(retval)