sporada/ocr
1
0
Fork 0
mirror of https://gitlab.ub.uni-bielefeld.de/sfb1288inf/ocr.git synced 2024-12-26 15:54:19 +00:00
Code Issues Projects Releases Wiki Activity

fix pipeline

Browse Source
This commit is contained in:
Patrick Jentsch 2020-06-23 15:19:27 +02:00
parent ec5b4eb521
commit aee9628e5e
2 changed files with 222 additions and 295 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

501
ocr
View File

@ -24,33 +24,30 @@ TESSERACT_MODELS = ['deu', 'eng', 'enm', 'fra', 'frk', 'frm', 'ita', 'por',
def parse_args(): def parse_args():
parser = ArgumentParser(description='OCR Pipeline utilizing tesseract.') parser = ArgumentParser(description='OCR Pipeline utilizing tesseract.')
parser.add_argument('i', help='Input directory for OCR. One PDf equals one\ parser.add_argument('-i', '--input-directory',
job') help='Input directory (only PDF files get processed)',
parser.add_argument('o', help='Output directory containing OCR results.') required=True)
parser.add_argument('-o', '--output-directory',
help='Output directory',
required=True)
parser.add_argument('-l', '--language', parser.add_argument('-l', '--language',
choices=TESSERACT_MODELS, choices=TESSERACT_MODELS,
required=True) required=True)
parser.add_argument('--binarize', parser.add_argument('--binarize',
action='store_true', action='store_true',
help='Use ocropy binarisation as preprocessing step.') help='Use ocropy binarisation as preprocessing step.')
parser.add_argument('--keep-intermediates', parser.add_argument('--compress',
action='store_true', action='store_true',
help='Keep intermediate files for debugging etc.', help='Compress the final PDF result file.')
required=False) parser.add_argument('--log-dir')
parser.add_argument('--n-cores', parser.add_argument('--n-cores',
default=min(4, multiprocessing.cpu_count()), default=min(4, multiprocessing.cpu_count()),
help='Total number of cores available.', help='Total number of cores available.',
type=int, type=int)
required=False) parser.add_argument('--zip',
parser.add_argument('--zip', help='Zips all results in different archives \ help='Zips all results in different archives depending'
depending on result types. Also zips \ ' on result types. Also zips everything into one '
everything into one archive.', 'archive.')
required=False)
parser.add_argument('-c', '--compress',
help='Compress the final PDF result file.',
required=False,
action='store_true')
parser.add_argument('--log_dir')
return parser.parse_args() return parser.parse_args()
@ -62,11 +59,10 @@ class OCRPipelineJob:
class OCRPipeline(WorkflowRunner): class OCRPipeline(WorkflowRunner):
def __init__(self, binarize, jobs, keep_intermediates, lang, n_cores, def __init__(self, binarize, jobs, lang, n_cores, output_dir, zip,
output_dir, zip, compress): compress):
self.binarize = binarize self.binarize = binarize
self.jobs = jobs self.jobs = jobs
self.keep_intermediates = keep_intermediates
self.lang = lang self.lang = lang
self.n_cores = n_cores self.n_cores = n_cores
self.output_dir = output_dir self.output_dir = output_dir
@ -79,36 +75,26 @@ class OCRPipeline(WorkflowRunner):
''' '''
' ################################################## ' ##################################################
' # mkdir_jobs # ' # setup output directory #
' ################################################## ' ##################################################
''' '''
setup_output_directory_jobs = []
mkdir_jobs = []
for i, job in enumerate(self.jobs): for i, job in enumerate(self.jobs):
output_dir = os.path.join(job.output_dir, 'tmp') intermediate_dir = os.path.join(job.output_dir, 'tmp')
poco_dir = os.path.join(job.output_dir, 'PoCo')
cmd = 'mkdir' cmd = 'mkdir'
cmd += ' -p' cmd += ' -p'
cmd += ' "{}"'.format(output_dir) cmd += ' "{}"'.format(intermediate_dir)
cmd += ' "{}"'.format(os.path.join(poco_dir, 'hocr')) cmd += ' "{}"'.format(os.path.join(job.output_dir, 'poco'))
cmd += ' "{}"'.format(os.path.join(poco_dir, 'tiff')) lbl = 'setup_output_directory_-_{}'.format(i)
if self.keep_intermediates: setup_output_directory_jobs.append(self.addTask(command=cmd,
cmd += ' "{}"'.format(os.path.join(output_dir, 'hocr')) label=lbl))
cmd += ' "{}"'.format(os.path.join(output_dir, 'pdf'))
cmd += ' "{}"'.format(os.path.join(output_dir, 'tiff'))
cmd += ' "{}"'.format(os.path.join(output_dir, 'txt'))
if self.binarize:
cmd += ' "{}"'.format(os.path.join(output_dir, 'bin.png'))
cmd += ' "{}"'.format(os.path.join(output_dir, 'nrm.png'))
lbl = 'mkdir_job_-_{}'.format(i)
mkdir_jobs.append(self.addTask(command=cmd, label=lbl))
''' '''
' ################################################## ' ##################################################
' # pdftoppm_jobs # ' # split input #
' ################################################## ' ##################################################
''' '''
pdftoppm_jobs = [] split_input_jobs = []
n_cores = min(self.n_cores, max(1, int(self.n_cores / len(self.jobs)))) n_cores = min(self.n_cores, max(1, int(self.n_cores / len(self.jobs))))
for i, job in enumerate(self.jobs): for i, job in enumerate(self.jobs):
output_dir = os.path.join(job.output_dir, 'tmp') output_dir = os.path.join(job.output_dir, 'tmp')
@ -118,24 +104,26 @@ class OCRPipeline(WorkflowRunner):
cmd += ' -tiff' cmd += ' -tiff'
cmd += ' -tiffcompression lzw' cmd += ' -tiffcompression lzw'
cmd += ' "{}" "{}"'.format(job.file, output_file_base) cmd += ' "{}" "{}"'.format(job.file, output_file_base)
deps = mkdir_jobs deps = 'setup_output_directory_-_{}'.format(i)
lbl = 'pdftoppm_job_-_{}'.format(i) lbl = 'split_input_-_{}'.format(i)
pdftoppm_jobs.append(self.addTask(command=cmd, dependencies=deps, split_input_jobs.append(self.addTask(command=cmd,
label=lbl, nCores=n_cores)) dependencies=deps,
label=lbl,
nCores=n_cores))
if self.binarize: if self.binarize:
''' '''
' The ocropus_nlbin_jobs list is created based on the output files ' The binarization_jobs list is created based on the output files
' of the pdftoppm_jobs. So wait until they are finished. ' of the split_jobs. So wait until they are finished.
''' '''
self.waitForTasks() self.waitForTasks()
''' '''
' ################################################## ' ##################################################
' # ocropus_nlbin_jobs # ' # binarization #
' ################################################## ' ##################################################
''' '''
ocropus_nlbin_jobs = [] binarization_jobs = []
''' '''
' We run ocropus-nlbin with either four or, if there are less then ' We run ocropus-nlbin with either four or, if there are less then
' four cores available for this workflow, the available core ' four cores available for this workflow, the available core
@ -152,24 +140,25 @@ class OCRPipeline(WorkflowRunner):
cmd = 'ocropus-nlbin "{}"'.format('" "'.join(files)) cmd = 'ocropus-nlbin "{}"'.format('" "'.join(files))
cmd += ' -o "{}"'.format(output_dir) cmd += ' -o "{}"'.format(output_dir)
cmd += ' -Q "{}"'.format(n_cores) cmd += ' -Q "{}"'.format(n_cores)
deps = pdftoppm_jobs deps = 'split_input_-_{}'.format(i)
lbl = 'ocropus_nlbin_job_-_{}'.format(i) lbl = 'binarization_-_{}'.format(i)
ocropus_nlbin_jobs.append( binarization_jobs.append(self.addTask(command=cmd,
self.addTask(command=cmd, dependencies=deps, label=lbl, dependencies=deps,
nCores=n_cores)) label=lbl,
nCores=n_cores))
''' '''
' The post_ocropus_nlbin_jobs are created based on the output files ' The post_binarization_jobs are created based on the output files
' of the ocropus_nlbin_jobs. So wait until they are finished. ' of the binarization_jobs. So wait until they are finished.
''' '''
self.waitForTasks() self.waitForTasks()
''' '''
' ################################################## ' ##################################################
' # post_ocropus_nlbin_jobs # ' # post binarization #
' ################################################## ' ##################################################
''' '''
post_ocropus_nlbin_jobs = [] post_binarization_jobs = []
for i, job in enumerate(self.jobs): for i, job in enumerate(self.jobs):
input_dir = os.path.join(job.output_dir, 'tmp') input_dir = os.path.join(job.output_dir, 'tmp')
output_dir = input_dir output_dir = input_dir
@ -182,26 +171,26 @@ class OCRPipeline(WorkflowRunner):
output_file = os.path.join(output_dir, 'page-{}.bin.png'.format(int(file.split('.', 1)[0]))) # noqa output_file = os.path.join(output_dir, 'page-{}.bin.png'.format(int(file.split('.', 1)[0]))) # noqa
cmd = 'mv "{}" "{}"'.format(os.path.join(output_dir, file), cmd = 'mv "{}" "{}"'.format(os.path.join(output_dir, file),
output_file) output_file)
deps = ocropus_nlbin_jobs deps = 'binarization_-_{}'.format(i)
lbl = 'post_ocropus_nlbin_job_-_{}-{}'.format(i, number) lbl = 'post_binarization_-_{}-{}'.format(i, number)
post_ocropus_nlbin_jobs.append( post_binarization_jobs.append(
self.addTask(command=cmd, dependencies=deps, self.addTask(command=cmd, dependencies=deps, label=lbl)
label=lbl)) )
number += 1 number += 1
''' '''
' The tesseract_jobs are created based of the output files of either ' The ocr_jobs are created based of the output files of either the
' the pdftoppm_jobs or post_ocropus_nlbin_jobs. So wait until they are ' split_jobs or post_binarization_jobs. So wait until they are
' finished. ' finished.
''' '''
self.waitForTasks() self.waitForTasks()
''' '''
' ################################################## ' ##################################################
' # tesseract_jobs # ' # ocr #
' ################################################## ' ##################################################
''' '''
tesseract_jobs = [] ocr_jobs = []
''' '''
' Tesseract runs fastest with four cores. So we run it with either four ' Tesseract runs fastest with four cores. So we run it with either four
' or, if there are less then four cores available for this workflow, ' or, if there are less then four cores available for this workflow,
@ -211,8 +200,7 @@ class OCRPipeline(WorkflowRunner):
for i, job in enumerate(self.jobs): for i, job in enumerate(self.jobs):
input_dir = os.path.join(job.output_dir, 'tmp') input_dir = os.path.join(job.output_dir, 'tmp')
output_dir = input_dir output_dir = input_dir
files = filter(lambda x: x.endswith('.bin.png' if self.binarize else '.tif'), # noqa files = filter(lambda x: x.endswith('.bin.png' if self.binarize else '.tif'), os.listdir(input_dir)) # noqa
os.listdir(input_dir))
files.sort(key=lambda x: int(re.search(r'\d+', x).group(0))) files.sort(key=lambda x: int(re.search(r'\d+', x).group(0)))
files = map(lambda x: os.path.join(input_dir, x), files) files = map(lambda x: os.path.join(input_dir, x), files)
number = 0 number = 0
@ -221,14 +209,17 @@ class OCRPipeline(WorkflowRunner):
cmd = 'tesseract "{}" "{}"'.format(file, output_file_base) cmd = 'tesseract "{}" "{}"'.format(file, output_file_base)
cmd += ' -l "{}"'.format(self.lang) cmd += ' -l "{}"'.format(self.lang)
cmd += ' hocr pdf txt' cmd += ' hocr pdf txt'
cmd += ' && '
cmd += 'sed -i \'s+{}/++g\' "{}".hocr'.format(input_dir, output_file_base) # noqa
if self.binarize: if self.binarize:
deps = post_ocropus_nlbin_jobs deps = 'post_binarization_-_{}-{}'.format(i, number)
else: else:
deps = pdftoppm_jobs deps = 'split_input_-_{}'.format(i)
label = 'tesseract_jobs_-_{}-{}'.format(i, number) label = 'ocr_-_{}-{}'.format(i, number)
tesseract_jobs.append( ocr_jobs.append(self.addTask(command=cmd,
self.addTask(command=cmd, dependencies=deps, label=label, dependencies=deps,
nCores=n_cores)) label=label,
nCores=n_cores))
number += 1 number += 1
''' '''
@ -239,190 +230,171 @@ class OCRPipeline(WorkflowRunner):
''' '''
' ################################################## ' ##################################################
' # hocrtotei_jobs # ' # combined pdf creation #
' ################################################## ' ##################################################
''' '''
hocrtotei_jobs = [] combined_pdf_creation_jobs = []
for i, job in enumerate(self.jobs): for i, job in enumerate(self.jobs):
input_dir = os.path.join(job.output_dir, 'tmp') input_dir = os.path.join(job.output_dir, 'tmp')
files = filter(lambda x: x.endswith('.hocr'), files = filter(lambda x: x.endswith('.pdf'),
os.listdir(input_dir)) os.listdir(input_dir))
files.sort(key=lambda x: int(re.search(r'\d+', x).group(0))) files.sort(key=lambda x: int(re.search(r'\d+', x).group(0)))
files = map(lambda x: os.path.join(input_dir, x), files) files = map(lambda x: os.path.join(input_dir, x), files)
output_file = os.path.join(job.output_dir,
'{}.xml'.format(job.name))
cmd = 'hocrtotei "{}" "{}"'.format('" "'.join(files), output_file)
deps = tesseract_jobs
lbl = 'hocrtotei_job_-_{}'.format(i)
hocrtotei_jobs.append(self.addTask(command=cmd, dependencies=deps,
label=lbl))
'''
' ##################################################
' # hocr_poco_jobs #
' ##################################################
'''
hocr_poco_jobs = []
for i, job in enumerate(self.jobs):
input_dir = os.path.join(job.output_dir, 'tmp')
files = filter(lambda x: x.endswith('.hocr'),
os.listdir(input_dir))
files.sort(key=lambda x: int(re.search(r'\d+', x).group(0)))
files = map(lambda x: os.path.join(input_dir, x), files)
# set relative file paths into hocr
relative_files = map(lambda x: os.path.join('..',
'tiff',
os.path.basename(x).replace('.hocr', '.tif')), # noqa
files)
for file, relative_file in zip(files, relative_files):
with open(file, 'r+') as f:
html = f.read()
html = html.replace(file.replace('.hocr', '.tif'),
relative_file)
f.seek(0)
f.truncate(0) # deletes content of file to write new html
f.write(html)
output_path_base = os.path.join(job.output_dir, 'PoCo')
output_path = os.path.join(output_path_base, 'hocr')
cmd = 'cp "{}" "{}"'.format('" "'.join(files), output_path)
deps = tesseract_jobs
lbl = 'hocr_poco_jobs-_{}'.format(i)
hocr_poco_jobs.append(self.addTask(command=cmd, dependencies=deps,
label=lbl))
'''
' ##################################################
' # tiff_poco_jobs #
' ##################################################
'''
tiff_poco_jobs = []
for i, job in enumerate(self.jobs):
input_dir = os.path.join(job.output_dir, 'tmp')
files = filter(lambda x: x.endswith('.tif'),
os.listdir(input_dir))
files.sort(key=lambda x: int(re.search(r'\d+', x).group(0)))
files = map(lambda x: os.path.join(input_dir, x), files)
output_path_base = os.path.join(job.output_dir, 'PoCo')
output_path = os.path.join(output_path_base, 'tiff')
cmd = 'cp "{}" "{}"'.format('" "'.join(files), output_path)
deps = tesseract_jobs
lbl = 'tiff_poco_jobs-_{}'.format(i)
tiff_poco_jobs.append(self.addTask(command=cmd, dependencies=deps,
label=lbl))
'''
' ##################################################
' # pdfunite_jobs #
' ##################################################
'''
pdfunite_jobs = []
for i, job in enumerate(self.jobs):
input_dir = os.path.join(job.output_dir, 'tmp')
files = filter(lambda x: x.endswith('.pdf'), os.listdir(input_dir))
files.sort(key=lambda x: int(re.search(r'\d+', x).group(0)))
files = map(lambda x: os.path.join(input_dir, x), files)
output_file = os.path.join(job.output_dir, output_file = os.path.join(job.output_dir,
'{}.pdf'.format(job.name)) '{}.pdf'.format(job.name))
cmd = 'pdfunite "{}" "{}"'.format('" "'.join(files), output_file) cmd = 'pdfunite "{}" "{}"'.format('" "'.join(files), output_file)
deps = tesseract_jobs deps = filter(lambda x: x.startswith('ocr_-_{}'.format(i)),
lbl = 'pdfunite_job_-_{}'.format(i) ocr_jobs)
pdfunite_jobs.append(self.addTask(command=cmd, dependencies=deps, lbl = 'combined_pdf_creation_-_{}'.format(i)
label=lbl)) combined_pdf_creation_jobs.append(self.addTask(command=cmd,
dependencies=deps,
label=lbl))
''' '''
' ################################################## ' ##################################################
' # cat_jobs # ' # combined txt creation #
' ################################################## ' ##################################################
''' '''
cat_jobs = [] combined_txt_creation_jobs = []
for i, job in enumerate(self.jobs): for i, job in enumerate(self.jobs):
input_dir = os.path.join(job.output_dir, 'tmp') input_dir = os.path.join(job.output_dir, 'tmp')
files = filter(lambda x: x.endswith('.txt'), os.listdir(input_dir)) files = filter(lambda x: x.endswith('.txt'), os.listdir(input_dir))
files.sort(key=lambda x: int(re.search(r'\d+', x).group(0))) files.sort(key=lambda x: int(re.search(r'\d+', x).group(0)))
files = map(lambda x: os.path.join(input_dir, x), files) files = map(lambda x: os.path.join(input_dir, x), files)
output_file = os.path.join(job.output_dir, output_file = os.path.join(job.output_dir, '{}.txt'.format(job.name)) # noqa
'{}.txt'.format(job.name))
cmd = 'cat "{}" > "{}"'.format('" "'.join(files), output_file) cmd = 'cat "{}" > "{}"'.format('" "'.join(files), output_file)
deps = tesseract_jobs deps = filter(lambda x: x.startswith('ocr_-_{}'.format(i)),
lbl = 'cat_job_-_{}'.format(i) ocr_jobs)
cat_jobs.append(self.addTask(command=cmd, dependencies=deps, lbl = 'combined_txt_creation_-_{}'.format(i)
label=lbl)) combined_txt_creation_jobs.append(self.addTask(command=cmd,
dependencies=deps,
label=lbl))
'''
' ##################################################
' # tei p5 creation #
' ##################################################
'''
tei_p5_creation_jobs = []
for i, job in enumerate(self.jobs):
input_dir = os.path.join(job.output_dir, 'tmp')
files = filter(lambda x: x.endswith('.hocr'), os.listdir(input_dir)) # noqa
files.sort(key=lambda x: int(re.search(r'\d+', x).group(0)))
files = map(lambda x: os.path.join(input_dir, x), files)
output_file = os.path.join(job.output_dir, '{}.xml'.format(job.name)) # noqa
cmd = 'hocrtotei "{}" "{}"'.format('" "'.join(files), output_file)
deps = filter(lambda x: x.startswith('ocr_-_{}'.format(i)),
ocr_jobs)
lbl = 'tei_p5_creation_-_{}'.format(i)
tei_p5_creation_jobs.append(self.addTask(command=cmd,
dependencies=deps,
label=lbl))
'''
' ##################################################
' # poco bundle creation #
' ##################################################
'''
poco_bundle_creation_jobs = []
for i, job in enumerate(self.jobs):
input_dir = os.path.join(job.output_dir, 'tmp')
output_dir = os.path.join(job.output_dir, 'poco')
cmd = 'mv "{}"/*.hocr "{}"'.format(input_dir, output_dir)
cmd += ' && '
cmd += 'mv "{}"/*.{} "{}"'.format(input_dir, 'bin.png' if self.binarize else 'tif', output_dir) # noqa
deps = filter(lambda x: x.startswith('ocr_-_{}'.format(i)),
ocr_jobs)
deps.append('tei_p5_creation_-_{}'.format(i))
lbl = 'poco_bundle_creation_-_{}'.format(i)
poco_bundle_creation_jobs.append(self.addTask(command=cmd,
dependencies=deps,
label=lbl))
''' '''
' The following jobs are created based of the output files of the ' The following jobs are created based of the output files of the
' pdfunite_jobs. So wait until they are finished. ' combined_pdf_creation_jobs. So wait until they are finished.
''' '''
self.waitForTasks() self.waitForTasks()
''' '''
' ################################################## ' ##################################################
' # compress_jobs # ' # pdf compression #
' ################################################## ' ##################################################
''' '''
compress_jobs = [] pdf_compression_jobs = []
if self.compress: if self.compress:
for i, job in enumerate(self.jobs): for i, job in enumerate(self.jobs):
print(os.listdir(job.output_dir)) file = filter(lambda x: x.endswith('.pdf'), os.listdir(job.output_dir))[0] # noqa
file = filter(lambda x: x.endswith('.pdf'),
os.listdir(job.output_dir))[0]
original_file = os.path.join(job.output_dir, file) original_file = os.path.join(job.output_dir, file)
compressed_file = os.path.join(job.output_dir, 'c_' + file) compressed_file = os.path.join(job.output_dir, 'c_' + file)
cmd = ('gs ' cmd = 'gs'
+ '-sDEVICE=pdfwrite ' cmd += ' -sDEVICE=pdfwrite'
+ '-dCompatibilityLevel=1.4 ' cmd += ' -dCompatibilityLevel=1.4'
+ '-dPDFSETTINGS=/ebook ' cmd += ' -dPDFSETTINGS=/ebook'
+ '-dNOPAUSE ' cmd += ' -dNOPAUSE'
+ '-dQUIET ' cmd += ' -dQUIET'
+ '-dBATCH ' cmd += ' -dBATCH'
+ '-sOutputFile={o} {i} ').format(o=compressed_file, cmd += ' -sOutputFile="{}"'.format(compressed_file)
i=original_file) cmd += ' "{}"'.format(original_file)
cmd += '&& rm {original_f} '.format(original_f=original_file) cmd += ' && '
cmd += ('&& mv {compressed_f} ' cmd += 'mv "{}" "{}"'.format(compressed_file, original_file)
+ '{original_f} ').format(compressed_f=compressed_file, deps = 'combined_pdf_creation_-_{}'.format(i)
original_f=original_file) lbl = 'pdf_compression_-_{}'.format(i)
deps = (hocrtotei_jobs pdf_compression_jobs.append(self.addTask(command=cmd,
+ tesseract_jobs dependencies=deps,
+ pdfunite_jobs label=lbl))
+ cat_jobs
+ hocr_poco_jobs
+ tiff_poco_jobs)
lbl = 'compress_job_-_{}'.format(i)
compress_jobs.append(self.addTask(command=cmd,
dependencies=deps,
label=lbl))
''' '''
' ################################################## ' ##################################################
' # zip_jobs # ' # cleanup #
' ################################################## ' ##################################################
''' '''
zip_jobs = [] cleanup_jobs = []
deps = (hocrtotei_jobs for i, job in enumerate(self.jobs):
+ tesseract_jobs input_dir = os.path.join(job.output_dir, 'tmp')
+ pdfunite_jobs cmd = 'rm -r "{}"'.format(input_dir)
+ cat_jobs if self.compress:
+ hocr_poco_jobs deps = ['pdf_compression_-_{}'.format(i)]
+ tiff_poco_jobs else:
+ compress_jobs) deps = ['combined_pdf_creation_-_{}'.format(i)]
deps.append('combined_txt_creation_-_{}'.format(i))
deps.append('poco_bundle_creation_-_{}'.format(i))
deps.append('tei_p5_creation_-_{}'.format(i))
lbl = 'cleanup_-_{}'.format(i)
cleanup_jobs.append(self.addTask(command=cmd,
dependencies=deps,
label=lbl))
'''
' ##################################################
' # zip creation #
' ##################################################
'''
zip_creation_jobs = []
if self.zip is not None: if self.zip is not None:
# Remove .zip file extension if provided # Remove .zip file extension if provided
if self.zip.endswith('.zip'): if self.zip.endswith('.zip'):
self.zip = self.zip[:-4] self.zip = self.zip[:-4]
self.zip = self.zip if self.zip else 'output' self.zip = self.zip if self.zip else 'output'
# zip ALL # zip all files
cmd = 'cd "{}"'.format(self.output_dir) cmd = 'cd "{}"'.format(self.output_dir)
cmd += ' && ' cmd += ' && '
cmd += 'zip' cmd += 'zip'
cmd += ' -r' cmd += ' -r'
cmd += ' "{}".all.zip .'.format(self.zip) cmd += ' "{}".all.zip .'.format(self.zip)
cmd += ' -x "pyflow.data*" "*tmp*"' cmd += ' -x "pyflow.data*" "*tmp*"'
cmd += ' -i "*.pdf" "*.txt" "*.xml" "*.hocr" "*.tif"' cmd += ' -i "*.pdf" "*.txt" "*.xml" "*.hocr" "*.{}"'.format('bin.png' if self.binarize else 'tif') # noqa
cmd += ' && ' cmd += ' && '
cmd += 'cd -' cmd += 'cd -'
lbl = 'zip_job_-_all' deps = (pdf_compression_jobs if self.compress else
zip_jobs.append(self.addTask(command=cmd, dependencies=deps, combined_pdf_creation_jobs)
label=lbl)) deps += combined_txt_creation_jobs
# zip PDFs deps += poco_bundle_creation_jobs
lbl = 'zip_creation_-_all'
zip_creation_jobs.append(self.addTask(command=cmd,
dependencies=deps,
label=lbl))
# zip PDF files
cmd = 'cd "{}"'.format(self.output_dir) cmd = 'cd "{}"'.format(self.output_dir)
cmd += ' && ' cmd += ' && '
cmd += 'zip' cmd += 'zip'
@ -432,11 +404,13 @@ class OCRPipeline(WorkflowRunner):
cmd += ' -i "*.pdf"' cmd += ' -i "*.pdf"'
cmd += ' && ' cmd += ' && '
cmd += 'cd -' cmd += 'cd -'
deps = deps + ['zip_job_-_all'] deps = (pdf_compression_jobs if self.compress else
lbl = 'zip_job_-_pdf' combined_pdf_creation_jobs)
zip_jobs.append(self.addTask(command=cmd, dependencies=deps, lbl = 'zip_creation_-_pdf'
label=lbl)) zip_creation_jobs.append(self.addTask(command=cmd,
# zip TXTs dependencies=deps,
label=lbl))
# zip TXT files
cmd = 'cd "{}"'.format(self.output_dir) cmd = 'cd "{}"'.format(self.output_dir)
cmd += ' && ' cmd += ' && '
cmd += 'zip' cmd += 'zip'
@ -446,11 +420,12 @@ class OCRPipeline(WorkflowRunner):
cmd += ' -i "*.txt"' cmd += ' -i "*.txt"'
cmd += ' && ' cmd += ' && '
cmd += 'cd -' cmd += 'cd -'
deps = deps + ['zip_job_-_all'] deps = combined_txt_creation_jobs
lbl = 'zip_job_-_txt' lbl = 'zip_creation_-_txt'
zip_jobs.append(self.addTask(command=cmd, dependencies=deps, zip_creation_jobs.append(self.addTask(command=cmd,
label=lbl)) dependencies=deps,
# zip XMLs label=lbl))
# zip XML files
cmd = 'cd "{}"'.format(self.output_dir) cmd = 'cd "{}"'.format(self.output_dir)
cmd += ' && ' cmd += ' && '
cmd += 'zip' cmd += 'zip'
@ -460,80 +435,26 @@ class OCRPipeline(WorkflowRunner):
cmd += ' -i "*.xml"' cmd += ' -i "*.xml"'
cmd += ' && ' cmd += ' && '
cmd += 'cd -' cmd += 'cd -'
deps = deps + ['zip_job_-_all'] deps = tei_p5_creation_jobs
lbl = 'zip_job_-_xml' lbl = 'zip_creation_-_xml'
zip_jobs.append(self.addTask(command=cmd, dependencies=deps, zip_creation_jobs.append(self.addTask(command=cmd,
label=lbl)) dependencies=deps,
# zip PoCo files label=lbl))
poco_paths = [] # zip PoCo bundles
poco_names = []
for i, job in enumerate(self.jobs):
poco_paths.append(os.path.join(os.path.basename(job.output_dir), # noqa
'PoCo'))
poco_names.append(job.output_dir)
cmd = 'cd "{}"'.format(self.output_dir) cmd = 'cd "{}"'.format(self.output_dir)
cmd += ' && ' cmd += ' && '
cmd += 'zip' cmd += 'zip'
cmd += ' -r' cmd += ' -r'
cmd += ' "{}".poco.zip'.format(self.zip) cmd += ' "{}".poco.zip .'.format(self.zip)
cmd += ' "{}"'.format('" "'.join(poco_paths)) cmd += ' -x "pyflow.data*" "*tmp*"'
cmd += ' -i "*.hocr" "*.{}"'.format('bin.png' if self.binarize else 'tif') # noqa
cmd += ' && ' cmd += ' && '
cmd += 'cd -' cmd += 'cd -'
deps = deps + ['zip_job_-_all'] deps = poco_bundle_creation_jobs
lbl = 'zip_job_-_poco_{}'.format(i) lbl = 'zip_creation_-_poco'
zip_jobs.append(self.addTask(command=cmd, dependencies=deps, zip_creation_jobs.append(self.addTask(command=cmd,
label=lbl)) dependencies=deps,
label=lbl))
'''
' ##################################################
' # mv_jobs #
' ##################################################
'''
mv_jobs = []
if self.keep_intermediates:
for i, job in enumerate(self.jobs):
input_dir = os.path.join(job.output_dir, 'tmp')
output_dir = input_dir
cmd = 'mv "{}"/*.hocr "{}"'.format(
input_dir, os.path.join(output_dir, 'hocr'))
cmd += ' && '
cmd += 'mv "{}"/*.pdf "{}"'.format(input_dir, os.path.join(output_dir, 'pdf')) # noqa
cmd += ' && '
cmd += 'mv "{}"/*.tif "{}"'.format(input_dir, os.path.join(output_dir, 'tiff')) # noqa
cmd += ' && '
cmd += 'mv "{}"/*.txt "{}"'.format(input_dir, os.path.join(output_dir, 'txt')) # noqa
if self.binarize:
cmd += ' && '
cmd += 'mv "{}"/*.bin.png "{}"'.format(input_dir, os.path.join(output_dir, 'bin.png')) # noqa
cmd += ' && '
cmd += 'mv "{}"/*.nrm.png "{}"'.format(input_dir, os.path.join(output_dir, 'nrm.png')) # noqa
deps = (hocrtotei_jobs
+ tesseract_jobs
+ pdfunite_jobs
+ cat_jobs
+ hocr_poco_jobs
+ tiff_poco_jobs,
+ compress_jobs
+ zip_jobs)
lbl = 'mv_job_-_{}'.format(i)
mv_jobs.append(self.addTask(command=cmd, dependencies=deps,
label=lbl))
else:
for i, job in enumerate(self.jobs):
input_dir = os.path.join(job.output_dir, 'tmp')
cmd = 'rm -r "{}"'.format(input_dir)
deps = (hocrtotei_jobs
+ tesseract_jobs
+ pdfunite_jobs
+ cat_jobs
+ hocr_poco_jobs
+ tiff_poco_jobs
+ compress_jobs
+ zip_jobs)
lbl = 'mv_job_-_{}'.format(i)
mv_jobs.append(self.addTask(command=cmd, dependencies=deps,
label=lbl))
def collect_jobs(input_dir, output_dir): def collect_jobs(input_dir, output_dir):
@ -550,12 +471,14 @@ def collect_jobs(input_dir, output_dir):
def main(): def main():
args = parse_args() args = parse_args()
jobs = collect_jobs(args.i, args.o) jobs = collect_jobs(args.input_directory, args.output_directory)
ocr_pipeline = OCRPipeline(args.binarize, jobs, args.keep_intermediates, ocr_pipeline = OCRPipeline(args.binarize, jobs, args.language,
args.language, args.n_cores, args.o, args.zip, args.n_cores, args.output_directory, args.zip,
args.compress) args.compress)
retval = ocr_pipeline.run(dataDirRoot=(args.log_dir or args.o), retval = ocr_pipeline.run(
nCores=args.n_cores) dataDirRoot=(args.log_dir or args.output_directory),
nCores=args.n_cores
)
sys.exit(retval) sys.exit(retval)

16
wrapper/ocr
View File

@ -12,17 +12,21 @@ UID = str(os.getuid())
GID = str(os.getgid()) GID = str(os.getgid())
parser = ArgumentParser(add_help=False) parser = ArgumentParser(add_help=False)
parser.add_argument('-i') parser.add_argument('-i', '--input-directory')
parser.add_argument('-o') parser.add_argument('-o', '--output-directory')
args, remaining_args = parser.parse_known_args() args, remaining_args = parser.parse_known_args()
cmd = ['docker', 'run', '--rm', '-it', '-u', '{}:{}'.format(UID, GID)] cmd = ['docker', 'run', '--rm', '-it', '-u', '{}:{}'.format(UID, GID)]
if args.o is not None: if args.output_directory is not None:
cmd += ['-v', '{}:{}'.format(os.path.abspath(args.o), CONTAINER_OUTPUT_DIR)] cmd += ['-v', '{}:{}'.format(os.path.abspath(args.output_directory),
CONTAINER_OUTPUT_DIR)]
remaining_args.insert(0, CONTAINER_OUTPUT_DIR) remaining_args.insert(0, CONTAINER_OUTPUT_DIR)
if args.i is not None: remaining_args.insert(0, '-o')
cmd += ['-v', '{}:{}'.format(os.path.abspath(args.i), CONTAINER_INPUT_DIR)] if args.input_directory is not None:
cmd += ['-v', '{}:{}'.format(os.path.abspath(args.input_directory),
CONTAINER_INPUT_DIR)]
remaining_args.insert(0, CONTAINER_INPUT_DIR) remaining_args.insert(0, CONTAINER_INPUT_DIR)
remaining_args.insert(0, '-i')
cmd.append(CONTAINER_IMAGE) cmd.append(CONTAINER_IMAGE)
cmd += remaining_args cmd += remaining_args