mirror of
https://gitlab.ub.uni-bielefeld.de/sfb1288inf/ocr.git
synced 2024-12-25 18:14:17 +00:00
Add possibility to use an intermediate dir
This commit is contained in:
Patrick Jentsch
parent
6d90d43699
commit
ac4b5c2fd8
@ -76,8 +76,10 @@ RUN chmod 644 /usr/local/share/tessdata/*.traineddata
|
||||
## Install Pipeline ##
|
||||
RUN apt-get install -y --no-install-recommends \
|
||||
ghostscript \
|
||||
python-pip \
|
||||
python3.7 \
|
||||
zip
|
||||
zip \
|
||||
&& pip install natsort
|
||||
COPY "hocrtotei" "ocr" "/usr/local/bin/"
|
||||
|
||||
|
||||
|
||||
329
ocr
329
ocr
@ -12,11 +12,12 @@ Authors: Patrick Jentsch <p.jentsch@uni-bielefeld.de
|
||||
|
||||
|
||||
from argparse import ArgumentParser
|
||||
from natsort import natsorted
|
||||
from pyflow import WorkflowRunner
|
||||
import multiprocessing
|
||||
import os
|
||||
import re
|
||||
import sys
|
||||
import tempfile
|
||||
|
||||
|
||||
TESSERACT_MODELS = ['deu', 'eng', 'enm', 'fra', 'frk', 'frm', 'ita', 'por',
|
||||
@ -42,6 +43,7 @@ def parse_args():
|
||||
default=min(4, multiprocessing.cpu_count()),
|
||||
help='Total number of cores available.',
|
||||
type=int)
|
||||
parser.add_argument('--intermediate-directory')
|
||||
parser.add_argument('--zip',
|
||||
help='Zips all results in different archives depending'
|
||||
' on result types. Also zips everything into one '
|
||||
@ -50,20 +52,49 @@ def parse_args():
|
||||
|
||||
|
||||
class OCRPipelineJob:
|
||||
def __init__(self, file, output_dir):
|
||||
"""An OCR pipeline job class
|
||||
|
||||
Each input file of the pipeline is represented as an OCR pipeline job,
|
||||
which holds all necessary information for the pipeline to process it.
|
||||
|
||||
Arguments:
|
||||
file -- Path to the file
|
||||
output_dir -- Path to a directory, where job results a stored
|
||||
intermediate_dir -- Path to a directory, where intermediate files are
|
||||
stored.
|
||||
"""
|
||||
|
||||
def __init__(self, file, output_dir, intermediate_dir):
|
||||
self.file = file
|
||||
self.intermediate_dir = intermediate_dir
|
||||
self.name = os.path.basename(file).rsplit('.', 1)[0]
|
||||
self.output_dir = output_dir
|
||||
|
||||
|
||||
class OCRPipeline(WorkflowRunner):
|
||||
def __init__(self, binarize, jobs, lang, n_cores, output_dir, zip):
|
||||
self.binarize = binarize
|
||||
self.jobs = jobs
|
||||
def __init__(self, input_dir, lang, output_dir, binarize, intermediate_dir,
|
||||
n_cores, zip):
|
||||
self.input_dir = input_dir
|
||||
self.lang = lang
|
||||
self.n_cores = n_cores
|
||||
self.output_dir = output_dir
|
||||
self.zip = zip
|
||||
self.binarize = binarize
|
||||
if intermediate_dir is None:
|
||||
self.intermediate_dir = os.path.join(output_dir, 'tmp')
|
||||
else:
|
||||
self.intermediate_dir = tempfile.mkdtemp(dir=intermediate_dir)
|
||||
self.n_cores = n_cores
|
||||
if zip is None:
|
||||
self.zip = zip
|
||||
else:
|
||||
if zip.lower().endswith('.zip'):
|
||||
# Remove .zip file extension if provided
|
||||
self.zip = zip[:-4]
|
||||
self.zip = self.zip if self.zip else 'output'
|
||||
else:
|
||||
self.zip = zip
|
||||
self.jobs = collect_jobs(self.input_dir,
|
||||
self.output_dir,
|
||||
self.intermediate_dir)
|
||||
|
||||
def workflow(self):
|
||||
if not self.jobs:
|
||||
@ -74,26 +105,26 @@ class OCRPipeline(WorkflowRunner):
|
||||
' # setup output directory #
|
||||
' ##################################################
|
||||
'''
|
||||
setup_output_directory_jobs = []
|
||||
setup_output_directory_tasks = []
|
||||
for i, job in enumerate(self.jobs):
|
||||
intermediate_dir = os.path.join(job.output_dir, 'tmp')
|
||||
cmd = 'mkdir'
|
||||
cmd += ' -p'
|
||||
cmd += ' "{}"'.format(intermediate_dir)
|
||||
cmd += ' "{}"'.format(job.intermediate_dir)
|
||||
cmd += ' "{}"'.format(os.path.join(job.output_dir, 'poco'))
|
||||
lbl = 'setup_output_directory_-_{}'.format(i)
|
||||
setup_output_directory_jobs.append(self.addTask(command=cmd,
|
||||
label=lbl))
|
||||
task = self.addTask(command=cmd, label=lbl)
|
||||
setup_output_directory_tasks.append(task)
|
||||
|
||||
'''
|
||||
' ##################################################
|
||||
' # split input #
|
||||
' ##################################################
|
||||
'''
|
||||
split_input_jobs = []
|
||||
split_input_tasks = []
|
||||
n_cores = min(self.n_cores, max(1, int(self.n_cores / len(self.jobs))))
|
||||
for i, job in enumerate(self.jobs):
|
||||
output_dir = os.path.join(job.output_dir, 'tmp')
|
||||
input_file = job.file
|
||||
output_file = '{}/page-%d.tif'.format(job.intermediate_dir)
|
||||
cmd = 'gs'
|
||||
cmd += ' -dBATCH'
|
||||
cmd += ' -dNOPAUSE'
|
||||
@ -102,19 +133,17 @@ class OCRPipeline(WorkflowRunner):
|
||||
cmd += ' -r300'
|
||||
cmd += ' -sDEVICE=tiff24nc'
|
||||
cmd += ' -sCompression=lzw'
|
||||
cmd += ' "-sOutputFile={}/page-%d.tif"'.format(output_dir)
|
||||
cmd += ' "{}"'.format(job.file)
|
||||
cmd += ' "-sOutputFile={}"'.format(output_file)
|
||||
cmd += ' "{}"'.format(input_file)
|
||||
deps = 'setup_output_directory_-_{}'.format(i)
|
||||
lbl = 'split_input_-_{}'.format(i)
|
||||
split_input_jobs.append(self.addTask(command=cmd,
|
||||
dependencies=deps,
|
||||
label=lbl,
|
||||
nCores=n_cores))
|
||||
task = self.addTask(command=cmd, dependencies=deps, label=lbl, nCores=n_cores) # noqa
|
||||
split_input_tasks.append(task)
|
||||
|
||||
if self.binarize:
|
||||
'''
|
||||
' The binarization_jobs list is created based on the output files
|
||||
' of the split_jobs. So wait until they are finished.
|
||||
' The binarization_tasks list is created based on the output files
|
||||
' of the split_tasks. So wait until they are finished.
|
||||
'''
|
||||
self.waitForTasks()
|
||||
|
||||
@ -123,7 +152,7 @@ class OCRPipeline(WorkflowRunner):
|
||||
' # binarization #
|
||||
' ##################################################
|
||||
'''
|
||||
binarization_jobs = []
|
||||
binarization_tasks = []
|
||||
'''
|
||||
' We run ocropus-nlbin with either four or, if there are less then
|
||||
' four cores available for this workflow, the available core
|
||||
@ -131,27 +160,21 @@ class OCRPipeline(WorkflowRunner):
|
||||
'''
|
||||
n_cores = min(4, self.n_cores)
|
||||
for i, job in enumerate(self.jobs):
|
||||
input_dir = os.path.join(job.output_dir, 'tmp')
|
||||
output_dir = input_dir
|
||||
files = filter(lambda x: x.endswith('.tif'),
|
||||
os.listdir(input_dir))
|
||||
files.sort(key=lambda x: int(re.search(r'\d+', x).group(0)))
|
||||
input_dir = job.intermediate_dir
|
||||
output_dir = job.intermediate_dir
|
||||
files = filter(lambda x: x.endswith('.tif'), os.listdir(input_dir)) # noqa
|
||||
files = natsorted(files)
|
||||
files = map(lambda x: os.path.join(input_dir, x), files)
|
||||
cmd = 'ocropus-nlbin "{}"'.format('" "'.join(files))
|
||||
cmd += ' --nocheck'
|
||||
cmd += ' --output "{}"'.format(output_dir)
|
||||
cmd += ' --parallel "{}"'.format(n_cores)
|
||||
print(cmd)
|
||||
deps = 'split_input_-_{}'.format(i)
|
||||
lbl = 'binarization_-_{}'.format(i)
|
||||
binarization_jobs.append(self.addTask(command=cmd,
|
||||
dependencies=deps,
|
||||
label=lbl,
|
||||
nCores=n_cores))
|
||||
task = self.addTask(command=cmd, dependencies=deps, label=lbl, nCores=n_cores) # noqa
|
||||
binarization_tasks.append(task)
|
||||
|
||||
'''
|
||||
' The post_binarization_jobs are created based on the output files
|
||||
' of the binarization_jobs. So wait until they are finished.
|
||||
'''
|
||||
self.waitForTasks()
|
||||
|
||||
'''
|
||||
@ -160,10 +183,9 @@ class OCRPipeline(WorkflowRunner):
|
||||
' ##################################################
|
||||
'''
|
||||
for i, job in enumerate(self.jobs):
|
||||
input_dir = os.path.join(job.output_dir, 'tmp')
|
||||
output_dir = input_dir
|
||||
files = filter(lambda x: x.endswith('.bin.png'),
|
||||
os.listdir(input_dir))
|
||||
input_dir = job.intermediate_dir
|
||||
output_dir = job.intermediate_dir
|
||||
files = filter(lambda x: x.endswith('.bin.png'), os.listdir(input_dir)) # noqa
|
||||
for file in files:
|
||||
# int conversion is done in order to trim leading zeros
|
||||
page_number = int(file.split('.', 1)[0])
|
||||
@ -172,8 +194,8 @@ class OCRPipeline(WorkflowRunner):
|
||||
os.path.join(output_dir, output_file))
|
||||
|
||||
'''
|
||||
' The ocr_jobs are created based of the output files of either the
|
||||
' split_jobs or post_binarization_jobs. So wait until they are
|
||||
' The ocr_tasks are created based of the output files of either the
|
||||
' split_tasks or binarization_tasks. So wait until they are
|
||||
' finished.
|
||||
'''
|
||||
self.waitForTasks()
|
||||
@ -183,7 +205,7 @@ class OCRPipeline(WorkflowRunner):
|
||||
' # ocr #
|
||||
' ##################################################
|
||||
'''
|
||||
ocr_jobs = []
|
||||
ocr_tasks = []
|
||||
'''
|
||||
' Tesseract runs fastest with four cores. So we run it with either four
|
||||
' or, if there are less then four cores available for this workflow,
|
||||
@ -191,33 +213,34 @@ class OCRPipeline(WorkflowRunner):
|
||||
'''
|
||||
n_cores = min(4, self.n_cores)
|
||||
for i, job in enumerate(self.jobs):
|
||||
input_dir = os.path.join(job.output_dir, 'tmp')
|
||||
output_dir = input_dir
|
||||
files = filter(lambda x: x.endswith('.bin.png' if self.binarize else '.tif'), os.listdir(input_dir)) # noqa
|
||||
files.sort(key=lambda x: int(re.search(r'\d+', x).group(0)))
|
||||
input_dir = job.intermediate_dir
|
||||
output_dir = job.intermediate_dir
|
||||
files = os.listdir(input_dir)
|
||||
if self.binarize:
|
||||
deps = 'binarization_-_{}'.format(i)
|
||||
files = filter(lambda x: x.endswith('.bin.png'), files)
|
||||
else:
|
||||
deps = 'split_input_-_{}'.format(i)
|
||||
files = filter(lambda x: x.endswith('.tif'), files)
|
||||
files = natsorted(files)
|
||||
files = map(lambda x: os.path.join(input_dir, x), files)
|
||||
number = 0
|
||||
for file in files:
|
||||
output_file_base = os.path.join(output_dir, file.rsplit('.', 2 if self.binarize else 1)[0]) # noqa
|
||||
for j, file in enumerate(files):
|
||||
if self.binarize:
|
||||
output_file_base = os.path.join(output_dir, file.rsplit('.', 2)[0]) # noqa
|
||||
else:
|
||||
output_file_base = os.path.join(output_dir, file.rsplit('.', 1)[0]) # noqa
|
||||
cmd = 'tesseract "{}" "{}"'.format(file, output_file_base)
|
||||
cmd += ' -l "{}"'.format(self.lang)
|
||||
cmd += ' hocr pdf txt'
|
||||
cmd += ' && '
|
||||
cmd += 'sed -i \'s+{}/++g\' "{}".hocr'.format(input_dir, output_file_base) # noqa
|
||||
if self.binarize:
|
||||
deps = 'binarization_-_{}'.format(i)
|
||||
else:
|
||||
deps = 'split_input_-_{}'.format(i)
|
||||
label = 'ocr_-_{}-{}'.format(i, number)
|
||||
ocr_jobs.append(self.addTask(command=cmd,
|
||||
dependencies=deps,
|
||||
label=label,
|
||||
nCores=n_cores))
|
||||
number += 1
|
||||
lbl = 'ocr_-_{}-{}'.format(i, j)
|
||||
task = self.addTask(command=cmd, dependencies=deps, label=lbl, nCores=n_cores) # noqa
|
||||
ocr_tasks.append(task)
|
||||
|
||||
'''
|
||||
' The following jobs are created based of the output files of the
|
||||
' ocr_jobs. So wait until they are finished.
|
||||
' ocr_tasks. So wait until they are finished.
|
||||
'''
|
||||
self.waitForTasks()
|
||||
|
||||
@ -226,16 +249,14 @@ class OCRPipeline(WorkflowRunner):
|
||||
' # combined pdf creation #
|
||||
' ##################################################
|
||||
'''
|
||||
combined_pdf_creation_jobs = []
|
||||
combined_pdf_creation_tasks = []
|
||||
n_cores = min(self.n_cores, max(1, int(self.n_cores / len(self.jobs))))
|
||||
for i, job in enumerate(self.jobs):
|
||||
input_dir = os.path.join(job.output_dir, 'tmp')
|
||||
output_dir = job.output_dir
|
||||
files = filter(lambda x: x.endswith('.pdf'),
|
||||
os.listdir(input_dir))
|
||||
files.sort(key=lambda x: int(re.search(r'\d+', x).group(0)))
|
||||
input_dir = job.intermediate_dir
|
||||
output_file = os.path.join(job.output_dir, '{}.pdf'.format(job.name)) # noqa
|
||||
files = filter(lambda x: x.endswith('.pdf'), os.listdir(input_dir))
|
||||
files = natsorted(files)
|
||||
files = map(lambda x: os.path.join(input_dir, x), files)
|
||||
output_file = os.path.join(output_dir, '{}.pdf'.format(job.name))
|
||||
cmd = 'gs'
|
||||
cmd += ' -dBATCH'
|
||||
cmd += ' -dNOPAUSE'
|
||||
@ -245,77 +266,75 @@ class OCRPipeline(WorkflowRunner):
|
||||
cmd += ' -sDEVICE=pdfwrite'
|
||||
cmd += ' "-sOutputFile={}"'.format(output_file)
|
||||
cmd += ' "{}"'.format('" "'.join(files))
|
||||
deps = filter(lambda x: x.startswith('ocr_-_{}'.format(i)),
|
||||
ocr_jobs)
|
||||
deps = filter(lambda x: x.startswith('ocr_-_{}'.format(i)), ocr_tasks) # noqa
|
||||
lbl = 'combined_pdf_creation_-_{}'.format(i)
|
||||
combined_pdf_creation_jobs.append(self.addTask(command=cmd,
|
||||
dependencies=deps,
|
||||
label=lbl,
|
||||
nCores=n_cores))
|
||||
task = self.addTask(command=cmd, dependencies=deps, label=lbl, nCores=n_cores) # noqa
|
||||
combined_pdf_creation_tasks.append(task)
|
||||
|
||||
'''
|
||||
' ##################################################
|
||||
' # combined txt creation #
|
||||
' ##################################################
|
||||
'''
|
||||
combined_txt_creation_jobs = []
|
||||
combined_txt_creation_tasks = []
|
||||
for i, job in enumerate(self.jobs):
|
||||
input_dir = os.path.join(job.output_dir, 'tmp')
|
||||
files = filter(lambda x: x.endswith('.txt'), os.listdir(input_dir))
|
||||
files.sort(key=lambda x: int(re.search(r'\d+', x).group(0)))
|
||||
files = map(lambda x: os.path.join(input_dir, x), files)
|
||||
input_dir = job.intermediate_dir
|
||||
output_file = os.path.join(job.output_dir, '{}.txt'.format(job.name)) # noqa
|
||||
files = filter(lambda x: x.endswith('.txt'), os.listdir(input_dir))
|
||||
files = natsorted(files)
|
||||
files = map(lambda x: os.path.join(input_dir, x), files)
|
||||
cmd = 'cat "{}" > "{}"'.format('" "'.join(files), output_file)
|
||||
deps = filter(lambda x: x.startswith('ocr_-_{}'.format(i)),
|
||||
ocr_jobs)
|
||||
deps = filter(lambda x: x.startswith('ocr_-_{}'.format(i)), ocr_tasks) # noqa
|
||||
lbl = 'combined_txt_creation_-_{}'.format(i)
|
||||
combined_txt_creation_jobs.append(self.addTask(command=cmd,
|
||||
dependencies=deps,
|
||||
label=lbl))
|
||||
task = self.addTask(command=cmd, dependencies=deps, label=lbl)
|
||||
combined_txt_creation_tasks.append(task)
|
||||
|
||||
'''
|
||||
' ##################################################
|
||||
' # tei p5 creation #
|
||||
' ##################################################
|
||||
'''
|
||||
tei_p5_creation_jobs = []
|
||||
tei_p5_creation_tasks = []
|
||||
for i, job in enumerate(self.jobs):
|
||||
input_dir = os.path.join(job.output_dir, 'tmp')
|
||||
files = filter(lambda x: x.endswith('.hocr'), os.listdir(input_dir)) # noqa
|
||||
files.sort(key=lambda x: int(re.search(r'\d+', x).group(0)))
|
||||
files = map(lambda x: os.path.join(input_dir, x), files)
|
||||
input_dir = job.intermediate_dir
|
||||
output_file = os.path.join(job.output_dir, '{}.xml'.format(job.name)) # noqa
|
||||
cmd = 'hocrtotei "{}" "{}"'.format('" "'.join(files), output_file)
|
||||
deps = filter(lambda x: x.startswith('ocr_-_{}'.format(i)),
|
||||
ocr_jobs)
|
||||
files = filter(lambda x: x.endswith('.hocr'),
|
||||
os.listdir(input_dir))
|
||||
files = natsorted(files)
|
||||
files = map(lambda x: os.path.join(input_dir, x), files)
|
||||
cmd = 'hocrtotei "{}" "{}"'.format('" "'.join(files),
|
||||
output_file)
|
||||
deps = filter(lambda x: x.startswith('ocr_-_{}'.format(i)), ocr_tasks) # noqa
|
||||
lbl = 'tei_p5_creation_-_{}'.format(i)
|
||||
tei_p5_creation_jobs.append(self.addTask(command=cmd,
|
||||
dependencies=deps,
|
||||
label=lbl))
|
||||
task = self.addTask(command=cmd, dependencies=deps, label=lbl)
|
||||
tei_p5_creation_tasks.append(task)
|
||||
|
||||
'''
|
||||
' ##################################################
|
||||
' # poco bundle creation #
|
||||
' ##################################################
|
||||
'''
|
||||
poco_bundle_creation_jobs = []
|
||||
poco_bundle_creation_tasks = []
|
||||
for i, job in enumerate(self.jobs):
|
||||
input_dir = os.path.join(job.output_dir, 'tmp')
|
||||
input_dir = job.intermediate_dir
|
||||
output_dir = os.path.join(job.output_dir, 'poco')
|
||||
cmd = 'mv "{}"/*.hocr "{}"'.format(input_dir, output_dir)
|
||||
cmd += ' && '
|
||||
cmd += 'mv "{}"/*.{} "{}"'.format(input_dir, 'bin.png' if self.binarize else 'tif', output_dir) # noqa
|
||||
deps = filter(lambda x: x.startswith('ocr_-_{}'.format(i)),
|
||||
ocr_jobs)
|
||||
files = os.listdir(input_dir)
|
||||
if self.binarize:
|
||||
files = filter(lambda x: x.endswith(('.bin.png', '.hocr')), files) # noqa
|
||||
else:
|
||||
files = filter(lambda x: x.endswith(('.tif', '.hocr')), files)
|
||||
files = natsorted(files)
|
||||
files = map(lambda x: os.path.join(input_dir, x), files)
|
||||
cmd = 'mv "{}" "{}"'.format('" "'.join(files), output_dir)
|
||||
deps = filter(lambda x: x.startswith('ocr_-_{}'.format(i)), ocr_tasks) # noqa
|
||||
deps.append('tei_p5_creation_-_{}'.format(i))
|
||||
lbl = 'poco_bundle_creation_-_{}'.format(i)
|
||||
poco_bundle_creation_jobs.append(self.addTask(command=cmd,
|
||||
dependencies=deps,
|
||||
label=lbl))
|
||||
task = self.addTask(command=cmd, dependencies=deps, label=lbl)
|
||||
poco_bundle_creation_tasks.append(task)
|
||||
|
||||
'''
|
||||
' The following jobs are created based of the output files of the
|
||||
' combined_pdf_creation_jobs. So wait until they are finished.
|
||||
' combined_pdf_creation_tasks. So wait until they are finished.
|
||||
'''
|
||||
self.waitForTasks()
|
||||
|
||||
@ -324,126 +343,126 @@ class OCRPipeline(WorkflowRunner):
|
||||
' # cleanup #
|
||||
' ##################################################
|
||||
'''
|
||||
cleanup_jobs = []
|
||||
cleanup_tasks = []
|
||||
for i, job in enumerate(self.jobs):
|
||||
input_dir = os.path.join(job.output_dir, 'tmp')
|
||||
input_dir = job.intermediate_dir
|
||||
cmd = 'rm -r "{}"'.format(input_dir)
|
||||
deps = ['combined_pdf_creation_-_{}'.format(i)]
|
||||
deps.append('combined_txt_creation_-_{}'.format(i))
|
||||
deps.append('poco_bundle_creation_-_{}'.format(i))
|
||||
deps.append('tei_p5_creation_-_{}'.format(i))
|
||||
lbl = 'cleanup_-_{}'.format(i)
|
||||
cleanup_jobs.append(self.addTask(command=cmd,
|
||||
dependencies=deps,
|
||||
label=lbl))
|
||||
deps = ['combined_pdf_creation_-_{}'.format(i),
|
||||
'combined_txt_creation_-_{}'.format(i),
|
||||
'poco_bundle_creation_-_{}'.format(i),
|
||||
'tei_p5_creation_-_{}'.format(i)]
|
||||
lbl = 'job_cleanup_-_{}'.format(i)
|
||||
task = self.addTask(command=cmd, dependencies=deps, label=lbl)
|
||||
cleanup_tasks.append(task)
|
||||
|
||||
input_dir = self.intermediate_dir
|
||||
cmd = 'rm -r "{}"'.format(input_dir)
|
||||
deps = cleanup_tasks
|
||||
lbl = 'pipeline_cleanup'
|
||||
task = self.addTask(command=cmd, dependencies=deps, label=lbl)
|
||||
cleanup_tasks.append(task)
|
||||
|
||||
self.waitForTasks()
|
||||
|
||||
'''
|
||||
' ##################################################
|
||||
' # zip creation #
|
||||
' ##################################################
|
||||
'''
|
||||
zip_creation_jobs = []
|
||||
zip_creation_tasks = []
|
||||
if self.zip is not None:
|
||||
# Remove .zip file extension if provided
|
||||
if self.zip.endswith('.zip'):
|
||||
self.zip = self.zip[:-4]
|
||||
self.zip = self.zip if self.zip else 'output'
|
||||
# zip all files
|
||||
cmd = 'cd "{}"'.format(self.output_dir)
|
||||
cmd += ' && '
|
||||
cmd += 'zip'
|
||||
cmd += ' -r'
|
||||
cmd += ' "{}".all.zip .'.format(self.zip)
|
||||
cmd += ' "{}.all.zip" .'.format(self.zip)
|
||||
cmd += ' -x "pyflow.data*" "*tmp*"'
|
||||
cmd += ' -i "*.pdf" "*.txt" "*.xml" "*.hocr" "*.{}"'.format('bin.png' if self.binarize else 'tif') # noqa
|
||||
cmd += ' && '
|
||||
cmd += 'cd -'
|
||||
deps = combined_pdf_creation_jobs
|
||||
deps += combined_txt_creation_jobs
|
||||
deps += poco_bundle_creation_jobs
|
||||
deps = combined_pdf_creation_tasks + combined_txt_creation_tasks + poco_bundle_creation_tasks # noqa
|
||||
lbl = 'zip_creation_-_all'
|
||||
zip_creation_jobs.append(self.addTask(command=cmd,
|
||||
dependencies=deps,
|
||||
label=lbl))
|
||||
task = self.addTask(command=cmd, dependencies=deps, label=lbl)
|
||||
zip_creation_tasks.append(task)
|
||||
# zip PDF files
|
||||
cmd = 'cd "{}"'.format(self.output_dir)
|
||||
cmd += ' && '
|
||||
cmd += 'zip'
|
||||
cmd += ' -r'
|
||||
cmd += ' "{}".pdf.zip .'.format(self.zip)
|
||||
cmd += ' "{}.pdf.zip" .'.format(self.zip)
|
||||
cmd += ' -x "pyflow.data*" "*tmp*"'
|
||||
cmd += ' -i "*.pdf"'
|
||||
cmd += ' && '
|
||||
cmd += 'cd -'
|
||||
deps = combined_pdf_creation_jobs
|
||||
deps = combined_pdf_creation_tasks
|
||||
lbl = 'zip_creation_-_pdf'
|
||||
zip_creation_jobs.append(self.addTask(command=cmd,
|
||||
dependencies=deps,
|
||||
label=lbl))
|
||||
task = self.addTask(command=cmd, dependencies=deps, label=lbl)
|
||||
zip_creation_tasks.append(task)
|
||||
# zip TXT files
|
||||
cmd = 'cd "{}"'.format(self.output_dir)
|
||||
cmd += ' && '
|
||||
cmd += 'zip'
|
||||
cmd += ' -r'
|
||||
cmd += ' "{}".txt.zip .'.format(self.zip)
|
||||
cmd += ' "{}.txt.zip" .'.format(self.zip)
|
||||
cmd += ' -x "pyflow.data*" "*tmp*"'
|
||||
cmd += ' -i "*.txt"'
|
||||
cmd += ' && '
|
||||
cmd += 'cd -'
|
||||
deps = combined_txt_creation_jobs
|
||||
deps = combined_txt_creation_tasks
|
||||
lbl = 'zip_creation_-_txt'
|
||||
zip_creation_jobs.append(self.addTask(command=cmd,
|
||||
dependencies=deps,
|
||||
label=lbl))
|
||||
task = self.addTask(command=cmd, dependencies=deps, label=lbl)
|
||||
zip_creation_tasks.append(task)
|
||||
# zip XML files
|
||||
cmd = 'cd "{}"'.format(self.output_dir)
|
||||
cmd += ' && '
|
||||
cmd += 'zip'
|
||||
cmd += ' -r'
|
||||
cmd += ' "{}".xml.zip .'.format(self.zip)
|
||||
cmd += ' "{}.xml.zip" .'.format(self.zip)
|
||||
cmd += ' -x "pyflow.data*" "*tmp*"'
|
||||
cmd += ' -i "*.xml"'
|
||||
cmd += ' && '
|
||||
cmd += 'cd -'
|
||||
deps = tei_p5_creation_jobs
|
||||
deps = tei_p5_creation_tasks
|
||||
lbl = 'zip_creation_-_xml'
|
||||
zip_creation_jobs.append(self.addTask(command=cmd,
|
||||
dependencies=deps,
|
||||
label=lbl))
|
||||
task = self.addTask(command=cmd, dependencies=deps, label=lbl)
|
||||
zip_creation_tasks.append(task)
|
||||
# zip PoCo bundles
|
||||
cmd = 'cd "{}"'.format(self.output_dir)
|
||||
cmd += ' && '
|
||||
cmd += 'zip'
|
||||
cmd += ' -r'
|
||||
cmd += ' "{}".poco.zip .'.format(self.zip)
|
||||
cmd += ' "{}.poco.zip" .'.format(self.zip)
|
||||
cmd += ' -x "pyflow.data*" "*tmp*"'
|
||||
cmd += ' -i "*.hocr" "*.{}"'.format('bin.png' if self.binarize else 'tif') # noqa
|
||||
cmd += ' && '
|
||||
cmd += 'cd -'
|
||||
deps = poco_bundle_creation_jobs
|
||||
deps = poco_bundle_creation_tasks
|
||||
lbl = 'zip_creation_-_poco'
|
||||
zip_creation_jobs.append(self.addTask(command=cmd,
|
||||
dependencies=deps,
|
||||
label=lbl))
|
||||
task = self.addTask(command=cmd, dependencies=deps, label=lbl)
|
||||
zip_creation_tasks.append(task)
|
||||
|
||||
|
||||
def collect_jobs(input_dir, output_dir):
|
||||
def collect_jobs(input_dir, output_dir, intermediate_dir):
|
||||
jobs = []
|
||||
for file in os.listdir(input_dir):
|
||||
if os.path.isdir(os.path.join(input_dir, file)):
|
||||
jobs += collect_jobs(os.path.join(input_dir, file),
|
||||
os.path.join(output_dir, file))
|
||||
elif file.endswith('.pdf'):
|
||||
jobs.append(OCRPipelineJob(os.path.join(input_dir, file),
|
||||
os.path.join(output_dir, file)))
|
||||
elif file.lower().endswith('.pdf'):
|
||||
job = OCRPipelineJob(os.path.join(input_dir, file),
|
||||
os.path.join(output_dir, file),
|
||||
os.path.join(intermediate_dir, file))
|
||||
jobs.append(job)
|
||||
return jobs
|
||||
|
||||
|
||||
def main():
|
||||
args = parse_args()
|
||||
jobs = collect_jobs(args.input_directory, args.output_directory)
|
||||
ocr_pipeline = OCRPipeline(args.binarize, jobs, args.language,
|
||||
args.n_cores, args.output_directory, args.zip)
|
||||
ocr_pipeline = OCRPipeline(args.input_directory, args.language,
|
||||
args.output_directory, args.binarize,
|
||||
args.intermediate_directory, args.n_cores,
|
||||
args.zip)
|
||||
retval = ocr_pipeline.run(
|
||||
dataDirRoot=(args.log_dir or args.output_directory),
|
||||
nCores=args.n_cores
|
||||
|
||||
@ -7,6 +7,7 @@ import subprocess
|
||||
|
||||
CONTAINER_IMAGE = 'gitlab.ub.uni-bielefeld.de:4567/sfb1288inf/ocr:latest'
|
||||
CONTAINER_INPUT_DIR = '/input'
|
||||
CONTAINER_INTERMEDIATE_DIR = '/intermediate'
|
||||
CONTAINER_OUTPUT_DIR = '/output'
|
||||
UID = str(os.getuid())
|
||||
GID = str(os.getgid())
|
||||
@ -14,9 +15,15 @@ GID = str(os.getgid())
|
||||
parser = ArgumentParser(add_help=False)
|
||||
parser.add_argument('-i', '--input-directory')
|
||||
parser.add_argument('-o', '--output-directory')
|
||||
parser.add_argument('--intermediate-directory')
|
||||
args, remaining_args = parser.parse_known_args()
|
||||
|
||||
cmd = ['docker', 'run', '--rm', '-it', '-u', '{}:{}'.format(UID, GID)]
|
||||
if args.intermediate_directory is not None:
|
||||
cmd += ['-v', '{}:{}'.format(os.path.abspath(args.intermediate_directory),
|
||||
CONTAINER_INTERMEDIATE_DIR)]
|
||||
remaining_args.insert(0, CONTAINER_INTERMEDIATE_DIR)
|
||||
remaining_args.insert(0, '--intermediate-directory')
|
||||
if args.output_directory is not None:
|
||||
cmd += ['-v', '{}:{}'.format(os.path.abspath(args.output_directory),
|
||||
CONTAINER_OUTPUT_DIR)]
|
||||
|
||||
Loading…
Reference in New Issue
Block a user