sporada/ocr
1
0
Fork 0
mirror of https://gitlab.ub.uni-bielefeld.de/sfb1288inf/ocr.git synced 2025-07-01 12:30:33 +00:00
Code Issues Projects Releases Wiki Activity

Correct order for output files.

Browse Source
This commit is contained in:
Patrick Jentsch
2019-05-13 15:03:43 +02:00
parent 937abb8c8d
commit 843151e547
3 changed files with 174 additions and 80 deletions
Download Patch File Download Diff File Expand all files Collapse all files

189
ocr
View File

@ -19,32 +19,27 @@ from pyflow import WorkflowRunner
''' TODO:
' Implement --end-page: Last page to ocr
' Implement --memMb: Total amount of memory (RAM) available for this workflow. Default: 2048 * nCores
' Implement --memMb: Total amount of memory (RAM) available for this workflow.
' Default: 2048 * nCores
' Implement --rotate: Rotate pages from input (90, 180, 270)
' Implement --split-pages: Split pages in half after possible rotation
' Implement --start-page: First page to ocr
'''
def parse_arguments():
parser = argparse.ArgumentParser(
"Performs OCR of (historical) documents utilizing OCRopus for \
preprocessing and Tesseract OCR for OCR. Available outputs are HOCR, \
PDF, shrinked PDF, and simple DTAbf (TEI P5 compliant). Software \
requirements: imagemagick, ocropus, pdftk, pdftoppm, poppler-utils, \
pyflow, python2.7, tesseract"
pyflow, python2.7, python3.5, tesseract"
)
parser.add_argument("-i",
dest="inputDir",
help="Input directory.",
required=True)
parser.add_argument("-l",
dest='lang',
help="Language for OCR",
required=True)
parser.add_argument("-o",
dest="outputDir",
help="Output directory.",
required=True)
parser.add_argument("--skip-binarization",
action='store_true',
default=False,
@ -67,14 +62,16 @@ def parse_arguments():
class OCRWorkflow(WorkflowRunner):
def __init__(self, jobs, skipBinarization, keepIntermediates, lang, nCores):
self.jobs = jobs
self.skipBinarization = skipBinarization
self.keepIntermediates = keepIntermediates
self.lang = lang
self.nCores = nCores
self.defaultNCores = min(nCores, max(1, int(nCores / len(jobs))))
def __init__(self, args):
self.jobs = analyze_jobs()
self.skipBinarization = args.skipBinarization
self.keepIntermediates = args.keepIntermediates
self.lang = args.lang
self.nCores = args.nCores
self.defaultNCores = min(
self.nCores,
max(1, int(self.nCores / len(self.jobs)))
)
def workflow(self):
###
@ -93,10 +90,17 @@ class OCRWorkflow(WorkflowRunner):
)
if not self.skipBinarization:
cmd += ' "%s" "%s"' % (
os.path.join(job["output_dir"], "tmp", "binarized_png"),
os.path.join(job["output_dir"], "tmp", "normalized_png"),
os.path.join(job["output_dir"], "tmp", "bin.png"),
os.path.join(job["output_dir"], "tmp", "nrm.png"),
)
create_output_directories_jobs.append(
self.addTask(
command=cmd,
label="create_output_directories_job_-_%i" % (
create_output_directories_job_number
),
nCores=self.defaultNCores)
)
create_output_directories_jobs.append(self.addTask(label="create_output_directories_job_-_%i" % (create_output_directories_job_number), command=cmd, nCores=self.defaultNCores))
###
# Task "split_job": split input file into one tiff file per page
@ -116,7 +120,14 @@ class OCRWorkflow(WorkflowRunner):
job["path"],
os.path.join(job["output_dir"], "tmp", "page")
)
split_jobs.append(self.addTask(label="split_job_-_%i" % (split_job_number), command=cmd, dependencies=create_output_directories_jobs, nCores=self.defaultNCores))
split_jobs.append(
self.addTask(
command=cmd,
dependencies=create_output_directories_jobs,
label="split_job_-_%i" % (split_job_number),
nCores=self.defaultNCores
)
)
###
# Task "ocropus_nlbin_job": binarize tiff files from previous split
@ -132,12 +143,21 @@ class OCRWorkflow(WorkflowRunner):
if not self.skipBinarization:
for job in self.jobs:
binarization_job_number += 1
cmd = 'ocropus-nlbin --output "%s" --parallel "%i" $(ls "%s"/*.tif | sort -V)' % (
cmd = 'ocropus-nlbin --output "%s" --parallel "%i" $(ls --quoting-style=shell-escape -v "%s"/*.tif)' % (
os.path.join(job["output_dir"], "tmp"),
binarization_job_nCores,
os.path.join(job["output_dir"], "tmp")
)
binarization_jobs.append(self.addTask(label="binarization_job_-_%i" % (binarization_job_number), command=cmd, dependencies=split_jobs, nCores=binarization_job_nCores))
binarization_jobs.append(
self.addTask(
command=cmd,
dependencies=split_jobs,
label="binarization_job_-_%i" % (
binarization_job_number
),
nCores=binarization_job_nCores
)
)
###
# Task "post_binarization_job": Normalize file names from binarization
@ -152,9 +172,21 @@ class OCRWorkflow(WorkflowRunner):
post_binarization_job_number += 1
cmd = 'mv "%s" "%s"' % (
os.path.join(job["output_dir"], "tmp", file),
os.path.join(job["output_dir"], "tmp", "page-%i.%s" % (int(file.split(".", 1)[0]), file.split(".", 1)[1])),
os.path.join(job["output_dir"], "tmp", "page-%i.%s" % (
int(file.split(".", 1)[0]),
file.split(".", 1)[1])
),
)
post_binarization_jobs.append(
self.addTask(
command=cmd,
dependencies=binarization_jobs,
label="post_binarization_job_-_%i" % (
post_binarization_job_number
),
nCores=self.defaultNCores
)
)
post_binarization_jobs.append(self.addTask(label="post_binarization_job_-_%i" % (post_binarization_job_number), command=cmd, dependencies=binarization_jobs, nCores=self.defaultNCores))
###
# Task "ocr_job": perform OCR
@ -165,8 +197,8 @@ class OCRWorkflow(WorkflowRunner):
ocr_job_number = 0
'''
' Tesseract runs fastest with four cores. So we run it with either four
' or, if there are less then four cores available for this workflow, the
' available core number.
' or, if there are less then four cores available for this workflow,
' the available core number.
'''
ocr_job_nCores = min(4, self.nCores)
'''
@ -183,7 +215,14 @@ class OCRWorkflow(WorkflowRunner):
os.path.join(job["output_dir"], "tmp", file.rsplit(".", 1 if self.skipBinarization else 2)[0]),
self.lang
)
ocr_jobs.append(self.addTask(label="ocr_job_-_%i" % (ocr_job_number), command=cmd, dependencies=post_binarization_jobs, nCores=ocr_job_nCores))
ocr_jobs.append(
self.addTask(
command=cmd,
dependencies=post_binarization_jobs,
label="ocr_job_-_%i" % (ocr_job_number),
nCores=ocr_job_nCores
)
)
###
# Task "hocr_to_tei_job": create TEI P5 file from hocr files
@ -197,7 +236,14 @@ class OCRWorkflow(WorkflowRunner):
os.path.join(job["output_dir"], "tmp"),
os.path.join(job["output_dir"], job["filename"].rsplit(".", 1)[0] + ".xml")
)
hocr_to_tei_jobs.append(self.addTask(label="hocr_to_tei_job_-_%i" % (hocr_to_tei_job_number), command=cmd, dependencies=ocr_jobs, nCores=self.defaultNCores))
hocr_to_tei_jobs.append(
self.addTask(
command=cmd,
dependencies=ocr_jobs,
label="hocr_to_tei_job_-_%i" % (hocr_to_tei_job_number),
nCores=self.defaultNCores
)
)
###
# Task "pdf_merge_job": Merge PDF files
@ -207,11 +253,18 @@ class OCRWorkflow(WorkflowRunner):
pdf_merge_job_number = 0
for job in self.jobs:
pdf_merge_job_number += 1
cmd = 'pdftk $(ls "%s"/*.pdf | sort -V) cat output "%s"' % (
cmd = '(ls --quoting-style=shell-escape -v "%s"/*.pdf && echo "\'%s\'") | xargs pdfunite' % (
os.path.join(job["output_dir"], "tmp"),
os.path.join(job["output_dir"], job["filename"].rsplit(".", 1)[0] + ".pdf")
)
pdf_merge_jobs.append(self.addTask(label="pdf_merge_job_-_%i" % (pdf_merge_job_number), command=cmd, dependencies=ocr_jobs, nCores=self.defaultNCores))
pdf_merge_jobs.append(
self.addTask(
command=cmd,
dependencies=ocr_jobs,
label="pdf_merge_job_-_%i" % (pdf_merge_job_number),
nCores=self.defaultNCores
)
)
###
# Task "txt_merge_job": Merge .txt files
@ -221,11 +274,18 @@ class OCRWorkflow(WorkflowRunner):
txt_merge_job_number = 0
for job in self.jobs:
txt_merge_job_number += 1
cmd = 'cat $(ls "%s"/*.txt | sort -V) > "%s"' % (
cmd = 'ls --quoting-style=shell-escape -v "%s"/*.txt | xargs cat > "%s"' % (
os.path.join(job["output_dir"], "tmp"),
os.path.join(job["output_dir"], job["filename"].rsplit(".", 1)[0] + ".txt")
)
txt_merge_jobs.append(self.addTask(label="txt_merge_job_-_%i" % (txt_merge_job_number), command=cmd, dependencies=ocr_jobs, nCores=self.defaultNCores))
txt_merge_jobs.append(
self.addTask(
command=cmd,
dependencies=ocr_jobs,
label="txt_merge_job_-_%i" % (txt_merge_job_number),
nCores=self.defaultNCores
)
)
###
# Task "cleanup_job": remove temporary files
@ -236,35 +296,59 @@ class OCRWorkflow(WorkflowRunner):
if self.keepIntermediates:
for job in self.jobs:
cleanup_job_counter += 1
cmd = 'mv "%s"/*.hocr "%s" && mv "%s"/*.pdf "%s" && mv "%s"/*.tif "%s" && mv "%s"/*.txt "%s"' % (
cmd = 'mv "%s"/*.hocr "%s"' % (
os.path.join(job["output_dir"], "tmp"),
os.path.join(job["output_dir"], "tmp", "hocr"),
)
cmd += ' && mv "%s"/*.pdf "%s"' % (
os.path.join(job["output_dir"], "tmp"),
os.path.join(job["output_dir"], "tmp", "pdf"),
)
cmd += ' && mv "%s"/*.tif "%s"' % (
os.path.join(job["output_dir"], "tmp"),
os.path.join(job["output_dir"], "tmp", "tiff"),
)
cmd += ' && mv "%s"/*.txt "%s"' % (
os.path.join(job["output_dir"], "tmp"),
os.path.join(job["output_dir"], "tmp", "txt")
os.path.join(job["output_dir"], "tmp", "txt"),
)
if not self.skipBinarization:
cmd += ' && mv "%s"/*.bin.png "%s" && mv "%s"/*.nrm.png "%s"' % (
cmd += ' && mv "%s"/*.bin.png "%s"' % (
os.path.join(job["output_dir"], "tmp"),
os.path.join(job["output_dir"], "tmp", "binarized_png"),
os.path.join(job["output_dir"], "tmp"),
os.path.join(job["output_dir"], "tmp", "normalized_png"),
os.path.join(job["output_dir"], "tmp", "bin.png"),
)
cleanup_jobs.append(self.addTask(label="cleanup_job_-_%i" % (cleanup_job_counter), command=cmd, dependencies=hocr_to_tei_jobs + pdf_merge_jobs + txt_merge_jobs, nCores=self.defaultNCores))
cmd += ' && mv "%s"/*.nrm.png "%s"' % (
os.path.join(job["output_dir"], "tmp"),
os.path.join(job["output_dir"], "tmp", "nrm.png"),
)
cleanup_jobs.append(
self.addTask(
command=cmd,
dependencies=hocr_to_tei_jobs + pdf_merge_jobs + txt_merge_jobs,
label="cleanup_job_-_%i" % (cleanup_job_counter),
nCores=self.defaultNCores
)
)
else:
for job in self.jobs:
cleanup_job_counter += 1
cmd = 'rm -r "%s"' % (
os.path.join(job["output_dir"], "tmp")
)
cleanup_jobs.append(self.addTask(label="cleanup_job_-_%i" % (cleanup_job_counter), command=cmd, dependencies=hocr_to_tei_jobs + pdf_merge_jobs + txt_merge_jobs), nCores=self.defaultNCores)
cleanup_jobs.append(
self.addTask(
label="cleanup_job_-_%i" % (cleanup_job_counter),
command=cmd,
dependencies=hocr_to_tei_jobs + pdf_merge_jobs + txt_merge_jobs,
nCores=self.defaultNCores
)
)
def analyze_jobs(inputDir, outputDir):
def analyze_jobs():
inputDir = "/files_for_ocr"
jobs = []
outputDir = "/files_from_ocr"
for file in os.listdir(inputDir):
if os.path.isdir(os.path.join(inputDir, file)):
@ -273,7 +357,13 @@ def analyze_jobs(inputDir, outputDir):
os.path.join(outputDir, file)
)
elif file.endswith((".pdf", ".tif", ".tiff")):
jobs.append({"filename": file, "output_dir": os.path.join(outputDir, file), "path": os.path.join(inputDir, file)})
jobs.append(
{
"filename": file,
"output_dir": os.path.join(outputDir, file),
"path": os.path.join(inputDir, file)
}
)
return jobs
@ -281,15 +371,10 @@ def analyze_jobs(inputDir, outputDir):
def main():
args = parse_arguments()
wflow = OCRWorkflow(
analyze_jobs(args.inputDir, args.outputDir),
args.skipBinarization,
args.keepIntermediates,
args.lang,
args.nCores
)
wflow = OCRWorkflow(args)
retval = wflow.run(dataDirRoot="/files_from_ocr", nCores=args.nCores)
retval = wflow.run(nCores=args.nCores)
sys.exit(retval)