sporada/ocr
1
0
Fork 0
mirror of https://gitlab.ub.uni-bielefeld.de/sfb1288inf/ocr.git synced 2024-12-27 08:44:17 +00:00
Code Issues Projects Releases Wiki Activity

Update

Browse Source
This commit is contained in:
Patrick Jentsch 2018-12-18 16:39:33 +01:00
parent 2cfcc2b2c2
commit fc5ae7bd37
1 changed files with 80 additions and 133 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

213
ocr_pyflow
View File

@ -31,125 +31,72 @@ from pyflow import WorkflowRunner
''' TODO:
' Implement --end-page: Last page to ocr
' Implement --memMb: Total amount of memory (RAM) available for this workflow. Default: 2048 * nCores
' Implement --rotate: Rotate pages from input (90, 180, 270)
' Implement --split-pages: Split pages in half after possible rotation
' Implement --start-page: First page to ocr
'''
def parse_arguments(): def parse_arguments():
parser = argparse.ArgumentParser( parser = argparse.ArgumentParser("Performs OCR of (historical) documents utilizing OCRopus for preprocessing and Tesseract OCR \
"Performs OCR of (historical) documents utilizing OCRopus for preprocessing and Tesseract OCR \
for OCR. Available outputs are HOCR, PDF, shrinked PDF, and simple DTAbf \ for OCR. Available outputs are HOCR, PDF, shrinked PDF, and simple DTAbf \
(TEI P5 compliant). Software requirements: imagemagick, ocropus, pdftk, pdftoppm, poppler-utils, pyflow, python2.7, tesseract") (TEI P5 compliant). Software requirements: imagemagick, ocropus, pdftk, pdftoppm, poppler-utils, pyflow, python2.7, tesseract")
parser.add_argument("-i", "--input-directory", parser.add_argument("-i",
dest="input_dir", dest="inputDir",
help="Input directory with input (Multipage-)TIFs or PDFs. For each input file an OCR-Run is \ help="Input directory.",
instantiated and output is created.",
required=True) required=True)
parser.add_argument("-o", "--output-directory", parser.add_argument("-l",
dest="output_dir", choices=["deu", "deu_frak", "eng", "enm", "fra", "spa", "frm"],
help="Directory, where output directories are created if necessary. \
Default: %s" % (os.path.join(os.path.curdir, "ocr_pyflow")),
required=False,
default=(os.path.join(os.path.curdir, "ocr_pyflow")))
parser.add_argument("--skip-pdf-processing",
dest="skip_pdf",
help="Skip detection of PDFs as input.",
default=False,
action='store_true',
required=False)
parser.add_argument("--skip-image-processing",
dest="skip_images",
help="Skip detection of images as input.",
default=False,
action='store_true',
required=False)
parser.add_argument("--start-page",
dest='startp',
help="NOT IMPLEMENTED! First page to ocr.",
default=-1,
required=False, type=int)
parser.add_argument("--end-page",
dest='endp',
default=-1,
help="NOT IMPLEMENTED! Last page to ocr.",
required=False, type=int)
parser.add_argument("-r", "--rotate-pages",
dest='rotate',
default='norotation',
help="NOT IMPLEMENTED! Rotate pages from input. Values: clockwise, counterclockwise, \
upsidedown. Default: norotation",
required=False,
choices=['clockwise', 'counterclockwise', 'upsidedown', 'norotation'])
parser.add_argument("-s", "--split-pages",
dest='split',
default=False,
help="NOT IMPLEMENTED! Split pages in half after possible rotation. Default: Not performed.",
required=False,
action='store_true')
parser.add_argument("--ppi-import",
dest="ppi_in",
help="NOT IMPLEMENTED! Scaling for input images. Default: 300 ppi.",
default=300,
required=False,
type=int)
parser.add_argument("-l", "--language",
dest='lang', dest='lang',
help="Language for OCR", help="Language for OCR",
required=True, required=True)
type=str) parser.add_argument("-o",
parser.add_argument("-p", "--create-pdf", dest="outputDir",
help="Output directory.",
required=True)
parser.add_argument("--skip-image",
action='store_true',
default=False,
dest="skip_images",
help="Skip detection of images as input.",
required=False)
parser.add_argument("--skip-pdf",
action='store_true',
default=False,
dest="skip_pdf",
help="Skip detection of PDFs as input.",
required=False)
parser.add_argument("--pdf",
action='store_true',
default=False,
dest='pdf', dest='pdf',
default=False, help="Create PDF files.",
action='store_true',
required=False) required=False)
parser.add_argument("-c", "--compress-pdf", parser.add_argument("--keep-intermediates",
dest='comp',
help="NOT IMPLEMENTED!",
default=False,
action='store_true', action='store_true',
required=False)
parser.add_argument("--ppi-export",
dest="ppi_out",
help="NOT IMPLEMENTED! Scaling for output images in PDF. Default: 150 ppi.",
default=150,
required=False,
type=int)
parser.add_argument("-k", "--keep-intermediate",
dest="intermediate",
help="Keep intermediate files. Default: False",
default=False, default=False,
action='store_true') dest="keepIntermediates",
parser.add_argument("--cores", help="Keep intermediate files.",
dest='nCores', required=False)
help="Amount of CPUs to use for parallel jobs. Default: Number of available CPUs", parser.add_argument("--nCores",
default=multiprocessing.cpu_count(), default=multiprocessing.cpu_count(),
dest="nCores",
help="Total number of cores available.",
required=False, required=False,
type=int) type=int)
parser.add_argument("--is-continued", dest='continued', return parser.parse_args()
help="NOT IMPLEMENTED! Enables continuing an erroneous or paused workflow. MUST use the \
same dataDirRoot as before.",
default=False,
required=False,
action='store_true')
parser.add_argument("--memory",
dest='mem',
help="NOT IMPLEMENTED! Total amount of memory (RAM) available for this workflow. \
Default: %i" % (8192),
default=8192,
required=False,
type=int)
args = parser.parse_args()
return (args)
class OCRWorkflow(WorkflowRunner): class OCRWorkflow(WorkflowRunner):
def __init__(self, pdfImageJobs, inputDir, outputDir, lang, pdf, intermediate, nCores, memMb): def __init__(self, jobs, lang, pdf, keepIntermediates, nCores):
self.pdfImageJobs = pdfImageJobs self.jobs = jobs
self.outputDir = outputDir
self.inputDir = inputDir
self.lang = lang self.lang = lang
self.pdf = pdf self.pdf = pdf
self.intermediate = intermediate self.keepIntermediates = keepIntermediates
self.nCores = nCores self.nCores = nCores
self.memMb = memMb
@ -160,7 +107,7 @@ class OCRWorkflow(WorkflowRunner):
### ###
mkdir_jobs = [] mkdir_jobs = []
mkdir_job_number = 0 mkdir_job_number = 0
for job in self.pdfImageJobs["images"] + self.pdfImageJobs["pdf"]: for job in self.jobs["images"] + self.jobs["pdf"]:
mkdir_job_number += 1 mkdir_job_number += 1
cmd = "mkdir -p %s %s %s %s" % ( cmd = "mkdir -p %s %s %s %s" % (
os.path.join(job["output_dir"], "hocr_files"), os.path.join(job["output_dir"], "hocr_files"),
@ -176,7 +123,7 @@ class OCRWorkflow(WorkflowRunner):
### ###
split_jobs = [] split_jobs = []
split_job_number = 0 split_job_number = 0
for job in self.pdfImageJobs["images"]: for job in self.jobs["images"]:
split_job_number += 1 split_job_number += 1
# TODO: Make the following command work # TODO: Make the following command work
''' '''
@ -191,13 +138,13 @@ class OCRWorkflow(WorkflowRunner):
os.path.join(job["output_dir"], "tmp", "tiff_files", os.path.basename(job["path"]).rsplit(".", 1)[0] + ".pdf"), os.path.join(job["output_dir"], "tmp", "tiff_files", os.path.basename(job["path"]).rsplit(".", 1)[0] + ".pdf"),
os.path.join(job["output_dir"], "tmp", "tiff_files", os.path.basename(job["path"]).rsplit(".", 1)[0]), os.path.join(job["output_dir"], "tmp", "tiff_files", os.path.basename(job["path"]).rsplit(".", 1)[0]),
os.path.join(job["output_dir"], "tmp", "tiff_files", os.path.basename(job["path"]).rsplit(".", 1)[0] + ".pdf")) os.path.join(job["output_dir"], "tmp", "tiff_files", os.path.basename(job["path"]).rsplit(".", 1)[0] + ".pdf"))
split_jobs.append(self.addTask(label="split_job_-_%i" % (split_job_number), command=cmd, dependencies=mkdir_jobs, nCores=1, memMb=1024)) split_jobs.append(self.addTask(label="split_job_-_%i" % (split_job_number), command=cmd, dependencies=mkdir_jobs))
for job in self.pdfImageJobs["pdf"]: for job in self.jobs["pdf"]:
split_job_number += 1 split_job_number += 1
cmd = "pdftoppm %s %s -tiff -r 300 -tiffcompression lzw -cropbox" % ( cmd = "pdftoppm %s %s -tiff -r 300 -tiffcompression lzw -cropbox" % (
job["path"], job["path"],
os.path.join(job["output_dir"], "tmp", "tiff_files", os.path.basename(job["path"]).rsplit(".", 1)[0])) os.path.join(job["output_dir"], "tmp", "tiff_files", os.path.basename(job["path"]).rsplit(".", 1)[0]))
split_jobs.append(self.addTask(label="split_job_-_%i" % (split_job_number), command=cmd, dependencies=mkdir_jobs, nCores=1, memMb=1024)) split_jobs.append(self.addTask(label="split_job_-_%i" % (split_job_number), command=cmd, dependencies=mkdir_jobs))
### ###
@ -206,13 +153,12 @@ class OCRWorkflow(WorkflowRunner):
### ###
ocropusnlbin_jobs = [] ocropusnlbin_jobs = []
ocropusnlbin_job_number = 0 ocropusnlbin_job_number = 0
for job in self.pdfImageJobs["images"] + self.pdfImageJobs["pdf"]: for job in self.jobs["images"] + self.jobs["pdf"]:
ocropusnlbin_job_number += 1 ocropusnlbin_job_number += 1
cmd = "ocropus-nlbin -Q %i -o %s %s" % ( cmd = "ocropus-nlbin -o %s %s" % (
self.nCores,
os.path.join(job["output_dir"], "tmp", "ocropus-nlbin"), os.path.join(job["output_dir"], "tmp", "ocropus-nlbin"),
os.path.join(job["output_dir"], "tmp", "tiff_files", os.path.basename(job["path"]).rsplit(".", 1)[0] + "-*.tif")) os.path.join(job["output_dir"], "tmp", "tiff_files", os.path.basename(job["path"]).rsplit(".", 1)[0] + "-*.tif"))
ocropusnlbin_jobs.append(self.addTask(label="ocropusnlbin_job_-_%i" % (ocropusnlbin_job_number), command=cmd, dependencies=split_jobs, nCores=self.nCores, memMb=self.memMb)) ocropusnlbin_jobs.append(self.addTask(label="ocropusnlbin_job_-_%i" % (ocropusnlbin_job_number), command=cmd, dependencies=split_jobs))
### ###
@ -222,7 +168,7 @@ class OCRWorkflow(WorkflowRunner):
self.waitForTasks() self.waitForTasks()
tesseract_jobs = [] tesseract_jobs = []
tesseract_job_number = 0 tesseract_job_number = 0
for job in self.pdfImageJobs["images"] + self.pdfImageJobs["pdf"]: for job in self.jobs["images"] + self.jobs["pdf"]:
# This list is empty if you don't wait for ocropus_nlbin_jobs to complete # This list is empty if you don't wait for ocropus_nlbin_jobs to complete
for file in filter(lambda x: x.endswith(".bin.png"), os.listdir(os.path.join(job["output_dir"], "tmp", "ocropus-nlbin"))): for file in filter(lambda x: x.endswith(".bin.png"), os.listdir(os.path.join(job["output_dir"], "tmp", "ocropus-nlbin"))):
tesseract_job_number += 1 tesseract_job_number += 1
@ -231,7 +177,7 @@ class OCRWorkflow(WorkflowRunner):
os.path.join(job["output_dir"], "tmp", "tesseract", file.rsplit(".", 2)[0]), os.path.join(job["output_dir"], "tmp", "tesseract", file.rsplit(".", 2)[0]),
self.lang, self.lang,
"pdf" if self.pdf else "") "pdf" if self.pdf else "")
tesseract_jobs.append(self.addTask(label="tesseract_job_-_%i" % (tesseract_job_number), command=cmd, dependencies=ocropusnlbin_jobs, nCores=1, memMb=2048)) tesseract_jobs.append(self.addTask(label="tesseract_job_-_%i" % (tesseract_job_number), command=cmd, dependencies=ocropusnlbin_jobs, nCores=min(4, self.nCores)))
### ###
@ -241,12 +187,12 @@ class OCRWorkflow(WorkflowRunner):
pdf_merge_jobs = [] pdf_merge_jobs = []
pdf_merge_job_number = 0 pdf_merge_job_number = 0
if self.pdf: if self.pdf:
for job in self.pdfImageJobs["images"] + self.pdfImageJobs["pdf"]: for job in self.jobs["images"] + self.jobs["pdf"]:
pdf_merge_job_number += 1 pdf_merge_job_number += 1
cmd = "pdftk %s cat output %s" % ( cmd = "pdftk %s cat output %s" % (
os.path.join(job["output_dir"], "tmp", "tesseract", "*.pdf"), os.path.join(job["output_dir"], "tmp", "tesseract", "*.pdf"),
os.path.join(job["output_dir"], os.path.basename(job["path"].rsplit(".", 1)[0] + ".pdf"))) os.path.join(job["output_dir"], os.path.basename(job["path"].rsplit(".", 1)[0] + ".pdf")))
pdf_merge_jobs.append(self.addTask(label="pdf_merge_job_-_%i" % (pdf_merge_job_number), command=cmd, dependencies=tesseract_jobs, nCores=1, memMb=4096)) pdf_merge_jobs.append(self.addTask(label="pdf_merge_job_-_%i" % (pdf_merge_job_number), command=cmd, dependencies=tesseract_jobs))
### ###
@ -256,11 +202,11 @@ class OCRWorkflow(WorkflowRunner):
pdf_to_txt_jobs = [] pdf_to_txt_jobs = []
pdf_to_txt_job_number = 0 pdf_to_txt_job_number = 0
if self.pdf: if self.pdf:
for job in self.pdfImageJobs["images"] + self.pdfImageJobs["pdf"]: for job in self.jobs["images"] + self.jobs["pdf"]:
pdf_to_txt_job_number += 1 pdf_to_txt_job_number += 1
cmd = "pdftotext -raw %s" % ( cmd = "pdftotext -raw %s" % (
os.path.join(job["output_dir"], os.path.basename(job["path"].rsplit(".", 1)[0] + ".pdf"))) os.path.join(job["output_dir"], os.path.basename(job["path"].rsplit(".", 1)[0] + ".pdf")))
pdf_merge_jobs.append(self.addTask(label="pdf_to_txt_job_-_%i" % (pdf_to_txt_job_number), command=cmd, dependencies=pdf_merge_jobs, nCores=1, memMb=4096)) pdf_merge_jobs.append(self.addTask(label="pdf_to_txt_job_-_%i" % (pdf_to_txt_job_number), command=cmd, dependencies=pdf_merge_jobs))
### ###
@ -269,7 +215,7 @@ class OCRWorkflow(WorkflowRunner):
### ###
move_hocr_jobs = [] move_hocr_jobs = []
move_hocr_job_number = 0 move_hocr_job_number = 0
for job in self.pdfImageJobs["images"] + self.pdfImageJobs["pdf"]: for job in self.jobs["images"] + self.jobs["pdf"]:
move_hocr_job_number += 1 move_hocr_job_number += 1
cmd = "mv %s %s" % ( cmd = "mv %s %s" % (
os.path.join(job["output_dir"], "tmp", "tesseract", "*.hocr"), os.path.join(job["output_dir"], "tmp", "tesseract", "*.hocr"),
@ -277,18 +223,18 @@ class OCRWorkflow(WorkflowRunner):
move_hocr_jobs.append(self.addTask(label="move_hocr_job_-_%i" % (move_hocr_job_number), command=cmd, dependencies=tesseract_jobs)) move_hocr_jobs.append(self.addTask(label="move_hocr_job_-_%i" % (move_hocr_job_number), command=cmd, dependencies=tesseract_jobs))
### ###Total amount of memory (RAM) available for this workflow. Default: 2048 * nCores"
# Task "hocr_to_teip5_job": create TEI P5 file from hocr files # Task "hocr_to_teip5_job": create TEI P5 file from hocr files
# Dependencies: move_hocr_jobs # Dependencies: move_hocr_jobs
### ###
hocr_to_teip5_jobs = [] hocr_to_teip5_jobs = []
hocr_to_teip5_job_number = 0 hocr_to_teip5_job_number = 0
for job in self.pdfImageJobs["images"] + self.pdfImageJobs["pdf"]: for job in self.jobs["images"] + self.jobs["pdf"]:
hocr_to_teip5_job_number += 1 hocr_to_teip5_job_number += 1
cmd = "parse_hocr %s %s" % ( cmd = "parse_hocr %s %s" % (
os.path.join(job["output_dir"], "hocr_files"), os.path.join(job["output_dir"], "hocr_files"),
os.path.join(os.path.join(job["output_dir"], os.path.basename(job["path"]).rsplit(".", 1)[0] + ".xml"))) os.path.join(os.path.join(job["output_dir"], os.path.basename(job["path"]).rsplit(".", 1)[0] + ".xml")))
hocr_to_teip5_jobs.append(self.addTask(label="hocr_to_teip5_job_-_%i" % (hocr_to_teip5_job_number), command=cmd, dependencies=move_hocr_jobs, nCores=1, memMb=250)) hocr_to_teip5_jobs.append(self.addTask(label="hocr_to_teip5_job_-_%i" % (hocr_to_teip5_job_number), command=cmd, dependencies=move_hocr_jobs))
### ###
@ -298,33 +244,33 @@ class OCRWorkflow(WorkflowRunner):
self.waitForTasks() self.waitForTasks()
cleanup_jobs = [] cleanup_jobs = []
cleanup_job_counter = 0 cleanup_job_counter = 0
if not self.intermediate: if not self.keepIntermediates:
for job in self.pdfImageJobs["images"] + self.pdfImageJobs["pdf"]: for job in self.jobs["images"] + self.jobs["pdf"]:
cleanup_job_counter += 1 cleanup_job_counter += 1
cmd = "rm -r %s" % (os.path.join(job["output_dir"], "tmp")) cmd = "rm -r %s" % (os.path.join(job["output_dir"], "tmp"))
cleanup_jobs.append(self.addTask(label="cleanup_job_-_%i" % (cleanup_job_counter), command=cmd)) cleanup_jobs.append(self.addTask(label="cleanup_job_-_%i" % (cleanup_job_counter), command=cmd))
def analyze_jobs(input_dir, output_dir, skip_pdf=False, skip_images=False): def analyze_jobs(inputDir, outputDir, skip_pdf=False, skip_images=False):
files = os.listdir(input_dir) files = os.listdir(inputDir)
images = [] images = []
pdf = [] pdf = []
if not skip_images: if not skip_images:
for file in filter(lambda x: x.endswith(".tif") or x.endswith(".tiff"), files): for file in filter(lambda x: x.endswith(".tif") or x.endswith(".tiff"), files):
images.append({"path": os.path.join(input_dir, file), "output_dir": os.path.join(output_dir, file.rsplit(".", 1)[0])}) images.append({"path": os.path.join(inputDir, file), "output_dir": os.path.join(outputDir, file.rsplit(".", 1)[0])})
if not skip_pdf: if not skip_pdf:
for file in filter(lambda x: x.endswith(".pdf"), files): for file in filter(lambda x: x.endswith(".pdf"), files):
pdf.append({"path": os.path.join(input_dir, file), "output_dir": os.path.join(output_dir, file.rsplit(".", 1)[0])}) pdf.append({"path": os.path.join(inputDir, file), "output_dir": os.path.join(outputDir, file.rsplit(".", 1)[0])})
for subdir in filter(lambda x: os.path.isdir(os.path.join(input_dir, x)), files): for subDir in filter(lambda x: os.path.isdir(os.path.join(inputDir, x)), files):
subdir_files = os.listdir(os.path.join(input_dir, subdir)) subDirFiles = os.listdir(os.path.join(inputDir, subDir))
if not skip_images: if not skip_images:
for file in filter(lambda x: x.endswith(".tif") or x.endswith(".tiff"), subdir_files): for file in filter(lambda x: x.endswith(".tif") or x.endswith(".tiff"), subDirFiles):
images.append({"path": os.path.join(input_dir, subdir, file), "output_dir": os.path.join(output_dir, subdir, file.rsplit(".", 1)[0])}) images.append({"path": os.path.join(inputDir, subDir, file), "output_dir": os.path.join(outputDir, subDir, file.rsplit(".", 1)[0])})
if not skip_pdf: if not skip_pdf:
for file in filter(lambda x: x.endswith(".pdf"), subdir_files): for file in filter(lambda x: x.endswith(".pdf"), subDirFiles):
pdf.append({"path": os.path.join(input_dir, subdir, file), "output_dir": os.path.join(output_dir, subdir, file.rsplit(".", 1)[0])}) pdf.append({"path": os.path.join(inputDir, subDir, file), "output_dir": os.path.join(outputDir, subDir, file.rsplit(".", 1)[0])})
return {"pdf": pdf, "images": images} return {"pdf": pdf, "images": images}
@ -345,10 +291,11 @@ def normalize_input_filenames(path):
def main(): def main():
args = parse_arguments() args = parse_arguments()
normalize_input_filenames(args.input_dir) normalize_input_filenames(args.inputDir)
jobs = analyze_jobs(args.input_dir, args.output_dir, skip_pdf=args.skip_pdf, skip_images=args.skip_images) jobs = analyze_jobs(args.inputDir, args.outputDir, skip_pdf=args.skip_pdf, skip_images=args.skip_images)
wflow = OCRWorkflow(jobs, args.input_dir, args.output_dir, args.lang, args.pdf, args.intermediate, args.nCores, args.mem)
retval = wflow.run(mode="local", nCores=args.nCores, memMb=args.mem) wflow = OCRWorkflow(jobs, args.lang, args.pdf, args.keepIntermediates, args.nCores)
retval = wflow.run(nCores=args.nCores)
sys.exit(retval) sys.exit(retval)