Compare commits

10 Commits

6 changed files with 266 additions and 420 deletions

View File

@ -1,8 +1,5 @@
image: docker:19.03.13
variables:
DOCKER_TLS_CERTDIR: "/certs"
services:
- docker:19.03.13-dind
@ -10,6 +7,10 @@ stages:
- build
- push
variables:
DOCKER_TLS_CERTDIR: "/certs"
INTERMEDIATE_IMAGE_TAG: $CI_REGISTRY_IMAGE:$CI_COMMIT_REF_NAME-$CI_COMMIT_SHA
.reg_setup:
before_script:
- apk add --no-cache curl
@ -28,8 +29,6 @@ build_image:
stage: build
tags:
- docker
variables:
INTERMEDIATE_IMAGE_TAG: $CI_REGISTRY_IMAGE:$CI_COMMIT_SHA
push_master:
extends:
@ -47,7 +46,6 @@ push_master:
- docker
variables:
IMAGE_TAG: $CI_REGISTRY_IMAGE:latest
INTERMEDIATE_IMAGE_TAG: $CI_REGISTRY_IMAGE:$CI_COMMIT_SHA
push_other:
extends:
@ -68,4 +66,3 @@ push_other:
- docker
variables:
IMAGE_TAG: $CI_REGISTRY_IMAGE:$CI_COMMIT_REF_NAME
INTERMEDIATE_IMAGE_TAG: $CI_REGISTRY_IMAGE:$CI_COMMIT_SHA

View File

@ -1,47 +1,53 @@
FROM debian:buster-slim
LABEL authors="Patrick Jentsch <p.jentsch@uni-bielefeld.de>"
LABEL authors="Patrick Jentsch <p.jentsch@uni-bielefeld.de>, Stephan Porada <porada@posteo.de>"
ENV LANG=C.UTF-8
RUN apt-get update
RUN apt-get update \
&& apt-get install --no-install-recommends --yes \
wget
# Install pipeline dependencies #
# Install the OCR pipeline and it's dependencies #
## Install pyFlow ##
ENV PYFLOW_RELEASE=1.1.20
ADD "https://github.com/Illumina/pyflow/releases/download/v${PYFLOW_RELEASE}/pyflow-${PYFLOW_RELEASE}.tar.gz" .
RUN tar -xzf "pyflow-${PYFLOW_RELEASE}.tar.gz" \
&& cd "pyflow-${PYFLOW_RELEASE}" \
ENV PYFLOW_VERSION=1.1.20
RUN wget --no-check-certificate --quiet \
"https://github.com/Illumina/pyflow/releases/download/v${PYFLOW_VERSION}/pyflow-${PYFLOW_VERSION}.tar.gz" \
&& tar -xzf "pyflow-${PYFLOW_VERSION}.tar.gz" \
&& cd "pyflow-${PYFLOW_VERSION}" \
&& apt-get install --no-install-recommends --yes \
python2.7 \
&& python2.7 setup.py build install \
&& cd .. \
&& rm -r "pyflow-${PYFLOW_RELEASE}" "pyflow-${PYFLOW_RELEASE}.tar.gz"
&& cd - > /dev/null \
&& rm -r "pyflow-${PYFLOW_VERSION}" "pyflow-${PYFLOW_VERSION}.tar.gz"
## Install ocropy ##
ENV OCROPY_RELEASE=1.3.3
ADD "https://github.com/tmbdev/ocropy/archive/v${OCROPY_RELEASE}.tar.gz" .
RUN tar -xzf "v${OCROPY_RELEASE}.tar.gz" \
&& cd "ocropy-${OCROPY_RELEASE}" \
ENV OCROPY_VERSION=1.3.3
RUN wget --no-check-certificate --quiet \
"https://github.com/tmbdev/ocropy/archive/v${OCROPY_VERSION}.tar.gz" \
&& tar -xzf "v${OCROPY_VERSION}.tar.gz" \
&& cd "ocropy-${OCROPY_VERSION}" \
&& apt-get install --no-install-recommends --yes \
python2.7 \
python-pil \
python-tk \
$(cat PACKAGES) \
&& python2.7 setup.py install \
&& cd .. \
&& rm -r "ocropy-${OCROPY_RELEASE}" "v${OCROPY_RELEASE}.tar.gz"
&& cd - > /dev/null \
&& rm -r "ocropy-${OCROPY_VERSION}" "v${OCROPY_VERSION}.tar.gz"
## Install Tesseract OCR ##
ENV TESSERACT_RELEASE=4.1.1
ADD "https://github.com/tesseract-ocr/tesseract/archive/${TESSERACT_RELEASE}.tar.gz" .
RUN tar -xzf "${TESSERACT_RELEASE}.tar.gz" \
&& cd "tesseract-${TESSERACT_RELEASE}" \
ENV TESSERACT_VERSION=4.1.1
RUN wget --no-check-certificate --quiet \
"https://github.com/tesseract-ocr/tesseract/archive/${TESSERACT_VERSION}.tar.gz" \
&& tar -xzf "${TESSERACT_VERSION}.tar.gz" \
&& cd "tesseract-${TESSERACT_VERSION}" \
&& apt-get install --no-install-recommends --yes \
autoconf \
automake \
@ -60,43 +66,32 @@ RUN tar -xzf "${TESSERACT_RELEASE}.tar.gz" \
&& make install \
&& ldconfig \
&& cd - > /dev/null \
&& rm -r "tesseract-${TESSERACT_RELEASE}" "${TESSERACT_RELEASE}.tar.gz"
&& rm -r "tesseract-${TESSERACT_VERSION}" "${TESSERACT_VERSION}.tar.gz"
ENV TESSDATA_BEST_RELEASE=4.1.0
ADD "https://github.com/tesseract-ocr/tessdata_best/archive/${TESSDATA_BEST_RELEASE}.tar.gz" .
RUN tar -xzf "${TESSDATA_BEST_RELEASE}.tar.gz" \
&& mv "tessdata_best-${TESSDATA_BEST_RELEASE}/ara.traineddata" "/usr/local/share/tessdata/" \
&& mv "tessdata_best-${TESSDATA_BEST_RELEASE}/chi_tra.traineddata" "/usr/local/share/tessdata/" \
&& mv "tessdata_best-${TESSDATA_BEST_RELEASE}/dan.traineddata" "/usr/local/share/tessdata/" \
&& mv "tessdata_best-${TESSDATA_BEST_RELEASE}/deu.traineddata" "/usr/local/share/tessdata/" \
&& mv "tessdata_best-${TESSDATA_BEST_RELEASE}/ell.traineddata" "/usr/local/share/tessdata/" \
&& mv "tessdata_best-${TESSDATA_BEST_RELEASE}/eng.traineddata" "/usr/local/share/tessdata/" \
&& mv "tessdata_best-${TESSDATA_BEST_RELEASE}/enm.traineddata" "/usr/local/share/tessdata/" \
&& mv "tessdata_best-${TESSDATA_BEST_RELEASE}/fra.traineddata" "/usr/local/share/tessdata/" \
&& mv "tessdata_best-${TESSDATA_BEST_RELEASE}/frk.traineddata" "/usr/local/share/tessdata/" \
&& mv "tessdata_best-${TESSDATA_BEST_RELEASE}/frm.traineddata" "/usr/local/share/tessdata/" \
&& mv "tessdata_best-${TESSDATA_BEST_RELEASE}/ita.traineddata" "/usr/local/share/tessdata/" \
&& mv "tessdata_best-${TESSDATA_BEST_RELEASE}/por.traineddata" "/usr/local/share/tessdata/" \
&& mv "tessdata_best-${TESSDATA_BEST_RELEASE}/rus.traineddata" "/usr/local/share/tessdata/" \
&& mv "tessdata_best-${TESSDATA_BEST_RELEASE}/spa.traineddata" "/usr/local/share/tessdata/" \
&& rm -r "tessdata_best-${TESSDATA_BEST_RELEASE}" "${TESSDATA_BEST_RELEASE}.tar.gz"
ENV TESSERACT_MODELS="ara,chi_tra,dan,deu,ell,eng,enm,fra,frk,frm,ita,por,rus,spa"
ENV TESSDATA_BEST_VERSION=4.1.0
RUN wget --no-check-certificate --quiet \
"https://github.com/tesseract-ocr/tessdata_best/archive/${TESSDATA_BEST_VERSION}.tar.gz" \
&& tar -xzf "${TESSDATA_BEST_VERSION}.tar.gz" \
&& for tesseract_model in $(echo ${TESSERACT_MODELS} | tr "," "\n"); do mv "tessdata_best-${TESSDATA_BEST_VERSION}/${tesseract_model}.traineddata" "/usr/local/share/tessdata/"; done \
&& rm -r "tessdata_best-${TESSDATA_BEST_VERSION}" "${TESSDATA_BEST_VERSION}.tar.gz"
## Further dependencies ##
RUN apt-get install --no-install-recommends --yes \
procps \
ghostscript \
python-pip \
python3.7 \
zip \
&& pip install natsort
RUN rm -r /var/lib/apt/lists/*
rename \
zip
## Install Pipeline ##
COPY hocrtotei ocr /usr/local/bin/
RUN rm -r /var/lib/apt/lists/*
ENTRYPOINT ["ocr"]
CMD ["--help"]

View File

@ -3,13 +3,14 @@
This software implements a heavily parallelized pipeline to recognize text in PDF files. It is used for nopaque's OCR service but you can also use it standalone, for that purpose a convenient wrapper script is provided.
## Software used in this pipeline implementation
- Official Debian Docker image (buster-slim) and programs from its free repositories: https://hub.docker.com/_/debian
- Official Debian Docker image (buster-slim): https://hub.docker.com/_/debian
- Software from Debian Buster's free repositories
- ocropy (1.3.3): https://github.com/ocropus/ocropy/releases/tag/v1.3.3
- pyFlow (1.1.20): https://github.com/Illumina/pyflow/releases/tag/v1.1.20
- Tesseract OCR (4.1.1): https://github.com/tesseract-ocr/tesseract/releases/tag/4.1.1
- tessdata_best (4.1.0): https://github.com/tesseract-ocr/tessdata_best/releases/tag/4.1.0
## Use this image
1. Create input and output directories for the pipeline.
@ -19,10 +20,10 @@ mkdir -p /<my_data_location>/input /<my_data_location>/output
2. Place your PDF files inside `/<my_data_location>/input`. Files should all contain text of the same language.
3. Start the pipeline process. Check the [Pipeline arguments](#pipeline-arguments) section for more details.
3. Start the pipeline process. Check the pipeline help (`ocr --help`) for more details.
```
# Option one: Use the wrapper script
## Install the wrapper script (only on first run). Get it from https://gitlab.ub.uni-bielefeld.de/sfb1288inf/ocr/-/raw/1.0.0/wrapper/ocr, make it executeable and add it to your ${PATH}
## Install the wrapper script (only on first run). Get it from https://gitlab.ub.uni-bielefeld.de/sfb1288inf/ocr/-/raw/development/wrapper/ocr, make it executeable and add it to your ${PATH}
cd /<my_data_location>
ocr -i input -l <language_code> -o output <optional_pipeline_arguments>
@ -33,51 +34,11 @@ docker run \
-u $(id -u $USER):$(id -g $USER) \
-v /<my_data_location>/input:/input \
-v /<my_data_location>/output:/output \
gitlab.ub.uni-bielefeld.de:4567/sfb1288inf/ocr:1.0.0 \
-i /input \
-l <language_code>
-o /output \
gitlab.ub.uni-bielefeld.de:4567/sfb1288inf/ocr:development \
-i /ocr_pipeline/input \
-l <language_code> \
-o /ocr_pipeline/output \
<optional_pipeline_arguments>
```
4. Check your results in the `/<my_data_location>/output` directory.
```
### Pipeline arguments
`-l languagecode`
* Tells tesseract which language will be used.
* options = ara (Arabic), chi_tra (Chinese - Traditional), dan (Danish), deu (German), ell (Greek, Modern (1453-)), eng (English), enm (Middle englisch), fra (French), frk (German Fraktur), frm (Middle french), ita (Italian), por (Portuguese), rus (Russian), spa (Spanish)
* required = True
`--keep-intermediates`
* If set, all intermediate files created during the OCR process will be
kept.
* default = False
* required = False
`--nCores corenumber`
* Sets the number of CPU cores being used during the OCR process.
* default = min(4, multiprocessing.cpu_count())
* required = False
`--skip-binarisation`
* Used to skip binarization with ocropus. If skipped, only the tesseract binarization is used.
* default = False
``` bash
# Example with all arguments used
docker run \
--rm \
-it \
-u $(id -u $USER):$(id -g $USER) \
-v "$HOME"/ocr/input:/input \
-v "$HOME"/ocr/output:/output \
gitlab.ub.uni-bielefeld.de:4567/sfb1288inf/ocr:1.0.0 \
-i /input \
-l eng \
-o /output \
--keep_intermediates \
--nCores 8 \
--skip-binarisation
```

View File

@ -1,49 +1,57 @@
#!/usr/bin/env python3.7
# coding=utf-8
""""Merges hOCR files into a TEI file."""
""""Convert hOCR to TEI XML."""
from xml.sax.saxutils import escape
from argparse import ArgumentParser
import re
import xml.etree.ElementTree as ET
parser = ArgumentParser(description='Merges hOCR files into a TEI file.')
parser.add_argument('i', metavar='hOCR-sourcefile', nargs='+')
parser.add_argument('o', metavar='TEI-destfile',)
parser = ArgumentParser(description='Convert hOCR to TEI XML.')
parser.add_argument('input', metavar='Path to hOCR input file')
parser.add_argument('output', metavar='Path to TEI output file')
args = parser.parse_args()
output_file = open(args.o, 'w')
output_file.write(
'<?xml version="1.0" encoding="UTF-8"?>\n'
+ '<TEI xmlns="http://www.tei-c.org/ns/1.0" xml:id="dtabf">\n'
+ ' <teiHeader>\n'
+ ' <fileDesc>\n'
+ ' <titleStmt/>\n'
+ ' <publicationStmt/>\n'
+ ' <sourceDesc/>\n'
+ ' </fileDesc>\n'
+ ' <encodingDesc/>\n'
+ ' <profileDesc/>\n'
+ ' </teiHeader>\n'
+ ' <text>\n'
+ ' <body>\n'
)
for index, input_file in enumerate(args.i):
tree = ET.parse(input_file)
output_file.write(' <pb n="%i"/>\n' % (index + 1))
for para in tree.findall('.//*[@class="ocr_par"]'):
output_file.write(' <p>\n')
tei = ''
tei += '<TEI xmlns="http://www.tei-c.org/ns/1.0">\n'
tei += ' <teiHeader>\n'
tei += ' <fileDesc>\n'
tei += ' <titleStmt>\n'
tei += ' <title></title>\n'
tei += ' </titleStmt>\n'
tei += ' <publicationStmt>\n'
tei += ' <p></p>\n'
tei += ' </publicationStmt>\n'
tei += ' <sourceDesc>\n'
tei += ' <p></p>\n'
tei += ' </sourceDesc>\n'
tei += ' </fileDesc>\n'
tei += ' </teiHeader>\n'
tei += ' <text>\n'
tei += ' <body>\n'
# Conversion start
hocr = ET.parse(args.input)
for page in hocr.findall('.//*[@class="ocr_page"]'):
page_properties = page.attrib.get('title')
facsimile = re.search(r'image \"(.*?)\"', page_properties).group(1)
page_number = re.search(r'ppageno (\d+)', page_properties).group(1)
tei += ' <pb facs="{}" n="{}"/>\n'.format(facsimile, page_number)
for para in page.findall('.//*[@class="ocr_par"]'):
tei += ' <p>\n'
for line in para.findall('.//*[@class="ocr_line"]'):
first_word_in_line = True
tei += ' <lb/>'
indent = ''
for word in line.findall('.//*[@class="ocrx_word"]'):
if word.text is not None:
output_file.write((' ' if first_word_in_line else ' ') + escape(word.text.strip()))
first_word_in_line = False
if not first_word_in_line:
output_file.write('<lb/>\n')
output_file.write(' </p>\n')
output_file.write(
' </body>\n'
+ ' </text>\n'
+ '</TEI>')
output_file.close()
tei += indent + escape(word.text.strip())
indent = ' '
tei += '\n'
tei += ' </p>\n'
# Conversion end
tei += ' </body>\n'
tei += ' </text>\n'
tei += '</TEI>\n'
with open(args.output, 'w') as tei_file:
tei_file.write(tei)

412
ocr
View File

@ -1,54 +1,17 @@
#!/usr/bin/env python2.7
# coding=utf-8
"""An OCR pipeline for PDF file processing."""
"""OCR pipeline for PDF file processing."""
__author__ = 'Patrick Jentsch <p.jentsch@uni-bielefeld.de>'
__author__ = 'Patrick Jentsch <p.jentsch@uni-bielefeld.de>,' \
'Stephan Porada <porada@posteo.de>'
__version__ = '1.0.0'
from argparse import ArgumentParser
from natsort import natsorted
from pyflow import WorkflowRunner
import multiprocessing
import os
import sys
import tempfile
TESSERACT_MODELS = ['deu', 'eng', 'enm', 'fra', 'frk', 'frm', 'ita', 'por', 'spa'] # noqa
def parse_args():
parser = ArgumentParser(
description='An OCR pipeline for PDF file processing.',
prog='OCR pipeline'
)
parser.add_argument('-i', '--input-directory',
help='Input directory (only PDF files get processed)',
required=True)
parser.add_argument('-o', '--output-directory',
help='Output directory',
required=True)
parser.add_argument('-l', '--language',
choices=TESSERACT_MODELS,
required=True)
parser.add_argument('--binarize',
action='store_true',
help='Use ocropy binarisation as preprocessing step.')
parser.add_argument('--log-dir')
parser.add_argument('--n-cores',
default=min(4, multiprocessing.cpu_count()),
help='Total number of cores available.',
type=int)
parser.add_argument('--intermediate-directory')
parser.add_argument('--zip',
help='Zips all results in different archives depending'
' on result types. Also zips everything into one '
'archive.')
parser.add_argument('-v', '--version',
action='version',
version='%(prog)s {}'.format(__version__))
return parser.parse_args()
class OCRPipelineJob:
@ -60,41 +23,23 @@ class OCRPipelineJob:
Arguments:
file -- Path to the file
output_dir -- Path to a directory, where job results a stored
intermediate_dir -- Path to a directory, where intermediate files are
stored.
"""
def __init__(self, file, output_dir, intermediate_dir):
def __init__(self, file, output_dir):
self.file = file
self.intermediate_dir = intermediate_dir
self.name = os.path.basename(file).rsplit('.', 1)[0]
self.output_dir = output_dir
self.page_dir = os.path.join(output_dir, 'pages')
class OCRPipeline(WorkflowRunner):
def __init__(self, input_dir, lang, output_dir, binarize, intermediate_dir,
n_cores, zip):
def __init__(self, input_dir, lang, output_dir, binarize, zip):
self.input_dir = input_dir
self.lang = lang
self.output_dir = output_dir
self.binarize = binarize
if intermediate_dir is None:
self.intermediate_dir = os.path.join(output_dir, 'tmp')
else:
self.intermediate_dir = tempfile.mkdtemp(dir=intermediate_dir)
self.n_cores = n_cores
if zip is None:
self.zip = zip
else:
if zip.lower().endswith('.zip'):
# Remove .zip file extension if provided
self.zip = zip[:-4]
self.zip = self.zip if self.zip else 'output'
else:
self.zip = zip
self.jobs = collect_jobs(self.input_dir,
self.output_dir,
self.intermediate_dir)
self.zip = zip
self.jobs = collect_jobs(self.input_dir, self.output_dir)
def workflow(self):
if not self.jobs:
@ -107,10 +52,7 @@ class OCRPipeline(WorkflowRunner):
'''
setup_output_directory_tasks = []
for i, job in enumerate(self.jobs):
cmd = 'mkdir'
cmd += ' -p'
cmd += ' "{}"'.format(job.intermediate_dir)
cmd += ' "{}"'.format(os.path.join(job.output_dir, 'poco'))
cmd = 'mkdir -p "{}"'.format(job.page_dir)
lbl = 'setup_output_directory_-_{}'.format(i)
task = self.addTask(command=cmd, label=lbl)
setup_output_directory_tasks.append(task)
@ -121,10 +63,10 @@ class OCRPipeline(WorkflowRunner):
' ##################################################
'''
split_input_tasks = []
n_cores = min(self.n_cores, max(1, int(self.n_cores / len(self.jobs))))
n_cores = max(1, int(self.getNCores() / len(self.jobs)))
for i, job in enumerate(self.jobs):
input_file = job.file
output_file = '{}/page-%d.tif'.format(job.intermediate_dir)
output_file = '{}/page-%d.tif'.format(job.page_dir)
cmd = 'gs'
cmd += ' -dBATCH'
cmd += ' -dNOPAUSE'
@ -137,15 +79,24 @@ class OCRPipeline(WorkflowRunner):
cmd += ' "{}"'.format(input_file)
deps = 'setup_output_directory_-_{}'.format(i)
lbl = 'split_input_-_{}'.format(i)
task = self.addTask(command=cmd, dependencies=deps, label=lbl, nCores=n_cores) # noqa
task = self.addTask(command=cmd, dependencies=deps, label=lbl,
nCores=n_cores)
split_input_tasks.append(task)
if self.binarize:
'''
' The binarization_tasks list is created based on the output files
' of the split_tasks. So wait until they are finished.
' ##################################################
' # pre binarization #
' ##################################################
'''
self.waitForTasks()
pre_binarization_tasks = []
for i, job in enumerate(self.jobs):
input_file = os.path.join(job.output_dir, 'binarization_input_files.txt') # noqa
cmd = 'ls -dv "{}/"* >> "{}"'.format(job.page_dir, input_file)
deps = 'split_input_-_{}'.format(i)
lbl = 'pre_binarization_-_{}'.format(i)
task = self.addTask(command=cmd, dependencies=deps, label=lbl)
pre_binarization_tasks.append(task)
'''
' ##################################################
@ -153,52 +104,55 @@ class OCRPipeline(WorkflowRunner):
' ##################################################
'''
binarization_tasks = []
'''
' We run ocropus-nlbin with either four or, if there are less then
' four cores available for this workflow, the available core
' number.
'''
n_cores = min(4, self.n_cores)
n_cores = self.getNCores()
mem_mb = self.getMemMb()
for i, job in enumerate(self.jobs):
input_dir = job.intermediate_dir
output_dir = job.intermediate_dir
files = filter(lambda x: x.endswith('.tif'), os.listdir(input_dir)) # noqa
files = natsorted(files)
files = map(lambda x: os.path.join(input_dir, x), files)
cmd = 'ocropus-nlbin "{}"'.format('" "'.join(files))
input_file = os.path.join(job.output_dir, 'binarization_input_files.txt') # noqa
cmd = 'ocropus-nlbin "@{}"'.format(input_file)
cmd += ' --nocheck'
cmd += ' --output "{}"'.format(output_dir)
cmd += ' --output "{}"'.format(job.page_dir)
cmd += ' --parallel "{}"'.format(n_cores)
print(cmd)
deps = 'split_input_-_{}'.format(i)
deps = 'pre_binarization_-_{}'.format(i)
lbl = 'binarization_-_{}'.format(i)
task = self.addTask(command=cmd, dependencies=deps, label=lbl, nCores=n_cores) # noqa
task = self.addTask(command=cmd, dependencies=deps, label=lbl,
memMb=mem_mb, nCores=n_cores)
binarization_tasks.append(task)
self.waitForTasks()
'''
' ##################################################
' # Renaming of binarization output files #
' # post binarization #
' ##################################################
'''
post_binarization_tasks = []
for i, job in enumerate(self.jobs):
input_dir = job.intermediate_dir
output_dir = job.intermediate_dir
files = filter(lambda x: x.endswith('.bin.png'), os.listdir(input_dir)) # noqa
for file in files:
# int conversion is done in order to trim leading zeros
page_number = int(file.split('.', 1)[0])
output_file = 'page-{}.bin.png'.format(page_number)
os.rename(os.path.join(output_dir, file),
os.path.join(output_dir, output_file))
input_file = os.path.join(job.output_dir, 'binarization_input_files.txt') # noqa
cmd = 'rm "{}"'.format(input_file)
cmd += ' && '
cmd += 'cd "{}"'.format(job.page_dir)
cmd += ' && '
cmd += 'rm *.{nrm.png,tif}'
cmd += ' && '
cmd += 'rename \'s/^0*/page-/\' *'
cmd += ' && '
cmd += 'cd -'
deps = 'binarization_-_{}'.format(i)
lbl = 'post_binarization_-_{}'.format(i)
task = self.addTask(command=cmd, dependencies=deps, label=lbl)
post_binarization_tasks.append(task)
'''
' The ocr_tasks are created based of the output files of either the
' split_tasks or binarization_tasks. So wait until they are
' finished.
' ##################################################
' # pre ocr #
' ##################################################
'''
self.waitForTasks()
pre_ocr_tasks = []
for i, job in enumerate(self.jobs):
input_file = os.path.join(job.output_dir, 'ocr_input_files.txt')
cmd = 'ls -dv "{}/"* >> "{}"'.format(job.page_dir, input_file)
deps = 'post_binarization_-_{}'.format(i) if self.binarize else 'split_input_-_{}'.format(i) # noqa
lbl = 'pre_ocr_-_{}'.format(i)
task = self.addTask(command=cmd, dependencies=deps, label=lbl)
pre_ocr_tasks.append(task)
'''
' ##################################################
@ -206,157 +160,51 @@ class OCRPipeline(WorkflowRunner):
' ##################################################
'''
ocr_tasks = []
n_cores = min(4, self.getNCores())
mem_mb = min(n_cores * 2048, self.getMemMb())
for i, job in enumerate(self.jobs):
input_dir = job.intermediate_dir
output_dir = job.intermediate_dir
files = os.listdir(input_dir)
if self.binarize:
deps = 'binarization_-_{}'.format(i)
files = filter(lambda x: x.endswith('.bin.png'), files)
else:
deps = 'split_input_-_{}'.format(i)
files = filter(lambda x: x.endswith('.tif'), files)
files = natsorted(files)
files = map(lambda x: os.path.join(input_dir, x), files)
for j, file in enumerate(files):
if self.binarize:
output_file_base = os.path.join(output_dir, file.rsplit('.', 2)[0]) # noqa
else:
output_file_base = os.path.join(output_dir, file.rsplit('.', 1)[0]) # noqa
cmd = 'tesseract "{}" "{}"'.format(file, output_file_base)
cmd += ' -l "{}"'.format(self.lang)
cmd += ' hocr pdf txt'
cmd += ' && '
cmd += 'sed -i \'s+{}/++g\' "{}".hocr'.format(input_dir, output_file_base) # noqa
lbl = 'ocr_-_{}-{}'.format(i, j)
task = self.addTask(command=cmd, dependencies=deps, label=lbl, env={"OMP_THREAD_LIMIT": "1"}) # noqa
ocr_tasks.append(task)
'''
' The following jobs are created based of the output files of the
' ocr_tasks. So wait until they are finished.
'''
self.waitForTasks()
input_file = os.path.join(job.output_dir, 'ocr_input_files.txt')
output_file_base = os.path.join(job.output_dir, job.name)
cmd = 'tesseract "{}" "{}"'.format(input_file, output_file_base)
cmd += ' -l "{}"'.format(self.lang)
cmd += ' hocr pdf txt'
deps = 'pre_ocr_-_{}'.format(i)
lbl = 'ocr_-_{}'.format(i)
task = self.addTask(command=cmd, dependencies=deps,
env={'OMP_THREAD_LIMIT': '{}'.format(n_cores)},
label=lbl, memMb=mem_mb, nCores=n_cores)
ocr_tasks.append(task)
'''
' ##################################################
' # combined pdf creation #
' # post ocr #
' ##################################################
'''
combined_pdf_creation_tasks = []
n_cores = min(self.n_cores, max(1, int(self.n_cores / len(self.jobs))))
post_ocr_tasks = []
for i, job in enumerate(self.jobs):
input_dir = job.intermediate_dir
output_file = os.path.join(job.output_dir, '{}.pdf'.format(job.name)) # noqa
files = filter(lambda x: x.endswith('.pdf'), os.listdir(input_dir))
files = natsorted(files)
files = map(lambda x: os.path.join(input_dir, x), files)
cmd = 'gs'
cmd += ' -dBATCH'
cmd += ' -dNOPAUSE'
cmd += ' -dNumRenderingThreads={}'.format(n_cores)
cmd += ' -dPDFSETTINGS=/ebook'
cmd += ' -dQUIET'
cmd += ' -sDEVICE=pdfwrite'
cmd += ' "-sOutputFile={}"'.format(output_file)
cmd += ' "{}"'.format('" "'.join(files))
deps = filter(lambda x: x.startswith('ocr_-_{}'.format(i)), ocr_tasks) # noqa
lbl = 'combined_pdf_creation_-_{}'.format(i)
task = self.addTask(command=cmd, dependencies=deps, label=lbl, nCores=n_cores) # noqa
combined_pdf_creation_tasks.append(task)
'''
' ##################################################
' # combined txt creation #
' ##################################################
'''
combined_txt_creation_tasks = []
for i, job in enumerate(self.jobs):
input_dir = job.intermediate_dir
output_file = os.path.join(job.output_dir, '{}.txt'.format(job.name)) # noqa
files = filter(lambda x: x.endswith('.txt'), os.listdir(input_dir))
files = natsorted(files)
files = map(lambda x: os.path.join(input_dir, x), files)
cmd = 'cat "{}" > "{}"'.format('" "'.join(files), output_file)
deps = filter(lambda x: x.startswith('ocr_-_{}'.format(i)), ocr_tasks) # noqa
lbl = 'combined_txt_creation_-_{}'.format(i)
input_file = os.path.join(job.output_dir, 'ocr_input_files.txt')
output_file_base = os.path.join(job.output_dir, job.name)
cmd = 'rm "{}"'.format(input_file)
cmd += ' && '
cmd += 'sed -i \'s+{}+pages+g\' "{}.hocr"'.format(job.page_dir, output_file_base) # noqa
deps = 'ocr_-_{}'.format(i)
lbl = 'post_ocr_-_{}'.format(i)
task = self.addTask(command=cmd, dependencies=deps, label=lbl)
combined_txt_creation_tasks.append(task)
post_ocr_tasks.append(task)
'''
' ##################################################
' # tei p5 creation #
' # hocr to tei #
' ##################################################
'''
tei_p5_creation_tasks = []
hocr_to_tei_tasks = []
for i, job in enumerate(self.jobs):
input_dir = job.intermediate_dir
output_file = os.path.join(job.output_dir, '{}.xml'.format(job.name)) # noqa
files = filter(lambda x: x.endswith('.hocr'),
os.listdir(input_dir))
files = natsorted(files)
files = map(lambda x: os.path.join(input_dir, x), files)
cmd = 'hocrtotei "{}" "{}"'.format('" "'.join(files),
output_file)
deps = filter(lambda x: x.startswith('ocr_-_{}'.format(i)), ocr_tasks) # noqa
lbl = 'tei_p5_creation_-_{}'.format(i)
output_file_base = os.path.join(job.output_dir, job.name)
cmd = 'hocrtotei "{}.hocr" "{}.xml"'.format(output_file_base, output_file_base) # noqa
deps = 'post_ocr_-_{}'.format(i)
lbl = 'hocr_to_tei_-_{}'.format(i)
task = self.addTask(command=cmd, dependencies=deps, label=lbl)
tei_p5_creation_tasks.append(task)
'''
' ##################################################
' # poco bundle creation #
' ##################################################
'''
poco_bundle_creation_tasks = []
for i, job in enumerate(self.jobs):
input_dir = job.intermediate_dir
output_dir = os.path.join(job.output_dir, 'poco')
files = os.listdir(input_dir)
if self.binarize:
files = filter(lambda x: x.endswith(('.bin.png', '.hocr')), files) # noqa
else:
files = filter(lambda x: x.endswith(('.tif', '.hocr')), files)
files = natsorted(files)
files = map(lambda x: os.path.join(input_dir, x), files)
cmd = 'mv "{}" "{}"'.format('" "'.join(files), output_dir)
deps = filter(lambda x: x.startswith('ocr_-_{}'.format(i)), ocr_tasks) # noqa
deps.append('tei_p5_creation_-_{}'.format(i))
lbl = 'poco_bundle_creation_-_{}'.format(i)
task = self.addTask(command=cmd, dependencies=deps, label=lbl)
poco_bundle_creation_tasks.append(task)
'''
' The following jobs are created based of the output files of the
' combined_pdf_creation_tasks. So wait until they are finished.
'''
self.waitForTasks()
'''
' ##################################################
' # cleanup #
' ##################################################
'''
cleanup_tasks = []
for i, job in enumerate(self.jobs):
input_dir = job.intermediate_dir
cmd = 'rm -r "{}"'.format(input_dir)
deps = ['combined_pdf_creation_-_{}'.format(i),
'combined_txt_creation_-_{}'.format(i),
'poco_bundle_creation_-_{}'.format(i),
'tei_p5_creation_-_{}'.format(i)]
lbl = 'job_cleanup_-_{}'.format(i)
task = self.addTask(command=cmd, dependencies=deps, label=lbl)
cleanup_tasks.append(task)
input_dir = self.intermediate_dir
cmd = 'rm -r "{}"'.format(input_dir)
deps = cleanup_tasks
lbl = 'pipeline_cleanup'
task = self.addTask(command=cmd, dependencies=deps, label=lbl)
cleanup_tasks.append(task)
self.waitForTasks()
hocr_to_tei_tasks.append(task)
'''
' ##################################################
@ -375,7 +223,7 @@ class OCRPipeline(WorkflowRunner):
cmd += ' -i "*.pdf" "*.txt" "*.xml" "*.hocr" "*.{}"'.format('bin.png' if self.binarize else 'tif') # noqa
cmd += ' && '
cmd += 'cd -'
deps = combined_pdf_creation_tasks + combined_txt_creation_tasks + poco_bundle_creation_tasks # noqa
deps = hocr_to_tei_tasks
lbl = 'zip_creation_-_all'
task = self.addTask(command=cmd, dependencies=deps, label=lbl)
zip_creation_tasks.append(task)
@ -389,7 +237,7 @@ class OCRPipeline(WorkflowRunner):
cmd += ' -i "*.pdf"'
cmd += ' && '
cmd += 'cd -'
deps = combined_pdf_creation_tasks
deps = ocr_tasks
lbl = 'zip_creation_-_pdf'
task = self.addTask(command=cmd, dependencies=deps, label=lbl)
zip_creation_tasks.append(task)
@ -403,7 +251,7 @@ class OCRPipeline(WorkflowRunner):
cmd += ' -i "*.txt"'
cmd += ' && '
cmd += 'cd -'
deps = combined_txt_creation_tasks
deps = ocr_tasks
lbl = 'zip_creation_-_txt'
task = self.addTask(command=cmd, dependencies=deps, label=lbl)
zip_creation_tasks.append(task)
@ -417,7 +265,7 @@ class OCRPipeline(WorkflowRunner):
cmd += ' -i "*.xml"'
cmd += ' && '
cmd += 'cd -'
deps = tei_p5_creation_tasks
deps = hocr_to_tei_tasks
lbl = 'zip_creation_-_xml'
task = self.addTask(command=cmd, dependencies=deps, label=lbl)
zip_creation_tasks.append(task)
@ -431,37 +279,79 @@ class OCRPipeline(WorkflowRunner):
cmd += ' -i "*.hocr" "*.{}"'.format('bin.png' if self.binarize else 'tif') # noqa
cmd += ' && '
cmd += 'cd -'
deps = poco_bundle_creation_tasks
deps = post_ocr_tasks
lbl = 'zip_creation_-_poco'
task = self.addTask(command=cmd, dependencies=deps, label=lbl)
zip_creation_tasks.append(task)
def collect_jobs(input_dir, output_dir, intermediate_dir):
def collect_jobs(input_dir, output_dir):
jobs = []
for file in os.listdir(input_dir):
if os.path.isdir(os.path.join(input_dir, file)):
jobs += collect_jobs(os.path.join(input_dir, file),
os.path.join(output_dir, file),
os.path.join(intermediate_dir, file))
elif file.lower().endswith('.pdf'):
continue
if file.lower().endswith('.pdf'):
job = OCRPipelineJob(os.path.join(input_dir, file),
os.path.join(output_dir, file),
os.path.join(intermediate_dir, file))
os.path.join(output_dir, file))
jobs.append(job)
return jobs
def parse_args():
parser = ArgumentParser(description='OCR pipeline for PDF file processing',
prog='OCR pipeline')
parser.add_argument('-i', '--input-dir',
help='Input directory',
required=True)
parser.add_argument('-o', '--output-dir',
help='Output directory',
required=True)
parser.add_argument('-l', '--language',
choices=list(map(lambda x: x[:-12], filter(lambda x: x.endswith('.traineddata'), os.listdir('/usr/local/share/tessdata')))), # noqa
help='Language of the input '
'(3-character ISO 639-2 language codes)',
required=True)
parser.add_argument('--binarize',
action='store_true',
help='Add binarization as a preprocessing step')
parser.add_argument('--log-dir',
help='Logging directory')
parser.add_argument('--mem-mb',
help='Amount of system memory to be used (Default: min(--n-cores * 2048, available system memory))', # noqa
type=int)
parser.add_argument('--n-cores',
default=min(4, multiprocessing.cpu_count()),
help='Number of CPU threads to be used (Default: min(4, number of CPUs))', # noqa
type=int)
parser.add_argument('--zip',
help='Create one zip file per filetype')
parser.add_argument('-v', '--version',
action='version',
help='Returns the current version of the OCR pipeline',
version='%(prog)s {}'.format(__version__))
args = parser.parse_args()
# Set some tricky default values and check for insufficient input
if args.log_dir is None:
args.log_dir = args.output_dir
if args.n_cores < 1:
raise Exception('--n-cores must be greater or equal 1')
if args.mem_mb is None:
max_mem_mb = int(os.popen('free -t -m').readlines()[-1].split()[1:][0])
args.mem_mb = min(args.n_cores * 2048, max_mem_mb)
if args.mem_mb < 2048:
raise Exception('--mem-mb must be greater or equal 2048')
if args.zip is not None and args.zip.lower().endswith('.zip'):
# Remove .zip file extension if provided
args.zip = args.zip[:-4]
args.zip = args.zip if args.zip else 'output'
return args
def main():
args = parse_args()
ocr_pipeline = OCRPipeline(args.input_directory, args.language,
args.output_directory, args.binarize,
args.intermediate_directory, args.n_cores,
args.zip)
retval = ocr_pipeline.run(
dataDirRoot=(args.log_dir or args.output_directory),
nCores=args.n_cores
)
ocr_pipeline = OCRPipeline(args.input_dir, args.language, args.output_dir, args.binarize, args.zip) # noqa
retval = ocr_pipeline.run(dataDirRoot=args.log_dir, memMb=args.mem_mb, nCores=args.n_cores) # noqa
sys.exit(retval)

View File

@ -1,43 +1,38 @@
#!/usr/bin/env python3
# coding=utf-8
"""A wrapper to execute the OCR pipeline in a Docker container"""
from argparse import ArgumentParser
import os
import subprocess
import sys
CONTAINER_IMAGE_TAG = '1.0.0'
CONTAINER_IMAGE = 'gitlab.ub.uni-bielefeld.de:4567/sfb1288inf/ocr:{}'.format(CONTAINER_IMAGE_TAG) # noqa
CONTAINER_IMAGE = 'gitlab.ub.uni-bielefeld.de:4567/sfb1288inf/ocr:1.0.0'
CONTAINER_INPUT_DIR = '/input'
CONTAINER_INTERMEDIATE_DIR = '/intermediate'
CONTAINER_OUTPUT_DIR = '/output'
CONTAINER_LOG_DIR = '/logs'
UID = str(os.getuid())
GID = str(os.getgid())
parser = ArgumentParser(add_help=False)
parser.add_argument('-i', '--input-directory')
parser.add_argument('-o', '--output-directory')
parser.add_argument('--intermediate-directory')
parser.add_argument('-i', '--input-dir')
parser.add_argument('-o', '--output-dir')
parser.add_argument('--log-dir')
args, remaining_args = parser.parse_known_args()
cmd = ['docker', 'run', '--rm', '-it', '-u', '{}:{}'.format(UID, GID)]
if args.intermediate_directory is not None:
cmd += ['-v', '{}:{}'.format(os.path.abspath(args.intermediate_directory),
CONTAINER_INTERMEDIATE_DIR)]
remaining_args.insert(0, CONTAINER_INTERMEDIATE_DIR)
remaining_args.insert(0, '--intermediate-directory')
if args.output_directory is not None:
cmd += ['-v', '{}:{}'.format(os.path.abspath(args.output_directory),
CONTAINER_OUTPUT_DIR)]
remaining_args.insert(0, CONTAINER_OUTPUT_DIR)
remaining_args.insert(0, '-o')
if args.input_directory is not None:
cmd += ['-v', '{}:{}'.format(os.path.abspath(args.input_directory),
CONTAINER_INPUT_DIR)]
remaining_args.insert(0, CONTAINER_INPUT_DIR)
remaining_args.insert(0, '-i')
if args.input_dir is not None:
mapping = os.path.abspath(args.input_dir) + ':' + CONTAINER_INPUT_DIR
cmd += ['-v', mapping]
remaining_args += ['-i', CONTAINER_INPUT_DIR]
if args.output_dir is not None:
mapping = os.path.abspath(args.output_dir) + ':' + CONTAINER_OUTPUT_DIR
cmd += ['-v', mapping]
remaining_args += ['-o', CONTAINER_OUTPUT_DIR]
if args.log_dir is not None:
mapping = os.path.abspath(args.log_dir) + ':' + CONTAINER_LOG_DIR
cmd += ['-v', mapping]
remaining_args += ['--log-dir', CONTAINER_LOG_DIR]
cmd.append(CONTAINER_IMAGE)
cmd += remaining_args
subprocess.run(cmd)
sys.exit(subprocess.run(cmd).returncode)