mirror of
https://gitlab.ub.uni-bielefeld.de/sfb1288inf/ocr.git
synced 2025-07-09 18:23:17 +00:00
Compare commits
20 Commits
613bceb4ff
...
master
Author | SHA1 | Date | |
---|---|---|---|
ca1803ab8a | |||
4518ca1c83 | |||
aeab9b7802 | |||
00c4b17018 | |||
c057d324cf | |||
f51a8c4546 | |||
c640d9743f | |||
e3fd679b38 | |||
8a3816121c | |||
e1b78b6ba4 | |||
a0760487ae | |||
a798457c43 | |||
e2da0fb839 | |||
e78f667438 | |||
41f70da8eb | |||
6db7f70446 | |||
947658a7d8 | |||
acbf61be05 | |||
104598039e | |||
da29659a9b |
@ -1,8 +1,5 @@
|
|||||||
image: docker:19.03.13
|
image: docker:19.03.13
|
||||||
|
|
||||||
variables:
|
|
||||||
DOCKER_TLS_CERTDIR: "/certs"
|
|
||||||
|
|
||||||
services:
|
services:
|
||||||
- docker:19.03.13-dind
|
- docker:19.03.13-dind
|
||||||
|
|
||||||
@ -10,6 +7,10 @@ stages:
|
|||||||
- build
|
- build
|
||||||
- push
|
- push
|
||||||
|
|
||||||
|
variables:
|
||||||
|
DOCKER_TLS_CERTDIR: "/certs"
|
||||||
|
INTERMEDIATE_IMAGE_TAG: $CI_REGISTRY_IMAGE:$CI_COMMIT_REF_NAME-$CI_COMMIT_SHA
|
||||||
|
|
||||||
.reg_setup:
|
.reg_setup:
|
||||||
before_script:
|
before_script:
|
||||||
- apk add --no-cache curl
|
- apk add --no-cache curl
|
||||||
@ -28,8 +29,6 @@ build_image:
|
|||||||
stage: build
|
stage: build
|
||||||
tags:
|
tags:
|
||||||
- docker
|
- docker
|
||||||
variables:
|
|
||||||
INTERMEDIATE_IMAGE_TAG: $CI_REGISTRY_IMAGE:$CI_COMMIT_SHA
|
|
||||||
|
|
||||||
push_master:
|
push_master:
|
||||||
extends:
|
extends:
|
||||||
@ -47,7 +46,6 @@ push_master:
|
|||||||
- docker
|
- docker
|
||||||
variables:
|
variables:
|
||||||
IMAGE_TAG: $CI_REGISTRY_IMAGE:latest
|
IMAGE_TAG: $CI_REGISTRY_IMAGE:latest
|
||||||
INTERMEDIATE_IMAGE_TAG: $CI_REGISTRY_IMAGE:$CI_COMMIT_SHA
|
|
||||||
|
|
||||||
push_other:
|
push_other:
|
||||||
extends:
|
extends:
|
||||||
@ -68,4 +66,3 @@ push_other:
|
|||||||
- docker
|
- docker
|
||||||
variables:
|
variables:
|
||||||
IMAGE_TAG: $CI_REGISTRY_IMAGE:$CI_COMMIT_REF_NAME
|
IMAGE_TAG: $CI_REGISTRY_IMAGE:$CI_COMMIT_REF_NAME
|
||||||
INTERMEDIATE_IMAGE_TAG: $CI_REGISTRY_IMAGE:$CI_COMMIT_SHA
|
|
||||||
|
86
Dockerfile
86
Dockerfile
@ -1,47 +1,59 @@
|
|||||||
FROM debian:buster-slim
|
FROM debian:buster-slim
|
||||||
|
|
||||||
|
|
||||||
LABEL authors="Patrick Jentsch <p.jentsch@uni-bielefeld.de>"
|
LABEL authors="Patrick Jentsch <p.jentsch@uni-bielefeld.de>, Stephan Porada <porada@posteo.de>"
|
||||||
|
|
||||||
|
|
||||||
ENV LANG=C.UTF-8
|
ENV LANG=C.UTF-8
|
||||||
|
|
||||||
|
|
||||||
RUN apt-get update
|
RUN apt-get update \
|
||||||
|
&& apt-get install --no-install-recommends --yes \
|
||||||
|
ghostscript \
|
||||||
|
procps \
|
||||||
|
python3.7 \
|
||||||
|
python3-pip \
|
||||||
|
rename \
|
||||||
|
wget \
|
||||||
|
zip \
|
||||||
|
&& python3 -m pip install lxml
|
||||||
|
|
||||||
|
# Install the OCR pipeline and it's dependencies #
|
||||||
# Install pipeline dependencies #
|
|
||||||
## Install pyFlow ##
|
## Install pyFlow ##
|
||||||
ENV PYFLOW_RELEASE=1.1.20
|
ENV PYFLOW_VERSION=1.1.20
|
||||||
ADD "https://github.com/Illumina/pyflow/releases/download/v${PYFLOW_RELEASE}/pyflow-${PYFLOW_RELEASE}.tar.gz" .
|
RUN wget --no-check-certificate --quiet \
|
||||||
RUN tar -xzf "pyflow-${PYFLOW_RELEASE}.tar.gz" \
|
"https://github.com/Illumina/pyflow/releases/download/v${PYFLOW_VERSION}/pyflow-${PYFLOW_VERSION}.tar.gz" \
|
||||||
&& cd "pyflow-${PYFLOW_RELEASE}" \
|
&& tar -xzf "pyflow-${PYFLOW_VERSION}.tar.gz" \
|
||||||
|
&& cd "pyflow-${PYFLOW_VERSION}" \
|
||||||
&& apt-get install --no-install-recommends --yes \
|
&& apt-get install --no-install-recommends --yes \
|
||||||
python2.7 \
|
python2.7 \
|
||||||
&& python2.7 setup.py build install \
|
&& python2.7 setup.py build install \
|
||||||
&& cd .. \
|
&& cd - > /dev/null \
|
||||||
&& rm -r "pyflow-${PYFLOW_RELEASE}" "pyflow-${PYFLOW_RELEASE}.tar.gz"
|
&& rm -r "pyflow-${PYFLOW_VERSION}" "pyflow-${PYFLOW_VERSION}.tar.gz"
|
||||||
|
|
||||||
|
|
||||||
## Install ocropy ##
|
## Install ocropy ##
|
||||||
ENV OCROPY_RELEASE=1.3.3
|
ENV OCROPY_VERSION=1.3.3
|
||||||
ADD "https://github.com/tmbdev/ocropy/archive/v${OCROPY_RELEASE}.tar.gz" .
|
RUN wget --no-check-certificate --quiet \
|
||||||
RUN tar -xzf "v${OCROPY_RELEASE}.tar.gz" \
|
"https://github.com/tmbdev/ocropy/archive/v${OCROPY_VERSION}.tar.gz" \
|
||||||
&& cd "ocropy-${OCROPY_RELEASE}" \
|
&& tar -xzf "v${OCROPY_VERSION}.tar.gz" \
|
||||||
|
&& cd "ocropy-${OCROPY_VERSION}" \
|
||||||
&& apt-get install --no-install-recommends --yes \
|
&& apt-get install --no-install-recommends --yes \
|
||||||
|
python2.7 \
|
||||||
python-pil \
|
python-pil \
|
||||||
python-tk \
|
python-tk \
|
||||||
$(cat PACKAGES) \
|
$(cat PACKAGES) \
|
||||||
&& python2.7 setup.py install \
|
&& python2.7 setup.py install \
|
||||||
&& cd .. \
|
&& cd - > /dev/null \
|
||||||
&& rm -r "ocropy-${OCROPY_RELEASE}" "v${OCROPY_RELEASE}.tar.gz"
|
&& rm -r "ocropy-${OCROPY_VERSION}" "v${OCROPY_VERSION}.tar.gz"
|
||||||
|
|
||||||
|
|
||||||
## Install Tesseract OCR ##
|
## Install Tesseract OCR ##
|
||||||
ENV TESSERACT_RELEASE=4.1.1
|
ENV TESSERACT_VERSION=5.0.0
|
||||||
ADD "https://github.com/tesseract-ocr/tesseract/archive/${TESSERACT_RELEASE}.tar.gz" .
|
RUN wget --no-check-certificate --quiet \
|
||||||
RUN tar -xzf "${TESSERACT_RELEASE}.tar.gz" \
|
"https://github.com/tesseract-ocr/tesseract/archive/${TESSERACT_VERSION}.tar.gz" \
|
||||||
&& cd "tesseract-${TESSERACT_RELEASE}" \
|
&& tar -xzf "${TESSERACT_VERSION}.tar.gz" \
|
||||||
|
&& cd "tesseract-${TESSERACT_VERSION}" \
|
||||||
&& apt-get install --no-install-recommends --yes \
|
&& apt-get install --no-install-recommends --yes \
|
||||||
autoconf \
|
autoconf \
|
||||||
automake \
|
automake \
|
||||||
@ -55,47 +67,19 @@ RUN tar -xzf "${TESSERACT_RELEASE}.tar.gz" \
|
|||||||
pkg-config \
|
pkg-config \
|
||||||
zlib1g-dev \
|
zlib1g-dev \
|
||||||
&& ./autogen.sh \
|
&& ./autogen.sh \
|
||||||
&& ./configure \
|
&& ./configure --disable-openmp --disable-shared 'CXXFLAGS=-g -O2 -fno-math-errno -Wall -Wextra -Wpedantic' \
|
||||||
&& make \
|
&& make \
|
||||||
&& make install \
|
&& make install \
|
||||||
&& ldconfig \
|
&& ldconfig \
|
||||||
&& cd - > /dev/null \
|
&& cd - > /dev/null \
|
||||||
&& rm -r "tesseract-${TESSERACT_RELEASE}" "${TESSERACT_RELEASE}.tar.gz"
|
&& rm -r "tesseract-${TESSERACT_VERSION}" "${TESSERACT_VERSION}.tar.gz"
|
||||||
|
|
||||||
ENV TESSDATA_BEST_RELEASE=4.1.0
|
|
||||||
ADD "https://github.com/tesseract-ocr/tessdata_best/archive/${TESSDATA_BEST_RELEASE}.tar.gz" .
|
|
||||||
RUN tar -xzf "${TESSDATA_BEST_RELEASE}.tar.gz" \
|
|
||||||
&& mv "tessdata_best-${TESSDATA_BEST_RELEASE}/ara.traineddata" "/usr/local/share/tessdata/" \
|
|
||||||
&& mv "tessdata_best-${TESSDATA_BEST_RELEASE}/chi_tra.traineddata" "/usr/local/share/tessdata/" \
|
|
||||||
&& mv "tessdata_best-${TESSDATA_BEST_RELEASE}/dan.traineddata" "/usr/local/share/tessdata/" \
|
|
||||||
&& mv "tessdata_best-${TESSDATA_BEST_RELEASE}/deu.traineddata" "/usr/local/share/tessdata/" \
|
|
||||||
&& mv "tessdata_best-${TESSDATA_BEST_RELEASE}/ell.traineddata" "/usr/local/share/tessdata/" \
|
|
||||||
&& mv "tessdata_best-${TESSDATA_BEST_RELEASE}/eng.traineddata" "/usr/local/share/tessdata/" \
|
|
||||||
&& mv "tessdata_best-${TESSDATA_BEST_RELEASE}/enm.traineddata" "/usr/local/share/tessdata/" \
|
|
||||||
&& mv "tessdata_best-${TESSDATA_BEST_RELEASE}/fra.traineddata" "/usr/local/share/tessdata/" \
|
|
||||||
&& mv "tessdata_best-${TESSDATA_BEST_RELEASE}/frk.traineddata" "/usr/local/share/tessdata/" \
|
|
||||||
&& mv "tessdata_best-${TESSDATA_BEST_RELEASE}/frm.traineddata" "/usr/local/share/tessdata/" \
|
|
||||||
&& mv "tessdata_best-${TESSDATA_BEST_RELEASE}/ita.traineddata" "/usr/local/share/tessdata/" \
|
|
||||||
&& mv "tessdata_best-${TESSDATA_BEST_RELEASE}/por.traineddata" "/usr/local/share/tessdata/" \
|
|
||||||
&& mv "tessdata_best-${TESSDATA_BEST_RELEASE}/rus.traineddata" "/usr/local/share/tessdata/" \
|
|
||||||
&& mv "tessdata_best-${TESSDATA_BEST_RELEASE}/spa.traineddata" "/usr/local/share/tessdata/" \
|
|
||||||
&& rm -r "tessdata_best-${TESSDATA_BEST_RELEASE}" "${TESSDATA_BEST_RELEASE}.tar.gz"
|
|
||||||
|
|
||||||
|
|
||||||
## Further dependencies ##
|
|
||||||
RUN apt-get install --no-install-recommends --yes \
|
|
||||||
ghostscript \
|
|
||||||
python-pip \
|
|
||||||
python3.7 \
|
|
||||||
zip \
|
|
||||||
&& pip install natsort
|
|
||||||
|
|
||||||
|
|
||||||
RUN rm -r /var/lib/apt/lists/*
|
RUN rm -r /var/lib/apt/lists/*
|
||||||
|
|
||||||
|
|
||||||
## Install Pipeline ##
|
## Install Pipeline ##
|
||||||
COPY hocrtotei ocr /usr/local/bin/
|
COPY hocr2tei hocr-combine ocr /usr/local/bin/
|
||||||
|
|
||||||
|
|
||||||
ENTRYPOINT ["ocr"]
|
ENTRYPOINT ["ocr"]
|
||||||
|
21
LICENSE
Normal file
21
LICENSE
Normal file
@ -0,0 +1,21 @@
|
|||||||
|
MIT License
|
||||||
|
|
||||||
|
Copyright (c) 2021 Bielefeld University - CRC 1288 - INF
|
||||||
|
|
||||||
|
Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||||
|
of this software and associated documentation files (the "Software"), to deal
|
||||||
|
in the Software without restriction, including without limitation the rights
|
||||||
|
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
||||||
|
copies of the Software, and to permit persons to whom the Software is
|
||||||
|
furnished to do so, subject to the following conditions:
|
||||||
|
|
||||||
|
The above copyright notice and this permission notice shall be included in all
|
||||||
|
copies or substantial portions of the Software.
|
||||||
|
|
||||||
|
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||||
|
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||||
|
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||||
|
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||||
|
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
||||||
|
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
||||||
|
SOFTWARE.
|
106
README.md
106
README.md
@ -1,83 +1,49 @@
|
|||||||
# OCR - Optical Character Recognition
|
# OCR - Optical Character Recognition
|
||||||
|
|
||||||
This software implements a heavily parallelized pipeline to recognize text in PDF files. It is used for nopaque's OCR service but you can also use it standalone, for that purpose a convenient wrapper script is provided.
|
This software implements a heavily parallelized pipeline to recognize text in PDF files. It is used for nopaque's OCR service but you can also use it standalone, for that purpose a convenient wrapper script is provided. The pipeline is designed to run on Linux operating systems, but with some tweaks it should also run on Windows with WSL installed.
|
||||||
|
|
||||||
## Software used in this pipeline implementation
|
## Software used in this pipeline implementation
|
||||||
- Official Debian Docker image (buster-slim) and programs from its free repositories: https://hub.docker.com/_/debian
|
|
||||||
|
- Official Debian Docker image (buster-slim): https://hub.docker.com/_/debian
|
||||||
|
- Software from Debian Buster's free repositories
|
||||||
- ocropy (1.3.3): https://github.com/ocropus/ocropy/releases/tag/v1.3.3
|
- ocropy (1.3.3): https://github.com/ocropus/ocropy/releases/tag/v1.3.3
|
||||||
- pyFlow (1.1.20): https://github.com/Illumina/pyflow/releases/tag/v1.1.20
|
- pyFlow (1.1.20): https://github.com/Illumina/pyflow/releases/tag/v1.1.20
|
||||||
- Tesseract OCR (4.1.1): https://github.com/tesseract-ocr/tesseract/releases/tag/4.1.1
|
- Tesseract OCR (5.0.0): https://github.com/tesseract-ocr/tesseract/releases/tag/5.0.0
|
||||||
- tessdata_best (4.1.0): https://github.com/tesseract-ocr/tessdata_best/releases/tag/4.1.0
|
|
||||||
|
|
||||||
|
## Installation
|
||||||
|
|
||||||
## Use this image
|
1. Install Docker and Python 3.
|
||||||
|
2. Clone this repository: `git clone https://gitlab.ub.uni-bielefeld.de/sfb1288inf/ocr.git`
|
||||||
|
3. Build the Docker image: `docker build -t gitlab.ub.uni-bielefeld.de:4567/sfb1288inf/ocr:v0.1.0 ocr`
|
||||||
|
4. Add the wrapper script (`wrapper/ocr` relative to this README file) to your `${PATH}`.
|
||||||
|
5. Create working directories for the pipeline: `mkdir -p /<my_data_location>/{input,models,output}`.
|
||||||
|
6. Place your Tesseract OCR model(s) inside `/<my_data_location>/models`.
|
||||||
|
|
||||||
1. Create input and output directories for the pipeline.
|
## Use the Pipeline
|
||||||
``` bash
|
|
||||||
mkdir -p /<my_data_location>/input /<my_data_location>/output
|
|
||||||
```
|
|
||||||
|
|
||||||
2. Place your PDF files inside `/<my_data_location>/input`. Files should all contain text of the same language.
|
1. Place your PDF files inside `/<my_data_location>/input`. Files should all contain text of the same language.
|
||||||
|
2. Clear your `/<my_data_location>/output` directory.
|
||||||
3. Start the pipeline process. Check the [Pipeline arguments](#pipeline-arguments) section for more details.
|
3. Start the pipeline process. Check the pipeline help (`ocr --help`) for more details.
|
||||||
```
|
```bash
|
||||||
# Option one: Use the wrapper script
|
|
||||||
## Install the wrapper script (only on first run). Get it from https://gitlab.ub.uni-bielefeld.de/sfb1288inf/ocr/-/raw/1.0.0/wrapper/ocr, make it executeable and add it to your ${PATH}
|
|
||||||
cd /<my_data_location>
|
cd /<my_data_location>
|
||||||
ocr -i input -l <language_code> -o output <optional_pipeline_arguments>
|
# <model_code> is the model filename without the ".traineddata" suffix
|
||||||
|
ocr \
|
||||||
# Option two: Classic Docker style
|
--input-dir input \
|
||||||
docker run \
|
--output-dir output \
|
||||||
--rm \
|
--model-file models/<model>
|
||||||
-it \
|
-m <model_code> <optional_pipeline_arguments>
|
||||||
-u $(id -u $USER):$(id -g $USER) \
|
# More then one model
|
||||||
-v /<my_data_location>/input:/input \
|
ocr \
|
||||||
-v /<my_data_location>/output:/output \
|
--input-dir input \
|
||||||
gitlab.ub.uni-bielefeld.de:4567/sfb1288inf/ocr:1.0.0 \
|
--output-dir output \
|
||||||
-i /input \
|
--model-file models/<model1>
|
||||||
-l <language_code>
|
--model-file models/<model2>
|
||||||
-o /output \
|
-m <model1_code>+<model2_code> <optional_pipeline_arguments>
|
||||||
<optional_pipeline_arguments>
|
# Instead of multiple --model-file statements, you can also use
|
||||||
|
ocr \
|
||||||
|
--input-dir input \
|
||||||
|
--output-dir output \
|
||||||
|
--model-file models/*
|
||||||
|
-m <model1_code>+<model2_code> <optional_pipeline_arguments>
|
||||||
```
|
```
|
||||||
|
|
||||||
4. Check your results in the `/<my_data_location>/output` directory.
|
4. Check your results in the `/<my_data_location>/output` directory.
|
||||||
```
|
|
||||||
|
|
||||||
### Pipeline arguments
|
|
||||||
|
|
||||||
`-l languagecode`
|
|
||||||
* Tells tesseract which language will be used.
|
|
||||||
* options = ara (Arabic), chi_tra (Chinese - Traditional), dan (Danish), deu (German), ell (Greek, Modern (1453-)), eng (English), enm (Middle englisch), fra (French), frk (German Fraktur), frm (Middle french), ita (Italian), por (Portuguese), rus (Russian), spa (Spanish)
|
|
||||||
* required = True
|
|
||||||
|
|
||||||
`--keep-intermediates`
|
|
||||||
* If set, all intermediate files created during the OCR process will be
|
|
||||||
kept.
|
|
||||||
* default = False
|
|
||||||
* required = False
|
|
||||||
|
|
||||||
`--nCores corenumber`
|
|
||||||
* Sets the number of CPU cores being used during the OCR process.
|
|
||||||
* default = min(4, multiprocessing.cpu_count())
|
|
||||||
* required = False
|
|
||||||
|
|
||||||
`--skip-binarisation`
|
|
||||||
* Used to skip binarization with ocropus. If skipped, only the tesseract binarization is used.
|
|
||||||
* default = False
|
|
||||||
|
|
||||||
``` bash
|
|
||||||
# Example with all arguments used
|
|
||||||
docker run \
|
|
||||||
--rm \
|
|
||||||
-it \
|
|
||||||
-u $(id -u $USER):$(id -g $USER) \
|
|
||||||
-v "$HOME"/ocr/input:/input \
|
|
||||||
-v "$HOME"/ocr/output:/output \
|
|
||||||
gitlab.ub.uni-bielefeld.de:4567/sfb1288inf/ocr:1.0.0 \
|
|
||||||
-i /input \
|
|
||||||
-l eng \
|
|
||||||
-o /output \
|
|
||||||
--keep_intermediates \
|
|
||||||
--nCores 8 \
|
|
||||||
--skip-binarisation
|
|
||||||
```
|
|
||||||
|
44
hocr-combine
Executable file
44
hocr-combine
Executable file
@ -0,0 +1,44 @@
|
|||||||
|
#!/usr/bin/env python3.7
|
||||||
|
# coding=utf-8
|
||||||
|
|
||||||
|
''' Combine multiple hOCR files. '''
|
||||||
|
|
||||||
|
from argparse import ArgumentParser
|
||||||
|
from lxml import html
|
||||||
|
|
||||||
|
|
||||||
|
parser = ArgumentParser(description='Combine multiple hOCR files.')
|
||||||
|
parser.add_argument(
|
||||||
|
'-i', '--input-file',
|
||||||
|
help='Input file',
|
||||||
|
nargs='+',
|
||||||
|
required=True
|
||||||
|
)
|
||||||
|
parser.add_argument(
|
||||||
|
'-o', '--output-file',
|
||||||
|
help='Output file',
|
||||||
|
required=True
|
||||||
|
)
|
||||||
|
args = parser.parse_args()
|
||||||
|
|
||||||
|
|
||||||
|
for input_file in args.input_file:
|
||||||
|
input_files = []
|
||||||
|
if input_file.startswith('@'):
|
||||||
|
with open(input_file[1:], 'r') as f:
|
||||||
|
input_files += [x for x in f.read().split("\n") if x != '']
|
||||||
|
else:
|
||||||
|
input_files.append(input_file)
|
||||||
|
if len(input_files) == 0:
|
||||||
|
exit(1)
|
||||||
|
|
||||||
|
|
||||||
|
hocr = html.parse(input_files[0])
|
||||||
|
hocr_body = hocr.find('body')
|
||||||
|
for input_file in input_files[1:]:
|
||||||
|
for ocr_page in html.parse(input_file).findall('//div[@class="ocr_page"]'):
|
||||||
|
hocr_body.append(ocr_page)
|
||||||
|
|
||||||
|
|
||||||
|
with open(args.output_file, 'wb') as f:
|
||||||
|
hocr.write(f, encoding='UTF-8', method='html')
|
68
hocr2tei
Executable file
68
hocr2tei
Executable file
@ -0,0 +1,68 @@
|
|||||||
|
#!/usr/bin/env python3.7
|
||||||
|
# coding=utf-8
|
||||||
|
|
||||||
|
''' Convert hOCR to TEI XML. '''
|
||||||
|
|
||||||
|
from argparse import ArgumentParser
|
||||||
|
from lxml import html
|
||||||
|
from xml.sax.saxutils import escape
|
||||||
|
import re
|
||||||
|
|
||||||
|
|
||||||
|
parser = ArgumentParser(description='Convert hOCR to TEI XML.')
|
||||||
|
parser.add_argument(
|
||||||
|
'-i', '--input-file',
|
||||||
|
help='Input file',
|
||||||
|
required=True
|
||||||
|
)
|
||||||
|
parser.add_argument(
|
||||||
|
'-o', '--output-file',
|
||||||
|
help='Output file',
|
||||||
|
required=True
|
||||||
|
)
|
||||||
|
args = parser.parse_args()
|
||||||
|
|
||||||
|
|
||||||
|
tei = ''
|
||||||
|
tei += '<TEI xmlns="http://www.tei-c.org/ns/1.0">\n'
|
||||||
|
tei += ' <teiHeader>\n'
|
||||||
|
tei += ' <fileDesc>\n'
|
||||||
|
tei += ' <titleStmt>\n'
|
||||||
|
tei += ' <title></title>\n'
|
||||||
|
tei += ' </titleStmt>\n'
|
||||||
|
tei += ' <publicationStmt>\n'
|
||||||
|
tei += ' <p></p>\n'
|
||||||
|
tei += ' </publicationStmt>\n'
|
||||||
|
tei += ' <sourceDesc>\n'
|
||||||
|
tei += ' <p></p>\n'
|
||||||
|
tei += ' </sourceDesc>\n'
|
||||||
|
tei += ' </fileDesc>\n'
|
||||||
|
tei += ' </teiHeader>\n'
|
||||||
|
tei += ' <text>\n'
|
||||||
|
tei += ' <body>\n'
|
||||||
|
hocr = html.parse(args.input_file)
|
||||||
|
for ocr_page in hocr.findall('.//div[@class="ocr_page"]'):
|
||||||
|
ocr_page_title_attrib = ocr_page.attrib.get('title')
|
||||||
|
facsimile = re.search(r'image \"(.*?)\"', ocr_page_title_attrib).group(1)
|
||||||
|
page_number = re.search(r'ppageno (\d+)', ocr_page_title_attrib).group(1)
|
||||||
|
tei += f' <pb facs="{facsimile}" n="{page_number}"/>\n'
|
||||||
|
for ocr_par in ocr_page.findall('.//p[@class="ocr_par"]'):
|
||||||
|
tei += ' <p>\n'
|
||||||
|
for ocr_line in ocr_par.findall('.//span[@class="ocr_line"]'):
|
||||||
|
tei += ' <lb/>'
|
||||||
|
is_first_word_in_line = True
|
||||||
|
for ocrx_word in ocr_line.findall('.//span[@class="ocrx_word"]'):
|
||||||
|
if ocrx_word.text is not None:
|
||||||
|
if not is_first_word_in_line:
|
||||||
|
tei += ' '
|
||||||
|
tei += escape(ocrx_word.text)
|
||||||
|
is_first_word_in_line = False
|
||||||
|
tei += '\n'
|
||||||
|
tei += ' </p>\n'
|
||||||
|
tei += ' </body>\n'
|
||||||
|
tei += ' </text>\n'
|
||||||
|
tei += '</TEI>\n'
|
||||||
|
|
||||||
|
|
||||||
|
with open(args.output_file, 'w') as f:
|
||||||
|
f.write(tei)
|
49
hocrtotei
49
hocrtotei
@ -1,49 +0,0 @@
|
|||||||
#!/usr/bin/env python3.7
|
|
||||||
# coding=utf-8
|
|
||||||
|
|
||||||
""""Merges hOCR files into a TEI file."""
|
|
||||||
|
|
||||||
from xml.sax.saxutils import escape
|
|
||||||
from argparse import ArgumentParser
|
|
||||||
import xml.etree.ElementTree as ET
|
|
||||||
|
|
||||||
parser = ArgumentParser(description='Merges hOCR files into a TEI file.')
|
|
||||||
parser.add_argument('i', metavar='hOCR-sourcefile', nargs='+')
|
|
||||||
parser.add_argument('o', metavar='TEI-destfile',)
|
|
||||||
args = parser.parse_args()
|
|
||||||
|
|
||||||
output_file = open(args.o, 'w')
|
|
||||||
output_file.write(
|
|
||||||
'<?xml version="1.0" encoding="UTF-8"?>\n'
|
|
||||||
+ '<TEI xmlns="http://www.tei-c.org/ns/1.0" xml:id="dtabf">\n'
|
|
||||||
+ ' <teiHeader>\n'
|
|
||||||
+ ' <fileDesc>\n'
|
|
||||||
+ ' <titleStmt/>\n'
|
|
||||||
+ ' <publicationStmt/>\n'
|
|
||||||
+ ' <sourceDesc/>\n'
|
|
||||||
+ ' </fileDesc>\n'
|
|
||||||
+ ' <encodingDesc/>\n'
|
|
||||||
+ ' <profileDesc/>\n'
|
|
||||||
+ ' </teiHeader>\n'
|
|
||||||
+ ' <text>\n'
|
|
||||||
+ ' <body>\n'
|
|
||||||
)
|
|
||||||
for index, input_file in enumerate(args.i):
|
|
||||||
tree = ET.parse(input_file)
|
|
||||||
output_file.write(' <pb n="%i"/>\n' % (index + 1))
|
|
||||||
for para in tree.findall('.//*[@class="ocr_par"]'):
|
|
||||||
output_file.write(' <p>\n')
|
|
||||||
for line in para.findall('.//*[@class="ocr_line"]'):
|
|
||||||
first_word_in_line = True
|
|
||||||
for word in line.findall('.//*[@class="ocrx_word"]'):
|
|
||||||
if word.text is not None:
|
|
||||||
output_file.write((' ' if first_word_in_line else ' ') + escape(word.text.strip()))
|
|
||||||
first_word_in_line = False
|
|
||||||
if not first_word_in_line:
|
|
||||||
output_file.write('<lb/>\n')
|
|
||||||
output_file.write(' </p>\n')
|
|
||||||
output_file.write(
|
|
||||||
' </body>\n'
|
|
||||||
+ ' </text>\n'
|
|
||||||
+ '</TEI>')
|
|
||||||
output_file.close()
|
|
51
wrapper/ocr
51
wrapper/ocr
@ -1,43 +1,44 @@
|
|||||||
#!/usr/bin/env python3
|
#!/usr/bin/env python3
|
||||||
# coding=utf-8
|
# coding=utf-8
|
||||||
|
|
||||||
"""A wrapper to execute the OCR pipeline in a Docker container"""
|
|
||||||
|
|
||||||
from argparse import ArgumentParser
|
from argparse import ArgumentParser
|
||||||
import os
|
import os
|
||||||
import subprocess
|
import subprocess
|
||||||
|
import sys
|
||||||
|
|
||||||
CONTAINER_IMAGE_TAG = '1.0.0'
|
CONTAINER_IMAGE = 'gitlab.ub.uni-bielefeld.de:4567/sfb1288inf/ocr:v0.1.0'
|
||||||
CONTAINER_IMAGE = 'gitlab.ub.uni-bielefeld.de:4567/sfb1288inf/ocr:{}'.format(CONTAINER_IMAGE_TAG) # noqa
|
|
||||||
CONTAINER_INPUT_DIR = '/input'
|
CONTAINER_INPUT_DIR = '/input'
|
||||||
CONTAINER_INTERMEDIATE_DIR = '/intermediate'
|
|
||||||
CONTAINER_OUTPUT_DIR = '/output'
|
CONTAINER_OUTPUT_DIR = '/output'
|
||||||
|
CONTAINER_MODELS_DIR = '/usr/local/share/tessdata'
|
||||||
|
CONTAINER_LOG_DIR = '/logs'
|
||||||
UID = str(os.getuid())
|
UID = str(os.getuid())
|
||||||
GID = str(os.getgid())
|
GID = str(os.getgid())
|
||||||
|
|
||||||
parser = ArgumentParser(add_help=False)
|
parser = ArgumentParser(add_help=False)
|
||||||
parser.add_argument('-i', '--input-directory')
|
parser.add_argument('-i', '--input-dir')
|
||||||
parser.add_argument('-o', '--output-directory')
|
parser.add_argument('-o', '--output-dir')
|
||||||
parser.add_argument('--intermediate-directory')
|
parser.add_argument('-t', '--model-file', action='extend', nargs='+')
|
||||||
|
parser.add_argument('--log-dir')
|
||||||
args, remaining_args = parser.parse_known_args()
|
args, remaining_args = parser.parse_known_args()
|
||||||
|
|
||||||
cmd = ['docker', 'run', '--rm', '-it', '-u', '{}:{}'.format(UID, GID)]
|
cmd = ['docker', 'run', '--rm', '-it', '-u', f'{UID}:{GID}']
|
||||||
if args.intermediate_directory is not None:
|
if args.input_dir is not None:
|
||||||
cmd += ['-v', '{}:{}'.format(os.path.abspath(args.intermediate_directory),
|
mapping = f'{os.path.abspath(args.input_dir)}:{CONTAINER_INPUT_DIR}'
|
||||||
CONTAINER_INTERMEDIATE_DIR)]
|
cmd += ['-v', mapping]
|
||||||
remaining_args.insert(0, CONTAINER_INTERMEDIATE_DIR)
|
remaining_args += ['-i', CONTAINER_INPUT_DIR]
|
||||||
remaining_args.insert(0, '--intermediate-directory')
|
if args.output_dir is not None:
|
||||||
if args.output_directory is not None:
|
mapping = f'{os.path.abspath(args.output_dir)}:{CONTAINER_OUTPUT_DIR}'
|
||||||
cmd += ['-v', '{}:{}'.format(os.path.abspath(args.output_directory),
|
cmd += ['-v', mapping]
|
||||||
CONTAINER_OUTPUT_DIR)]
|
remaining_args += ['-o', CONTAINER_OUTPUT_DIR]
|
||||||
remaining_args.insert(0, CONTAINER_OUTPUT_DIR)
|
if args.model_file is not None:
|
||||||
remaining_args.insert(0, '-o')
|
for model_file in args.model_file:
|
||||||
if args.input_directory is not None:
|
mapping = f'{os.path.abspath(model_file)}:{CONTAINER_MODELS_DIR}/{os.path.basename(model_file)}' # noqa
|
||||||
cmd += ['-v', '{}:{}'.format(os.path.abspath(args.input_directory),
|
cmd += ['-v', mapping]
|
||||||
CONTAINER_INPUT_DIR)]
|
if args.log_dir is not None:
|
||||||
remaining_args.insert(0, CONTAINER_INPUT_DIR)
|
mapping = '{os.path.abspath(args.log_dir)}:{CONTAINER_LOG_DIR}'
|
||||||
remaining_args.insert(0, '-i')
|
cmd += ['-v', mapping]
|
||||||
|
remaining_args += ['--log-dir', CONTAINER_LOG_DIR]
|
||||||
cmd.append(CONTAINER_IMAGE)
|
cmd.append(CONTAINER_IMAGE)
|
||||||
cmd += remaining_args
|
cmd += remaining_args
|
||||||
|
|
||||||
subprocess.run(cmd)
|
sys.exit(subprocess.run(cmd).returncode)
|
||||||
|
Reference in New Issue
Block a user