sporada/ocr
1
0
Fork 0
mirror of https://gitlab.ub.uni-bielefeld.de/sfb1288inf/ocr.git synced 2024-12-26 15:04:18 +00:00
Code Issues Projects Releases Wiki Activity

Update

Browse Source
This commit is contained in:
Patrick Jentsch 2019-04-25 11:40:27 +02:00
parent d25204d6a9
commit efbf6f24e6
2 changed files with 20 additions and 17 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

18
Dockerfile
View File

@ -42,18 +42,20 @@ RUN wget -nv https://github.com/Illumina/pyflow/releases/download/v"$PYFLOW_VERS
# Install Tesseract OCR and Data Files # Install Tesseract OCR and Data Files
RUN echo "deb https://notesalexp.org/tesseract-ocr/stretch/ stretch main" >> /etc/apt/sources.list && \ RUN echo "deb https://notesalexp.org/tesseract-ocr/stretch/ stretch main" >> /etc/apt/sources.list && \
echo "deb https://notesalexp.org/tesseract-ocr/tessdata_best/ stretch main" >> /etc/apt/sources.list && \
echo "deb https://notesalexp.org/tesseract-ocr/tessdata_legacy/ stretch main" >> /etc/apt/sources.list && \
wget -O - https://notesalexp.org/debian/alexp_key.asc | apt-key add - && \ wget -O - https://notesalexp.org/debian/alexp_key.asc | apt-key add - && \
apt-get update && \ apt-get update && \
apt-get install -y --no-install-recommends \ apt-get install -y --no-install-recommends \
tesseract-ocr \ tesseract-ocr \
tesseract-ocr-deu \ tesseract-ocr-deu-best \
tesseract-ocr-frk \ tesseract-ocr-deu-frak-legacy \
tesseract-ocr-eng \ tesseract-ocr-eng-best \
tesseract-ocr-enm \ tesseract-ocr-enm-best \
tesseract-ocr-fra \ tesseract-ocr-fra-best \
tesseract-ocr-frm \ tesseract-ocr-frm-best \
tesseract-ocr-por \ tesseract-ocr-por-best \
tesseract-ocr-spa tesseract-ocr-spa-best
COPY ocr /usr/local/bin COPY ocr /usr/local/bin
COPY hocrtotei /usr/local/bin COPY hocrtotei /usr/local/bin

19
ocr
View File

@ -73,6 +73,7 @@ class OCRWorkflow(WorkflowRunner):
self.keepIntermediates = keepIntermediates self.keepIntermediates = keepIntermediates
self.lang = lang self.lang = lang
self.nCores = nCores self.nCores = nCores
self.defaultNCores = min(nCores, max(1, int(nCores / len(jobs))))
def workflow(self): def workflow(self):
@ -95,7 +96,7 @@ class OCRWorkflow(WorkflowRunner):
os.path.join(job["output_dir"], "tmp", "binarized_png"), os.path.join(job["output_dir"], "tmp", "binarized_png"),
os.path.join(job["output_dir"], "tmp", "normalized_png"), os.path.join(job["output_dir"], "tmp", "normalized_png"),
) )
create_output_directories_jobs.append(self.addTask(label="create_output_directories_job_-_%i" % (create_output_directories_job_number), command=cmd)) create_output_directories_jobs.append(self.addTask(label="create_output_directories_job_-_%i" % (create_output_directories_job_number), command=cmd, nCores=self.defaultNCores))
### ###
# Task "split_job": split input file into one tiff file per page # Task "split_job": split input file into one tiff file per page
@ -115,7 +116,7 @@ class OCRWorkflow(WorkflowRunner):
job["path"], job["path"],
os.path.join(job["output_dir"], "tmp", "page") os.path.join(job["output_dir"], "tmp", "page")
) )
split_jobs.append(self.addTask(label="split_job_-_%i" % (split_job_number), command=cmd, dependencies=create_output_directories_jobs)) split_jobs.append(self.addTask(label="split_job_-_%i" % (split_job_number), command=cmd, dependencies=create_output_directories_jobs, nCores=self.defaultNCores))
### ###
# Task "ocropus_nlbin_job": binarize tiff files from previous split # Task "ocropus_nlbin_job": binarize tiff files from previous split
@ -153,7 +154,7 @@ class OCRWorkflow(WorkflowRunner):
os.path.join(job["output_dir"], "tmp", file), os.path.join(job["output_dir"], "tmp", file),
os.path.join(job["output_dir"], "tmp", "page-%i.%s" % (int(file.split(".", 1)[0]), file.split(".", 1)[1])), os.path.join(job["output_dir"], "tmp", "page-%i.%s" % (int(file.split(".", 1)[0]), file.split(".", 1)[1])),
) )
post_binarization_jobs.append(self.addTask(label="post_binarization_job_-_%i" % (post_binarization_job_number), command=cmd, dependencies=binarization_jobs)) post_binarization_jobs.append(self.addTask(label="post_binarization_job_-_%i" % (post_binarization_job_number), command=cmd, dependencies=binarization_jobs, nCores=self.defaultNCores))
### ###
# Task "ocr_job": perform OCR # Task "ocr_job": perform OCR
@ -196,7 +197,7 @@ class OCRWorkflow(WorkflowRunner):
os.path.join(job["output_dir"], "tmp"), os.path.join(job["output_dir"], "tmp"),
os.path.join(job["output_dir"], job["filename"].rsplit(".", 1)[0] + ".xml") os.path.join(job["output_dir"], job["filename"].rsplit(".", 1)[0] + ".xml")
) )
hocr_to_tei_jobs.append(self.addTask(label="hocr_to_tei_job_-_%i" % (hocr_to_tei_job_number), command=cmd, dependencies=ocr_jobs)) hocr_to_tei_jobs.append(self.addTask(label="hocr_to_tei_job_-_%i" % (hocr_to_tei_job_number), command=cmd, dependencies=ocr_jobs, nCores=self.defaultNCores))
### ###
# Task "pdf_merge_job": Merge PDF files # Task "pdf_merge_job": Merge PDF files
@ -210,7 +211,7 @@ class OCRWorkflow(WorkflowRunner):
os.path.join(job["output_dir"], "tmp"), os.path.join(job["output_dir"], "tmp"),
os.path.join(job["output_dir"], job["filename"].rsplit(".", 1)[0] + ".pdf") os.path.join(job["output_dir"], job["filename"].rsplit(".", 1)[0] + ".pdf")
) )
pdf_merge_jobs.append(self.addTask(label="pdf_merge_job_-_%i" % (pdf_merge_job_number), command=cmd, dependencies=ocr_jobs)) pdf_merge_jobs.append(self.addTask(label="pdf_merge_job_-_%i" % (pdf_merge_job_number), command=cmd, dependencies=ocr_jobs, nCores=self.defaultNCores))
### ###
# Task "txt_merge_job": Merge .txt files # Task "txt_merge_job": Merge .txt files
@ -224,11 +225,11 @@ class OCRWorkflow(WorkflowRunner):
os.path.join(job["output_dir"], "tmp"), os.path.join(job["output_dir"], "tmp"),
os.path.join(job["output_dir"], job["filename"].rsplit(".", 1)[0] + ".txt") os.path.join(job["output_dir"], job["filename"].rsplit(".", 1)[0] + ".txt")
) )
txt_merge_jobs.append(self.addTask(label="txt_merge_job_-_%i" % (txt_merge_job_number), command=cmd, dependencies=ocr_jobs)) txt_merge_jobs.append(self.addTask(label="txt_merge_job_-_%i" % (txt_merge_job_number), command=cmd, dependencies=ocr_jobs, nCores=self.defaultNCores))
### ###
# Task "cleanup_job": remove temporary files # Task "cleanup_job": remove temporary files
# Dependencies: hocr_to_teip5_jobs + pdf_merge_jobs + txt_merge_jobs # Dependencies: hocr_to_tei_jobs + pdf_merge_jobs + txt_merge_jobs
### ###
cleanup_jobs = [] cleanup_jobs = []
cleanup_job_counter = 0 cleanup_job_counter = 0
@ -252,14 +253,14 @@ class OCRWorkflow(WorkflowRunner):
os.path.join(job["output_dir"], "tmp"), os.path.join(job["output_dir"], "tmp"),
os.path.join(job["output_dir"], "tmp", "normalized_png"), os.path.join(job["output_dir"], "tmp", "normalized_png"),
) )
cleanup_jobs.append(self.addTask(label="cleanup_job_-_%i" % (cleanup_job_counter), command=cmd, dependencies=hocr_to_tei_jobs + pdf_merge_jobs + txt_merge_jobs)) cleanup_jobs.append(self.addTask(label="cleanup_job_-_%i" % (cleanup_job_counter), command=cmd, dependencies=hocr_to_tei_jobs + pdf_merge_jobs + txt_merge_jobs, nCores=self.defaultNCores))
else: else:
for job in self.jobs: for job in self.jobs:
cleanup_job_counter += 1 cleanup_job_counter += 1
cmd = 'rm -r "%s"' % ( cmd = 'rm -r "%s"' % (
os.path.join(job["output_dir"], "tmp") os.path.join(job["output_dir"], "tmp")
) )
cleanup_jobs.append(self.addTask(label="cleanup_job_-_%i" % (cleanup_job_counter), command=cmd, dependencies=hocr_to_tei_jobs + pdf_merge_jobs + txt_merge_jobs)) cleanup_jobs.append(self.addTask(label="cleanup_job_-_%i" % (cleanup_job_counter), command=cmd, dependencies=hocr_to_tei_jobs + pdf_merge_jobs + txt_merge_jobs), nCores=self.defaultNCores)
def analyze_jobs(inputDir, outputDir): def analyze_jobs(inputDir, outputDir):