From efbf6f24e6d95db375d66d85570682da11bdec98 Mon Sep 17 00:00:00 2001 From: Patrick Jentsch Date: Thu, 25 Apr 2019 11:40:27 +0200 Subject: [PATCH] Update --- Dockerfile | 18 ++++++++++-------- ocr | 19 ++++++++++--------- 2 files changed, 20 insertions(+), 17 deletions(-) diff --git a/Dockerfile b/Dockerfile index 169c358..579a09c 100644 --- a/Dockerfile +++ b/Dockerfile @@ -42,18 +42,20 @@ RUN wget -nv https://github.com/Illumina/pyflow/releases/download/v"$PYFLOW_VERS # Install Tesseract OCR and Data Files RUN echo "deb https://notesalexp.org/tesseract-ocr/stretch/ stretch main" >> /etc/apt/sources.list && \ + echo "deb https://notesalexp.org/tesseract-ocr/tessdata_best/ stretch main" >> /etc/apt/sources.list && \ + echo "deb https://notesalexp.org/tesseract-ocr/tessdata_legacy/ stretch main" >> /etc/apt/sources.list && \ wget -O - https://notesalexp.org/debian/alexp_key.asc | apt-key add - && \ apt-get update && \ apt-get install -y --no-install-recommends \ tesseract-ocr \ - tesseract-ocr-deu \ - tesseract-ocr-frk \ - tesseract-ocr-eng \ - tesseract-ocr-enm \ - tesseract-ocr-fra \ - tesseract-ocr-frm \ - tesseract-ocr-por \ - tesseract-ocr-spa + tesseract-ocr-deu-best \ + tesseract-ocr-deu-frak-legacy \ + tesseract-ocr-eng-best \ + tesseract-ocr-enm-best \ + tesseract-ocr-fra-best \ + tesseract-ocr-frm-best \ + tesseract-ocr-por-best \ + tesseract-ocr-spa-best COPY ocr /usr/local/bin COPY hocrtotei /usr/local/bin diff --git a/ocr b/ocr index b2c3032..f672a4a 100755 --- a/ocr +++ b/ocr @@ -73,6 +73,7 @@ class OCRWorkflow(WorkflowRunner): self.keepIntermediates = keepIntermediates self.lang = lang self.nCores = nCores + self.defaultNCores = min(nCores, max(1, int(nCores / len(jobs)))) def workflow(self): @@ -95,7 +96,7 @@ class OCRWorkflow(WorkflowRunner): os.path.join(job["output_dir"], "tmp", "binarized_png"), os.path.join(job["output_dir"], "tmp", "normalized_png"), ) - create_output_directories_jobs.append(self.addTask(label="create_output_directories_job_-_%i" % (create_output_directories_job_number), command=cmd)) + create_output_directories_jobs.append(self.addTask(label="create_output_directories_job_-_%i" % (create_output_directories_job_number), command=cmd, nCores=self.defaultNCores)) ### # Task "split_job": split input file into one tiff file per page @@ -115,7 +116,7 @@ class OCRWorkflow(WorkflowRunner): job["path"], os.path.join(job["output_dir"], "tmp", "page") ) - split_jobs.append(self.addTask(label="split_job_-_%i" % (split_job_number), command=cmd, dependencies=create_output_directories_jobs)) + split_jobs.append(self.addTask(label="split_job_-_%i" % (split_job_number), command=cmd, dependencies=create_output_directories_jobs, nCores=self.defaultNCores)) ### # Task "ocropus_nlbin_job": binarize tiff files from previous split @@ -153,7 +154,7 @@ class OCRWorkflow(WorkflowRunner): os.path.join(job["output_dir"], "tmp", file), os.path.join(job["output_dir"], "tmp", "page-%i.%s" % (int(file.split(".", 1)[0]), file.split(".", 1)[1])), ) - post_binarization_jobs.append(self.addTask(label="post_binarization_job_-_%i" % (post_binarization_job_number), command=cmd, dependencies=binarization_jobs)) + post_binarization_jobs.append(self.addTask(label="post_binarization_job_-_%i" % (post_binarization_job_number), command=cmd, dependencies=binarization_jobs, nCores=self.defaultNCores)) ### # Task "ocr_job": perform OCR @@ -196,7 +197,7 @@ class OCRWorkflow(WorkflowRunner): os.path.join(job["output_dir"], "tmp"), os.path.join(job["output_dir"], job["filename"].rsplit(".", 1)[0] + ".xml") ) - hocr_to_tei_jobs.append(self.addTask(label="hocr_to_tei_job_-_%i" % (hocr_to_tei_job_number), command=cmd, dependencies=ocr_jobs)) + hocr_to_tei_jobs.append(self.addTask(label="hocr_to_tei_job_-_%i" % (hocr_to_tei_job_number), command=cmd, dependencies=ocr_jobs, nCores=self.defaultNCores)) ### # Task "pdf_merge_job": Merge PDF files @@ -210,7 +211,7 @@ class OCRWorkflow(WorkflowRunner): os.path.join(job["output_dir"], "tmp"), os.path.join(job["output_dir"], job["filename"].rsplit(".", 1)[0] + ".pdf") ) - pdf_merge_jobs.append(self.addTask(label="pdf_merge_job_-_%i" % (pdf_merge_job_number), command=cmd, dependencies=ocr_jobs)) + pdf_merge_jobs.append(self.addTask(label="pdf_merge_job_-_%i" % (pdf_merge_job_number), command=cmd, dependencies=ocr_jobs, nCores=self.defaultNCores)) ### # Task "txt_merge_job": Merge .txt files @@ -224,11 +225,11 @@ class OCRWorkflow(WorkflowRunner): os.path.join(job["output_dir"], "tmp"), os.path.join(job["output_dir"], job["filename"].rsplit(".", 1)[0] + ".txt") ) - txt_merge_jobs.append(self.addTask(label="txt_merge_job_-_%i" % (txt_merge_job_number), command=cmd, dependencies=ocr_jobs)) + txt_merge_jobs.append(self.addTask(label="txt_merge_job_-_%i" % (txt_merge_job_number), command=cmd, dependencies=ocr_jobs, nCores=self.defaultNCores)) ### # Task "cleanup_job": remove temporary files - # Dependencies: hocr_to_teip5_jobs + pdf_merge_jobs + txt_merge_jobs + # Dependencies: hocr_to_tei_jobs + pdf_merge_jobs + txt_merge_jobs ### cleanup_jobs = [] cleanup_job_counter = 0 @@ -252,14 +253,14 @@ class OCRWorkflow(WorkflowRunner): os.path.join(job["output_dir"], "tmp"), os.path.join(job["output_dir"], "tmp", "normalized_png"), ) - cleanup_jobs.append(self.addTask(label="cleanup_job_-_%i" % (cleanup_job_counter), command=cmd, dependencies=hocr_to_tei_jobs + pdf_merge_jobs + txt_merge_jobs)) + cleanup_jobs.append(self.addTask(label="cleanup_job_-_%i" % (cleanup_job_counter), command=cmd, dependencies=hocr_to_tei_jobs + pdf_merge_jobs + txt_merge_jobs, nCores=self.defaultNCores)) else: for job in self.jobs: cleanup_job_counter += 1 cmd = 'rm -r "%s"' % ( os.path.join(job["output_dir"], "tmp") ) - cleanup_jobs.append(self.addTask(label="cleanup_job_-_%i" % (cleanup_job_counter), command=cmd, dependencies=hocr_to_tei_jobs + pdf_merge_jobs + txt_merge_jobs)) + cleanup_jobs.append(self.addTask(label="cleanup_job_-_%i" % (cleanup_job_counter), command=cmd, dependencies=hocr_to_tei_jobs + pdf_merge_jobs + txt_merge_jobs), nCores=self.defaultNCores) def analyze_jobs(inputDir, outputDir):