From ac4b5c2fd85c998d9333856a8fdfc4a087c7dedd Mon Sep 17 00:00:00 2001
From: Patrick Jentsch <pjentsch@sfb1288inf-Notebook.fritz.box>
Date: Tue, 22 Sep 2020 17:44:32 +0200
Subject: [PATCH] Add possibility to use an intermediate dir

---
 Dockerfile  |   4 +-
 ocr         | 329 +++++++++++++++++++++++++++-------------------------
 wrapper/ocr |   7 ++
 3 files changed, 184 insertions(+), 156 deletions(-)

diff --git a/Dockerfile b/Dockerfile
index 3ed80ef..2a299fc 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -76,8 +76,10 @@ RUN chmod 644 /usr/local/share/tessdata/*.traineddata
 ## Install Pipeline ##
 RUN apt-get install -y --no-install-recommends \
       ghostscript \
+      python-pip \
       python3.7 \
-      zip
+      zip \
+ && pip install natsort
 COPY "hocrtotei" "ocr" "/usr/local/bin/"
 
 
diff --git a/ocr b/ocr
index 10ee021..ab50ebf 100755
--- a/ocr
+++ b/ocr
@@ -12,11 +12,12 @@ Authors: Patrick Jentsch <p.jentsch@uni-bielefeld.de
 
 
 from argparse import ArgumentParser
+from natsort import natsorted
 from pyflow import WorkflowRunner
 import multiprocessing
 import os
-import re
 import sys
+import tempfile
 
 
 TESSERACT_MODELS = ['deu', 'eng', 'enm', 'fra', 'frk', 'frm', 'ita', 'por',
@@ -42,6 +43,7 @@ def parse_args():
                         default=min(4, multiprocessing.cpu_count()),
                         help='Total number of cores available.',
                         type=int)
+    parser.add_argument('--intermediate-directory')
     parser.add_argument('--zip',
                         help='Zips all results in different archives depending'
                              ' on result types. Also zips everything into one '
@@ -50,20 +52,49 @@ def parse_args():
 
 
 class OCRPipelineJob:
-    def __init__(self, file, output_dir):
+    """An OCR pipeline job class
+
+    Each input file of the pipeline is represented as an OCR pipeline job,
+    which holds all necessary information for the pipeline to process it.
+
+    Arguments:
+    file -- Path to the file
+    output_dir -- Path to a directory, where job results a stored
+    intermediate_dir -- Path to a directory, where intermediate files are
+                        stored.
+    """
+
+    def __init__(self, file, output_dir, intermediate_dir):
         self.file = file
+        self.intermediate_dir = intermediate_dir
         self.name = os.path.basename(file).rsplit('.', 1)[0]
         self.output_dir = output_dir
 
 
 class OCRPipeline(WorkflowRunner):
-    def __init__(self, binarize, jobs, lang, n_cores, output_dir, zip):
-        self.binarize = binarize
-        self.jobs = jobs
+    def __init__(self, input_dir, lang, output_dir, binarize, intermediate_dir,
+                 n_cores, zip):
+        self.input_dir = input_dir
         self.lang = lang
-        self.n_cores = n_cores
         self.output_dir = output_dir
-        self.zip = zip
+        self.binarize = binarize
+        if intermediate_dir is None:
+            self.intermediate_dir = os.path.join(output_dir, 'tmp')
+        else:
+            self.intermediate_dir = tempfile.mkdtemp(dir=intermediate_dir)
+        self.n_cores = n_cores
+        if zip is None:
+            self.zip = zip
+        else:
+            if zip.lower().endswith('.zip'):
+                # Remove .zip file extension if provided
+                self.zip = zip[:-4]
+                self.zip = self.zip if self.zip else 'output'
+            else:
+                self.zip = zip
+        self.jobs = collect_jobs(self.input_dir,
+                                 self.output_dir,
+                                 self.intermediate_dir)
 
     def workflow(self):
         if not self.jobs:
@@ -74,26 +105,26 @@ class OCRPipeline(WorkflowRunner):
         ' # setup output directory                         #
         ' ##################################################
         '''
-        setup_output_directory_jobs = []
+        setup_output_directory_tasks = []
         for i, job in enumerate(self.jobs):
-            intermediate_dir = os.path.join(job.output_dir, 'tmp')
             cmd = 'mkdir'
             cmd += ' -p'
-            cmd += ' "{}"'.format(intermediate_dir)
+            cmd += ' "{}"'.format(job.intermediate_dir)
             cmd += ' "{}"'.format(os.path.join(job.output_dir, 'poco'))
             lbl = 'setup_output_directory_-_{}'.format(i)
-            setup_output_directory_jobs.append(self.addTask(command=cmd,
-                                                            label=lbl))
+            task = self.addTask(command=cmd, label=lbl)
+            setup_output_directory_tasks.append(task)
 
         '''
         ' ##################################################
         ' # split input                                    #
         ' ##################################################
         '''
-        split_input_jobs = []
+        split_input_tasks = []
         n_cores = min(self.n_cores, max(1, int(self.n_cores / len(self.jobs))))
         for i, job in enumerate(self.jobs):
-            output_dir = os.path.join(job.output_dir, 'tmp')
+            input_file = job.file
+            output_file = '{}/page-%d.tif'.format(job.intermediate_dir)
             cmd = 'gs'
             cmd += ' -dBATCH'
             cmd += ' -dNOPAUSE'
@@ -102,19 +133,17 @@ class OCRPipeline(WorkflowRunner):
             cmd += ' -r300'
             cmd += ' -sDEVICE=tiff24nc'
             cmd += ' -sCompression=lzw'
-            cmd += ' "-sOutputFile={}/page-%d.tif"'.format(output_dir)
-            cmd += ' "{}"'.format(job.file)
+            cmd += ' "-sOutputFile={}"'.format(output_file)
+            cmd += ' "{}"'.format(input_file)
             deps = 'setup_output_directory_-_{}'.format(i)
             lbl = 'split_input_-_{}'.format(i)
-            split_input_jobs.append(self.addTask(command=cmd,
-                                                 dependencies=deps,
-                                                 label=lbl,
-                                                 nCores=n_cores))
+            task = self.addTask(command=cmd, dependencies=deps, label=lbl, nCores=n_cores)  # noqa
+            split_input_tasks.append(task)
 
         if self.binarize:
             '''
-            ' The binarization_jobs list is created based on the output files
-            ' of the split_jobs. So wait until they are finished.
+            ' The binarization_tasks list is created based on the output files
+            ' of the split_tasks. So wait until they are finished.
             '''
             self.waitForTasks()
 
@@ -123,7 +152,7 @@ class OCRPipeline(WorkflowRunner):
             ' # binarization                                   #
             ' ##################################################
             '''
-            binarization_jobs = []
+            binarization_tasks = []
             '''
             ' We run ocropus-nlbin with either four or, if there are less then
             ' four cores available for this workflow, the available core
@@ -131,27 +160,21 @@ class OCRPipeline(WorkflowRunner):
             '''
             n_cores = min(4, self.n_cores)
             for i, job in enumerate(self.jobs):
-                input_dir = os.path.join(job.output_dir, 'tmp')
-                output_dir = input_dir
-                files = filter(lambda x: x.endswith('.tif'),
-                               os.listdir(input_dir))
-                files.sort(key=lambda x: int(re.search(r'\d+', x).group(0)))
+                input_dir = job.intermediate_dir
+                output_dir = job.intermediate_dir
+                files = filter(lambda x: x.endswith('.tif'), os.listdir(input_dir))  # noqa
+                files = natsorted(files)
                 files = map(lambda x: os.path.join(input_dir, x), files)
                 cmd = 'ocropus-nlbin "{}"'.format('" "'.join(files))
                 cmd += ' --nocheck'
                 cmd += ' --output "{}"'.format(output_dir)
                 cmd += ' --parallel "{}"'.format(n_cores)
+                print(cmd)
                 deps = 'split_input_-_{}'.format(i)
                 lbl = 'binarization_-_{}'.format(i)
-                binarization_jobs.append(self.addTask(command=cmd,
-                                                      dependencies=deps,
-                                                      label=lbl,
-                                                      nCores=n_cores))
+                task = self.addTask(command=cmd, dependencies=deps, label=lbl, nCores=n_cores)  # noqa
+                binarization_tasks.append(task)
 
-            '''
-            ' The post_binarization_jobs are created based on the output files
-            ' of the binarization_jobs. So wait until they are finished.
-            '''
             self.waitForTasks()
 
             '''
@@ -160,10 +183,9 @@ class OCRPipeline(WorkflowRunner):
             ' ##################################################
             '''
             for i, job in enumerate(self.jobs):
-                input_dir = os.path.join(job.output_dir, 'tmp')
-                output_dir = input_dir
-                files = filter(lambda x: x.endswith('.bin.png'),
-                               os.listdir(input_dir))
+                input_dir = job.intermediate_dir
+                output_dir = job.intermediate_dir
+                files = filter(lambda x: x.endswith('.bin.png'), os.listdir(input_dir))  # noqa
                 for file in files:
                     # int conversion is done in order to trim leading zeros
                     page_number = int(file.split('.', 1)[0])
@@ -172,8 +194,8 @@ class OCRPipeline(WorkflowRunner):
                               os.path.join(output_dir, output_file))
 
         '''
-        ' The ocr_jobs are created based of the output files of either the
-        ' split_jobs or post_binarization_jobs. So wait until they are
+        ' The ocr_tasks are created based of the output files of either the
+        ' split_tasks or binarization_tasks. So wait until they are
         ' finished.
         '''
         self.waitForTasks()
@@ -183,7 +205,7 @@ class OCRPipeline(WorkflowRunner):
         ' # ocr                                            #
         ' ##################################################
         '''
-        ocr_jobs = []
+        ocr_tasks = []
         '''
         ' Tesseract runs fastest with four cores. So we run it with either four
         ' or, if there are less then four cores available for this workflow,
@@ -191,33 +213,34 @@ class OCRPipeline(WorkflowRunner):
         '''
         n_cores = min(4, self.n_cores)
         for i, job in enumerate(self.jobs):
-            input_dir = os.path.join(job.output_dir, 'tmp')
-            output_dir = input_dir
-            files = filter(lambda x: x.endswith('.bin.png' if self.binarize else '.tif'), os.listdir(input_dir))  # noqa
-            files.sort(key=lambda x: int(re.search(r'\d+', x).group(0)))
+            input_dir = job.intermediate_dir
+            output_dir = job.intermediate_dir
+            files = os.listdir(input_dir)
+            if self.binarize:
+                deps = 'binarization_-_{}'.format(i)
+                files = filter(lambda x: x.endswith('.bin.png'), files)
+            else:
+                deps = 'split_input_-_{}'.format(i)
+                files = filter(lambda x: x.endswith('.tif'), files)
+            files = natsorted(files)
             files = map(lambda x: os.path.join(input_dir, x), files)
-            number = 0
-            for file in files:
-                output_file_base = os.path.join(output_dir, file.rsplit('.', 2 if self.binarize else 1)[0])  # noqa
+            for j, file in enumerate(files):
+                if self.binarize:
+                    output_file_base = os.path.join(output_dir, file.rsplit('.', 2)[0])  # noqa
+                else:
+                    output_file_base = os.path.join(output_dir, file.rsplit('.', 1)[0])  # noqa
                 cmd = 'tesseract "{}" "{}"'.format(file, output_file_base)
                 cmd += ' -l "{}"'.format(self.lang)
                 cmd += ' hocr pdf txt'
                 cmd += ' && '
                 cmd += 'sed -i \'s+{}/++g\' "{}".hocr'.format(input_dir, output_file_base)  # noqa
-                if self.binarize:
-                    deps = 'binarization_-_{}'.format(i)
-                else:
-                    deps = 'split_input_-_{}'.format(i)
-                label = 'ocr_-_{}-{}'.format(i, number)
-                ocr_jobs.append(self.addTask(command=cmd,
-                                             dependencies=deps,
-                                             label=label,
-                                             nCores=n_cores))
-                number += 1
+                lbl = 'ocr_-_{}-{}'.format(i, j)
+                task = self.addTask(command=cmd, dependencies=deps, label=lbl, nCores=n_cores)  # noqa
+                ocr_tasks.append(task)
 
         '''
         ' The following jobs are created based of the output files of the
-        ' ocr_jobs. So wait until they are finished.
+        ' ocr_tasks. So wait until they are finished.
         '''
         self.waitForTasks()
 
@@ -226,16 +249,14 @@ class OCRPipeline(WorkflowRunner):
         ' # combined pdf creation                          #
         ' ##################################################
         '''
-        combined_pdf_creation_jobs = []
+        combined_pdf_creation_tasks = []
         n_cores = min(self.n_cores, max(1, int(self.n_cores / len(self.jobs))))
         for i, job in enumerate(self.jobs):
-            input_dir = os.path.join(job.output_dir, 'tmp')
-            output_dir = job.output_dir
-            files = filter(lambda x: x.endswith('.pdf'),
-                           os.listdir(input_dir))
-            files.sort(key=lambda x: int(re.search(r'\d+', x).group(0)))
+            input_dir = job.intermediate_dir
+            output_file = os.path.join(job.output_dir, '{}.pdf'.format(job.name))  # noqa
+            files = filter(lambda x: x.endswith('.pdf'), os.listdir(input_dir))
+            files = natsorted(files)
             files = map(lambda x: os.path.join(input_dir, x), files)
-            output_file = os.path.join(output_dir, '{}.pdf'.format(job.name))
             cmd = 'gs'
             cmd += ' -dBATCH'
             cmd += ' -dNOPAUSE'
@@ -245,77 +266,75 @@ class OCRPipeline(WorkflowRunner):
             cmd += ' -sDEVICE=pdfwrite'
             cmd += ' "-sOutputFile={}"'.format(output_file)
             cmd += ' "{}"'.format('" "'.join(files))
-            deps = filter(lambda x: x.startswith('ocr_-_{}'.format(i)),
-                          ocr_jobs)
+            deps = filter(lambda x: x.startswith('ocr_-_{}'.format(i)), ocr_tasks)  # noqa
             lbl = 'combined_pdf_creation_-_{}'.format(i)
-            combined_pdf_creation_jobs.append(self.addTask(command=cmd,
-                                                           dependencies=deps,
-                                                           label=lbl,
-                                                           nCores=n_cores))
+            task = self.addTask(command=cmd, dependencies=deps, label=lbl, nCores=n_cores)  # noqa
+            combined_pdf_creation_tasks.append(task)
 
         '''
         ' ##################################################
         ' # combined txt creation                          #
         ' ##################################################
         '''
-        combined_txt_creation_jobs = []
+        combined_txt_creation_tasks = []
         for i, job in enumerate(self.jobs):
-            input_dir = os.path.join(job.output_dir, 'tmp')
-            files = filter(lambda x: x.endswith('.txt'), os.listdir(input_dir))
-            files.sort(key=lambda x: int(re.search(r'\d+', x).group(0)))
-            files = map(lambda x: os.path.join(input_dir, x), files)
+            input_dir = job.intermediate_dir
             output_file = os.path.join(job.output_dir, '{}.txt'.format(job.name))  # noqa
+            files = filter(lambda x: x.endswith('.txt'), os.listdir(input_dir))
+            files = natsorted(files)
+            files = map(lambda x: os.path.join(input_dir, x), files)
             cmd = 'cat "{}" > "{}"'.format('" "'.join(files), output_file)
-            deps = filter(lambda x: x.startswith('ocr_-_{}'.format(i)),
-                          ocr_jobs)
+            deps = filter(lambda x: x.startswith('ocr_-_{}'.format(i)), ocr_tasks)  # noqa
             lbl = 'combined_txt_creation_-_{}'.format(i)
-            combined_txt_creation_jobs.append(self.addTask(command=cmd,
-                                                           dependencies=deps,
-                                                           label=lbl))
+            task = self.addTask(command=cmd, dependencies=deps, label=lbl)
+            combined_txt_creation_tasks.append(task)
 
         '''
         ' ##################################################
         ' # tei p5 creation                                #
         ' ##################################################
         '''
-        tei_p5_creation_jobs = []
+        tei_p5_creation_tasks = []
         for i, job in enumerate(self.jobs):
-            input_dir = os.path.join(job.output_dir, 'tmp')
-            files = filter(lambda x: x.endswith('.hocr'), os.listdir(input_dir))  # noqa
-            files.sort(key=lambda x: int(re.search(r'\d+', x).group(0)))
-            files = map(lambda x: os.path.join(input_dir, x), files)
+            input_dir = job.intermediate_dir
             output_file = os.path.join(job.output_dir, '{}.xml'.format(job.name))  # noqa
-            cmd = 'hocrtotei "{}" "{}"'.format('" "'.join(files), output_file)
-            deps = filter(lambda x: x.startswith('ocr_-_{}'.format(i)),
-                          ocr_jobs)
+            files = filter(lambda x: x.endswith('.hocr'),
+                           os.listdir(input_dir))
+            files = natsorted(files)
+            files = map(lambda x: os.path.join(input_dir, x), files)
+            cmd = 'hocrtotei "{}" "{}"'.format('" "'.join(files),
+                                               output_file)
+            deps = filter(lambda x: x.startswith('ocr_-_{}'.format(i)), ocr_tasks)  # noqa
             lbl = 'tei_p5_creation_-_{}'.format(i)
-            tei_p5_creation_jobs.append(self.addTask(command=cmd,
-                                                     dependencies=deps,
-                                                     label=lbl))
+            task = self.addTask(command=cmd, dependencies=deps, label=lbl)
+            tei_p5_creation_tasks.append(task)
 
         '''
         ' ##################################################
         ' # poco bundle creation                           #
         ' ##################################################
         '''
-        poco_bundle_creation_jobs = []
+        poco_bundle_creation_tasks = []
         for i, job in enumerate(self.jobs):
-            input_dir = os.path.join(job.output_dir, 'tmp')
+            input_dir = job.intermediate_dir
             output_dir = os.path.join(job.output_dir, 'poco')
-            cmd = 'mv "{}"/*.hocr "{}"'.format(input_dir, output_dir)
-            cmd += ' && '
-            cmd += 'mv "{}"/*.{} "{}"'.format(input_dir, 'bin.png' if self.binarize else 'tif', output_dir)  # noqa
-            deps = filter(lambda x: x.startswith('ocr_-_{}'.format(i)),
-                          ocr_jobs)
+            files = os.listdir(input_dir)
+            if self.binarize:
+                files = filter(lambda x: x.endswith(('.bin.png', '.hocr')), files)  # noqa
+            else:
+                files = filter(lambda x: x.endswith(('.tif', '.hocr')), files)
+            files = natsorted(files)
+            files = map(lambda x: os.path.join(input_dir, x), files)
+            cmd = 'mv "{}" "{}"'.format('" "'.join(files), output_dir)
+            deps = filter(lambda x: x.startswith('ocr_-_{}'.format(i)), ocr_tasks)  # noqa
             deps.append('tei_p5_creation_-_{}'.format(i))
             lbl = 'poco_bundle_creation_-_{}'.format(i)
-            poco_bundle_creation_jobs.append(self.addTask(command=cmd,
-                                                          dependencies=deps,
-                                                          label=lbl))
+            task = self.addTask(command=cmd, dependencies=deps, label=lbl)
+            poco_bundle_creation_tasks.append(task)
 
         '''
         ' The following jobs are created based of the output files of the
-        ' combined_pdf_creation_jobs. So wait until they are finished.
+        ' combined_pdf_creation_tasks. So wait until they are finished.
         '''
         self.waitForTasks()
 
@@ -324,126 +343,126 @@ class OCRPipeline(WorkflowRunner):
         ' # cleanup                                        #
         ' ##################################################
         '''
-        cleanup_jobs = []
+        cleanup_tasks = []
         for i, job in enumerate(self.jobs):
-            input_dir = os.path.join(job.output_dir, 'tmp')
+            input_dir = job.intermediate_dir
             cmd = 'rm -r "{}"'.format(input_dir)
-            deps = ['combined_pdf_creation_-_{}'.format(i)]
-            deps.append('combined_txt_creation_-_{}'.format(i))
-            deps.append('poco_bundle_creation_-_{}'.format(i))
-            deps.append('tei_p5_creation_-_{}'.format(i))
-            lbl = 'cleanup_-_{}'.format(i)
-            cleanup_jobs.append(self.addTask(command=cmd,
-                                             dependencies=deps,
-                                             label=lbl))
+            deps = ['combined_pdf_creation_-_{}'.format(i),
+                    'combined_txt_creation_-_{}'.format(i),
+                    'poco_bundle_creation_-_{}'.format(i),
+                    'tei_p5_creation_-_{}'.format(i)]
+            lbl = 'job_cleanup_-_{}'.format(i)
+            task = self.addTask(command=cmd, dependencies=deps, label=lbl)
+            cleanup_tasks.append(task)
+
+        input_dir = self.intermediate_dir
+        cmd = 'rm -r "{}"'.format(input_dir)
+        deps = cleanup_tasks
+        lbl = 'pipeline_cleanup'
+        task = self.addTask(command=cmd, dependencies=deps, label=lbl)
+        cleanup_tasks.append(task)
+
+        self.waitForTasks()
 
         '''
         ' ##################################################
         ' # zip creation                                   #
         ' ##################################################
         '''
-        zip_creation_jobs = []
+        zip_creation_tasks = []
         if self.zip is not None:
-            # Remove .zip file extension if provided
-            if self.zip.endswith('.zip'):
-                self.zip = self.zip[:-4]
-                self.zip = self.zip if self.zip else 'output'
             # zip all files
             cmd = 'cd "{}"'.format(self.output_dir)
             cmd += ' && '
             cmd += 'zip'
             cmd += ' -r'
-            cmd += ' "{}".all.zip .'.format(self.zip)
+            cmd += ' "{}.all.zip" .'.format(self.zip)
             cmd += ' -x "pyflow.data*" "*tmp*"'
             cmd += ' -i "*.pdf" "*.txt" "*.xml" "*.hocr" "*.{}"'.format('bin.png' if self.binarize else 'tif')  # noqa
             cmd += ' && '
             cmd += 'cd -'
-            deps = combined_pdf_creation_jobs
-            deps += combined_txt_creation_jobs
-            deps += poco_bundle_creation_jobs
+            deps = combined_pdf_creation_tasks + combined_txt_creation_tasks + poco_bundle_creation_tasks  # noqa
             lbl = 'zip_creation_-_all'
-            zip_creation_jobs.append(self.addTask(command=cmd,
-                                                  dependencies=deps,
-                                                  label=lbl))
+            task = self.addTask(command=cmd, dependencies=deps, label=lbl)
+            zip_creation_tasks.append(task)
             # zip PDF files
             cmd = 'cd "{}"'.format(self.output_dir)
             cmd += ' && '
             cmd += 'zip'
             cmd += ' -r'
-            cmd += ' "{}".pdf.zip .'.format(self.zip)
+            cmd += ' "{}.pdf.zip" .'.format(self.zip)
             cmd += ' -x "pyflow.data*" "*tmp*"'
             cmd += ' -i "*.pdf"'
             cmd += ' && '
             cmd += 'cd -'
-            deps = combined_pdf_creation_jobs
+            deps = combined_pdf_creation_tasks
             lbl = 'zip_creation_-_pdf'
-            zip_creation_jobs.append(self.addTask(command=cmd,
-                                                  dependencies=deps,
-                                                  label=lbl))
+            task = self.addTask(command=cmd, dependencies=deps, label=lbl)
+            zip_creation_tasks.append(task)
             # zip TXT files
             cmd = 'cd "{}"'.format(self.output_dir)
             cmd += ' && '
             cmd += 'zip'
             cmd += ' -r'
-            cmd += ' "{}".txt.zip .'.format(self.zip)
+            cmd += ' "{}.txt.zip" .'.format(self.zip)
             cmd += ' -x "pyflow.data*" "*tmp*"'
             cmd += ' -i "*.txt"'
             cmd += ' && '
             cmd += 'cd -'
-            deps = combined_txt_creation_jobs
+            deps = combined_txt_creation_tasks
             lbl = 'zip_creation_-_txt'
-            zip_creation_jobs.append(self.addTask(command=cmd,
-                                                  dependencies=deps,
-                                                  label=lbl))
+            task = self.addTask(command=cmd, dependencies=deps, label=lbl)
+            zip_creation_tasks.append(task)
             # zip XML files
             cmd = 'cd "{}"'.format(self.output_dir)
             cmd += ' && '
             cmd += 'zip'
             cmd += ' -r'
-            cmd += ' "{}".xml.zip .'.format(self.zip)
+            cmd += ' "{}.xml.zip" .'.format(self.zip)
             cmd += ' -x "pyflow.data*" "*tmp*"'
             cmd += ' -i "*.xml"'
             cmd += ' && '
             cmd += 'cd -'
-            deps = tei_p5_creation_jobs
+            deps = tei_p5_creation_tasks
             lbl = 'zip_creation_-_xml'
-            zip_creation_jobs.append(self.addTask(command=cmd,
-                                                  dependencies=deps,
-                                                  label=lbl))
+            task = self.addTask(command=cmd, dependencies=deps, label=lbl)
+            zip_creation_tasks.append(task)
             # zip PoCo bundles
             cmd = 'cd "{}"'.format(self.output_dir)
             cmd += ' && '
             cmd += 'zip'
             cmd += ' -r'
-            cmd += ' "{}".poco.zip .'.format(self.zip)
+            cmd += ' "{}.poco.zip" .'.format(self.zip)
             cmd += ' -x "pyflow.data*" "*tmp*"'
             cmd += ' -i "*.hocr" "*.{}"'.format('bin.png' if self.binarize else 'tif')  # noqa
             cmd += ' && '
             cmd += 'cd -'
-            deps = poco_bundle_creation_jobs
+            deps = poco_bundle_creation_tasks
             lbl = 'zip_creation_-_poco'
-            zip_creation_jobs.append(self.addTask(command=cmd,
-                                                  dependencies=deps,
-                                                  label=lbl))
+            task = self.addTask(command=cmd, dependencies=deps, label=lbl)
+            zip_creation_tasks.append(task)
 
 
-def collect_jobs(input_dir, output_dir):
+def collect_jobs(input_dir, output_dir, intermediate_dir):
     jobs = []
     for file in os.listdir(input_dir):
         if os.path.isdir(os.path.join(input_dir, file)):
             jobs += collect_jobs(os.path.join(input_dir, file),
                                  os.path.join(output_dir, file))
-        elif file.endswith('.pdf'):
-            jobs.append(OCRPipelineJob(os.path.join(input_dir, file),
-                                       os.path.join(output_dir, file)))
+        elif file.lower().endswith('.pdf'):
+            job = OCRPipelineJob(os.path.join(input_dir, file),
+                                 os.path.join(output_dir, file),
+                                 os.path.join(intermediate_dir, file))
+            jobs.append(job)
     return jobs
 
 
 def main():
     args = parse_args()
-    jobs = collect_jobs(args.input_directory, args.output_directory)
-    ocr_pipeline = OCRPipeline(args.binarize, jobs, args.language,
-                               args.n_cores, args.output_directory, args.zip)
+    ocr_pipeline = OCRPipeline(args.input_directory, args.language,
+                               args.output_directory, args.binarize,
+                               args.intermediate_directory, args.n_cores,
+                               args.zip)
     retval = ocr_pipeline.run(
         dataDirRoot=(args.log_dir or args.output_directory),
         nCores=args.n_cores
diff --git a/wrapper/ocr b/wrapper/ocr
index 5b3e68f..5c908a9 100755
--- a/wrapper/ocr
+++ b/wrapper/ocr
@@ -7,6 +7,7 @@ import subprocess
 
 CONTAINER_IMAGE = 'gitlab.ub.uni-bielefeld.de:4567/sfb1288inf/ocr:latest'
 CONTAINER_INPUT_DIR = '/input'
+CONTAINER_INTERMEDIATE_DIR = '/intermediate'
 CONTAINER_OUTPUT_DIR = '/output'
 UID = str(os.getuid())
 GID = str(os.getgid())
@@ -14,9 +15,15 @@ GID = str(os.getgid())
 parser = ArgumentParser(add_help=False)
 parser.add_argument('-i', '--input-directory')
 parser.add_argument('-o', '--output-directory')
+parser.add_argument('--intermediate-directory')
 args, remaining_args = parser.parse_known_args()
 
 cmd = ['docker', 'run', '--rm', '-it', '-u', '{}:{}'.format(UID, GID)]
+if args.intermediate_directory is not None:
+    cmd += ['-v', '{}:{}'.format(os.path.abspath(args.intermediate_directory),
+                                 CONTAINER_INTERMEDIATE_DIR)]
+    remaining_args.insert(0, CONTAINER_INTERMEDIATE_DIR)
+    remaining_args.insert(0, '--intermediate-directory')
 if args.output_directory is not None:
     cmd += ['-v', '{}:{}'.format(os.path.abspath(args.output_directory),
                                  CONTAINER_OUTPUT_DIR)]