fix pipeline

ocr

@@ -24,33 +24,30 @@ TESSERACT_MODELS = ['deu', 'eng', 'enm', 'fra', 'frk', 'frm', 'ita', 'por',
 
 def parse_args():
     parser = ArgumentParser(description='OCR Pipeline utilizing tesseract.')
-    parser.add_argument('i', help='Input directory for OCR. One PDf equals one\
+    parser.add_argument('-i', '--input-directory',
-                                   job')
+                        help='Input directory (only PDF files get processed)',
-    parser.add_argument('o', help='Output directory containing OCR results.')
+                        required=True)
+    parser.add_argument('-o', '--output-directory',
+                        help='Output directory',
+                        required=True)
     parser.add_argument('-l', '--language',
                         choices=TESSERACT_MODELS,
                         required=True)
     parser.add_argument('--binarize',
                         action='store_true',
                         help='Use ocropy binarisation as preprocessing step.')
-    parser.add_argument('--keep-intermediates',
+    parser.add_argument('--compress',
                         action='store_true',
-                        help='Keep intermediate files for debugging etc.',
+                        help='Compress the final PDF result file.')
-                        required=False)
+    parser.add_argument('--log-dir')
     parser.add_argument('--n-cores',
                         default=min(4, multiprocessing.cpu_count()),
                         help='Total number of cores available.',
-                        type=int,
+                        type=int)
-                        required=False)
+    parser.add_argument('--zip',
-    parser.add_argument('--zip', help='Zips all results in different archives \
+                        help='Zips all results in different archives depending'
-                                       depending on result types. Also zips   \
+                             ' on result types. Also zips everything into one '
-                                       everything into one archive.',
+                             'archive.')
-                        required=False)
-    parser.add_argument('-c', '--compress',
-                        help='Compress the final PDF result file.',
-                        required=False,
-                        action='store_true')
-    parser.add_argument('--log_dir')
     return parser.parse_args()
 
 
@@ -62,11 +59,10 @@ class OCRPipelineJob:
 
 
 class OCRPipeline(WorkflowRunner):
-    def __init__(self, binarize, jobs, keep_intermediates, lang, n_cores,
+    def __init__(self, binarize, jobs, lang, n_cores, output_dir, zip,
-                 output_dir, zip, compress):
+                 compress):
         self.binarize = binarize
         self.jobs = jobs
-        self.keep_intermediates = keep_intermediates
         self.lang = lang
         self.n_cores = n_cores
         self.output_dir = output_dir
@@ -79,36 +75,26 @@ class OCRPipeline(WorkflowRunner):
 
         '''
         ' ##################################################
-        ' # mkdir_jobs                                     #
+        ' # setup output directory                         #
         ' ##################################################
         '''
-
+        setup_output_directory_jobs = []
-        mkdir_jobs = []
         for i, job in enumerate(self.jobs):
-            output_dir = os.path.join(job.output_dir, 'tmp')
+            intermediate_dir = os.path.join(job.output_dir, 'tmp')
-            poco_dir = os.path.join(job.output_dir, 'PoCo')
             cmd = 'mkdir'
             cmd += ' -p'
-            cmd += ' "{}"'.format(output_dir)
+            cmd += ' "{}"'.format(intermediate_dir)
-            cmd += ' "{}"'.format(os.path.join(poco_dir, 'hocr'))
+            cmd += ' "{}"'.format(os.path.join(job.output_dir, 'poco'))
-            cmd += ' "{}"'.format(os.path.join(poco_dir, 'tiff'))
+            lbl = 'setup_output_directory_-_{}'.format(i)
-            if self.keep_intermediates:
+            setup_output_directory_jobs.append(self.addTask(command=cmd,
-                cmd += ' "{}"'.format(os.path.join(output_dir, 'hocr'))
+                                                            label=lbl))
-                cmd += ' "{}"'.format(os.path.join(output_dir, 'pdf'))
-                cmd += ' "{}"'.format(os.path.join(output_dir, 'tiff'))
-                cmd += ' "{}"'.format(os.path.join(output_dir, 'txt'))
-                if self.binarize:
-                    cmd += ' "{}"'.format(os.path.join(output_dir, 'bin.png'))
-                    cmd += ' "{}"'.format(os.path.join(output_dir, 'nrm.png'))
-            lbl = 'mkdir_job_-_{}'.format(i)
-            mkdir_jobs.append(self.addTask(command=cmd, label=lbl))
 
         '''
         ' ##################################################
-        ' # pdftoppm_jobs                                  #
+        ' # split input                                    #
         ' ##################################################
         '''
-        pdftoppm_jobs = []
+        split_input_jobs = []
         n_cores = min(self.n_cores, max(1, int(self.n_cores / len(self.jobs))))
         for i, job in enumerate(self.jobs):
             output_dir = os.path.join(job.output_dir, 'tmp')
@@ -118,24 +104,26 @@ class OCRPipeline(WorkflowRunner):
             cmd += ' -tiff'
             cmd += ' -tiffcompression lzw'
             cmd += ' "{}" "{}"'.format(job.file, output_file_base)
-            deps = mkdir_jobs
+            deps = 'setup_output_directory_-_{}'.format(i)
-            lbl = 'pdftoppm_job_-_{}'.format(i)
+            lbl = 'split_input_-_{}'.format(i)
-            pdftoppm_jobs.append(self.addTask(command=cmd, dependencies=deps,
+            split_input_jobs.append(self.addTask(command=cmd,
-                                              label=lbl, nCores=n_cores))
+                                                 dependencies=deps,
+                                                 label=lbl,
+                                                 nCores=n_cores))
 
         if self.binarize:
             '''
-            ' The ocropus_nlbin_jobs list is created based on the output files
+            ' The binarization_jobs list is created based on the output files
-            ' of the pdftoppm_jobs. So wait until they are finished.
+            ' of the split_jobs. So wait until they are finished.
             '''
             self.waitForTasks()
 
             '''
             ' ##################################################
-            ' # ocropus_nlbin_jobs                             #
+            ' # binarization                                   #
             ' ##################################################
             '''
-            ocropus_nlbin_jobs = []
+            binarization_jobs = []
             '''
             ' We run ocropus-nlbin with either four or, if there are less then
             ' four cores available for this workflow, the available core
@@ -152,24 +140,25 @@ class OCRPipeline(WorkflowRunner):
                 cmd = 'ocropus-nlbin "{}"'.format('" "'.join(files))
                 cmd += ' -o "{}"'.format(output_dir)
                 cmd += ' -Q "{}"'.format(n_cores)
-                deps = pdftoppm_jobs
+                deps = 'split_input_-_{}'.format(i)
-                lbl = 'ocropus_nlbin_job_-_{}'.format(i)
+                lbl = 'binarization_-_{}'.format(i)
-                ocropus_nlbin_jobs.append(
+                binarization_jobs.append(self.addTask(command=cmd,
-                    self.addTask(command=cmd, dependencies=deps, label=lbl,
+                                                      dependencies=deps,
+                                                      label=lbl,
                                                       nCores=n_cores))
 
             '''
-            ' The post_ocropus_nlbin_jobs are created based on the output files
+            ' The post_binarization_jobs are created based on the output files
-            ' of the ocropus_nlbin_jobs. So wait until they are finished.
+            ' of the binarization_jobs. So wait until they are finished.
             '''
             self.waitForTasks()
 
             '''
             ' ##################################################
-            ' # post_ocropus_nlbin_jobs                        #
+            ' # post binarization                              #
             ' ##################################################
             '''
-            post_ocropus_nlbin_jobs = []
+            post_binarization_jobs = []
             for i, job in enumerate(self.jobs):
                 input_dir = os.path.join(job.output_dir, 'tmp')
                 output_dir = input_dir
@@ -182,26 +171,26 @@ class OCRPipeline(WorkflowRunner):
                     output_file = os.path.join(output_dir, 'page-{}.bin.png'.format(int(file.split('.', 1)[0])))  # noqa
                     cmd = 'mv "{}" "{}"'.format(os.path.join(output_dir, file),
                                                 output_file)
-                    deps = ocropus_nlbin_jobs
+                    deps = 'binarization_-_{}'.format(i)
-                    lbl = 'post_ocropus_nlbin_job_-_{}-{}'.format(i, number)
+                    lbl = 'post_binarization_-_{}-{}'.format(i, number)
-                    post_ocropus_nlbin_jobs.append(
+                    post_binarization_jobs.append(
-                        self.addTask(command=cmd, dependencies=deps,
+                        self.addTask(command=cmd, dependencies=deps, label=lbl)
-                                     label=lbl))
+                    )
                     number += 1
 
         '''
-        ' The tesseract_jobs are created based of the output files of either
+        ' The ocr_jobs are created based of the output files of either the
-        ' the pdftoppm_jobs or post_ocropus_nlbin_jobs. So wait until they are
+        ' split_jobs or post_binarization_jobs. So wait until they are
         ' finished.
         '''
         self.waitForTasks()
 
         '''
         ' ##################################################
-        ' # tesseract_jobs                                 #
+        ' # ocr                                            #
         ' ##################################################
         '''
-        tesseract_jobs = []
+        ocr_jobs = []
         '''
         ' Tesseract runs fastest with four cores. So we run it with either four
         ' or, if there are less then four cores available for this workflow,
@@ -211,8 +200,7 @@ class OCRPipeline(WorkflowRunner):
         for i, job in enumerate(self.jobs):
             input_dir = os.path.join(job.output_dir, 'tmp')
             output_dir = input_dir
-            files = filter(lambda x: x.endswith('.bin.png' if self.binarize else '.tif'),  # noqa
+            files = filter(lambda x: x.endswith('.bin.png' if self.binarize else '.tif'), os.listdir(input_dir))  # noqa
-                           os.listdir(input_dir))
             files.sort(key=lambda x: int(re.search(r'\d+', x).group(0)))
             files = map(lambda x: os.path.join(input_dir, x), files)
             number = 0
@@ -221,13 +209,16 @@ class OCRPipeline(WorkflowRunner):
                 cmd = 'tesseract "{}" "{}"'.format(file, output_file_base)
                 cmd += ' -l "{}"'.format(self.lang)
                 cmd += ' hocr pdf txt'
+                cmd += ' && '
+                cmd += 'sed -i \'s+{}/++g\' "{}".hocr'.format(input_dir, output_file_base)  # noqa
                 if self.binarize:
-                    deps = post_ocropus_nlbin_jobs
+                    deps = 'post_binarization_-_{}-{}'.format(i, number)
                 else:
-                    deps = pdftoppm_jobs
+                    deps = 'split_input_-_{}'.format(i)
-                label = 'tesseract_jobs_-_{}-{}'.format(i, number)
+                label = 'ocr_-_{}-{}'.format(i, number)
-                tesseract_jobs.append(
+                ocr_jobs.append(self.addTask(command=cmd,
-                    self.addTask(command=cmd, dependencies=deps, label=label,
+                                             dependencies=deps,
+                                             label=label,
                                              nCores=n_cores))
                 number += 1
 
@@ -239,190 +230,171 @@ class OCRPipeline(WorkflowRunner):
 
         '''
         ' ##################################################
-        ' # hocrtotei_jobs                                 #
+        ' # combined pdf creation                          #
         ' ##################################################
         '''
-        hocrtotei_jobs = []
+        combined_pdf_creation_jobs = []
         for i, job in enumerate(self.jobs):
             input_dir = os.path.join(job.output_dir, 'tmp')
-            files = filter(lambda x: x.endswith('.hocr'),
+            files = filter(lambda x: x.endswith('.pdf'),
                            os.listdir(input_dir))
             files.sort(key=lambda x: int(re.search(r'\d+', x).group(0)))
             files = map(lambda x: os.path.join(input_dir, x), files)
-            output_file = os.path.join(job.output_dir,
-                                       '{}.xml'.format(job.name))
-            cmd = 'hocrtotei "{}" "{}"'.format('" "'.join(files), output_file)
-            deps = tesseract_jobs
-            lbl = 'hocrtotei_job_-_{}'.format(i)
-            hocrtotei_jobs.append(self.addTask(command=cmd, dependencies=deps,
-                                               label=lbl))
-
-        '''
-        ' ##################################################
-        ' # hocr_poco_jobs                                 #
-        ' ##################################################
-        '''
-
-        hocr_poco_jobs = []
-        for i, job in enumerate(self.jobs):
-            input_dir = os.path.join(job.output_dir, 'tmp')
-            files = filter(lambda x: x.endswith('.hocr'),
-                           os.listdir(input_dir))
-            files.sort(key=lambda x: int(re.search(r'\d+', x).group(0)))
-            files = map(lambda x: os.path.join(input_dir, x), files)
-            # set relative file paths into hocr
-            relative_files = map(lambda x: os.path.join('..',
-                                                        'tiff',
-                                                        os.path.basename(x).replace('.hocr', '.tif')),  # noqa
-                                 files)
-            for file, relative_file in zip(files, relative_files):
-                with open(file, 'r+') as f:
-                    html = f.read()
-                    html = html.replace(file.replace('.hocr', '.tif'),
-                                        relative_file)
-                    f.seek(0)
-                    f.truncate(0)  # deletes content of file to write new html
-                    f.write(html)
-            output_path_base = os.path.join(job.output_dir, 'PoCo')
-            output_path = os.path.join(output_path_base, 'hocr')
-            cmd = 'cp "{}" "{}"'.format('" "'.join(files), output_path)
-            deps = tesseract_jobs
-            lbl = 'hocr_poco_jobs-_{}'.format(i)
-            hocr_poco_jobs.append(self.addTask(command=cmd, dependencies=deps,
-                                               label=lbl))
-        '''
-        ' ##################################################
-        ' # tiff_poco_jobs                                 #
-        ' ##################################################
-        '''
-
-        tiff_poco_jobs = []
-        for i, job in enumerate(self.jobs):
-            input_dir = os.path.join(job.output_dir, 'tmp')
-            files = filter(lambda x: x.endswith('.tif'),
-                           os.listdir(input_dir))
-            files.sort(key=lambda x: int(re.search(r'\d+', x).group(0)))
-            files = map(lambda x: os.path.join(input_dir, x), files)
-            output_path_base = os.path.join(job.output_dir, 'PoCo')
-            output_path = os.path.join(output_path_base, 'tiff')
-            cmd = 'cp "{}" "{}"'.format('" "'.join(files), output_path)
-            deps = tesseract_jobs
-            lbl = 'tiff_poco_jobs-_{}'.format(i)
-            tiff_poco_jobs.append(self.addTask(command=cmd, dependencies=deps,
-                                               label=lbl))
-
-        '''
-        ' ##################################################
-        ' # pdfunite_jobs                                  #
-        ' ##################################################
-        '''
-        pdfunite_jobs = []
-        for i, job in enumerate(self.jobs):
-            input_dir = os.path.join(job.output_dir, 'tmp')
-            files = filter(lambda x: x.endswith('.pdf'), os.listdir(input_dir))
-            files.sort(key=lambda x: int(re.search(r'\d+', x).group(0)))
-            files = map(lambda x: os.path.join(input_dir, x), files)
             output_file = os.path.join(job.output_dir,
                                        '{}.pdf'.format(job.name))
             cmd = 'pdfunite "{}" "{}"'.format('" "'.join(files), output_file)
-            deps = tesseract_jobs
+            deps = filter(lambda x: x.startswith('ocr_-_{}'.format(i)),
-            lbl = 'pdfunite_job_-_{}'.format(i)
+                          ocr_jobs)
-            pdfunite_jobs.append(self.addTask(command=cmd, dependencies=deps,
+            lbl = 'combined_pdf_creation_-_{}'.format(i)
-                                              label=lbl))
+            combined_pdf_creation_jobs.append(self.addTask(command=cmd,
-
-        '''
-        ' ##################################################
-        ' # cat_jobs                                       #
-        ' ##################################################
-        '''
-        cat_jobs = []
-        for i, job in enumerate(self.jobs):
-            input_dir = os.path.join(job.output_dir, 'tmp')
-            files = filter(lambda x: x.endswith('.txt'), os.listdir(input_dir))
-            files.sort(key=lambda x: int(re.search(r'\d+', x).group(0)))
-            files = map(lambda x: os.path.join(input_dir, x), files)
-            output_file = os.path.join(job.output_dir,
-                                       '{}.txt'.format(job.name))
-            cmd = 'cat "{}" > "{}"'.format('" "'.join(files), output_file)
-            deps = tesseract_jobs
-            lbl = 'cat_job_-_{}'.format(i)
-            cat_jobs.append(self.addTask(command=cmd, dependencies=deps,
-                                         label=lbl))
-        '''
-        ' The following jobs are created based of the output files of the
-        ' pdfunite_jobs. So wait until they are finished.
-        '''
-        self.waitForTasks()
-
-        '''
-        ' ##################################################
-        ' # compress_jobs                                  #
-        ' ##################################################
-        '''
-        compress_jobs = []
-        if self.compress:
-            for i, job in enumerate(self.jobs):
-                print(os.listdir(job.output_dir))
-                file = filter(lambda x: x.endswith('.pdf'),
-                              os.listdir(job.output_dir))[0]
-                original_file = os.path.join(job.output_dir, file)
-                compressed_file = os.path.join(job.output_dir, 'c_' + file)
-                cmd = ('gs '
-                       + '-sDEVICE=pdfwrite '
-                       + '-dCompatibilityLevel=1.4 '
-                       + '-dPDFSETTINGS=/ebook '
-                       + '-dNOPAUSE '
-                       + '-dQUIET '
-                       + '-dBATCH '
-                       + '-sOutputFile={o} {i} ').format(o=compressed_file,
-                                                         i=original_file)
-                cmd += '&& rm {original_f} '.format(original_f=original_file)
-                cmd += ('&& mv {compressed_f} '
-                        + '{original_f} ').format(compressed_f=compressed_file,
-                                                  original_f=original_file)
-                deps = (hocrtotei_jobs
-                        + tesseract_jobs
-                        + pdfunite_jobs
-                        + cat_jobs
-                        + hocr_poco_jobs
-                        + tiff_poco_jobs)
-                lbl = 'compress_job_-_{}'.format(i)
-                compress_jobs.append(self.addTask(command=cmd,
                                                            dependencies=deps,
                                                            label=lbl))
 
         '''
         ' ##################################################
-        ' # zip_jobs                                       #
+        ' # combined txt creation                          #
         ' ##################################################
         '''
-        zip_jobs = []
+        combined_txt_creation_jobs = []
-        deps = (hocrtotei_jobs
+        for i, job in enumerate(self.jobs):
-                + tesseract_jobs
+            input_dir = os.path.join(job.output_dir, 'tmp')
-                + pdfunite_jobs
+            files = filter(lambda x: x.endswith('.txt'), os.listdir(input_dir))
-                + cat_jobs
+            files.sort(key=lambda x: int(re.search(r'\d+', x).group(0)))
-                + hocr_poco_jobs
+            files = map(lambda x: os.path.join(input_dir, x), files)
-                + tiff_poco_jobs
+            output_file = os.path.join(job.output_dir, '{}.txt'.format(job.name))  # noqa
-                + compress_jobs)
+            cmd = 'cat "{}" > "{}"'.format('" "'.join(files), output_file)
+            deps = filter(lambda x: x.startswith('ocr_-_{}'.format(i)),
+                          ocr_jobs)
+            lbl = 'combined_txt_creation_-_{}'.format(i)
+            combined_txt_creation_jobs.append(self.addTask(command=cmd,
+                                                           dependencies=deps,
+                                                           label=lbl))
+
+        '''
+        ' ##################################################
+        ' # tei p5 creation                                #
+        ' ##################################################
+        '''
+        tei_p5_creation_jobs = []
+        for i, job in enumerate(self.jobs):
+            input_dir = os.path.join(job.output_dir, 'tmp')
+            files = filter(lambda x: x.endswith('.hocr'), os.listdir(input_dir))  # noqa
+            files.sort(key=lambda x: int(re.search(r'\d+', x).group(0)))
+            files = map(lambda x: os.path.join(input_dir, x), files)
+            output_file = os.path.join(job.output_dir, '{}.xml'.format(job.name))  # noqa
+            cmd = 'hocrtotei "{}" "{}"'.format('" "'.join(files), output_file)
+            deps = filter(lambda x: x.startswith('ocr_-_{}'.format(i)),
+                          ocr_jobs)
+            lbl = 'tei_p5_creation_-_{}'.format(i)
+            tei_p5_creation_jobs.append(self.addTask(command=cmd,
+                                                     dependencies=deps,
+                                                     label=lbl))
+
+        '''
+        ' ##################################################
+        ' # poco bundle creation                           #
+        ' ##################################################
+        '''
+        poco_bundle_creation_jobs = []
+        for i, job in enumerate(self.jobs):
+            input_dir = os.path.join(job.output_dir, 'tmp')
+            output_dir = os.path.join(job.output_dir, 'poco')
+            cmd = 'mv "{}"/*.hocr "{}"'.format(input_dir, output_dir)
+            cmd += ' && '
+            cmd += 'mv "{}"/*.{} "{}"'.format(input_dir, 'bin.png' if self.binarize else 'tif', output_dir)  # noqa
+            deps = filter(lambda x: x.startswith('ocr_-_{}'.format(i)),
+                          ocr_jobs)
+            deps.append('tei_p5_creation_-_{}'.format(i))
+            lbl = 'poco_bundle_creation_-_{}'.format(i)
+            poco_bundle_creation_jobs.append(self.addTask(command=cmd,
+                                                          dependencies=deps,
+                                                          label=lbl))
+
+        '''
+        ' The following jobs are created based of the output files of the
+        ' combined_pdf_creation_jobs. So wait until they are finished.
+        '''
+        self.waitForTasks()
+
+        '''
+        ' ##################################################
+        ' # pdf compression                                #
+        ' ##################################################
+        '''
+        pdf_compression_jobs = []
+        if self.compress:
+            for i, job in enumerate(self.jobs):
+                file = filter(lambda x: x.endswith('.pdf'), os.listdir(job.output_dir))[0]  # noqa
+                original_file = os.path.join(job.output_dir, file)
+                compressed_file = os.path.join(job.output_dir, 'c_' + file)
+                cmd = 'gs'
+                cmd += ' -sDEVICE=pdfwrite'
+                cmd += ' -dCompatibilityLevel=1.4'
+                cmd += ' -dPDFSETTINGS=/ebook'
+                cmd += ' -dNOPAUSE'
+                cmd += ' -dQUIET'
+                cmd += ' -dBATCH'
+                cmd += ' -sOutputFile="{}"'.format(compressed_file)
+                cmd += ' "{}"'.format(original_file)
+                cmd += ' && '
+                cmd += 'mv "{}" "{}"'.format(compressed_file, original_file)
+                deps = 'combined_pdf_creation_-_{}'.format(i)
+                lbl = 'pdf_compression_-_{}'.format(i)
+                pdf_compression_jobs.append(self.addTask(command=cmd,
+                                                         dependencies=deps,
+                                                         label=lbl))
+
+        '''
+        ' ##################################################
+        ' # cleanup                                        #
+        ' ##################################################
+        '''
+        cleanup_jobs = []
+        for i, job in enumerate(self.jobs):
+            input_dir = os.path.join(job.output_dir, 'tmp')
+            cmd = 'rm -r "{}"'.format(input_dir)
+            if self.compress:
+                deps = ['pdf_compression_-_{}'.format(i)]
+            else:
+                deps = ['combined_pdf_creation_-_{}'.format(i)]
+            deps.append('combined_txt_creation_-_{}'.format(i))
+            deps.append('poco_bundle_creation_-_{}'.format(i))
+            deps.append('tei_p5_creation_-_{}'.format(i))
+            lbl = 'cleanup_-_{}'.format(i)
+            cleanup_jobs.append(self.addTask(command=cmd,
+                                             dependencies=deps,
+                                             label=lbl))
+
+        '''
+        ' ##################################################
+        ' # zip creation                                   #
+        ' ##################################################
+        '''
+        zip_creation_jobs = []
         if self.zip is not None:
             # Remove .zip file extension if provided
             if self.zip.endswith('.zip'):
                 self.zip = self.zip[:-4]
                 self.zip = self.zip if self.zip else 'output'
-        # zip ALL
+            # zip all files
             cmd = 'cd "{}"'.format(self.output_dir)
             cmd += ' && '
             cmd += 'zip'
             cmd += ' -r'
             cmd += ' "{}".all.zip .'.format(self.zip)
             cmd += ' -x "pyflow.data*" "*tmp*"'
-            cmd += ' -i "*.pdf" "*.txt" "*.xml" "*.hocr" "*.tif"'
+            cmd += ' -i "*.pdf" "*.txt" "*.xml" "*.hocr" "*.{}"'.format('bin.png' if self.binarize else 'tif')  # noqa
             cmd += ' && '
             cmd += 'cd -'
-            lbl = 'zip_job_-_all'
+            deps = (pdf_compression_jobs if self.compress else
-            zip_jobs.append(self.addTask(command=cmd, dependencies=deps,
+                    combined_pdf_creation_jobs)
+            deps += combined_txt_creation_jobs
+            deps += poco_bundle_creation_jobs
+            lbl = 'zip_creation_-_all'
+            zip_creation_jobs.append(self.addTask(command=cmd,
+                                                  dependencies=deps,
                                                   label=lbl))
-        # zip PDFs
+            # zip PDF files
             cmd = 'cd "{}"'.format(self.output_dir)
             cmd += ' && '
             cmd += 'zip'
@@ -432,11 +404,13 @@ class OCRPipeline(WorkflowRunner):
             cmd += ' -i "*.pdf"'
             cmd += ' && '
             cmd += 'cd -'
-            deps = deps + ['zip_job_-_all']
+            deps = (pdf_compression_jobs if self.compress else
-            lbl = 'zip_job_-_pdf'
+                    combined_pdf_creation_jobs)
-            zip_jobs.append(self.addTask(command=cmd, dependencies=deps,
+            lbl = 'zip_creation_-_pdf'
+            zip_creation_jobs.append(self.addTask(command=cmd,
+                                                  dependencies=deps,
                                                   label=lbl))
-        # zip TXTs
+            # zip TXT files
             cmd = 'cd "{}"'.format(self.output_dir)
             cmd += ' && '
             cmd += 'zip'
@@ -446,11 +420,12 @@ class OCRPipeline(WorkflowRunner):
             cmd += ' -i "*.txt"'
             cmd += ' && '
             cmd += 'cd -'
-            deps = deps + ['zip_job_-_all']
+            deps = combined_txt_creation_jobs
-            lbl = 'zip_job_-_txt'
+            lbl = 'zip_creation_-_txt'
-            zip_jobs.append(self.addTask(command=cmd, dependencies=deps,
+            zip_creation_jobs.append(self.addTask(command=cmd,
+                                                  dependencies=deps,
                                                   label=lbl))
-        # zip XMLs
+            # zip XML files
             cmd = 'cd "{}"'.format(self.output_dir)
             cmd += ' && '
             cmd += 'zip'
@@ -460,79 +435,25 @@ class OCRPipeline(WorkflowRunner):
             cmd += ' -i "*.xml"'
             cmd += ' && '
             cmd += 'cd -'
-            deps = deps + ['zip_job_-_all']
+            deps = tei_p5_creation_jobs
-            lbl = 'zip_job_-_xml'
+            lbl = 'zip_creation_-_xml'
-            zip_jobs.append(self.addTask(command=cmd, dependencies=deps,
+            zip_creation_jobs.append(self.addTask(command=cmd,
+                                                  dependencies=deps,
                                                   label=lbl))
-        # zip PoCo files
+            # zip PoCo bundles
-            poco_paths = []
-            poco_names = []
-            for i, job in enumerate(self.jobs):
-                poco_paths.append(os.path.join(os.path.basename(job.output_dir),  # noqa
-                                                                'PoCo'))
-                poco_names.append(job.output_dir)
-
             cmd = 'cd "{}"'.format(self.output_dir)
             cmd += ' && '
             cmd += 'zip'
             cmd += ' -r'
-            cmd += ' "{}".poco.zip'.format(self.zip)
+            cmd += ' "{}".poco.zip .'.format(self.zip)
-            cmd += ' "{}"'.format('" "'.join(poco_paths))
+            cmd += ' -x "pyflow.data*" "*tmp*"'
+            cmd += ' -i "*.hocr" "*.{}"'.format('bin.png' if self.binarize else 'tif')  # noqa
             cmd += ' && '
             cmd += 'cd -'
-            deps = deps + ['zip_job_-_all']
+            deps = poco_bundle_creation_jobs
-            lbl = 'zip_job_-_poco_{}'.format(i)
+            lbl = 'zip_creation_-_poco'
-            zip_jobs.append(self.addTask(command=cmd, dependencies=deps,
+            zip_creation_jobs.append(self.addTask(command=cmd,
-                                         label=lbl))
+                                                  dependencies=deps,
-
-        '''
-        ' ##################################################
-        ' # mv_jobs                                        #
-        ' ##################################################
-        '''
-        mv_jobs = []
-        if self.keep_intermediates:
-            for i, job in enumerate(self.jobs):
-                input_dir = os.path.join(job.output_dir, 'tmp')
-                output_dir = input_dir
-                cmd = 'mv "{}"/*.hocr "{}"'.format(
-                    input_dir, os.path.join(output_dir, 'hocr'))
-                cmd += ' && '
-                cmd += 'mv "{}"/*.pdf "{}"'.format(input_dir, os.path.join(output_dir, 'pdf'))  # noqa
-                cmd += ' && '
-                cmd += 'mv "{}"/*.tif "{}"'.format(input_dir, os.path.join(output_dir, 'tiff'))  # noqa
-                cmd += ' && '
-                cmd += 'mv "{}"/*.txt "{}"'.format(input_dir, os.path.join(output_dir, 'txt'))  # noqa
-                if self.binarize:
-                    cmd += ' && '
-                    cmd += 'mv "{}"/*.bin.png "{}"'.format(input_dir, os.path.join(output_dir, 'bin.png'))  # noqa
-                    cmd += ' && '
-                    cmd += 'mv "{}"/*.nrm.png "{}"'.format(input_dir, os.path.join(output_dir, 'nrm.png'))  # noqa
-                deps = (hocrtotei_jobs
-                        + tesseract_jobs
-                        + pdfunite_jobs
-                        + cat_jobs
-                        + hocr_poco_jobs
-                        + tiff_poco_jobs,
-                        + compress_jobs
-                        + zip_jobs)
-                lbl = 'mv_job_-_{}'.format(i)
-                mv_jobs.append(self.addTask(command=cmd, dependencies=deps,
-                                            label=lbl))
-        else:
-            for i, job in enumerate(self.jobs):
-                input_dir = os.path.join(job.output_dir, 'tmp')
-                cmd = 'rm -r "{}"'.format(input_dir)
-                deps = (hocrtotei_jobs
-                        + tesseract_jobs
-                        + pdfunite_jobs
-                        + cat_jobs
-                        + hocr_poco_jobs
-                        + tiff_poco_jobs
-                        + compress_jobs
-                        + zip_jobs)
-                lbl = 'mv_job_-_{}'.format(i)
-                mv_jobs.append(self.addTask(command=cmd, dependencies=deps,
                                                   label=lbl))
 
 
@@ -550,12 +471,14 @@ def collect_jobs(input_dir, output_dir):
 
 def main():
     args = parse_args()
-    jobs = collect_jobs(args.i, args.o)
+    jobs = collect_jobs(args.input_directory, args.output_directory)
-    ocr_pipeline = OCRPipeline(args.binarize, jobs, args.keep_intermediates,
+    ocr_pipeline = OCRPipeline(args.binarize, jobs, args.language,
-                               args.language, args.n_cores, args.o, args.zip,
+                               args.n_cores, args.output_directory, args.zip,
                                args.compress)
-    retval = ocr_pipeline.run(dataDirRoot=(args.log_dir or args.o),
+    retval = ocr_pipeline.run(
-                              nCores=args.n_cores)
+        dataDirRoot=(args.log_dir or args.output_directory),
+        nCores=args.n_cores
+    )
     sys.exit(retval)
 
 

wrapper/ocr

@@ -12,17 +12,21 @@ UID = str(os.getuid())
 GID = str(os.getgid())
 
 parser = ArgumentParser(add_help=False)
-parser.add_argument('-i')
+parser.add_argument('-i', '--input-directory')
-parser.add_argument('-o')
+parser.add_argument('-o', '--output-directory')
 args, remaining_args = parser.parse_known_args()
 
 cmd = ['docker', 'run', '--rm', '-it', '-u', '{}:{}'.format(UID, GID)]
-if args.o is not None:
+if args.output_directory is not None:
-    cmd += ['-v', '{}:{}'.format(os.path.abspath(args.o), CONTAINER_OUTPUT_DIR)]
+    cmd += ['-v', '{}:{}'.format(os.path.abspath(args.output_directory),
+                                 CONTAINER_OUTPUT_DIR)]
     remaining_args.insert(0, CONTAINER_OUTPUT_DIR)
-if args.i is not None:
+    remaining_args.insert(0, '-o')
-    cmd += ['-v', '{}:{}'.format(os.path.abspath(args.i), CONTAINER_INPUT_DIR)]
+if args.input_directory is not None:
+    cmd += ['-v', '{}:{}'.format(os.path.abspath(args.input_directory),
+                                 CONTAINER_INPUT_DIR)]
     remaining_args.insert(0, CONTAINER_INPUT_DIR)
+    remaining_args.insert(0, '-i')
 cmd.append(CONTAINER_IMAGE)
 cmd += remaining_args