sporada/nopaque
1
0
Fork 0
mirror of https://gitlab.ub.uni-bielefeld.de/sfb1288inf/nopaque.git synced 2025-11-03 20:02:47 +00:00
Code Issues Projects Releases Wiki Activity

Rename all services, use scss, cleanup, add sandpaper conversion script

Browse Source
This commit is contained in:
Patrick Jentsch
2022-04-04 13:31:09 +02:00
parent 8fd59f8078
commit ce997e69ea
31 changed files with 1361 additions and 303 deletions
Download Patch File Download Diff File Expand all files Collapse all files

58
app/daemon/job_utils.py
View File

@@ -22,34 +22,46 @@ class CheckJobsMixin:
def create_job_service(self, job):
''' # Docker service settings # '''
''' ## Service specific settings ## '''
if job.service == 'file-setup':
if job.service == 'file-setup-pipeline':
mem_mb = 512
n_cores = 2
executable = 'file-setup'
image = f'{current_app.config["NOPAQUE_DOCKER_IMAGE_PREFIX"]}file-setup:v{job.service_version}' # noqa
elif job.service == 'tesseract-ocr':
mem_mb = 2048
executable = 'file-setup-pipeline'
image = f'{current_app.config["NOPAQUE_DOCKER_IMAGE_PREFIX"]}file-setup-pipeline:v{job.service_version}' # noqa
elif job.service == 'tesseract-ocr-pipeline':
mem_mb = 1024
n_cores = 4
executable = 'ocr'
image = f'{current_app.config["NOPAQUE_DOCKER_IMAGE_PREFIX"]}ocr:v{job.service_version}' # noqa
elif job.service == 'spacy-nlp':
executable = 'tesseract-ocr-pipeline'
image = f'{current_app.config["NOPAQUE_DOCKER_IMAGE_PREFIX"]}tesseract-ocr-pipeline:v{job.service_version}' # noqa
elif job.service == 'transkribus-htr-pipeline':
mem_mb = 1024
n_cores = 4
executable = 'transkribus-htr-pipeline'
image = f'{current_app.config["NOPAQUE_DOCKER_IMAGE_PREFIX"]}transkribus-htr-pipeline:v{job.service_version}' # noqa
elif job.service == 'spacy-nlp-pipeline':
mem_mb = 1024
n_cores = 1
executable = 'nlp'
image = f'{current_app.config["NOPAQUE_DOCKER_IMAGE_PREFIX"]}nlp:v{job.service_version}' # noqa
executable = 'spacy-nlp-pipeline'
image = f'{current_app.config["NOPAQUE_DOCKER_IMAGE_PREFIX"]}spacy-nlp-pipeline:v{job.service_version}' # noqa
''' ## Command ## '''
command = f'{executable} -i /input -o /output'
command += ' --log-dir /logs'
command += f' --mem-mb {mem_mb}'
command += f' --n-cores {n_cores}'
service_args = json.loads(job.service_args)
if job.service == 'spacy-nlp':
command += f' -m {service_args["model"]}'
if 'encoding_detection' in service_args and service_args['encoding_detection']: # noqa
if job.service == 'spacy-nlp-pipeline':
command += f' -m {job.service_args["model"]}'
if 'encoding_detection' in job.service_args and job.service_args['encoding_detection']: # noqa
command += ' --check-encoding'
elif job.service == 'tesseract-ocr':
command += f' -m {service_args["model"]}'
if 'binarization' in service_args and service_args['binarization']:
elif job.service == 'tesseract-ocr-pipeline':
command += f' -m {job.service_args["model"]}'
if 'binarization' in job.service_args and job.service_args['binarization']:
command += ' --binarize'
elif job.service == 'transkribus-htr-pipeline':
command += f' -m {job.service_args["model"]}'
readcoop_username = current_app.config.get('NOPAQUE_READCOOP_USERNAME')
command += f' --readcoop-username "{readcoop_username}"'
readcoop_password = current_app.config.get('NOPAQUE_READCOOP_PASSWORD')
command += f' --readcoop-password "{readcoop_password}"'
if 'binarization' in job.service_args and job.service_args['binarization']:
command += ' --binarize'
''' ## Constraints ## '''
constraints = ['node.role==worker']
@@ -63,16 +75,15 @@ class CheckJobsMixin:
mounts = []
''' ### Input mount(s) ### '''
input_mount_target_base = '/input'
if job.service == 'file-setup':
if job.service == 'file-setup-pipeline':
input_mount_target_base += f'/{secure_filename(job.title)}'
for job_input in job.inputs:
input_mount_source = job_input.path
input_mount_target = f'/{input_mount_target_base}/{job_input.filename}' # noqa
input_mount_target = f'{input_mount_target_base}/{job_input.filename}' # noqa
input_mount = f'{input_mount_source}:{input_mount_target}:ro'
mounts.append(input_mount)
if job.service == 'tesseract-ocr':
service_args = json.loads(job.service_args)
model = TesseractOCRModel.query.get(service_args['model'])
if job.service == 'tesseract-ocr-pipeline':
model = TesseractOCRModel.query.get(job.service_args['model'])
if model is None:
job.status = JobStatus.FAILED
return
@@ -114,7 +125,8 @@ class CheckJobsMixin:
mounts=mounts,
name=name,
resources=resources,
restart_policy=restart_policy
restart_policy=restart_policy,
user='1000:1000'
)
except docker.errors.APIError as e:
current_app.logger.error(