From f65ab646e9e6d7a1dfb9d1fcf1ac78407c28cebe Mon Sep 17 00:00:00 2001 From: Patrick Jentsch Date: Wed, 14 Apr 2021 12:00:09 +0200 Subject: [PATCH] implement fixed version numbers --- web/app/services/__init__.py | 18 ++++++------- web/app/tasks/job_utils.py | 49 +++++++++++++++++++++++------------- 2 files changed, 40 insertions(+), 27 deletions(-) diff --git a/web/app/services/__init__.py b/web/app/services/__init__.py index f374b18b..6732fd88 100644 --- a/web/app/services/__init__.py +++ b/web/app/services/__init__.py @@ -8,12 +8,12 @@ SERVICES = { 'file-setup': { 'name': 'File setup', 'versions': { - 'latest': '1.0.0', - '1.0.0': { + 'latest': '1.0.0b', + '1.0.0b': { 'publishing_data': { 'date': None, 'title': 'nopaque File setup service', - 'url': 'https://gitlab.ub.uni-bielefeld.de/sfb1288inf/file-setup/-/tree/1.0.0', # noqa + 'url': 'https://gitlab.ub.uni-bielefeld.de/sfb1288inf/file-setup/-/tree/1.0.0b', # noqa 'version': '1.0.0' } } @@ -22,8 +22,8 @@ SERVICES = { 'nlp': { 'name': 'Natural Language Processing', 'versions': { - 'latest': '1.0.0', - '1.0.0': { + 'latest': '1.0.0b', + '1.0.0b': { 'check_encoding': True, 'models': { 'de': 'German', @@ -36,7 +36,7 @@ SERVICES = { 'publishing_data': { 'date': None, 'title': 'nopaque NLP service', - 'url': 'https://gitlab.ub.uni-bielefeld.de/sfb1288inf/nlp/-/tree/1.0.0', # noqa + 'url': 'https://gitlab.ub.uni-bielefeld.de/sfb1288inf/nlp/-/tree/1.0.0b', # noqa 'version': '1.0.0' } } @@ -45,8 +45,8 @@ SERVICES = { 'ocr': { 'name': 'Optical Character Recognition', 'versions': { - 'latest': '1.0.0', - '1.0.0': { + 'latest': '1.0.0b', + '1.0.0b': { 'binarization': True, 'models': { 'ara': 'Arabic', @@ -67,7 +67,7 @@ SERVICES = { 'publishing_data': { 'date': None, 'title': 'nopaque OCR service', - 'url': 'https://gitlab.ub.uni-bielefeld.de/sfb1288inf/ocr/-/tree/1.0.0', # noqa + 'url': 'https://gitlab.ub.uni-bielefeld.de/sfb1288inf/ocr/-/tree/1.0.0b', # noqa 'version': '1.0.0' } } diff --git a/web/app/tasks/job_utils.py b/web/app/tasks/job_utils.py index cdd0160d..96f9e437 100644 --- a/web/app/tasks/job_utils.py +++ b/web/app/tasks/job_utils.py @@ -9,20 +9,22 @@ import json import os +# TODO: Integrate the service_settings into app/services/__init__.py service_settings = { 'file-setup': { - 'ressources': docker.types.Resources(mem_reservation=1024 * (10 ** 6), - cpu_reservation=1 * (10 ** 9)) + 'default_args': ' --mem-mb 2048 --n-cores 2', + 'ressources': docker.types.Resources(cpu_reservation=2 * (10 ** 9), + mem_reservation=2048 * (10 ** 6)) }, 'nlp': { - 'default_args': ' --n-cores 2 --mem-mb 2048', - 'ressources': docker.types.Resources(mem_reservation=2048 * (10 ** 6), - cpu_reservation=2 * (10 ** 9)) + 'default_args': ' --mem-mb 2048 --n-cores 2', + 'ressources': docker.types.Resources(cpu_reservation=2 * (10 ** 9), + mem_reservation=2048 * (10 ** 6)) }, 'ocr': { - 'default_args': ' --n-cores 4 --mem-mb 4096', - 'ressources': docker.types.Resources(mem_reservation=4096 * (10 ** 6), - cpu_reservation=4 * (10 ** 9)) + 'default_args': ' --mem-mb 4096 --n-cores 4', + 'ressources': docker.types.Resources(cpu_reservation=4 * (10 ** 9), + mem_reservation=4096 * (10 ** 6)) } } @@ -42,25 +44,36 @@ class CheckJobsMixin: self.remove_job_service(job) def create_job_service(self, job): - cmd = '{} -i /files -o /files/output'.format(job.service) - if 'default_args' in service_settings[job.service]: - cmd += service_settings[job.service]['default_args'] - if job.service == 'file-setup': - cmd += ' -f {}'.format(secure_filename(job.title)) - cmd += ' --log-dir /files' + cmd = job.service + cmd += ' -i /input' + cmd += ' -o /output' + cmd += ' --log-dir /input' cmd += ' --zip [{}]_{}'.format(job.service, secure_filename(job.title)) + cmd += service_settings[job.service]['default_args'] cmd += ' ' + ' '.join(json.loads(job.service_args)) - ressources = service_settings[job.service]['ressources'] + # Setup input mount + input_mount_src = job.path + input_mount_dest = os.path.abspath('/input') + if job.service == 'file-setup': + input_mount_dest = os.path.join(input_mount_dest, secure_filename(job.title)) # noqa + input_mount = '{}:{}:rw'.format(input_mount_src, input_mount_dest) + # Setup output mount + output_mount_src = os.path.join(job.path, 'output') + output_mount_dest = os.path.abspath('/output') + os.makedirs(output_mount_src) + output_mount = '{}:{}:rw'.format(output_mount_src, output_mount_dest) + logging.warning(input_mount) + logging.warning(output_mount) service_kwargs = {'command': cmd, 'constraints': ['node.role==worker'], 'labels': {'origin': 'nopaque', 'type': 'job', 'job_id': str(job.id)}, - 'mounts': [job.path + ':/files:rw'], + 'mounts': [input_mount, output_mount], 'name': 'job_{}'.format(job.id), - 'resources': ressources, + 'resources': service_settings[job.service]['ressources'], # noqa 'restart_policy': docker.types.RestartPolicy()} - service_image = 'gitlab.ub.uni-bielefeld.de:4567/sfb1288inf/{}:latest'.format(job.service) # noqa + service_image = 'gitlab.ub.uni-bielefeld.de:4567/sfb1288inf/{}:{}'.format(job.service, job.service_version) # noqa try: self.docker.services.create(service_image, **service_kwargs) except docker.errors.APIError as e: