sporada/nopaque
1
0
Fork 0
mirror of https://gitlab.ub.uni-bielefeld.de/sfb1288inf/nopaque.git synced 2024-11-15 01:05:42 +00:00
Code Issues Projects Releases Wiki Activity

implement fixed version numbers

Browse Source
This commit is contained in:
Patrick Jentsch 2021-04-14 12:00:09 +02:00
parent 2f9ecf8048
commit f65ab646e9
2 changed files with 40 additions and 27 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

18
web/app/services/__init__.py
View File

@ -8,12 +8,12 @@ SERVICES = {
'file-setup': {
'name': 'File setup',
'versions': {
'latest': '1.0.0',
'1.0.0': {
'latest': '1.0.0b',
'1.0.0b': {
'publishing_data': {
'date': None,
'title': 'nopaque File setup service',
'url': 'https://gitlab.ub.uni-bielefeld.de/sfb1288inf/file-setup/-/tree/1.0.0', # noqa
'url': 'https://gitlab.ub.uni-bielefeld.de/sfb1288inf/file-setup/-/tree/1.0.0b', # noqa
'version': '1.0.0'
}
}
@ -22,8 +22,8 @@ SERVICES = {
'nlp': {
'name': 'Natural Language Processing',
'versions': {
'latest': '1.0.0',
'1.0.0': {
'latest': '1.0.0b',
'1.0.0b': {
'check_encoding': True,
'models': {
'de': 'German',
@ -36,7 +36,7 @@ SERVICES = {
'publishing_data': {
'date': None,
'title': 'nopaque NLP service',
'url': 'https://gitlab.ub.uni-bielefeld.de/sfb1288inf/nlp/-/tree/1.0.0', # noqa
'url': 'https://gitlab.ub.uni-bielefeld.de/sfb1288inf/nlp/-/tree/1.0.0b', # noqa
'version': '1.0.0'
}
}
@ -45,8 +45,8 @@ SERVICES = {
'ocr': {
'name': 'Optical Character Recognition',
'versions': {
'latest': '1.0.0',
'1.0.0': {
'latest': '1.0.0b',
'1.0.0b': {
'binarization': True,
'models': {
'ara': 'Arabic',
@ -67,7 +67,7 @@ SERVICES = {
'publishing_data': {
'date': None,
'title': 'nopaque OCR service',
'url': 'https://gitlab.ub.uni-bielefeld.de/sfb1288inf/ocr/-/tree/1.0.0', # noqa
'url': 'https://gitlab.ub.uni-bielefeld.de/sfb1288inf/ocr/-/tree/1.0.0b', # noqa
'version': '1.0.0'
}
}

49
web/app/tasks/job_utils.py
View File

@ -9,20 +9,22 @@ import json
import os
# TODO: Integrate the service_settings into app/services/__init__.py
service_settings = {
'file-setup': {
'ressources': docker.types.Resources(mem_reservation=1024 * (10 ** 6),
cpu_reservation=1 * (10 ** 9))
'default_args': ' --mem-mb 2048 --n-cores 2',
'ressources': docker.types.Resources(cpu_reservation=2 * (10 ** 9),
mem_reservation=2048 * (10 ** 6))
},
'nlp': {
'default_args': ' --n-cores 2 --mem-mb 2048',
'ressources': docker.types.Resources(mem_reservation=2048 * (10 ** 6),
cpu_reservation=2 * (10 ** 9))
'default_args': ' --mem-mb 2048 --n-cores 2',
'ressources': docker.types.Resources(cpu_reservation=2 * (10 ** 9),
mem_reservation=2048 * (10 ** 6))
},
'ocr': {
'default_args': ' --n-cores 4 --mem-mb 4096',
'ressources': docker.types.Resources(mem_reservation=4096 * (10 ** 6),
cpu_reservation=4 * (10 ** 9))
'default_args': ' --mem-mb 4096 --n-cores 4',
'ressources': docker.types.Resources(cpu_reservation=4 * (10 ** 9),
mem_reservation=4096 * (10 ** 6))
}
}
@ -42,25 +44,36 @@ class CheckJobsMixin:
self.remove_job_service(job)
def create_job_service(self, job):
cmd = '{} -i /files -o /files/output'.format(job.service)
if 'default_args' in service_settings[job.service]:
cmd += service_settings[job.service]['default_args']
if job.service == 'file-setup':
cmd += ' -f {}'.format(secure_filename(job.title))
cmd += ' --log-dir /files'
cmd = job.service
cmd += ' -i /input'
cmd += ' -o /output'
cmd += ' --log-dir /input'
cmd += ' --zip [{}]_{}'.format(job.service, secure_filename(job.title))
cmd += service_settings[job.service]['default_args']
cmd += ' ' + ' '.join(json.loads(job.service_args))
ressources = service_settings[job.service]['ressources']
# Setup input mount
input_mount_src = job.path
input_mount_dest = os.path.abspath('/input')
if job.service == 'file-setup':
input_mount_dest = os.path.join(input_mount_dest, secure_filename(job.title)) # noqa
input_mount = '{}:{}:rw'.format(input_mount_src, input_mount_dest)
# Setup output mount
output_mount_src = os.path.join(job.path, 'output')
output_mount_dest = os.path.abspath('/output')
os.makedirs(output_mount_src)
output_mount = '{}:{}:rw'.format(output_mount_src, output_mount_dest)
logging.warning(input_mount)
logging.warning(output_mount)
service_kwargs = {'command': cmd,
'constraints': ['node.role==worker'],
'labels': {'origin': 'nopaque',
'type': 'job',
'job_id': str(job.id)},
'mounts': [job.path + ':/files:rw'],
'mounts': [input_mount, output_mount],
'name': 'job_{}'.format(job.id),
'resources': ressources,
'resources': service_settings[job.service]['ressources'], # noqa
'restart_policy': docker.types.RestartPolicy()}
service_image = 'gitlab.ub.uni-bielefeld.de:4567/sfb1288inf/{}:latest'.format(job.service) # noqa
service_image = 'gitlab.ub.uni-bielefeld.de:4567/sfb1288inf/{}:{}'.format(job.service, job.service_version) # noqa
try:
self.docker.services.create(service_image, **service_kwargs)
except docker.errors.APIError as e: