sporada/nopaque
1
0
Fork 0
mirror of https://gitlab.ub.uni-bielefeld.de/sfb1288inf/nopaque.git synced 2024-11-15 01:05:42 +00:00
Code Issues Projects Releases Wiki Activity

implement fixed version numbers

Browse Source
This commit is contained in:
Patrick Jentsch 2021-04-14 12:00:09 +02:00
parent 2f9ecf8048
commit f65ab646e9
2 changed files with 40 additions and 27 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

18
web/app/services/__init__.py
View File

@ -8,12 +8,12 @@ SERVICES = {
'file-setup': { 'file-setup': {
'name': 'File setup', 'name': 'File setup',
'versions': { 'versions': {
'latest': '1.0.0', 'latest': '1.0.0b',
'1.0.0': { '1.0.0b': {
'publishing_data': { 'publishing_data': {
'date': None, 'date': None,
'title': 'nopaque File setup service', 'title': 'nopaque File setup service',
'url': 'https://gitlab.ub.uni-bielefeld.de/sfb1288inf/file-setup/-/tree/1.0.0', # noqa 'url': 'https://gitlab.ub.uni-bielefeld.de/sfb1288inf/file-setup/-/tree/1.0.0b', # noqa
'version': '1.0.0' 'version': '1.0.0'
} }
} }
@ -22,8 +22,8 @@ SERVICES = {
'nlp': { 'nlp': {
'name': 'Natural Language Processing', 'name': 'Natural Language Processing',
'versions': { 'versions': {
'latest': '1.0.0', 'latest': '1.0.0b',
'1.0.0': { '1.0.0b': {
'check_encoding': True, 'check_encoding': True,
'models': { 'models': {
'de': 'German', 'de': 'German',
@ -36,7 +36,7 @@ SERVICES = {
'publishing_data': { 'publishing_data': {
'date': None, 'date': None,
'title': 'nopaque NLP service', 'title': 'nopaque NLP service',
'url': 'https://gitlab.ub.uni-bielefeld.de/sfb1288inf/nlp/-/tree/1.0.0', # noqa 'url': 'https://gitlab.ub.uni-bielefeld.de/sfb1288inf/nlp/-/tree/1.0.0b', # noqa
'version': '1.0.0' 'version': '1.0.0'
} }
} }
@ -45,8 +45,8 @@ SERVICES = {
'ocr': { 'ocr': {
'name': 'Optical Character Recognition', 'name': 'Optical Character Recognition',
'versions': { 'versions': {
'latest': '1.0.0', 'latest': '1.0.0b',
'1.0.0': { '1.0.0b': {
'binarization': True, 'binarization': True,
'models': { 'models': {
'ara': 'Arabic', 'ara': 'Arabic',
@ -67,7 +67,7 @@ SERVICES = {
'publishing_data': { 'publishing_data': {
'date': None, 'date': None,
'title': 'nopaque OCR service', 'title': 'nopaque OCR service',
'url': 'https://gitlab.ub.uni-bielefeld.de/sfb1288inf/ocr/-/tree/1.0.0', # noqa 'url': 'https://gitlab.ub.uni-bielefeld.de/sfb1288inf/ocr/-/tree/1.0.0b', # noqa
'version': '1.0.0' 'version': '1.0.0'
} }
} }

49
web/app/tasks/job_utils.py
View File

@ -9,20 +9,22 @@ import json
import os import os
# TODO: Integrate the service_settings into app/services/__init__.py
service_settings = { service_settings = {
'file-setup': { 'file-setup': {
'ressources': docker.types.Resources(mem_reservation=1024 * (10 ** 6), 'default_args': ' --mem-mb 2048 --n-cores 2',
cpu_reservation=1 * (10 ** 9)) 'ressources': docker.types.Resources(cpu_reservation=2 * (10 ** 9),
mem_reservation=2048 * (10 ** 6))
}, },
'nlp': { 'nlp': {
'default_args': ' --n-cores 2 --mem-mb 2048', 'default_args': ' --mem-mb 2048 --n-cores 2',
'ressources': docker.types.Resources(mem_reservation=2048 * (10 ** 6), 'ressources': docker.types.Resources(cpu_reservation=2 * (10 ** 9),
cpu_reservation=2 * (10 ** 9)) mem_reservation=2048 * (10 ** 6))
}, },
'ocr': { 'ocr': {
'default_args': ' --n-cores 4 --mem-mb 4096', 'default_args': ' --mem-mb 4096 --n-cores 4',
'ressources': docker.types.Resources(mem_reservation=4096 * (10 ** 6), 'ressources': docker.types.Resources(cpu_reservation=4 * (10 ** 9),
cpu_reservation=4 * (10 ** 9)) mem_reservation=4096 * (10 ** 6))
} }
} }
@ -42,25 +44,36 @@ class CheckJobsMixin:
self.remove_job_service(job) self.remove_job_service(job)
def create_job_service(self, job): def create_job_service(self, job):
cmd = '{} -i /files -o /files/output'.format(job.service) cmd = job.service
if 'default_args' in service_settings[job.service]: cmd += ' -i /input'
cmd += service_settings[job.service]['default_args'] cmd += ' -o /output'
if job.service == 'file-setup': cmd += ' --log-dir /input'
cmd += ' -f {}'.format(secure_filename(job.title))
cmd += ' --log-dir /files'
cmd += ' --zip [{}]_{}'.format(job.service, secure_filename(job.title)) cmd += ' --zip [{}]_{}'.format(job.service, secure_filename(job.title))
cmd += service_settings[job.service]['default_args']
cmd += ' ' + ' '.join(json.loads(job.service_args)) cmd += ' ' + ' '.join(json.loads(job.service_args))
ressources = service_settings[job.service]['ressources'] # Setup input mount
input_mount_src = job.path
input_mount_dest = os.path.abspath('/input')
if job.service == 'file-setup':
input_mount_dest = os.path.join(input_mount_dest, secure_filename(job.title)) # noqa
input_mount = '{}:{}:rw'.format(input_mount_src, input_mount_dest)
# Setup output mount
output_mount_src = os.path.join(job.path, 'output')
output_mount_dest = os.path.abspath('/output')
os.makedirs(output_mount_src)
output_mount = '{}:{}:rw'.format(output_mount_src, output_mount_dest)
logging.warning(input_mount)
logging.warning(output_mount)
service_kwargs = {'command': cmd, service_kwargs = {'command': cmd,
'constraints': ['node.role==worker'], 'constraints': ['node.role==worker'],
'labels': {'origin': 'nopaque', 'labels': {'origin': 'nopaque',
'type': 'job', 'type': 'job',
'job_id': str(job.id)}, 'job_id': str(job.id)},
'mounts': [job.path + ':/files:rw'], 'mounts': [input_mount, output_mount],
'name': 'job_{}'.format(job.id), 'name': 'job_{}'.format(job.id),
'resources': ressources, 'resources': service_settings[job.service]['ressources'], # noqa
'restart_policy': docker.types.RestartPolicy()} 'restart_policy': docker.types.RestartPolicy()}
service_image = 'gitlab.ub.uni-bielefeld.de:4567/sfb1288inf/{}:latest'.format(job.service) # noqa service_image = 'gitlab.ub.uni-bielefeld.de:4567/sfb1288inf/{}:{}'.format(job.service, job.service_version) # noqa
try: try:
self.docker.services.create(service_image, **service_kwargs) self.docker.services.create(service_image, **service_kwargs)
except docker.errors.APIError as e: except docker.errors.APIError as e: