sporada/nopaque
1
0
Fork 0
mirror of https://gitlab.ub.uni-bielefeld.de/sfb1288inf/nopaque.git synced 2025-06-26 16:00:34 +00:00
Code Issues Projects Releases Wiki Activity

Update service pages and how version data is gathered.

Browse Source
This commit is contained in:
Patrick Jentsch
2021-03-26 13:10:42 +01:00
parent 55d94ea329
commit 2f9ecf8048
9 changed files with 133 additions and 43 deletions
Download Patch File Download Diff File Expand all files Collapse all files

34
web/app/tasks/job_utils.py
View File

@ -9,13 +9,31 @@ import json
import os
service_settings = {
'file-setup': {
'ressources': docker.types.Resources(mem_reservation=1024 * (10 ** 6),
cpu_reservation=1 * (10 ** 9))
},
'nlp': {
'default_args': ' --n-cores 2 --mem-mb 2048',
'ressources': docker.types.Resources(mem_reservation=2048 * (10 ** 6),
cpu_reservation=2 * (10 ** 9))
},
'ocr': {
'default_args': ' --n-cores 4 --mem-mb 4096',
'ressources': docker.types.Resources(mem_reservation=4096 * (10 ** 6),
cpu_reservation=4 * (10 ** 9))
}
}
class CheckJobsMixin:
def check_jobs(self):
jobs = Job.query.all()
canceling_jobs = list(filter(lambda job: job.status == 'canceling', jobs))
canceling_jobs = list(filter(lambda job: job.status == 'canceling', jobs)) # noqa
queued_jobs = list(filter(lambda job: job.status == 'queued', jobs))
running_jobs = list(filter(lambda job: job.status == 'running', jobs))
submitted_jobs = list(filter(lambda job: job.status == 'submitted', jobs))
submitted_jobs = list(filter(lambda job: job.status == 'submitted', jobs)) # noqa
for job in submitted_jobs:
self.create_job_service(job)
for job in queued_jobs + running_jobs:
@ -25,26 +43,24 @@ class CheckJobsMixin:
def create_job_service(self, job):
cmd = '{} -i /files -o /files/output'.format(job.service)
if 'default_args' in service_settings[job.service]:
cmd += service_settings[job.service]['default_args']
if job.service == 'file-setup':
cmd += ' -f {}'.format(secure_filename(job.title))
ressources = docker.types.Resources(cpu_reservation=4 * (10 ** 9), mem_reservation=4096 * (10 ** 6)) # noqa
elif job.service == 'nlp':
ressources = docker.types.Resources(cpu_reservation=2 * (10 ** 9), mem_reservation=4096 * (10 ** 6)) # noqa
elif job.service == 'ocr':
ressources = docker.types.Resources(cpu_reservation=4 * (10 ** 9), mem_reservation=8192 * (10 ** 6)) # noqa
cmd += ' --log-dir /files'
cmd += ' --zip [{}]_{}'.format(job.service, secure_filename(job.title))
cmd += ' ' + ' '.join(json.loads(job.service_args))
ressources = service_settings[job.service]['ressources']
service_kwargs = {'command': cmd,
'constraints': ['node.role==worker'],
'labels': {'origin': 'nopaque',
'type': 'service.{}'.format(job.service),
'type': 'job',
'job_id': str(job.id)},
'mounts': [job.path + ':/files:rw'],
'name': 'job_{}'.format(job.id),
'resources': ressources,
'restart_policy': docker.types.RestartPolicy()}
service_image = 'gitlab.ub.uni-bielefeld.de:4567/sfb1288inf/{}:{}'.format(job.service, job.service_version) # noqa
service_image = 'gitlab.ub.uni-bielefeld.de:4567/sfb1288inf/{}:latest'.format(job.service) # noqa
try:
self.docker.services.create(service_image, **service_kwargs)
except docker.errors.APIError as e: