mirror of
https://gitlab.ub.uni-bielefeld.de/sfb1288inf/nopaque.git
synced 2024-11-15 01:05:42 +00:00
112 lines
4.4 KiB
Python
112 lines
4.4 KiB
Python
from datetime import datetime
|
|
from . import docker_client
|
|
from .. import db
|
|
from ..email import create_message, send
|
|
from ..models import JobResult
|
|
import docker
|
|
import logging
|
|
import json
|
|
import os
|
|
|
|
|
|
def create_job_service(job):
|
|
cmd = '{} -i /files -o /files/output'.format(job.service)
|
|
if job.service == 'file-setup':
|
|
cmd += ' -f {}'.format(job.secure_filename)
|
|
cmd += ' --log-dir /files'
|
|
cmd += ' --zip [{}]_{}'.format(job.service, job.secure_filename)
|
|
cmd += ' ' + ' '.join(json.loads(job.service_args))
|
|
service_kwargs = {'command': cmd,
|
|
'constraints': ['node.role==worker'],
|
|
'labels': {'origin': 'nopaque',
|
|
'type': 'service.{}'.format(job.service),
|
|
'job_id': str(job.id)},
|
|
'mounts': [job.path + ':/files:rw'],
|
|
'name': 'job_{}'.format(job.id),
|
|
'resources': docker.types.Resources(
|
|
cpu_reservation=job.n_cores * (10 ** 9),
|
|
mem_reservation=job.mem_mb * (10 ** 6)
|
|
),
|
|
'restart_policy': docker.types.RestartPolicy()}
|
|
service_image = ('gitlab.ub.uni-bielefeld.de:4567/sfb1288inf/'
|
|
+ job.service + ':' + job.service_version)
|
|
try:
|
|
docker_client.services.create(service_image, **service_kwargs)
|
|
except docker.errors.APIError as e:
|
|
logging.error('Create "{}" service raised '.format(service_kwargs['name']) # noqa
|
|
+ '[docker-APIError] The server returned an error. '
|
|
+ 'Details: {}'.format(e))
|
|
else:
|
|
job.status = 'queued'
|
|
msg = create_message(
|
|
job.creator.email,
|
|
'Status update for your Job "{}"'.format(job.title),
|
|
'tasks/email/notification',
|
|
job=job
|
|
)
|
|
send(msg)
|
|
|
|
|
|
def checkout_job_service(job):
|
|
service_name = 'job_{}'.format(job.id)
|
|
try:
|
|
service = docker_client.services.get(service_name)
|
|
except docker.errors.NotFound:
|
|
logging.error('Get "{}" service raised '.format(service_name)
|
|
+ '[docker-NotFound] The service does not exist. '
|
|
+ '(job.status: {} -> failed)'.format(job.status))
|
|
job.status = 'failed'
|
|
except docker.errors.APIError as e:
|
|
logging.error('Get "{}" service raised '.format(service_name)
|
|
+ '[docker-APIError] The server returned an error. '
|
|
+ 'Details: {}'.format(e))
|
|
except docker.errors.InvalidVersion:
|
|
logging.error('Get "{}" service raised '.format(service_name)
|
|
+ '[docker-InvalidVersion] One of the arguments is '
|
|
+ 'not supported with the current API version.')
|
|
else:
|
|
service_tasks = service.tasks()
|
|
if not service_tasks:
|
|
return
|
|
task_state = service_tasks[0].get('Status').get('State')
|
|
if job.status == 'queued' and task_state != 'pending':
|
|
job.status = 'running'
|
|
elif job.status == 'running' and task_state == 'complete':
|
|
service.remove()
|
|
job.end_date = datetime.utcnow()
|
|
job.status = task_state
|
|
if task_state == 'complete':
|
|
job_results_dir = os.path.join(job.path, 'output')
|
|
job_results = filter(lambda x: x.endswith('.zip'),
|
|
os.listdir(job_results_dir))
|
|
for job_result in job_results:
|
|
job_result = JobResult(filename=job_result, job=job)
|
|
db.session.add(job_result)
|
|
elif job.status == 'running' and task_state == 'failed':
|
|
service.remove()
|
|
job.end_date = datetime.utcnow()
|
|
job.status = task_state
|
|
finally:
|
|
# TODO: send email
|
|
msg = create_message(
|
|
job.creator.email,
|
|
'[nopaque] Status update for your Job "{}"'.format(job.title),
|
|
'tasks/email/notification',
|
|
job=job
|
|
)
|
|
send(msg)
|
|
pass
|
|
|
|
|
|
def remove_job_service(job):
|
|
service_name = 'job_{}'.format(job.id)
|
|
try:
|
|
service = docker_client.services.get(service_name)
|
|
except docker.errors.NotFound:
|
|
# TODO: send email
|
|
job.status = 'canceled'
|
|
# TODO: handle docker.errors.APIError and docker.errors.InvalidVersion
|
|
else:
|
|
service.update(mounts=None)
|
|
service.remove()
|