nopaque/web/app/tasks/job_utils.py

from datetime import datetime
from . import docker_client
from .. import db
from ..email import create_message, send
from ..models import JobResult
import docker
import logging
import json
import os


def create_job_service(job):
    cmd = '{} -i /files -o /files/output'.format(job.service)
    if job.service == 'file-setup':
        cmd += ' -f {}'.format(job.secure_filename)
    cmd += ' --log-dir /files'
    cmd += ' --zip [{}]_{}'.format(job.service, job.secure_filename)
    cmd += ' ' + ' '.join(json.loads(job.service_args))
    service_kwargs = {'command': cmd,
                      'constraints': ['node.role==worker'],
                      'labels': {'origin': 'nopaque',
                                 'type': 'service.{}'.format(job.service),
                                 'job_id': str(job.id)},
                      'mounts': [job.path + ':/files:rw'],
                      'name': 'job_{}'.format(job.id),
                      'resources': docker.types.Resources(
                          cpu_reservation=job.n_cores * (10 ** 9),
                          mem_reservation=job.mem_mb * (10 ** 6)
                      ),
                      'restart_policy': docker.types.RestartPolicy()}
    service_image = ('gitlab.ub.uni-bielefeld.de:4567/sfb1288inf/'
                     + job.service + ':' + job.service_version)
    try:
        docker_client.services.create(service_image, **service_kwargs)
    except docker.errors.APIError as e:
        logging.error('Create "{}" service raised '.format(service_kwargs['name'])  # noqa
                      + '[docker-APIError] The server returned an error. '
                      + 'Details: {}'.format(e))
    else:
        job.status = 'queued'
        msg = create_message(
            job.creator.email,
            'Status update for your Job "{}"'.format(job.title),
            'tasks/email/notification',
            job=job
        )
        send(msg)


def checkout_job_service(job):
    service_name = 'job_{}'.format(job.id)
    try:
        service = docker_client.services.get(service_name)
    except docker.errors.NotFound:
        logging.error('Get "{}" service raised '.format(service_name)
                      + '[docker-NotFound] The service does not exist. '
                      + '(job.status: {} -> failed)'.format(job.status))
        job.status = 'failed'
    except docker.errors.APIError as e:
        logging.error('Get "{}" service raised '.format(service_name)
                      + '[docker-APIError] The server returned an error. '
                      + 'Details: {}'.format(e))
    except docker.errors.InvalidVersion:
        logging.error('Get "{}" service raised '.format(service_name)
                      + '[docker-InvalidVersion] One of the arguments is '
                      + 'not supported with the current API version.')
    else:
        service_tasks = service.tasks()
        if not service_tasks:
            return
        task_state = service_tasks[0].get('Status').get('State')
        if job.status == 'queued' and task_state != 'pending':
            job.status = 'running'
        elif job.status == 'running' and task_state == 'complete':
            service.remove()
            job.end_date = datetime.utcnow()
            job.status = task_state
            if task_state == 'complete':
                job_results_dir = os.path.join(job.path, 'output')
                job_results = filter(lambda x: x.endswith('.zip'),
                                     os.listdir(job_results_dir))
                for job_result in job_results:
                    job_result = JobResult(filename=job_result, job=job)
                    db.session.add(job_result)
        elif job.status == 'running' and task_state == 'failed':
            service.remove()
            job.end_date = datetime.utcnow()
            job.status = task_state
    finally:
        # TODO: send email
        msg = create_message(
            job.creator.email,
            '[nopaque] Status update for your Job "{}"'.format(job.title),
            'tasks/email/notification',
            job=job
        )
        send(msg)
        pass


def remove_job_service(job):
    service_name = 'job_{}'.format(job.id)
    try:
        service = docker_client.services.get(service_name)
    except docker.errors.NotFound:
        # TODO: send email
        job.status = 'canceled'
    # TODO: handle docker.errors.APIError and docker.errors.InvalidVersion
    else:
        service.update(mounts=None)
        service.remove()