nopaque/web/app/tasks/corpus_utils.py

from . import docker_client
import docker
import logging
import os
import shutil


def create_build_corpus_service(corpus):
    corpus_data_dir = os.path.join(corpus.path, 'data')
    shutil.rmtree(corpus_data_dir, ignore_errors=True)
    os.mkdir(corpus_data_dir)
    corpus_registry_dir = os.path.join(corpus.path, 'registry')
    shutil.rmtree(corpus_registry_dir, ignore_errors=True)
    os.mkdir(corpus_registry_dir)
    corpus_file = os.path.join(corpus.path, 'merged', 'corpus.vrt')
    service_kwargs = {
        'command': 'docker-entrypoint.sh build-corpus',
        'constraints': ['node.role==worker'],
        'labels': {'origin': 'nopaque',
                   'type': 'corpus.prepare',
                   'corpus_id': str(corpus.id)},
        'mounts': [corpus_file + ':/root/files/corpus.vrt:ro',
                   corpus_data_dir + ':/corpora/data:rw',
                   corpus_registry_dir + ':/usr/local/share/cwb/registry:rw'],
        'name': 'build-corpus_{}'.format(corpus.id),
        'restart_policy': docker.types.RestartPolicy()
    }
    service_image = 'gitlab.ub.uni-bielefeld.de:4567/sfb1288inf/cqpserver:latest'  # noqa
    try:
        docker_client.services.create(service_image, **service_kwargs)
    except docker.errors.APIError as e:
        logging.error('Create "{}" service raised '.format(service_kwargs['name'])  # noqa
                      + '[docker-APIError] The server returned an error. '
                      + 'Details: {}'.format(e))
    else:
        corpus.status = 'queued'


def checkout_build_corpus_service(corpus):
    service_name = 'build-corpus_{}'.format(corpus.id)
    try:
        service = docker_client.services.get(service_name)
    except docker.errors.NotFound:
        logging.error('Get "{}" service raised '.format(service_name)
                      + '[docker-NotFound] The service does not exist. '
                      + '(corpus.status: {} -> failed)'.format(corpus.status))
        corpus.status = 'failed'
    except docker.errors.APIError as e:
        logging.error('Get "{}" service raised '.format(service_name)
                      + '[docker-APIError] The server returned an error. '
                      + 'Details: {}'.format(e))
    except docker.errors.InvalidVersion:
        logging.error('Get "{}" service raised '.format(service_name)
                      + '[docker-InvalidVersion] One of the arguments is '
                      + 'not supported with the current API version.')
    else:
        service_tasks = service.tasks()
        if not service_tasks:
            return
        task_state = service_tasks[0].get('Status').get('State')
        if corpus.status == 'queued' and task_state != 'pending':
            corpus.status = 'running'
        elif corpus.status == 'running' and task_state in ['complete', 'failed']:  # noqa
            try:
                service.remove()
            except docker.errors.APIError as e:
                logging.error('Remove "{}" service raised '.format(service_name)  # noqa
                              + '[docker-APIError] The server returned an error. '  # noqa
                              + 'Details: {}'.format(e))
                return
            else:
                corpus.status = 'prepared' if task_state == 'complete' \
                                else 'failed'


def create_cqpserver_container(corpus):
    corpus_data_dir = os.path.join(corpus.path, 'data')
    corpus_registry_dir = os.path.join(corpus.path, 'registry')
    container_kwargs = {
        'command': 'cqpserver',
        'detach': True,
        'volumes': [corpus_data_dir + ':/corpora/data:rw',
                    corpus_registry_dir + ':/usr/local/share/cwb/registry:rw'],
        'name': 'cqpserver_{}'.format(corpus.id),
        'network': 'nopaque_default'
    }
    container_image = 'gitlab.ub.uni-bielefeld.de:4567/sfb1288inf/cqpserver:latest'  # noqa
    # Check if a cqpserver container already exists. If this is the case,
    # remove it and create a new one
    try:
        container = docker_client.containers.get(container_kwargs['name'])
    except docker.errors.NotFound:
        pass
    except docker.errors.APIError as e:
        logging.error('Get "{}" container raised '.format(container_kwargs['name'])
                      + '[docker-APIError] The server returned an error. '
                      + 'Details: {}'.format(e))
        return
    else:
        try:
            container.remove(force=True)
        except docker.errors.APIError as e:
            logging.error('Remove "{}" container raised '.format(container_kwargs['name'])
                          + '[docker-APIError] The server returned an error. '
                          + 'Details: {}'.format(e))
            return
    try:
        docker_client.containers.run(container_image, **container_kwargs)
    except docker.errors.ContainerError:
        # This case should not occur, because detach is True.
        logging.error('Run "{}" container raised '.format(container_kwargs['name'])
                      + '[docker-ContainerError] The container exits with a '
                      + 'non-zero exit code and detach is False.')
        corpus.status = 'failed'
    except docker.errors.ImageNotFound:
        logging.error('Run "{}" container raised '.format(container_kwargs['name'])
                      + '[docker-ImageNotFound] The specified image does not '
                      + 'exist.')
        corpus.status = 'failed'
    except docker.errors.APIError as e:
        logging.error('Run "{}" container raised '.format(container_kwargs['name'])
                      + '[docker-APIError] The server returned an error. '
                      + 'Details: {}'.format(e))
    else:
        corpus.status = 'analysing'


def remove_cqpserver_container(corpus):
    container_name = 'cqpserver_{}'.format(corpus.id)
    try:
        container = docker_client.containers.get(container_name)
    except docker.errors.NotFound:
        pass
    except docker.errors.APIError as e:
        logging.error('Get "{}" container raised '.format(container_name)
                      + '[docker-APIError] The server returned an error. '
                      + 'Details: {}'.format(e))
        return
    else:
        try:
            container.remove(force=True)
        except docker.errors.APIError as e:
            logging.error('Remove "{}" container raised '.format(container_name)
                          + '[docker-APIError] The server returned an error. '
                          + 'Details: {}'.format(e))
            return
    corpus.status = 'prepared'