mirror of
https://gitlab.ub.uni-bielefeld.de/sfb1288inf/nopaque.git
synced 2025-06-20 13:00:36 +00:00
Simplify daemon logic
This commit is contained in:
Patrick Jentsch
35
web/app/tasks/__init__.py
Normal file
35
web/app/tasks/__init__.py
Normal file
@ -0,0 +1,35 @@
|
||||
from .. import db
|
||||
from ..models import Corpus, Job
|
||||
import docker
|
||||
|
||||
|
||||
docker_client = docker.from_env()
|
||||
from . import corpus_utils, job_utils # noqa
|
||||
|
||||
|
||||
def check_corpora():
|
||||
corpora = Corpus.query.all()
|
||||
for corpus in filter(lambda corpus: corpus.status == 'submitted', corpora):
|
||||
corpus_utils.create_build_corpus_service(corpus)
|
||||
for corpus in filter(lambda corpus: (corpus.status == 'queued'
|
||||
or corpus.status == 'running'),
|
||||
corpora):
|
||||
corpus_utils.checkout_build_corpus_service(corpus)
|
||||
for corpus in filter(lambda corpus: corpus.status == 'start analysis',
|
||||
corpora):
|
||||
corpus_utils.create_cqpserver_container(corpus)
|
||||
for corpus in filter(lambda corpus: corpus.status == 'stop analysis',
|
||||
corpora):
|
||||
corpus_utils.remove_cqpserver_container(corpus)
|
||||
db.session.commit()
|
||||
|
||||
|
||||
def check_jobs():
|
||||
jobs = Job.query.all()
|
||||
for job in filter(lambda job: job.status == 'submitted', jobs):
|
||||
job_utils.create_job_service(job)
|
||||
for job in filter(lambda job: job.status == 'queued', jobs):
|
||||
job_utils.checkout_job_service(job)
|
||||
for job in filter(lambda job: job.status == 'running', jobs):
|
||||
job_utils.checkout_job_service(job)
|
||||
db.session.commit()
|
||||
120
web/app/tasks/corpus_utils.py
Normal file
120
web/app/tasks/corpus_utils.py
Normal file
@ -0,0 +1,120 @@
|
||||
from flask import current_app
|
||||
from . import docker_client
|
||||
import docker
|
||||
import logging
|
||||
import os
|
||||
import shutil
|
||||
|
||||
|
||||
def create_build_corpus_service(corpus):
|
||||
corpus_dir = os.path.join(current_app.config['DATA_DIR'],
|
||||
str(corpus.user_id),
|
||||
'corpora',
|
||||
str(corpus.id))
|
||||
corpus_data_dir = os.path.join(corpus_dir, 'data')
|
||||
corpus_file = os.path.join(corpus_dir, 'merged', 'corpus.vrt')
|
||||
corpus_registry_dir = os.path.join(corpus_dir, 'registry')
|
||||
if os.path.exists(corpus_data_dir):
|
||||
shutil.rmtree(corpus_data_dir)
|
||||
if os.path.exists(corpus_registry_dir):
|
||||
shutil.rmtree(corpus_registry_dir)
|
||||
os.mkdir(corpus_data_dir)
|
||||
os.mkdir(corpus_registry_dir)
|
||||
service_args = {
|
||||
'command': 'docker-entrypoint.sh build-corpus',
|
||||
'constraints': ['node.role==worker'],
|
||||
'labels': {'origin': 'nopaque',
|
||||
'type': 'corpus.prepare',
|
||||
'corpus_id': str(corpus.id)},
|
||||
'mounts': [corpus_file + ':/root/files/corpus.vrt:ro',
|
||||
corpus_data_dir + ':/corpora/data:rw',
|
||||
corpus_registry_dir + ':/usr/local/share/cwb/registry:rw'],
|
||||
'name': 'build-corpus_{}'.format(corpus.id),
|
||||
'restart_policy': docker.types.RestartPolicy()
|
||||
}
|
||||
service_image = \
|
||||
'gitlab.ub.uni-bielefeld.de:4567/sfb1288inf/cqpserver:latest'
|
||||
try:
|
||||
docker_client.services.create(service_image, **service_args)
|
||||
except docker.errors.APIError as e:
|
||||
logging.error('create_build_corpus_service({}): '.format(corpus.id)
|
||||
+ '{} (status: {} -> failed)'.format(e, corpus.status))
|
||||
corpus.status = 'failed'
|
||||
else:
|
||||
corpus.status = 'queued'
|
||||
finally:
|
||||
# TODO: send email
|
||||
pass
|
||||
|
||||
|
||||
def checkout_build_corpus_service(corpus):
|
||||
service_name = 'build-corpus_{}'.format(corpus.id)
|
||||
try:
|
||||
service = docker_client.services.get(service_name)
|
||||
except docker.errors.NotFound as e:
|
||||
logging.error('checkout_build_corpus_service({}):'.format(corpus.id)
|
||||
+ ' {} (stauts: {} -> failed)'.format(e, corpus.status))
|
||||
corpus.status = 'failed'
|
||||
# TODO: handle docker.errors.APIError and docker.errors.InvalidVersion
|
||||
else:
|
||||
service_tasks = service.tasks()
|
||||
if not service_tasks:
|
||||
return
|
||||
task_state = service_tasks[0].get('Status').get('State')
|
||||
if corpus.status == 'queued' and task_state != 'pending':
|
||||
corpus.status = 'running'
|
||||
elif corpus.status == 'running' and task_state == 'complete':
|
||||
service.remove()
|
||||
corpus.status = 'prepared'
|
||||
elif corpus.status == 'running' and task_state == 'failed':
|
||||
service.remove()
|
||||
corpus.status = task_state
|
||||
finally:
|
||||
# TODO: send email
|
||||
pass
|
||||
|
||||
|
||||
def create_cqpserver_container(corpus):
|
||||
corpus_dir = os.path.join(current_app.config['DATA_DIR'],
|
||||
str(corpus.user_id),
|
||||
'corpora',
|
||||
str(corpus.id))
|
||||
corpus_data_dir = os.path.join(corpus_dir, 'data')
|
||||
corpus_registry_dir = os.path.join(corpus_dir, 'registry')
|
||||
container_args = {
|
||||
'command': 'cqpserver',
|
||||
'detach': True,
|
||||
'volumes': [corpus_data_dir + ':/corpora/data:rw',
|
||||
corpus_registry_dir + ':/usr/local/share/cwb/registry:rw'],
|
||||
'name': 'cqpserver_{}'.format(corpus.id),
|
||||
'network': 'nopaque_default'
|
||||
}
|
||||
container_image = \
|
||||
'gitlab.ub.uni-bielefeld.de:4567/sfb1288inf/cqpserver:latest'
|
||||
try:
|
||||
container = docker_client.containers.get(container_args['name'])
|
||||
except docker.errors.NotFound:
|
||||
pass
|
||||
except docker.errors.DockerException:
|
||||
return
|
||||
else:
|
||||
container.remove(force=True)
|
||||
try:
|
||||
docker_client.containers.run(container_image, **container_args)
|
||||
except docker.errors.DockerException:
|
||||
return
|
||||
else:
|
||||
corpus.status = 'analysing'
|
||||
|
||||
|
||||
def remove_cqpserver_container(corpus):
|
||||
container_name = 'cqpserver_{}'.format(corpus.id)
|
||||
try:
|
||||
container = docker_client.containers.get(container_name)
|
||||
except docker.errors.NotFound:
|
||||
pass
|
||||
except docker.errors.DockerException:
|
||||
return
|
||||
else:
|
||||
container.remove(force=True)
|
||||
corpus.status = 'prepared'
|
||||
101
web/app/tasks/job_utils.py
Normal file
101
web/app/tasks/job_utils.py
Normal file
@ -0,0 +1,101 @@
|
||||
from datetime import datetime
|
||||
from flask import current_app
|
||||
from . import docker_client
|
||||
from .. import db
|
||||
from ..models import JobResult
|
||||
import docker
|
||||
import logging
|
||||
import json
|
||||
import os
|
||||
|
||||
|
||||
def create_job_service(job):
|
||||
job_dir = os.path.join(current_app.config['DATA_DIR'],
|
||||
str(job.user_id),
|
||||
'jobs',
|
||||
str(job.id))
|
||||
cmd = '{} -i /files -o /files/output'.format(job.service)
|
||||
if job.service == 'file-setup':
|
||||
cmd += ' -f {}'.format(job.secure_filename)
|
||||
cmd += ' --log-dir /files'
|
||||
cmd += ' --zip [{}]_{}'.format(job.service, job.secure_filename)
|
||||
cmd += ' ' + ' '.join(json.loads(job.service_args))
|
||||
service_args = {'command': cmd,
|
||||
'constraints': ['node.role==worker'],
|
||||
'labels': {'origin': 'nopaque',
|
||||
'type': 'service.{}'.format(job.service),
|
||||
'job_id': str(job.id)},
|
||||
'mounts': [job_dir + ':/files:rw'],
|
||||
'name': 'job_{}'.format(job.id),
|
||||
'resources': docker.types.Resources(
|
||||
cpu_reservation=job.n_cores * (10 ** 9),
|
||||
mem_reservation=job.mem_mb * (10 ** 6)),
|
||||
'restart_policy': docker.types.RestartPolicy()}
|
||||
service_image = ('gitlab.ub.uni-bielefeld.de:4567/sfb1288inf/'
|
||||
+ job.service + ':' + job.service_version)
|
||||
try:
|
||||
docker_client.services.create(service_image, **service_args)
|
||||
except docker.errors.APIError as e:
|
||||
logging.error('create_job_service({}): {} '.format(job.id, e)
|
||||
+ '(status: {} -> failed)'.format(job.status))
|
||||
job.status = 'failed'
|
||||
else:
|
||||
job.status = 'queued'
|
||||
finally:
|
||||
# TODO: send email
|
||||
pass
|
||||
|
||||
|
||||
def checkout_job_service(job):
|
||||
service_name = 'job_{}'.format(job.id)
|
||||
try:
|
||||
service = docker_client.services.get(service_name)
|
||||
except docker.errors.NotFound as e:
|
||||
logging.error('checkout_job_service({}): {} '.format(job.id, e)
|
||||
+ '(status: {} -> submitted)'.format(job.status))
|
||||
job.status = 'submitted'
|
||||
# TODO: handle docker.errors.APIError and docker.errors.InvalidVersion
|
||||
else:
|
||||
service_tasks = service.tasks()
|
||||
if not service_tasks:
|
||||
return
|
||||
task_state = service_tasks[0].get('Status').get('State')
|
||||
if job.status == 'queued' and task_state != 'pending':
|
||||
job.status = 'running'
|
||||
elif job.status == 'queued' and task_state == 'complete':
|
||||
service.remove()
|
||||
job.end_date = datetime.utcnow()
|
||||
job.status = task_state
|
||||
if task_state == 'complete':
|
||||
results_dir = os.path.join(current_app.config['DATA_DIR'],
|
||||
str(job.user_id),
|
||||
'jobs',
|
||||
str(job.id),
|
||||
'output')
|
||||
results = filter(lambda x: x.endswith('.zip'),
|
||||
os.listdir(results_dir))
|
||||
for result in results:
|
||||
job_result = JobResult(dir=results_dir,
|
||||
filename=result,
|
||||
job_id=job.id)
|
||||
db.session.add(job_result)
|
||||
elif job.status == 'running' and task_state == 'failed':
|
||||
service.remove()
|
||||
job.end_date = datetime.utcnow()
|
||||
job.status = task_state
|
||||
finally:
|
||||
# TODO: send email
|
||||
pass
|
||||
|
||||
|
||||
def remove_job_service(job):
|
||||
service_name = 'job_{}'.format(job.id)
|
||||
try:
|
||||
service = docker_client.services.get(service_name)
|
||||
except docker.errors.NotFound:
|
||||
# TODO: send email
|
||||
job.status = 'canceled'
|
||||
# TODO: handle docker.errors.APIError and docker.errors.InvalidVersion
|
||||
else:
|
||||
service.update(mounts=None)
|
||||
service.remove()
|
||||
Reference in New Issue
Block a user