sporada/nopaque
1
0
Fork 0
mirror of https://gitlab.ub.uni-bielefeld.de/sfb1288inf/nopaque.git synced 2024-12-25 10:54:18 +00:00
Code Issues Projects Releases Wiki Activity

Change the APScheduler Logic and try to catch more errors in Daemon

Browse Source
This commit is contained in:
Patrick Jentsch 2022-07-04 14:11:10 +02:00
parent 76924956de
commit 5771e156ce
6 changed files with 56 additions and 104 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
app/__init__.py
View File

@ -33,7 +33,7 @@ def create_app(config: Config = Config) -> Flask:
app.config.from_object(config) app.config.from_object(config)
config.init_app(app) config.init_app(app)
docker_client.login( docker_client.login(
username=app.config['NOPAQUE_DOCKER_REGISTRY_USERNAME'], app.config['NOPAQUE_DOCKER_REGISTRY_USERNAME'],
password=app.config['NOPAQUE_DOCKER_REGISTRY_PASSWORD'], password=app.config['NOPAQUE_DOCKER_REGISTRY_PASSWORD'],
registry=app.config['NOPAQUE_DOCKER_REGISTRY'] registry=app.config['NOPAQUE_DOCKER_REGISTRY']
) )
@ -48,9 +48,6 @@ def create_app(config: Config = Config) -> Flask:
scheduler.init_app(app) scheduler.init_app(app)
socketio.init_app(app, message_queue=app.config['NOPAQUE_SOCKETIO_MESSAGE_QUEUE_URI']) # noqa socketio.init_app(app, message_queue=app.config['NOPAQUE_SOCKETIO_MESSAGE_QUEUE_URI']) # noqa
from . import tasks
tasks.register(app, scheduler)
from .admin import bp as admin_blueprint from .admin import bp as admin_blueprint
app.register_blueprint(admin_blueprint, url_prefix='/admin') app.register_blueprint(admin_blueprint, url_prefix='/admin')
@ -81,4 +78,7 @@ def create_app(config: Config = Config) -> Flask:
from .settings import bp as settings_blueprint from .settings import bp as settings_blueprint
app.register_blueprint(settings_blueprint, url_prefix='/settings') app.register_blueprint(settings_blueprint, url_prefix='/settings')
from .users import bp as users_blueprint
app.register_blueprint(users_blueprint, url_prefix='/users')
return app return app

4
app/daemon/__init__.py
View File

@ -1,9 +1,11 @@
from app import db from app import db
from flask import Flask
from .corpus_utils import check_corpora from .corpus_utils import check_corpora
from .job_utils import check_jobs from .job_utils import check_jobs
def daemon(): def daemon(app: Flask):
with app.app_context():
check_corpora() check_corpora()
check_jobs() check_jobs()
db.session.commit() db.session.commit()

73
app/daemon/corpus_utils.py
View File

@ -90,11 +90,8 @@ def _create_build_corpus_service(corpus):
restart_policy=restart_policy, restart_policy=restart_policy,
user='0:0' user='0:0'
) )
except docker.errors.APIError as e: except docker.errors.DockerException as e:
current_app.logger.error( current_app.logger.error(f'Create service "{name}" failed: {e}')
f'Create service "{name}" failed '
f'due to "docker.errors.APIError": {e}'
)
return return
corpus.status = CorpusStatus.QUEUED corpus.status = CorpusStatus.QUEUED
@ -103,17 +100,11 @@ def _checkout_build_corpus_service(corpus):
try: try:
service = docker_client.services.get(service_name) service = docker_client.services.get(service_name)
except docker.errors.NotFound as e: except docker.errors.NotFound as e:
current_app.logger.error( current_app.logger.error(f'Get service "{service_name}" failed: {e}')
f'Get service "{service_name}" failed '
f'due to "docker.errors.NotFound": {e}'
)
corpus.status = CorpusStatus.FAILED corpus.status = CorpusStatus.FAILED
return return
except docker.errors.APIError as e: except docker.errors.DockerException as e:
current_app.logger.error( current_app.logger.error(f'Get service "{service_name}" failed: {e}')
f'Get service "{service_name}" failed '
f'due to "docker.errors.APIError": {e}'
)
service_tasks = service.tasks() service_tasks = service.tasks()
if not service_tasks: if not service_tasks:
return return
@ -129,11 +120,8 @@ def _checkout_build_corpus_service(corpus):
return return
try: try:
service.remove() service.remove()
except docker.errors.APIError as e: except docker.errors.DockerException as e:
current_app.logger.error( current_app.logger.error(f'Remove service "{service_name}" failed: {e}')
f'Remove service "{service_name}" failed '
f'due to "docker.errors.APIError": {e}'
)
def _create_cqpserver_container(corpus): def _create_cqpserver_container(corpus):
''' # Docker container settings # ''' ''' # Docker container settings # '''
@ -174,20 +162,14 @@ def _create_cqpserver_container(corpus):
container = docker_client.containers.get(name) container = docker_client.containers.get(name)
except docker.errors.NotFound: except docker.errors.NotFound:
pass pass
except docker.errors.APIError as e: except docker.errors.DockerException as e:
current_app.logger.error( current_app.logger.error(f'Get container "{name}" failed: {e}')
f'Get container "{name}" failed '
f'due to "docker.errors.APIError": {e}'
)
return return
else: else:
try: try:
container.remove(force=True) container.remove(force=True)
except docker.errors.APIError as e: except docker.errors.DockerException as e:
current_app.logger.error( current_app.logger.error(f'Remove container "{name}" failed: {e}')
f'Remove container "{name}" failed '
f'due to "docker.errors.APIError": {e}'
)
return return
try: try:
docker_client.containers.run( docker_client.containers.run(
@ -207,11 +189,8 @@ def _create_cqpserver_container(corpus):
) )
corpus.status = CorpusStatus.FAILED corpus.status = CorpusStatus.FAILED
return return
except docker.errors.APIError as e: except docker.errors.DockerException as e:
current_app.logger.error( current_app.logger.error(f'Run container "{name}" failed: {e}')
f'Run container "{name}" failed '
f'due to "docker.errors.APIError" error: {e}'
)
return return
corpus.status = CorpusStatus.RUNNING_ANALYSIS_SESSION corpus.status = CorpusStatus.RUNNING_ANALYSIS_SESSION
@ -220,17 +199,11 @@ def _checkout_analysing_corpus_container(corpus):
try: try:
docker_client.containers.get(container_name) docker_client.containers.get(container_name)
except docker.errors.NotFound as e: except docker.errors.NotFound as e:
current_app.logger.error( current_app.logger.error(f'Get container "{container_name}" failed: {e}')
f'Get container "{container_name}" failed '
f'due to "docker.errors.NotFound": {e}'
)
corpus.num_analysis_sessions = 0 corpus.num_analysis_sessions = 0
corpus.status = CorpusStatus.BUILT corpus.status = CorpusStatus.BUILT
except docker.errors.APIError as e: except docker.errors.DockerException as e:
current_app.logger.error( current_app.logger.error(f'Get container "{container_name}" failed: {e}')
f'Get container "{container_name}" failed '
f'due to "docker.errors.APIError": {e}'
)
def _remove_cqpserver_container(corpus): def _remove_cqpserver_container(corpus):
container_name = f'cqpserver_{corpus.id}' container_name = f'cqpserver_{corpus.id}'
@ -239,16 +212,10 @@ def _remove_cqpserver_container(corpus):
except docker.errors.NotFound: except docker.errors.NotFound:
corpus.status = CorpusStatus.BUILT corpus.status = CorpusStatus.BUILT
return return
except docker.errors.APIError as e: except docker.errors.DockerException as e:
current_app.logger.error( current_app.logger.error(f'Get container "{container_name}" failed: {e}')
f'Get container "{container_name}" failed '
f'due to "docker.errors.APIError": {e}'
)
return return
try: try:
container.remove(force=True) container.remove(force=True)
except docker.errors.APIError as e: except docker.errors.DockerException as e:
current_app.logger.error( current_app.logger.error(f'Remove container "{container_name}" failed: {e}')
f'Remove container "{container_name}" failed '
f'due to "docker.errors.APIError": {e}'
)

47
app/daemon/job_utils.py
View File

@ -134,11 +134,8 @@ def _create_job_service(job):
restart_policy=restart_policy, restart_policy=restart_policy,
user='0:0' user='0:0'
) )
except docker.errors.APIError as e: except docker.errors.DockerException as e:
current_app.logger.error( current_app.logger.error(f'Create service "{name}" failed: {e}')
f'Create service "{name}" failed '
f'due to "docker.errors.APIError": {e}'
)
return return
job.status = JobStatus.QUEUED job.status = JobStatus.QUEUED
@ -147,17 +144,11 @@ def _checkout_job_service(job):
try: try:
service = docker_client.services.get(service_name) service = docker_client.services.get(service_name)
except docker.errors.NotFound as e: except docker.errors.NotFound as e:
current_app.logger.error( current_app.logger.error(f'Get service "{service_name}" failed: {e}')
f'Get service "{service_name}" failed '
f'due to "docker.errors.NotFound": {e}'
)
job.status = JobStatus.FAILED job.status = JobStatus.FAILED
return return
except docker.errors.APIError as e: except docker.errors.DockerException as e:
current_app.logger.error( current_app.logger.error(f'Get service "{service_name}" failed: {e}')
f'Get service "{service_name}" failed '
f'due to "docker.errors.APIError": {e}'
)
return return
service_tasks = service.tasks() service_tasks = service.tasks()
if not service_tasks: if not service_tasks:
@ -194,11 +185,8 @@ def _checkout_job_service(job):
job.end_date = datetime.utcnow() job.end_date = datetime.utcnow()
try: try:
service.remove() service.remove()
except docker.errors.APIError as e: except docker.errors.DockerException as e:
current_app.logger.error( current_app.logger.error(f'Remove service "{service_name}" failed: {e}')
f'Remove service "{service_name}" failed '
f'due to "docker.errors.APIError": {e}'
)
def _remove_job_service(job): def _remove_job_service(job):
service_name = f'job_{job.id}' service_name = f'job_{job.id}'
@ -207,24 +195,15 @@ def _remove_job_service(job):
except docker.errors.NotFound: except docker.errors.NotFound:
job.status = JobStatus.CANCELED job.status = JobStatus.CANCELED
return return
except docker.errors.APIError as e: except docker.errors.DockerException as e:
current_app.logger.error( current_app.logger.error(f'Get service "{service_name}" failed: {e}')
f'Get service "{service_name}" failed '
f'due to "docker.errors.APIError": {e}'
)
return return
try: try:
service.update(mounts=None) service.update(mounts=None)
except docker.errors.APIError as e: except docker.errors.DockerException as e:
current_app.logger.error( current_app.logger.error(f'Update service "{service_name}" failed: {e}')
f'Update service "{service_name}" failed '
f'due to "docker.errors.APIError": {e}'
)
return return
try: try:
service.remove() service.remove()
except docker.errors.APIError as e: except docker.errors.DockerException as e:
current_app.logger.error( current_app.logger.error(f'Remove "{service_name}" service failed: {e}')
f'Remove "{service_name}" service failed '
f'due to "docker.errors.APIError": {e}'
)

9
app/tasks.py
View File

@ -1,9 +0,0 @@
from app.daemon import daemon
def register(app, scheduler):
if app.config['NOPAQUE_IS_PRIMARY_INSTANCE']:
@scheduler.task('interval', id='daemon', seconds=3)
def daemon_task():
with app.app_context():
daemon()

13
config.py
View File

@ -18,6 +18,9 @@ class Config:
SESSION_COOKIE_SECURE = \ SESSION_COOKIE_SECURE = \
os.environ.get('SESSION_COOKIE_SECURE', 'false').lower() == 'true' os.environ.get('SESSION_COOKIE_SECURE', 'false').lower() == 'true'
''' # Flask-APScheduler # '''
JOBS = []
''' # Flask-Assets ''' ''' # Flask-Assets '''
ASSETS_DEBUG = os.environ.get('ASSETS_DEBUG', 'false').lower() == 'true' ASSETS_DEBUG = os.environ.get('ASSETS_DEBUG', 'false').lower() == 'true'
@ -112,6 +115,16 @@ class Config:
fmt=app.config['NOPAQUE_LOG_FORMAT'], fmt=app.config['NOPAQUE_LOG_FORMAT'],
datefmt=app.config['NOPAQUE_LOG_DATE_FORMAT'] datefmt=app.config['NOPAQUE_LOG_DATE_FORMAT']
) )
if app.config['NOPAQUE_IS_PRIMARY_INSTANCE']:
app.config['JOBS'].append(
{
"id": "daemon",
"func": "app.daemon:daemon",
"args": (app,),
"trigger": "interval",
"seconds": 3,
}
)
if app.config['NOPAQUE_LOG_STDERR_ENABLED']: if app.config['NOPAQUE_LOG_STDERR_ENABLED']:
stream_handler = logging.StreamHandler() stream_handler = logging.StreamHandler()
stream_handler.setFormatter(formatter) stream_handler.setFormatter(formatter)