From b3e467beaca50bb36553f502713987d657c63adb Mon Sep 17 00:00:00 2001 From: Patrick Jentsch Date: Wed, 10 Jun 2020 14:18:02 +0200 Subject: [PATCH 1/2] Update daemon --- daemon/nopaqued.py | 30 +++++++++++++++++++----------- daemon/tasks/check_corpora.py | 4 +++- daemon/tasks/check_jobs.py | 2 ++ daemon/tasks/decorators.py | 14 ++++++++++++++ daemon/tasks/notify.py | 2 ++ 5 files changed, 40 insertions(+), 12 deletions(-) create mode 100644 daemon/tasks/decorators.py diff --git a/daemon/nopaqued.py b/daemon/nopaqued.py index 9ec1bf5a..01eedb90 100644 --- a/daemon/nopaqued.py +++ b/daemon/nopaqued.py @@ -1,25 +1,33 @@ -from concurrent.futures import ThreadPoolExecutor +from logger.logger import init_logger from tasks.check_corpora import check_corpora from tasks.check_jobs import check_jobs from tasks.notify import notify +from time import sleep import os -# TODO: Check if thread is still alive and execute next thread after that -# TODO: Check line length - - def nopaqued(): - execute_notifications = bool(os.environ.get('NOPAQUE_EXECUTE_NOTIFICATIONS', True)) # noqa - # executing background functions + logger = init_logger() + NOPAQUE_EXECUTE_NOTIFICATIONS = os.environ.get('NOPAQUE_EXECUTE_NOTIFICATIONS', 'True').lower() == 'true' # noqa + threads = {'check_corpora': None, 'check_jobs': None, 'notify': None} + + threads['check_corpora'] = check_corpora() + threads['check_jobs'] = check_jobs() + threads['notify'] = notify(NOPAQUE_EXECUTE_NOTIFICATIONS) while True: - with ThreadPoolExecutor(max_workers=3) as executor: - executor.submit(check_jobs) - executor.submit(check_corpora) - executor.submit(notify, execute_notifications) + logger.warning('check_corpora: {}'.format(threads['check_corpora'].is_alive())) + if not threads['check_corpora'].is_alive(): + threads['check_corpora'] = check_corpora() + logger.warning('check_jobs: {}'.format(threads['check_jobs'].is_alive())) + if not threads['check_jobs'].is_alive(): + threads['check_jobs'] = check_jobs() + logger.warning('notify: {}'.format(threads['notify'].is_alive())) + if not threads['notify'].is_alive(): + threads['notify'] = notify(NOPAQUE_EXECUTE_NOTIFICATIONS) # If execute_notifications True mails are sent. # If execute_notifications False no mails are sent. # But notification status will be set nonetheless. + sleep(3) if __name__ == '__main__': diff --git a/daemon/tasks/check_corpora.py b/daemon/tasks/check_corpora.py index fb8e13fc..588d801d 100644 --- a/daemon/tasks/check_corpora.py +++ b/daemon/tasks/check_corpora.py @@ -1,11 +1,13 @@ from logger.logger import init_logger from tasks import Session, docker_client, NOPAQUE_STORAGE +from tasks.decorators import background from tasks.Models import Corpus import docker import os import shutil +@background def check_corpora(): c_session = Session() corpora = c_session.query(Corpus).all() @@ -101,7 +103,7 @@ def __create_cqpserver_container(corpus): 'volumes': [corpus_data_dir + ':/corpora/data:rw', corpus_registry_dir + ':/usr/local/share/cwb/registry:rw'], 'name': 'cqpserver_{}'.format(corpus.id), - 'network': 'opaque_default'} + 'network': 'nopaque_default'} container_image = ('gitlab.ub.uni-bielefeld.de:4567/sfb1288inf/cqpserver:latest') try: container = docker_client.containers.get(container_args['name']) diff --git a/daemon/tasks/check_jobs.py b/daemon/tasks/check_jobs.py index 09dd24ac..e779bbff 100644 --- a/daemon/tasks/check_jobs.py +++ b/daemon/tasks/check_jobs.py @@ -1,12 +1,14 @@ from datetime import datetime from logger.logger import init_logger from tasks import Session, docker_client, NOPAQUE_STORAGE +from tasks.decorators import background from tasks.Models import Job, NotificationData, NotificationEmailData, JobResult import docker import json import os +@background def check_jobs(): # logger = init_logger() cj_session = Session() diff --git a/daemon/tasks/decorators.py b/daemon/tasks/decorators.py new file mode 100644 index 00000000..040250a8 --- /dev/null +++ b/daemon/tasks/decorators.py @@ -0,0 +1,14 @@ +from functools import wraps +from threading import Thread + + +def background(f): + ''' + ' This decorator executes a function in a Thread. + ''' + @wraps(f) + def wrapped(*args, **kwargs): + thread = Thread(target=f, args=args, kwargs=kwargs) + thread.start() + return thread + return wrapped diff --git a/daemon/tasks/notify.py b/daemon/tasks/notify.py index 4d85be4f..a0ff75d4 100644 --- a/daemon/tasks/notify.py +++ b/daemon/tasks/notify.py @@ -2,10 +2,12 @@ from notify.notification import Notification from notify.service import NotificationService from sqlalchemy import asc from tasks import Session +from tasks.decorators import background from tasks.Models import NotificationEmailData import os +@background def notify(execute_flag): # If True mails are sent normaly # If False mails are not sent. Used to avoid sending mails for jobs that From 4735df7c6080df877c1d0db50028a69f6bf472c0 Mon Sep 17 00:00:00 2001 From: Patrick Jentsch Date: Wed, 10 Jun 2020 14:25:57 +0200 Subject: [PATCH 2/2] Add missing configuration to .env.tpl --- .env.tpl | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.env.tpl b/.env.tpl index 3a8c84e1..68a5c792 100644 --- a/.env.tpl +++ b/.env.tpl @@ -45,6 +45,8 @@ NOPAQUE_ADMIN= NOPAQUE_CONTACT= # Example: nopaque.localhost NOPAQUE_DOMAIN= +# Choose one: False, True +NOPAQUE_EXECUTE_NOTIFICATIONS= # Choose one: CRITICAL, ERROR, WARNING, INFO, DEBUG NOPAQUE_LOG_LEVEL= # Example: nopaque Admin