mirror of
https://gitlab.ub.uni-bielefeld.de/sfb1288inf/nopaque.git
synced 2024-12-24 10:34:17 +00:00
Avoid that threads of the same type can be executed twice or thrice etc.
This commit is contained in:
Stephan Porada
parent
9e88330433
commit
b56a8362ff
@ -28,7 +28,7 @@ WORKDIR /home/nopaqued
|
||||
COPY ["logger", "logger"]
|
||||
COPY ["notify", "notify"]
|
||||
COPY ["tasks", "tasks"]
|
||||
COPY ["decorators.py", "nopaqued.py", "requirements.txt", "./"]
|
||||
COPY ["nopaqued.py", "requirements.txt", "./"]
|
||||
RUN python -m venv venv \
|
||||
&& venv/bin/pip install --requirement requirements.txt \
|
||||
&& mkdir logs
|
||||
|
||||
@ -1,14 +0,0 @@
|
||||
from functools import wraps
|
||||
from threading import Thread
|
||||
|
||||
|
||||
def background(f):
|
||||
'''
|
||||
' This decorator executes a function in a Thread.
|
||||
'''
|
||||
@wraps(f)
|
||||
def wrapped(*args, **kwargs):
|
||||
thread = Thread(target=f, args=args, kwargs=kwargs)
|
||||
thread.start()
|
||||
return thread
|
||||
return wrapped
|
||||
@ -1,34 +1,23 @@
|
||||
from tasks.check_jobs import check_jobs
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
from tasks.check_corpora import check_corpora
|
||||
from tasks.check_jobs import check_jobs
|
||||
from tasks.notify import notify
|
||||
from time import sleep
|
||||
|
||||
|
||||
# TODO: Check if thread is still alive and execute next thread after that
|
||||
# TODO: Remove unnecessary commits
|
||||
# TODO: Check line length
|
||||
# check_jobs_thread = None
|
||||
# check_corpora_thread = None
|
||||
# notify_thread = None
|
||||
#
|
||||
#
|
||||
# def nopaqued():
|
||||
# # executing background functions
|
||||
# while True:
|
||||
# check_jobs_thread = check_jobs()
|
||||
# check_corpora_thread = check_corpora()
|
||||
# notify_thread = notify(True) # If True mails are sent. If False no mails are sent.
|
||||
# # But notification status will be set nonetheless.
|
||||
# sleep(3)
|
||||
|
||||
|
||||
def nopaqued():
|
||||
# executing background functions
|
||||
while True:
|
||||
check_jobs()
|
||||
check_corpora()
|
||||
notify(True) # If True mails are sent. If False no mails are sent.
|
||||
# But notification status will be set nonetheless.
|
||||
with ThreadPoolExecutor(max_workers=3) as executor:
|
||||
executor.submit(check_jobs)
|
||||
executor.submit(check_corpora)
|
||||
executor.submit(notify, True) # If True mails are sent.
|
||||
# If False no mails are sent.
|
||||
# But notification status will be set nonetheless.
|
||||
sleep(3)
|
||||
|
||||
|
||||
|
||||
@ -8,10 +8,10 @@ class NotificationService(object):
|
||||
def __init__(self, execute_flag):
|
||||
super(NotificationService, self).__init__()
|
||||
self.execute_flag = execute_flag # If True mails are sent normaly
|
||||
# If False mails are not sent. Used to avoid sending mails for jobs that
|
||||
# have been completed a long time ago. Use this if you implement notify
|
||||
# into an already existing nopaque instance. Change it to True after the
|
||||
# daemon has run one time with the flag set to False
|
||||
# If False mails are not sent. Used to avoid sending mails for jobs
|
||||
# that have been completed a long time ago. Use this if you implement
|
||||
# notify into an already existing nopaque instance. Change it to True
|
||||
# after the daemon has run one time with the flag set to False
|
||||
self.not_sent = {} # Holds due to an error unsent email notifications
|
||||
self.mail_limit_exceeded = False # Bool to show if the mail server
|
||||
# stoped sending mails due to exceeding its sending limit
|
||||
|
||||
@ -1,4 +1,3 @@
|
||||
from decorators import background
|
||||
from logger.logger import init_logger
|
||||
from tasks import Session, docker_client, NOPAQUE_STORAGE
|
||||
from tasks.Models import Corpus
|
||||
@ -7,7 +6,6 @@ import os
|
||||
import shutil
|
||||
|
||||
|
||||
@background
|
||||
def check_corpora():
|
||||
c_session = Session()
|
||||
corpora = c_session.query(Corpus).all()
|
||||
|
||||
@ -1,5 +1,4 @@
|
||||
from datetime import datetime
|
||||
from decorators import background
|
||||
from logger.logger import init_logger
|
||||
from tasks import Session, docker_client, NOPAQUE_STORAGE
|
||||
from tasks.Models import Job, NotificationData, NotificationEmailData, JobResult
|
||||
@ -8,7 +7,6 @@ import json
|
||||
import os
|
||||
|
||||
|
||||
@background
|
||||
def check_jobs():
|
||||
# logger = init_logger()
|
||||
cj_session = Session()
|
||||
@ -49,7 +47,8 @@ def __add_notification_data(job, notified_on_status, scoped_session):
|
||||
if (notification_exists == 0):
|
||||
notification_data = NotificationData(job_id=job.id)
|
||||
scoped_session.add(notification_data)
|
||||
scoped_session.commit() # If no commit job will have no NotificationData
|
||||
scoped_session.commit()
|
||||
# If no commit job will have no NotificationData
|
||||
# logger.warning('Created NotificationData for current Job.'))
|
||||
else:
|
||||
pass
|
||||
@ -70,10 +69,14 @@ def __create_job_service(job):
|
||||
job_dir = os.path.join(NOPAQUE_STORAGE, str(job.user_id), 'jobs',
|
||||
str(job.id))
|
||||
service_args = {'command': ('{} /files /files/output'.format(job.service)
|
||||
+ ' {}'.format(job.secure_filename if job.service == 'file-setup' else '')
|
||||
+ ' {}'.format(job.secure_filename
|
||||
if job.service == 'file-setup'
|
||||
else '')
|
||||
+ ' --log-dir /files'
|
||||
+ ' --zip [{}]_{}'.format(job.service, job.secure_filename)
|
||||
+ ' ' + ' '.join(json.loads(job.service_args))),
|
||||
+ ' --zip [{}]_{}'.format(job.service,
|
||||
job.secure_filename)
|
||||
+ ' ' + ' '.join(json.loads(job.service_args))
|
||||
),
|
||||
'constraints': ['node.role==worker'],
|
||||
'labels': {'origin': 'nopaque',
|
||||
'type': 'service.{}'.format(job.service),
|
||||
|
||||
@ -1,4 +1,3 @@
|
||||
from decorators import background
|
||||
from notify.notification import Notification
|
||||
from notify.service import NotificationService
|
||||
from sqlalchemy import asc
|
||||
@ -7,57 +6,6 @@ from tasks.Models import NotificationEmailData
|
||||
import os
|
||||
|
||||
|
||||
# Email notification functions
|
||||
def __create_mail_notifications(notification_service):
|
||||
mn_session = Session()
|
||||
notification_email_data = mn_session.query(NotificationEmailData).order_by(asc(NotificationEmailData.creation_date)).all()
|
||||
notifications = {}
|
||||
for data in notification_email_data:
|
||||
notification = Notification()
|
||||
notification.set_addresses(notification_service.email_address,
|
||||
data.job.user.email)
|
||||
subject_template = '[nopaque] Status update for your Job/Corpora: {title}!'
|
||||
subject_template_values_dict = {'title': data.job.title}
|
||||
domain = os.environ.get('NOPAQUE_DOMAIN')
|
||||
url = '{domain}/{jobs}/{id}'.format(domain=domain,
|
||||
jobs='jobs',
|
||||
id=data.job.id)
|
||||
body_template_values_dict = {'username': data.job.user.username,
|
||||
'id': data.job.id,
|
||||
'title': data.job.title,
|
||||
'status': data.notify_status,
|
||||
'time': data.creation_date,
|
||||
'url': url}
|
||||
notification.set_notification_content(subject_template,
|
||||
subject_template_values_dict,
|
||||
'notify/templates/notification_messages/notification.txt',
|
||||
'notify/templates/notification_messages/notification.html',
|
||||
body_template_values_dict)
|
||||
notifications[data.job.id] = notification
|
||||
# Using a dictionary for notifications avoids sending multiple mails
|
||||
# if the status of a job changes in a few seconds. The user will not get
|
||||
# swamped with mails for queued, running and complete if those happen in
|
||||
# in a few seconds. Only the last update will be sent. This depends on
|
||||
# the sleep time interval though.
|
||||
mn_session.delete(data)
|
||||
mn_session.commit()
|
||||
Session.remove()
|
||||
return notifications
|
||||
|
||||
|
||||
def __send_mail_notifications(notifications, notification_service):
|
||||
for key, notification in notifications.items():
|
||||
try:
|
||||
notification_service.send(notification)
|
||||
notification_service.mail_limit_exceeded = False
|
||||
except Exception as e:
|
||||
# Adds notifications to unsent if mail server exceded limit for
|
||||
# consecutive mail sending
|
||||
notification_service.not_sent[key] = notification
|
||||
notification_service.mail_limit_exceeded = True
|
||||
|
||||
|
||||
@background
|
||||
def notify(execute_flag):
|
||||
# If True mails are sent normaly
|
||||
# If False mails are not sent. Used to avoid sending mails for jobs that
|
||||
@ -85,3 +33,56 @@ def notify(execute_flag):
|
||||
notification_service.quit()
|
||||
except Exception as e:
|
||||
notification_service.not_sent.update(notifications)
|
||||
notification_service.quit()
|
||||
|
||||
|
||||
# Email notification functions
|
||||
def __create_mail_notifications(notification_service):
|
||||
mn_session = Session()
|
||||
notification_email_data = mn_session.query(NotificationEmailData).order_by(asc(NotificationEmailData.creation_date)).all()
|
||||
notifications = {}
|
||||
for data in notification_email_data:
|
||||
notification = Notification()
|
||||
notification.set_addresses(notification_service.email_address,
|
||||
data.job.user.email)
|
||||
subject_template = '[nopaque] Status update for your Job/Corpora: {title}!'
|
||||
subject_template_values_dict = {'title': data.job.title}
|
||||
domain = os.environ.get('NOPAQUE_DOMAIN')
|
||||
url = '{domain}/{jobs}/{id}'.format(domain=domain,
|
||||
jobs='jobs',
|
||||
id=data.job.id)
|
||||
body_template_values_dict = {'username': data.job.user.username,
|
||||
'id': data.job.id,
|
||||
'title': data.job.title,
|
||||
'status': data.notify_status,
|
||||
'time': data.creation_date,
|
||||
'url': url}
|
||||
txt_tmplt = 'notify/templates/notification_messages/notification.txt'
|
||||
html_tmplt = 'notify/templates/notification_messages/notification.html'
|
||||
notification.set_notification_content(subject_template,
|
||||
subject_template_values_dict,
|
||||
txt_tmplt,
|
||||
html_tmplt,
|
||||
body_template_values_dict)
|
||||
notifications[data.job.id] = notification
|
||||
# Using a dictionary for notifications avoids sending multiple mails
|
||||
# if the status of a job changes in a few seconds. The user will not
|
||||
# get swamped with mails for queued, running and complete if those
|
||||
# happen in in a few seconds. Only the last update will be sent.
|
||||
# This depends on the sleep time interval though.
|
||||
mn_session.delete(data)
|
||||
mn_session.commit()
|
||||
Session.remove()
|
||||
return notifications
|
||||
|
||||
|
||||
def __send_mail_notifications(notifications, notification_service):
|
||||
for key, notification in notifications.items():
|
||||
try:
|
||||
notification_service.send(notification)
|
||||
notification_service.mail_limit_exceeded = False
|
||||
except Exception as e:
|
||||
# Adds notifications to unsent if mail server exceded limit for
|
||||
# consecutive mail sending
|
||||
notification_service.not_sent[key] = notification
|
||||
notification_service.mail_limit_exceeded = True
|
||||
|
||||
Loading…
Reference in New Issue
Block a user