mirror of
https://gitlab.ub.uni-bielefeld.de/sfb1288inf/nopaque.git
synced 2025-06-13 09:30:40 +00:00
Avoid that threads of the same type can be executed twice or thrice etc.
This commit is contained in:
Stephan Porada
@ -1,4 +1,3 @@
|
||||
from decorators import background
|
||||
from logger.logger import init_logger
|
||||
from tasks import Session, docker_client, NOPAQUE_STORAGE
|
||||
from tasks.Models import Corpus
|
||||
@ -7,7 +6,6 @@ import os
|
||||
import shutil
|
||||
|
||||
|
||||
@background
|
||||
def check_corpora():
|
||||
c_session = Session()
|
||||
corpora = c_session.query(Corpus).all()
|
||||
@ -131,4 +129,4 @@ def __remove_cqpserver_container(corpus):
|
||||
return
|
||||
else:
|
||||
container.remove(force=True)
|
||||
corpus.status = 'prepared'
|
||||
corpus.status = 'prepared'
|
||||
|
||||
@ -1,5 +1,4 @@
|
||||
from datetime import datetime
|
||||
from decorators import background
|
||||
from logger.logger import init_logger
|
||||
from tasks import Session, docker_client, NOPAQUE_STORAGE
|
||||
from tasks.Models import Job, NotificationData, NotificationEmailData, JobResult
|
||||
@ -8,7 +7,6 @@ import json
|
||||
import os
|
||||
|
||||
|
||||
@background
|
||||
def check_jobs():
|
||||
# logger = init_logger()
|
||||
cj_session = Session()
|
||||
@ -49,7 +47,8 @@ def __add_notification_data(job, notified_on_status, scoped_session):
|
||||
if (notification_exists == 0):
|
||||
notification_data = NotificationData(job_id=job.id)
|
||||
scoped_session.add(notification_data)
|
||||
scoped_session.commit() # If no commit job will have no NotificationData
|
||||
scoped_session.commit()
|
||||
# If no commit job will have no NotificationData
|
||||
# logger.warning('Created NotificationData for current Job.'))
|
||||
else:
|
||||
pass
|
||||
@ -70,10 +69,14 @@ def __create_job_service(job):
|
||||
job_dir = os.path.join(NOPAQUE_STORAGE, str(job.user_id), 'jobs',
|
||||
str(job.id))
|
||||
service_args = {'command': ('{} /files /files/output'.format(job.service)
|
||||
+ ' {}'.format(job.secure_filename if job.service == 'file-setup' else '')
|
||||
+ ' {}'.format(job.secure_filename
|
||||
if job.service == 'file-setup'
|
||||
else '')
|
||||
+ ' --log-dir /files'
|
||||
+ ' --zip [{}]_{}'.format(job.service, job.secure_filename)
|
||||
+ ' ' + ' '.join(json.loads(job.service_args))),
|
||||
+ ' --zip [{}]_{}'.format(job.service,
|
||||
job.secure_filename)
|
||||
+ ' ' + ' '.join(json.loads(job.service_args))
|
||||
),
|
||||
'constraints': ['node.role==worker'],
|
||||
'labels': {'origin': 'nopaque',
|
||||
'type': 'service.{}'.format(job.service),
|
||||
@ -147,4 +150,4 @@ def __remove_job_service(job):
|
||||
return
|
||||
else:
|
||||
service.update(mounts=None)
|
||||
service.remove()
|
||||
service.remove()
|
||||
|
||||
@ -1,4 +1,3 @@
|
||||
from decorators import background
|
||||
from notify.notification import Notification
|
||||
from notify.service import NotificationService
|
||||
from sqlalchemy import asc
|
||||
@ -7,57 +6,6 @@ from tasks.Models import NotificationEmailData
|
||||
import os
|
||||
|
||||
|
||||
# Email notification functions
|
||||
def __create_mail_notifications(notification_service):
|
||||
mn_session = Session()
|
||||
notification_email_data = mn_session.query(NotificationEmailData).order_by(asc(NotificationEmailData.creation_date)).all()
|
||||
notifications = {}
|
||||
for data in notification_email_data:
|
||||
notification = Notification()
|
||||
notification.set_addresses(notification_service.email_address,
|
||||
data.job.user.email)
|
||||
subject_template = '[nopaque] Status update for your Job/Corpora: {title}!'
|
||||
subject_template_values_dict = {'title': data.job.title}
|
||||
domain = os.environ.get('NOPAQUE_DOMAIN')
|
||||
url = '{domain}/{jobs}/{id}'.format(domain=domain,
|
||||
jobs='jobs',
|
||||
id=data.job.id)
|
||||
body_template_values_dict = {'username': data.job.user.username,
|
||||
'id': data.job.id,
|
||||
'title': data.job.title,
|
||||
'status': data.notify_status,
|
||||
'time': data.creation_date,
|
||||
'url': url}
|
||||
notification.set_notification_content(subject_template,
|
||||
subject_template_values_dict,
|
||||
'notify/templates/notification_messages/notification.txt',
|
||||
'notify/templates/notification_messages/notification.html',
|
||||
body_template_values_dict)
|
||||
notifications[data.job.id] = notification
|
||||
# Using a dictionary for notifications avoids sending multiple mails
|
||||
# if the status of a job changes in a few seconds. The user will not get
|
||||
# swamped with mails for queued, running and complete if those happen in
|
||||
# in a few seconds. Only the last update will be sent. This depends on
|
||||
# the sleep time interval though.
|
||||
mn_session.delete(data)
|
||||
mn_session.commit()
|
||||
Session.remove()
|
||||
return notifications
|
||||
|
||||
|
||||
def __send_mail_notifications(notifications, notification_service):
|
||||
for key, notification in notifications.items():
|
||||
try:
|
||||
notification_service.send(notification)
|
||||
notification_service.mail_limit_exceeded = False
|
||||
except Exception as e:
|
||||
# Adds notifications to unsent if mail server exceded limit for
|
||||
# consecutive mail sending
|
||||
notification_service.not_sent[key] = notification
|
||||
notification_service.mail_limit_exceeded = True
|
||||
|
||||
|
||||
@background
|
||||
def notify(execute_flag):
|
||||
# If True mails are sent normaly
|
||||
# If False mails are not sent. Used to avoid sending mails for jobs that
|
||||
@ -84,4 +32,57 @@ def notify(execute_flag):
|
||||
notification_service.not_sent = {}
|
||||
notification_service.quit()
|
||||
except Exception as e:
|
||||
notification_service.not_sent.update(notifications)
|
||||
notification_service.not_sent.update(notifications)
|
||||
notification_service.quit()
|
||||
|
||||
|
||||
# Email notification functions
|
||||
def __create_mail_notifications(notification_service):
|
||||
mn_session = Session()
|
||||
notification_email_data = mn_session.query(NotificationEmailData).order_by(asc(NotificationEmailData.creation_date)).all()
|
||||
notifications = {}
|
||||
for data in notification_email_data:
|
||||
notification = Notification()
|
||||
notification.set_addresses(notification_service.email_address,
|
||||
data.job.user.email)
|
||||
subject_template = '[nopaque] Status update for your Job/Corpora: {title}!'
|
||||
subject_template_values_dict = {'title': data.job.title}
|
||||
domain = os.environ.get('NOPAQUE_DOMAIN')
|
||||
url = '{domain}/{jobs}/{id}'.format(domain=domain,
|
||||
jobs='jobs',
|
||||
id=data.job.id)
|
||||
body_template_values_dict = {'username': data.job.user.username,
|
||||
'id': data.job.id,
|
||||
'title': data.job.title,
|
||||
'status': data.notify_status,
|
||||
'time': data.creation_date,
|
||||
'url': url}
|
||||
txt_tmplt = 'notify/templates/notification_messages/notification.txt'
|
||||
html_tmplt = 'notify/templates/notification_messages/notification.html'
|
||||
notification.set_notification_content(subject_template,
|
||||
subject_template_values_dict,
|
||||
txt_tmplt,
|
||||
html_tmplt,
|
||||
body_template_values_dict)
|
||||
notifications[data.job.id] = notification
|
||||
# Using a dictionary for notifications avoids sending multiple mails
|
||||
# if the status of a job changes in a few seconds. The user will not
|
||||
# get swamped with mails for queued, running and complete if those
|
||||
# happen in in a few seconds. Only the last update will be sent.
|
||||
# This depends on the sleep time interval though.
|
||||
mn_session.delete(data)
|
||||
mn_session.commit()
|
||||
Session.remove()
|
||||
return notifications
|
||||
|
||||
|
||||
def __send_mail_notifications(notifications, notification_service):
|
||||
for key, notification in notifications.items():
|
||||
try:
|
||||
notification_service.send(notification)
|
||||
notification_service.mail_limit_exceeded = False
|
||||
except Exception as e:
|
||||
# Adds notifications to unsent if mail server exceded limit for
|
||||
# consecutive mail sending
|
||||
notification_service.not_sent[key] = notification
|
||||
notification_service.mail_limit_exceeded = True
|
||||
|
||||
Reference in New Issue
Block a user