From dff92cbf4de9d2021458c1b1badf98e4e419e64c Mon Sep 17 00:00:00 2001 From: Patrick Jentsch Date: Thu, 8 Oct 2020 12:34:02 +0200 Subject: [PATCH] Huge config update and smtp fix for daemon --- .env.tpl | 189 +++++++--- .gitignore | 37 +- daemon/.dockerignore | 5 +- daemon/Dockerfile | 6 +- daemon/boot.sh | 9 + daemon/config.py | 61 ++++ daemon/docker-entrypoint.sh | 9 - daemon/logger/__init__.py | 0 daemon/logger/logger.py | 30 -- daemon/nopaqued.py | 24 +- daemon/nopaqued.py.bak | 455 ------------------------ daemon/notify/notification.py | 11 +- daemon/notify/service.py | 45 +-- daemon/tasks/Models.py | 2 +- daemon/tasks/__init__.py | 21 +- daemon/tasks/check_corpora.py | 39 +- daemon/tasks/check_jobs.py | 75 ++-- daemon/tasks/notify.py | 98 ++--- docker-compose.development.yml | 25 ++ docker-compose.override.yml.tpl | 51 --- docker-compose.traefik.yml | 30 ++ docker-compose.yml | 40 +-- logs/dummy | 0 web/.dockerignore | 5 +- web/Dockerfile | 6 +- web/app/__init__.py | 21 +- web/app/auth/views.py | 2 +- web/app/corpora/events.py | 19 +- web/app/corpora/views.py | 6 +- web/app/email.py | 8 +- web/app/jobs/views.py | 4 +- web/app/main/forms.py | 12 - web/app/main/views.py | 14 - web/app/models.py | 16 +- web/app/query_results/views.py | 6 +- web/app/services/views.py | 2 +- web/app/templates/main/feedback.html.j2 | 35 -- web/app/templates/main/poster.html.j2 | 202 ----------- web/boot.sh | 21 ++ web/config.py | 154 ++++---- web/docker-entrypoint.sh | 16 - web/nopaque.py | 3 +- web/tests/test_basics.py | 3 - 43 files changed, 613 insertions(+), 1204 deletions(-) create mode 100755 daemon/boot.sh create mode 100644 daemon/config.py delete mode 100755 daemon/docker-entrypoint.sh delete mode 100644 daemon/logger/__init__.py delete mode 100644 daemon/logger/logger.py delete mode 100644 daemon/nopaqued.py.bak create mode 100644 docker-compose.development.yml delete mode 100644 docker-compose.override.yml.tpl create mode 100644 docker-compose.traefik.yml delete mode 100644 logs/dummy delete mode 100644 web/app/main/forms.py delete mode 100644 web/app/templates/main/feedback.html.j2 delete mode 100644 web/app/templates/main/poster.html.j2 create mode 100755 web/boot.sh delete mode 100755 web/docker-entrypoint.sh diff --git a/.env.tpl b/.env.tpl index 68a5c792..f9f9afeb 100644 --- a/.env.tpl +++ b/.env.tpl @@ -1,64 +1,145 @@ -### Build ### -# Bash: getent group docker | cut -d: -f3 -DOCKER_GID= -# Bash: id -g -GID= -# Bash: id -u -UID= +################################################################################ +# Docker # +################################################################################ +# DEFAULT: ./db +# NOTE: Use `.` as +# HOST_DB_DIR= + +# Example: 999 +# HINT: Use this bash command `getent group docker | cut -d: -f3` +HOST_DOCKER_GID= + +# Example: 1000 +# HINT: Use this bash command `id -g` +HOST_GID= + +# DEFAULT: ./mq +# NOTE: Use `.` as +# HOST_MQ_DIR= + +# DEFAULT: ./nopaqued.log +# NOTES: Use `.` as , +# This file must be present on container startup +# HOST_NOPAQUE_DAEMON_LOG_FILE= + +# DEFAULT: ./nopaque.log +# NOTES: Use `.` as , +# This file must be present on container startup +# HOST_NOPAQUE_LOG_FILE= + +# Example: 1000 +# HINT: Use this bash command `id -u` +HOST_UID= +################################################################################ +# Database (only PostgreSQL) # +################################################################################ +NOPAQUE_DB_HOST= -### Runtime ### -# Fill out these variables to use the Docker HTTP socket. When doing this, you -# can remove the Docker UNIX socket mount from the docker-compose file. -# Example: /home/nopaqued/.docker -# DOCKER_CERT_PATH= -# Example: host.docker.internal -# DOCKER_HOST= +NOPAQUE_DB_NAME= + +NOPAQUE_DB_PASSWORD= + +# DEFAULT: 5432 +# NOPAQUE_DB_PORT= + +NOPAQUE_DB_USERNAME= + + +################################################################################ +# SMTP # +################################################################################ +# EXAMPLE: nopaque Admin +NOPAQUE_SMTP_DEFAULT_SENDER= + +NOPAQUE_SMTP_PASSWORD= + +# EXAMPLE: smtp.example.com +NOPAQUE_SMTP_SERVER= + +# EXAMPLE: 587 +NOPAQUE_SMTP_PORT= + +# DEFAULT: False # Choose one: False, True -# DOCKER_TLS_VERIFY= +# NOPAQUE_SMTP_USE_SSL= -# Choose one: development, production, testing -FLASK_CONFIG= -# Bash: python -c "import uuid; print(uuid.uuid4().hex)" -SECRET_KEY= - -# Example: - -GITLAB_USERNAME= -# Example: - -GITLAB_PASSWORD= - -# Example: smtp.example.com -MAIL_SERVER= -# Example: 587 -MAIL_PORT= +# DEFAULT: False # Choose one: False, True -MAIL_USE_TLS= -# Example: nopaque@example.com -MAIL_USERNAME= -# Example: - -MAIL_PASSWORD= +# NOPAQUE_SMTP_USE_TLS= -# Example: nopaque@example.com -NOPAQUE_ADMIN= -# Example: nopaque@example.com -NOPAQUE_CONTACT= -# Example: nopaque.localhost -NOPAQUE_DOMAIN= +# EXAMPLE: nopaque@example.com +NOPAQUE_SMTP_USERNAME= + + +################################################################################ +# General # +################################################################################ +# Example: admin.nopaque@example.com +NOPAQUE_ADMIN_EMAIL_ADRESS= + +# Example: contact.nopaque@example.com +NOPAQUE_CONTACT_EMAIL_ADRESS= + +# DEFAULT: /mnt/nopaque +# NOTE: This must be a network share and it must be available on all Docker Swarm nodes +# NOPAQUE_DATA_DIR= + +# DEFAULT: False # Choose one: False, True -NOPAQUE_EXECUTE_NOTIFICATIONS= -# Choose one: CRITICAL, ERROR, WARNING, INFO, DEBUG -NOPAQUE_LOG_LEVEL= -# Example: nopaque Admin -NOPAQUE_MAIL_SENDER= +# NOPAQUE_DEBUG= + +# DEFAULT: localhost +# NOPAQUE_DOMAIN= + +# DEFAULT: 0 +# NOPAQUE_NUM_PROXIES= + +# DEFAULT: http # Choose one: http, https -NOPAQUE_PROTOCOL= -# Example: /mnt/nopaque -NOPAQUE_STORAGE= +# NOPAQUE_PROTOCOL= -# Example: nopaque -POSTGRES_DB_NAME= -# Example: - -POSTGRES_USER= -# Example: - -POSTGRES_PASSWORD= +# DEFAULT: 5 +# NOPAQUE_RESSOURCES_PER_PAGE= + +# DEFAULT: hard to guess string +# HINT: Use this bash command `python -c "import uuid; print(uuid.uuid4().hex)"` +NOPAQUE_SECRET_KEY= + +# DEFAULT: 10 +# NOPAQUE_USERS_PER_PAGE= + + +################################################################################ +# Logging # +################################################################################ +# DEFAULT: /nopaqued.log ~ /home/nopaqued/nopaqued.log +# NOTE: Use `.` as +# NOPAQUE_DAEMON_LOG_FILE= + +# DEFAULT: %Y-%m-%d %H:%M:%S +# NOPAQUE_LOG_DATE_FORMAT= + +# DEFAULT: /NOPAQUE.log ~ /home/NOPAQUE/NOPAQUE.log +# NOTE: Use `.` as +# NOPAQUE_LOG_FILE= + +# DEFAULT: [%(asctime)s] %(levelname)s in %(pathname)s (function: %(funcName)s, line: %(lineno)d): %(message)s +# NOPAQUE_LOG_FORMAT= + +# DEFAULT: ERROR +# Choose one: CRITICAL, ERROR, WARNING, INFO, DEBUG +# NOPAQUE_LOG_LEVEL= + + +################################################################################ +# Message queue # +################################################################################ +NOPAQUE_MQ_HOST= + +# EXAMPLE: 6379 +NOPAQUE_MQ_PORT= + +# Choose one of the supported types by Flask-SocketIO +NOPAQUE_MQ_TYPE= diff --git a/.gitignore b/.gitignore index 8ef892c5..7e80fd54 100644 --- a/.gitignore +++ b/.gitignore @@ -1,6 +1,37 @@ -docker-compose.override.yml nopaque.log nopaqued.log -.DS_Store -*.env + +*.py[cod] + +# C extensions +*.so + +# Docker related files +docker-compose.override.yml +db +mq + +# Environment files +.env + +# Installer logs +pip-log.txt + +# Packages +*.egg +*.egg-info +dist +build +eggs +parts +bin +var +sdist +develop-eggs +.installed.cfg +lib +lib64 __pycache__ + +# Virtual environment +venv diff --git a/daemon/.dockerignore b/daemon/.dockerignore index 96dbc1bd..21803000 100644 --- a/daemon/.dockerignore +++ b/daemon/.dockerignore @@ -1,3 +1,6 @@ +# Docker related files Dockerfile .dockerignore -*.bak + +# Packages +__pycache__ diff --git a/daemon/Dockerfile b/daemon/Dockerfile index 91bded0f..1d808831 100644 --- a/daemon/Dockerfile +++ b/daemon/Dockerfile @@ -1,7 +1,7 @@ FROM python:3.6-slim-stretch -LABEL maintainer="inf_sfb1288@lists.uni-bielefeld.de" +LABEL authors="Patrick Jentsch , Stephan Porada " ARG DOCKER_GID @@ -15,7 +15,7 @@ RUN apt-get update \ build-essential \ libpq-dev \ wait-for-it \ - && rm -rf /var/lib/apt/lists/* + && rm -r /var/lib/apt/lists/* RUN groupadd --gid ${DOCKER_GID} --system docker \ @@ -31,4 +31,4 @@ RUN python -m venv venv \ && mkdir logs -ENTRYPOINT ["./docker-entrypoint.sh"] +ENTRYPOINT ["./boot.sh"] diff --git a/daemon/boot.sh b/daemon/boot.sh new file mode 100755 index 00000000..ce652c64 --- /dev/null +++ b/daemon/boot.sh @@ -0,0 +1,9 @@ +#!/bin/bash +echo "Waiting for db..." +wait-for-it "${NOPAQUE_DB_HOST}:${NOPAQUE_DB_PORT:-5432}" --strict --timeout=0 +echo "Waiting for nopaque..." +wait-for-it nopaque:5000 --strict --timeout=0 + +source venv/bin/activate + +python nopaqued.py diff --git a/daemon/config.py b/daemon/config.py new file mode 100644 index 00000000..bbbb1b82 --- /dev/null +++ b/daemon/config.py @@ -0,0 +1,61 @@ +import logging +import os + + +root_dir = os.path.abspath(os.path.dirname(__file__)) + + +DEFAULT_DATA_DIR = os.path.join('/mnt/data') +DEFAULT_DB_PORT = '5432' +DEFAULT_DOMAIN = 'localhost' +DEFAULT_LOG_DATE_FORMAT = '%Y-%m-%d %H:%M:%S' +DEFAULT_LOG_FILE = os.path.join(root_dir, 'nopaqued.log') +DEFAULT_LOG_FORMAT = ('[%(asctime)s] %(levelname)s in %(pathname)s ' + '(function: %(funcName)s, line: %(lineno)d): ' + '%(message)s') +DEFAULT_LOG_LEVEL = 'ERROR' +DEFAULT_MAIL_USE_SSL = 'False' +DEFAULT_MAIL_USE_TLS = 'False' +DEFAULT_PROTOCOL = 'http' + + +class Config: + ''' ### Database ### ''' + DB_HOST = os.environ.get('NOPAQUE_DB_HOST') + DB_NAME = os.environ.get('NOPAQUE_DB_NAME') + DB_PASSWORD = os.environ.get('NOPAQUE_DB_PASSWORD') + DB_PORT = os.environ.get('NOPAQUE_DB_PORT', DEFAULT_DB_PORT) + DB_USERNAME = os.environ.get('NOPAQUE_DB_USERNAME') + SQLALCHEMY_DATABASE_URI = 'postgresql://{}:{}@{}:{}/{}'.format( + DB_USERNAME, DB_PASSWORD, DB_HOST, DB_PORT, DB_NAME) + + ''' ### SMTP ### ''' + SMTP_DEFAULT_SENDER = os.environ.get('NOPAQUE_SMTP_DEFAULT_SENDER') + SMTP_PASSWORD = os.environ.get('NOPAQUE_SMTP_PASSWORD') + SMTP_PORT = os.environ.get('NOPAQUE_SMTP_PORT') + SMTP_SERVER = os.environ.get('NOPAQUE_SMTP_SERVER') + SMTP_USERNAME = os.environ.get('NOPAQUE_SMTP_USERNAME') + SMTP_USE_SSL = os.environ.get('NOPAQUE_SMTP_USE_SSL', + DEFAULT_MAIL_USE_SSL).lower() == 'true' + SMTP_USE_TLS = os.environ.get('NOPAQUE_SMTP_USE_TLS', + DEFAULT_MAIL_USE_TLS).lower() == 'true' + + ''' ### General ### ''' + DATA_DIR = os.environ.get('NOPAQUE_DATA_DIR', DEFAULT_DATA_DIR) + DOMAIN = os.environ.get('NOPAQUE_DOMAIN', DEFAULT_DOMAIN) + PROTOCOL = os.environ.get('NOPAQUE_PROTOCOL', DEFAULT_PROTOCOL) + + ''' ### Logging ### ''' + LOG_DATE_FORMAT = os.environ.get('NOPAQUE_LOG_DATE_FORMAT', + DEFAULT_LOG_DATE_FORMAT) + LOG_FILE = os.environ.get('NOPAQUE_DAEMON_LOG_FILE', DEFAULT_LOG_FILE) + LOG_FORMAT = os.environ.get('NOPAQUE_LOG_FORMAT', DEFAULT_LOG_FORMAT) + LOG_LEVEL = os.environ.get('NOPAQUE_LOG_LEVEL', DEFAULT_LOG_LEVEL) + + def init_app(self): + # Configure logging according to the corresponding (LOG_*) config + # entries + logging.basicConfig(datefmt=self.LOG_DATE_FORMAT, + filename=self.LOG_FILE, + format=self.LOG_FORMAT, + level=self.LOG_LEVEL) diff --git a/daemon/docker-entrypoint.sh b/daemon/docker-entrypoint.sh deleted file mode 100755 index 637d29a0..00000000 --- a/daemon/docker-entrypoint.sh +++ /dev/null @@ -1,9 +0,0 @@ -#!/bin/bash - -echo "Waiting for db..." -wait-for-it db:5432 --strict --timeout=0 -echo "Waiting for web..." -wait-for-it web:5000 --strict --timeout=0 - -source venv/bin/activate -python nopaqued.py diff --git a/daemon/logger/__init__.py b/daemon/logger/__init__.py deleted file mode 100644 index e69de29b..00000000 diff --git a/daemon/logger/logger.py b/daemon/logger/logger.py deleted file mode 100644 index 297ec964..00000000 --- a/daemon/logger/logger.py +++ /dev/null @@ -1,30 +0,0 @@ -import os -import logging - - -def init_logger(): - ''' - Functions initiates a logger instance. - ''' - os.makedirs('logs', exist_ok=True) - logging.basicConfig(filename='logs/nopaqued.log', - format='[%(asctime)s] %(levelname)s in ' - '%(pathname)s:%(lineno)d - %(message)s', - datefmt='%Y-%m-%d %H:%M:%S', filemode='w') - NOPAQUE_LOG_LEVEL = os.environ.get('NOPAQUE_LOG_LEVEL') - if NOPAQUE_LOG_LEVEL is None: - FLASK_CONFIG = os.environ.get('FLASK_CONFIG') - if FLASK_CONFIG == 'development': - logging.basicConfig(level='DEBUG') - elif FLASK_CONFIG == 'testing': - # TODO: Set an appropriate log level - pass - elif FLASK_CONFIG == 'production': - logging.basicConfig(level='ERROR') - else: - logging.basicConfig(level=NOPAQUE_LOG_LEVEL) - return logging.getLogger(__name__) - - -if __name__ == '__main__': - init_logger() diff --git a/daemon/nopaqued.py b/daemon/nopaqued.py index 6ff3595a..8832c091 100644 --- a/daemon/nopaqued.py +++ b/daemon/nopaqued.py @@ -2,26 +2,20 @@ from tasks.check_corpora import check_corpora from tasks.check_jobs import check_jobs from tasks.notify import notify from time import sleep -import os def nopaqued(): - NOPAQUE_EXECUTE_NOTIFICATIONS = os.environ.get('NOPAQUE_EXECUTE_NOTIFICATIONS', 'True').lower() == 'true' # noqa - threads = {'check_corpora': None, 'check_jobs': None, 'notify': None} + check_corpora_thread = check_corpora() + check_jobs_thread = check_jobs() + notify_thread = notify() - threads['check_corpora'] = check_corpora() - threads['check_jobs'] = check_jobs() - threads['notify'] = notify(NOPAQUE_EXECUTE_NOTIFICATIONS) while True: - if not threads['check_corpora'].is_alive(): - threads['check_corpora'] = check_corpora() - if not threads['check_jobs'].is_alive(): - threads['check_jobs'] = check_jobs() - if not threads['notify'].is_alive(): - threads['notify'] = notify(NOPAQUE_EXECUTE_NOTIFICATIONS) - # If execute_notifications True mails are sent. - # If execute_notifications False no mails are sent. - # But notification status will be set nonetheless. + if not check_corpora_thread.is_alive(): + check_corpora_thread = check_corpora() + if not check_jobs_thread.is_alive(): + check_jobs_thread = check_jobs() + if not notify_thread.is_alive(): + notify_thread = notify() sleep(3) diff --git a/daemon/nopaqued.py.bak b/daemon/nopaqued.py.bak deleted file mode 100644 index 500568aa..00000000 --- a/daemon/nopaqued.py.bak +++ /dev/null @@ -1,455 +0,0 @@ -from notify.notification import Notification -from notify.service import NotificationService -from sqlalchemy import create_engine, asc -from sqlalchemy.orm import Session, relationship -from sqlalchemy.ext.automap import automap_base -from datetime import datetime -from time import sleep -import docker -import json -import logging -import os -import shutil - - -''' Global constants ''' -NOPAQUE_STORAGE = os.environ.get('NOPAQUE_STORAGE') - -''' Global variables ''' -docker_client = None -session = None - - -# Classes for database models -Base = automap_base() - - -class Corpus(Base): - __tablename__ = 'corpora' - files = relationship('CorpusFile', collection_class=set) - - -class CorpusFile(Base): - __tablename__ = 'corpus_files' - - -class Job(Base): - __tablename__ = 'jobs' - inputs = relationship('JobInput', collection_class=set) - results = relationship('JobResult', collection_class=set) - notification_data = relationship('NotificationData', collection_class=list) - notification_email_data = relationship('NotificationEmailData', collection_class=list) - - -class NotificationData(Base): - __tablename__ = 'notification_data' - job = relationship('Job', collection_class=set) - - -class NotificationEmailData(Base): - __tablename__ = 'notification_email_data' - job = relationship('Job', collection_class=set) - - -class JobInput(Base): - __tablename__ = 'job_results' - - -class JobResult(Base): - __tablename__ = 'job_results' - - -class User(Base): - __tablename__ = 'users' - jobs = relationship('Job', collection_class=set) - corpora = relationship('Corpus', collection_class=set) - - -def check_corpora(): - corpora = session.query(Corpus).all() - for corpus in filter(lambda corpus: corpus.status == 'submitted', corpora): - __create_build_corpus_service(corpus) - for corpus in filter(lambda corpus: (corpus.status == 'queued' - or corpus.status == 'running'), - corpora): - __checkout_build_corpus_service(corpus) - for corpus in filter(lambda corpus: corpus.status == 'start analysis', - corpora): - __create_cqpserver_container(corpus) - for corpus in filter(lambda corpus: corpus.status == 'stop analysis', - corpora): - __remove_cqpserver_container(corpus) - - -def __create_build_corpus_service(corpus): - corpus_dir = os.path.join(NOPAQUE_STORAGE, str(corpus.user_id), - 'corpora', str(corpus.id)) - corpus_data_dir = os.path.join(corpus_dir, 'data') - corpus_file = os.path.join(corpus_dir, 'merged', 'corpus.vrt') - corpus_registry_dir = os.path.join(corpus_dir, 'registry') - if os.path.exists(corpus_data_dir): - shutil.rmtree(corpus_data_dir) - if os.path.exists(corpus_registry_dir): - shutil.rmtree(corpus_registry_dir) - os.mkdir(corpus_data_dir) - os.mkdir(corpus_registry_dir) - service_args = {'command': 'docker-entrypoint.sh build-corpus', - 'constraints': ['node.role==worker'], - 'labels': {'origin': 'nopaque', - 'type': 'corpus.prepare', - 'corpus_id': str(corpus.id)}, - 'mounts': [corpus_file + ':/root/files/corpus.vrt:ro', - corpus_data_dir + ':/corpora/data:rw', - corpus_registry_dir + ':/usr/local/share/cwb/registry:rw'], - 'name': 'build-corpus_{}'.format(corpus.id), - 'restart_policy': docker.types.RestartPolicy()} - service_image = ('gitlab.ub.uni-bielefeld.de:4567/sfb1288inf/cqpserver:latest') - try: - service = docker_client.services.get(service_args['name']) - except docker.errors.NotFound: - pass - except docker.errors.DockerException: - return - else: - service.remove() - try: - docker_client.services.create(service_image, **service_args) - except docker.errors.DockerException: - corpus.status = 'failed' - else: - corpus.status = 'queued' - - -def __checkout_build_corpus_service(corpus): - service_name = 'build-corpus_{}'.format(corpus.id) - try: - service = docker_client.services.get(service_name) - except docker.errors.NotFound: - logger.error('__checkout_build_corpus_service({}):'.format(corpus.id) - + ' The service does not exist.' - + ' (stauts: {} -> failed)'.format(corpus.status)) - corpus.status = 'failed' - return - except docker.errors.DockerException: - return - service_tasks = service.tasks() - if not service_tasks: - return - task_state = service_tasks[0].get('Status').get('State') - if corpus.status == 'queued' and task_state != 'pending': - corpus.status = 'running' - elif corpus.status == 'running' and task_state == 'complete': - service.remove() - corpus.status = 'prepared' - elif corpus.status == 'running' and task_state == 'failed': - service.remove() - corpus.status = task_state - - -def __create_cqpserver_container(corpus): - corpus_dir = os.path.join(NOPAQUE_STORAGE, str(corpus.user_id), - 'corpora', str(corpus.id)) - corpus_data_dir = os.path.join(corpus_dir, 'data') - corpus_registry_dir = os.path.join(corpus_dir, 'registry') - container_args = {'command': 'cqpserver', - 'detach': True, - 'volumes': [corpus_data_dir + ':/corpora/data:rw', - corpus_registry_dir + ':/usr/local/share/cwb/registry:rw'], - 'name': 'cqpserver_{}'.format(corpus.id), - 'network': 'opaque_default'} - container_image = ('gitlab.ub.uni-bielefeld.de:4567/sfb1288inf/cqpserver:latest') - try: - container = docker_client.containers.get(container_args['name']) - except docker.errors.NotFound: - pass - except docker.errors.DockerException: - return - else: - container.remove(force=True) - try: - docker_client.containers.run(container_image, **container_args) - except docker.errors.DockerException: - return - else: - corpus.status = 'analysing' - - -def __remove_cqpserver_container(corpus): - container_name = 'cqpserver_{}'.format(corpus.id) - try: - container = docker_client.containers.get(container_name) - except docker.errors.NotFound: - pass - except docker.errors.DockerException: - return - else: - container.remove(force=True) - corpus.status = 'prepared' - - -def check_jobs(): - jobs = session.query(Job).all() - for job in filter(lambda job: job.status == 'submitted', jobs): - __create_job_service(job) - for job in filter(lambda job: (job.status == 'queued'), jobs): - __checkout_job_service(job) - # __add_notification_data(job, 'queued') - for job in filter(lambda job: (job.status == 'running'), jobs): - __checkout_job_service(job) - # __add_notification_data(job, 'running') - # for job in filter(lambda job: job.status == 'complete', jobs): - # __add_notification_data(job, 'complete') - # for job in filter(lambda job: job.status == 'failed', jobs): - #__add_notification_data(job, 'failed') - for job in filter(lambda job: job.status == 'canceling', jobs): - __remove_job_service(job) - - -def __add_notification_data(job, notified_on_status): - # checks if user wants any notifications at all - if (job.user.setting_job_status_mail_notifications == 'none'): - # logger.warning('User does not want any notifications!') - return - # checks if user wants only notification on completed jobs - elif (job.user.setting_job_status_mail_notifications == 'end' - and notified_on_status != 'complete'): - # logger.warning('User only wants notifications on job completed!') - return - else: - # check if a job already has associated NotificationData - notification_exists = len(job.notification_data) - # create notification_data for current job if there is none - if (notification_exists == 0): - notification_data = NotificationData(job_id=job.id) - session.add(notification_data) - session.commit() # If no commit job will have no NotificationData - # logger.warning('Created NotificationData for current Job.')) - else: - pass - # logger.warning('Job already had notification: {}'.format(notification_exists)) - if (job.notification_data[0].notified_on != notified_on_status): - notification_email_data = NotificationEmailData(job_id=job.id) - notification_email_data.notify_status = notified_on_status - notification_email_data.creation_date = datetime.utcnow() - job.notification_data[0].notified_on = notified_on_status - session.add(notification_email_data) - # logger.warning('Created NotificationEmailData for current Job.') - else: - # logger.warning('NotificationEmailData has already been created for current Job!') - pass - - -def __create_job_service(job): - job_dir = os.path.join(NOPAQUE_STORAGE, str(job.user_id), 'jobs', - str(job.id)) - service_args = {'command': ('{} /files /files/output'.format(job.service) - + ' {}'.format(job.secure_filename if job.service == 'file-setup' else '') - + ' --log-dir /files' - + ' --zip [{}]_{}'.format(job.service, job.secure_filename) - + ' ' + ' '.join(json.loads(job.service_args))), - 'constraints': ['node.role==worker'], - 'labels': {'origin': 'nopaque', - 'type': 'service.{}'.format(job.service), - 'job_id': str(job.id)}, - 'mounts': [job_dir + ':/files:rw'], - 'name': 'job_{}'.format(job.id), - 'resources': docker.types.Resources( - cpu_reservation=job.n_cores * (10 ** 9), - mem_reservation=job.mem_mb * (10 ** 6)), - 'restart_policy': docker.types.RestartPolicy()} - service_image = ('gitlab.ub.uni-bielefeld.de:4567/sfb1288inf/' - + job.service + ':' + job.service_version) - try: - service = docker_client.services.get(service_args['name']) - except docker.errors.NotFound: - pass - except docker.errors.DockerException: - return - else: - service.remove() - try: - docker_client.services.create(service_image, **service_args) - except docker.errors.DockerException: - job.status = 'failed' - else: - job.status = 'queued' - - -def __checkout_job_service(job): - service_name = 'job_{}'.format(job.id) - try: - service = docker_client.services.get(service_name) - except docker.errors.NotFound: - logger.error('__checkout_job_service({}):'.format(job.id) - + ' The service does not exist.' - + ' (stauts: {} -> failed)'.format(job.status)) - job.status = 'failed' - return - except docker.errors.DockerException: - return - service_tasks = service.tasks() - if not service_tasks: - return - task_state = service_tasks[0].get('Status').get('State') - if job.status == 'queued' and task_state != 'pending': - job.status = 'running' - elif (job.status == 'running' - and (task_state == 'complete' or task_state == 'failed')): - service.remove() - job.end_date = datetime.utcnow() - job.status = task_state - if task_state == 'complete': - results_dir = os.path.join(NOPAQUE_STORAGE, str(job.user_id), - 'jobs', str(job.id), 'output') - results = filter(lambda x: x.endswith('.zip'), - os.listdir(results_dir)) - for result in results: - job_result = JobResult(dir=results_dir, filename=result, - job_id=job.id) - session.add(job_result) - - -def __remove_job_service(job): - service_name = 'job_{}'.format(job.id) - try: - service = docker_client.services.get(service_name) - except docker.errors.NotFound: - job.status = 'canceled' - except docker.errors.DockerException: - return - else: - service.update(mounts=None) - service.remove() - - -def handle_jobs(): - check_jobs() - - -def handle_corpora(): - check_corpora() - - -# Email notification functions -def create_mail_notifications(notification_service): - notification_email_data = session.query(NotificationEmailData).order_by(asc(NotificationEmailData.creation_date)).all() - notifications = {} - for data in notification_email_data: - notification = Notification() - notification.set_addresses(notification_service.email_address, - data.job.user.email) - subject_template = '[nopaque] Status update for your Job/Corpora: {title}!' - subject_template_values_dict = {'title': data.job.title} - protocol = os.environ.get('NOPAQUE_PROTOCOL') - domain = os.environ.get('NOPAQUE_DOMAIN') - url = '{protocol}://{domain}/{jobs}/{id}'.format( - protocol=protocol, domain=domain, jobs='jobs', id=data.job.id) - body_template_values_dict = {'username': data.job.user.username, - 'id': data.job.id, - 'title': data.job.title, - 'status': data.notify_status, - 'time': data.creation_date, - 'url': url} - notification.set_notification_content(subject_template, - subject_template_values_dict, - 'templates/notification_messages/notification.txt', - 'templates/notification_messages/notification.html', - body_template_values_dict) - notifications[data.job.id] = notification - # Using a dictionary for notifications avoids sending multiple mails - # if the status of a job changes in a few seconds. The user will not get - # swamped with mails for queued, running and complete if those happen in - # in a few seconds. Only the last update will be sent. - session.delete(data) - return notifications - - -def send_mail_notifications(notifications, notification_service): - for key, notification in notifications.items(): - try: - notification_service.send(notification) - notification_service.mail_limit_exceeded = False - except Exception as e: - # Adds notifications to unsent if mail server exceded limit for - # consecutive mail sending - notification_service.not_sent[key] = notification - notification_service.mail_limit_exceeded = True - - -def notify(): - # Initialize notification service - notification_service = NotificationService() - notification_service.get_smtp_configs() - notification_service.set_server() - # create notifications (content, recipient etc.) - notifications = create_mail_notifications(notification_service) - # only login and send mails if there are any notifications - if (len(notifications) > 0): - try: - notification_service.login() - # combine new and unsent notifications - notifications.update(notification_service.not_sent) - # send all notifications - send_mail_notifications(notifications, notification_service) - # remove unsent notifications because they have been sent now - # but only if mail limit has not been exceeded - if (notification_service.mail_limit_exceeded is not True): - notification_service.not_sent = {} - notification_service.quit() - except Exception as e: - notification_service.not_sent.update(notifications) - - -# Logger functions # -def init_logger(): - ''' - Functions initiates a logger instance. - ''' - global logger - - if not os.path.isfile('logs/nopaqued.log'): - file_path = os.path.join(os.getcwd(), 'logs/nopaqued.log') - log = open(file_path, 'w+') - log.close() - logging.basicConfig(datefmt='%Y-%m-%d %H:%M:%S', - filemode='w', filename='logs/nopaqued.log', - format='%(asctime)s - %(levelname)s - %(name)s - ' - '%(filename)s - %(lineno)d - %(message)s') - logger = logging.getLogger(__name__) - if os.environ.get('FLASK_CONFIG') == 'development': - logger.setLevel(logging.DEBUG) - if os.environ.get('FLASK_CONFIG') == 'production': - logger.setLevel(logging.WARNING) - - -def nopaqued(): - global Base - global docker_client - global session - - engine = create_engine( - 'postgresql://{}:{}@db/{}'.format( - os.environ.get('POSTGRES_USER'), - os.environ.get('POSTGRES_PASSWORD'), - os.environ.get('POSTGRES_DB_NAME'))) - Base.prepare(engine, reflect=True) - session = Session(engine) - session.commit() - - docker_client = docker.from_env() - docker_client.login(password=os.environ.get('GITLAB_PASSWORD'), - registry="gitlab.ub.uni-bielefeld.de:4567", - username=os.environ.get('GITLAB_USERNAME')) - - # executing background functions - while True: - handle_jobs() - handle_corpora() - # notify() - session.commit() - sleep(3) - - -if __name__ == '__main__': - init_logger() - nopaqued() diff --git a/daemon/notify/notification.py b/daemon/notify/notification.py index f6063386..488471c3 100644 --- a/daemon/notify/notification.py +++ b/daemon/notify/notification.py @@ -11,16 +11,17 @@ class Notification(EmailMessage): body_html_template_path, body_template_values_dict): # Create subject with subject_template_values_dict - self['subject'] = subject_template.format(**subject_template_values_dict) + self['subject'] = subject_template.format( + **subject_template_values_dict) # Open template files and insert values from body_template_values_dict with open(body_txt_template_path) as nfile: - self.body_txt = nfile.read().format(**body_template_values_dict) + self.body = nfile.read().format(**body_template_values_dict) with open(body_html_template_path) as nfile: - self.body_html = nfile.read().format(**body_template_values_dict) + self.html = nfile.read().format(**body_template_values_dict) # Set txt of email - self.set_content(self.body_txt) + self.set_content(self.body) # Set html alternative - self.add_alternative(self.body_html, subtype='html') + self.add_alternative(self.html, subtype='html') def set_addresses(self, sender, recipient): self['From'] = sender diff --git a/daemon/notify/service.py b/daemon/notify/service.py index 0b08037d..633fb386 100644 --- a/daemon/notify/service.py +++ b/daemon/notify/service.py @@ -1,41 +1,16 @@ -import os -import smtplib - - -class NotificationService(object): +class NotificationService: """This is a nopaque notifcation service object.""" - def __init__(self, execute_flag): - super(NotificationService, self).__init__() - self.execute_flag = execute_flag # If True mails are sent normaly - # If False mails are not sent. Used to avoid sending mails for jobs - # that have been completed a long time ago. Use this if you implement - # notify into an already existing nopaque instance. Change it to True - # after the daemon has run one time with the flag set to False - self.not_sent = {} # Holds due to an error unsent email notifications - self.mail_limit_exceeded = False # Bool to show if the mail server - # stoped sending mails due to exceeding its sending limit - - def get_smtp_configs(self): - self.password = os.environ.get('MAIL_PASSWORD') - self.port = os.environ.get('MAIL_PORT') - self.server_str = os.environ.get('MAIL_SERVER') - self.tls = os.environ.get('MAIL_USE_TLS') - self.username = os.environ.get('MAIL_USERNAME').split("@")[0] - self.email_address = os.environ.get('MAIL_USERNAME') - - def set_server(self): - self.smtp_server = smtplib.SMTP(host=self.server_str, port=self.port) - - def login(self): - self.smtp_server.starttls() - self.smtp_server.login(self.username, self.password) + def __init__(self, smtp): + # Bool to show if the mail server stoped sending mails due to exceeding + # its sending limit + self.mail_limit_exceeded = False + # Holds due to an error unsent email notifications + self.not_sent = {} + self.smtp = smtp def send(self, email): - if self.execute_flag: - self.smtp_server.send_message(email) - else: - return + self.smtp.send_message(email) def quit(self): - self.smtp_server.quit() + self.smtp.quit() diff --git a/daemon/tasks/Models.py b/daemon/tasks/Models.py index 42cc4021..1f113142 100644 --- a/daemon/tasks/Models.py +++ b/daemon/tasks/Models.py @@ -1,6 +1,6 @@ from sqlalchemy.ext.automap import automap_base from sqlalchemy.orm import relationship -from tasks import engine +from . import engine Base = automap_base() diff --git a/daemon/tasks/__init__.py b/daemon/tasks/__init__.py index e3e6eb51..89ed03e7 100644 --- a/daemon/tasks/__init__.py +++ b/daemon/tasks/__init__.py @@ -1,22 +1,11 @@ +from config import Config from sqlalchemy import create_engine from sqlalchemy.orm import scoped_session, sessionmaker -import os import docker -''' Global constants ''' -NOPAQUE_STORAGE = os.environ.get('NOPAQUE_STORAGE') -''' Docker client ''' +config = Config() +config.init_app() docker_client = docker.from_env() -docker_client.login(password=os.environ.get('GITLAB_PASSWORD'), - registry="gitlab.ub.uni-bielefeld.de:4567", - username=os.environ.get('GITLAB_USERNAME')) - -''' Scoped session ''' -engine = create_engine( - 'postgresql://{}:{}@db/{}'.format( - os.environ.get('POSTGRES_USER'), - os.environ.get('POSTGRES_PASSWORD'), - os.environ.get('POSTGRES_DB_NAME'))) -session_factory = sessionmaker(bind=engine) -Session = scoped_session(session_factory) +engine = create_engine(config.SQLALCHEMY_DATABASE_URI) +Session = scoped_session(sessionmaker(bind=engine)) diff --git a/daemon/tasks/check_corpora.py b/daemon/tasks/check_corpora.py index 588d801d..c91e57d6 100644 --- a/daemon/tasks/check_corpora.py +++ b/daemon/tasks/check_corpora.py @@ -1,16 +1,16 @@ -from logger.logger import init_logger -from tasks import Session, docker_client, NOPAQUE_STORAGE -from tasks.decorators import background -from tasks.Models import Corpus +from . import config, docker_client, Session +from .decorators import background +from .models import Corpus import docker +import logging import os import shutil @background def check_corpora(): - c_session = Session() - corpora = c_session.query(Corpus).all() + session = Session() + corpora = session.query(Corpus).all() for corpus in filter(lambda corpus: corpus.status == 'submitted', corpora): __create_build_corpus_service(corpus) for corpus in filter(lambda corpus: (corpus.status == 'queued' @@ -23,13 +23,15 @@ def check_corpora(): for corpus in filter(lambda corpus: corpus.status == 'stop analysis', corpora): __remove_cqpserver_container(corpus) - c_session.commit() + session.commit() Session.remove() def __create_build_corpus_service(corpus): - corpus_dir = os.path.join(NOPAQUE_STORAGE, str(corpus.user_id), - 'corpora', str(corpus.id)) + corpus_dir = os.path.join(config.DATA_DIR, + str(corpus.user_id), + 'corpora', + str(corpus.id)) corpus_data_dir = os.path.join(corpus_dir, 'data') corpus_file = os.path.join(corpus_dir, 'merged', 'corpus.vrt') corpus_registry_dir = os.path.join(corpus_dir, 'registry') @@ -49,7 +51,8 @@ def __create_build_corpus_service(corpus): corpus_registry_dir + ':/usr/local/share/cwb/registry:rw'], 'name': 'build-corpus_{}'.format(corpus.id), 'restart_policy': docker.types.RestartPolicy()} - service_image = ('gitlab.ub.uni-bielefeld.de:4567/sfb1288inf/cqpserver:latest') + service_image = \ + 'gitlab.ub.uni-bielefeld.de:4567/sfb1288inf/cqpserver:latest' try: service = docker_client.services.get(service_args['name']) except docker.errors.NotFound: @@ -67,14 +70,13 @@ def __create_build_corpus_service(corpus): def __checkout_build_corpus_service(corpus): - logger = init_logger() service_name = 'build-corpus_{}'.format(corpus.id) try: service = docker_client.services.get(service_name) except docker.errors.NotFound: - logger.error('__checkout_build_corpus_service({}):'.format(corpus.id) - + ' The service does not exist.' - + ' (stauts: {} -> failed)'.format(corpus.status)) + logging.error('__checkout_build_corpus_service({}):'.format(corpus.id) + + ' The service does not exist.' + + ' (stauts: {} -> failed)'.format(corpus.status)) corpus.status = 'failed' return except docker.errors.DockerException: @@ -94,8 +96,10 @@ def __checkout_build_corpus_service(corpus): def __create_cqpserver_container(corpus): - corpus_dir = os.path.join(NOPAQUE_STORAGE, str(corpus.user_id), - 'corpora', str(corpus.id)) + corpus_dir = os.path.join(config.DATA_DIR, + str(corpus.user_id), + 'corpora', + str(corpus.id)) corpus_data_dir = os.path.join(corpus_dir, 'data') corpus_registry_dir = os.path.join(corpus_dir, 'registry') container_args = {'command': 'cqpserver', @@ -104,7 +108,8 @@ def __create_cqpserver_container(corpus): corpus_registry_dir + ':/usr/local/share/cwb/registry:rw'], 'name': 'cqpserver_{}'.format(corpus.id), 'network': 'nopaque_default'} - container_image = ('gitlab.ub.uni-bielefeld.de:4567/sfb1288inf/cqpserver:latest') + container_image = \ + 'gitlab.ub.uni-bielefeld.de:4567/sfb1288inf/cqpserver:latest' try: container = docker_client.containers.get(container_args['name']) except docker.errors.NotFound: diff --git a/daemon/tasks/check_jobs.py b/daemon/tasks/check_jobs.py index 3d96988c..d8812ef3 100644 --- a/daemon/tasks/check_jobs.py +++ b/daemon/tasks/check_jobs.py @@ -1,46 +1,42 @@ from datetime import datetime -from logger.logger import init_logger -from tasks import Session, docker_client, NOPAQUE_STORAGE -from tasks.decorators import background -from tasks.Models import Job, NotificationData, NotificationEmailData, JobResult +from . import config, docker_client, Session +from .decorators import background +from .models import Job, JobResult, NotificationData, NotificationEmailData import docker +import logging import json import os @background def check_jobs(): - # logger = init_logger() - cj_session = Session() - jobs = cj_session.query(Job).all() + session = Session() + jobs = session.query(Job).all() for job in filter(lambda job: job.status == 'submitted', jobs): __create_job_service(job) - for job in filter(lambda job: (job.status == 'queued'), jobs): - __checkout_job_service(job, cj_session) - __add_notification_data(job, 'queued', cj_session) - for job in filter(lambda job: (job.status == 'running'), jobs): - __checkout_job_service(job, cj_session) - __add_notification_data(job, 'running', cj_session) + for job in filter(lambda job: job.status == 'queued', jobs): + __checkout_job_service(job, session) + __add_notification_data(job, 'queued', session) + for job in filter(lambda job: job.status == 'running', jobs): + __checkout_job_service(job, session) + __add_notification_data(job, 'running', session) for job in filter(lambda job: job.status == 'complete', jobs): - __add_notification_data(job, 'complete', cj_session) + __add_notification_data(job, 'complete', session) for job in filter(lambda job: job.status == 'failed', jobs): - __add_notification_data(job, 'failed', cj_session) + __add_notification_data(job, 'failed', session) for job in filter(lambda job: job.status == 'canceling', jobs): __remove_job_service(job) - cj_session.commit() + session.commit() Session.remove() -def __add_notification_data(job, notified_on_status, scoped_session): - logger = init_logger() +def __add_notification_data(job, notified_on_status, session): # checks if user wants any notifications at all if (job.user.setting_job_status_mail_notifications == 'none'): - # logger.warning('User does not want any notifications!') return # checks if user wants only notification on completed jobs elif (job.user.setting_job_status_mail_notifications == 'end' and notified_on_status != 'complete'): - # logger.warning('User only wants notifications on job completed!') return else: # check if a job already has associated NotificationData @@ -48,27 +44,21 @@ def __add_notification_data(job, notified_on_status, scoped_session): # create notification_data for current job if there is none if (notification_exists == 0): notification_data = NotificationData(job_id=job.id) - scoped_session.add(notification_data) - scoped_session.commit() + session.add(notification_data) # If no commit job will have no NotificationData - # logger.warning('Created NotificationData for current Job.')) - else: - pass - # logger.warning('Job already had notification: {}'.format(notification_exists)) + session.commit() if (job.notification_data[0].notified_on != notified_on_status): notification_email_data = NotificationEmailData(job_id=job.id) notification_email_data.notify_status = notified_on_status notification_email_data.creation_date = datetime.utcnow() job.notification_data[0].notified_on = notified_on_status - scoped_session.add(notification_email_data) - logger.warning('Created NotificationEmailData for current Job.') - else: - # logger.warning('NotificationEmailData has already been created for current Job!') - pass + session.add(notification_email_data) def __create_job_service(job): - job_dir = os.path.join(NOPAQUE_STORAGE, str(job.user_id), 'jobs', + job_dir = os.path.join(config.DATA_DIR, + str(job.user_id), + 'jobs', str(job.id)) cmd = '{} -i /files -o /files/output'.format(job.service) if job.service == 'file-setup': @@ -105,15 +95,14 @@ def __create_job_service(job): job.status = 'queued' -def __checkout_job_service(job, scoped_session): - logger = init_logger() +def __checkout_job_service(job, session): service_name = 'job_{}'.format(job.id) try: service = docker_client.services.get(service_name) except docker.errors.NotFound: - logger.error('__checkout_job_service({}):'.format(job.id) - + ' The service does not exist.' - + ' (stauts: {} -> failed)'.format(job.status)) + logging.error('__checkout_job_service({}): '.format(job.id) + + 'The service does not exist. ' + + '(status: {} -> failed)'.format(job.status)) job.status = 'failed' return except docker.errors.DockerException: @@ -130,14 +119,18 @@ def __checkout_job_service(job, scoped_session): job.end_date = datetime.utcnow() job.status = task_state if task_state == 'complete': - results_dir = os.path.join(NOPAQUE_STORAGE, str(job.user_id), - 'jobs', str(job.id), 'output') + results_dir = os.path.join(config.DATA_DIR, + str(job.user_id), + 'jobs', + str(job.id), + 'output') results = filter(lambda x: x.endswith('.zip'), os.listdir(results_dir)) for result in results: - job_result = JobResult(dir=results_dir, filename=result, + job_result = JobResult(dir=results_dir, + filename=result, job_id=job.id) - scoped_session.add(job_result) + session.add(job_result) def __remove_job_service(job): diff --git a/daemon/tasks/notify.py b/daemon/tasks/notify.py index a0ff75d4..e2976a69 100644 --- a/daemon/tasks/notify.py +++ b/daemon/tasks/notify.py @@ -1,58 +1,71 @@ from notify.notification import Notification from notify.service import NotificationService from sqlalchemy import asc -from tasks import Session -from tasks.decorators import background -from tasks.Models import NotificationEmailData -import os +from . import config, Session +from .decorators import background +from .models import NotificationEmailData +import logging +import smtplib @background -def notify(execute_flag): - # If True mails are sent normaly - # If False mails are not sent. Used to avoid sending mails for jobs that - # have been completed a long time ago. Use this if you implement notify - # into an already existing nopaque instance. Change it to True after the - # daemon has run one time with the flag set to False. - # Initialize notification service - notification_service = NotificationService(execute_flag) - notification_service.get_smtp_configs() - notification_service.set_server() +def notify(): + session = Session() + if config.SMTP_USE_SSL: + smtp = smtplib.SMTP_SSL(host=config.SMTP_SERVER, port=config.SMTP_PORT) + else: + smtp = smtplib.SMTP(host=config.SMTP_SERVER, port=config.SMTP_PORT) + if config.SMTP_USE_TLS: + smtp.starttls() + try: + smtp.login(config.SMTP_USERNAME, config.SMTP_PASSWORD) + except smtplib.SMTPHeloError: + logging.warning('The server didn’t reply properly to the HELO ' + 'greeting.') + return + except smtplib.SMTPAuthenticationError as e: + logging.warning('The server didn’t accept the username/password ' + 'combination.') + logging.warning(e) + return + except smtplib.SMTPNotSupportedError: + logging.warning('The AUTH command is not supported by the server.') + return + except smtplib.SMTPException: + logging.warning('No suitable authentication method was found.') + return + notification_service = NotificationService(smtp) # create notifications (content, recipient etc.) - notifications = __create_mail_notifications(notification_service) + notifications = __create_mail_notifications(notification_service, session) # only login and send mails if there are any notifications if (len(notifications) > 0): - try: - notification_service.login() - # combine new and unsent notifications - notifications.update(notification_service.not_sent) - # send all notifications - __send_mail_notifications(notifications, notification_service) - # remove unsent notifications because they have been sent now - # but only if mail limit has not been exceeded - if (notification_service.mail_limit_exceeded is not True): - notification_service.not_sent = {} - notification_service.quit() - except Exception as e: - notification_service.not_sent.update(notifications) - notification_service.quit() + # combine new and unsent notifications + notifications.update(notification_service.not_sent) + # send all notifications + __send_mail_notifications(notifications, notification_service) + # remove unsent notifications because they have been sent now + # but only if mail limit has not been exceeded + if (notification_service.mail_limit_exceeded is not True): + notification_service.not_sent = {} + smtp.quit() + Session.remove() # Email notification functions -def __create_mail_notifications(notification_service): - mn_session = Session() - notification_email_data = mn_session.query(NotificationEmailData).order_by(asc(NotificationEmailData.creation_date)).all() +def __create_mail_notifications(notification_service, session): + notification_email_data = session.query(NotificationEmailData).order_by(asc(NotificationEmailData.creation_date)).all() # noqa notifications = {} for data in notification_email_data: notification = Notification() - notification.set_addresses(notification_service.email_address, + notification.set_addresses(config.SMTP_DEFAULT_SENDER, data.job.user.email) - subject_template = '[nopaque] Status update for your Job/Corpora: {title}!' + subject_template = ('[nopaque] Status update for your Job/Corpora: ' + '{title}!') subject_template_values_dict = {'title': data.job.title} - protocol = os.environ.get('NOPAQUE_PROTOCOL') - domain = os.environ.get('NOPAQUE_DOMAIN') - url = '{protocol}://{domain}/{jobs}/{id}'.format( - protocol=protocol, domain=domain, jobs='jobs', id=data.job.id) + url = '{}://{}/{}/{}'.format(config.PROTOCOL, + config.DOMAIN, + 'jobs', + data.job.id) body_template_values_dict = {'username': data.job.user.username, 'id': data.job.id, 'title': data.job.title, @@ -72,9 +85,8 @@ def __create_mail_notifications(notification_service): # get swamped with mails for queued, running and complete if those # happen in in a few seconds. Only the last update will be sent. # This depends on the sleep time interval though. - mn_session.delete(data) - mn_session.commit() - Session.remove() + session.delete(data) + session.commit() return notifications @@ -83,8 +95,10 @@ def __send_mail_notifications(notifications, notification_service): try: notification_service.send(notification) notification_service.mail_limit_exceeded = False - except Exception as e: + except Exception: # Adds notifications to unsent if mail server exceded limit for # consecutive mail sending + logging.warning('limit') notification_service.not_sent[key] = notification notification_service.mail_limit_exceeded = True + notification_service.not_sent.update(notifications) diff --git a/docker-compose.development.yml b/docker-compose.development.yml new file mode 100644 index 00000000..d0542d88 --- /dev/null +++ b/docker-compose.development.yml @@ -0,0 +1,25 @@ +version: "3.5" + +services: + nopaque: + ports: + - "5000:5000" + volumes: + # Mount code as volumes + - "./web/app:/home/nopaque/app" + - "./web/boot.sh:/home/nopaque/boot.sh" + - "./web/config.py:/home/nopaque/config.py" + - "./web/migrations:/home/nopaque/migrations" + - "./web/nopaque.py:/home/nopaque/nopaque.py" + - "./web/requirements.txt:/home/nopaque/requirements.txt" + - "./web/tests:/home/nopaque/tests" + nopaqued: + volumes: + # Mount code as volumes + - "./daemon/boot.sh:/home/nopaqued/boot.sh" + - "./daemon/config.py:/home/nopaqued/config.py" + - "./daemon/logger:/home/nopaqued/logger" + - "./daemon/nopaqued.py:/home/nopaqued/nopaqued.py" + - "./daemon/notify:/home/nopaqued/notify" + - "./daemon/requirements.txt:/home/nopaqued/requirements.txt" + - "./daemon/tasks:/home/nopaqued/tasks" diff --git a/docker-compose.override.yml.tpl b/docker-compose.override.yml.tpl deleted file mode 100644 index c9d673a5..00000000 --- a/docker-compose.override.yml.tpl +++ /dev/null @@ -1,51 +0,0 @@ -version: "3.5" - -networks: - reverse-proxy: - external: - name: reverse-proxy - -services: - web: - labels: - - "traefik.docker.network=reverse-proxy" - - "traefik.enable=true" - ### ### - - "traefik.http.middlewares.nopaque-header.headers.customrequestheaders.X-Forwarded-Proto=http" - - "traefik.http.routers.nopaque.entrypoints=web" - - "traefik.http.routers.nopaque.middlewares=nopaque-header, redirect-to-https@file" - - "traefik.http.routers.nopaque.rule=Host(`${NOPAQUE_DOMAIN}`)" - ### ### - ### ### - - "traefik.http.middlewares.nopaque-secure-header.headers.customrequestheaders.X-Forwarded-Proto=https" - - "traefik.http.routers.nopaque-secure.entrypoints=web-secure" - - "traefik.http.routers.nopaque-secure.middlewares=hsts-header@file, nopaque-secure-header" - - "traefik.http.routers.nopaque-secure.rule=Host(`${NOPAQUE_DOMAIN}`)" - - "traefik.http.routers.nopaque-secure.tls.options=intermediate@file" - ### ### - ### ### - # - "traefik.http.middlewares.nopaque-basicauth.basicauth.users=:" - # - "traefik.http.routers.nopaque.middlewares=nopaque-basicauth, nopaque-header, redirect-to-https@file" - # - "traefik.http.routers.nopaque-secure.middlewares=nopaque-basicauth, hsts-header@file, nopaque-secure-header" - ### ### - networks: - - default - - reverse-proxy - volumes: - # Mount code as volumes - - "./web/app:/home/nopaque/app" - - "./web/migrations:/home/nopaque/migrations" - - "./web/tests:/home/nopaque/tests" - - "./web/config.py:/home/nopaque/config.py" - - "./web/docker-entrypoint.sh:/home/nopaque/docker-entrypoint.sh" - - "./web/nopaque.py:/home/nopaque/nopaque.py" - - "./web/requirements.txt:/home/nopaque/requirements.txt" - daemon: - volumes: - # Mount code as volumes - - "./daemon/logger:/home/nopaqued/logger" - - "./daemon/notify:/home/nopaqued/notify" - - "./daemon/tasks:/home/nopaqued/tasks" - - "./daemon/docker-entrypoint.sh:/home/nopaqued/docker-entrypoint.sh" - - "./daemon/nopaqued.py:/home/nopaqued/nopaqued.py" - - "./daemon/requirements.txt:/home/nopaqued/requirements.txt" diff --git a/docker-compose.traefik.yml b/docker-compose.traefik.yml new file mode 100644 index 00000000..53fbdb9f --- /dev/null +++ b/docker-compose.traefik.yml @@ -0,0 +1,30 @@ +################################################################################ +# Don't forget to set the NOPAQUE_NUM_PROXIES variable in your .env # +################################################################################ +version: "3.5" + +networks: + reverse-proxy: + external: + name: reverse-proxy + +services: + nopaque: + labels: + - "traefik.docker.network=reverse-proxy" + - "traefik.enable=true" + ### ### + - "traefik.http.routers.nopaque.entrypoints=web" + - "traefik.http.routers.nopaque.middlewares=redirect-to-https@file" + - "traefik.http.routers.nopaque.rule=Host(`${NOPAQUE_DOMAIN:-localhost}`)" + ### ### + ### ### + - "traefik.http.routers.nopaque-secure.entrypoints=web-secure" + - "traefik.http.routers.nopaque-secure.middlewares=hsts-header@file" + - "traefik.http.routers.nopaque-secure.rule=Host(`${NOPAQUE_DOMAIN:-localhost}`)" + - "traefik.http.routers.nopaque-secure.tls.certresolver=" + - "traefik.http.routers.nopaque-secure.tls.options=intermediate@file" + ### ### + networks: + - default + - reverse-proxy diff --git a/docker-compose.yml b/docker-compose.yml index ae210132..9e5bad1a 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,49 +1,49 @@ version: "3.5" -volumes: - redis-trash1: - services: - web: + nopaque: build: args: - GID: ${GID} - UID: ${UID} + GID: ${HOST_GID} + UID: ${HOST_UID} context: ./web depends_on: - db - - redis + - mq env_file: .env image: nopaque/web restart: unless-stopped volumes: - - "./logs:/home/nopaque/logs" - - "${NOPAQUE_STORAGE}:${NOPAQUE_STORAGE}" - daemon: + - "${NOPAQUE_DATA_DIR:-/mnt/nopaque}:${NOPAQUE_DATA_DIR:-/mnt/nopaque}" + - "${HOST_NOPAQUE_LOG_FILE-./nopaque.log}:${NOPAQUE_LOG_FILE:-/home/nopaque/nopaque.log}" + nopaqued: build: args: - DOCKER_GID: ${DOCKER_GID} - GID: ${GID} - UID: ${UID} + DOCKER_GID: ${HOST_DOCKER_GID} + GID: ${HOST_GID} + UID: ${HOST_UID} context: ./daemon depends_on: - db - - web + - nopaque env_file: .env image: nopaque/daemon restart: unless-stopped volumes: - "/var/run/docker.sock:/var/run/docker.sock" - - "./logs:/home/nopaqued/logs" - - "${NOPAQUE_STORAGE}:${NOPAQUE_STORAGE}" + - "${NOPAQUE_DATA_DIR:-/mnt/nopaque}:${NOPAQUE_DATA_DIR:-/mnt/nopaque}" + - "${HOST_NOPAQUE_DAEMON_LOG_FILE-./nopaqued.log}:${NOPAQUE_DAEMON_LOG_FILE:-/home/nopaqued/nopaqued.log}" db: - env_file: .env + environment: + - POSTGRES_DB_NAME=${NOPAQUE_DB_NAME} + - POSTGRES_USER=${NOPAQUE_DB_USERNAME} + - POSTGRES_PASSWORD=${NOPAQUE_DB_PASSWORD} image: postgres:11 restart: unless-stopped volumes: - - "/srv/nopaque/db:/var/lib/postgresql/data" - redis: + - "${HOST_DB_DIR:-./db}:/var/lib/postgresql/data" + mq: image: redis:6 restart: unless-stopped volumes: - - "redis-trash1:/data" + - "${HOST_MQ_DIR:-./mq}:/data" diff --git a/logs/dummy b/logs/dummy deleted file mode 100644 index e69de29b..00000000 diff --git a/web/.dockerignore b/web/.dockerignore index 96dbc1bd..21803000 100644 --- a/web/.dockerignore +++ b/web/.dockerignore @@ -1,3 +1,6 @@ +# Docker related files Dockerfile .dockerignore -*.bak + +# Packages +__pycache__ diff --git a/web/Dockerfile b/web/Dockerfile index 57c19cbd..dc4e149c 100644 --- a/web/Dockerfile +++ b/web/Dockerfile @@ -1,7 +1,7 @@ FROM python:3.6-slim-stretch -LABEL maintainer="inf_sfb1288@lists.uni-bielefeld.de" +LABEL authors="Patrick Jentsch , Stephan Porada " ARG UID @@ -18,7 +18,7 @@ RUN apt-get update \ build-essential \ libpq-dev \ wait-for-it \ - && rm -rf /var/lib/apt/lists/* + && rm -r /var/lib/apt/lists/* RUN groupadd --gid ${GID} --system nopaque \ @@ -33,4 +33,4 @@ RUN python -m venv venv \ && mkdir logs -ENTRYPOINT ["./docker-entrypoint.sh"] +ENTRYPOINT ["./boot.sh"] diff --git a/web/app/__init__.py b/web/app/__init__.py index 6244a0e4..302a3fa1 100644 --- a/web/app/__init__.py +++ b/web/app/__init__.py @@ -1,15 +1,14 @@ -from config import config +from config import Config from flask import Flask from flask_login import LoginManager from flask_mail import Mail from flask_paranoid import Paranoid from flask_socketio import SocketIO from flask_sqlalchemy import SQLAlchemy -import logging +config = Config() db = SQLAlchemy() -logger = logging.getLogger(__name__) login_manager = LoginManager() login_manager.login_view = 'auth.login' mail = Mail() @@ -18,44 +17,36 @@ paranoid.redirect_view = '/' socketio = SocketIO() -def create_app(config_name): +def create_app(): app = Flask(__name__) - app.config.from_object(config[config_name]) + app.config.from_object(config) - config[config_name].init_app(app) + config.init_app(app) db.init_app(app) login_manager.init_app(app) mail.init_app(app) paranoid.init_app(app) - socketio.init_app(app, message_queue='redis://redis:6379/') + socketio.init_app(app, message_queue=config.SOCKETIO_MESSAGE_QUEUE_URI) from . import events from .admin import admin as admin_blueprint app.register_blueprint(admin_blueprint, url_prefix='/admin') - from .auth import auth as auth_blueprint app.register_blueprint(auth_blueprint, url_prefix='/auth') - from .content import content as content_blueprint app.register_blueprint(content_blueprint, url_prefix='/content') - from .corpora import corpora as corpora_blueprint app.register_blueprint(corpora_blueprint, url_prefix='/corpora') - from .jobs import jobs as jobs_blueprint app.register_blueprint(jobs_blueprint, url_prefix='/jobs') - from .main import main as main_blueprint app.register_blueprint(main_blueprint) - from .profile import profile as profile_blueprint app.register_blueprint(profile_blueprint, url_prefix='/profile') - from .query_results import query_results as query_results_blueprint app.register_blueprint(query_results_blueprint, url_prefix='/query_results') - from .services import services as services_blueprint app.register_blueprint(services_blueprint, url_prefix='/services') diff --git a/web/app/auth/views.py b/web/app/auth/views.py index 6b87f744..c0fe6934 100644 --- a/web/app/auth/views.py +++ b/web/app/auth/views.py @@ -65,7 +65,7 @@ def register(): username=registration_form.username.data) db.session.add(user) db.session.commit() - user_dir = os.path.join(current_app.config['NOPAQUE_STORAGE'], + user_dir = os.path.join(current_app.config['DATA_DIR'], str(user.id)) if os.path.exists(user_dir): shutil.rmtree(user_dir) diff --git a/web/app/corpora/events.py b/web/app/corpora/events.py index 7357d817..ced63a3a 100644 --- a/web/app/corpora/events.py +++ b/web/app/corpora/events.py @@ -9,8 +9,6 @@ import cqi import math from datetime import datetime -import time -from app import logger ''' ' A dictionary containing lists of, with corpus ids associated, Socket.IO @@ -41,7 +39,8 @@ def corpus_analysis_get_meta_data(corpus_id): metadata['corpus_name'] = db_corpus.title metadata['corpus_description'] = db_corpus.description metadata['corpus_creation_date'] = db_corpus.creation_date.isoformat() - metadata['corpus_last_edited_date'] = db_corpus.last_edited_date.isoformat() + metadata['corpus_last_edited_date'] = \ + db_corpus.last_edited_date.isoformat() client = corpus_analysis_clients.get(request.sid) if client is None: response = {'code': 424, 'desc': 'No client found for this session', @@ -61,18 +60,20 @@ def corpus_analysis_get_meta_data(corpus_id): metadata['corpus_size_tokens'] = client_corpus.attrs['size'] text_attr = client_corpus.structural_attributes.get('text') - struct_attrs = client_corpus.structural_attributes.list(filters={'part_of': text_attr}) + struct_attrs = client_corpus.structural_attributes.list( + filters={'part_of': text_attr}) text_ids = range(0, (text_attr.attrs['size'])) texts_metadata = {} for text_id in text_ids: texts_metadata[text_id] = {} for struct_attr in struct_attrs: - texts_metadata[text_id][struct_attr.attrs['name'][(len(text_attr.attrs['name']) + 1):]] = struct_attr.values_by_ids(list(range(struct_attr.attrs['size'])))[text_id] + texts_metadata[text_id][struct_attr.attrs['name'][(len(text_attr.attrs['name']) + 1):]] = struct_attr.values_by_ids(list(range(struct_attr.attrs['size'])))[text_id] # noqa metadata['corpus_all_texts'] = texts_metadata metadata['corpus_analysis_date'] = datetime.utcnow().isoformat() metadata['corpus_cqi_py_protocol_version'] = client.api.version metadata['corpus_cqi_py_package_version'] = cqi.__version__ - metadata['corpus_cqpserver_version'] = 'CQPserver v3.4.22' # TODO: make this dynamically + # TODO: make this dynamically + metadata['corpus_cqpserver_version'] = 'CQPserver v3.4.22' # write some metadata to the db db_corpus.current_nr_of_tokens = metadata['corpus_size_tokens'] @@ -133,7 +134,7 @@ def corpus_analysis_query(query): if (results.attrs['size'] == 0): progress = 100 else: - progress = ((chunk_start + chunk_size) / results.attrs['size']) * 100 + progress = ((chunk_start + chunk_size) / results.attrs['size']) * 100 # noqa progress = min(100, int(math.ceil(progress))) response = {'code': 200, 'desc': None, 'msg': 'OK', 'payload': {'chunk': chunk, 'progress': progress}} @@ -202,7 +203,9 @@ def corpus_analysis_get_match_with_full_context(payload): 'payload': payload, 'type': type, 'data_indexes': data_indexes} - socketio.emit('corpus_analysis_get_match_with_full_context', response, room=request.sid) + socketio.emit('corpus_analysis_get_match_with_full_context', + response, + room=request.sid) client.status = 'ready' diff --git a/web/app/corpora/views.py b/web/app/corpora/views.py index 2cbc3f6a..234d6207 100644 --- a/web/app/corpora/views.py +++ b/web/app/corpora/views.py @@ -21,7 +21,7 @@ def add_corpus(): status='unprepared', title=add_corpus_form.title.data) db.session.add(corpus) db.session.commit() - dir = os.path.join(current_app.config['NOPAQUE_STORAGE'], + dir = os.path.join(current_app.config['DATA_DIR'], str(corpus.user_id), 'corpora', str(corpus.id)) try: os.makedirs(dir) @@ -109,7 +109,7 @@ def add_corpus_file(corpus_id): # Save the file dir = os.path.join(str(corpus.user_id), 'corpora', str(corpus.id)) add_corpus_file_form.file.data.save( - os.path.join(current_app.config['NOPAQUE_STORAGE'], dir, + os.path.join(current_app.config['DATA_DIR'], dir, add_corpus_file_form.file.data.filename)) corpus_file = CorpusFile( address=add_corpus_file_form.address.data, @@ -163,7 +163,7 @@ def download_corpus_file(corpus_id, corpus_file_id): if not (corpus_file.corpus.creator == current_user or current_user.is_administrator()): abort(403) - dir = os.path.join(current_app.config['NOPAQUE_STORAGE'], + dir = os.path.join(current_app.config['DATA_DIR'], corpus_file.dir) return send_from_directory(as_attachment=True, directory=dir, filename=corpus_file.filename) diff --git a/web/app/email.py b/web/app/email.py index b6dd4e4e..4969b05e 100644 --- a/web/app/email.py +++ b/web/app/email.py @@ -1,15 +1,11 @@ -from flask import current_app, render_template +from flask import render_template from flask_mail import Message from . import mail from .decorators import background def create_message(recipient, subject, template, **kwargs): - app = current_app._get_current_object() - sender = app.config['NOPAQUE_MAIL_SENDER'] - subject_prefix = app.config['NOPAQUE_MAIL_SUBJECT_PREFIX'] - msg = Message('{} {}'.format(subject_prefix, subject), - recipients=[recipient], sender=sender) + msg = Message('[nopaque] {}'.format(subject), recipients=[recipient]) msg.body = render_template('{}.txt.j2'.format(template), **kwargs) msg.html = render_template('{}.html.j2'.format(template), **kwargs) return msg diff --git a/web/app/jobs/views.py b/web/app/jobs/views.py index 557413cc..a92013f7 100644 --- a/web/app/jobs/views.py +++ b/web/app/jobs/views.py @@ -44,7 +44,7 @@ def download_job_input(job_id, job_input_id): if not (job_input.job.creator == current_user or current_user.is_administrator()): abort(403) - dir = os.path.join(current_app.config['NOPAQUE_STORAGE'], + dir = os.path.join(current_app.config['DATA_DIR'], job_input.dir) return send_from_directory(as_attachment=True, directory=dir, filename=job_input.filename) @@ -72,7 +72,7 @@ def download_job_result(job_id, job_result_id): if not (job_result.job.creator == current_user or current_user.is_administrator()): abort(403) - dir = os.path.join(current_app.config['NOPAQUE_STORAGE'], + dir = os.path.join(current_app.config['DATA_DIR'], job_result.dir) return send_from_directory(as_attachment=True, directory=dir, filename=job_result.filename) diff --git a/web/app/main/forms.py b/web/app/main/forms.py deleted file mode 100644 index d209a6ba..00000000 --- a/web/app/main/forms.py +++ /dev/null @@ -1,12 +0,0 @@ -from flask_wtf import FlaskForm -from wtforms import DecimalField, StringField, SubmitField, TextAreaField -from wtforms.validators import DataRequired, Email, Length, NumberRange - - -class FeedbackForm(FlaskForm): - email = StringField('Email', validators=[DataRequired(), Email()]) - feedback = TextAreaField('Feedback', validators=[Length(0, 255)]) - like_range = DecimalField('How would you rate nopaque?', - validators=[DataRequired(), - NumberRange(min=1, max=10)]) - submit = SubmitField('Send feedback') diff --git a/web/app/main/views.py b/web/app/main/views.py index 15009e42..eefb3043 100644 --- a/web/app/main/views.py +++ b/web/app/main/views.py @@ -1,8 +1,6 @@ from flask import flash, redirect, render_template, url_for from flask_login import login_required, login_user from . import main -from .forms import FeedbackForm -from .. import logger from ..auth.forms import LoginForm from ..models import User @@ -28,18 +26,6 @@ def dashboard(): return render_template('main/dashboard.html.j2', title='Dashboard') -@main.route('/feedback', methods=['GET', 'POST']) -@login_required -def feedback(): - feedback_form = FeedbackForm(prefix='feedback-form') - if feedback_form.validate_on_submit(): - logger.warning(feedback_form.email) - logger.warning(feedback_form.feedback) - logger.warning(feedback_form.like_range) - return render_template('main/feedback.html.j2', - feedback_form=feedback_form, title='Feedback') - - @main.route('/poster', methods=['GET', 'POST']) def poster(): login_form = LoginForm(prefix='login-form') diff --git a/web/app/models.py b/web/app/models.py index d377272c..e28a5b06 100644 --- a/web/app/models.py +++ b/web/app/models.py @@ -166,7 +166,7 @@ class User(UserMixin, db.Model): def __init__(self, **kwargs): super(User, self).__init__(**kwargs) if self.role is None: - if self.email == current_app.config['NOPAQUE_ADMIN']: + if self.email == current_app.config['ADMIN_EMAIL_ADRESS']: self.role = Role.query.filter_by(name='Administrator').first() if self.role is None: self.role = Role.query.filter_by(default=True).first() @@ -251,7 +251,7 @@ class User(UserMixin, db.Model): ''' Delete the user and its corpora and jobs from database and filesystem. ''' - user_dir = os.path.join(current_app.config['NOPAQUE_STORAGE'], + user_dir = os.path.join(current_app.config['DATA_DIR'], str(self.id)) shutil.rmtree(user_dir, ignore_errors=True) db.session.delete(self) @@ -383,7 +383,7 @@ class Job(db.Model): db.session.commit() sleep(1) db.session.refresh(self) - job_dir = os.path.join(current_app.config['NOPAQUE_STORAGE'], + job_dir = os.path.join(current_app.config['DATA_DIR'], str(self.user_id), 'jobs', str(self.id)) @@ -397,7 +397,7 @@ class Job(db.Model): if self.status != 'failed': raise Exception('Could not restart job: status is not "failed"') - job_dir = os.path.join(current_app.config['NOPAQUE_STORAGE'], + job_dir = os.path.join(current_app.config['DATA_DIR'], str(self.user_id), 'jobs', str(self.id)) @@ -508,7 +508,7 @@ class CorpusFile(db.Model): title = db.Column(db.String(255)) def delete(self): - corpus_file_path = os.path.join(current_app.config['NOPAQUE_STORAGE'], + corpus_file_path = os.path.join(current_app.config['DATA_DIR'], str(self.corpus.user_id), 'corpora', str(self.corpus_id), @@ -570,7 +570,7 @@ class Corpus(db.Model): 'files': {file.id: file.to_dict() for file in self.files}} def build(self): - corpus_dir = os.path.join(current_app.config['NOPAQUE_STORAGE'], + corpus_dir = os.path.join(current_app.config['DATA_DIR'], str(self.user_id), 'corpora', str(self.id)) @@ -606,7 +606,7 @@ class Corpus(db.Model): self.status = 'submitted' def delete(self): - corpus_dir = os.path.join(current_app.config['NOPAQUE_STORAGE'], + corpus_dir = os.path.join(current_app.config['DATA_DIR'], str(self.user_id), 'corpora', str(self.id)) @@ -636,7 +636,7 @@ class QueryResult(db.Model): title = db.Column(db.String(32)) def delete(self): - query_result_dir = os.path.join(current_app.config['NOPAQUE_STORAGE'], + query_result_dir = os.path.join(current_app.config['DATA_DIR'], str(self.user_id), 'query_results', str(self.id)) diff --git a/web/app/query_results/views.py b/web/app/query_results/views.py index ac21a749..ff6eae5f 100644 --- a/web/app/query_results/views.py +++ b/web/app/query_results/views.py @@ -31,7 +31,7 @@ def add_query_result(): db.session.add(query_result) db.session.commit() # create paths to save the uploaded json file - query_result_dir = os.path.join(current_app.config['NOPAQUE_STORAGE'], + query_result_dir = os.path.join(current_app.config['DATA_DIR'], str(current_user.id), 'query_results', str(query_result.id)) @@ -106,7 +106,7 @@ def inspect_query_result(query_result_id): prefix='inspect-display-options-form' ) query_result_file_path = os.path.join( - current_app.config['NOPAQUE_STORAGE'], + current_app.config['DATA_DIR'], str(current_user.id), 'query_results', str(query_result.id), @@ -141,7 +141,7 @@ def download_query_result(query_result_id): if not (query_result.creator == current_user or current_user.is_administrator()): abort(403) - query_result_dir = os.path.join(current_app.config['NOPAQUE_STORAGE'], + query_result_dir = os.path.join(current_app.config['DATA_DIR'], str(current_user.id), 'query_results', str(query_result.id)) diff --git a/web/app/services/views.py b/web/app/services/views.py index 3c8d0b08..6fbf2ef0 100644 --- a/web/app/services/views.py +++ b/web/app/services/views.py @@ -55,7 +55,7 @@ def service(service): db.session.add(job) db.session.commit() relative_dir = os.path.join(str(job.user_id), 'jobs', str(job.id)) - absolut_dir = os.path.join(current_app.config['NOPAQUE_STORAGE'], + absolut_dir = os.path.join(current_app.config['DATA_DIR'], relative_dir) try: os.makedirs(absolut_dir) diff --git a/web/app/templates/main/feedback.html.j2 b/web/app/templates/main/feedback.html.j2 deleted file mode 100644 index 2e47fb8d..00000000 --- a/web/app/templates/main/feedback.html.j2 +++ /dev/null @@ -1,35 +0,0 @@ -{% extends "nopaque.html.j2" %} - -{% block page_content %} - -
-
-
- {{ feedback_form.hidden_tag() }} -
-

- {{ feedback_form.like_range.label }} - {{ feedback_form.like_range(class='validate', type='range', min=1, max=10) }} -

-
- email - {{ feedback_form.email(class='validate', type='email') }} - {{ feedback_form.email.label }} - {% for error in feedback_form.email.errors %} - {{ error }} - {% endfor %} -
-
- mode_edit - {{ feedback_form.feedback(class='materialize-textarea', data_length=255) }} - {{ feedback_form.feedback.label }} -
-
-
- {{ M.render_field(feedback_form.submit, material_icon='send') }} -
-
-
-
- -{% endblock %} diff --git a/web/app/templates/main/poster.html.j2 b/web/app/templates/main/poster.html.j2 deleted file mode 100644 index ba20e8b3..00000000 --- a/web/app/templates/main/poster.html.j2 +++ /dev/null @@ -1,202 +0,0 @@ -{% extends "nopaque.html.j2" %} - -{% set parallax = True %} - -{% block page_content %} - - -
-
-
-
-

nopaque

-

From text to data to analysis

-

Patrick Jentsch, Stephan Porada and Helene Schlicht

-
-
-

 

-
-
-
-
-

 

- -
-
-

nopaque is a web application that helps to convert heterogeneous textual source material into standard-compliant research data for subsequent analysis. nopaque is designed to accompany your research process.

-

The web application is developed within the DFG-funded Collaborative Research Center (SFB) 1288 "Practices of Comparison" by the subproject INF "Data Infrastructure and Digital Humanities".

-
-
-
-
-
-
-
- -
- -
- -
-
-
-
-
-

Why you should use nopaque

-

nopaque is a custom-built web application for researchers who want to get out more of their images and texts without having to bother about the technical side of things. You can focus on what really interests you, nopaque does the rest.

-

nopaque’s utilization of container virtualization guarantees high interoperability, reusability and reproducibility of research results. All processing steps are carried out in containers created on demand, based on static images with fixed software versions including all dependencies.

-
- -
-
-
- flash_on -

Speeds up your work

-

All tools provided by nopaque are carefully selected to provide a complete tool suite without being held up by compatibility issues.

-
-
- cloud -

Cloud infrastructure

-

All computational work is processed within nopaque’s cloud infrastructure. You don't need to install any software. Great, right?

-
-
- group -

User friendly

-

You can start right away without having to read mile-long manuals. All services come with default settings that make it easy for you to just get going. Also great, right?

-
-
- settings -

Meshing processes

-

No matter where you step in, nopaque facilitates and accompanies your research. Its workflow perfectly ties in with your research process.

-
-
-
-
-
-
-
- -
- -
- -
-
-
-
-
-

What nopaque can do for you

-

All services and processes are logically linked and built upon each other. You can follow them step by step or directly choose the one that suits your needs best. And while the process is computed in nopaque’s cloud infrastructure, you can just keep working.

-
- -
-
-
-
- burst_mode -

File setup

-

Digital copies of text based research data (books, letters, etc.) often comprise various files and formats. nopaque converts and merges those files to facilitate further processing and the application of other services.

-
-
- find_in_page -

Optical Character Recognition

-

nopaque converts your image data – like photos or scans – into text data through OCR making it machine readable. This step enables you to proceed with further computational analysis of your documents.

-
-
- format_textdirection_l_to_r -

Natural Language Processing

-

By means of computational linguistic data processing (tokenization, lemmatization, part-of-speech tagging and named-entity recognition) nopaque extracts additional information from your text.

-
-
- search -

Corpus analysis

-

nopaque lets you create and upload as many text corpora as you want. It makes use of CQP Query Language, which allows for complex search requests with the aid of metadata and NLP tags.

-
-
-
-
-
-
-
- -
- -
- -
-
-
-
- - -
-
-
- {{ login_form.hidden_tag() }} -
- Registration and Log in -
- person - {{ login_form.user(class='validate') }} - {{ login_form.user.label }} - {% for error in login_form.user.errors %} - {{ error }} - {% endfor %} -
-
- vpn_key - {{ login_form.password(class='validate') }} - {{ login_form.password.label }} - {% for error in login_form.password.errors %} - {{ error }} - {% endfor %} -
-
-
- Forgot your password? - | - No account yet? -
-
- {{ materialize.submit_button(login_form.submit) }} -
-
-
-
-
-
-
-
-
-
-
-{% endblock %} diff --git a/web/boot.sh b/web/boot.sh new file mode 100755 index 00000000..871e149c --- /dev/null +++ b/web/boot.sh @@ -0,0 +1,21 @@ +#!/bin/bash +echo "Waiting for db..." +wait-for-it "${NOPAQUE_DB_HOST}:${NOPAQUE_DB_PORT:-5432}" --strict --timeout=0 +echo "Waiting for mq..." +wait-for-it "${NOPAQUE_MQ_HOST}:${NOPAQUE_MQ_PORT}" --strict --timeout=0 + +source venv/bin/activate + +if [ "$#" -eq 0 ]; then + flask deploy + python nopaque.py +elif [[ "$1" == "flask" ]]; then + exec ${@:1} +else + echo "$0 [COMMAND]" + echo "" + echo "nopaque startup script" + echo "" + echo "Management Commands:" + echo " flask" +fi diff --git a/web/config.py b/web/config.py index 7bb300ff..94640d99 100644 --- a/web/config.py +++ b/web/config.py @@ -1,85 +1,97 @@ from werkzeug.middleware.proxy_fix import ProxyFix -import os import logging +import os + + +root_dir = os.path.abspath(os.path.dirname(__file__)) + + +DEFAULT_DATA_DIR = os.path.join('/mnt/data') +DEFAULT_DB_PORT = '5432' +DEFAULT_DEBUG = 'False' +DEFAULT_LOG_DATE_FORMAT = '%Y-%m-%d %H:%M:%S' +DEFAULT_LOG_FILE = os.path.join(root_dir, 'nopaque.log') +DEFAULT_LOG_FORMAT = ('[%(asctime)s] %(levelname)s in %(pathname)s ' + '(function: %(funcName)s, line: %(lineno)d): ' + '%(message)s') +DEFAULT_LOG_LEVEL = 'ERROR' +DEFAULT_SMTP_USE_SSL = 'False' +DEFAULT_SMTP_USE_TLS = 'False' +DEFAULT_NUM_PROXIES = '0' +DEFAULT_PROTOCOL = 'http' +DEFAULT_RESSOURCES_PER_PAGE = '5' +DEFAULT_USERS_PER_PAGE = '10' +DEFAULT_SECRET_KEY = 'hard to guess string' class Config: - ''' ### Flask ### ''' - SECRET_KEY = os.environ.get('SECRET_KEY') or 'hard to guess string' - - ''' ### Flask-Mail ### ''' - MAIL_SERVER = os.environ.get('MAIL_SERVER') - MAIL_PORT = int(os.environ.get('MAIL_PORT')) - MAIL_USE_TLS = os.environ.get('MAIL_USE_TLS').lower() == 'true' - MAIL_USERNAME = os.environ.get('MAIL_USERNAME') - MAIL_PASSWORD = os.environ.get('MAIL_PASSWORD') - - ''' ### Flask-SQLAlchemy ### ''' - SQLALCHEMY_DATABASE_URI = 'postgresql://{}:{}@db/{}'.format( - os.environ.get('POSTGRES_USER'), - os.environ.get('POSTGRES_PASSWORD'), - os.environ.get('POSTGRES_DB_NAME')) + ''' ### Database ### ''' + DB_HOST = os.environ.get('NOPAQUE_DB_HOST') + DB_NAME = os.environ.get('NOPAQUE_DB_NAME') + DB_PASSWORD = os.environ.get('NOPAQUE_DB_PASSWORD') + DB_PORT = os.environ.get('NOPAQUE_DB_PORT', DEFAULT_DB_PORT) + DB_USERNAME = os.environ.get('NOPAQUE_DB_USERNAME') + SQLALCHEMY_DATABASE_URI = 'postgresql://{}:{}@{}:{}/{}'.format( + DB_USERNAME, DB_PASSWORD, DB_HOST, DB_PORT, DB_NAME) SQLALCHEMY_RECORD_QUERIES = True SQLALCHEMY_TRACK_MODIFICATIONS = False - ''' ### nopaque ### ''' - NOPAQUE_ADMIN = os.environ.get('NOPAQUE_ADMIN') - NOPAQUE_CONTACT = os.environ.get('NOPAQUE_CONTACT') - NOPAQUE_MAIL_SENDER = os.environ.get('NOPAQUE_MAIL_SENDER') - NOPAQUE_MAIL_SUBJECT_PREFIX = '[nopaque]' - NOPAQUE_PROTOCOL = os.environ.get('NOPAQUE_PROTOCOL') - NOPAQUE_STORAGE = os.environ.get('NOPAQUE_STORAGE') + ''' ### Email ### ''' + MAIL_DEFAULT_SENDER = os.environ.get('NOPAQUE_SMTP_DEFAULT_SENDER') + MAIL_PASSWORD = os.environ.get('NOPAQUE_SMTP_PASSWORD') + MAIL_PORT = os.environ.get('NOPAQUE_SMTP_PORT') + MAIL_SERVER = os.environ.get('NOPAQUE_SMTP_SERVER') + MAIL_USERNAME = os.environ.get('NOPAQUE_SMTP_USERNAME') + MAIL_USE_SSL = os.environ.get('NOPAQUE_SMTP_USE_SSL', + DEFAULT_SMTP_USE_SSL).lower() == 'true' + MAIL_USE_TLS = os.environ.get('NOPAQUE_SMTP_USE_TLS', + DEFAULT_SMTP_USE_TLS).lower() == 'true' - os.makedirs('logs', exist_ok=True) - logging.basicConfig(filename='logs/nopaque.log', - format='[%(asctime)s] %(levelname)s in ' - '%(pathname)s:%(lineno)d - %(message)s', - datefmt='%Y-%m-%d %H:%M:%S', filemode='w') - - ''' ### Security enhancements ### ''' - if NOPAQUE_PROTOCOL == 'https': - ''' ### Flask ### ''' - SESSION_COOKIE_SECURE = True - - ''' ### Flask-Login ### ''' + ''' ### General ### ''' + ADMIN_EMAIL_ADRESS = os.environ.get('NOPAQUE_ADMIN_EMAIL_ADRESS') + CONTACT_EMAIL_ADRESS = os.environ.get('NOPAQUE_CONTACT_EMAIL_ADRESS') + DATA_DIR = os.environ.get('NOPAQUE_DATA_DIR', DEFAULT_DATA_DIR) + DEBUG = os.environ.get('NOPAQUE_DEBUG', DEFAULT_DEBUG).lower() == 'true' + NUM_PROXIES = int(os.environ.get('NOPAQUE_NUM_PROXIES', + DEFAULT_NUM_PROXIES)) + PROTOCOL = os.environ.get('NOPAQUE_PROTOCOL', DEFAULT_PROTOCOL) + RESSOURCES_PER_PAGE = int(os.environ.get('NOPAQUE_RESSOURCES_PER_PAGE', + DEFAULT_RESSOURCES_PER_PAGE)) + SECRET_KEY = os.environ.get('NOPAQUE_SECRET_KEY', DEFAULT_SECRET_KEY) + USERS_PER_PAGE = int(os.environ.get('NOPAQUE_USERS_PER_PAGE', + DEFAULT_USERS_PER_PAGE)) + if PROTOCOL == 'https': REMEMBER_COOKIE_HTTPONLY = True REMEMBER_COOKIE_SECURE = True + SESSION_COOKIE_SECURE = True - @staticmethod - def init_app(app): - proxy_fix_kwargs = {'x_for': 1, 'x_host': 1, 'x_port': 1, 'x_proto': 1} - app.wsgi_app = ProxyFix(app.wsgi_app, **proxy_fix_kwargs) + ''' ### Logging ### ''' + LOG_DATE_FORMAT = os.environ.get('NOPAQUE_LOG_DATE_FORMAT', + DEFAULT_LOG_DATE_FORMAT) + LOG_FILE = os.environ.get('NOPAQUE_LOG_FILE', DEFAULT_LOG_FILE) + LOG_FORMAT = os.environ.get('NOPAQUE_LOG_FORMAT', DEFAULT_LOG_FORMAT) + LOG_LEVEL = os.environ.get('NOPAQUE_LOG_LEVEL', DEFAULT_LOG_LEVEL) + ''' ### Message queue ### ''' + MQ_HOST = os.environ.get('NOPAQUE_MQ_HOST') + MQ_PORT = os.environ.get('NOPAQUE_MQ_PORT') + MQ_TYPE = os.environ.get('NOPAQUE_MQ_TYPE') + SOCKETIO_MESSAGE_QUEUE_URI = \ + '{}://{}:{}/'.format(MQ_TYPE, MQ_HOST, MQ_PORT) -class DevelopmentConfig(Config): - ''' ### Flask ### ''' - DEBUG = True - - ''' ### nopaque ### ''' - NOPAQUE_LOG_LEVEL = os.environ.get('NOPAQUE_LOG_LEVEL') or 'DEBUG' - logging.basicConfig(level=NOPAQUE_LOG_LEVEL) - - -class TestingConfig(Config): - ''' ### Flask ### ''' - TESTING = True - - ''' ### Flask-SQLAlchemy ### ''' - SQLALCHEMY_DATABASE_URI = 'sqlite://' - - ''' ### Flask-WTF ### ''' - WTF_CSRF_ENABLED = False - - -class ProductionConfig(Config): - ''' ### nopaque ### ''' - NOPAQUE_LOG_LEVEL = os.environ.get('NOPAQUE_LOG_LEVEL') or 'ERROR' - logging.basicConfig(level=NOPAQUE_LOG_LEVEL) - - -config = { - 'development': DevelopmentConfig, - 'testing': TestingConfig, - 'production': ProductionConfig, - 'default': DevelopmentConfig, -} + def init_app(self, app): + # Configure logging according to the corresponding (LOG_*) config + # entries + logging.basicConfig(datefmt=self.LOG_DATE_FORMAT, + filename=self.LOG_FILE, + format=self.LOG_FORMAT, + level=self.LOG_LEVEL) + # Apply the ProxyFix middleware if nopaque is running behind reverse + # proxies. (NUM_PROXIES indicates the number of reverse proxies running + # in front of nopaque) + if self.NUM_PROXIES > 0: + app.wsgi_app = ProxyFix(app.wsgi_app, + x_for=self.NUM_PROXIES, + x_host=self.NUM_PROXIES, + x_port=self.NUM_PROXIES, + x_proto=self.NUM_PROXIES) diff --git a/web/docker-entrypoint.sh b/web/docker-entrypoint.sh deleted file mode 100755 index 51c56c92..00000000 --- a/web/docker-entrypoint.sh +++ /dev/null @@ -1,16 +0,0 @@ -#!/bin/bash - -echo "Waiting for db..." -wait-for-it db:5432 --strict --timeout=0 -echo "Waiting for redis..." -wait-for-it redis:6379 --strict --timeout=0 - -source venv/bin/activate -if [ $# -eq 0 ]; then - flask deploy - python nopaque.py -elif [ $1 == "flask" ]; then - flask ${@:2} -else - echo "$0 [flask [options]]" -fi diff --git a/web/nopaque.py b/web/nopaque.py index 56b2bbeb..b56fba8b 100644 --- a/web/nopaque.py +++ b/web/nopaque.py @@ -5,9 +5,8 @@ from app.models import (Corpus, CorpusFile, Job, JobInput, JobResult, NotificationData, NotificationEmailData, QueryResult, Role, User) from flask_migrate import Migrate, upgrade -import os -app = create_app(os.getenv('FLASK_CONFIG') or 'default') +app = create_app() migrate = Migrate(app, db, compare_type=True) diff --git a/web/tests/test_basics.py b/web/tests/test_basics.py index 0fdf4983..e52943de 100644 --- a/web/tests/test_basics.py +++ b/web/tests/test_basics.py @@ -17,6 +17,3 @@ class BasicsTestCase(unittest.TestCase): def test_app_exists(self): self.assertFalse(current_app is None) - - def test_app_is_testing(self): - self.assertTrue(current_app.config['TESTING'])