From eedd75ed69d0f66b6e3e6e1b327dfc1f21553e1f Mon Sep 17 00:00:00 2001 From: Patrick Jentsch Date: Wed, 21 Oct 2020 13:07:10 +0200 Subject: [PATCH 1/2] Next big config update. Check the .env.tpl and db.env.tpl --- .env.tpl | 122 +++++++------ .gitignore | 8 +- README.md | 7 +- daemon/Dockerfile | 4 +- daemon/app/__init__.py | 30 ++++ daemon/{tasks => app}/decorators.py | 0 daemon/{tasks => app}/models.py | 0 daemon/{notify => app/tasks}/__init__.py | 0 daemon/{ => app}/tasks/check_corpora.py | 7 +- daemon/{ => app}/tasks/check_jobs.py | 7 +- daemon/app/tasks/libnotify/__init__.py | 0 .../tasks/libnotify}/notification.py | 0 .../tasks/libnotify}/service.py | 0 .../libnotify/templates}/notification.html | 0 .../libnotify/templates}/notification.txt | 0 daemon/{ => app}/tasks/notify.py | 21 ++- daemon/boot.sh | 5 - daemon/config.py | 102 ++++++----- daemon/nopaqued.py | 26 +-- daemon/requirements.txt | 1 + daemon/tasks/__init__.py | 11 -- db.env.tpl | 5 + docker-compose.development.yml | 4 +- docker-compose.traefik.yml | 7 +- docker-compose.yml | 5 +- web/Dockerfile | 3 +- web/app/__init__.py | 13 +- web/boot.sh | 14 +- web/config.py | 167 +++++++++--------- web/nopaque.py | 26 ++- web/requirements.txt | 7 +- 31 files changed, 337 insertions(+), 265 deletions(-) create mode 100644 daemon/app/__init__.py rename daemon/{tasks => app}/decorators.py (100%) rename daemon/{tasks => app}/models.py (100%) rename daemon/{notify => app/tasks}/__init__.py (100%) rename daemon/{ => app}/tasks/check_corpora.py (97%) rename daemon/{ => app}/tasks/check_jobs.py (96%) create mode 100644 daemon/app/tasks/libnotify/__init__.py rename daemon/{notify => app/tasks/libnotify}/notification.py (100%) rename daemon/{notify => app/tasks/libnotify}/service.py (100%) rename daemon/{notify/templates/notification_messages => app/tasks/libnotify/templates}/notification.html (100%) rename daemon/{notify/templates/notification_messages => app/tasks/libnotify/templates}/notification.txt (100%) rename daemon/{ => app}/tasks/notify.py (89%) delete mode 100644 daemon/tasks/__init__.py create mode 100644 db.env.tpl diff --git a/.env.tpl b/.env.tpl index eea32232..ca0657d5 100644 --- a/.env.tpl +++ b/.env.tpl @@ -5,6 +5,10 @@ # NOTE: Use `.` as # HOST_DB_DIR= +# DEFAULT: ./mq +# NOTE: Use `.` as +# HOST_MQ_DIR= + # Example: 999 # HINT: Use this bash command `getent group docker | cut -d: -f3` HOST_DOCKER_GID= @@ -13,10 +17,6 @@ HOST_DOCKER_GID= # HINT: Use this bash command `id -g` HOST_GID= -# DEFAULT: ./mq -# NOTE: Use `.` as -# HOST_MQ_DIR= - # DEFAULT: ./nopaqued.log # NOTES: Use `.` as , # This file must be present on container startup @@ -33,22 +33,38 @@ HOST_UID= ################################################################################ -# Database (only PostgreSQL) # +# Cookies # ################################################################################ -NOPAQUE_DB_HOST= +# CHOOSE ONE: False, True +# DEFAULT: False +# HINT: Set to true if you redirect http to https +# NOPAQUE_REMEMBER_COOKIE_SECURE= -NOPAQUE_DB_NAME= - -NOPAQUE_DB_PASSWORD= - -# DEFAULT: 5432 -# NOPAQUE_DB_PORT= - -NOPAQUE_DB_USERNAME= +# CHOOSE ONE: False, True +# DEFAULT: False +# HINT: Set to true if you redirect http to https +# NOPAQUE_SESSION_COOKIE_SECURE= ################################################################################ -# SMTP # +# Database # +# DATABASE_URI blueprint: # +# - dialect[+driver]://username:password@host[:port]/database # +# - sqlite is not supported # +# - values in square brackets are optional # +################################################################################ +# DEFAULT: postgresql://nopaque:nopaque@db/nopaque +# NOPAQUE_DATABASE_URL= + +# DEFAULT: postgresql://nopaque:nopaque@db/nopaque_dev +# NOPAQUE_DEV_DATABASE_URL= + +# DEFAULT: postgresql://nopaque:nopaque@db/nopaque_test +# NOPAQUE_TEST_DATABASE_URL= + + +################################################################################ +# Email # ################################################################################ # EXAMPLE: nopaque Admin NOPAQUE_SMTP_DEFAULT_SENDER= @@ -61,12 +77,12 @@ NOPAQUE_SMTP_SERVER= # EXAMPLE: 587 NOPAQUE_SMTP_PORT= +# CHOOSE ONE: False, True # DEFAULT: False -# Choose one: False, True # NOPAQUE_SMTP_USE_SSL= +# CHOOSE ONE: False, True # DEFAULT: False -# Choose one: False, True # NOPAQUE_SMTP_USE_TLS= # EXAMPLE: nopaque@example.com @@ -76,78 +92,84 @@ NOPAQUE_SMTP_USERNAME= ################################################################################ # General # ################################################################################ -# Example: admin.nopaque@example.com +# EXAMPLE: admin.nopaque@example.com NOPAQUE_ADMIN_EMAIL_ADRESS= -# Example: contact.nopaque@example.com +# DEFAULT: development +# CHOOSE ONE: development, production, testing +# NOPAQUE_CONFIG= + +# EXAMPLE: contact.nopaque@example.com NOPAQUE_CONTACT_EMAIL_ADRESS= # DEFAULT: /mnt/nopaque # NOTE: This must be a network share and it must be available on all Docker Swarm nodes # NOPAQUE_DATA_DIR= -# DEFAULT: False -# Choose one: False, True -# NOPAQUE_DEBUG= - # DEFAULT: localhost # NOPAQUE_DOMAIN= -# DEFAULT: 0 -# NOPAQUE_NUM_PROXIES= - +# CHOOSE ONE: http, https # DEFAULT: http -# Choose one: http, https # NOPAQUE_PROTOCOL= -# DEFAULT: True -# Choose one: False, True -# NOPAQUE_REMEMBER_COOKIE_HTTPONLY= - -# DEFAULT: False -# Choose one: False, True -# HINT: Set to true if you redirect http to https -# NOPAQUE_REMEMBER_COOKIE_SECURE= - # DEFAULT: hard to guess string # HINT: Use this bash command `python -c "import uuid; print(uuid.uuid4().hex)"` # NOPAQUE_SECRET_KEY= -# DEFAULT: False -# Choose one: False, True -# HINT: Set to true if you redirect http to https -# NOPAQUE_SESSION_COOKIE_SECURE= - ################################################################################ # Logging # ################################################################################ -# DEFAULT: /nopaqued.log ~ /home/nopaqued/nopaqued.log +# DEFAULT: /home/nopaqued/nopaqued.log ~ /home/nopaqued/nopaqued.log # NOTE: Use `.` as # NOPAQUE_DAEMON_LOG_FILE= # DEFAULT: %Y-%m-%d %H:%M:%S # NOPAQUE_LOG_DATE_FORMAT= -# DEFAULT: /NOPAQUE.log ~ /home/NOPAQUE/NOPAQUE.log +# DEFAULT: /nopaque.log ~ /home/nopaque/nopaque.log # NOTE: Use `.` as # NOPAQUE_LOG_FILE= # DEFAULT: [%(asctime)s] %(levelname)s in %(pathname)s (function: %(funcName)s, line: %(lineno)d): %(message)s # NOPAQUE_LOG_FORMAT= -# DEFAULT: ERROR -# Choose one: CRITICAL, ERROR, WARNING, INFO, DEBUG +# DEFAULT: WARNING +# CHOOSE ONE: CRITICAL, ERROR, WARNING, INFO, DEBUG # NOPAQUE_LOG_LEVEL= ################################################################################ # Message queue # +# MESSAGE_QUEUE_URI blueprint: # +# - transport://[userid:password]@hostname[:port]/[virtual_host] # +# - values in square brackets are optional # ################################################################################ -NOPAQUE_MQ_HOST= +# DEFAULT: None +# HINT: A message queue is not required when using a single server process +# NOPAQUE_SOCKETIO_MESSAGE_QUEUE_URI= -# EXAMPLE: 6379 -NOPAQUE_MQ_PORT= -# Choose one of the supported types by Flask-SocketIO -NOPAQUE_MQ_TYPE= +################################################################################ +# Proxy fix # +################################################################################ +# DEFAULT: 0 +# Number of values to trust for X-Forwarded-For +# NOPAQUE_NUM_PROXIES_X_FOR= + +# DEFAULT: 0 +# Number of values to trust for X-Forwarded-Host +# NOPAQUE_NUM_PROXIES_X_HOST= + +# DEFAULT: 0 +# Number of values to trust for X-Forwarded-Port +# NOPAQUE_NUM_PROXIES_X_PORT= + +# DEFAULT: 0 +# Number of values to trust for X-Forwarded-Prefix +# NOPAQUE_NUM_PROXIES_X_PREFIX= + +# DEFAULT: 0 +# Number of values to trust for X-Forwarded-Proto +# NOPAQUE_NUM_PROXIES_X_PROTO= diff --git a/.gitignore b/.gitignore index 7e80fd54..de6e4247 100644 --- a/.gitignore +++ b/.gitignore @@ -1,6 +1,3 @@ -nopaque.log -nopaqued.log - *.py[cod] # C extensions @@ -12,11 +9,14 @@ db mq # Environment files -.env +*.env # Installer logs pip-log.txt +# Log files +*.log + # Packages *.egg *.egg-info diff --git a/README.md b/README.md index 35d9d06f..8c233226 100644 --- a/README.md +++ b/README.md @@ -27,7 +27,7 @@ username@hostname:~$ docker run \ -p 139:139 \ -p 445:445 \ dperson/samba \ - -p -s "nopaque;/srv/samba/nopaque;no;no;no;nopaque" -u "nopaque;nopaque" + -p -r -s "nopaque;/srv/samba/nopaque;no;no;no;nopaque" -u "nopaque;nopaque" # Mount the Samba share on all swarm nodes (managers and workers) username@hostname:~$ sudo mkdir /mnt/nopaque @@ -39,9 +39,12 @@ username@hostname:~$ sudo mount --types cifs --options gid=${USER},password=nopa ``` bash # Clone the nopaque repository username@hostname:~$ git clone https://gitlab.ub.uni-bielefeld.de/sfb1288inf/nopaque.git +username@hostname:~$ cp db.env.tpl db.env username@hostname:~$ cp .env.tpl .env -# Fill out the variables within this file. +# Fill out the variables within these files. +username@hostname:~$ db.env username@hostname:~$ .env +# Create docker-compose.override.yml file username@hostname:~$ touch docker-compose.override.yml # Tweak the docker-compose.override.yml to satisfy your needs. (You can find examples in docker-compose..yml) username@hostname:~$ docker-compose.override.yml diff --git a/daemon/Dockerfile b/daemon/Dockerfile index d03d7e0f..c5c7191c 100644 --- a/daemon/Dockerfile +++ b/daemon/Dockerfile @@ -14,7 +14,6 @@ RUN apt-get update \ && apt-get install --no-install-recommends --yes \ build-essential \ libpq-dev \ - wait-for-it \ && rm -r /var/lib/apt/lists/* @@ -27,8 +26,7 @@ WORKDIR /home/nopaqued COPY --chown=nopaqued:nopaqued [".", "."] RUN python -m venv venv \ - && venv/bin/pip install --requirement requirements.txt \ - && mkdir logs + && venv/bin/pip install --requirement requirements.txt ENTRYPOINT ["./boot.sh"] diff --git a/daemon/app/__init__.py b/daemon/app/__init__.py new file mode 100644 index 00000000..4c060fa3 --- /dev/null +++ b/daemon/app/__init__.py @@ -0,0 +1,30 @@ +from config import config +from sqlalchemy import create_engine +from sqlalchemy.orm import scoped_session, sessionmaker +from time import sleep +import docker +import os + + +configuration = config[os.environ.get('NOPAQUE_CONFIG', 'development')] +docker_client = docker.from_env() +engine = create_engine(configuration.SQLALCHEMY_DATABASE_URI) +Session = scoped_session(sessionmaker(bind=engine)) + + +def run(): + from .tasks.check_corpora import check_corpora + check_corpora_thread = check_corpora() + from .tasks.check_jobs import check_jobs + check_jobs_thread = check_jobs() + from .tasks.notify import notify + notify_thread = notify() + + while True: + if not check_corpora_thread.is_alive(): + check_corpora_thread = check_corpora() + if not check_jobs_thread.is_alive(): + check_jobs_thread = check_jobs() + if not notify_thread.is_alive(): + notify_thread = notify() + sleep(3) diff --git a/daemon/tasks/decorators.py b/daemon/app/decorators.py similarity index 100% rename from daemon/tasks/decorators.py rename to daemon/app/decorators.py diff --git a/daemon/tasks/models.py b/daemon/app/models.py similarity index 100% rename from daemon/tasks/models.py rename to daemon/app/models.py diff --git a/daemon/notify/__init__.py b/daemon/app/tasks/__init__.py similarity index 100% rename from daemon/notify/__init__.py rename to daemon/app/tasks/__init__.py diff --git a/daemon/tasks/check_corpora.py b/daemon/app/tasks/check_corpora.py similarity index 97% rename from daemon/tasks/check_corpora.py rename to daemon/app/tasks/check_corpora.py index c91e57d6..6ecffea5 100644 --- a/daemon/tasks/check_corpora.py +++ b/daemon/app/tasks/check_corpora.py @@ -1,6 +1,7 @@ -from . import config, docker_client, Session -from .decorators import background -from .models import Corpus +from .. import configuration as config +from .. import docker_client, Session +from ..decorators import background +from ..models import Corpus import docker import logging import os diff --git a/daemon/tasks/check_jobs.py b/daemon/app/tasks/check_jobs.py similarity index 96% rename from daemon/tasks/check_jobs.py rename to daemon/app/tasks/check_jobs.py index d8812ef3..f5530e1e 100644 --- a/daemon/tasks/check_jobs.py +++ b/daemon/app/tasks/check_jobs.py @@ -1,7 +1,8 @@ from datetime import datetime -from . import config, docker_client, Session -from .decorators import background -from .models import Job, JobResult, NotificationData, NotificationEmailData +from .. import configuration as config +from .. import docker_client, Session +from ..decorators import background +from ..models import Job, JobResult, NotificationData, NotificationEmailData import docker import logging import json diff --git a/daemon/app/tasks/libnotify/__init__.py b/daemon/app/tasks/libnotify/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/daemon/notify/notification.py b/daemon/app/tasks/libnotify/notification.py similarity index 100% rename from daemon/notify/notification.py rename to daemon/app/tasks/libnotify/notification.py diff --git a/daemon/notify/service.py b/daemon/app/tasks/libnotify/service.py similarity index 100% rename from daemon/notify/service.py rename to daemon/app/tasks/libnotify/service.py diff --git a/daemon/notify/templates/notification_messages/notification.html b/daemon/app/tasks/libnotify/templates/notification.html similarity index 100% rename from daemon/notify/templates/notification_messages/notification.html rename to daemon/app/tasks/libnotify/templates/notification.html diff --git a/daemon/notify/templates/notification_messages/notification.txt b/daemon/app/tasks/libnotify/templates/notification.txt similarity index 100% rename from daemon/notify/templates/notification_messages/notification.txt rename to daemon/app/tasks/libnotify/templates/notification.txt diff --git a/daemon/tasks/notify.py b/daemon/app/tasks/notify.py similarity index 89% rename from daemon/tasks/notify.py rename to daemon/app/tasks/notify.py index e2976a69..5d3d23f3 100644 --- a/daemon/tasks/notify.py +++ b/daemon/app/tasks/notify.py @@ -1,13 +1,18 @@ -from notify.notification import Notification -from notify.service import NotificationService from sqlalchemy import asc -from . import config, Session -from .decorators import background -from .models import NotificationEmailData +from .libnotify.notification import Notification +from .libnotify.service import NotificationService +from .. import configuration as config +from .. import Session +from ..decorators import background +from ..models import NotificationEmailData import logging +import os import smtplib +ROOT_DIR = os.path.abspath(os.path.dirname(__file__)) + + @background def notify(): session = Session() @@ -72,8 +77,10 @@ def __create_mail_notifications(notification_service, session): 'status': data.notify_status, 'time': data.creation_date, 'url': url} - txt_tmplt = 'notify/templates/notification_messages/notification.txt' - html_tmplt = 'notify/templates/notification_messages/notification.html' + txt_tmplt = os.path.join(ROOT_DIR, + 'libnotify/templates/notification.txt') + html_tmplt = os.path.join(ROOT_DIR, + 'libnotify/templates/notification.html') notification.set_notification_content(subject_template, subject_template_values_dict, txt_tmplt, diff --git a/daemon/boot.sh b/daemon/boot.sh index d3e27ee8..53127dd0 100755 --- a/daemon/boot.sh +++ b/daemon/boot.sh @@ -1,8 +1,3 @@ #!/bin/bash -echo "Waiting for db..." -wait-for-it "${NOPAQUE_DB_HOST}:${NOPAQUE_DB_PORT:-5432}" --strict --timeout=0 -echo "Waiting for nopaque..." -wait-for-it nopaque:5000 --strict --timeout=0 - source venv/bin/activate python nopaqued.py diff --git a/daemon/config.py b/daemon/config.py index 590aa4b6..8729b563 100644 --- a/daemon/config.py +++ b/daemon/config.py @@ -2,60 +2,70 @@ import logging import os -root_dir = os.path.abspath(os.path.dirname(__file__)) - - -DEFAULT_DATA_DIR = os.path.join('/mnt/nopaque') -DEFAULT_DB_PORT = '5432' -DEFAULT_DOMAIN = 'localhost' -DEFAULT_LOG_DATE_FORMAT = '%Y-%m-%d %H:%M:%S' -DEFAULT_LOG_FILE = os.path.join(root_dir, 'nopaqued.log') -DEFAULT_LOG_FORMAT = ('[%(asctime)s] %(levelname)s in %(pathname)s ' - '(function: %(funcName)s, line: %(lineno)d): ' - '%(message)s') -DEFAULT_LOG_LEVEL = 'ERROR' -DEFAULT_MAIL_USE_SSL = 'False' -DEFAULT_MAIL_USE_TLS = 'False' -DEFAULT_PROTOCOL = 'http' +ROOT_DIR = os.path.abspath(os.path.dirname(__file__)) class Config: - ''' ### Database ### ''' - DB_HOST = os.environ.get('NOPAQUE_DB_HOST') - DB_NAME = os.environ.get('NOPAQUE_DB_NAME') - DB_PASSWORD = os.environ.get('NOPAQUE_DB_PASSWORD') - DB_PORT = os.environ.get('NOPAQUE_DB_PORT', DEFAULT_DB_PORT) - DB_USERNAME = os.environ.get('NOPAQUE_DB_USERNAME') - SQLALCHEMY_DATABASE_URI = 'postgresql://{}:{}@{}:{}/{}'.format( - DB_USERNAME, DB_PASSWORD, DB_HOST, DB_PORT, DB_NAME) - - ''' ### SMTP ### ''' + ''' # Email # ''' SMTP_DEFAULT_SENDER = os.environ.get('NOPAQUE_SMTP_DEFAULT_SENDER') SMTP_PASSWORD = os.environ.get('NOPAQUE_SMTP_PASSWORD') - SMTP_PORT = os.environ.get('NOPAQUE_SMTP_PORT') + SMTP_PORT = int(os.environ.get('NOPAQUE_SMTP_PORT')) SMTP_SERVER = os.environ.get('NOPAQUE_SMTP_SERVER') SMTP_USERNAME = os.environ.get('NOPAQUE_SMTP_USERNAME') - SMTP_USE_SSL = os.environ.get('NOPAQUE_SMTP_USE_SSL', - DEFAULT_MAIL_USE_SSL).lower() == 'true' - SMTP_USE_TLS = os.environ.get('NOPAQUE_SMTP_USE_TLS', - DEFAULT_MAIL_USE_TLS).lower() == 'true' + SMTP_USE_SSL = os.environ.get( + 'NOPAQUE_SMTP_USE_SSL', 'false').lower() == 'true' + SMTP_USE_TLS = os.environ.get( + 'NOPAQUE_SMTP_USE_TLS', 'false').lower() == 'true' - ''' ### General ### ''' - DATA_DIR = os.environ.get('NOPAQUE_DATA_DIR', DEFAULT_DATA_DIR) - DOMAIN = os.environ.get('NOPAQUE_DOMAIN', DEFAULT_DOMAIN) - PROTOCOL = os.environ.get('NOPAQUE_PROTOCOL', DEFAULT_PROTOCOL) + ''' # General # ''' + DATA_DIR = os.environ.get('NOPAQUE_DATA_DIR', '/mnt/nopaque') + DOMAIN = os.environ.get('NOPAQUE_DOMAIN', 'localhost') + PROTOCOL = os.environ.get('NOPAQUE_PROTOCOL', 'http') + SECRET_KEY = os.environ.get('NOPAQUE_SECRET_KEY', 'hard to guess string') - ''' ### Logging ### ''' + ''' # Logging # ''' LOG_DATE_FORMAT = os.environ.get('NOPAQUE_LOG_DATE_FORMAT', - DEFAULT_LOG_DATE_FORMAT) - LOG_FILE = os.environ.get('NOPAQUE_DAEMON_LOG_FILE', DEFAULT_LOG_FILE) - LOG_FORMAT = os.environ.get('NOPAQUE_LOG_FORMAT', DEFAULT_LOG_FORMAT) - LOG_LEVEL = os.environ.get('NOPAQUE_LOG_LEVEL', DEFAULT_LOG_LEVEL) + '%Y-%m-%d %H:%M:%S') + LOG_FILE = os.environ.get('NOPAQUED_LOG_FILE', + os.path.join(ROOT_DIR, 'nopaqued.log')) + LOG_FORMAT = os.environ.get( + 'NOPAQUE_LOG_FORMAT', + '[%(asctime)s] %(levelname)s in ' + '%(pathname)s (function: %(funcName)s, line: %(lineno)d): %(message)s' + ) + LOG_LEVEL = os.environ.get('NOPAQUE_LOG_LEVEL', 'WARNING') - def init_app(self): - # Configure logging according to the corresponding (LOG_*) config - # entries - logging.basicConfig(datefmt=self.LOG_DATE_FORMAT, - filename=self.LOG_FILE, - format=self.LOG_FORMAT, - level=self.LOG_LEVEL) + @classmethod + def init(cls): + # Set up logging according to the corresponding (LOG_*) variables + logging.basicConfig(datefmt=cls.LOG_DATE_FORMAT, + filename=cls.LOG_FILE, + format=cls.LOG_FORMAT, + level=cls.LOG_LEVEL) + + +class DevelopmentConfig(Config): + ''' # Database # ''' + SQLALCHEMY_DATABASE_URI = os.environ.get( + 'NOPAQUE_DEV_DATABASE_URL', + 'sqlite:///' + os.path.join(ROOT_DIR, 'data-dev.sqlite') + ) + + +class ProductionConfig(Config): + ''' # Database # ''' + SQLALCHEMY_DATABASE_URI = os.environ.get( + 'NOPAQUE_DATABASE_URL', + 'sqlite:///' + os.path.join(ROOT_DIR, 'data.sqlite') + ) + + +class TestingConfig(Config): + ''' # Database # ''' + SQLALCHEMY_DATABASE_URI = os.environ.get( + 'NOPAQUE_TEST_DATABASE_URL', 'sqlite://') + + +config = {'development': DevelopmentConfig, + 'production': ProductionConfig, + 'testing': TestingConfig} diff --git a/daemon/nopaqued.py b/daemon/nopaqued.py index 8832c091..7fbb79dc 100644 --- a/daemon/nopaqued.py +++ b/daemon/nopaqued.py @@ -1,23 +1,13 @@ -from tasks.check_corpora import check_corpora -from tasks.check_jobs import check_jobs -from tasks.notify import notify -from time import sleep +from dotenv import load_dotenv +from app import run +import os -def nopaqued(): - check_corpora_thread = check_corpora() - check_jobs_thread = check_jobs() - notify_thread = notify() - - while True: - if not check_corpora_thread.is_alive(): - check_corpora_thread = check_corpora() - if not check_jobs_thread.is_alive(): - check_jobs_thread = check_jobs() - if not notify_thread.is_alive(): - notify_thread = notify() - sleep(3) +# Load environment variables +DOTENV_FILE = os.path.join(os.path.dirname(__file__), '.env') +if os.path.exists(DOTENV_FILE): + load_dotenv(DOTENV_FILE) if __name__ == '__main__': - nopaqued() + run() diff --git a/daemon/requirements.txt b/daemon/requirements.txt index bafd77ed..de767e32 100644 --- a/daemon/requirements.txt +++ b/daemon/requirements.txt @@ -1,3 +1,4 @@ docker psycopg2 +python-dotenv SQLAlchemy diff --git a/daemon/tasks/__init__.py b/daemon/tasks/__init__.py deleted file mode 100644 index 89ed03e7..00000000 --- a/daemon/tasks/__init__.py +++ /dev/null @@ -1,11 +0,0 @@ -from config import Config -from sqlalchemy import create_engine -from sqlalchemy.orm import scoped_session, sessionmaker -import docker - - -config = Config() -config.init_app() -docker_client = docker.from_env() -engine = create_engine(config.SQLALCHEMY_DATABASE_URI) -Session = scoped_session(sessionmaker(bind=engine)) diff --git a/db.env.tpl b/db.env.tpl new file mode 100644 index 00000000..88ee89de --- /dev/null +++ b/db.env.tpl @@ -0,0 +1,5 @@ +POSTGRES_DB_NAME= + +POSTGRES_USER= + +POSTGRES_PASSWORD= diff --git a/docker-compose.development.yml b/docker-compose.development.yml index d0542d88..0a3248db 100644 --- a/docker-compose.development.yml +++ b/docker-compose.development.yml @@ -16,10 +16,8 @@ services: nopaqued: volumes: # Mount code as volumes + - "./daemon/app:/home/nopaqued/app" - "./daemon/boot.sh:/home/nopaqued/boot.sh" - "./daemon/config.py:/home/nopaqued/config.py" - - "./daemon/logger:/home/nopaqued/logger" - "./daemon/nopaqued.py:/home/nopaqued/nopaqued.py" - - "./daemon/notify:/home/nopaqued/notify" - "./daemon/requirements.txt:/home/nopaqued/requirements.txt" - - "./daemon/tasks:/home/nopaqued/tasks" diff --git a/docker-compose.traefik.yml b/docker-compose.traefik.yml index 195a5702..5cefda8f 100644 --- a/docker-compose.traefik.yml +++ b/docker-compose.traefik.yml @@ -1,5 +1,6 @@ ################################################################################ -# Don't forget to set the NOPAQUE_NUM_PROXIES variable in your .env # +# Don't forget to set the proxy variables in your nopaque.env # +# Traefik sets the X_FOR, X_HOST, X_PORT and X_PROTO headers by default # ################################################################################ version: "3.5" @@ -17,13 +18,13 @@ services: - "traefik.http.middlewares.nopaque-header.headers.customrequestheaders.X-Forwarded-Proto=http" - "traefik.http.routers.nopaque.entrypoints=web" - "traefik.http.routers.nopaque.middlewares=nopaque-header, redirect-to-https@file" - - "traefik.http.routers.nopaque.rule=Host(`${NOPAQUE_DOMAIN:-localhost}`)" + - "traefik.http.routers.nopaque.rule=Host(``)" ### ### ### ### - "traefik.http.middlewares.nopaque-secure-header.headers.customrequestheaders.X-Forwarded-Proto=https" - "traefik.http.routers.nopaque-secure.entrypoints=web-secure" - "traefik.http.routers.nopaque-secure.middlewares=hsts-header@file, nopaque-secure-header" - - "traefik.http.routers.nopaque-secure.rule=Host(`${NOPAQUE_DOMAIN:-localhost}`)" + - "traefik.http.routers.nopaque-secure.rule=Host(``)" - "traefik.http.routers.nopaque-secure.tls.certresolver=" - "traefik.http.routers.nopaque-secure.tls.options=intermediate@file" ### ### diff --git a/docker-compose.yml b/docker-compose.yml index 9e5bad1a..b6f7a9f0 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -34,10 +34,7 @@ services: - "${NOPAQUE_DATA_DIR:-/mnt/nopaque}:${NOPAQUE_DATA_DIR:-/mnt/nopaque}" - "${HOST_NOPAQUE_DAEMON_LOG_FILE-./nopaqued.log}:${NOPAQUE_DAEMON_LOG_FILE:-/home/nopaqued/nopaqued.log}" db: - environment: - - POSTGRES_DB_NAME=${NOPAQUE_DB_NAME} - - POSTGRES_USER=${NOPAQUE_DB_USERNAME} - - POSTGRES_PASSWORD=${NOPAQUE_DB_PASSWORD} + env_file: db.env image: postgres:11 restart: unless-stopped volumes: diff --git a/web/Dockerfile b/web/Dockerfile index c467b259..216964cc 100644 --- a/web/Dockerfile +++ b/web/Dockerfile @@ -29,8 +29,7 @@ WORKDIR /home/nopaque COPY --chown=nopaque:nopaque [".", "."] RUN python -m venv venv \ - && venv/bin/pip install --requirement requirements.txt \ - && mkdir logs + && venv/bin/pip install --requirement requirements.txt ENTRYPOINT ["./boot.sh"] diff --git a/web/app/__init__.py b/web/app/__init__.py index 0588b334..7a5d8b68 100644 --- a/web/app/__init__.py +++ b/web/app/__init__.py @@ -1,4 +1,4 @@ -from config import Config +from config import config from flask import Flask from flask_login import LoginManager from flask_mail import Mail @@ -7,7 +7,6 @@ from flask_socketio import SocketIO from flask_sqlalchemy import SQLAlchemy -config = Config() db = SQLAlchemy() login_manager = LoginManager() login_manager.login_view = 'auth.login' @@ -17,19 +16,19 @@ paranoid.redirect_view = '/' socketio = SocketIO() -def create_app(): +def create_app(config_name): app = Flask(__name__) - app.config.from_object(config) + app.config.from_object(config[config_name]) - config.init_app(app) + config[config_name].init_app(app) db.init_app(app) login_manager.init_app(app) mail.init_app(app) paranoid.init_app(app) - socketio.init_app(app, message_queue=config.SOCKETIO_MESSAGE_QUEUE_URI) + socketio.init_app( + app, message_queue=config[config_name].SOCKETIO_MESSAGE_QUEUE_URI) from . import events - from .admin import admin as admin_blueprint app.register_blueprint(admin_blueprint, url_prefix='/admin') from .auth import auth as auth_blueprint diff --git a/web/boot.sh b/web/boot.sh index 4a79f81a..a390d4c4 100755 --- a/web/boot.sh +++ b/web/boot.sh @@ -1,12 +1,14 @@ #!/bin/bash -echo "Waiting for db..." -wait-for-it "${NOPAQUE_DB_HOST}:${NOPAQUE_DB_PORT:-5432}" --strict --timeout=0 -echo "Waiting for mq..." -wait-for-it "${NOPAQUE_MQ_HOST}:${NOPAQUE_MQ_PORT}" --strict --timeout=0 - source venv/bin/activate if [[ "$#" -eq 0 ]]; then - flask deploy + while true; do + flask deploy + if [[ "$?" == "0" ]]; then + break + fi + echo Deploy command failed, retrying in 5 secs... + sleep 5 + done python nopaque.py elif [[ "$1" == "flask" ]]; then exec ${@:1} diff --git a/web/config.py b/web/config.py index 066592a0..0f78e6ad 100644 --- a/web/config.py +++ b/web/config.py @@ -3,100 +3,107 @@ import logging import os -root_dir = os.path.abspath(os.path.dirname(__file__)) - - -DEFAULT_DATA_DIR = os.path.join('/mnt/nopaque') -DEFAULT_DB_PORT = '5432' -DEFAULT_DEBUG = 'False' -DEFAULT_LOG_DATE_FORMAT = '%Y-%m-%d %H:%M:%S' -DEFAULT_LOG_FILE = os.path.join(root_dir, 'nopaque.log') -DEFAULT_LOG_FORMAT = ('[%(asctime)s] %(levelname)s in %(pathname)s ' - '(function: %(funcName)s, line: %(lineno)d): ' - '%(message)s') -DEFAULT_LOG_LEVEL = 'ERROR' -DEFAULT_SMTP_USE_SSL = 'False' -DEFAULT_SMTP_USE_TLS = 'False' -DEFAULT_NUM_PROXIES = '0' -DEFAULT_PROTOCOL = 'http' -DEFAULT_REMEMBER_COOKIE_HTTPONLY = 'True' -DEFAULT_REMEMBER_COOKIE_SECURE = 'False' -DEFAULT_SECRET_KEY = 'hard to guess string' -DEFAULT_SESSION_COOKIE_SECURE = 'False' +ROOT_DIR = os.path.abspath(os.path.dirname(__file__)) class Config: - ''' ### Database ### ''' - DB_HOST = os.environ.get('NOPAQUE_DB_HOST') - DB_NAME = os.environ.get('NOPAQUE_DB_NAME') - DB_PASSWORD = os.environ.get('NOPAQUE_DB_PASSWORD') - DB_PORT = os.environ.get('NOPAQUE_DB_PORT', DEFAULT_DB_PORT) - DB_USERNAME = os.environ.get('NOPAQUE_DB_USERNAME') - SQLALCHEMY_DATABASE_URI = 'postgresql://{}:{}@{}:{}/{}'.format( - DB_USERNAME, DB_PASSWORD, DB_HOST, DB_PORT, DB_NAME) + ''' # Cookies # ''' + REMEMBER_COOKIE_HTTPONLY = True + REMEMBER_COOKIE_SECURE = os.environ.get( + 'NOPAQUE_REMEMBER_COOKIE_SECURE', 'false').lower() == 'true' + SESSION_COOKIE_SECURE = os.environ.get( + 'NOPAQUE_SESSION_COOKIE_SECURE', 'false').lower() == 'true' + + ''' # Database # ''' SQLALCHEMY_RECORD_QUERIES = True SQLALCHEMY_TRACK_MODIFICATIONS = False - ''' ### Email ### ''' + ''' # Email # ''' MAIL_DEFAULT_SENDER = os.environ.get('NOPAQUE_SMTP_DEFAULT_SENDER') MAIL_PASSWORD = os.environ.get('NOPAQUE_SMTP_PASSWORD') - MAIL_PORT = os.environ.get('NOPAQUE_SMTP_PORT') + MAIL_PORT = int(os.environ.get('NOPAQUE_SMTP_PORT')) MAIL_SERVER = os.environ.get('NOPAQUE_SMTP_SERVER') MAIL_USERNAME = os.environ.get('NOPAQUE_SMTP_USERNAME') - MAIL_USE_SSL = os.environ.get('NOPAQUE_SMTP_USE_SSL', - DEFAULT_SMTP_USE_SSL).lower() == 'true' - MAIL_USE_TLS = os.environ.get('NOPAQUE_SMTP_USE_TLS', - DEFAULT_SMTP_USE_TLS).lower() == 'true' + MAIL_USE_SSL = os.environ.get( + 'NOPAQUE_SMTP_USE_SSL', 'false').lower() == 'true' + MAIL_USE_TLS = os.environ.get( + 'NOPAQUE_SMTP_USE_TLS', 'false').lower() == 'true' - ''' ### General ### ''' + ''' # General # ''' ADMIN_EMAIL_ADRESS = os.environ.get('NOPAQUE_ADMIN_EMAIL_ADRESS') CONTACT_EMAIL_ADRESS = os.environ.get('NOPAQUE_CONTACT_EMAIL_ADRESS') - DATA_DIR = os.environ.get('NOPAQUE_DATA_DIR', DEFAULT_DATA_DIR) - DEBUG = os.environ.get('NOPAQUE_DEBUG', DEFAULT_DEBUG).lower() == 'true' - NUM_PROXIES = int(os.environ.get('NOPAQUE_NUM_PROXIES', - DEFAULT_NUM_PROXIES)) - PROTOCOL = os.environ.get('NOPAQUE_PROTOCOL', DEFAULT_PROTOCOL) - REMEMBER_COOKIE_HTTPONLY = os.environ.get( - 'NOPAQUE_REMEMBER_COOKIE_HTTPONLY', - DEFAULT_REMEMBER_COOKIE_HTTPONLY - ).lower() == 'true' - REMEMBER_COOKIE_SECURE = os.environ.get( - 'NOPAQUE_REMEMBER_COOKIE_SECURE', - DEFAULT_REMEMBER_COOKIE_SECURE - ).lower() == 'true' - SECRET_KEY = os.environ.get('RECIPY_SECRET_KEY', DEFAULT_SECRET_KEY) - SESSION_COOKIE_SECURE = os.environ.get( - 'NOPAQUE_SESSION_COOKIE_SECURE', - DEFAULT_SESSION_COOKIE_SECURE - ).lower() == 'true' + DATA_DIR = os.environ.get('NOPAQUE_DATA_DIR', '/mnt/nopaque') + SECRET_KEY = os.environ.get('NOPAQUE_SECRET_KEY', 'hard to guess string') - ''' ### Logging ### ''' + ''' # Logging # ''' LOG_DATE_FORMAT = os.environ.get('NOPAQUE_LOG_DATE_FORMAT', - DEFAULT_LOG_DATE_FORMAT) - LOG_FILE = os.environ.get('NOPAQUE_LOG_FILE', DEFAULT_LOG_FILE) - LOG_FORMAT = os.environ.get('NOPAQUE_LOG_FORMAT', DEFAULT_LOG_FORMAT) - LOG_LEVEL = os.environ.get('NOPAQUE_LOG_LEVEL', DEFAULT_LOG_LEVEL) + '%Y-%m-%d %H:%M:%S') + LOG_FILE = os.environ.get('NOPAQUE_LOG_FILE', + os.path.join(ROOT_DIR, 'nopaque.log')) + LOG_FORMAT = os.environ.get( + 'NOPAQUE_LOG_FORMAT', + '[%(asctime)s] %(levelname)s in ' + '%(pathname)s (function: %(funcName)s, line: %(lineno)d): %(message)s' + ) + LOG_LEVEL = os.environ.get('NOPAQUE_LOG_LEVEL', 'WARNING') - ''' ### Message queue ### ''' - MQ_HOST = os.environ.get('NOPAQUE_MQ_HOST') - MQ_PORT = os.environ.get('NOPAQUE_MQ_PORT') - MQ_TYPE = os.environ.get('NOPAQUE_MQ_TYPE') - SOCKETIO_MESSAGE_QUEUE_URI = \ - '{}://{}:{}/'.format(MQ_TYPE, MQ_HOST, MQ_PORT) + ''' # Message queue # ''' + SOCKETIO_MESSAGE_QUEUE_URI = os.environ.get( + 'NOPAQUE_SOCKETIO_MESSAGE_QUEUE_URI') - def init_app(self, app): - # Configure logging according to the corresponding (LOG_*) config - # entries - logging.basicConfig(datefmt=self.LOG_DATE_FORMAT, - filename=self.LOG_FILE, - format=self.LOG_FORMAT, - level=self.LOG_LEVEL) - # Apply the ProxyFix middleware if nopaque is running behind reverse - # proxies. (NUM_PROXIES indicates the number of reverse proxies running - # in front of nopaque) - if self.NUM_PROXIES > 0: - app.wsgi_app = ProxyFix(app.wsgi_app, - x_for=self.NUM_PROXIES, - x_host=self.NUM_PROXIES, - x_port=self.NUM_PROXIES, - x_proto=self.NUM_PROXIES) + ''' # Proxy fix # ''' + PROXY_FIX_X_FOR = int(os.environ.get('NOPAQUE_PROXY_FIX_X_FOR', '0')) + PROXY_FIX_X_HOST = int(os.environ.get('NOPAQUE_PROXY_FIX_X_HOST', '0')) + PROXY_FIX_X_PORT = int(os.environ.get('NOPAQUE_PROXY_FIX_X_PORT', '0')) + PROXY_FIX_X_PREFIX = int(os.environ.get('NOPAQUE_PROXY_FIX_X_PREFIX', '0')) + PROXY_FIX_X_PROTO = int(os.environ.get('NOPAQUE_PROXY_FIX_X_PROTO', '0')) + + @classmethod + def init_app(cls, app): + # Set up logging according to the corresponding (LOG_*) variables + logging.basicConfig(datefmt=cls.LOG_DATE_FORMAT, + filename=cls.LOG_FILE, + format=cls.LOG_FORMAT, + level=cls.LOG_LEVEL) + # Set up and apply the ProxyFix middleware according to the + # corresponding (PROXY_FIX_*) variables + app.wsgi_app = ProxyFix(app.wsgi_app, + x_for=cls.PROXY_FIX_X_FOR, + x_host=cls.PROXY_FIX_X_HOST, + x_port=cls.PROXY_FIX_X_PORT, + x_prefix=cls.PROXY_FIX_X_PREFIX, + x_proto=cls.PROXY_FIX_X_PROTO) + + +class DevelopmentConfig(Config): + ''' # Database # ''' + SQLALCHEMY_DATABASE_URI = os.environ.get( + 'NOPAQUE_DEV_DATABASE_URL', + 'postgresql://nopaque:nopaque@db/nopaque_dev' + ) + + ''' # General # ''' + DEBUG = True + + +class ProductionConfig(Config): + ''' # Database # ''' + SQLALCHEMY_DATABASE_URI = os.environ.get( + 'NOPAQUE_DATABASE_URL', 'postgresql://nopaque:nopaque@db/nopaque') + + +class TestingConfig(Config): + ''' # Database # ''' + SQLALCHEMY_DATABASE_URI = os.environ.get( + 'NOPAQUE_TEST_DATABASE_URL', + 'postgresql://nopaque:nopaque@db/nopaque_test' + ) + + ''' # General # ''' + TESTING = True + WTF_CSRF_ENABLED = False + + +config = {'development': DevelopmentConfig, + 'production': ProductionConfig, + 'testing': TestingConfig} diff --git a/web/nopaque.py b/web/nopaque.py index b56fba8b..8884e833 100644 --- a/web/nopaque.py +++ b/web/nopaque.py @@ -1,12 +1,28 @@ +# First things first: apply monkey patch, so that no code gets executed without +# patched libraries! import eventlet -eventlet.monkey_patch() # noqa -from app import create_app, db, socketio + + +eventlet.monkey_patch() + + +from dotenv import load_dotenv # noqa +import os # noqa + +# Load environment variables +DOTENV_FILE = os.path.join(os.path.dirname(__file__), '.env') +if os.path.exists(DOTENV_FILE): + load_dotenv(DOTENV_FILE) + + +from app import create_app, db, socketio # noqa from app.models import (Corpus, CorpusFile, Job, JobInput, JobResult, NotificationData, NotificationEmailData, QueryResult, - Role, User) -from flask_migrate import Migrate, upgrade + Role, User) # noqa +from flask_migrate import Migrate, upgrade # noqa -app = create_app() + +app = create_app(os.environ.get('NOPAQUE_CONFIG', 'development')) migrate = Migrate(app, db, compare_type=True) diff --git a/web/requirements.txt b/web/requirements.txt index a5f15a10..0d7f6e68 100644 --- a/web/requirements.txt +++ b/web/requirements.txt @@ -1,6 +1,5 @@ cqi dnspython==1.16.0 -email_validator eventlet Flask Flask-Login @@ -11,6 +10,8 @@ Flask-SocketIO Flask-SQLAlchemy Flask-WTF jsonpatch -psycopg2 -redis jsonschema +psycopg2 +python-dotenv +redis +wtforms[email] From 0ef7355ca11807fb02c4001164af5a30b83c599a Mon Sep 17 00:00:00 2001 From: Patrick Jentsch Date: Wed, 21 Oct 2020 13:20:46 +0200 Subject: [PATCH 2/2] Add init call to config initialization --- daemon/app/__init__.py | 1 + 1 file changed, 1 insertion(+) diff --git a/daemon/app/__init__.py b/daemon/app/__init__.py index 4c060fa3..eaccafd2 100644 --- a/daemon/app/__init__.py +++ b/daemon/app/__init__.py @@ -7,6 +7,7 @@ import os configuration = config[os.environ.get('NOPAQUE_CONFIG', 'development')] +configuration.init() docker_client = docker.from_env() engine = create_engine(configuration.SQLALCHEMY_DATABASE_URI) Session = scoped_session(sessionmaker(bind=engine))