Merge branch 'development' of gitlab.ub.uni-bielefeld.de:sfb1288inf/nopaque into development

.env.tpl

@@ -5,6 +5,10 @@
 # NOTE: Use `.` as <project-root-dir>
 # HOST_DB_DIR=
 
+# DEFAULT: ./mq
+# NOTE: Use `.` as <project-root-dir>
+# HOST_MQ_DIR=
+
 # Example: 999
 # HINT: Use this bash command `getent group docker | cut -d: -f3`
 HOST_DOCKER_GID=
@@ -13,10 +17,6 @@ HOST_DOCKER_GID=
 # HINT: Use this bash command `id -g`
 HOST_GID=
 
-# DEFAULT: ./mq
-# NOTE: Use `.` as <project-root-dir>
-# HOST_MQ_DIR=
-
 # DEFAULT: ./nopaqued.log
 # NOTES: Use `.` as <project-root-dir>,
 #        This file must be present on container startup
@@ -33,22 +33,38 @@ HOST_UID=
 
 
 ################################################################################
-# Database (only PostgreSQL)                                                   #
+# Cookies                                                                      #
 ################################################################################
-NOPAQUE_DB_HOST=
+# CHOOSE ONE: False, True
+# DEFAULT: False
+# HINT: Set to true if you redirect http to https
+# NOPAQUE_REMEMBER_COOKIE_SECURE=
 
-NOPAQUE_DB_NAME=
+# CHOOSE ONE: False, True
-
+# DEFAULT: False
-NOPAQUE_DB_PASSWORD=
+# HINT: Set to true if you redirect http to https
-
+# NOPAQUE_SESSION_COOKIE_SECURE=
-# DEFAULT: 5432
-# NOPAQUE_DB_PORT=
-
-NOPAQUE_DB_USERNAME=
 
 
 ################################################################################
-# SMTP                                                                         #
+# Database                                                                     #
+# DATABASE_URI blueprint:                                                      #
+#   - dialect[+driver]://username:password@host[:port]/database                #
+#     - sqlite is not supported                                                #
+#     - values in square brackets are optional                                 #
+################################################################################
+# DEFAULT: postgresql://nopaque:nopaque@db/nopaque
+# NOPAQUE_DATABASE_URL=
+
+# DEFAULT: postgresql://nopaque:nopaque@db/nopaque_dev
+# NOPAQUE_DEV_DATABASE_URL=
+
+# DEFAULT: postgresql://nopaque:nopaque@db/nopaque_test
+# NOPAQUE_TEST_DATABASE_URL=
+
+
+################################################################################
+# Email                                                                        #
 ################################################################################
 # EXAMPLE: nopaque Admin <nopaque@example.com>
 NOPAQUE_SMTP_DEFAULT_SENDER=
@@ -61,12 +77,12 @@ NOPAQUE_SMTP_SERVER=
 # EXAMPLE: 587
 NOPAQUE_SMTP_PORT=
 
+# CHOOSE ONE: False, True
 # DEFAULT: False
-# Choose one: False, True
 # NOPAQUE_SMTP_USE_SSL=
 
+# CHOOSE ONE: False, True
 # DEFAULT: False
-# Choose one: False, True
 # NOPAQUE_SMTP_USE_TLS=
 
 # EXAMPLE: nopaque@example.com
@@ -76,78 +92,84 @@ NOPAQUE_SMTP_USERNAME=
 ################################################################################
 # General                                                                      #
 ################################################################################
-# Example: admin.nopaque@example.com
+# EXAMPLE: admin.nopaque@example.com
 NOPAQUE_ADMIN_EMAIL_ADRESS=
 
-# Example: contact.nopaque@example.com
+# DEFAULT: development
+# CHOOSE ONE: development, production, testing
+# NOPAQUE_CONFIG=
+
+# EXAMPLE: contact.nopaque@example.com
 NOPAQUE_CONTACT_EMAIL_ADRESS=
 
 # DEFAULT: /mnt/nopaque
 # NOTE: This must be a network share and it must be available on all Docker Swarm nodes
 # NOPAQUE_DATA_DIR=
 
-# DEFAULT: False
-# Choose one: False, True
-# NOPAQUE_DEBUG=
-
 # DEFAULT: localhost
 # NOPAQUE_DOMAIN=
 
-# DEFAULT: 0
+# CHOOSE ONE: http, https
-# NOPAQUE_NUM_PROXIES=
-
 # DEFAULT: http
-# Choose one: http, https
 # NOPAQUE_PROTOCOL=
 
-# DEFAULT: True
-# Choose one: False, True
-# NOPAQUE_REMEMBER_COOKIE_HTTPONLY=
-
-# DEFAULT: False
-# Choose one: False, True
-# HINT: Set to true if you redirect http to https
-# NOPAQUE_REMEMBER_COOKIE_SECURE=
-
 # DEFAULT: hard to guess string
 # HINT: Use this bash command `python -c "import uuid; print(uuid.uuid4().hex)"`
 # NOPAQUE_SECRET_KEY=
 
-# DEFAULT: False
-# Choose one: False, True
-# HINT: Set to true if you redirect http to https
-# NOPAQUE_SESSION_COOKIE_SECURE=
-
 
 ################################################################################
 # Logging                                                                      #
 ################################################################################
-# DEFAULT: <nopaqued-root-dir>/nopaqued.log ~ /home/nopaqued/nopaqued.log
+# DEFAULT: /home/nopaqued/nopaqued.log ~ /home/nopaqued/nopaqued.log
 # NOTE: Use `.` as <nopaqued-root-dir>
 # NOPAQUE_DAEMON_LOG_FILE=
 
 # DEFAULT: %Y-%m-%d %H:%M:%S
 # NOPAQUE_LOG_DATE_FORMAT=
 
-# DEFAULT: <nopaque-root-dir>/NOPAQUE.log ~ /home/NOPAQUE/NOPAQUE.log
+# DEFAULT: <nopaque-root-dir>/nopaque.log ~ /home/nopaque/nopaque.log
 # NOTE: Use `.` as <nopaque-root-dir>
 # NOPAQUE_LOG_FILE=
 
 # DEFAULT: [%(asctime)s] %(levelname)s in %(pathname)s (function: %(funcName)s, line: %(lineno)d): %(message)s
 # NOPAQUE_LOG_FORMAT=
 
-# DEFAULT: ERROR
+# DEFAULT: WARNING
-# Choose one: CRITICAL, ERROR, WARNING, INFO, DEBUG
+# CHOOSE ONE: CRITICAL, ERROR, WARNING, INFO, DEBUG
 # NOPAQUE_LOG_LEVEL=
 
 
 ################################################################################
 # Message queue                                                                #
+# MESSAGE_QUEUE_URI blueprint:                                                 #
+#   - transport://[userid:password]@hostname[:port]/[virtual_host]             #
+#     - values in square brackets are optional                                 #
 ################################################################################
-NOPAQUE_MQ_HOST=
+# DEFAULT: None
+# HINT: A message queue is not required when using a single server process
+# NOPAQUE_SOCKETIO_MESSAGE_QUEUE_URI=
 
-# EXAMPLE: 6379
-NOPAQUE_MQ_PORT=
 
-# Choose one of the supported types by Flask-SocketIO
+################################################################################
-NOPAQUE_MQ_TYPE=
+# Proxy fix                                                                    #
+################################################################################
+# DEFAULT: 0
+# Number of values to trust for X-Forwarded-For
+# NOPAQUE_NUM_PROXIES_X_FOR=
+
+# DEFAULT: 0
+# Number of values to trust for X-Forwarded-Host
+# NOPAQUE_NUM_PROXIES_X_HOST=
+
+# DEFAULT: 0
+# Number of values to trust for X-Forwarded-Port
+# NOPAQUE_NUM_PROXIES_X_PORT=
+
+# DEFAULT: 0
+# Number of values to trust for X-Forwarded-Prefix
+# NOPAQUE_NUM_PROXIES_X_PREFIX=
+
+# DEFAULT: 0
+# Number of values to trust for X-Forwarded-Proto
+# NOPAQUE_NUM_PROXIES_X_PROTO=

.gitignore

@@ -1,6 +1,3 @@
-nopaque.log
-nopaqued.log
-
 *.py[cod]
 
 # C extensions
@@ -12,11 +9,14 @@ db
 mq
 
 # Environment files
-.env
+*.env
 
 # Installer logs
 pip-log.txt
 
+# Log files
+*.log
+
 # Packages
 *.egg
 *.egg-info

README.md

@@ -27,7 +27,7 @@ username@hostname:~$ docker run \
                        -p 139:139 \
                        -p 445:445 \
                        dperson/samba \
-                         -p -s "nopaque;/srv/samba/nopaque;no;no;no;nopaque" -u "nopaque;nopaque"
+                         -p -r -s "nopaque;/srv/samba/nopaque;no;no;no;nopaque" -u "nopaque;nopaque"
 
 # Mount the Samba share on all swarm nodes (managers and workers)
 username@hostname:~$ sudo mkdir /mnt/nopaque
@@ -39,9 +39,12 @@ username@hostname:~$ sudo mount --types cifs --options gid=${USER},password=nopa
 ``` bash
 # Clone the nopaque repository
 username@hostname:~$ git clone https://gitlab.ub.uni-bielefeld.de/sfb1288inf/nopaque.git
+username@hostname:~$ cp db.env.tpl db.env
 username@hostname:~$ cp .env.tpl .env
-# Fill out the variables within this file.
+# Fill out the variables within these files.
+username@hostname:~$ <YOUR EDITOR> db.env
 username@hostname:~$ <YOUR EDITOR> .env
+# Create docker-compose.override.yml file
 username@hostname:~$ touch docker-compose.override.yml
 # Tweak the docker-compose.override.yml to satisfy your needs. (You can find examples in docker-compose.<example>.yml)
 username@hostname:~$ <YOUR EDITOR> docker-compose.override.yml

daemon/Dockerfile

@@ -14,7 +14,6 @@ RUN apt-get update \
  && apt-get install --no-install-recommends --yes \
       build-essential \
       libpq-dev \
-      wait-for-it \
  && rm -r /var/lib/apt/lists/*
 
 
@@ -27,8 +26,7 @@ WORKDIR /home/nopaqued
 
 COPY --chown=nopaqued:nopaqued [".", "."]
 RUN python -m venv venv \
- && venv/bin/pip install --requirement requirements.txt \
+ && venv/bin/pip install --requirement requirements.txt
- && mkdir logs
 
 
 ENTRYPOINT ["./boot.sh"]

daemon/app/__init__.py

@@ -0,0 +1,31 @@
+from config import config
+from sqlalchemy import create_engine
+from sqlalchemy.orm import scoped_session, sessionmaker
+from time import sleep
+import docker
+import os
+
+
+configuration = config[os.environ.get('NOPAQUE_CONFIG', 'development')]
+configuration.init()
+docker_client = docker.from_env()
+engine = create_engine(configuration.SQLALCHEMY_DATABASE_URI)
+Session = scoped_session(sessionmaker(bind=engine))
+
+
+def run():
+    from .tasks.check_corpora import check_corpora
+    check_corpora_thread = check_corpora()
+    from .tasks.check_jobs import check_jobs
+    check_jobs_thread = check_jobs()
+    from .tasks.notify import notify
+    notify_thread = notify()
+
+    while True:
+        if not check_corpora_thread.is_alive():
+            check_corpora_thread = check_corpora()
+        if not check_jobs_thread.is_alive():
+            check_jobs_thread = check_jobs()
+        if not notify_thread.is_alive():
+            notify_thread = notify()
+        sleep(3)

daemon/app/tasks/check_corpora.py

@@ -1,6 +1,7 @@
-from . import config, docker_client, Session
+from .. import configuration as config
-from .decorators import background
+from .. import docker_client, Session
-from .models import Corpus
+from ..decorators import background
+from ..models import Corpus
 import docker
 import logging
 import os

daemon/app/tasks/check_jobs.py

@@ -1,7 +1,8 @@
 from datetime import datetime
-from . import config, docker_client, Session
+from .. import configuration as config
-from .decorators import background
+from .. import docker_client, Session
-from .models import Job, JobResult, NotificationData, NotificationEmailData
+from ..decorators import background
+from ..models import Job, JobResult, NotificationData, NotificationEmailData
 import docker
 import logging
 import json

daemon/app/tasks/notify.py

@@ -1,13 +1,18 @@
-from notify.notification import Notification
-from notify.service import NotificationService
 from sqlalchemy import asc
-from . import config, Session
+from .libnotify.notification import Notification
-from .decorators import background
+from .libnotify.service import NotificationService
-from .models import NotificationEmailData
+from .. import configuration as config
+from .. import Session
+from ..decorators import background
+from ..models import NotificationEmailData
 import logging
+import os
 import smtplib
 
 
+ROOT_DIR = os.path.abspath(os.path.dirname(__file__))
+
+
 @background
 def notify():
     session = Session()
@@ -72,8 +77,10 @@ def __create_mail_notifications(notification_service, session):
                                      'status': data.notify_status,
                                      'time': data.creation_date,
                                      'url': url}
-        txt_tmplt = 'notify/templates/notification_messages/notification.txt'
+        txt_tmplt = os.path.join(ROOT_DIR,
-        html_tmplt = 'notify/templates/notification_messages/notification.html'
+                                 'libnotify/templates/notification.txt')
+        html_tmplt = os.path.join(ROOT_DIR,
+                                  'libnotify/templates/notification.html')
         notification.set_notification_content(subject_template,
                                               subject_template_values_dict,
                                               txt_tmplt,

daemon/boot.sh

@@ -1,8 +1,3 @@
 #!/bin/bash
-echo "Waiting for db..."
-wait-for-it "${NOPAQUE_DB_HOST}:${NOPAQUE_DB_PORT:-5432}" --strict --timeout=0
-echo "Waiting for nopaque..."
-wait-for-it nopaque:5000 --strict --timeout=0
-
 source venv/bin/activate
 python nopaqued.py

daemon/config.py

@@ -2,60 +2,70 @@ import logging
 import os
 
 
-root_dir = os.path.abspath(os.path.dirname(__file__))
+ROOT_DIR = os.path.abspath(os.path.dirname(__file__))
-
-
-DEFAULT_DATA_DIR = os.path.join('/mnt/nopaque')
-DEFAULT_DB_PORT = '5432'
-DEFAULT_DOMAIN = 'localhost'
-DEFAULT_LOG_DATE_FORMAT = '%Y-%m-%d %H:%M:%S'
-DEFAULT_LOG_FILE = os.path.join(root_dir, 'nopaqued.log')
-DEFAULT_LOG_FORMAT = ('[%(asctime)s] %(levelname)s in %(pathname)s '
-                      '(function: %(funcName)s, line: %(lineno)d): '
-                      '%(message)s')
-DEFAULT_LOG_LEVEL = 'ERROR'
-DEFAULT_MAIL_USE_SSL = 'False'
-DEFAULT_MAIL_USE_TLS = 'False'
-DEFAULT_PROTOCOL = 'http'
 
 
 class Config:
-    ''' ### Database ### '''
+    ''' # Email # '''
-    DB_HOST = os.environ.get('NOPAQUE_DB_HOST')
-    DB_NAME = os.environ.get('NOPAQUE_DB_NAME')
-    DB_PASSWORD = os.environ.get('NOPAQUE_DB_PASSWORD')
-    DB_PORT = os.environ.get('NOPAQUE_DB_PORT', DEFAULT_DB_PORT)
-    DB_USERNAME = os.environ.get('NOPAQUE_DB_USERNAME')
-    SQLALCHEMY_DATABASE_URI = 'postgresql://{}:{}@{}:{}/{}'.format(
-        DB_USERNAME, DB_PASSWORD, DB_HOST, DB_PORT, DB_NAME)
-
-    ''' ### SMTP ### '''
     SMTP_DEFAULT_SENDER = os.environ.get('NOPAQUE_SMTP_DEFAULT_SENDER')
     SMTP_PASSWORD = os.environ.get('NOPAQUE_SMTP_PASSWORD')
-    SMTP_PORT = os.environ.get('NOPAQUE_SMTP_PORT')
+    SMTP_PORT = int(os.environ.get('NOPAQUE_SMTP_PORT'))
     SMTP_SERVER = os.environ.get('NOPAQUE_SMTP_SERVER')
     SMTP_USERNAME = os.environ.get('NOPAQUE_SMTP_USERNAME')
-    SMTP_USE_SSL = os.environ.get('NOPAQUE_SMTP_USE_SSL',
+    SMTP_USE_SSL = os.environ.get(
-                                  DEFAULT_MAIL_USE_SSL).lower() == 'true'
+        'NOPAQUE_SMTP_USE_SSL', 'false').lower() == 'true'
-    SMTP_USE_TLS = os.environ.get('NOPAQUE_SMTP_USE_TLS',
+    SMTP_USE_TLS = os.environ.get(
-                                  DEFAULT_MAIL_USE_TLS).lower() == 'true'
+        'NOPAQUE_SMTP_USE_TLS', 'false').lower() == 'true'
 
-    ''' ### General ### '''
+    ''' # General # '''
-    DATA_DIR = os.environ.get('NOPAQUE_DATA_DIR', DEFAULT_DATA_DIR)
+    DATA_DIR = os.environ.get('NOPAQUE_DATA_DIR', '/mnt/nopaque')
-    DOMAIN = os.environ.get('NOPAQUE_DOMAIN', DEFAULT_DOMAIN)
+    DOMAIN = os.environ.get('NOPAQUE_DOMAIN', 'localhost')
-    PROTOCOL = os.environ.get('NOPAQUE_PROTOCOL', DEFAULT_PROTOCOL)
+    PROTOCOL = os.environ.get('NOPAQUE_PROTOCOL', 'http')
+    SECRET_KEY = os.environ.get('NOPAQUE_SECRET_KEY', 'hard to guess string')
 
-    ''' ### Logging ### '''
+    ''' # Logging # '''
     LOG_DATE_FORMAT = os.environ.get('NOPAQUE_LOG_DATE_FORMAT',
-                                     DEFAULT_LOG_DATE_FORMAT)
+                                     '%Y-%m-%d %H:%M:%S')
-    LOG_FILE = os.environ.get('NOPAQUE_DAEMON_LOG_FILE', DEFAULT_LOG_FILE)
+    LOG_FILE = os.environ.get('NOPAQUED_LOG_FILE',
-    LOG_FORMAT = os.environ.get('NOPAQUE_LOG_FORMAT', DEFAULT_LOG_FORMAT)
+                              os.path.join(ROOT_DIR, 'nopaqued.log'))
-    LOG_LEVEL = os.environ.get('NOPAQUE_LOG_LEVEL', DEFAULT_LOG_LEVEL)
+    LOG_FORMAT = os.environ.get(
+        'NOPAQUE_LOG_FORMAT',
+        '[%(asctime)s] %(levelname)s in '
+        '%(pathname)s (function: %(funcName)s, line: %(lineno)d): %(message)s'
+    )
+    LOG_LEVEL = os.environ.get('NOPAQUE_LOG_LEVEL', 'WARNING')
 
-    def init_app(self):
+    @classmethod
-        # Configure logging according to the corresponding (LOG_*) config
+    def init(cls):
-        # entries
+        # Set up logging according to the corresponding (LOG_*) variables
-        logging.basicConfig(datefmt=self.LOG_DATE_FORMAT,
+        logging.basicConfig(datefmt=cls.LOG_DATE_FORMAT,
-                            filename=self.LOG_FILE,
+                            filename=cls.LOG_FILE,
-                            format=self.LOG_FORMAT,
+                            format=cls.LOG_FORMAT,
-                            level=self.LOG_LEVEL)
+                            level=cls.LOG_LEVEL)
+
+
+class DevelopmentConfig(Config):
+    ''' # Database # '''
+    SQLALCHEMY_DATABASE_URI = os.environ.get(
+        'NOPAQUE_DEV_DATABASE_URL',
+        'sqlite:///' + os.path.join(ROOT_DIR, 'data-dev.sqlite')
+    )
+
+
+class ProductionConfig(Config):
+    ''' # Database # '''
+    SQLALCHEMY_DATABASE_URI = os.environ.get(
+        'NOPAQUE_DATABASE_URL',
+        'sqlite:///' + os.path.join(ROOT_DIR, 'data.sqlite')
+    )
+
+
+class TestingConfig(Config):
+    ''' # Database # '''
+    SQLALCHEMY_DATABASE_URI = os.environ.get(
+        'NOPAQUE_TEST_DATABASE_URL', 'sqlite://')
+
+
+config = {'development': DevelopmentConfig,
+          'production': ProductionConfig,
+          'testing': TestingConfig}

daemon/nopaqued.py

@@ -1,23 +1,13 @@
-from tasks.check_corpora import check_corpora
+from dotenv import load_dotenv
-from tasks.check_jobs import check_jobs
+from app import run
-from tasks.notify import notify
+import os
-from time import sleep
 
 
-def nopaqued():
+# Load environment variables
-    check_corpora_thread = check_corpora()
+DOTENV_FILE = os.path.join(os.path.dirname(__file__), '.env')
-    check_jobs_thread = check_jobs()
+if os.path.exists(DOTENV_FILE):
-    notify_thread = notify()
+    load_dotenv(DOTENV_FILE)
-
-    while True:
-        if not check_corpora_thread.is_alive():
-            check_corpora_thread = check_corpora()
-        if not check_jobs_thread.is_alive():
-            check_jobs_thread = check_jobs()
-        if not notify_thread.is_alive():
-            notify_thread = notify()
-        sleep(3)
 
 
 if __name__ == '__main__':
-    nopaqued()
+    run()

daemon/requirements.txt

@@ -1,3 +1,4 @@
 docker
 psycopg2
+python-dotenv
 SQLAlchemy

daemon/tasks/__init__.py

@@ -1,11 +0,0 @@
-from config import Config
-from sqlalchemy import create_engine
-from sqlalchemy.orm import scoped_session, sessionmaker
-import docker
-
-
-config = Config()
-config.init_app()
-docker_client = docker.from_env()
-engine = create_engine(config.SQLALCHEMY_DATABASE_URI)
-Session = scoped_session(sessionmaker(bind=engine))

db.env.tpl

@@ -0,0 +1,5 @@
+POSTGRES_DB_NAME=
+
+POSTGRES_USER=
+
+POSTGRES_PASSWORD=

docker-compose.development.yml

@@ -16,10 +16,8 @@ services:
   nopaqued:
     volumes:
       # Mount code as volumes
+      - "./daemon/app:/home/nopaqued/app"
       - "./daemon/boot.sh:/home/nopaqued/boot.sh"
       - "./daemon/config.py:/home/nopaqued/config.py"
-      - "./daemon/logger:/home/nopaqued/logger"
       - "./daemon/nopaqued.py:/home/nopaqued/nopaqued.py"
-      - "./daemon/notify:/home/nopaqued/notify"
       - "./daemon/requirements.txt:/home/nopaqued/requirements.txt"
-      - "./daemon/tasks:/home/nopaqued/tasks"

docker-compose.traefik.yml

@@ -1,5 +1,6 @@
 ################################################################################
-# Don't forget to set the NOPAQUE_NUM_PROXIES variable in your .env            #
+# Don't forget to set the proxy variables in your nopaque.env                  #
+# Traefik sets the X_FOR, X_HOST, X_PORT and X_PROTO headers by default        #
 ################################################################################
 version: "3.5"
 
@@ -17,13 +18,13 @@ services:
       - "traefik.http.middlewares.nopaque-header.headers.customrequestheaders.X-Forwarded-Proto=http"
       - "traefik.http.routers.nopaque.entrypoints=web"
       - "traefik.http.routers.nopaque.middlewares=nopaque-header, redirect-to-https@file"
-      - "traefik.http.routers.nopaque.rule=Host(`${NOPAQUE_DOMAIN:-localhost}`)"
+      - "traefik.http.routers.nopaque.rule=Host(`<DOMAIN>`)"
       ### </http> ###
       ### <https> ###
       - "traefik.http.middlewares.nopaque-secure-header.headers.customrequestheaders.X-Forwarded-Proto=https"
       - "traefik.http.routers.nopaque-secure.entrypoints=web-secure"
       - "traefik.http.routers.nopaque-secure.middlewares=hsts-header@file, nopaque-secure-header"
-      - "traefik.http.routers.nopaque-secure.rule=Host(`${NOPAQUE_DOMAIN:-localhost}`)"
+      - "traefik.http.routers.nopaque-secure.rule=Host(`<DOMAIN>`)"
       - "traefik.http.routers.nopaque-secure.tls.certresolver=<CERTRESOLVER>"
       - "traefik.http.routers.nopaque-secure.tls.options=intermediate@file"
       ### </https> ###

docker-compose.yml

@@ -34,10 +34,7 @@ services:
       - "${NOPAQUE_DATA_DIR:-/mnt/nopaque}:${NOPAQUE_DATA_DIR:-/mnt/nopaque}"
       - "${HOST_NOPAQUE_DAEMON_LOG_FILE-./nopaqued.log}:${NOPAQUE_DAEMON_LOG_FILE:-/home/nopaqued/nopaqued.log}"
   db:
-    environment:
+    env_file: db.env
-      - POSTGRES_DB_NAME=${NOPAQUE_DB_NAME}
-      - POSTGRES_USER=${NOPAQUE_DB_USERNAME}
-      - POSTGRES_PASSWORD=${NOPAQUE_DB_PASSWORD}
     image: postgres:11
     restart: unless-stopped
     volumes:

web/Dockerfile

@@ -29,8 +29,7 @@ WORKDIR /home/nopaque
 
 COPY --chown=nopaque:nopaque [".", "."]
 RUN python -m venv venv \
- && venv/bin/pip install --requirement requirements.txt \
+ && venv/bin/pip install --requirement requirements.txt
- && mkdir logs
 
 
 ENTRYPOINT ["./boot.sh"]

web/app/__init__.py

@@ -1,4 +1,4 @@
-from config import Config
+from config import config
 from flask import Flask
 from flask_login import LoginManager
 from flask_mail import Mail
@@ -7,7 +7,6 @@ from flask_socketio import SocketIO
 from flask_sqlalchemy import SQLAlchemy
 
 
-config = Config()
 db = SQLAlchemy()
 login_manager = LoginManager()
 login_manager.login_view = 'auth.login'
@@ -17,19 +16,19 @@ paranoid.redirect_view = '/'
 socketio = SocketIO()
 
 
-def create_app():
+def create_app(config_name):
     app = Flask(__name__)
-    app.config.from_object(config)
+    app.config.from_object(config[config_name])
 
-    config.init_app(app)
+    config[config_name].init_app(app)
     db.init_app(app)
     login_manager.init_app(app)
     mail.init_app(app)
     paranoid.init_app(app)
-    socketio.init_app(app, message_queue=config.SOCKETIO_MESSAGE_QUEUE_URI)
+    socketio.init_app(
+        app, message_queue=config[config_name].SOCKETIO_MESSAGE_QUEUE_URI)
 
     from . import events
-
     from .admin import admin as admin_blueprint
     app.register_blueprint(admin_blueprint, url_prefix='/admin')
     from .auth import auth as auth_blueprint

web/boot.sh

@@ -1,12 +1,14 @@
 #!/bin/bash
-echo "Waiting for db..."
-wait-for-it "${NOPAQUE_DB_HOST}:${NOPAQUE_DB_PORT:-5432}" --strict --timeout=0
-echo "Waiting for mq..."
-wait-for-it "${NOPAQUE_MQ_HOST}:${NOPAQUE_MQ_PORT}" --strict --timeout=0
-
 source venv/bin/activate
 if [[ "$#" -eq 0 ]]; then
+    while true; do
         flask deploy
+        if [[ "$?" == "0" ]]; then
+            break
+        fi
+        echo Deploy command failed, retrying in 5 secs...
+        sleep 5
+    done
     python nopaque.py
 elif [[ "$1" == "flask" ]]; then
     exec ${@:1}

web/config.py

@@ -3,100 +3,107 @@ import logging
 import os
 
 
-root_dir = os.path.abspath(os.path.dirname(__file__))
+ROOT_DIR = os.path.abspath(os.path.dirname(__file__))
-
-
-DEFAULT_DATA_DIR = os.path.join('/mnt/nopaque')
-DEFAULT_DB_PORT = '5432'
-DEFAULT_DEBUG = 'False'
-DEFAULT_LOG_DATE_FORMAT = '%Y-%m-%d %H:%M:%S'
-DEFAULT_LOG_FILE = os.path.join(root_dir, 'nopaque.log')
-DEFAULT_LOG_FORMAT = ('[%(asctime)s] %(levelname)s in %(pathname)s '
-                      '(function: %(funcName)s, line: %(lineno)d): '
-                      '%(message)s')
-DEFAULT_LOG_LEVEL = 'ERROR'
-DEFAULT_SMTP_USE_SSL = 'False'
-DEFAULT_SMTP_USE_TLS = 'False'
-DEFAULT_NUM_PROXIES = '0'
-DEFAULT_PROTOCOL = 'http'
-DEFAULT_REMEMBER_COOKIE_HTTPONLY = 'True'
-DEFAULT_REMEMBER_COOKIE_SECURE = 'False'
-DEFAULT_SECRET_KEY = 'hard to guess string'
-DEFAULT_SESSION_COOKIE_SECURE = 'False'
 
 
 class Config:
-    ''' ### Database ### '''
+    ''' # Cookies # '''
-    DB_HOST = os.environ.get('NOPAQUE_DB_HOST')
+    REMEMBER_COOKIE_HTTPONLY = True
-    DB_NAME = os.environ.get('NOPAQUE_DB_NAME')
+    REMEMBER_COOKIE_SECURE = os.environ.get(
-    DB_PASSWORD = os.environ.get('NOPAQUE_DB_PASSWORD')
+        'NOPAQUE_REMEMBER_COOKIE_SECURE', 'false').lower() == 'true'
-    DB_PORT = os.environ.get('NOPAQUE_DB_PORT', DEFAULT_DB_PORT)
+    SESSION_COOKIE_SECURE = os.environ.get(
-    DB_USERNAME = os.environ.get('NOPAQUE_DB_USERNAME')
+        'NOPAQUE_SESSION_COOKIE_SECURE', 'false').lower() == 'true'
-    SQLALCHEMY_DATABASE_URI = 'postgresql://{}:{}@{}:{}/{}'.format(
+
-        DB_USERNAME, DB_PASSWORD, DB_HOST, DB_PORT, DB_NAME)
+    ''' # Database # '''
     SQLALCHEMY_RECORD_QUERIES = True
     SQLALCHEMY_TRACK_MODIFICATIONS = False
 
-    ''' ### Email ### '''
+    ''' # Email # '''
     MAIL_DEFAULT_SENDER = os.environ.get('NOPAQUE_SMTP_DEFAULT_SENDER')
     MAIL_PASSWORD = os.environ.get('NOPAQUE_SMTP_PASSWORD')
-    MAIL_PORT = os.environ.get('NOPAQUE_SMTP_PORT')
+    MAIL_PORT = int(os.environ.get('NOPAQUE_SMTP_PORT'))
     MAIL_SERVER = os.environ.get('NOPAQUE_SMTP_SERVER')
     MAIL_USERNAME = os.environ.get('NOPAQUE_SMTP_USERNAME')
-    MAIL_USE_SSL = os.environ.get('NOPAQUE_SMTP_USE_SSL',
+    MAIL_USE_SSL = os.environ.get(
-                                  DEFAULT_SMTP_USE_SSL).lower() == 'true'
+        'NOPAQUE_SMTP_USE_SSL', 'false').lower() == 'true'
-    MAIL_USE_TLS = os.environ.get('NOPAQUE_SMTP_USE_TLS',
+    MAIL_USE_TLS = os.environ.get(
-                                  DEFAULT_SMTP_USE_TLS).lower() == 'true'
+        'NOPAQUE_SMTP_USE_TLS', 'false').lower() == 'true'
 
-    ''' ### General ### '''
+    ''' # General # '''
     ADMIN_EMAIL_ADRESS = os.environ.get('NOPAQUE_ADMIN_EMAIL_ADRESS')
     CONTACT_EMAIL_ADRESS = os.environ.get('NOPAQUE_CONTACT_EMAIL_ADRESS')
-    DATA_DIR = os.environ.get('NOPAQUE_DATA_DIR', DEFAULT_DATA_DIR)
+    DATA_DIR = os.environ.get('NOPAQUE_DATA_DIR', '/mnt/nopaque')
-    DEBUG = os.environ.get('NOPAQUE_DEBUG', DEFAULT_DEBUG).lower() == 'true'
+    SECRET_KEY = os.environ.get('NOPAQUE_SECRET_KEY', 'hard to guess string')
-    NUM_PROXIES = int(os.environ.get('NOPAQUE_NUM_PROXIES',
-                                     DEFAULT_NUM_PROXIES))
-    PROTOCOL = os.environ.get('NOPAQUE_PROTOCOL', DEFAULT_PROTOCOL)
-    REMEMBER_COOKIE_HTTPONLY = os.environ.get(
-        'NOPAQUE_REMEMBER_COOKIE_HTTPONLY',
-        DEFAULT_REMEMBER_COOKIE_HTTPONLY
-    ).lower() == 'true'
-    REMEMBER_COOKIE_SECURE = os.environ.get(
-        'NOPAQUE_REMEMBER_COOKIE_SECURE',
-        DEFAULT_REMEMBER_COOKIE_SECURE
-    ).lower() == 'true'
-    SECRET_KEY = os.environ.get('RECIPY_SECRET_KEY', DEFAULT_SECRET_KEY)
-    SESSION_COOKIE_SECURE = os.environ.get(
-        'NOPAQUE_SESSION_COOKIE_SECURE',
-        DEFAULT_SESSION_COOKIE_SECURE
-    ).lower() == 'true'
 
-    ''' ### Logging ### '''
+    ''' # Logging # '''
     LOG_DATE_FORMAT = os.environ.get('NOPAQUE_LOG_DATE_FORMAT',
-                                     DEFAULT_LOG_DATE_FORMAT)
+                                     '%Y-%m-%d %H:%M:%S')
-    LOG_FILE = os.environ.get('NOPAQUE_LOG_FILE', DEFAULT_LOG_FILE)
+    LOG_FILE = os.environ.get('NOPAQUE_LOG_FILE',
-    LOG_FORMAT = os.environ.get('NOPAQUE_LOG_FORMAT', DEFAULT_LOG_FORMAT)
+                              os.path.join(ROOT_DIR, 'nopaque.log'))
-    LOG_LEVEL = os.environ.get('NOPAQUE_LOG_LEVEL', DEFAULT_LOG_LEVEL)
+    LOG_FORMAT = os.environ.get(
+        'NOPAQUE_LOG_FORMAT',
+        '[%(asctime)s] %(levelname)s in '
+        '%(pathname)s (function: %(funcName)s, line: %(lineno)d): %(message)s'
+    )
+    LOG_LEVEL = os.environ.get('NOPAQUE_LOG_LEVEL', 'WARNING')
 
-    ''' ### Message queue ### '''
+    ''' # Message queue # '''
-    MQ_HOST = os.environ.get('NOPAQUE_MQ_HOST')
+    SOCKETIO_MESSAGE_QUEUE_URI = os.environ.get(
-    MQ_PORT = os.environ.get('NOPAQUE_MQ_PORT')
+        'NOPAQUE_SOCKETIO_MESSAGE_QUEUE_URI')
-    MQ_TYPE = os.environ.get('NOPAQUE_MQ_TYPE')
-    SOCKETIO_MESSAGE_QUEUE_URI = \
-        '{}://{}:{}/'.format(MQ_TYPE, MQ_HOST, MQ_PORT)
 
-    def init_app(self, app):
+    ''' # Proxy fix # '''
-        # Configure logging according to the corresponding (LOG_*) config
+    PROXY_FIX_X_FOR = int(os.environ.get('NOPAQUE_PROXY_FIX_X_FOR', '0'))
-        # entries
+    PROXY_FIX_X_HOST = int(os.environ.get('NOPAQUE_PROXY_FIX_X_HOST', '0'))
-        logging.basicConfig(datefmt=self.LOG_DATE_FORMAT,
+    PROXY_FIX_X_PORT = int(os.environ.get('NOPAQUE_PROXY_FIX_X_PORT', '0'))
-                            filename=self.LOG_FILE,
+    PROXY_FIX_X_PREFIX = int(os.environ.get('NOPAQUE_PROXY_FIX_X_PREFIX', '0'))
-                            format=self.LOG_FORMAT,
+    PROXY_FIX_X_PROTO = int(os.environ.get('NOPAQUE_PROXY_FIX_X_PROTO', '0'))
-                            level=self.LOG_LEVEL)
+
-        # Apply the ProxyFix middleware if nopaque is running behind reverse
+    @classmethod
-        # proxies. (NUM_PROXIES indicates the number of reverse proxies running
+    def init_app(cls, app):
-        # in front of nopaque)
+        # Set up logging according to the corresponding (LOG_*) variables
-        if self.NUM_PROXIES > 0:
+        logging.basicConfig(datefmt=cls.LOG_DATE_FORMAT,
+                            filename=cls.LOG_FILE,
+                            format=cls.LOG_FORMAT,
+                            level=cls.LOG_LEVEL)
+        # Set up and apply the ProxyFix middleware according to the
+        # corresponding (PROXY_FIX_*) variables
         app.wsgi_app = ProxyFix(app.wsgi_app,
-                                    x_for=self.NUM_PROXIES,
+                                x_for=cls.PROXY_FIX_X_FOR,
-                                    x_host=self.NUM_PROXIES,
+                                x_host=cls.PROXY_FIX_X_HOST,
-                                    x_port=self.NUM_PROXIES,
+                                x_port=cls.PROXY_FIX_X_PORT,
-                                    x_proto=self.NUM_PROXIES)
+                                x_prefix=cls.PROXY_FIX_X_PREFIX,
+                                x_proto=cls.PROXY_FIX_X_PROTO)
+
+
+class DevelopmentConfig(Config):
+    ''' # Database # '''
+    SQLALCHEMY_DATABASE_URI = os.environ.get(
+        'NOPAQUE_DEV_DATABASE_URL',
+        'postgresql://nopaque:nopaque@db/nopaque_dev'
+    )
+
+    ''' # General # '''
+    DEBUG = True
+
+
+class ProductionConfig(Config):
+    ''' # Database # '''
+    SQLALCHEMY_DATABASE_URI = os.environ.get(
+        'NOPAQUE_DATABASE_URL', 'postgresql://nopaque:nopaque@db/nopaque')
+
+
+class TestingConfig(Config):
+    ''' # Database # '''
+    SQLALCHEMY_DATABASE_URI = os.environ.get(
+        'NOPAQUE_TEST_DATABASE_URL',
+        'postgresql://nopaque:nopaque@db/nopaque_test'
+    )
+
+    ''' # General # '''
+    TESTING = True
+    WTF_CSRF_ENABLED = False
+
+
+config = {'development': DevelopmentConfig,
+          'production': ProductionConfig,
+          'testing': TestingConfig}

web/nopaque.py

@@ -1,12 +1,28 @@
+# First things first: apply monkey patch, so that no code gets executed without
+# patched libraries!
 import eventlet
-eventlet.monkey_patch()  # noqa
+
-from app import create_app, db, socketio
+
+eventlet.monkey_patch()
+
+
+from dotenv import load_dotenv  # noqa
+import os  # noqa
+
+# Load environment variables
+DOTENV_FILE = os.path.join(os.path.dirname(__file__), '.env')
+if os.path.exists(DOTENV_FILE):
+    load_dotenv(DOTENV_FILE)
+
+
+from app import create_app, db, socketio  # noqa
 from app.models import (Corpus, CorpusFile, Job, JobInput, JobResult,
                         NotificationData, NotificationEmailData, QueryResult,
-                        Role, User)
+                        Role, User)  # noqa
-from flask_migrate import Migrate, upgrade
+from flask_migrate import Migrate, upgrade  # noqa
 
-app = create_app()
+
+app = create_app(os.environ.get('NOPAQUE_CONFIG', 'development'))
 migrate = Migrate(app, db, compare_type=True)
 
 

web/requirements.txt

@@ -1,6 +1,5 @@
 cqi
 dnspython==1.16.0
-email_validator
 eventlet
 Flask
 Flask-Login
@@ -11,6 +10,8 @@ Flask-SocketIO
 Flask-SQLAlchemy
 Flask-WTF
 jsonpatch
-psycopg2
-redis
 jsonschema
+psycopg2
+python-dotenv
+redis
+wtforms[email]