Enable advanced logging features

.env.tpl

@@ -1,13 +1,9 @@
 ################################################################################
 # Docker                                                                       #
 ################################################################################
-# DEFAULT: ./data/db
+# DEFAULT: ./data
-# NOTE: Use `.` as <project-root-dir>
+# NOTE: Use `.` as <project-basedir>
-# HOST_DB_DIR=
+# HOST_DATA_DIR=
-
-# DEFAULT: ./data/mq
-# NOTE: Use `.` as <project-root-dir>
-# HOST_MQ_DIR=
 
 # Example: 1000
 # HINT: Use this bash command `id -u`
@@ -21,10 +17,9 @@ HOST_GID=
 # HINT: Use this bash command `getent group docker | cut -d: -f3`
 HOST_DOCKER_GID=
 
-# DEFAULT: ./nopaque.log
+# DEFAULT: ./logs
-# NOTES: Use `.` as <project-root-dir>,
+# NOTES: Use `.` as <project-basedir>
-#        This file must be present on container startup
+# HOST_LOG_DIR=
-# HOST_LOG_FILE=
 
 
 ################################################################################
@@ -91,8 +86,9 @@ MAIL_USERNAME=
 # Flask-SQLAlchemy                                                             #
 # https://flask-sqlalchemy.palletsprojects.com/en/2.x/config/                  #
 ################################################################################
-# DEFAULT: 'sqlite:///<nopaque-root-dir>/app.db'
+# DEFAULT: 'sqlite:///<nopaque-basedir>/app.db'
-# NOTE: Use `.` as <nopaque-root-dir>
+# NOTE: Use `.` as <nopaque-basedir>,
+#       Don't use a SQLite database when using Docker
 # SQLALCHEMY_DATABASE_URI=
 
 
@@ -125,17 +121,21 @@ NOPAQUE_SOCKETIO_MESSAGE_QUEUE_URI=
 # DEFAULT: %Y-%m-%d %H:%M:%S
 # NOPAQUE_LOG_DATE_FORMAT=
 
-# DEFAULT: <nopaque-root-dir>/nopaque.log
+# DEFAULT: <nopaque-basedir>/logs
-# NOTE: Use `.` as <nopaque-root-dir>
+# NOTE: Use `.` as <nopaque-basedir>
-# NOPAQUE_LOG_FILE=
+# NOPAQUE_LOG_DIR=
 
 # DEFAULT: [%(asctime)s] %(levelname)s in %(pathname)s (function: %(funcName)s, line: %(lineno)d): %(message)s
 # NOPAQUE_LOG_FORMAT=
 
-# DEFAULT: WARNING
+# DEFAULT: INFO
 # CHOOSE ONE: CRITICAL, ERROR, WARNING, INFO, DEBUG
 # NOPAQUE_LOG_LEVEL=
 
+# CHOOSE ONE: False, True
+# DEFAULT: False
+# NOPAQUE_LOG_TO_STDOUT=
+
 # DEFAULT: 0
 # Number of values to trust for X-Forwarded-For
 # NOPAQUE_PROXY_FIX_X_FOR=

.gitignore

@@ -18,8 +18,8 @@ data/**
 # Installer logs
 pip-log.txt
 
-# Log files
+# Log folder
-*.log
+logs
 
 # Packages
 *.egg

app/__init__.py

@@ -33,7 +33,9 @@ def create_app(config_class=Config):
     migrate.init_app(app, db)
     paranoid.init_app(app)
     socketio.init_app(
-        app, message_queue=app.config['NOPAQUE_SOCKETIO_MESSAGE_QUEUE_URI'])
+        app,
+        message_queue=app.config.get('NOPAQUE_SOCKETIO_MESSAGE_QUEUE_URI')
+    )
 
     from .events import socketio as socketio_events
     from .events import sqlalchemy as sqlalchemy_events

config.py

@@ -1,4 +1,5 @@
 from dotenv import load_dotenv
+from logging.handlers import RotatingFileHandler
 from werkzeug.middleware.proxy_fix import ProxyFix
 import logging
 import os
@@ -50,31 +51,66 @@ class Config:
     NOPAQUE_SOCKETIO_MESSAGE_QUEUE_URI = \
         os.environ.get('NOPAQUE_SOCKETIO_MESSAGE_QUEUE_URI')
 
+    NOPAQUE_LOG_DATE_FORMAT = os.environ.get('NOPAQUE_LOG_DATE_FORMAT',
+                                             '%Y-%m-%d %H:%M:%S')
+    NOPAQUE_LOG_DIR = os.environ.get('NOPAQUE_LOG_DIR',
+                                     os.path.join(basedir, 'logs'))
+    NOPAQUE_LOG_FORMAT = os.environ.get(
+        'NOPAQUE_LOG_DATE_FORMAT',
+        '[%(asctime)s] %(levelname)s in '
+        '%(pathname)s (function: %(funcName)s, line: %(lineno)d): %(message)s'
+    )
+    NOPAQUE_LOG_LEVEL = os.environ.get('NOPAQUE_LOG_LEVEL', 'INFO')
+    NOPAQUE_LOG_TO_STDOUT = \
+        os.environ.get('NOPAQUE_LOG_TO_STDOUT', 'false').lower() == 'true'
+
+    NOPAQUE_PROXY_FIX_X_FOR = \
+        int(os.environ.get('NOPAQUE_PROXY_FIX_X_FOR', '0'))
+    NOPAQUE_PROXY_FIX_X_HOST = \
+        int(os.environ.get('NOPAQUE_PROXY_FIX_X_HOST', '0'))
+    NOPAQUE_PROXY_FIX_X_PORT = \
+        int(os.environ.get('NOPAQUE_PROXY_FIX_X_PORT', '0'))
+    NOPAQUE_PROXY_FIX_X_PREFIX = \
+        int(os.environ.get('NOPAQUE_PROXY_FIX_X_PREFIX', '0'))
+    NOPAQUE_PROXY_FIX_X_PROTO = \
+        int(os.environ.get('NOPAQUE_PROXY_FIX_X_PROTO', '0'))
+
     @classmethod
     def init_app(cls, app):
         # Set up logging according to the corresponding (NOPAQUE_LOG_*)
-        # environment variables
+        # configurations
-        basic_config_kwargs = {
+        # ... But first remove all existing handlers
-            'datefmt': os.environ.get('NOPAQUE_LOG_DATE_FORMAT',
+        for handler in app.logger.handlers:
-                                      '%Y-%m-%d %H:%M:%S'),
+            app.logger.removeHandler(handler)
-            'filename': os.environ.get('NOPAQUE_LOG_FILE',
+        formatter = logging.Formatter(
-                                       os.path.join(basedir, 'nopaque.log')),
+            fmt=app.config.get('NOPAQUE_LOG_FORMAT'),
-            'format': os.environ.get(
+            datefmt=app.config.get('NOPAQUE_LOG_DATE_FORMAT')
-                'NOPAQUE_LOG_FORMAT',
+        )
-                '[%(asctime)s] %(levelname)s in '
+        if app.config.get('NOPAQUE_LOG_TO_STDOUT'):
-                '%(pathname)s (function: %(funcName)s, line: %(lineno)d): '
+            stream_handler = logging.StreamHandler()
-                '%(message)s'
+            stream_handler.setFormatter(formatter)
-            ),
+            stream_handler.setLevel(app.config.get('NOPAQUE_LOG_LEVEL'))
-            'level': os.environ.get('NOPAQUE_LOG_LEVEL', 'WARNING')
+            app.logger.addHandler(stream_handler)
-        }
+        else:
-        logging.basicConfig(**basic_config_kwargs)
+            if not os.path.exists(app.config.get('NOPAQUE_LOG_DIR')):
+                os.mkdir(app.config.get('NOPAQUE_LOG_DIR'))
+            rotating_file_handler = RotatingFileHandler(
+                os.path.join(app.config.get('NOPAQUE_LOG_DIR'), 'nopaque.log'),
+                maxBytes=10240,
+                backupCount=10
+            )
+            rotating_file_handler.setFormatter(formatter)
+            rotating_file_handler.setLevel(app.config.get('NOPAQUE_LOG_LEVEL'))
+            app.logger.addHandler(rotating_file_handler)
+        app.logger.setLevel(app.config.get('NOPAQUE_LOG_LEVEL'))
+
         # Set up and apply the ProxyFix middleware according to the
-        # corresponding (NOPAQUE_PROXY_FIX_*) environment variables
+        # corresponding (NOPAQUE_PROXY_FIX_*) configurations
-        proxy_fix_kwargs = {
+        app.wsgi_app = ProxyFix(
-            'x_for': int(os.environ.get('NOPAQUE_PROXY_FIX_X_FOR', '0')),
+            app.wsgi_app,
-            'x_host': int(os.environ.get('NOPAQUE_PROXY_FIX_X_HOST', '0')),
+            x_for=app.config.get('NOPAQUE_PROXY_FIX_X_FOR'),
-            'x_port': int(os.environ.get('NOPAQUE_PROXY_FIX_X_PORT', '0')),
+            x_host=app.config.get('NOPAQUE_PROXY_FIX_X_HOST'),
-            'x_prefix': int(os.environ.get('NOPAQUE_PROXY_FIX_X_PREFIX', '0')),
+            x_port=app.config.get('NOPAQUE_PROXY_FIX_X_PORT'),
-            'x_proto': int(os.environ.get('NOPAQUE_PROXY_FIX_X_PROTO', '0'))
+            x_prefix=app.config.get('NOPAQUE_PROXY_FIX_X_PREFIX'),
-        }
+            x_proto=app.config.get('NOPAQUE_PROXY_FIX_X_PROTO')
-        app.wsgi_app = ProxyFix(app.wsgi_app, **proxy_fix_kwargs)
+        )

docker-compose.yml

@@ -6,13 +6,13 @@ services:
     image: postgres:11
     restart: unless-stopped
     volumes:
-      - "${HOST_DB_DIR:-./data/db}:/var/lib/postgresql/data"
+      - "${HOST_DATA_DIR:-./data}/db:/var/lib/postgresql/data"
 
   mq:
     image: redis:6
     restart: unless-stopped
     volumes:
-      - "${HOST_MQ_DIR:-./data/mq}:/data"
+      - "${HOST_DATA_DIR:-./data}/mq:/data"
 
   nopaque:
     build:
@@ -30,4 +30,4 @@ services:
     volumes:
       - "/var/run/docker.sock:/var/run/docker.sock"
       - "${NOPAQUE_DATA_DIR:-/mnt/nopaque}:${NOPAQUE_DATA_DIR:-/mnt/nopaque}"
-      - "${HOST_NOPAQUE_LOG_FILE-./nopaque.log}:${NOPAQUE_LOG_FILE:-/home/nopaque/nopaque.log}"
+      - "${HOST_LOG_DIR-./logs}:${NOPAQUE_LOG_DIR:-/home/nopaque/logs}"

nopaque.py

@@ -4,7 +4,7 @@ import eventlet
 eventlet.monkey_patch()
 
 
-from app import db, cli, create_app, socketio  # noqa
+from app import db, cli, create_app  # noqa
 from app.models import (Corpus, CorpusFile, Job, JobInput, JobResult,
                         QueryResult, Role, User)  # noqa