From e2c68a1a808ec98b0af836417a70c4b5f33837f4 Mon Sep 17 00:00:00 2001 From: Patrick Jentsch Date: Wed, 15 Sep 2021 17:58:17 +0200 Subject: [PATCH] Enable advanced logging features --- .env.tpl | 34 +++++++++---------- .gitignore | 4 +-- app/__init__.py | 4 ++- config.py | 84 +++++++++++++++++++++++++++++++++------------- docker-compose.yml | 6 ++-- nopaque.py | 2 +- 6 files changed, 86 insertions(+), 48 deletions(-) diff --git a/.env.tpl b/.env.tpl index d05e16ae..f0e99b98 100644 --- a/.env.tpl +++ b/.env.tpl @@ -1,13 +1,9 @@ ################################################################################ # Docker # ################################################################################ -# DEFAULT: ./data/db -# NOTE: Use `.` as -# HOST_DB_DIR= - -# DEFAULT: ./data/mq -# NOTE: Use `.` as -# HOST_MQ_DIR= +# DEFAULT: ./data +# NOTE: Use `.` as +# HOST_DATA_DIR= # Example: 1000 # HINT: Use this bash command `id -u` @@ -21,10 +17,9 @@ HOST_GID= # HINT: Use this bash command `getent group docker | cut -d: -f3` HOST_DOCKER_GID= -# DEFAULT: ./nopaque.log -# NOTES: Use `.` as , -# This file must be present on container startup -# HOST_LOG_FILE= +# DEFAULT: ./logs +# NOTES: Use `.` as +# HOST_LOG_DIR= ################################################################################ @@ -91,8 +86,9 @@ MAIL_USERNAME= # Flask-SQLAlchemy # # https://flask-sqlalchemy.palletsprojects.com/en/2.x/config/ # ################################################################################ -# DEFAULT: 'sqlite:////app.db' -# NOTE: Use `.` as +# DEFAULT: 'sqlite:////app.db' +# NOTE: Use `.` as , +# Don't use a SQLite database when using Docker # SQLALCHEMY_DATABASE_URI= @@ -125,17 +121,21 @@ NOPAQUE_SOCKETIO_MESSAGE_QUEUE_URI= # DEFAULT: %Y-%m-%d %H:%M:%S # NOPAQUE_LOG_DATE_FORMAT= -# DEFAULT: /nopaque.log -# NOTE: Use `.` as -# NOPAQUE_LOG_FILE= +# DEFAULT: /logs +# NOTE: Use `.` as +# NOPAQUE_LOG_DIR= # DEFAULT: [%(asctime)s] %(levelname)s in %(pathname)s (function: %(funcName)s, line: %(lineno)d): %(message)s # NOPAQUE_LOG_FORMAT= -# DEFAULT: WARNING +# DEFAULT: INFO # CHOOSE ONE: CRITICAL, ERROR, WARNING, INFO, DEBUG # NOPAQUE_LOG_LEVEL= +# CHOOSE ONE: False, True +# DEFAULT: False +# NOPAQUE_LOG_TO_STDOUT= + # DEFAULT: 0 # Number of values to trust for X-Forwarded-For # NOPAQUE_PROXY_FIX_X_FOR= diff --git a/.gitignore b/.gitignore index 5c56af6d..f8f58ae1 100644 --- a/.gitignore +++ b/.gitignore @@ -18,8 +18,8 @@ data/** # Installer logs pip-log.txt -# Log files -*.log +# Log folder +logs # Packages *.egg diff --git a/app/__init__.py b/app/__init__.py index c03a25ea..37dc1a4b 100644 --- a/app/__init__.py +++ b/app/__init__.py @@ -33,7 +33,9 @@ def create_app(config_class=Config): migrate.init_app(app, db) paranoid.init_app(app) socketio.init_app( - app, message_queue=app.config['NOPAQUE_SOCKETIO_MESSAGE_QUEUE_URI']) + app, + message_queue=app.config.get('NOPAQUE_SOCKETIO_MESSAGE_QUEUE_URI') + ) from .events import socketio as socketio_events from .events import sqlalchemy as sqlalchemy_events diff --git a/config.py b/config.py index 2c99b412..c0c3f1a8 100644 --- a/config.py +++ b/config.py @@ -1,4 +1,5 @@ from dotenv import load_dotenv +from logging.handlers import RotatingFileHandler from werkzeug.middleware.proxy_fix import ProxyFix import logging import os @@ -50,31 +51,66 @@ class Config: NOPAQUE_SOCKETIO_MESSAGE_QUEUE_URI = \ os.environ.get('NOPAQUE_SOCKETIO_MESSAGE_QUEUE_URI') + NOPAQUE_LOG_DATE_FORMAT = os.environ.get('NOPAQUE_LOG_DATE_FORMAT', + '%Y-%m-%d %H:%M:%S') + NOPAQUE_LOG_DIR = os.environ.get('NOPAQUE_LOG_DIR', + os.path.join(basedir, 'logs')) + NOPAQUE_LOG_FORMAT = os.environ.get( + 'NOPAQUE_LOG_DATE_FORMAT', + '[%(asctime)s] %(levelname)s in ' + '%(pathname)s (function: %(funcName)s, line: %(lineno)d): %(message)s' + ) + NOPAQUE_LOG_LEVEL = os.environ.get('NOPAQUE_LOG_LEVEL', 'INFO') + NOPAQUE_LOG_TO_STDOUT = \ + os.environ.get('NOPAQUE_LOG_TO_STDOUT', 'false').lower() == 'true' + + NOPAQUE_PROXY_FIX_X_FOR = \ + int(os.environ.get('NOPAQUE_PROXY_FIX_X_FOR', '0')) + NOPAQUE_PROXY_FIX_X_HOST = \ + int(os.environ.get('NOPAQUE_PROXY_FIX_X_HOST', '0')) + NOPAQUE_PROXY_FIX_X_PORT = \ + int(os.environ.get('NOPAQUE_PROXY_FIX_X_PORT', '0')) + NOPAQUE_PROXY_FIX_X_PREFIX = \ + int(os.environ.get('NOPAQUE_PROXY_FIX_X_PREFIX', '0')) + NOPAQUE_PROXY_FIX_X_PROTO = \ + int(os.environ.get('NOPAQUE_PROXY_FIX_X_PROTO', '0')) + @classmethod def init_app(cls, app): # Set up logging according to the corresponding (NOPAQUE_LOG_*) - # environment variables - basic_config_kwargs = { - 'datefmt': os.environ.get('NOPAQUE_LOG_DATE_FORMAT', - '%Y-%m-%d %H:%M:%S'), - 'filename': os.environ.get('NOPAQUE_LOG_FILE', - os.path.join(basedir, 'nopaque.log')), - 'format': os.environ.get( - 'NOPAQUE_LOG_FORMAT', - '[%(asctime)s] %(levelname)s in ' - '%(pathname)s (function: %(funcName)s, line: %(lineno)d): ' - '%(message)s' - ), - 'level': os.environ.get('NOPAQUE_LOG_LEVEL', 'WARNING') - } - logging.basicConfig(**basic_config_kwargs) + # configurations + # ... But first remove all existing handlers + for handler in app.logger.handlers: + app.logger.removeHandler(handler) + formatter = logging.Formatter( + fmt=app.config.get('NOPAQUE_LOG_FORMAT'), + datefmt=app.config.get('NOPAQUE_LOG_DATE_FORMAT') + ) + if app.config.get('NOPAQUE_LOG_TO_STDOUT'): + stream_handler = logging.StreamHandler() + stream_handler.setFormatter(formatter) + stream_handler.setLevel(app.config.get('NOPAQUE_LOG_LEVEL')) + app.logger.addHandler(stream_handler) + else: + if not os.path.exists(app.config.get('NOPAQUE_LOG_DIR')): + os.mkdir(app.config.get('NOPAQUE_LOG_DIR')) + rotating_file_handler = RotatingFileHandler( + os.path.join(app.config.get('NOPAQUE_LOG_DIR'), 'nopaque.log'), + maxBytes=10240, + backupCount=10 + ) + rotating_file_handler.setFormatter(formatter) + rotating_file_handler.setLevel(app.config.get('NOPAQUE_LOG_LEVEL')) + app.logger.addHandler(rotating_file_handler) + app.logger.setLevel(app.config.get('NOPAQUE_LOG_LEVEL')) + # Set up and apply the ProxyFix middleware according to the - # corresponding (NOPAQUE_PROXY_FIX_*) environment variables - proxy_fix_kwargs = { - 'x_for': int(os.environ.get('NOPAQUE_PROXY_FIX_X_FOR', '0')), - 'x_host': int(os.environ.get('NOPAQUE_PROXY_FIX_X_HOST', '0')), - 'x_port': int(os.environ.get('NOPAQUE_PROXY_FIX_X_PORT', '0')), - 'x_prefix': int(os.environ.get('NOPAQUE_PROXY_FIX_X_PREFIX', '0')), - 'x_proto': int(os.environ.get('NOPAQUE_PROXY_FIX_X_PROTO', '0')) - } - app.wsgi_app = ProxyFix(app.wsgi_app, **proxy_fix_kwargs) + # corresponding (NOPAQUE_PROXY_FIX_*) configurations + app.wsgi_app = ProxyFix( + app.wsgi_app, + x_for=app.config.get('NOPAQUE_PROXY_FIX_X_FOR'), + x_host=app.config.get('NOPAQUE_PROXY_FIX_X_HOST'), + x_port=app.config.get('NOPAQUE_PROXY_FIX_X_PORT'), + x_prefix=app.config.get('NOPAQUE_PROXY_FIX_X_PREFIX'), + x_proto=app.config.get('NOPAQUE_PROXY_FIX_X_PROTO') + ) diff --git a/docker-compose.yml b/docker-compose.yml index e5c1ecef..f668e00d 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -6,13 +6,13 @@ services: image: postgres:11 restart: unless-stopped volumes: - - "${HOST_DB_DIR:-./data/db}:/var/lib/postgresql/data" + - "${HOST_DATA_DIR:-./data}/db:/var/lib/postgresql/data" mq: image: redis:6 restart: unless-stopped volumes: - - "${HOST_MQ_DIR:-./data/mq}:/data" + - "${HOST_DATA_DIR:-./data}/mq:/data" nopaque: build: @@ -30,4 +30,4 @@ services: volumes: - "/var/run/docker.sock:/var/run/docker.sock" - "${NOPAQUE_DATA_DIR:-/mnt/nopaque}:${NOPAQUE_DATA_DIR:-/mnt/nopaque}" - - "${HOST_NOPAQUE_LOG_FILE-./nopaque.log}:${NOPAQUE_LOG_FILE:-/home/nopaque/nopaque.log}" + - "${HOST_LOG_DIR-./logs}:${NOPAQUE_LOG_DIR:-/home/nopaque/logs}" diff --git a/nopaque.py b/nopaque.py index faa1dd2d..30324bb9 100644 --- a/nopaque.py +++ b/nopaque.py @@ -4,7 +4,7 @@ import eventlet eventlet.monkey_patch() -from app import db, cli, create_app, socketio # noqa +from app import db, cli, create_app # noqa from app.models import (Corpus, CorpusFile, Job, JobInput, JobResult, QueryResult, Role, User) # noqa