sporada/nopaque
1
0
Fork 0
mirror of https://gitlab.ub.uni-bielefeld.de/sfb1288inf/nopaque.git synced 2025-11-03 20:02:47 +00:00
Code Issues Projects Releases Wiki Activity

Enable advanced logging features

Browse Source
This commit is contained in:
Patrick Jentsch
2021-09-15 17:58:17 +02:00
parent 727553412f
commit e2c68a1a80
6 changed files with 86 additions and 48 deletions
Download Patch File Download Diff File Expand all files Collapse all files

34
.env.tpl
View File

@@ -1,13 +1,9 @@
################################################################################
# Docker #
################################################################################
# DEFAULT: ./data/db
# NOTE: Use `.` as <project-root-dir>
# HOST_DB_DIR=
# DEFAULT: ./data/mq
# NOTE: Use `.` as <project-root-dir>
# HOST_MQ_DIR=
# DEFAULT: ./data
# NOTE: Use `.` as <project-basedir>
# HOST_DATA_DIR=
# Example: 1000
# HINT: Use this bash command `id -u`
@@ -21,10 +17,9 @@ HOST_GID=
# HINT: Use this bash command `getent group docker | cut -d: -f3`
HOST_DOCKER_GID=
# DEFAULT: ./nopaque.log
# NOTES: Use `.` as <project-root-dir>,
# This file must be present on container startup
# HOST_LOG_FILE=
# DEFAULT: ./logs
# NOTES: Use `.` as <project-basedir>
# HOST_LOG_DIR=
################################################################################
@@ -91,8 +86,9 @@ MAIL_USERNAME=
# Flask-SQLAlchemy #
# https://flask-sqlalchemy.palletsprojects.com/en/2.x/config/ #
################################################################################
# DEFAULT: 'sqlite:///<nopaque-root-dir>/app.db'
# NOTE: Use `.` as <nopaque-root-dir>
# DEFAULT: 'sqlite:///<nopaque-basedir>/app.db'
# NOTE: Use `.` as <nopaque-basedir>,
# Don't use a SQLite database when using Docker
# SQLALCHEMY_DATABASE_URI=
@@ -125,17 +121,21 @@ NOPAQUE_SOCKETIO_MESSAGE_QUEUE_URI=
# DEFAULT: %Y-%m-%d %H:%M:%S
# NOPAQUE_LOG_DATE_FORMAT=
# DEFAULT: <nopaque-root-dir>/nopaque.log
# NOTE: Use `.` as <nopaque-root-dir>
# NOPAQUE_LOG_FILE=
# DEFAULT: <nopaque-basedir>/logs
# NOTE: Use `.` as <nopaque-basedir>
# NOPAQUE_LOG_DIR=
# DEFAULT: [%(asctime)s] %(levelname)s in %(pathname)s (function: %(funcName)s, line: %(lineno)d): %(message)s
# NOPAQUE_LOG_FORMAT=
# DEFAULT: WARNING
# DEFAULT: INFO
# CHOOSE ONE: CRITICAL, ERROR, WARNING, INFO, DEBUG
# NOPAQUE_LOG_LEVEL=
# CHOOSE ONE: False, True
# DEFAULT: False
# NOPAQUE_LOG_TO_STDOUT=
# DEFAULT: 0
# Number of values to trust for X-Forwarded-For
# NOPAQUE_PROXY_FIX_X_FOR=

4
.gitignore vendored
View File

@@ -18,8 +18,8 @@ data/**
# Installer logs
pip-log.txt
# Log files
*.log
# Log folder
logs
# Packages
*.egg

4
app/__init__.py
View File

@@ -33,7 +33,9 @@ def create_app(config_class=Config):
migrate.init_app(app, db)
paranoid.init_app(app)
socketio.init_app(
app, message_queue=app.config['NOPAQUE_SOCKETIO_MESSAGE_QUEUE_URI'])
app,
message_queue=app.config.get('NOPAQUE_SOCKETIO_MESSAGE_QUEUE_URI')
)
from .events import socketio as socketio_events
from .events import sqlalchemy as sqlalchemy_events

84
config.py
View File

@@ -1,4 +1,5 @@
from dotenv import load_dotenv
from logging.handlers import RotatingFileHandler
from werkzeug.middleware.proxy_fix import ProxyFix
import logging
import os
@@ -50,31 +51,66 @@ class Config:
NOPAQUE_SOCKETIO_MESSAGE_QUEUE_URI = \
os.environ.get('NOPAQUE_SOCKETIO_MESSAGE_QUEUE_URI')
NOPAQUE_LOG_DATE_FORMAT = os.environ.get('NOPAQUE_LOG_DATE_FORMAT',
'%Y-%m-%d %H:%M:%S')
NOPAQUE_LOG_DIR = os.environ.get('NOPAQUE_LOG_DIR',
os.path.join(basedir, 'logs'))
NOPAQUE_LOG_FORMAT = os.environ.get(
'NOPAQUE_LOG_DATE_FORMAT',
'[%(asctime)s] %(levelname)s in '
'%(pathname)s (function: %(funcName)s, line: %(lineno)d): %(message)s'
)
NOPAQUE_LOG_LEVEL = os.environ.get('NOPAQUE_LOG_LEVEL', 'INFO')
NOPAQUE_LOG_TO_STDOUT = \
os.environ.get('NOPAQUE_LOG_TO_STDOUT', 'false').lower() == 'true'
NOPAQUE_PROXY_FIX_X_FOR = \
int(os.environ.get('NOPAQUE_PROXY_FIX_X_FOR', '0'))
NOPAQUE_PROXY_FIX_X_HOST = \
int(os.environ.get('NOPAQUE_PROXY_FIX_X_HOST', '0'))
NOPAQUE_PROXY_FIX_X_PORT = \
int(os.environ.get('NOPAQUE_PROXY_FIX_X_PORT', '0'))
NOPAQUE_PROXY_FIX_X_PREFIX = \
int(os.environ.get('NOPAQUE_PROXY_FIX_X_PREFIX', '0'))
NOPAQUE_PROXY_FIX_X_PROTO = \
int(os.environ.get('NOPAQUE_PROXY_FIX_X_PROTO', '0'))
@classmethod
def init_app(cls, app):
# Set up logging according to the corresponding (NOPAQUE_LOG_*)
# environment variables
basic_config_kwargs = {
'datefmt': os.environ.get('NOPAQUE_LOG_DATE_FORMAT',
'%Y-%m-%d %H:%M:%S'),
'filename': os.environ.get('NOPAQUE_LOG_FILE',
os.path.join(basedir, 'nopaque.log')),
'format': os.environ.get(
'NOPAQUE_LOG_FORMAT',
'[%(asctime)s] %(levelname)s in '
'%(pathname)s (function: %(funcName)s, line: %(lineno)d): '
'%(message)s'
),
'level': os.environ.get('NOPAQUE_LOG_LEVEL', 'WARNING')
}
logging.basicConfig(**basic_config_kwargs)
# configurations
# ... But first remove all existing handlers
for handler in app.logger.handlers:
app.logger.removeHandler(handler)
formatter = logging.Formatter(
fmt=app.config.get('NOPAQUE_LOG_FORMAT'),
datefmt=app.config.get('NOPAQUE_LOG_DATE_FORMAT')
)
if app.config.get('NOPAQUE_LOG_TO_STDOUT'):
stream_handler = logging.StreamHandler()
stream_handler.setFormatter(formatter)
stream_handler.setLevel(app.config.get('NOPAQUE_LOG_LEVEL'))
app.logger.addHandler(stream_handler)
else:
if not os.path.exists(app.config.get('NOPAQUE_LOG_DIR')):
os.mkdir(app.config.get('NOPAQUE_LOG_DIR'))
rotating_file_handler = RotatingFileHandler(
os.path.join(app.config.get('NOPAQUE_LOG_DIR'), 'nopaque.log'),
maxBytes=10240,
backupCount=10
)
rotating_file_handler.setFormatter(formatter)
rotating_file_handler.setLevel(app.config.get('NOPAQUE_LOG_LEVEL'))
app.logger.addHandler(rotating_file_handler)
app.logger.setLevel(app.config.get('NOPAQUE_LOG_LEVEL'))
# Set up and apply the ProxyFix middleware according to the
# corresponding (NOPAQUE_PROXY_FIX_*) environment variables
proxy_fix_kwargs = {
'x_for': int(os.environ.get('NOPAQUE_PROXY_FIX_X_FOR', '0')),
'x_host': int(os.environ.get('NOPAQUE_PROXY_FIX_X_HOST', '0')),
'x_port': int(os.environ.get('NOPAQUE_PROXY_FIX_X_PORT', '0')),
'x_prefix': int(os.environ.get('NOPAQUE_PROXY_FIX_X_PREFIX', '0')),
'x_proto': int(os.environ.get('NOPAQUE_PROXY_FIX_X_PROTO', '0'))
}
app.wsgi_app = ProxyFix(app.wsgi_app, **proxy_fix_kwargs)
# corresponding (NOPAQUE_PROXY_FIX_*) configurations
app.wsgi_app = ProxyFix(
app.wsgi_app,
x_for=app.config.get('NOPAQUE_PROXY_FIX_X_FOR'),
x_host=app.config.get('NOPAQUE_PROXY_FIX_X_HOST'),
x_port=app.config.get('NOPAQUE_PROXY_FIX_X_PORT'),
x_prefix=app.config.get('NOPAQUE_PROXY_FIX_X_PREFIX'),
x_proto=app.config.get('NOPAQUE_PROXY_FIX_X_PROTO')
)

6
docker-compose.yml
View File

@@ -6,13 +6,13 @@ services:
image: postgres:11
restart: unless-stopped
volumes:
- "${HOST_DB_DIR:-./data/db}:/var/lib/postgresql/data"
- "${HOST_DATA_DIR:-./data}/db:/var/lib/postgresql/data"
mq:
image: redis:6
restart: unless-stopped
volumes:
- "${HOST_MQ_DIR:-./data/mq}:/data"
- "${HOST_DATA_DIR:-./data}/mq:/data"
nopaque:
build:
@@ -30,4 +30,4 @@ services:
volumes:
- "/var/run/docker.sock:/var/run/docker.sock"
- "${NOPAQUE_DATA_DIR:-/mnt/nopaque}:${NOPAQUE_DATA_DIR:-/mnt/nopaque}"
- "${HOST_NOPAQUE_LOG_FILE-./nopaque.log}:${NOPAQUE_LOG_FILE:-/home/nopaque/nopaque.log}"
- "${HOST_LOG_DIR-./logs}:${NOPAQUE_LOG_DIR:-/home/nopaque/logs}"

2
nopaque.py
View File

@@ -4,7 +4,7 @@ import eventlet
eventlet.monkey_patch()
from app import db, cli, create_app, socketio # noqa
from app import db, cli, create_app # noqa
from app.models import (Corpus, CorpusFile, Job, JobInput, JobResult,
QueryResult, Role, User) # noqa