sporada/nopaque
1
0
Fork 0
mirror of https://gitlab.ub.uni-bielefeld.de/sfb1288inf/nopaque.git synced 2025-11-04 04:12:45 +00:00
Code Issues Projects Releases Wiki Activity

Enable advanced logging features

Browse Source
This commit is contained in:
Patrick Jentsch
2021-09-15 17:58:17 +02:00
parent 727553412f
commit e2c68a1a80
6 changed files with 86 additions and 48 deletions
Download Patch File Download Diff File Expand all files Collapse all files

34
.env.tpl
View File

@@ -1,13 +1,9 @@
################################################################################ ################################################################################
# Docker # # Docker #
################################################################################ ################################################################################
# DEFAULT: ./data/db # DEFAULT: ./data
# NOTE: Use `.` as <project-root-dir> # NOTE: Use `.` as <project-basedir>
# HOST_DB_DIR= # HOST_DATA_DIR=
# DEFAULT: ./data/mq
# NOTE: Use `.` as <project-root-dir>
# HOST_MQ_DIR=
# Example: 1000 # Example: 1000
# HINT: Use this bash command `id -u` # HINT: Use this bash command `id -u`
@@ -21,10 +17,9 @@ HOST_GID=
# HINT: Use this bash command `getent group docker | cut -d: -f3` # HINT: Use this bash command `getent group docker | cut -d: -f3`
HOST_DOCKER_GID= HOST_DOCKER_GID=
# DEFAULT: ./nopaque.log # DEFAULT: ./logs
# NOTES: Use `.` as <project-root-dir>, # NOTES: Use `.` as <project-basedir>
# This file must be present on container startup # HOST_LOG_DIR=
# HOST_LOG_FILE=
################################################################################ ################################################################################
@@ -91,8 +86,9 @@ MAIL_USERNAME=
# Flask-SQLAlchemy # # Flask-SQLAlchemy #
# https://flask-sqlalchemy.palletsprojects.com/en/2.x/config/ # # https://flask-sqlalchemy.palletsprojects.com/en/2.x/config/ #
################################################################################ ################################################################################
# DEFAULT: 'sqlite:///<nopaque-root-dir>/app.db' # DEFAULT: 'sqlite:///<nopaque-basedir>/app.db'
# NOTE: Use `.` as <nopaque-root-dir> # NOTE: Use `.` as <nopaque-basedir>,
# Don't use a SQLite database when using Docker
# SQLALCHEMY_DATABASE_URI= # SQLALCHEMY_DATABASE_URI=
@@ -125,17 +121,21 @@ NOPAQUE_SOCKETIO_MESSAGE_QUEUE_URI=
# DEFAULT: %Y-%m-%d %H:%M:%S # DEFAULT: %Y-%m-%d %H:%M:%S
# NOPAQUE_LOG_DATE_FORMAT= # NOPAQUE_LOG_DATE_FORMAT=
# DEFAULT: <nopaque-root-dir>/nopaque.log # DEFAULT: <nopaque-basedir>/logs
# NOTE: Use `.` as <nopaque-root-dir> # NOTE: Use `.` as <nopaque-basedir>
# NOPAQUE_LOG_FILE= # NOPAQUE_LOG_DIR=
# DEFAULT: [%(asctime)s] %(levelname)s in %(pathname)s (function: %(funcName)s, line: %(lineno)d): %(message)s # DEFAULT: [%(asctime)s] %(levelname)s in %(pathname)s (function: %(funcName)s, line: %(lineno)d): %(message)s
# NOPAQUE_LOG_FORMAT= # NOPAQUE_LOG_FORMAT=
# DEFAULT: WARNING # DEFAULT: INFO
# CHOOSE ONE: CRITICAL, ERROR, WARNING, INFO, DEBUG # CHOOSE ONE: CRITICAL, ERROR, WARNING, INFO, DEBUG
# NOPAQUE_LOG_LEVEL= # NOPAQUE_LOG_LEVEL=
# CHOOSE ONE: False, True
# DEFAULT: False
# NOPAQUE_LOG_TO_STDOUT=
# DEFAULT: 0 # DEFAULT: 0
# Number of values to trust for X-Forwarded-For # Number of values to trust for X-Forwarded-For
# NOPAQUE_PROXY_FIX_X_FOR= # NOPAQUE_PROXY_FIX_X_FOR=

4
.gitignore vendored
View File

@@ -18,8 +18,8 @@ data/**
# Installer logs # Installer logs
pip-log.txt pip-log.txt
# Log files # Log folder
*.log logs
# Packages # Packages
*.egg *.egg

4
app/__init__.py
View File

@@ -33,7 +33,9 @@ def create_app(config_class=Config):
migrate.init_app(app, db) migrate.init_app(app, db)
paranoid.init_app(app) paranoid.init_app(app)
socketio.init_app( socketio.init_app(
app, message_queue=app.config['NOPAQUE_SOCKETIO_MESSAGE_QUEUE_URI']) app,
message_queue=app.config.get('NOPAQUE_SOCKETIO_MESSAGE_QUEUE_URI')
)
from .events import socketio as socketio_events from .events import socketio as socketio_events
from .events import sqlalchemy as sqlalchemy_events from .events import sqlalchemy as sqlalchemy_events

84
config.py
View File

@@ -1,4 +1,5 @@
from dotenv import load_dotenv from dotenv import load_dotenv
from logging.handlers import RotatingFileHandler
from werkzeug.middleware.proxy_fix import ProxyFix from werkzeug.middleware.proxy_fix import ProxyFix
import logging import logging
import os import os
@@ -50,31 +51,66 @@ class Config:
NOPAQUE_SOCKETIO_MESSAGE_QUEUE_URI = \ NOPAQUE_SOCKETIO_MESSAGE_QUEUE_URI = \
os.environ.get('NOPAQUE_SOCKETIO_MESSAGE_QUEUE_URI') os.environ.get('NOPAQUE_SOCKETIO_MESSAGE_QUEUE_URI')
NOPAQUE_LOG_DATE_FORMAT = os.environ.get('NOPAQUE_LOG_DATE_FORMAT',
'%Y-%m-%d %H:%M:%S')
NOPAQUE_LOG_DIR = os.environ.get('NOPAQUE_LOG_DIR',
os.path.join(basedir, 'logs'))
NOPAQUE_LOG_FORMAT = os.environ.get(
'NOPAQUE_LOG_DATE_FORMAT',
'[%(asctime)s] %(levelname)s in '
'%(pathname)s (function: %(funcName)s, line: %(lineno)d): %(message)s'
)
NOPAQUE_LOG_LEVEL = os.environ.get('NOPAQUE_LOG_LEVEL', 'INFO')
NOPAQUE_LOG_TO_STDOUT = \
os.environ.get('NOPAQUE_LOG_TO_STDOUT', 'false').lower() == 'true'
NOPAQUE_PROXY_FIX_X_FOR = \
int(os.environ.get('NOPAQUE_PROXY_FIX_X_FOR', '0'))
NOPAQUE_PROXY_FIX_X_HOST = \
int(os.environ.get('NOPAQUE_PROXY_FIX_X_HOST', '0'))
NOPAQUE_PROXY_FIX_X_PORT = \
int(os.environ.get('NOPAQUE_PROXY_FIX_X_PORT', '0'))
NOPAQUE_PROXY_FIX_X_PREFIX = \
int(os.environ.get('NOPAQUE_PROXY_FIX_X_PREFIX', '0'))
NOPAQUE_PROXY_FIX_X_PROTO = \
int(os.environ.get('NOPAQUE_PROXY_FIX_X_PROTO', '0'))
@classmethod @classmethod
def init_app(cls, app): def init_app(cls, app):
# Set up logging according to the corresponding (NOPAQUE_LOG_*) # Set up logging according to the corresponding (NOPAQUE_LOG_*)
# environment variables # configurations
basic_config_kwargs = { # ... But first remove all existing handlers
'datefmt': os.environ.get('NOPAQUE_LOG_DATE_FORMAT', for handler in app.logger.handlers:
'%Y-%m-%d %H:%M:%S'), app.logger.removeHandler(handler)
'filename': os.environ.get('NOPAQUE_LOG_FILE', formatter = logging.Formatter(
os.path.join(basedir, 'nopaque.log')), fmt=app.config.get('NOPAQUE_LOG_FORMAT'),
'format': os.environ.get( datefmt=app.config.get('NOPAQUE_LOG_DATE_FORMAT')
'NOPAQUE_LOG_FORMAT', )
'[%(asctime)s] %(levelname)s in ' if app.config.get('NOPAQUE_LOG_TO_STDOUT'):
'%(pathname)s (function: %(funcName)s, line: %(lineno)d): ' stream_handler = logging.StreamHandler()
'%(message)s' stream_handler.setFormatter(formatter)
), stream_handler.setLevel(app.config.get('NOPAQUE_LOG_LEVEL'))
'level': os.environ.get('NOPAQUE_LOG_LEVEL', 'WARNING') app.logger.addHandler(stream_handler)
} else:
logging.basicConfig(**basic_config_kwargs) if not os.path.exists(app.config.get('NOPAQUE_LOG_DIR')):
os.mkdir(app.config.get('NOPAQUE_LOG_DIR'))
rotating_file_handler = RotatingFileHandler(
os.path.join(app.config.get('NOPAQUE_LOG_DIR'), 'nopaque.log'),
maxBytes=10240,
backupCount=10
)
rotating_file_handler.setFormatter(formatter)
rotating_file_handler.setLevel(app.config.get('NOPAQUE_LOG_LEVEL'))
app.logger.addHandler(rotating_file_handler)
app.logger.setLevel(app.config.get('NOPAQUE_LOG_LEVEL'))
# Set up and apply the ProxyFix middleware according to the # Set up and apply the ProxyFix middleware according to the
# corresponding (NOPAQUE_PROXY_FIX_*) environment variables # corresponding (NOPAQUE_PROXY_FIX_*) configurations
proxy_fix_kwargs = { app.wsgi_app = ProxyFix(
'x_for': int(os.environ.get('NOPAQUE_PROXY_FIX_X_FOR', '0')), app.wsgi_app,
'x_host': int(os.environ.get('NOPAQUE_PROXY_FIX_X_HOST', '0')), x_for=app.config.get('NOPAQUE_PROXY_FIX_X_FOR'),
'x_port': int(os.environ.get('NOPAQUE_PROXY_FIX_X_PORT', '0')), x_host=app.config.get('NOPAQUE_PROXY_FIX_X_HOST'),
'x_prefix': int(os.environ.get('NOPAQUE_PROXY_FIX_X_PREFIX', '0')), x_port=app.config.get('NOPAQUE_PROXY_FIX_X_PORT'),
'x_proto': int(os.environ.get('NOPAQUE_PROXY_FIX_X_PROTO', '0')) x_prefix=app.config.get('NOPAQUE_PROXY_FIX_X_PREFIX'),
} x_proto=app.config.get('NOPAQUE_PROXY_FIX_X_PROTO')
app.wsgi_app = ProxyFix(app.wsgi_app, **proxy_fix_kwargs) )

6
docker-compose.yml
View File

@@ -6,13 +6,13 @@ services:
image: postgres:11 image: postgres:11
restart: unless-stopped restart: unless-stopped
volumes: volumes:
- "${HOST_DB_DIR:-./data/db}:/var/lib/postgresql/data" - "${HOST_DATA_DIR:-./data}/db:/var/lib/postgresql/data"
mq: mq:
image: redis:6 image: redis:6
restart: unless-stopped restart: unless-stopped
volumes: volumes:
- "${HOST_MQ_DIR:-./data/mq}:/data" - "${HOST_DATA_DIR:-./data}/mq:/data"
nopaque: nopaque:
build: build:
@@ -30,4 +30,4 @@ services:
volumes: volumes:
- "/var/run/docker.sock:/var/run/docker.sock" - "/var/run/docker.sock:/var/run/docker.sock"
- "${NOPAQUE_DATA_DIR:-/mnt/nopaque}:${NOPAQUE_DATA_DIR:-/mnt/nopaque}" - "${NOPAQUE_DATA_DIR:-/mnt/nopaque}:${NOPAQUE_DATA_DIR:-/mnt/nopaque}"
- "${HOST_NOPAQUE_LOG_FILE-./nopaque.log}:${NOPAQUE_LOG_FILE:-/home/nopaque/nopaque.log}" - "${HOST_LOG_DIR-./logs}:${NOPAQUE_LOG_DIR:-/home/nopaque/logs}"

2
nopaque.py
View File

@@ -4,7 +4,7 @@ import eventlet
eventlet.monkey_patch() eventlet.monkey_patch()
from app import db, cli, create_app, socketio # noqa from app import db, cli, create_app # noqa
from app.models import (Corpus, CorpusFile, Job, JobInput, JobResult, from app.models import (Corpus, CorpusFile, Job, JobInput, JobResult,
QueryResult, Role, User) # noqa QueryResult, Role, User) # noqa