Compare commits

...

5 Commits

Author SHA1 Message Date
Patrick Jentsch
81c6f32a35 Simplify logging configuration 2024-08-01 16:29:06 +02:00
Patrick Jentsch
94548ac30c Move sheduler start logic 2024-08-01 12:10:33 +02:00
Patrick Jentsch
158190de1a Codesstyle enhancements 2024-08-01 12:00:52 +02:00
Patrick Jentsch
13e4d461c7 Update .env.tpl 2024-08-01 12:00:34 +02:00
Patrick Jentsch
e51dcafa6f Update vscode settings.json 2024-08-01 11:59:50 +02:00
12 changed files with 160 additions and 188 deletions

View File

@ -1,32 +1,20 @@
##############################################################################
# Variables for use in Docker Compose YAML files #
# Environment variables used by Docker Compose config files. #
##############################################################################
# HINT: Use this bash command `id -u`
# NOTE: 0 (= root user) is not allowed
HOST_UID=
# HINT: Use this bash command `id -g`
# NOTE: 0 (= root group) is not allowed
HOST_GID=
# HINT: Use this bash command `getent group docker | cut -d: -f3`
HOST_DOCKER_GID=
# DEFAULT: nopaque
# DOCKER_DEFAULT_NETWORK_NAME=
# DEFAULT: ./volumes/db/data
# NOTE: Use `.` as <project-basedir>
# DOCKER_DB_SERVICE_DATA_VOLUME_SOURCE_PATH=
# DEFAULT: ./volumes/mq/data
# NOTE: Use `.` as <project-basedir>
# DOCKER_MQ_SERVICE_DATA_VOLUME_SOURCE_PATH=
NOPAQUE_DOCKER_NETWORK_NAME=nopaque
# NOTE: This must be a network share and it must be available on all
# Docker Swarm nodes, mounted to the same path with the same
# user and group ownership.
DOCKER_NOPAQUE_SERVICE_DATA_VOLUME_SOURCE_PATH=
# DEFAULT: ./volumes/nopaque/logs
# NOTE: Use `.` as <project-basedir>
# DOCKER_NOPAQUE_SERVICE_LOGS_VOLUME_SOURCE_PATH=.
# Docker Swarm nodes, mounted to the same path.
HOST_NOPAQUE_DATA_PATH=/mnt/nopaque

18
.vscode/settings.json vendored
View File

@ -1,19 +1,7 @@
{
"editor.rulers": [79],
"editor.tabSize": 2,
"files.insertFinalNewline": true,
"[css]": {
"editor.tabSize": 2
},
"[html]": {
"editor.tabSize": 2
},
"[javascript]": {
"editor.tabSize": 2
},
"[jinja-html]": {
"editor.tabSize": 2
},
"[scss]": {
"editor.tabSize": 2
}
"files.trimFinalNewlines": true,
"files.trimTrailingWhitespace": true
}

View File

@ -2,7 +2,9 @@ from flask import current_app
from flask_migrate import upgrade
from pathlib import Path
from typing import List
from app import db
from app.models import (
Corpus,
CorpusFollowerRole,
Role,
SpaCyNLPPipelineModel,
@ -15,7 +17,7 @@ from . import bp
@bp.cli.command('deploy')
def deploy():
''' Run deployment tasks. '''
# Make default directories
print('Make default directories')
base_dir = current_app.config['NOPAQUE_DATA_DIR']
default_dirs: List[Path] = [
@ -28,11 +30,9 @@ def deploy():
if not default_dir.is_dir():
raise NotADirectoryError(f'{default_dir} is not a directory')
# migrate database to latest revision
print('Migrate database to latest revision')
upgrade()
# Insert/Update default database values
print('Insert/Update default Roles')
Role.insert_defaults()
print('Insert/Update default Users')
@ -44,4 +44,9 @@ def deploy():
print('Insert/Update default TesseractOCRPipelineModels')
TesseractOCRPipelineModel.insert_defaults()
print('Stop running analysis sessions')
for corpus in Corpus.query.filter(Corpus.num_analysis_sessions > 0).all():
corpus.num_analysis_sessions = 0
db.session.commit()
# TODO: Implement checks for if the nopaque network exists

View File

@ -20,6 +20,7 @@ class Config:
APIFAIRY_UI_PATH = '/api'
''' # Flask # '''
APPLICATION_ROOT = os.environ.get('APPLICATION_ROOT', '/')
PREFERRED_URL_SCHEME = os.environ.get('PREFERRED_URL_SCHEME', 'http')
SECRET_KEY = os.environ.get('SECRET_KEY', 'hard to guess string')
SERVER_NAME = os.environ.get('SERVER_NAME', 'localhost:5000')
@ -34,7 +35,7 @@ class Config:
''' # Flask-Hashids '''
HASHIDS_MIN_LENGTH = int(os.environ.get('HASHIDS_MIN_LENGTH', '16'))
HASHIDS_SALT=os.environ.get('HASHIDS_SALT', 'hard to guess string')
HASHIDS_SALT = os.environ.get('HASHIDS_SALT', 'hard to guess string')
''' # Flask-Login # '''
REMEMBER_COOKIE_SECURE = \
@ -85,17 +86,15 @@ class Config:
'[%(asctime)s] %(levelname)s in '
'%(pathname)s (function: %(funcName)s, line: %(lineno)d): %(message)s'
)
NOPAQUE_LOG_LEVEL = os.environ.get('NOPAQUE_LOG_LEVEL', 'INFO')
NOPAQUE_LOG_FILE_ENABLED = \
os.environ.get('NOPAQUE_LOG_FILE_ENABLED', 'true').lower() == 'true'
NOPAQUE_LOG_FILE_DIR = \
os.environ.get('NOPAQUE_LOGS_PATH', os.path.join(basedir, 'logs'))
os.environ.get('NOPAQUE_LOG_FILE_ENABLED', 'false').lower() == 'true'
NOPAQUE_LOG_FILE_DIR = Path(os.environ.get('NOPAQUE_LOG_FILE_DIR', '/var/log/nopaque'))
NOPAQUE_LOG_FILE_LEVEL = \
os.environ.get('NOPAQUE_LOG_FILE_LEVEL', NOPAQUE_LOG_LEVEL)
os.environ.get('NOPAQUE_LOG_FILE_LEVEL', None)
NOPAQUE_LOG_STDERR_ENABLED = \
os.environ.get('NOPAQUE_LOG_STDERR_ENABLED', 'false').lower() == 'true'
os.environ.get('NOPAQUE_LOG_STDERR_ENABLED', 'true').lower() == 'true'
NOPAQUE_LOG_STDERR_LEVEL = \
os.environ.get('NOPAQUE_LOG_STDERR_LEVEL', NOPAQUE_LOG_LEVEL)
os.environ.get('NOPAQUE_LOG_STDERR_LEVEL', None)
NOPAQUE_PROXY_FIX_ENABLED = \
os.environ.get('NOPAQUE_PROXY_FIX_ENABLED', 'false').lower() == 'true'
@ -119,42 +118,35 @@ class Config:
@staticmethod
def init_app(app: Flask):
# Set up logging according to the corresponding (NOPAQUE_LOG_*)
# configurations
app.logger.setLevel(app.config['NOPAQUE_LOG_LEVEL'])
# Remove existing handlers
for handler in app.logger.handlers:
app.logger.removeHandler(handler)
# Setup handlers
formatter = logging.Formatter(
log_formatter = logging.Formatter(
fmt=app.config['NOPAQUE_LOG_FORMAT'],
datefmt=app.config['NOPAQUE_LOG_DATE_FORMAT']
)
if app.config['NOPAQUE_IS_PRIMARY_INSTANCE']:
app.config['JOBS'].append(
{
"id": "daemon",
"func": "app.daemon:daemon",
"args": (app,),
"trigger": "interval",
"seconds": 3,
}
)
if app.config['NOPAQUE_LOG_STDERR_ENABLED']:
log_stderr_level: str | None = app.config['NOPAQUE_LOG_STDERR_LEVEL']
stream_handler = logging.StreamHandler()
stream_handler.setFormatter(formatter)
stream_handler.setLevel(app.config['NOPAQUE_LOG_STDERR_LEVEL'])
stream_handler.setFormatter(log_formatter)
if log_stderr_level is not None:
stream_handler.setLevel(log_stderr_level)
app.logger.addHandler(stream_handler)
if app.config['NOPAQUE_LOG_FILE_ENABLED']:
if not os.path.exists(app.config['NOPAQUE_LOG_FILE_DIR']):
os.mkdir(app.config['NOPAQUE_LOG_FILE_DIR'])
log_file_dir: Path = app.config['NOPAQUE_LOG_FILE_DIR']
log_file_level: str | None = app.config['NOPAQUE_LOG_FILE_LEVEL']
if not log_file_dir.exists():
log_file_dir.mkdir()
rotating_file_handler = RotatingFileHandler(
os.path.join(app.config['NOPAQUE_LOG_FILE_DIR'], 'nopaque.log'), # noqa
log_file_dir / 'nopaque.log',
maxBytes=10_240,
backupCount=10
)
rotating_file_handler.setFormatter(formatter)
rotating_file_handler.setLevel(app.config['NOPAQUE_LOG_FILE_LEVEL']) # noqa
rotating_file_handler.setFormatter(log_formatter)
if log_file_level is not None:
rotating_file_handler.setLevel(log_file_level)
app.logger.addHandler(rotating_file_handler)
if app.config['NOPAQUE_PROXY_FIX_ENABLED']:
@ -168,3 +160,14 @@ class Config:
x_prefix=app.config['NOPAQUE_PROXY_FIX_X_PREFIX'],
x_proto=app.config['NOPAQUE_PROXY_FIX_X_PROTO']
)
if app.config['NOPAQUE_IS_PRIMARY_INSTANCE']:
app.config['JOBS'].append(
{
"id": "daemon",
"func": "app.daemon:daemon",
"args": (app,),
"trigger": "interval",
"seconds": 3,
}
)

View File

@ -1,5 +1,5 @@
##############################################################################
# Environment variables to configure the db service in docker-compose.yml. #
# Environment variables used by the Docker db service. #
# #
# More information about the environment variables can be found here: #
# https://hub.docker.com/_/postgres #

View File

@ -1,62 +1,55 @@
# The docker-compose.yml file is not meant to be modified itself.
# Instead use the following files for configurations:
# - .env: Environment variables for the docker-compose.yml file.
# - db.env: Environment variables for the database service.
# - nopaque.env: Environment variables for the nopaque service.
# - docker-compose.override.yml: Override the docker-compose.yml file.
# - Don't change too much here, it's meant for configurations like exposing
# ports for development or adding labels for e.g. traefik.
# This file is not meant to be modified, use the following files instead:
# - `.env`: Environment variables available within Docker Compose config files.
# - `db.env`: Environment variables for the database service.
# - `nopaque.env`: Environment variables for the nopaque service.
# - `docker-compose.override.yml`: Override the docker-compose.yml file.
# - The `docker-compose` directory includes examples for this.
networks:
default:
name: "${DOCKER_DEFAULT_NETWORK_NAME:-nopaque}"
nopaque:
attachable: true
driver: "overlay"
name: "${NOPAQUE_DOCKER_NETWORK_NAME}"
services:
db:
env_file: db.env
image: postgres:11
restart: unless-stopped
env_file: "db.env"
image: "postgres:11"
networks:
- "nopaque"
restart: "unless-stopped"
volumes:
- type: bind
source: "${DOCKER_DB_SERVICE_DATA_VOLUME_SOURCE_PATH:-./volumes/db/data}"
target: "/var/lib/postgresql/data"
- "./volumes/db/data:/var/lib/postgresql/data"
mq:
image: redis:6
restart: unless-stopped
image: "redis:6"
networks:
- "nopaque"
restart: "unless-stopped"
volumes:
- type: bind
source: "${DOCKER_MQ_SERVICE_DATA_VOLUME_SOURCE_PATH:-./volumes/mq/data}"
target: "/data"
- "./volumes/mq/data:/data"
nopaque:
build: .
depends_on:
- db
- mq
- "db"
- "mq"
env_file:
- nopaque.env
- "nopaque.env"
environment:
# This section overrides the values set in the nopaque.env file. Do not
# override the environment variables in a docker-compose.override.yml
# file unless you really know what you are doing.
- NOPAQUE_UID=${HOST_UID}
- NOPAQUE_GID=${HOST_GID}
- DOCKER_GID=${HOST_DOCKER_GID}
- NOPAQUE_DATA_PATH=${DOCKER_NOPAQUE_SERVICE_DATA_VOLUME_SOURCE_PATH}
- NOPAQUE_DOCKER_NETWORK_NAME=${DOCKER_DEFAULT_NETWORK_NAME:-nopaque}
- NOPAQUE_LOGS_PATH=/home/nopaque/logs
image: nopaque:latest
restart: unless-stopped
# DANGER: Don't change the following environment variables within a
# Docker Compose config file, use the `.env` file instead.
HOST_UID: "${HOST_UID}"
HOST_GID: "${HOST_GID}"
HOST_DOCKER_GID: "${HOST_DOCKER_GID}"
NOPAQUE_DATA_PATH: "${HOST_NOPAQUE_DATA_PATH}"
NOPAQUE_DOCKER_NETWORK_NAME: "${NOPAQUE_DOCKER_NETWORK_NAME}"
image: "nopaque:latest"
networks:
- "nopaque"
restart: "unless-stopped"
volumes:
- type: bind
source: "/var/run/docker.sock"
target: "/var/run/docker.sock"
# TODO: Make this less quirky. The target path should be variable.
# In order to achieve this, a cifs volume needs to be configured.
- type: bind
source: "${DOCKER_NOPAQUE_SERVICE_DATA_VOLUME_SOURCE_PATH}"
target: "${DOCKER_NOPAQUE_SERVICE_DATA_VOLUME_SOURCE_PATH}"
- type: bind
source: "${DOCKER_NOPAQUE_SERVICE_LOGS_VOLUME_SOURCE_PATH:-./volumes/nopaque/logs}"
target: "/home/nopaque/logs"
- "/var/run/docker.sock:/var/run/docker.sock"
# DANGER: Don't change the following mount within a Docker Compose
# config file, use the `.env` file instead.
- "${HOST_NOPAQUE_DATA_PATH}:${HOST_NOPAQUE_DATA_PATH}"

View File

@ -1,7 +1,7 @@
services:
nopaque:
environment:
- FLASK_DEBUG=True
FLASK_DEBUG: "True"
ports:
- "5000:5000"
volumes:

View File

@ -1,4 +1,4 @@
services:
nopaque:
environment:
- NOPAQUE_IS_PRIMARY_INSTANCE=False
NOPAQUE_IS_PRIMARY_INSTANCE: "False"

View File

@ -6,29 +6,26 @@ networks:
services:
nopaque:
environment:
- NOPAQUE_PROXY_FIX_ENABLED=True
- NOPAQUE_PROXY_FIX_X_FOR=1
- NOPAQUE_PROXY_FIX_X_HOST=1
- NOPAQUE_PROXY_FIX_X_PORT=1
- NOPAQUE_PROXY_FIX_X_PREFIX=0
- NOPAQUE_PROXY_FIX_X_PROTO=1
NOPAQUE_PROXY_FIX_ENABLED: "True"
NOPAQUE_PROXY_FIX_X_FOR: "1"
NOPAQUE_PROXY_FIX_X_HOST: "1"
NOPAQUE_PROXY_FIX_X_PORT: "1"
NOPAQUE_PROXY_FIX_X_PREFIX: "0"
NOPAQUE_PROXY_FIX_X_PROTO: "1"
labels:
- "traefik.docker.network=traefik"
- "traefik.enable=true"
### <http> ###
# HTTP
- "traefik.http.routers.http-nopaque.entrypoints=http"
- "traefik.http.routers.http-nopaque.middlewares=redirect-to-https@file"
# Replace <nopaque-domain> with your domain
- "traefik.http.routers.http-nopaque.rule=Host(`<nopaque-domain>`)"
### </http> ###
### <https> ###
# HTTPS
- "traefik.http.routers.https-nopaque.entrypoints=https"
- "traefik.http.routers.https-nopaque.middlewares=hsts-header@file"
# Replace <nopaque-domain> with your domain
- "traefik.http.routers.https-nopaque.rule=Host(`<nopaque-domain>`)"
- "traefik.http.routers.https-nopaque.tls.certresolver=<CERTRESOLVER>"
- "traefik.http.routers.https-nopaque.tls.options=intermediate@file"
### </https> ###
networks:
- default
- traefik
- "nopaque"
- "traefik"

View File

@ -7,25 +7,48 @@ CHECK_MARK="\xE2\x9C\x93"
CROSS_MARK="\xE2\x9D\x8C"
if [[ "${NOPAQUE_UID}" == "0" ]]; then
# Check if necessary environment variables are set
if [[ -z "${HOST_DOCKER_GID}" ]]; then
echo "Environment variable \"HOST_DOCKER_GID\" not set."
exit 1
fi
if [[ -z "${HOST_UID}" ]]; then
echo "Environment variable \"HOST_UID\" not set."
exit 1
fi
if [[ -z "${HOST_GID}" ]]; then
echo "Environment variable \"HOST_GID\" not set."
exit 1
fi
# Check if the UID or GID are set to "0" (root). We want an unprivileged user.
if [[ "${HOST_UID}" == "0" ]]; then
echo -e "${RED_COLOR}${CROSS_MARK}${NO_COLOR}"
echo "Running as root is not allowed"
echo "\"0\" is not allowed for HOST_UID"
exit 1
fi
if [[ "${HOST_GID}" == "0" ]]; then
echo -e "${RED_COLOR}${CROSS_MARK}${NO_COLOR}"
echo "\"0\" is not allowed for HOST_GID"
exit 1
fi
echo "Set container UID and GIDs to match the host system..."
##############################################################################
# docker GID #
# Update docker GID #
##############################################################################
if [[ "${DOCKER_GID}" == "$(getent group docker | cut -d: -f3)" ]]; then
DOCKER_GID=$(getent group docker | cut -d: -f3)
if [[ "${DOCKER_GID}" == "${HOST_DOCKER_GID}" ]]; then
echo -n "- docker GID is already matching..."
echo -e "${GREEN_COLOR}${CHECK_MARK}${NO_COLOR}"
else
echo -n "- Updating docker GID ($(getent group docker | cut -d: -f3) -> ${DOCKER_GID})... "
groupmod --gid "${DOCKER_GID}" docker > /dev/null
echo -n "- Updating docker GID (${DOCKER_GID} -> ${HOST_DOCKER_GID})... "
groupmod --gid "${HOST_DOCKER_GID}" docker > /dev/null
if [[ "${?}" == "0" ]]; then
echo -e "${GREEN_COLOR}${CHECK_MARK}${NO_COLOR}"
else
@ -36,16 +59,16 @@ fi
##############################################################################
# nopaque GID #
# Update nopaque GID #
##############################################################################
if [[ "${NOPAQUE_GID}" == "$(id -g nopaque)" ]]; then
NOPAQUE_GID=$(id -g nopaque)
if [[ "${NOPAQUE_GID}" == "${HOST_GID}" ]]; then
echo -n "- nopaque GID is already matching..."
echo -e "${GREEN_COLOR}${CHECK_MARK}${NO_COLOR}"
else
echo -n "- Updating nopaque GID ($(id -g nopaque) -> ${NOPAQUE_GID})... "
groupmod --gid "${NOPAQUE_GID}" nopaque > /dev/null
echo -n "- Updating nopaque GID (${NOPAQUE_GID} -> ${HOST_GID})... "
groupmod --gid "${HOST_GID}" nopaque > /dev/null
if [[ "${?}" == "0" ]]; then
HAS_NOPAQUE_GID_CHANGED=true
echo -e "${GREEN_COLOR}${CHECK_MARK}${NO_COLOR}"
else
echo -e "${RED_COLOR}${CROSS_MARK}${NO_COLOR}"
@ -64,14 +87,15 @@ fi
##############################################################################
# nopaque UID #
# Update nopaque UID #
##############################################################################
if [[ "${NOPAQUE_UID}" == "$(id -u nopaque)" ]]; then
NOPAQUE_UID=$(id -u nopaque)
if [[ "${NOPAQUE_UID}" == "${HOST_UID}" ]]; then
echo -n "- nopaque UID is already matching..."
echo -e "${GREEN_COLOR}${CHECK_MARK}${NO_COLOR}"
else
echo -n "- Updating nopaque UID ($(id -u nopaque) -> ${NOPAQUE_UID})... "
usermod --uid "${NOPAQUE_UID}" nopaque > /dev/null
echo -n "- Updating nopaque UID (${NOPAQUE_UID} -> ${HOST_UID})... "
usermod --uid "${HOST_UID}" nopaque > /dev/null
if [[ "${?}" == "0" ]]; then
echo -e "${GREEN_COLOR}${CHECK_MARK}${NO_COLOR}"
else

View File

@ -1,9 +1,5 @@
##############################################################################
# Environment variables to configure the nopaque. #
# - When running nopaque with Docker Compose, these variables are set in the #
# `docker-compose.yml` file. #
# - When running nopaque without Docker, these variables are loaded by #
# nopaque in the config.py file #
# Environment variables used by nopaque. #
##############################################################################
@ -11,6 +7,9 @@
# Flask #
# https://flask.palletsprojects.com/en/1.1.x/config/ #
##############################################################################
# DEFAULT: /
# APPLICATION_ROOT=
# CHOOSE ONE: http, https
# DEFAULT: http
# PREFERRED_URL_SCHEME=
@ -112,12 +111,12 @@ SQLALCHEMY_DATABASE_URI=
NOPAQUE_ADMIN=
# DEFAULT: /mnt/nopaque
# NOTES:
# NOTES:
# - This must be a network share and it must be available on all
# Docker Swarm nodes, mounted to the same path with the same
# user and group ownership
# Docker Swarm nodes, mounted to the same path.
# - When running with Docker Compose, this gets overwritten in the
# `docker-compose.yml` file
# `docker-compose.yml` file with the value of the `HOST_NOPAQUE_DATA_PATH`
# environment variable from the `.env` file.
# NOPAQUE_DATA_PATH=
# CHOOSE ONE: False, True
@ -142,30 +141,22 @@ NOPAQUE_DOCKER_REGISTRY_PASSWORD=
# DEFAULT: [%(asctime)s] %(levelname)s in %(pathname)s (function: %(funcName)s, line: %(lineno)d): %(message)s
# NOPAQUE_LOG_FORMAT=
# DEFAULT: INFO
# CHOOSE ONE: CRITICAL, ERROR, WARNING, INFO, DEBUG
# NOPAQUE_LOG_LEVEL=
# CHOOSE ONE: False, True
# DEFAULT: True
# DEFAULT: False
# NOPAQUE_LOG_FILE_ENABLED=
# DEFAULT: <nopaque-basedir>/logs
# NOTES:
# - Use `.` as <nopaque-basedir>
# - When running with Docker Compose, this gets overwritten in the
# `docker-compose.yml` file
# NOPAQUE_LOGS_PATH=
# DEFAULT: /var/log/nopaque
# NOPAQUE_LOG_FILE_DIR=
# DEFAULT: NOPAQUE_LOG_LEVEL
# DEFAULT: DEBUG if FLASK_DEBUG == True else WARNING
# CHOOSE ONE: CRITICAL, ERROR, WARNING, INFO, DEBUG
# NOPAQUE_LOG_FILE_LEVEL=
# CHOOSE ONE: False, True
# DEFAULT: False
# DEFAULT: True
# NOPAQUE_LOG_STDERR_ENABLED=
# DEFAULT: NOPAQUE_LOG_LEVEL
# DEFAULT: DEBUG if FLASK_DEBUG == True else WARNING
# CHOOSE ONE: CRITICAL, ERROR, WARNING, INFO, DEBUG
# NOPAQUE_LOG_STDERR_LEVEL=

23
wsgi.py
View File

@ -8,20 +8,6 @@ from flask import Flask # noqa
from typing import Any, Dict # noqa
from app import create_app, db, scheduler, socketio # noqa
from app import models # noqa
# from app.models import (
# Avatar,
# Corpus,
# CorpusFile,
# CorpusFollowerAssociation,
# CorpusFollowerRole,
# Job,
# JobInput,
# JobResult,
# Role,
# TesseractOCRPipelineModel,
# SpaCyNLPPipelineModel,
# User
# ) # noqa
app: Flask = create_app()
@ -48,12 +34,9 @@ def make_shell_context() -> Dict[str, Any]:
def main():
with app.app_context():
if app.config['NOPAQUE_IS_PRIMARY_INSTANCE']:
for corpus in models.Corpus.query.filter(models.Corpus.num_analysis_sessions > 0).all():
corpus.num_analysis_sessions = 0
db.session.commit()
scheduler.start()
if app.config['NOPAQUE_IS_PRIMARY_INSTANCE']:
with app.app_context():
scheduler.start()
socketio.run(app, host='0.0.0.0')