Merge branch 'development' of gitlab.ub.uni-bielefeld.de:sfb1288inf/nopaque into development

This commit is contained in:
Stephan Porada 2020-10-23 16:11:15 +02:00
commit 21af48bc52
31 changed files with 338 additions and 265 deletions

122
.env.tpl
View File

@ -5,6 +5,10 @@
# NOTE: Use `.` as <project-root-dir>
# HOST_DB_DIR=
# DEFAULT: ./mq
# NOTE: Use `.` as <project-root-dir>
# HOST_MQ_DIR=
# Example: 999
# HINT: Use this bash command `getent group docker | cut -d: -f3`
HOST_DOCKER_GID=
@ -13,10 +17,6 @@ HOST_DOCKER_GID=
# HINT: Use this bash command `id -g`
HOST_GID=
# DEFAULT: ./mq
# NOTE: Use `.` as <project-root-dir>
# HOST_MQ_DIR=
# DEFAULT: ./nopaqued.log
# NOTES: Use `.` as <project-root-dir>,
# This file must be present on container startup
@ -33,22 +33,38 @@ HOST_UID=
################################################################################
# Database (only PostgreSQL) #
# Cookies #
################################################################################
NOPAQUE_DB_HOST=
# CHOOSE ONE: False, True
# DEFAULT: False
# HINT: Set to true if you redirect http to https
# NOPAQUE_REMEMBER_COOKIE_SECURE=
NOPAQUE_DB_NAME=
NOPAQUE_DB_PASSWORD=
# DEFAULT: 5432
# NOPAQUE_DB_PORT=
NOPAQUE_DB_USERNAME=
# CHOOSE ONE: False, True
# DEFAULT: False
# HINT: Set to true if you redirect http to https
# NOPAQUE_SESSION_COOKIE_SECURE=
################################################################################
# SMTP #
# Database #
# DATABASE_URI blueprint: #
# - dialect[+driver]://username:password@host[:port]/database #
# - sqlite is not supported #
# - values in square brackets are optional #
################################################################################
# DEFAULT: postgresql://nopaque:nopaque@db/nopaque
# NOPAQUE_DATABASE_URL=
# DEFAULT: postgresql://nopaque:nopaque@db/nopaque_dev
# NOPAQUE_DEV_DATABASE_URL=
# DEFAULT: postgresql://nopaque:nopaque@db/nopaque_test
# NOPAQUE_TEST_DATABASE_URL=
################################################################################
# Email #
################################################################################
# EXAMPLE: nopaque Admin <nopaque@example.com>
NOPAQUE_SMTP_DEFAULT_SENDER=
@ -61,12 +77,12 @@ NOPAQUE_SMTP_SERVER=
# EXAMPLE: 587
NOPAQUE_SMTP_PORT=
# CHOOSE ONE: False, True
# DEFAULT: False
# Choose one: False, True
# NOPAQUE_SMTP_USE_SSL=
# CHOOSE ONE: False, True
# DEFAULT: False
# Choose one: False, True
# NOPAQUE_SMTP_USE_TLS=
# EXAMPLE: nopaque@example.com
@ -76,78 +92,84 @@ NOPAQUE_SMTP_USERNAME=
################################################################################
# General #
################################################################################
# Example: admin.nopaque@example.com
# EXAMPLE: admin.nopaque@example.com
NOPAQUE_ADMIN_EMAIL_ADRESS=
# Example: contact.nopaque@example.com
# DEFAULT: development
# CHOOSE ONE: development, production, testing
# NOPAQUE_CONFIG=
# EXAMPLE: contact.nopaque@example.com
NOPAQUE_CONTACT_EMAIL_ADRESS=
# DEFAULT: /mnt/nopaque
# NOTE: This must be a network share and it must be available on all Docker Swarm nodes
# NOPAQUE_DATA_DIR=
# DEFAULT: False
# Choose one: False, True
# NOPAQUE_DEBUG=
# DEFAULT: localhost
# NOPAQUE_DOMAIN=
# DEFAULT: 0
# NOPAQUE_NUM_PROXIES=
# CHOOSE ONE: http, https
# DEFAULT: http
# Choose one: http, https
# NOPAQUE_PROTOCOL=
# DEFAULT: True
# Choose one: False, True
# NOPAQUE_REMEMBER_COOKIE_HTTPONLY=
# DEFAULT: False
# Choose one: False, True
# HINT: Set to true if you redirect http to https
# NOPAQUE_REMEMBER_COOKIE_SECURE=
# DEFAULT: hard to guess string
# HINT: Use this bash command `python -c "import uuid; print(uuid.uuid4().hex)"`
# NOPAQUE_SECRET_KEY=
# DEFAULT: False
# Choose one: False, True
# HINT: Set to true if you redirect http to https
# NOPAQUE_SESSION_COOKIE_SECURE=
################################################################################
# Logging #
################################################################################
# DEFAULT: <nopaqued-root-dir>/nopaqued.log ~ /home/nopaqued/nopaqued.log
# DEFAULT: /home/nopaqued/nopaqued.log ~ /home/nopaqued/nopaqued.log
# NOTE: Use `.` as <nopaqued-root-dir>
# NOPAQUE_DAEMON_LOG_FILE=
# DEFAULT: %Y-%m-%d %H:%M:%S
# NOPAQUE_LOG_DATE_FORMAT=
# DEFAULT: <nopaque-root-dir>/NOPAQUE.log ~ /home/NOPAQUE/NOPAQUE.log
# DEFAULT: <nopaque-root-dir>/nopaque.log ~ /home/nopaque/nopaque.log
# NOTE: Use `.` as <nopaque-root-dir>
# NOPAQUE_LOG_FILE=
# DEFAULT: [%(asctime)s] %(levelname)s in %(pathname)s (function: %(funcName)s, line: %(lineno)d): %(message)s
# NOPAQUE_LOG_FORMAT=
# DEFAULT: ERROR
# Choose one: CRITICAL, ERROR, WARNING, INFO, DEBUG
# DEFAULT: WARNING
# CHOOSE ONE: CRITICAL, ERROR, WARNING, INFO, DEBUG
# NOPAQUE_LOG_LEVEL=
################################################################################
# Message queue #
# MESSAGE_QUEUE_URI blueprint: #
# - transport://[userid:password]@hostname[:port]/[virtual_host] #
# - values in square brackets are optional #
################################################################################
NOPAQUE_MQ_HOST=
# DEFAULT: None
# HINT: A message queue is not required when using a single server process
# NOPAQUE_SOCKETIO_MESSAGE_QUEUE_URI=
# EXAMPLE: 6379
NOPAQUE_MQ_PORT=
# Choose one of the supported types by Flask-SocketIO
NOPAQUE_MQ_TYPE=
################################################################################
# Proxy fix #
################################################################################
# DEFAULT: 0
# Number of values to trust for X-Forwarded-For
# NOPAQUE_NUM_PROXIES_X_FOR=
# DEFAULT: 0
# Number of values to trust for X-Forwarded-Host
# NOPAQUE_NUM_PROXIES_X_HOST=
# DEFAULT: 0
# Number of values to trust for X-Forwarded-Port
# NOPAQUE_NUM_PROXIES_X_PORT=
# DEFAULT: 0
# Number of values to trust for X-Forwarded-Prefix
# NOPAQUE_NUM_PROXIES_X_PREFIX=
# DEFAULT: 0
# Number of values to trust for X-Forwarded-Proto
# NOPAQUE_NUM_PROXIES_X_PROTO=

8
.gitignore vendored
View File

@ -1,6 +1,3 @@
nopaque.log
nopaqued.log
*.py[cod]
# C extensions
@ -12,11 +9,14 @@ db
mq
# Environment files
.env
*.env
# Installer logs
pip-log.txt
# Log files
*.log
# Packages
*.egg
*.egg-info

View File

@ -27,7 +27,7 @@ username@hostname:~$ docker run \
-p 139:139 \
-p 445:445 \
dperson/samba \
-p -s "nopaque;/srv/samba/nopaque;no;no;no;nopaque" -u "nopaque;nopaque"
-p -r -s "nopaque;/srv/samba/nopaque;no;no;no;nopaque" -u "nopaque;nopaque"
# Mount the Samba share on all swarm nodes (managers and workers)
username@hostname:~$ sudo mkdir /mnt/nopaque
@ -39,9 +39,12 @@ username@hostname:~$ sudo mount --types cifs --options gid=${USER},password=nopa
``` bash
# Clone the nopaque repository
username@hostname:~$ git clone https://gitlab.ub.uni-bielefeld.de/sfb1288inf/nopaque.git
username@hostname:~$ cp db.env.tpl db.env
username@hostname:~$ cp .env.tpl .env
# Fill out the variables within this file.
# Fill out the variables within these files.
username@hostname:~$ <YOUR EDITOR> db.env
username@hostname:~$ <YOUR EDITOR> .env
# Create docker-compose.override.yml file
username@hostname:~$ touch docker-compose.override.yml
# Tweak the docker-compose.override.yml to satisfy your needs. (You can find examples in docker-compose.<example>.yml)
username@hostname:~$ <YOUR EDITOR> docker-compose.override.yml

View File

@ -14,7 +14,6 @@ RUN apt-get update \
&& apt-get install --no-install-recommends --yes \
build-essential \
libpq-dev \
wait-for-it \
&& rm -r /var/lib/apt/lists/*
@ -27,8 +26,7 @@ WORKDIR /home/nopaqued
COPY --chown=nopaqued:nopaqued [".", "."]
RUN python -m venv venv \
&& venv/bin/pip install --requirement requirements.txt \
&& mkdir logs
&& venv/bin/pip install --requirement requirements.txt
ENTRYPOINT ["./boot.sh"]

31
daemon/app/__init__.py Normal file
View File

@ -0,0 +1,31 @@
from config import config
from sqlalchemy import create_engine
from sqlalchemy.orm import scoped_session, sessionmaker
from time import sleep
import docker
import os
configuration = config[os.environ.get('NOPAQUE_CONFIG', 'development')]
configuration.init()
docker_client = docker.from_env()
engine = create_engine(configuration.SQLALCHEMY_DATABASE_URI)
Session = scoped_session(sessionmaker(bind=engine))
def run():
from .tasks.check_corpora import check_corpora
check_corpora_thread = check_corpora()
from .tasks.check_jobs import check_jobs
check_jobs_thread = check_jobs()
from .tasks.notify import notify
notify_thread = notify()
while True:
if not check_corpora_thread.is_alive():
check_corpora_thread = check_corpora()
if not check_jobs_thread.is_alive():
check_jobs_thread = check_jobs()
if not notify_thread.is_alive():
notify_thread = notify()
sleep(3)

View File

@ -1,6 +1,7 @@
from . import config, docker_client, Session
from .decorators import background
from .models import Corpus
from .. import configuration as config
from .. import docker_client, Session
from ..decorators import background
from ..models import Corpus
import docker
import logging
import os

View File

@ -1,7 +1,8 @@
from datetime import datetime
from . import config, docker_client, Session
from .decorators import background
from .models import Job, JobResult, NotificationData, NotificationEmailData
from .. import configuration as config
from .. import docker_client, Session
from ..decorators import background
from ..models import Job, JobResult, NotificationData, NotificationEmailData
import docker
import logging
import json

View File

View File

@ -1,13 +1,18 @@
from notify.notification import Notification
from notify.service import NotificationService
from sqlalchemy import asc
from . import config, Session
from .decorators import background
from .models import NotificationEmailData
from .libnotify.notification import Notification
from .libnotify.service import NotificationService
from .. import configuration as config
from .. import Session
from ..decorators import background
from ..models import NotificationEmailData
import logging
import os
import smtplib
ROOT_DIR = os.path.abspath(os.path.dirname(__file__))
@background
def notify():
session = Session()
@ -72,8 +77,10 @@ def __create_mail_notifications(notification_service, session):
'status': data.notify_status,
'time': data.creation_date,
'url': url}
txt_tmplt = 'notify/templates/notification_messages/notification.txt'
html_tmplt = 'notify/templates/notification_messages/notification.html'
txt_tmplt = os.path.join(ROOT_DIR,
'libnotify/templates/notification.txt')
html_tmplt = os.path.join(ROOT_DIR,
'libnotify/templates/notification.html')
notification.set_notification_content(subject_template,
subject_template_values_dict,
txt_tmplt,

View File

@ -1,8 +1,3 @@
#!/bin/bash
echo "Waiting for db..."
wait-for-it "${NOPAQUE_DB_HOST}:${NOPAQUE_DB_PORT:-5432}" --strict --timeout=0
echo "Waiting for nopaque..."
wait-for-it nopaque:5000 --strict --timeout=0
source venv/bin/activate
python nopaqued.py

View File

@ -2,60 +2,70 @@ import logging
import os
root_dir = os.path.abspath(os.path.dirname(__file__))
DEFAULT_DATA_DIR = os.path.join('/mnt/nopaque')
DEFAULT_DB_PORT = '5432'
DEFAULT_DOMAIN = 'localhost'
DEFAULT_LOG_DATE_FORMAT = '%Y-%m-%d %H:%M:%S'
DEFAULT_LOG_FILE = os.path.join(root_dir, 'nopaqued.log')
DEFAULT_LOG_FORMAT = ('[%(asctime)s] %(levelname)s in %(pathname)s '
'(function: %(funcName)s, line: %(lineno)d): '
'%(message)s')
DEFAULT_LOG_LEVEL = 'ERROR'
DEFAULT_MAIL_USE_SSL = 'False'
DEFAULT_MAIL_USE_TLS = 'False'
DEFAULT_PROTOCOL = 'http'
ROOT_DIR = os.path.abspath(os.path.dirname(__file__))
class Config:
''' ### Database ### '''
DB_HOST = os.environ.get('NOPAQUE_DB_HOST')
DB_NAME = os.environ.get('NOPAQUE_DB_NAME')
DB_PASSWORD = os.environ.get('NOPAQUE_DB_PASSWORD')
DB_PORT = os.environ.get('NOPAQUE_DB_PORT', DEFAULT_DB_PORT)
DB_USERNAME = os.environ.get('NOPAQUE_DB_USERNAME')
SQLALCHEMY_DATABASE_URI = 'postgresql://{}:{}@{}:{}/{}'.format(
DB_USERNAME, DB_PASSWORD, DB_HOST, DB_PORT, DB_NAME)
''' ### SMTP ### '''
''' # Email # '''
SMTP_DEFAULT_SENDER = os.environ.get('NOPAQUE_SMTP_DEFAULT_SENDER')
SMTP_PASSWORD = os.environ.get('NOPAQUE_SMTP_PASSWORD')
SMTP_PORT = os.environ.get('NOPAQUE_SMTP_PORT')
SMTP_PORT = int(os.environ.get('NOPAQUE_SMTP_PORT'))
SMTP_SERVER = os.environ.get('NOPAQUE_SMTP_SERVER')
SMTP_USERNAME = os.environ.get('NOPAQUE_SMTP_USERNAME')
SMTP_USE_SSL = os.environ.get('NOPAQUE_SMTP_USE_SSL',
DEFAULT_MAIL_USE_SSL).lower() == 'true'
SMTP_USE_TLS = os.environ.get('NOPAQUE_SMTP_USE_TLS',
DEFAULT_MAIL_USE_TLS).lower() == 'true'
SMTP_USE_SSL = os.environ.get(
'NOPAQUE_SMTP_USE_SSL', 'false').lower() == 'true'
SMTP_USE_TLS = os.environ.get(
'NOPAQUE_SMTP_USE_TLS', 'false').lower() == 'true'
''' ### General ### '''
DATA_DIR = os.environ.get('NOPAQUE_DATA_DIR', DEFAULT_DATA_DIR)
DOMAIN = os.environ.get('NOPAQUE_DOMAIN', DEFAULT_DOMAIN)
PROTOCOL = os.environ.get('NOPAQUE_PROTOCOL', DEFAULT_PROTOCOL)
''' # General # '''
DATA_DIR = os.environ.get('NOPAQUE_DATA_DIR', '/mnt/nopaque')
DOMAIN = os.environ.get('NOPAQUE_DOMAIN', 'localhost')
PROTOCOL = os.environ.get('NOPAQUE_PROTOCOL', 'http')
SECRET_KEY = os.environ.get('NOPAQUE_SECRET_KEY', 'hard to guess string')
''' ### Logging ### '''
''' # Logging # '''
LOG_DATE_FORMAT = os.environ.get('NOPAQUE_LOG_DATE_FORMAT',
DEFAULT_LOG_DATE_FORMAT)
LOG_FILE = os.environ.get('NOPAQUE_DAEMON_LOG_FILE', DEFAULT_LOG_FILE)
LOG_FORMAT = os.environ.get('NOPAQUE_LOG_FORMAT', DEFAULT_LOG_FORMAT)
LOG_LEVEL = os.environ.get('NOPAQUE_LOG_LEVEL', DEFAULT_LOG_LEVEL)
'%Y-%m-%d %H:%M:%S')
LOG_FILE = os.environ.get('NOPAQUED_LOG_FILE',
os.path.join(ROOT_DIR, 'nopaqued.log'))
LOG_FORMAT = os.environ.get(
'NOPAQUE_LOG_FORMAT',
'[%(asctime)s] %(levelname)s in '
'%(pathname)s (function: %(funcName)s, line: %(lineno)d): %(message)s'
)
LOG_LEVEL = os.environ.get('NOPAQUE_LOG_LEVEL', 'WARNING')
def init_app(self):
# Configure logging according to the corresponding (LOG_*) config
# entries
logging.basicConfig(datefmt=self.LOG_DATE_FORMAT,
filename=self.LOG_FILE,
format=self.LOG_FORMAT,
level=self.LOG_LEVEL)
@classmethod
def init(cls):
# Set up logging according to the corresponding (LOG_*) variables
logging.basicConfig(datefmt=cls.LOG_DATE_FORMAT,
filename=cls.LOG_FILE,
format=cls.LOG_FORMAT,
level=cls.LOG_LEVEL)
class DevelopmentConfig(Config):
''' # Database # '''
SQLALCHEMY_DATABASE_URI = os.environ.get(
'NOPAQUE_DEV_DATABASE_URL',
'sqlite:///' + os.path.join(ROOT_DIR, 'data-dev.sqlite')
)
class ProductionConfig(Config):
''' # Database # '''
SQLALCHEMY_DATABASE_URI = os.environ.get(
'NOPAQUE_DATABASE_URL',
'sqlite:///' + os.path.join(ROOT_DIR, 'data.sqlite')
)
class TestingConfig(Config):
''' # Database # '''
SQLALCHEMY_DATABASE_URI = os.environ.get(
'NOPAQUE_TEST_DATABASE_URL', 'sqlite://')
config = {'development': DevelopmentConfig,
'production': ProductionConfig,
'testing': TestingConfig}

View File

@ -1,23 +1,13 @@
from tasks.check_corpora import check_corpora
from tasks.check_jobs import check_jobs
from tasks.notify import notify
from time import sleep
from dotenv import load_dotenv
from app import run
import os
def nopaqued():
check_corpora_thread = check_corpora()
check_jobs_thread = check_jobs()
notify_thread = notify()
while True:
if not check_corpora_thread.is_alive():
check_corpora_thread = check_corpora()
if not check_jobs_thread.is_alive():
check_jobs_thread = check_jobs()
if not notify_thread.is_alive():
notify_thread = notify()
sleep(3)
# Load environment variables
DOTENV_FILE = os.path.join(os.path.dirname(__file__), '.env')
if os.path.exists(DOTENV_FILE):
load_dotenv(DOTENV_FILE)
if __name__ == '__main__':
nopaqued()
run()

View File

@ -1,3 +1,4 @@
docker
psycopg2
python-dotenv
SQLAlchemy

View File

@ -1,11 +0,0 @@
from config import Config
from sqlalchemy import create_engine
from sqlalchemy.orm import scoped_session, sessionmaker
import docker
config = Config()
config.init_app()
docker_client = docker.from_env()
engine = create_engine(config.SQLALCHEMY_DATABASE_URI)
Session = scoped_session(sessionmaker(bind=engine))

5
db.env.tpl Normal file
View File

@ -0,0 +1,5 @@
POSTGRES_DB_NAME=
POSTGRES_USER=
POSTGRES_PASSWORD=

View File

@ -16,10 +16,8 @@ services:
nopaqued:
volumes:
# Mount code as volumes
- "./daemon/app:/home/nopaqued/app"
- "./daemon/boot.sh:/home/nopaqued/boot.sh"
- "./daemon/config.py:/home/nopaqued/config.py"
- "./daemon/logger:/home/nopaqued/logger"
- "./daemon/nopaqued.py:/home/nopaqued/nopaqued.py"
- "./daemon/notify:/home/nopaqued/notify"
- "./daemon/requirements.txt:/home/nopaqued/requirements.txt"
- "./daemon/tasks:/home/nopaqued/tasks"

View File

@ -1,5 +1,6 @@
################################################################################
# Don't forget to set the NOPAQUE_NUM_PROXIES variable in your .env #
# Don't forget to set the proxy variables in your nopaque.env #
# Traefik sets the X_FOR, X_HOST, X_PORT and X_PROTO headers by default #
################################################################################
version: "3.5"
@ -17,13 +18,13 @@ services:
- "traefik.http.middlewares.nopaque-header.headers.customrequestheaders.X-Forwarded-Proto=http"
- "traefik.http.routers.nopaque.entrypoints=web"
- "traefik.http.routers.nopaque.middlewares=nopaque-header, redirect-to-https@file"
- "traefik.http.routers.nopaque.rule=Host(`${NOPAQUE_DOMAIN:-localhost}`)"
- "traefik.http.routers.nopaque.rule=Host(`<DOMAIN>`)"
### </http> ###
### <https> ###
- "traefik.http.middlewares.nopaque-secure-header.headers.customrequestheaders.X-Forwarded-Proto=https"
- "traefik.http.routers.nopaque-secure.entrypoints=web-secure"
- "traefik.http.routers.nopaque-secure.middlewares=hsts-header@file, nopaque-secure-header"
- "traefik.http.routers.nopaque-secure.rule=Host(`${NOPAQUE_DOMAIN:-localhost}`)"
- "traefik.http.routers.nopaque-secure.rule=Host(`<DOMAIN>`)"
- "traefik.http.routers.nopaque-secure.tls.certresolver=<CERTRESOLVER>"
- "traefik.http.routers.nopaque-secure.tls.options=intermediate@file"
### </https> ###

View File

@ -34,10 +34,7 @@ services:
- "${NOPAQUE_DATA_DIR:-/mnt/nopaque}:${NOPAQUE_DATA_DIR:-/mnt/nopaque}"
- "${HOST_NOPAQUE_DAEMON_LOG_FILE-./nopaqued.log}:${NOPAQUE_DAEMON_LOG_FILE:-/home/nopaqued/nopaqued.log}"
db:
environment:
- POSTGRES_DB_NAME=${NOPAQUE_DB_NAME}
- POSTGRES_USER=${NOPAQUE_DB_USERNAME}
- POSTGRES_PASSWORD=${NOPAQUE_DB_PASSWORD}
env_file: db.env
image: postgres:11
restart: unless-stopped
volumes:

View File

@ -29,8 +29,7 @@ WORKDIR /home/nopaque
COPY --chown=nopaque:nopaque [".", "."]
RUN python -m venv venv \
&& venv/bin/pip install --requirement requirements.txt \
&& mkdir logs
&& venv/bin/pip install --requirement requirements.txt
ENTRYPOINT ["./boot.sh"]

View File

@ -1,4 +1,4 @@
from config import Config
from config import config
from flask import Flask
from flask_login import LoginManager
from flask_mail import Mail
@ -7,7 +7,6 @@ from flask_socketio import SocketIO
from flask_sqlalchemy import SQLAlchemy
config = Config()
db = SQLAlchemy()
login_manager = LoginManager()
login_manager.login_view = 'auth.login'
@ -17,19 +16,19 @@ paranoid.redirect_view = '/'
socketio = SocketIO()
def create_app():
def create_app(config_name):
app = Flask(__name__)
app.config.from_object(config)
app.config.from_object(config[config_name])
config.init_app(app)
config[config_name].init_app(app)
db.init_app(app)
login_manager.init_app(app)
mail.init_app(app)
paranoid.init_app(app)
socketio.init_app(app, message_queue=config.SOCKETIO_MESSAGE_QUEUE_URI)
socketio.init_app(
app, message_queue=config[config_name].SOCKETIO_MESSAGE_QUEUE_URI)
from . import events
from .admin import admin as admin_blueprint
app.register_blueprint(admin_blueprint, url_prefix='/admin')
from .auth import auth as auth_blueprint

View File

@ -1,12 +1,14 @@
#!/bin/bash
echo "Waiting for db..."
wait-for-it "${NOPAQUE_DB_HOST}:${NOPAQUE_DB_PORT:-5432}" --strict --timeout=0
echo "Waiting for mq..."
wait-for-it "${NOPAQUE_MQ_HOST}:${NOPAQUE_MQ_PORT}" --strict --timeout=0
source venv/bin/activate
if [[ "$#" -eq 0 ]]; then
while true; do
flask deploy
if [[ "$?" == "0" ]]; then
break
fi
echo Deploy command failed, retrying in 5 secs...
sleep 5
done
python nopaque.py
elif [[ "$1" == "flask" ]]; then
exec ${@:1}

View File

@ -3,100 +3,107 @@ import logging
import os
root_dir = os.path.abspath(os.path.dirname(__file__))
DEFAULT_DATA_DIR = os.path.join('/mnt/nopaque')
DEFAULT_DB_PORT = '5432'
DEFAULT_DEBUG = 'False'
DEFAULT_LOG_DATE_FORMAT = '%Y-%m-%d %H:%M:%S'
DEFAULT_LOG_FILE = os.path.join(root_dir, 'nopaque.log')
DEFAULT_LOG_FORMAT = ('[%(asctime)s] %(levelname)s in %(pathname)s '
'(function: %(funcName)s, line: %(lineno)d): '
'%(message)s')
DEFAULT_LOG_LEVEL = 'ERROR'
DEFAULT_SMTP_USE_SSL = 'False'
DEFAULT_SMTP_USE_TLS = 'False'
DEFAULT_NUM_PROXIES = '0'
DEFAULT_PROTOCOL = 'http'
DEFAULT_REMEMBER_COOKIE_HTTPONLY = 'True'
DEFAULT_REMEMBER_COOKIE_SECURE = 'False'
DEFAULT_SECRET_KEY = 'hard to guess string'
DEFAULT_SESSION_COOKIE_SECURE = 'False'
ROOT_DIR = os.path.abspath(os.path.dirname(__file__))
class Config:
''' ### Database ### '''
DB_HOST = os.environ.get('NOPAQUE_DB_HOST')
DB_NAME = os.environ.get('NOPAQUE_DB_NAME')
DB_PASSWORD = os.environ.get('NOPAQUE_DB_PASSWORD')
DB_PORT = os.environ.get('NOPAQUE_DB_PORT', DEFAULT_DB_PORT)
DB_USERNAME = os.environ.get('NOPAQUE_DB_USERNAME')
SQLALCHEMY_DATABASE_URI = 'postgresql://{}:{}@{}:{}/{}'.format(
DB_USERNAME, DB_PASSWORD, DB_HOST, DB_PORT, DB_NAME)
''' # Cookies # '''
REMEMBER_COOKIE_HTTPONLY = True
REMEMBER_COOKIE_SECURE = os.environ.get(
'NOPAQUE_REMEMBER_COOKIE_SECURE', 'false').lower() == 'true'
SESSION_COOKIE_SECURE = os.environ.get(
'NOPAQUE_SESSION_COOKIE_SECURE', 'false').lower() == 'true'
''' # Database # '''
SQLALCHEMY_RECORD_QUERIES = True
SQLALCHEMY_TRACK_MODIFICATIONS = False
''' ### Email ### '''
''' # Email # '''
MAIL_DEFAULT_SENDER = os.environ.get('NOPAQUE_SMTP_DEFAULT_SENDER')
MAIL_PASSWORD = os.environ.get('NOPAQUE_SMTP_PASSWORD')
MAIL_PORT = os.environ.get('NOPAQUE_SMTP_PORT')
MAIL_PORT = int(os.environ.get('NOPAQUE_SMTP_PORT'))
MAIL_SERVER = os.environ.get('NOPAQUE_SMTP_SERVER')
MAIL_USERNAME = os.environ.get('NOPAQUE_SMTP_USERNAME')
MAIL_USE_SSL = os.environ.get('NOPAQUE_SMTP_USE_SSL',
DEFAULT_SMTP_USE_SSL).lower() == 'true'
MAIL_USE_TLS = os.environ.get('NOPAQUE_SMTP_USE_TLS',
DEFAULT_SMTP_USE_TLS).lower() == 'true'
MAIL_USE_SSL = os.environ.get(
'NOPAQUE_SMTP_USE_SSL', 'false').lower() == 'true'
MAIL_USE_TLS = os.environ.get(
'NOPAQUE_SMTP_USE_TLS', 'false').lower() == 'true'
''' ### General ### '''
''' # General # '''
ADMIN_EMAIL_ADRESS = os.environ.get('NOPAQUE_ADMIN_EMAIL_ADRESS')
CONTACT_EMAIL_ADRESS = os.environ.get('NOPAQUE_CONTACT_EMAIL_ADRESS')
DATA_DIR = os.environ.get('NOPAQUE_DATA_DIR', DEFAULT_DATA_DIR)
DEBUG = os.environ.get('NOPAQUE_DEBUG', DEFAULT_DEBUG).lower() == 'true'
NUM_PROXIES = int(os.environ.get('NOPAQUE_NUM_PROXIES',
DEFAULT_NUM_PROXIES))
PROTOCOL = os.environ.get('NOPAQUE_PROTOCOL', DEFAULT_PROTOCOL)
REMEMBER_COOKIE_HTTPONLY = os.environ.get(
'NOPAQUE_REMEMBER_COOKIE_HTTPONLY',
DEFAULT_REMEMBER_COOKIE_HTTPONLY
).lower() == 'true'
REMEMBER_COOKIE_SECURE = os.environ.get(
'NOPAQUE_REMEMBER_COOKIE_SECURE',
DEFAULT_REMEMBER_COOKIE_SECURE
).lower() == 'true'
SECRET_KEY = os.environ.get('RECIPY_SECRET_KEY', DEFAULT_SECRET_KEY)
SESSION_COOKIE_SECURE = os.environ.get(
'NOPAQUE_SESSION_COOKIE_SECURE',
DEFAULT_SESSION_COOKIE_SECURE
).lower() == 'true'
DATA_DIR = os.environ.get('NOPAQUE_DATA_DIR', '/mnt/nopaque')
SECRET_KEY = os.environ.get('NOPAQUE_SECRET_KEY', 'hard to guess string')
''' ### Logging ### '''
''' # Logging # '''
LOG_DATE_FORMAT = os.environ.get('NOPAQUE_LOG_DATE_FORMAT',
DEFAULT_LOG_DATE_FORMAT)
LOG_FILE = os.environ.get('NOPAQUE_LOG_FILE', DEFAULT_LOG_FILE)
LOG_FORMAT = os.environ.get('NOPAQUE_LOG_FORMAT', DEFAULT_LOG_FORMAT)
LOG_LEVEL = os.environ.get('NOPAQUE_LOG_LEVEL', DEFAULT_LOG_LEVEL)
'%Y-%m-%d %H:%M:%S')
LOG_FILE = os.environ.get('NOPAQUE_LOG_FILE',
os.path.join(ROOT_DIR, 'nopaque.log'))
LOG_FORMAT = os.environ.get(
'NOPAQUE_LOG_FORMAT',
'[%(asctime)s] %(levelname)s in '
'%(pathname)s (function: %(funcName)s, line: %(lineno)d): %(message)s'
)
LOG_LEVEL = os.environ.get('NOPAQUE_LOG_LEVEL', 'WARNING')
''' ### Message queue ### '''
MQ_HOST = os.environ.get('NOPAQUE_MQ_HOST')
MQ_PORT = os.environ.get('NOPAQUE_MQ_PORT')
MQ_TYPE = os.environ.get('NOPAQUE_MQ_TYPE')
SOCKETIO_MESSAGE_QUEUE_URI = \
'{}://{}:{}/'.format(MQ_TYPE, MQ_HOST, MQ_PORT)
''' # Message queue # '''
SOCKETIO_MESSAGE_QUEUE_URI = os.environ.get(
'NOPAQUE_SOCKETIO_MESSAGE_QUEUE_URI')
def init_app(self, app):
# Configure logging according to the corresponding (LOG_*) config
# entries
logging.basicConfig(datefmt=self.LOG_DATE_FORMAT,
filename=self.LOG_FILE,
format=self.LOG_FORMAT,
level=self.LOG_LEVEL)
# Apply the ProxyFix middleware if nopaque is running behind reverse
# proxies. (NUM_PROXIES indicates the number of reverse proxies running
# in front of nopaque)
if self.NUM_PROXIES > 0:
''' # Proxy fix # '''
PROXY_FIX_X_FOR = int(os.environ.get('NOPAQUE_PROXY_FIX_X_FOR', '0'))
PROXY_FIX_X_HOST = int(os.environ.get('NOPAQUE_PROXY_FIX_X_HOST', '0'))
PROXY_FIX_X_PORT = int(os.environ.get('NOPAQUE_PROXY_FIX_X_PORT', '0'))
PROXY_FIX_X_PREFIX = int(os.environ.get('NOPAQUE_PROXY_FIX_X_PREFIX', '0'))
PROXY_FIX_X_PROTO = int(os.environ.get('NOPAQUE_PROXY_FIX_X_PROTO', '0'))
@classmethod
def init_app(cls, app):
# Set up logging according to the corresponding (LOG_*) variables
logging.basicConfig(datefmt=cls.LOG_DATE_FORMAT,
filename=cls.LOG_FILE,
format=cls.LOG_FORMAT,
level=cls.LOG_LEVEL)
# Set up and apply the ProxyFix middleware according to the
# corresponding (PROXY_FIX_*) variables
app.wsgi_app = ProxyFix(app.wsgi_app,
x_for=self.NUM_PROXIES,
x_host=self.NUM_PROXIES,
x_port=self.NUM_PROXIES,
x_proto=self.NUM_PROXIES)
x_for=cls.PROXY_FIX_X_FOR,
x_host=cls.PROXY_FIX_X_HOST,
x_port=cls.PROXY_FIX_X_PORT,
x_prefix=cls.PROXY_FIX_X_PREFIX,
x_proto=cls.PROXY_FIX_X_PROTO)
class DevelopmentConfig(Config):
''' # Database # '''
SQLALCHEMY_DATABASE_URI = os.environ.get(
'NOPAQUE_DEV_DATABASE_URL',
'postgresql://nopaque:nopaque@db/nopaque_dev'
)
''' # General # '''
DEBUG = True
class ProductionConfig(Config):
''' # Database # '''
SQLALCHEMY_DATABASE_URI = os.environ.get(
'NOPAQUE_DATABASE_URL', 'postgresql://nopaque:nopaque@db/nopaque')
class TestingConfig(Config):
''' # Database # '''
SQLALCHEMY_DATABASE_URI = os.environ.get(
'NOPAQUE_TEST_DATABASE_URL',
'postgresql://nopaque:nopaque@db/nopaque_test'
)
''' # General # '''
TESTING = True
WTF_CSRF_ENABLED = False
config = {'development': DevelopmentConfig,
'production': ProductionConfig,
'testing': TestingConfig}

View File

@ -1,12 +1,28 @@
# First things first: apply monkey patch, so that no code gets executed without
# patched libraries!
import eventlet
eventlet.monkey_patch() # noqa
from app import create_app, db, socketio
eventlet.monkey_patch()
from dotenv import load_dotenv # noqa
import os # noqa
# Load environment variables
DOTENV_FILE = os.path.join(os.path.dirname(__file__), '.env')
if os.path.exists(DOTENV_FILE):
load_dotenv(DOTENV_FILE)
from app import create_app, db, socketio # noqa
from app.models import (Corpus, CorpusFile, Job, JobInput, JobResult,
NotificationData, NotificationEmailData, QueryResult,
Role, User)
from flask_migrate import Migrate, upgrade
Role, User) # noqa
from flask_migrate import Migrate, upgrade # noqa
app = create_app()
app = create_app(os.environ.get('NOPAQUE_CONFIG', 'development'))
migrate = Migrate(app, db, compare_type=True)

View File

@ -1,6 +1,5 @@
cqi
dnspython==1.16.0
email_validator
eventlet
Flask
Flask-Login
@ -11,6 +10,8 @@ Flask-SocketIO
Flask-SQLAlchemy
Flask-WTF
jsonpatch
psycopg2
redis
jsonschema
psycopg2
python-dotenv
redis
wtforms[email]