diff --git a/README.md b/README.md index 91473370..b418de52 100644 --- a/README.md +++ b/README.md @@ -16,22 +16,14 @@ The generated computational workload is handled by a [Docker](https://docs.docke ### **Create network storage** -A shared network space is necessary so that all swarm members have access to all the data. To achieve this a [samba](https://www.samba.org/) share is used. -``` bash -# Example: Create a Samba share via Docker -# More details can be found under https://hub.docker.com/r/dperson/samba/ -username@hostname:~$ sudo mkdir -p /srv/samba/nopaque -username@hostname:~$ docker run \ - --name opaque_storage \ - -v /srv/samba/nopaque:/srv/samba/nopaque \ - -p 139:139 \ - -p 445:445 \ - dperson/samba \ - -p -r -s "nopaque;/srv/samba/nopaque;no;no;no;nopaque" -u "nopaque;nopaque" +A shared network space is necessary so that all swarm members have access to all the data. To achieve this a [samba](https://www.samba.org/) share can be used. +You can create a samba share by using [this](https://hub.docker.com/r/dperson/samba/) Docker image. + +``` bash # Mount the Samba share on all swarm nodes (managers and workers) username@hostname:~$ sudo mkdir /mnt/nopaque -username@hostname:~$ sudo mount --types cifs --options gid=${USER},password=nopaque,uid=${USER},user=nopaque,vers=3.0 ///nopaque /mnt/nopaque +username@hostname:~$ sudo mount --types cifs --options gid=${USER},password=nopaque,uid=${USER},user=nopaque,vers=3.0 /// /mnt/nopaque ``` ### **Download, configure and build nopaque** @@ -39,8 +31,8 @@ username@hostname:~$ sudo mount --types cifs --options gid=${USER},password=nopa ``` bash # Clone the nopaque repository username@hostname:~$ git clone https://gitlab.ub.uni-bielefeld.de/sfb1288inf/nopaque.git -# Create data directories for the database and message queue -username@hostname:~$ mkdir data/{db,mq} +# Create data directories +username@hostname:~$ mkdir data/{db,logs,mq} username@hostname:~$ cp db.env.tpl db.env username@hostname:~$ cp .env.tpl .env # Fill out the variables within these files. @@ -56,8 +48,6 @@ username@hostname:~$ docker-compose build ### Start your instance ``` bash -# Create log files -touch nopaque.log nopaqued.log # For background execution add the -d flag username@hostname:~$ docker-compose up # To scale your app use the following command after starting it normally @@ -65,5 +55,7 @@ username@hostname:~$ docker-compose -f docker-compose.yml \ -f docker-compose.override.yml -f docker-compose.scale.yml up - -d --no-recreate --scale nopaque= + -d + --no-recreate + --scale nopaque= ``` diff --git a/app/__init__.py b/app/__init__.py index 0b14dc23..457c0934 100644 --- a/app/__init__.py +++ b/app/__init__.py @@ -10,21 +10,23 @@ from hashids import Hashids import flask_assets -assets = flask_assets.Environment() -db = SQLAlchemy() -hashids = Hashids(min_length=32) # , salt=current_app.config.get('SECRET_KEY') -login = LoginManager() -login.login_view = 'auth.login' -login.login_message = 'Please log in to access this page.' -mail = Mail() -migrate = Migrate() -paranoid = Paranoid() -paranoid.redirect_view = '/' -socketio = SocketIO() +assets: flask_assets.Environment = flask_assets.Environment() +db: SQLAlchemy = SQLAlchemy() +# TODO: Add 'SECRET_KEY' from as 'salt' kwarg +hashids: Hashids = Hashids(min_length=32) +login: LoginManager = LoginManager() +login.login_view: str = 'auth.login' +login.login_message: str = 'Please log in to access this page.' +mail: Mail = Mail() +migrate: Migrate = Migrate() +paranoid: Paranoid = Paranoid() +paranoid.redirect_view: str = '/' +socketio: SocketIO = SocketIO() -def create_app(config_class=Config): - app = Flask(__name__) +def create_app(config_class: Config = Config) -> Flask: + ''' Creates an initialized Flask (WSGI Application) object. ''' + app: Flask = Flask(__name__) app.config.from_object(config_class) assets.init_app(app) @@ -35,13 +37,10 @@ def create_app(config_class=Config): migrate.init_app(app, db) paranoid.init_app(app) socketio.init_app( - app, - message_queue=app.config.get('NOPAQUE_SOCKETIO_MESSAGE_QUEUE_URI') - ) + app, message_queue=app.config['NOPAQUE_SOCKETIO_MESSAGE_QUEUE_URI']) - from .utils import HashidConverter, permission_context_processor + from .utils import HashidConverter app.url_map.converters['hashid'] = HashidConverter - app.context_processor(permission_context_processor) from .events import socketio as socketio_events from .events import sqlalchemy as sqlalchemy_events diff --git a/app/cli.py b/app/cli.py index 17720f58..d885ff12 100644 --- a/app/cli.py +++ b/app/cli.py @@ -6,7 +6,7 @@ from flask_migrate import upgrade def register(app): @app.cli.command() def deploy(): - """Run deployment tasks.""" + ''' Run deployment tasks. ''' # migrate database to latest revision upgrade() # create or update user roles @@ -14,27 +14,29 @@ def register(app): @app.cli.group() def daemon(): - """Daemon commands.""" + ''' Daemon commands. ''' pass @daemon.command('run') def run_daemon(): - """Run daemon""" + ''' Run daemon ''' + corpus: Corpus for corpus in Corpus.query.filter(Corpus.num_analysis_sessions > 0): corpus.num_analysis_sessions = 0 db.session.commit() from app.daemon import Daemon - daemon = Daemon() + daemon: Daemon = Daemon() daemon.run() @app.cli.group() def test(): - """Test commands.""" + ''' Test commands. ''' pass @test.command('run') def run_test(): - """Run unit tests.""" - import unittest - tests = unittest.TestLoader().discover('tests') - unittest.TextTestRunner(verbosity=2).run(tests) + ''' Run unit tests. ''' + from unittest import TestLoader, TextTestRunner + from unittest.suite import TestSuite + tests: TestSuite = TestLoader().discover('tests') + TextTestRunner(verbosity=2).run(tests) diff --git a/app/email.py b/app/email.py index 53f82ac6..1fb830c0 100644 --- a/app/email.py +++ b/app/email.py @@ -1,17 +1,25 @@ from flask import current_app, render_template from flask_mail import Message +from typing import Any, Text from . import mail from .decorators import background -def create_message(recipient, subject, template, **kwargs): - msg = Message('{} {}'.format(current_app.config['NOPAQUE_MAIL_SUBJECT_PREFIX'], subject), recipients=[recipient]) # noqa - msg.body = render_template('{}.txt.j2'.format(template), **kwargs) - msg.html = render_template('{}.html.j2'.format(template), **kwargs) +def create_message( + recipient: str, + subject: str, + template: str, + **kwargs: Any +) -> Message: + subject_prefix: str = current_app.config['NOPAQUE_MAIL_SUBJECT_PREFIX'] + msg: Message = Message( + f'{subject_prefix} {subject}', recipients=[recipient]) + msg.body: Text = render_template(f'{template}.txt.j2', **kwargs) + msg.html: Text = render_template(f'{template}.html.j2', **kwargs) return msg @background -def send(msg, *args, **kwargs): +def send(msg: Message, *args, **kwargs): with kwargs['app'].app_context(): mail.send(msg) diff --git a/app/utils.py b/app/utils.py index 04ca4a45..75d38b7c 100644 --- a/app/utils.py +++ b/app/utils.py @@ -1,15 +1,10 @@ from app import hashids from werkzeug.routing import BaseConverter -from .models import Permission class HashidConverter(BaseConverter): - def to_python(self, value): + def to_python(self, value: str) -> int: return hashids.decode(value)[0] - def to_url(self, value): + def to_url(self, value: int) -> str: return hashids.encode(value) - - -def permission_context_processor(): - return {'Permission': Permission} diff --git a/nopaque.py b/nopaque.py index 30324bb9..0045d02b 100644 --- a/nopaque.py +++ b/nopaque.py @@ -6,21 +6,33 @@ eventlet.monkey_patch() from app import db, cli, create_app # noqa from app.models import (Corpus, CorpusFile, Job, JobInput, JobResult, - QueryResult, Role, User) # noqa + Permission, QueryResult, Role, User) # noqa +from flask import Flask # noqa +from typing import Any, Dict # noqa -app = create_app() +app: Flask = create_app() cli.register(app) +@app.context_processor +def make_context() -> Dict[str, Any]: + ''' Adds variables to the template context. ''' + return {'Permission': Permission} + + @app.shell_context_processor -def make_shell_context(): - return {'Corpus': Corpus, - 'CorpusFile': CorpusFile, - 'db': db, - 'Job': Job, - 'JobInput': JobInput, - 'JobResult': JobResult, - 'QueryResult': QueryResult, - 'Role': Role, - 'User': User} +def make_shell_context() -> Dict[str, Any]: + ''' Adds variables to the shell context. ''' + return { + 'Corpus': Corpus, + 'CorpusFile': CorpusFile, + 'db': db, + 'Job': Job, + 'JobInput': JobInput, + 'JobResult': JobResult, + 'Permission': Permission, + 'QueryResult': QueryResult, + 'Role': Role, + 'User': User + }