First attempts to use type hinting

This commit is contained in:
Patrick Jentsch 2021-12-07 14:18:05 +01:00
parent 9d30dda733
commit e1004a0181
6 changed files with 77 additions and 69 deletions

View File

@ -16,22 +16,14 @@ The generated computational workload is handled by a [Docker](https://docs.docke
### **Create network storage**
A shared network space is necessary so that all swarm members have access to all the data. To achieve this a [samba](https://www.samba.org/) share is used.
``` bash
# Example: Create a Samba share via Docker
# More details can be found under https://hub.docker.com/r/dperson/samba/
username@hostname:~$ sudo mkdir -p /srv/samba/nopaque
username@hostname:~$ docker run \
--name opaque_storage \
-v /srv/samba/nopaque:/srv/samba/nopaque \
-p 139:139 \
-p 445:445 \
dperson/samba \
-p -r -s "nopaque;/srv/samba/nopaque;no;no;no;nopaque" -u "nopaque;nopaque"
A shared network space is necessary so that all swarm members have access to all the data. To achieve this a [samba](https://www.samba.org/) share can be used.
You can create a samba share by using [this](https://hub.docker.com/r/dperson/samba/) Docker image.
``` bash
# Mount the Samba share on all swarm nodes (managers and workers)
username@hostname:~$ sudo mkdir /mnt/nopaque
username@hostname:~$ sudo mount --types cifs --options gid=${USER},password=nopaque,uid=${USER},user=nopaque,vers=3.0 //<SAMBA-SERVER-IP>/nopaque /mnt/nopaque
username@hostname:~$ sudo mount --types cifs --options gid=${USER},password=nopaque,uid=${USER},user=nopaque,vers=3.0 //<SAMBA-SERVER-IP>/<SAMBA-SHARE-NAME> /mnt/nopaque
```
### **Download, configure and build nopaque**
@ -39,8 +31,8 @@ username@hostname:~$ sudo mount --types cifs --options gid=${USER},password=nopa
``` bash
# Clone the nopaque repository
username@hostname:~$ git clone https://gitlab.ub.uni-bielefeld.de/sfb1288inf/nopaque.git
# Create data directories for the database and message queue
username@hostname:~$ mkdir data/{db,mq}
# Create data directories
username@hostname:~$ mkdir data/{db,logs,mq}
username@hostname:~$ cp db.env.tpl db.env
username@hostname:~$ cp .env.tpl .env
# Fill out the variables within these files.
@ -56,8 +48,6 @@ username@hostname:~$ docker-compose build
### Start your instance
``` bash
# Create log files
touch nopaque.log nopaqued.log
# For background execution add the -d flag
username@hostname:~$ docker-compose up
# To scale your app use the following command after starting it normally
@ -65,5 +55,7 @@ username@hostname:~$ docker-compose -f docker-compose.yml \
-f docker-compose.override.yml
-f docker-compose.scale.yml
up
-d --no-recreate --scale nopaque=<NUM_INSTANCES>
-d
--no-recreate
--scale nopaque=<NUM_INSTANCES>
```

View File

@ -10,21 +10,23 @@ from hashids import Hashids
import flask_assets
assets = flask_assets.Environment()
db = SQLAlchemy()
hashids = Hashids(min_length=32) # , salt=current_app.config.get('SECRET_KEY')
login = LoginManager()
login.login_view = 'auth.login'
login.login_message = 'Please log in to access this page.'
mail = Mail()
migrate = Migrate()
paranoid = Paranoid()
paranoid.redirect_view = '/'
socketio = SocketIO()
assets: flask_assets.Environment = flask_assets.Environment()
db: SQLAlchemy = SQLAlchemy()
# TODO: Add 'SECRET_KEY' from as 'salt' kwarg
hashids: Hashids = Hashids(min_length=32)
login: LoginManager = LoginManager()
login.login_view: str = 'auth.login'
login.login_message: str = 'Please log in to access this page.'
mail: Mail = Mail()
migrate: Migrate = Migrate()
paranoid: Paranoid = Paranoid()
paranoid.redirect_view: str = '/'
socketio: SocketIO = SocketIO()
def create_app(config_class=Config):
app = Flask(__name__)
def create_app(config_class: Config = Config) -> Flask:
''' Creates an initialized Flask (WSGI Application) object. '''
app: Flask = Flask(__name__)
app.config.from_object(config_class)
assets.init_app(app)
@ -35,13 +37,10 @@ def create_app(config_class=Config):
migrate.init_app(app, db)
paranoid.init_app(app)
socketio.init_app(
app,
message_queue=app.config.get('NOPAQUE_SOCKETIO_MESSAGE_QUEUE_URI')
)
app, message_queue=app.config['NOPAQUE_SOCKETIO_MESSAGE_QUEUE_URI'])
from .utils import HashidConverter, permission_context_processor
from .utils import HashidConverter
app.url_map.converters['hashid'] = HashidConverter
app.context_processor(permission_context_processor)
from .events import socketio as socketio_events
from .events import sqlalchemy as sqlalchemy_events

View File

@ -6,7 +6,7 @@ from flask_migrate import upgrade
def register(app):
@app.cli.command()
def deploy():
"""Run deployment tasks."""
''' Run deployment tasks. '''
# migrate database to latest revision
upgrade()
# create or update user roles
@ -14,27 +14,29 @@ def register(app):
@app.cli.group()
def daemon():
"""Daemon commands."""
''' Daemon commands. '''
pass
@daemon.command('run')
def run_daemon():
"""Run daemon"""
''' Run daemon '''
corpus: Corpus
for corpus in Corpus.query.filter(Corpus.num_analysis_sessions > 0):
corpus.num_analysis_sessions = 0
db.session.commit()
from app.daemon import Daemon
daemon = Daemon()
daemon: Daemon = Daemon()
daemon.run()
@app.cli.group()
def test():
"""Test commands."""
''' Test commands. '''
pass
@test.command('run')
def run_test():
"""Run unit tests."""
import unittest
tests = unittest.TestLoader().discover('tests')
unittest.TextTestRunner(verbosity=2).run(tests)
''' Run unit tests. '''
from unittest import TestLoader, TextTestRunner
from unittest.suite import TestSuite
tests: TestSuite = TestLoader().discover('tests')
TextTestRunner(verbosity=2).run(tests)

View File

@ -1,17 +1,25 @@
from flask import current_app, render_template
from flask_mail import Message
from typing import Any, Text
from . import mail
from .decorators import background
def create_message(recipient, subject, template, **kwargs):
msg = Message('{} {}'.format(current_app.config['NOPAQUE_MAIL_SUBJECT_PREFIX'], subject), recipients=[recipient]) # noqa
msg.body = render_template('{}.txt.j2'.format(template), **kwargs)
msg.html = render_template('{}.html.j2'.format(template), **kwargs)
def create_message(
recipient: str,
subject: str,
template: str,
**kwargs: Any
) -> Message:
subject_prefix: str = current_app.config['NOPAQUE_MAIL_SUBJECT_PREFIX']
msg: Message = Message(
f'{subject_prefix} {subject}', recipients=[recipient])
msg.body: Text = render_template(f'{template}.txt.j2', **kwargs)
msg.html: Text = render_template(f'{template}.html.j2', **kwargs)
return msg
@background
def send(msg, *args, **kwargs):
def send(msg: Message, *args, **kwargs):
with kwargs['app'].app_context():
mail.send(msg)

View File

@ -1,15 +1,10 @@
from app import hashids
from werkzeug.routing import BaseConverter
from .models import Permission
class HashidConverter(BaseConverter):
def to_python(self, value):
def to_python(self, value: str) -> int:
return hashids.decode(value)[0]
def to_url(self, value):
def to_url(self, value: int) -> str:
return hashids.encode(value)
def permission_context_processor():
return {'Permission': Permission}

View File

@ -6,21 +6,33 @@ eventlet.monkey_patch()
from app import db, cli, create_app # noqa
from app.models import (Corpus, CorpusFile, Job, JobInput, JobResult,
QueryResult, Role, User) # noqa
Permission, QueryResult, Role, User) # noqa
from flask import Flask # noqa
from typing import Any, Dict # noqa
app = create_app()
app: Flask = create_app()
cli.register(app)
@app.context_processor
def make_context() -> Dict[str, Any]:
''' Adds variables to the template context. '''
return {'Permission': Permission}
@app.shell_context_processor
def make_shell_context():
return {'Corpus': Corpus,
'CorpusFile': CorpusFile,
'db': db,
'Job': Job,
'JobInput': JobInput,
'JobResult': JobResult,
'QueryResult': QueryResult,
'Role': Role,
'User': User}
def make_shell_context() -> Dict[str, Any]:
''' Adds variables to the shell context. '''
return {
'Corpus': Corpus,
'CorpusFile': CorpusFile,
'db': db,
'Job': Job,
'JobInput': JobInput,
'JobResult': JobResult,
'Permission': Permission,
'QueryResult': QueryResult,
'Role': Role,
'User': User
}