From f2806f94738c26e70521de67c8778e372c813a84 Mon Sep 17 00:00:00 2001 From: Patrick Jentsch Date: Wed, 4 Sep 2019 16:00:14 +0200 Subject: [PATCH 1/7] Codestyle. --- app/templates/main/jobs/job.html.j2 | 7 +++-- app/templates/services/nlp.html.j2 | 40 ++++++----------------------- app/templates/services/ocr.html.j2 | 40 ++++++----------------------- 3 files changed, 19 insertions(+), 68 deletions(-) diff --git a/app/templates/main/jobs/job.html.j2 b/app/templates/main/jobs/job.html.j2 index a9bfd2b5..12865aa2 100644 --- a/app/templates/main/jobs/job.html.j2 +++ b/app/templates/main/jobs/job.html.j2 @@ -161,11 +161,12 @@ + Files - - + + @@ -179,8 +180,6 @@ {% for result in files[file]['results'] %} {{ result }} {% endfor %} - {% else %} - None {% endif %} diff --git a/app/templates/services/nlp.html.j2 b/app/templates/services/nlp.html.j2 index 1ba9550b..8e394f08 100644 --- a/app/templates/services/nlp.html.j2 +++ b/app/templates/services/nlp.html.j2 @@ -13,56 +13,32 @@
- - layers - Tokenisierung - -

- Aufteilung eines Textes in Sätze und Wörter. Dies - ist zur weiteren Verarbeitung notwendig. -

+ layersTokenisierung +

Aufteilung eines Textes in Sätze und Wörter. Dies ist zur weiteren Verarbeitung notwendig.

- - layers - Lemmatisierung - -

- Reduktion der Flexionsformen eines Wortes auf dessen - Grundform. -

+ layersLemmatisierung +

Reduktion der Flexionsformen eines Wortes auf dessen Grundform.

- - layers - Part-of-speech-Tagging - -

- Kontext- und definitionsbezogene Zuordnung von Wörtern - und Satzzeichen zu Wortarten. -

+ layersPart-of-speech-Tagging +

Kontext- und definitionsbezogene Zuordnung von Wörtern und Satzzeichen zu Wortarten.

- - layers - Eigennamenerkennung - -

- Identifikation von Wörtern, die eine Entität - beschreiben, wie Firmen- und Personennamen. -

+ layersEigennamenerkennung +

Identifikation von Wörtern, die eine Entitätbeschreiben, wie Firmen- und Personennamen.

diff --git a/app/templates/services/ocr.html.j2 b/app/templates/services/ocr.html.j2 index 2fc3198c..3e272898 100644 --- a/app/templates/services/ocr.html.j2 +++ b/app/templates/services/ocr.html.j2 @@ -14,56 +14,32 @@
- - layers - Eingabe von Bilddaten - -

- Über ein Auftragsformular können Bilddaten in Form von - PDF-Dateien hochgeladen werden. -

+ layersEingabe von Bilddaten +

Über ein Auftragsformular können Bilddaten in Form von PDF-Dateien hochgeladen werden.

- - layers - Optische Zeichenerkennung - -

- Die optische Zeichenerkennung erfolgt in der - Recheninfrastruktur der Plattform. -

+ layersOptische Zeichenerkennung +

Die optische Zeichenerkennung erfolgt in der Recheninfrastruktur der Plattform.

- - layers - Fehlerkorrektur - -

- Je nach Qualität der Eingabedaten kann es zu - Fehlern kommen, die korrigiert werden sollten. -

+ layersFehlerkorrektur +

Je nach Qualität der Eingabedaten kann es zu Fehlern kommen, die korrigiert werden sollten.

- - layers - Weiterverarbeitung - -

- Die Textdaten können weiterverarbeitet[*] - oder in dieser Form bereits genutzt[*] werden. -

+ layersWeiterverarbeitung +

Die Textdaten können weiterverarbeitet[*] oder in dieser Form bereits genutzt[*] werden.

From a05e37ddb6a3be77195de59028ea1afe2dc0f01e Mon Sep 17 00:00:00 2001 From: Patrick Jentsch Date: Thu, 5 Sep 2019 10:12:27 +0200 Subject: [PATCH 2/7] Add .env example file. --- .env_example | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) create mode 100644 .env_example diff --git a/.env_example b/.env_example new file mode 100644 index 00000000..979f5da7 --- /dev/null +++ b/.env_example @@ -0,0 +1,17 @@ +### Flask ### +# SECRET_KEY= + +### Flask-Mail ### +MAIL_SERVER=smtp.example.com +MAIL_PORT=587 +MAIL_USE_TLS=true +MAIL_USERNAME=username@example.com +MAIL_PASSWORD=password +MAIL_DEFAULT_SENDER=username@example.com + +### Opaque ### +OPAQUE_ADMIN=admin.opaque@example.com +OPAQUE_STORAGE_DIRECTORY=/absolut/path/to/opaque/storage/directory + +### Flask-SQLAlchemy ### +# SQLALCHEMY_DATABASE_URI= From 40809d9c7b3d68242c60f849dc519ac12b41b44c Mon Sep 17 00:00:00 2001 From: Patrick Jentsch Date: Thu, 5 Sep 2019 10:12:40 +0200 Subject: [PATCH 3/7] Change setting names. --- app/email.py | 6 +----- app/main/views.py | 10 +++++----- config.py | 43 ++++++++++++++++++++++--------------------- 3 files changed, 28 insertions(+), 31 deletions(-) diff --git a/app/email.py b/app/email.py index 1e3fb3ab..072b7bc3 100644 --- a/app/email.py +++ b/app/email.py @@ -10,11 +10,7 @@ def send_async_email(app, msg): def send_email(to, subject, template, **kwargs): - subject = '{} {}'.format(current_app.config['OPAQUE_MAIL_SUBJECT_PREFIX'], - subject) - msg = Message(subject, - sender=current_app.config['OPAQUE_MAIL_SENDER'], - recipients=[to]) + msg = Message('[Opaque] {}'.format(subject), recipients=[to]) msg.body = render_template(template + '.txt.j2', **kwargs) msg.html = render_template(template + '.html.j2', **kwargs) thr = Thread(target=send_async_email, diff --git a/app/main/views.py b/app/main/views.py index efb2b0bd..9acc9061 100644 --- a/app/main/views.py +++ b/app/main/views.py @@ -21,7 +21,7 @@ def corpus(corpus_id): print('Corpus not found.') abort(404) - dir = os.path.join(current_app.config['OPAQUE_STORAGE'], + dir = os.path.join(current_app.config['OPAQUE_STORAGE_DIRECTORY'], str(current_user.id), 'corpora', str(corpus.id)) @@ -44,7 +44,7 @@ def corpus_download(corpus_id): if not file or not corpus: print('File not found.') abort(404) - dir = os.path.join(current_app.config['OPAQUE_STORAGE'], + dir = os.path.join(current_app.config['OPAQUE_STORAGE_DIRECTORY'], str(current_user.id), 'corpora', str(corpus.id)) @@ -66,7 +66,7 @@ def dashboard(): db.session.add(corpus) db.session.commit() - dir = os.path.join(app.config['OPAQUE_STORAGE'], + dir = os.path.join(app.config['OPAQUE_STORAGE_DIRECTORY'], str(corpus.user_id), 'corpora', str(corpus.id)) @@ -96,7 +96,7 @@ def job(job_id): print('Job not found.') abort(404) - dir = os.path.join(current_app.config['OPAQUE_STORAGE'], + dir = os.path.join(current_app.config['OPAQUE_STORAGE_DIRECTORY'], str(current_user.id), 'jobs', str(job.id)) @@ -130,7 +130,7 @@ def job_download(job_id): if not file or not job: print('File not found.') abort(404) - dir = os.path.join(current_app.config['OPAQUE_STORAGE'], + dir = os.path.join(current_app.config['OPAQUE_STORAGE_DIRECTORY'], str(current_user.id), 'jobs', str(job.id)) diff --git a/config.py b/config.py index 96aaa6d1..836cdcfe 100644 --- a/config.py +++ b/config.py @@ -1,47 +1,48 @@ import os -basedir = os.path.abspath(os.path.dirname(__file__)) - - class Config: + ''' ### Flask ### ''' + SECRET_KEY = os.environ.get('SECRET_KEY') or 'hard to guess string' + + ''' ### Flask-Mail ### ''' MAIL_SERVER = os.environ.get('MAIL_SERVER') MAIL_PORT = int(os.environ.get('MAIL_PORT')) MAIL_USE_TLS = os.environ.get('MAIL_USE_TLS').lower() == 'true' MAIL_USERNAME = os.environ.get('MAIL_USERNAME') MAIL_PASSWORD = os.environ.get('MAIL_PASSWORD') - OPAQUE_ADMIN = os.environ.get('OPAQUE_ADMIN') - OPAQUE_STORAGE = os.environ.get('OPAQUE_STORAGE') - OPAQUE_MAIL_SUBJECT_PREFIX = '[Opaque]' - OPAQUE_MAIL_SENDER = 'Opaque' - SECRET_KEY = os.environ.get('SECRET_KEY') or 'hard to guess string' + MAIL_DEFAULT_SENDER = os.environ.get('MAIL_DEFAULT_SENDER') + + ''' ### Flask-SQLAlchemy ### ''' SQLALCHEMY_TRACK_MODIFICATIONS = False + ''' ### Opaque ### ''' + OPAQUE_ADMIN = os.environ.get('OPAQUE_ADMIN') + OPAQUE_STORAGE_DIRECTORY = os.environ.get('OPAQUE_STORAGE_DIRECTORY') + @staticmethod def init_app(app): pass class DevelopmentConfig(Config): + ''' ### Flask ### ''' DEBUG = True - SQLALCHEMY_DATABASE_URI = 'sqlite:///' + os.path.join(basedir, - 'data_dev.sqlite') + + ''' ### Flask-SQLAlchemy ### ''' + SQLALCHEMY_DATABASE_URI = 'sqlite:///{}'.format( + os.path.join(os.path.dirname(os.path.abspath(__file__)), + 'data_dev.sqlite') + ) -class TestingConfig(Config): - TESTING = True - SQLALCHEMY_DATABASE_URI = os.environ.get('TEST_DATABASE_URL') or \ - 'sqlite://' - WTF_CSRF_ENABLED = False - - -# class ProductionConfig(Config): +class ProductionConfig(Config): + ''' ### Flask-SQLAlchemy ### ''' + SQLALCHEMY_DATABASE_URI = os.environ.get('SQLALCHEMY_DATABASE_URI') config = { 'development': DevelopmentConfig, - 'testing': TestingConfig, - # 'production': ProductionConfig, - + 'production': ProductionConfig, 'default': DevelopmentConfig } From 78710ed58ae03160a182adb466cc8c7082e5be93 Mon Sep 17 00:00:00 2001 From: Patrick Jentsch Date: Thu, 5 Sep 2019 10:20:18 +0200 Subject: [PATCH 4/7] Add config selection to .env example. --- .env_example | 1 + 1 file changed, 1 insertion(+) diff --git a/.env_example b/.env_example index 979f5da7..e74e5b7d 100644 --- a/.env_example +++ b/.env_example @@ -1,4 +1,5 @@ ### Flask ### +FLASK_CONFIG=production # SECRET_KEY= ### Flask-Mail ### From 78499ee73158ea175868efca10e645c540cbf10c Mon Sep 17 00:00:00 2001 From: Patrick Jentsch Date: Thu, 5 Sep 2019 10:20:41 +0200 Subject: [PATCH 5/7] Remove unused (since flask-socketio) FLASK_ENV variable. --- .flaskenv | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.flaskenv b/.flaskenv index f88b6e5b..47093cda 100644 --- a/.flaskenv +++ b/.flaskenv @@ -1,2 +1,2 @@ +### Flask ### FLASK_APP=opaque.py -FLASK_ENV=development From 0f0c40951bb37e43c746f43bd79255b5a3fd2eec Mon Sep 17 00:00:00 2001 From: Patrick Jentsch Date: Thu, 5 Sep 2019 12:00:31 +0200 Subject: [PATCH 6/7] Use fix path! --- .env_example | 1 - app/services/views.py | 4 ++-- config.py | 2 +- 3 files changed, 3 insertions(+), 4 deletions(-) diff --git a/.env_example b/.env_example index e74e5b7d..f328a3ed 100644 --- a/.env_example +++ b/.env_example @@ -12,7 +12,6 @@ MAIL_DEFAULT_SENDER=username@example.com ### Opaque ### OPAQUE_ADMIN=admin.opaque@example.com -OPAQUE_STORAGE_DIRECTORY=/absolut/path/to/opaque/storage/directory ### Flask-SQLAlchemy ### # SQLALCHEMY_DATABASE_URI= diff --git a/app/services/views.py b/app/services/views.py index 96247d57..52c36084 100644 --- a/app/services/views.py +++ b/app/services/views.py @@ -26,7 +26,7 @@ def nlp(): db.session.add(nlp_job) db.session.commit() - dir = os.path.join(current_app.config['OPAQUE_STORAGE'], + dir = os.path.join(current_app.config['OPAQUE_STORAGE_DIRECTORY'], str(nlp_job.user_id), 'jobs', str(nlp_job.id)) @@ -72,7 +72,7 @@ def ocr(): db.session.add(ocr_job) db.session.commit() - dir = os.path.join(current_app.config['OPAQUE_STORAGE'], + dir = os.path.join(current_app.config['OPAQUE_STORAGE_DIRECTORY'], str(ocr_job.user_id), 'jobs', str(ocr_job.id)) diff --git a/config.py b/config.py index 836cdcfe..fd0e246c 100644 --- a/config.py +++ b/config.py @@ -18,7 +18,7 @@ class Config: ''' ### Opaque ### ''' OPAQUE_ADMIN = os.environ.get('OPAQUE_ADMIN') - OPAQUE_STORAGE_DIRECTORY = os.environ.get('OPAQUE_STORAGE_DIRECTORY') + OPAQUE_STORAGE_DIRECTORY = '/opaque_storage' @staticmethod def init_app(app): From e3db2ecd1e52b4993b91e1a1c3501c0fc775de1d Mon Sep 17 00:00:00 2001 From: Patrick Jentsch Date: Thu, 5 Sep 2019 14:42:21 +0200 Subject: [PATCH 7/7] Load environment variables in container. So the python-dotenv module is no longer needed! --- Dockerfile | 38 +++++++++++++++++++++++++------------- docker-entrypoint.sh | 13 +++++++++++++ flask-entrypoint.sh | 3 --- opaque.py | 2 -- requirements.txt | 1 - 5 files changed, 38 insertions(+), 19 deletions(-) create mode 100755 docker-entrypoint.sh delete mode 100644 flask-entrypoint.sh diff --git a/Dockerfile b/Dockerfile index e6a7ea0b..e588a372 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,18 +1,30 @@ -# pull official base image -FROM python:3.6.9 +FROM python:3.6-alpine -# set environment varibles -ENV PYTHONDONTWRITEBYTECODE 1 -ENV PYTHONUNBUFFERED 1 -# set work directory -WORKDIR /opaque +RUN apk add build-base -# Copy the current directory contents into the container at /daemon -COPY . /opaque -# Install requirements -RUN pip install --trusted-host pypi.python.org -r requirements.txt +RUN adduser -D opaque +USER opaque -# set permissions for entrypoint -RUN chmod a+x flask-entrypoint.sh + +WORKDIR /home/opaque + + +COPY app app +COPY migrations migrations +COPY opaque.py config.py ./ +COPY requirements.txt requirements.txt + + +RUN python -m venv venv && \ + venv/bin/pip install -r requirements.txt + + +COPY docker-entrypoint.sh /usr/local/bin/ + + +EXPOSE 5000 + + +ENTRYPOINT ["docker-entrypoint.sh"] diff --git a/docker-entrypoint.sh b/docker-entrypoint.sh new file mode 100755 index 00000000..535d86a0 --- /dev/null +++ b/docker-entrypoint.sh @@ -0,0 +1,13 @@ +#!/bin/bash + +# If no argument is given, start Opaque +if [ $# -eq 0 ] +then + venv/bin/python opaque.py +else + if [[ $1 == "--create-db" ]] + then + flask db init + flask db upgrade + fi +fi diff --git a/flask-entrypoint.sh b/flask-entrypoint.sh deleted file mode 100644 index ed4b193f..00000000 --- a/flask-entrypoint.sh +++ /dev/null @@ -1,3 +0,0 @@ -#!/bin/bash -x - -python opaque.py diff --git a/opaque.py b/opaque.py index 3f8483af..8d4b0101 100644 --- a/opaque.py +++ b/opaque.py @@ -1,7 +1,5 @@ import eventlet eventlet.monkey_patch() -from dotenv import load_dotenv -load_dotenv() from app import create_app, db, socketio from app.models import Corpus, User, Role, Permission, Job from flask_migrate import Migrate diff --git a/requirements.txt b/requirements.txt index 2592d001..19d3e6ed 100644 --- a/requirements.txt +++ b/requirements.txt @@ -8,5 +8,4 @@ Flask-SQLAlchemy Flask-Table Flask-WTF jsonpatch -python-dotenv redis
InputsResultsInputsResults