mirror of
https://gitlab.ub.uni-bielefeld.de/sfb1288inf/nopaque.git
synced 2024-12-24 02:24:20 +00:00
Merge branch 'development' of gitlab.ub.uni-bielefeld.de:sfb1288inf/opaque into development
This commit is contained in:
Stephan Porada
commit
9ba7989fcc
17
.env_example
Normal file
17
.env_example
Normal file
@ -0,0 +1,17 @@
|
||||
### Flask ###
|
||||
FLASK_CONFIG=production
|
||||
# SECRET_KEY=
|
||||
|
||||
### Flask-Mail ###
|
||||
MAIL_SERVER=smtp.example.com
|
||||
MAIL_PORT=587
|
||||
MAIL_USE_TLS=true
|
||||
MAIL_USERNAME=username@example.com
|
||||
MAIL_PASSWORD=password
|
||||
MAIL_DEFAULT_SENDER=username@example.com
|
||||
|
||||
### Opaque ###
|
||||
OPAQUE_ADMIN=admin.opaque@example.com
|
||||
|
||||
### Flask-SQLAlchemy ###
|
||||
# SQLALCHEMY_DATABASE_URI=
|
||||
38
Dockerfile
38
Dockerfile
@ -1,18 +1,30 @@
|
||||
# pull official base image
|
||||
FROM python:3.6.9
|
||||
FROM python:3.6-alpine
|
||||
|
||||
# set environment varibles
|
||||
ENV PYTHONDONTWRITEBYTECODE 1
|
||||
ENV PYTHONUNBUFFERED 1
|
||||
|
||||
# set work directory
|
||||
WORKDIR /opaque
|
||||
RUN apk add build-base
|
||||
|
||||
# Copy the current directory contents into the container at /daemon
|
||||
COPY . /opaque
|
||||
|
||||
# Install requirements
|
||||
RUN pip install --trusted-host pypi.python.org -r requirements.txt
|
||||
RUN adduser -D opaque
|
||||
USER opaque
|
||||
|
||||
# set permissions for entrypoint
|
||||
RUN chmod a+x flask-entrypoint.sh
|
||||
|
||||
WORKDIR /home/opaque
|
||||
|
||||
|
||||
COPY app app
|
||||
COPY migrations migrations
|
||||
COPY opaque.py config.py ./
|
||||
COPY requirements.txt requirements.txt
|
||||
|
||||
|
||||
RUN python -m venv venv && \
|
||||
venv/bin/pip install -r requirements.txt
|
||||
|
||||
|
||||
COPY docker-entrypoint.sh /usr/local/bin/
|
||||
|
||||
|
||||
EXPOSE 5000
|
||||
|
||||
|
||||
ENTRYPOINT ["docker-entrypoint.sh"]
|
||||
|
||||
@ -10,11 +10,7 @@ def send_async_email(app, msg):
|
||||
|
||||
|
||||
def send_email(to, subject, template, **kwargs):
|
||||
subject = '{} {}'.format(current_app.config['OPAQUE_MAIL_SUBJECT_PREFIX'],
|
||||
subject)
|
||||
msg = Message(subject,
|
||||
sender=current_app.config['OPAQUE_MAIL_SENDER'],
|
||||
recipients=[to])
|
||||
msg = Message('[Opaque] {}'.format(subject), recipients=[to])
|
||||
msg.body = render_template(template + '.txt.j2', **kwargs)
|
||||
msg.html = render_template(template + '.html.j2', **kwargs)
|
||||
thr = Thread(target=send_async_email,
|
||||
|
||||
@ -21,7 +21,7 @@ def corpus(corpus_id):
|
||||
print('Corpus not found.')
|
||||
abort(404)
|
||||
|
||||
dir = os.path.join(current_app.config['OPAQUE_STORAGE'],
|
||||
dir = os.path.join(current_app.config['OPAQUE_STORAGE_DIRECTORY'],
|
||||
str(current_user.id),
|
||||
'corpora',
|
||||
str(corpus.id))
|
||||
@ -44,7 +44,7 @@ def corpus_download(corpus_id):
|
||||
if not file or not corpus:
|
||||
print('File not found.')
|
||||
abort(404)
|
||||
dir = os.path.join(current_app.config['OPAQUE_STORAGE'],
|
||||
dir = os.path.join(current_app.config['OPAQUE_STORAGE_DIRECTORY'],
|
||||
str(current_user.id),
|
||||
'corpora',
|
||||
str(corpus.id))
|
||||
@ -66,7 +66,7 @@ def dashboard():
|
||||
db.session.add(corpus)
|
||||
db.session.commit()
|
||||
|
||||
dir = os.path.join(app.config['OPAQUE_STORAGE'],
|
||||
dir = os.path.join(app.config['OPAQUE_STORAGE_DIRECTORY'],
|
||||
str(corpus.user_id),
|
||||
'corpora',
|
||||
str(corpus.id))
|
||||
@ -96,7 +96,7 @@ def job(job_id):
|
||||
print('Job not found.')
|
||||
abort(404)
|
||||
|
||||
dir = os.path.join(current_app.config['OPAQUE_STORAGE'],
|
||||
dir = os.path.join(current_app.config['OPAQUE_STORAGE_DIRECTORY'],
|
||||
str(current_user.id),
|
||||
'jobs',
|
||||
str(job.id))
|
||||
@ -130,7 +130,7 @@ def job_download(job_id):
|
||||
if not file or not job:
|
||||
print('File not found.')
|
||||
abort(404)
|
||||
dir = os.path.join(current_app.config['OPAQUE_STORAGE'],
|
||||
dir = os.path.join(current_app.config['OPAQUE_STORAGE_DIRECTORY'],
|
||||
str(current_user.id),
|
||||
'jobs',
|
||||
str(job.id))
|
||||
|
||||
@ -26,7 +26,7 @@ def nlp():
|
||||
db.session.add(nlp_job)
|
||||
db.session.commit()
|
||||
|
||||
dir = os.path.join(current_app.config['OPAQUE_STORAGE'],
|
||||
dir = os.path.join(current_app.config['OPAQUE_STORAGE_DIRECTORY'],
|
||||
str(nlp_job.user_id),
|
||||
'jobs',
|
||||
str(nlp_job.id))
|
||||
@ -72,7 +72,7 @@ def ocr():
|
||||
db.session.add(ocr_job)
|
||||
db.session.commit()
|
||||
|
||||
dir = os.path.join(current_app.config['OPAQUE_STORAGE'],
|
||||
dir = os.path.join(current_app.config['OPAQUE_STORAGE_DIRECTORY'],
|
||||
str(ocr_job.user_id),
|
||||
'jobs',
|
||||
str(ocr_job.id))
|
||||
|
||||
@ -161,11 +161,12 @@
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<span class="card-title">Files</span>
|
||||
<table>
|
||||
<thead>
|
||||
<tr>
|
||||
<th>Inputs</th>
|
||||
<th>Results</th>
|
||||
<th style="width: 50%;">Inputs</th>
|
||||
<th style="width: 50%;">Results</th>
|
||||
</tr>
|
||||
</thead>
|
||||
<tbody>
|
||||
@ -179,8 +180,6 @@
|
||||
{% for result in files[file]['results'] %}
|
||||
<a href="{{ url_for('main.job_download', job_id=job.id, file=files[file]['results'][result]['path']) }}" class="waves-effect waves-light btn-small">{{ result }}</a>
|
||||
{% endfor %}
|
||||
{% else %}
|
||||
None
|
||||
{% endif %}
|
||||
</td>
|
||||
</tr>
|
||||
|
||||
@ -13,56 +13,32 @@
|
||||
<div class="col s12 m6">
|
||||
<div class="card z-depth-0">
|
||||
<div class="card-content">
|
||||
<span class="card-title">
|
||||
<i class="material-icons blue-grey-text text-darken-2 left">layers</i>
|
||||
Tokenisierung
|
||||
</span>
|
||||
<p>
|
||||
Aufteilung eines Textes in Sätze und Wörter. Dies
|
||||
ist zur weiteren Verarbeitung notwendig.
|
||||
</p>
|
||||
<span class="card-title"><i class="material-icons blue-grey-text text-darken-2 left">layers</i>Tokenisierung</span>
|
||||
<p>Aufteilung eines Textes in Sätze und Wörter. Dies ist zur weiteren Verarbeitung notwendig.</p>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
<div class="col s12 m6">
|
||||
<div class="card z-depth-0">
|
||||
<div class="card-content">
|
||||
<span class="card-title">
|
||||
<i class="material-icons blue-grey-text text-darken-2 left">layers</i>
|
||||
Lemmatisierung
|
||||
</span>
|
||||
<p>
|
||||
Reduktion der Flexionsformen eines Wortes auf dessen
|
||||
Grundform.
|
||||
</p>
|
||||
<span class="card-title"><i class="material-icons blue-grey-text text-darken-2 left">layers</i>Lemmatisierung</span>
|
||||
<p>Reduktion der Flexionsformen eines Wortes auf dessen Grundform.<br><br></p>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
<div class="col s12 m6">
|
||||
<div class="card z-depth-0">
|
||||
<div class="card-content">
|
||||
<span class="card-title">
|
||||
<i class="material-icons blue-grey-text text-darken-2 left">layers</i>
|
||||
Part-of-speech-Tagging
|
||||
</span>
|
||||
<p>
|
||||
Kontext- und definitionsbezogene Zuordnung von Wörtern
|
||||
und Satzzeichen zu Wortarten.
|
||||
</p>
|
||||
<span class="card-title"><i class="material-icons blue-grey-text text-darken-2 left">layers</i>Part-of-speech-Tagging</span>
|
||||
<p>Kontext- und definitionsbezogene Zuordnung von Wörtern und Satzzeichen zu Wortarten.</p>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
<div class="col s12 m6">
|
||||
<div class="card z-depth-0">
|
||||
<div class="card-content">
|
||||
<span class="card-title">
|
||||
<i class="material-icons blue-grey-text text-darken-2 left">layers</i>
|
||||
Eigennamenerkennung
|
||||
</span>
|
||||
<p>
|
||||
Identifikation von Wörtern, die eine Entität
|
||||
beschreiben, wie Firmen- und Personennamen.
|
||||
</p>
|
||||
<span class="card-title"><i class="material-icons blue-grey-text text-darken-2 left">layers</i>Eigennamenerkennung</span>
|
||||
<p>Identifikation von Wörtern, die eine Entitätbeschreiben, wie Firmen- und Personennamen.</p>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
@ -14,56 +14,32 @@
|
||||
<div class="col s12 m6">
|
||||
<div class="card z-depth-0">
|
||||
<div class="card-content">
|
||||
<span class="card-title">
|
||||
<i class="material-icons blue-grey-text text-darken-2 left">layers</i>
|
||||
Eingabe von Bilddaten
|
||||
</span>
|
||||
<p>
|
||||
Über ein Auftragsformular können Bilddaten in Form von
|
||||
PDF-Dateien hochgeladen werden.
|
||||
</p>
|
||||
<span class="card-title"><i class="material-icons blue-grey-text text-darken-2 left">layers</i>Eingabe von Bilddaten</span>
|
||||
<p>Über ein Auftragsformular können Bilddaten in Form von PDF-Dateien hochgeladen werden.</p>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
<div class="col s12 m6">
|
||||
<div class="card z-depth-0">
|
||||
<div class="card-content">
|
||||
<span class="card-title">
|
||||
<i class="material-icons blue-grey-text text-darken-2 left">layers</i>
|
||||
Optische Zeichenerkennung
|
||||
</span>
|
||||
<p>
|
||||
Die optische Zeichenerkennung erfolgt in der
|
||||
Recheninfrastruktur der Plattform.
|
||||
</p>
|
||||
<span class="card-title"><i class="material-icons blue-grey-text text-darken-2 left">layers</i>Optische Zeichenerkennung</span>
|
||||
<p>Die optische Zeichenerkennung erfolgt in der Recheninfrastruktur der Plattform.</p>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
<div class="col s12 m6">
|
||||
<div class="card z-depth-0">
|
||||
<div class="card-content">
|
||||
<span class="card-title">
|
||||
<i class="material-icons blue-grey-text text-darken-2 left">layers</i>
|
||||
Fehlerkorrektur
|
||||
</span>
|
||||
<p>
|
||||
Je nach Qualität der Eingabedaten kann es zu
|
||||
Fehlern kommen, die korrigiert werden sollten.
|
||||
</p>
|
||||
<span class="card-title"><i class="material-icons blue-grey-text text-darken-2 left">layers</i>Fehlerkorrektur</span>
|
||||
<p>Je nach Qualität der Eingabedaten kann es zu Fehlern kommen, die korrigiert werden sollten.</p>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
<div class="col s12 m6">
|
||||
<div class="card z-depth-0">
|
||||
<div class="card-content">
|
||||
<span class="card-title">
|
||||
<i class="material-icons blue-grey-text text-darken-2 left">layers</i>
|
||||
Weiterverarbeitung
|
||||
</span>
|
||||
<p>
|
||||
Die Textdaten können weiterverarbeitet<a class="tooltipped" data-position="top" data-tooltip="Zum Beispiel durch die hier angebotene linguistische Datenverarbeitung."><sup>[*]</sup></a>
|
||||
oder in dieser Form bereits genutzt<a class="tooltipped" data-position="top" data-tooltip="Zum Beispiel mit dem Programm "AntConc"."><sup>[*]</sup></a> werden.
|
||||
</p>
|
||||
<span class="card-title"><i class="material-icons blue-grey-text text-darken-2 left">layers</i>Weiterverarbeitung</span>
|
||||
<p>Die Textdaten können weiterverarbeitet<a class="tooltipped" data-position="top" data-tooltip="Zum Beispiel durch die hier angebotene linguistische Datenverarbeitung."><sup>[*]</sup></a> oder in dieser Form bereits genutzt<a class="tooltipped" data-position="top" data-tooltip="Zum Beispiel mit dem Programm "AntConc"."><sup>[*]</sup></a> werden.</p>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
44
config.py
44
config.py
@ -1,29 +1,34 @@
|
||||
import os
|
||||
|
||||
|
||||
basedir = os.path.abspath(os.path.dirname(__file__))
|
||||
|
||||
|
||||
class Config:
|
||||
''' ### Flask ### '''
|
||||
SECRET_KEY = os.environ.get('SECRET_KEY') or 'hard to guess string'
|
||||
|
||||
''' ### Flask-Mail ### '''
|
||||
MAIL_SERVER = os.environ.get('MAIL_SERVER')
|
||||
MAIL_PORT = int(os.environ.get('MAIL_PORT'))
|
||||
MAIL_USE_TLS = os.environ.get('MAIL_USE_TLS').lower() == 'true'
|
||||
MAIL_USERNAME = os.environ.get('MAIL_USERNAME')
|
||||
MAIL_PASSWORD = os.environ.get('MAIL_PASSWORD')
|
||||
OPAQUE_ADMIN = os.environ.get('OPAQUE_ADMIN')
|
||||
OPAQUE_STORAGE = os.environ.get('OPAQUE_STORAGE')
|
||||
OPAQUE_MAIL_SUBJECT_PREFIX = '[Opaque]'
|
||||
OPAQUE_MAIL_SENDER = 'Opaque'
|
||||
SECRET_KEY = os.environ.get('SECRET_KEY') or 'hard to guess string'
|
||||
MAIL_DEFAULT_SENDER = os.environ.get('MAIL_DEFAULT_SENDER')
|
||||
|
||||
''' ### Flask-SQLAlchemy ### '''
|
||||
SQLALCHEMY_TRACK_MODIFICATIONS = False
|
||||
|
||||
''' ### Opaque ### '''
|
||||
OPAQUE_ADMIN = os.environ.get('OPAQUE_ADMIN')
|
||||
OPAQUE_STORAGE_DIRECTORY = '/opaque_storage'
|
||||
|
||||
@staticmethod
|
||||
def init_app(app):
|
||||
pass
|
||||
|
||||
|
||||
class DevelopmentConfig(Config):
|
||||
''' ### Flask ### '''
|
||||
DEBUG = True
|
||||
<<<<<<< HEAD
|
||||
# SQLALCHEMY_DATABASE_URI = 'sqlite:///' + os.path.join(basedir,
|
||||
# 'data_dev.sqlite')
|
||||
SQLALCHEMY_DATABASE_URI = \
|
||||
@ -34,22 +39,29 @@ class DevelopmentConfig(Config):
|
||||
port=os.environ.get('POSTGRES_PORT'),
|
||||
db=os.environ.get('POSTGRES_DB_NAME'))
|
||||
print(SQLALCHEMY_DATABASE_URI)
|
||||
=======
|
||||
>>>>>>> e3db2ecd1e52b4993b91e1a1c3501c0fc775de1d
|
||||
|
||||
''' ### Flask-SQLAlchemy ### '''
|
||||
SQLALCHEMY_DATABASE_URI = 'sqlite:///{}'.format(
|
||||
os.path.join(os.path.dirname(os.path.abspath(__file__)),
|
||||
'data_dev.sqlite')
|
||||
)
|
||||
|
||||
|
||||
class TestingConfig(Config):
|
||||
TESTING = True
|
||||
SQLALCHEMY_DATABASE_URI = os.environ.get('TEST_DATABASE_URL') or \
|
||||
'sqlite://'
|
||||
WTF_CSRF_ENABLED = False
|
||||
|
||||
|
||||
# class ProductionConfig(Config):
|
||||
class ProductionConfig(Config):
|
||||
''' ### Flask-SQLAlchemy ### '''
|
||||
SQLALCHEMY_DATABASE_URI = os.environ.get('SQLALCHEMY_DATABASE_URI')
|
||||
|
||||
|
||||
config = {
|
||||
'development': DevelopmentConfig,
|
||||
<<<<<<< HEAD
|
||||
'testing': TestingConfig,
|
||||
#'production': ProductionConfig,
|
||||
|
||||
=======
|
||||
'production': ProductionConfig,
|
||||
>>>>>>> e3db2ecd1e52b4993b91e1a1c3501c0fc775de1d
|
||||
'default': DevelopmentConfig
|
||||
}
|
||||
|
||||
13
docker-entrypoint.sh
Executable file
13
docker-entrypoint.sh
Executable file
@ -0,0 +1,13 @@
|
||||
#!/bin/bash
|
||||
|
||||
# If no argument is given, start Opaque
|
||||
if [ $# -eq 0 ]
|
||||
then
|
||||
venv/bin/python opaque.py
|
||||
else
|
||||
if [[ $1 == "--create-db" ]]
|
||||
then
|
||||
flask db init
|
||||
flask db upgrade
|
||||
fi
|
||||
fi
|
||||
@ -1,3 +0,0 @@
|
||||
#!/bin/bash -x
|
||||
|
||||
python opaque.py
|
||||
@ -1,7 +1,5 @@
|
||||
import eventlet
|
||||
eventlet.monkey_patch()
|
||||
from dotenv import load_dotenv
|
||||
load_dotenv()
|
||||
from app import create_app, db, socketio
|
||||
from app.models import Corpus, User, Role, Permission, Job
|
||||
from flask_migrate import Migrate
|
||||
|
||||
@ -8,6 +8,9 @@ Flask-SQLAlchemy
|
||||
Flask-Table
|
||||
Flask-WTF
|
||||
jsonpatch
|
||||
<<<<<<< HEAD
|
||||
psycopg2
|
||||
python-dotenv
|
||||
=======
|
||||
>>>>>>> e3db2ecd1e52b4993b91e1a1c3501c0fc775de1d
|
||||
redis
|
||||
|
||||
Loading…
Reference in New Issue
Block a user