sporada/nopaque
1
0
Fork 0
mirror of https://gitlab.ub.uni-bielefeld.de/sfb1288inf/nopaque.git synced 2025-06-16 02:50:40 +00:00
Code Issues Projects Releases Wiki Activity

Merge branch 'development' of gitlab.ub.uni-bielefeld.de:sfb1288inf/nopaque into development

Browse Source
This commit is contained in:
Stephan Porada
2020-10-08 12:50:42 +02:00
parent cf6f8e82ee ae9d357beb
commit fb76cf256c
43 changed files with 760 additions and 1200 deletions
Download Patch File Download Diff File Expand all files Collapse all files

5
web/.dockerignore
View File

@ -1,3 +1,6 @@
# Docker related files
Dockerfile
.dockerignore
*.bak
# Packages
__pycache__

6
web/Dockerfile
View File

@ -1,7 +1,7 @@
FROM python:3.6-slim-stretch
LABEL maintainer="inf_sfb1288@lists.uni-bielefeld.de"
LABEL authors="Patrick Jentsch <p.jentsch@uni-bielefeld.de>, Stephan Porada <sporada@uni-bielefeld.de>"
ARG UID
@ -18,7 +18,7 @@ RUN apt-get update \
build-essential \
libpq-dev \
wait-for-it \
&& rm -rf /var/lib/apt/lists/*
&& rm -r /var/lib/apt/lists/*
RUN groupadd --gid ${GID} --system nopaque \
@ -33,4 +33,4 @@ RUN python -m venv venv \
&& mkdir logs
ENTRYPOINT ["./docker-entrypoint.sh"]
ENTRYPOINT ["./boot.sh"]

20
web/app/__init__.py
View File

@ -1,15 +1,14 @@
from config import config
from config import Config
from flask import Flask
from flask_login import LoginManager
from flask_mail import Mail
from flask_paranoid import Paranoid
from flask_socketio import SocketIO
from flask_sqlalchemy import SQLAlchemy
import logging
config = Config()
db = SQLAlchemy()
logger = logging.getLogger(__name__)
login_manager = LoginManager()
login_manager.login_view = 'auth.login'
mail = Mail()
@ -18,40 +17,33 @@ paranoid.redirect_view = '/'
socketio = SocketIO()
def create_app(config_name):
def create_app():
app = Flask(__name__)
app.config.from_object(config[config_name])
app.config.from_object(config)
config[config_name].init_app(app)
config.init_app(app)
db.init_app(app)
login_manager.init_app(app)
mail.init_app(app)
paranoid.init_app(app)
socketio.init_app(app, message_queue='redis://redis:6379/')
socketio.init_app(app, message_queue=config.SOCKETIO_MESSAGE_QUEUE_URI)
from . import events
from .admin import admin as admin_blueprint
app.register_blueprint(admin_blueprint, url_prefix='/admin')
from .auth import auth as auth_blueprint
app.register_blueprint(auth_blueprint, url_prefix='/auth')
from .content import content as content_blueprint
app.register_blueprint(content_blueprint, url_prefix='/content')
from .corpora import corpora as corpora_blueprint
app.register_blueprint(corpora_blueprint, url_prefix='/corpora')
from .jobs import jobs as jobs_blueprint
app.register_blueprint(jobs_blueprint, url_prefix='/jobs')
from .main import main as main_blueprint
app.register_blueprint(main_blueprint)
from .profile import profile as profile_blueprint
app.register_blueprint(profile_blueprint, url_prefix='/profile')
from .services import services as services_blueprint
app.register_blueprint(services_blueprint, url_prefix='/services')

2
web/app/auth/views.py
View File

@ -65,7 +65,7 @@ def register():
username=registration_form.username.data)
db.session.add(user)
db.session.commit()
user_dir = os.path.join(current_app.config['NOPAQUE_STORAGE'],
user_dir = os.path.join(current_app.config['DATA_DIR'],
str(user.id))
if os.path.exists(user_dir):
shutil.rmtree(user_dir)

19
web/app/corpora/events.py
View File

@ -9,8 +9,6 @@ import cqi
import math
from datetime import datetime
import time
from app import logger
'''
' A dictionary containing lists of, with corpus ids associated, Socket.IO
@ -41,7 +39,8 @@ def corpus_analysis_get_meta_data(corpus_id):
metadata['corpus_name'] = db_corpus.title
metadata['corpus_description'] = db_corpus.description
metadata['corpus_creation_date'] = db_corpus.creation_date.isoformat()
metadata['corpus_last_edited_date'] = db_corpus.last_edited_date.isoformat()
metadata['corpus_last_edited_date'] = \
db_corpus.last_edited_date.isoformat()
client = corpus_analysis_clients.get(request.sid)
if client is None:
response = {'code': 424, 'desc': 'No client found for this session',
@ -61,18 +60,20 @@ def corpus_analysis_get_meta_data(corpus_id):
metadata['corpus_size_tokens'] = client_corpus.attrs['size']
text_attr = client_corpus.structural_attributes.get('text')
struct_attrs = client_corpus.structural_attributes.list(filters={'part_of': text_attr})
struct_attrs = client_corpus.structural_attributes.list(
filters={'part_of': text_attr})
text_ids = range(0, (text_attr.attrs['size']))
texts_metadata = {}
for text_id in text_ids:
texts_metadata[text_id] = {}
for struct_attr in struct_attrs:
texts_metadata[text_id][struct_attr.attrs['name'][(len(text_attr.attrs['name']) + 1):]] = struct_attr.values_by_ids(list(range(struct_attr.attrs['size'])))[text_id]
texts_metadata[text_id][struct_attr.attrs['name'][(len(text_attr.attrs['name']) + 1):]] = struct_attr.values_by_ids(list(range(struct_attr.attrs['size'])))[text_id] # noqa
metadata['corpus_all_texts'] = texts_metadata
metadata['corpus_analysis_date'] = datetime.utcnow().isoformat()
metadata['corpus_cqi_py_protocol_version'] = client.api.version
metadata['corpus_cqi_py_package_version'] = cqi.__version__
metadata['corpus_cqpserver_version'] = 'CQPserver v3.4.22' # TODO: make this dynamically
# TODO: make this dynamically
metadata['corpus_cqpserver_version'] = 'CQPserver v3.4.22'
# write some metadata to the db
db_corpus.current_nr_of_tokens = metadata['corpus_size_tokens']
@ -133,7 +134,7 @@ def corpus_analysis_query(query):
if (results.attrs['size'] == 0):
progress = 100
else:
progress = ((chunk_start + chunk_size) / results.attrs['size']) * 100
progress = ((chunk_start + chunk_size) / results.attrs['size']) * 100 # noqa
progress = min(100, int(math.ceil(progress)))
response = {'code': 200, 'desc': None, 'msg': 'OK',
'payload': {'chunk': chunk, 'progress': progress}}
@ -202,7 +203,9 @@ def corpus_analysis_get_match_with_full_context(payload):
'payload': payload,
'type': type,
'data_indexes': data_indexes}
socketio.emit('corpus_analysis_get_match_with_full_context', response, room=request.sid)
socketio.emit('corpus_analysis_get_match_with_full_context',
response,
room=request.sid)
client.status = 'ready'

6
web/app/corpora/views.py
View File

@ -23,7 +23,7 @@ def add_corpus():
status='unprepared', title=add_corpus_form.title.data)
db.session.add(corpus)
db.session.commit()
dir = os.path.join(current_app.config['NOPAQUE_STORAGE'],
dir = os.path.join(current_app.config['DATA_DIR'],
str(corpus.user_id), 'corpora', str(corpus.id))
try:
os.makedirs(dir)
@ -111,7 +111,7 @@ def add_corpus_file(corpus_id):
# Save the file
dir = os.path.join(str(corpus.user_id), 'corpora', str(corpus.id))
add_corpus_file_form.file.data.save(
os.path.join(current_app.config['NOPAQUE_STORAGE'], dir,
os.path.join(current_app.config['DATA_DIR'], dir,
add_corpus_file_form.file.data.filename))
corpus_file = CorpusFile(
address=add_corpus_file_form.address.data,
@ -165,7 +165,7 @@ def download_corpus_file(corpus_id, corpus_file_id):
if not (corpus_file.corpus.creator == current_user
or current_user.is_administrator()):
abort(403)
dir = os.path.join(current_app.config['NOPAQUE_STORAGE'],
dir = os.path.join(current_app.config['DATA_DIR'],
corpus_file.dir)
return send_from_directory(as_attachment=True, directory=dir,
filename=corpus_file.filename)

8
web/app/email.py
View File

@ -1,15 +1,11 @@
from flask import current_app, render_template
from flask import render_template
from flask_mail import Message
from . import mail
from .decorators import background
def create_message(recipient, subject, template, **kwargs):
app = current_app._get_current_object()
sender = app.config['NOPAQUE_MAIL_SENDER']
subject_prefix = app.config['NOPAQUE_MAIL_SUBJECT_PREFIX']
msg = Message('{} {}'.format(subject_prefix, subject),
recipients=[recipient], sender=sender)
msg = Message('[nopaque] {}'.format(subject), recipients=[recipient])
msg.body = render_template('{}.txt.j2'.format(template), **kwargs)
msg.html = render_template('{}.html.j2'.format(template), **kwargs)
return msg

4
web/app/jobs/views.py
View File

@ -44,7 +44,7 @@ def download_job_input(job_id, job_input_id):
if not (job_input.job.creator == current_user
or current_user.is_administrator()):
abort(403)
dir = os.path.join(current_app.config['NOPAQUE_STORAGE'],
dir = os.path.join(current_app.config['DATA_DIR'],
job_input.dir)
return send_from_directory(as_attachment=True, directory=dir,
filename=job_input.filename)
@ -72,7 +72,7 @@ def download_job_result(job_id, job_result_id):
if not (job_result.job.creator == current_user
or current_user.is_administrator()):
abort(403)
dir = os.path.join(current_app.config['NOPAQUE_STORAGE'],
dir = os.path.join(current_app.config['DATA_DIR'],
job_result.dir)
return send_from_directory(as_attachment=True, directory=dir,
filename=job_result.filename)

12
web/app/main/forms.py
View File

@ -1,12 +0,0 @@
from flask_wtf import FlaskForm
from wtforms import DecimalField, StringField, SubmitField, TextAreaField
from wtforms.validators import DataRequired, Email, Length, NumberRange
class FeedbackForm(FlaskForm):
email = StringField('Email', validators=[DataRequired(), Email()])
feedback = TextAreaField('Feedback', validators=[Length(0, 255)])
like_range = DecimalField('How would you rate nopaque?',
validators=[DataRequired(),
NumberRange(min=1, max=10)])
submit = SubmitField('Send feedback')

14
web/app/main/views.py
View File

@ -1,8 +1,6 @@
from flask import flash, redirect, render_template, url_for
from flask_login import login_required, login_user
from . import main
from .forms import FeedbackForm
from .. import logger
from ..auth.forms import LoginForm
from ..models import User
@ -28,18 +26,6 @@ def dashboard():
return render_template('main/dashboard.html.j2', title='Dashboard')
@main.route('/feedback', methods=['GET', 'POST'])
@login_required
def feedback():
feedback_form = FeedbackForm(prefix='feedback-form')
if feedback_form.validate_on_submit():
logger.warning(feedback_form.email)
logger.warning(feedback_form.feedback)
logger.warning(feedback_form.like_range)
return render_template('main/feedback.html.j2',
feedback_form=feedback_form, title='Feedback')
@main.route('/poster', methods=['GET', 'POST'])
def poster():
login_form = LoginForm(prefix='login-form')

16
web/app/models.py
View File

@ -166,7 +166,7 @@ class User(UserMixin, db.Model):
def __init__(self, **kwargs):
super(User, self).__init__(**kwargs)
if self.role is None:
if self.email == current_app.config['NOPAQUE_ADMIN']:
if self.email == current_app.config['ADMIN_EMAIL_ADRESS']:
self.role = Role.query.filter_by(name='Administrator').first()
if self.role is None:
self.role = Role.query.filter_by(default=True).first()
@ -251,7 +251,7 @@ class User(UserMixin, db.Model):
'''
Delete the user and its corpora and jobs from database and filesystem.
'''
user_dir = os.path.join(current_app.config['NOPAQUE_STORAGE'],
user_dir = os.path.join(current_app.config['DATA_DIR'],
str(self.id))
shutil.rmtree(user_dir, ignore_errors=True)
db.session.delete(self)
@ -383,7 +383,7 @@ class Job(db.Model):
db.session.commit()
sleep(1)
db.session.refresh(self)
job_dir = os.path.join(current_app.config['NOPAQUE_STORAGE'],
job_dir = os.path.join(current_app.config['DATA_DIR'],
str(self.user_id),
'jobs',
str(self.id))
@ -397,7 +397,7 @@ class Job(db.Model):
if self.status != 'failed':
raise Exception('Could not restart job: status is not "failed"')
job_dir = os.path.join(current_app.config['NOPAQUE_STORAGE'],
job_dir = os.path.join(current_app.config['DATA_DIR'],
str(self.user_id),
'jobs',
str(self.id))
@ -508,7 +508,7 @@ class CorpusFile(db.Model):
title = db.Column(db.String(255))
def delete(self):
corpus_file_path = os.path.join(current_app.config['NOPAQUE_STORAGE'],
corpus_file_path = os.path.join(current_app.config['DATA_DIR'],
str(self.corpus.user_id),
'corpora',
str(self.corpus_id),
@ -570,7 +570,7 @@ class Corpus(db.Model):
'files': {file.id: file.to_dict() for file in self.files}}
def build(self):
corpus_dir = os.path.join(current_app.config['NOPAQUE_STORAGE'],
corpus_dir = os.path.join(current_app.config['DATA_DIR'],
str(self.user_id),
'corpora',
str(self.id))
@ -606,7 +606,7 @@ class Corpus(db.Model):
self.status = 'submitted'
def delete(self):
corpus_dir = os.path.join(current_app.config['NOPAQUE_STORAGE'],
corpus_dir = os.path.join(current_app.config['DATA_DIR'],
str(self.user_id),
'corpora',
str(self.id))
@ -636,7 +636,7 @@ class QueryResult(db.Model):
title = db.Column(db.String(32))
def delete(self):
query_result_dir = os.path.join(current_app.config['NOPAQUE_STORAGE'],
query_result_dir = os.path.join(current_app.config['DATA_DIR'],
str(self.user_id),
'query_results',
str(self.id))

150
web/app/query_results/views.py Normal file
View File

@ -0,0 +1,150 @@
from . import query_results
from . import tasks
from .. import db
from ..corpora.forms import DisplayOptionsForm, InspectDisplayOptionsForm
from ..models import QueryResult
from .forms import AddQueryResultForm
from flask import (abort, current_app, flash, make_response, redirect,
render_template, request, send_from_directory, url_for)
from flask_login import current_user, login_required
import json
import os
from jsonschema import validate
@query_results.route('/add', methods=['GET', 'POST'])
@login_required
def add_query_result():
'''
View to import a result as a json file.
'''
add_query_result_form = AddQueryResultForm(prefix='add-query-result-form')
if add_query_result_form.is_submitted():
if not add_query_result_form.validate():
return make_response(add_query_result_form.errors, 400)
query_result = QueryResult(
creator=current_user,
description=add_query_result_form.description.data,
filename=add_query_result_form.file.data.filename,
title=add_query_result_form.title.data
)
db.session.add(query_result)
db.session.commit()
# create paths to save the uploaded json file
query_result_dir = os.path.join(current_app.config['DATA_DIR'],
str(current_user.id),
'query_results',
str(query_result.id))
try:
os.makedirs(query_result_dir)
except Exception:
db.session.delete(query_result)
db.session.commit()
flash('Internal Server Error', 'error')
redirect_url = url_for('query_results.add_query_result')
return make_response({'redirect_url': redirect_url}, 500)
# save the uploaded file
query_result_file_path = os.path.join(query_result_dir,
query_result.filename)
add_query_result_form.file.data.save(query_result_file_path)
# parse json from file
with open(query_result_file_path, 'r') as file:
query_result_file_content = json.load(file)
# parse json schema
with open('app/static/json_schema/nopaque_cqi_py_results_schema.json', 'r') as file: # noqa
schema = json.load(file)
try:
# validate imported json file
validate(instance=query_result_file_content, schema=schema)
except Exception:
tasks.delete_query_result(query_result.id)
flash('Uploaded file is invalid', 'result')
redirect_url = url_for('query_results.add_query_result')
return make_response({'redirect_url': redirect_url}, 201)
query_result_file_content.pop('matches')
query_result_file_content.pop('cpos_lookup')
query_result.query_metadata = query_result_file_content
db.session.commit()
flash('Query result added!', 'result')
redirect_url = url_for('query_results.query_result',
query_result_id=query_result.id)
return make_response({'redirect_url': redirect_url}, 201)
return render_template('corpora/query_results/add_query_result.html.j2',
add_query_result_form=add_query_result_form,
title='Add query result')
@query_results.route('/<int:query_result_id>')
@login_required
def query_result(query_result_id):
query_result = QueryResult.query.get_or_404(query_result_id)
if not (query_result.creator == current_user
or current_user.is_administrator()):
abort(403)
return render_template('corpora/query_results/query_result.html.j2',
query_result=query_result,
title='Query result')
@query_results.route('/<int:query_result_id>/inspect')
@login_required
def inspect_query_result(query_result_id):
'''
View to inspect imported result file in a corpus analysis like interface
'''
query_result = QueryResult.query.get_or_404(query_result_id)
query_metadata = query_result.query_metadata
if not (query_result.creator == current_user
or current_user.is_administrator()):
abort(403)
display_options_form = DisplayOptionsForm(
prefix='display-options-form',
results_per_page=request.args.get('results_per_page', 30),
result_context=request.args.get('context', 20)
)
inspect_display_options_form = InspectDisplayOptionsForm(
prefix='inspect-display-options-form'
)
query_result_file_path = os.path.join(
current_app.config['DATA_DIR'],
str(current_user.id),
'query_results',
str(query_result.id),
query_result.filename
)
with open(query_result_file_path, 'r') as query_result_file:
query_result_file_content = json.load(query_result_file)
return render_template('corpora/query_results/inspect.html.j2',
display_options_form=display_options_form,
inspect_display_options_form=inspect_display_options_form,
query_result_file_content=query_result_file_content,
query_metadata=query_metadata,
title='Inspect query result')
@query_results.route('/<int:query_result_id>/delete')
@login_required
def delete_query_result(query_result_id):
query_result = QueryResult.query.get_or_404(query_result_id)
if not (query_result.creator == current_user
or current_user.is_administrator()):
abort(403)
tasks.delete_query_result(query_result_id)
flash('Query result deleted!', 'result')
return redirect(url_for('services.service', service="corpus_analysis"))
@query_results.route('/<int:query_result_id>/download')
@login_required
def download_query_result(query_result_id):
query_result = QueryResult.query.get_or_404(query_result_id)
if not (query_result.creator == current_user
or current_user.is_administrator()):
abort(403)
query_result_dir = os.path.join(current_app.config['DATA_DIR'],
str(current_user.id),
'query_results',
str(query_result.id))
return send_from_directory(as_attachment=True,
directory=query_result_dir,
filename=query_result.filename)

2
web/app/services/views.py
View File

@ -55,7 +55,7 @@ def service(service):
db.session.add(job)
db.session.commit()
relative_dir = os.path.join(str(job.user_id), 'jobs', str(job.id))
absolut_dir = os.path.join(current_app.config['NOPAQUE_STORAGE'],
absolut_dir = os.path.join(current_app.config['DATA_DIR'],
relative_dir)
try:
os.makedirs(absolut_dir)

35
web/app/templates/main/feedback.html.j2
View File

@ -1,35 +0,0 @@
{% extends "nopaque.html.j2" %}
{% block page_content %}
<div class="col s12">
<div class="card">
<form method="POST">
{{ feedback_form.hidden_tag() }}
<div class="card-content">
<p class="range-field">
{{ feedback_form.like_range.label }}
{{ feedback_form.like_range(class='validate', type='range', min=1, max=10) }}
</p>
<div class="input-field">
<i class="material-icons prefix">email</i>
{{ feedback_form.email(class='validate', type='email') }}
{{ feedback_form.email.label }}
{% for error in feedback_form.email.errors %}
<span class="helper-text red-text">{{ error }}</span>
{% endfor %}
</div>
<div class="input-field">
<i class="material-icons prefix">mode_edit</i>
{{ feedback_form.feedback(class='materialize-textarea', data_length=255) }}
{{ feedback_form.feedback.label }}
</div>
</div>
<div class="card-action right-align">
{{ M.render_field(feedback_form.submit, material_icon='send') }}
</div>
</form>
</div>
</div>
{% endblock %}

202
web/app/templates/main/poster.html.j2
View File

@ -1,202 +0,0 @@
{% extends "nopaque.html.j2" %}
{% set parallax = True %}
{% block page_content %}
<style>
{% if request.args.get('print') == 'True' %}
html {
/* DIN 0 bei 150dpi */
width: 4697;
height: 7022px;
}
div.navbar-fixed {
transform: scale(3);
transform-origin: 0 0;
}
footer.page-footer {
transform: scale(3);
transform-origin: 0 0;
margin-top: 5496px;
}
.print-transform {
transform: scale(3);
transform-origin: 0 0;
}
{% endif %}
.parallax-container {
height: 321px;
}
</style>
<div class="print-transform">
<div class="section">
<div class="row container">
<div class="col s12 m5">
<h1>nopaque</h1>
<p>From text to data to analysis</p>
<p class="light">Patrick Jentsch, Stephan Porada and Helene Schlicht</p>
</div>
<div class="col s12 m7">
<p>&nbsp;</p>
<div class="card">
<div class="card-content">
<div class="row">
<div class="col s3">
<p>&nbsp;</p>
<img class="responsive-img" src="https://www.uni-bielefeld.de/sfb1288/images/Logo_SFB1288_DE_300dpi.png">
</div>
<div class="col s9">
<p>nopaque is a web application that helps to convert heterogeneous textual source material into standard-compliant research data for subsequent analysis. nopaque is designed to accompany your research process.</p>
<p>The web application is developed within the DFG-funded Collaborative Research Center (SFB) 1288 "Practices of Comparison" by the subproject INF "Data Infrastructure and Digital Humanities".</p>
</div>
</div>
</div>
</div>
</div>
</div>
</div>
<div class="parallax-container">
<img src="{{ url_for('static', filename='images/parallax_hq/books_antique_book_old.jpg') }}" width="100%" alt="" style="margin-top: -200px;">
</div>
<div class="section white scrollspy" id="information">
<div class="row container">
<div class="col s12">
<div class="row">
<div class="col s12">
<h3>Why you should use nopaque</h3>
<p>nopaque is a custom-built web application for researchers who want to get out more of their images and texts without having to bother about the technical side of things. You can focus on what really interests you, nopaque does the rest.</p>
<p>nopaque’s utilization of container virtualization guarantees high interoperability, reusability and reproducibility of research results. All processing steps are carried out in containers created on demand, based on static images with fixed software versions including all dependencies.</p>
</div>
<div class="col s12">
<div class="row">
<div class="col s12 m6 l3 center-align">
<i class="large material-icons" style="color: #ee6e73;">flash_on</i>
<p>Speeds up your work</p>
<p class="light">All tools provided by nopaque are carefully selected to provide a complete tool suite without being held up by compatibility issues.</p>
</div>
<div class="col s12 m6 l3 center-align">
<i class="large material-icons" style="color: #ee6e73;">cloud</i>
<p>Cloud infrastructure</p>
<p class="light">All computational work is processed within nopaque’s cloud infrastructure. You don't need to install any software. Great, right?</p>
</div>
<div class="col s12 m6 l3 center-align">
<i class="large material-icons" style="color: #ee6e73;">group</i>
<p>User friendly</p>
<p class="light">You can start right away without having to read mile-long manuals. All services come with default settings that make it easy for you to just get going. Also great, right?</p>
</div>
<div class="col s12 m6 l3 center-align">
<i class="large material-icons" style="color: #ee6e73;">settings</i>
<p>Meshing processes</p>
<p class="light">No matter where you step in, nopaque facilitates and accompanies your research. Its workflow perfectly ties in with your research process.</p>
</div>
</div>
</div>
</div>
</div>
</div>
</div>
<div class="parallax-container">
<img src="{{ url_for('static', filename='images/parallax_hq/concept_document_focus_letter.jpg') }}" width="100%" alt="" style="margin-top: -350px;">
</div>
<div class="section white scrollspy" id="services">
<div class="row container">
<div class="col s12">
<div class="row">
<div class="col s12">
<h3>What nopaque can do for you</h3>
<p>All services and processes are logically linked and built upon each other. You can follow them step by step or directly choose the one that suits your needs best. And while the process is computed in nopaque’s cloud infrastructure, you can just keep working.</p>
</div>
<div class="col s12">
<br class="hide-on-small-only">
<div class="row">
<div class="col s12 m6 l3 center-align">
<i class="large material-icons" style="color: #ee6e73;">burst_mode</i>
<p>File setup</p>
<p class="light">Digital copies of text based research data (books, letters, etc.) often comprise various files and formats. nopaque converts and merges those files to facilitate further processing and the application of other services.</p>
</div>
<div class="col s12 m6 l3 center-align">
<i class="large material-icons" style="color: #ee6e73;">find_in_page</i>
<p>Optical Character Recognition</p>
<p class="light">nopaque converts your image data – like photos or scans – into text data through OCR making it machine readable. This step enables you to proceed with further computational analysis of your documents.</p>
</div>
<div class="col s12 m6 l3 center-align">
<i class="large material-icons" style="color: #ee6e73;">format_textdirection_l_to_r</i>
<p>Natural Language Processing</p>
<p class="light">By means of computational linguistic data processing (tokenization, lemmatization, part-of-speech tagging and named-entity recognition) nopaque extracts additional information from your text.</p>
</div>
<div class="col s12 m6 l3 center-align">
<i class="large material-icons" style="color: #ee6e73;">search</i>
<p>Corpus analysis</p>
<p class="light">nopaque lets you create and upload as many text corpora as you want. It makes use of CQP Query Language, which allows for complex search requests with the aid of metadata and NLP tags.</p>
</div>
</div>
</div>
</div>
</div>
</div>
</div>
<div class="parallax-container">
<img src="{{ url_for('static', filename='images/parallax_hq/text_data_wide.png') }}" width="100%" alt="" style="margin-top: -450px;">
</div>
<div class="section white scrollspy" id="registration-and-log-in">
<div class="row container">
<div class="col s12">
<div class="row">
<!--
<div class="col s12 m4">
<h3>Registration and Log in</h3>
<p>Want to boost your research and get going? nopaque is free and no download is needed. Register now!</p>
<a class="btn waves-effect waves-light" href="{{ url_for('auth.register') }}"><i class="material-icons left">person_add</i>Register</a>
</div>-->
<div class="col s12">
<div class="card">
<form method="POST">
{{ login_form.hidden_tag() }}
<div class="card-content">
<span class="card-title">Registration and Log in</span>
<div class="input-field">
<i class="material-icons prefix">person</i>
{{ login_form.user(class='validate') }}
{{ login_form.user.label }}
{% for error in login_form.user.errors %}
<span class="helper-text red-text">{{ error }}</span>
{% endfor %}
</div>
<div class="input-field">
<i class="material-icons prefix">vpn_key</i>
{{ login_form.password(class='validate') }}
{{ login_form.password.label }}
{% for error in login_form.password.errors %}
<span class="helper-text red-text">{{ error }}</span>
{% endfor %}
</div>
<div class="row" style="margin-bottom: 0;">
<div class="col s6 left-align">
<a href="{{ url_for('auth.reset_password_request') }}">Forgot your password?</a>
|
<a href="{{ url_for('auth.reset_password_request') }}">No account yet?</a>
</div>
<div class="col s6 right-align">
{{ materialize.submit_button(login_form.submit) }}
</div>
</div>
</div>
</form>
</div>
</div>
</div>
</div>
</div>
</div>
</div>
{% endblock %}

21
web/boot.sh Executable file
View File

@ -0,0 +1,21 @@
#!/bin/bash
echo "Waiting for db..."
wait-for-it "${NOPAQUE_DB_HOST}:${NOPAQUE_DB_PORT:-5432}" --strict --timeout=0
echo "Waiting for mq..."
wait-for-it "${NOPAQUE_MQ_HOST}:${NOPAQUE_MQ_PORT}" --strict --timeout=0
source venv/bin/activate
if [ "$#" -eq 0 ]; then
flask deploy
python nopaque.py
elif [[ "$1" == "flask" ]]; then
exec ${@:1}
else
echo "$0 [COMMAND]"
echo ""
echo "nopaque startup script"
echo ""
echo "Management Commands:"
echo " flask"
fi

154
web/config.py
View File

@ -1,85 +1,97 @@
from werkzeug.middleware.proxy_fix import ProxyFix
import os
import logging
import os
root_dir = os.path.abspath(os.path.dirname(__file__))
DEFAULT_DATA_DIR = os.path.join('/mnt/nopaque')
DEFAULT_DB_PORT = '5432'
DEFAULT_DEBUG = 'False'
DEFAULT_LOG_DATE_FORMAT = '%Y-%m-%d %H:%M:%S'
DEFAULT_LOG_FILE = os.path.join(root_dir, 'nopaque.log')
DEFAULT_LOG_FORMAT = ('[%(asctime)s] %(levelname)s in %(pathname)s '
'(function: %(funcName)s, line: %(lineno)d): '
'%(message)s')
DEFAULT_LOG_LEVEL = 'ERROR'
DEFAULT_SMTP_USE_SSL = 'False'
DEFAULT_SMTP_USE_TLS = 'False'
DEFAULT_NUM_PROXIES = '0'
DEFAULT_PROTOCOL = 'http'
DEFAULT_RESSOURCES_PER_PAGE = '5'
DEFAULT_USERS_PER_PAGE = '10'
DEFAULT_SECRET_KEY = 'hard to guess string'
class Config:
''' ### Flask ### '''
SECRET_KEY = os.environ.get('SECRET_KEY') or 'hard to guess string'
''' ### Flask-Mail ### '''
MAIL_SERVER = os.environ.get('MAIL_SERVER')
MAIL_PORT = int(os.environ.get('MAIL_PORT'))
MAIL_USE_TLS = os.environ.get('MAIL_USE_TLS').lower() == 'true'
MAIL_USERNAME = os.environ.get('MAIL_USERNAME')
MAIL_PASSWORD = os.environ.get('MAIL_PASSWORD')
''' ### Flask-SQLAlchemy ### '''
SQLALCHEMY_DATABASE_URI = 'postgresql://{}:{}@db/{}'.format(
os.environ.get('POSTGRES_USER'),
os.environ.get('POSTGRES_PASSWORD'),
os.environ.get('POSTGRES_DB_NAME'))
''' ### Database ### '''
DB_HOST = os.environ.get('NOPAQUE_DB_HOST')
DB_NAME = os.environ.get('NOPAQUE_DB_NAME')
DB_PASSWORD = os.environ.get('NOPAQUE_DB_PASSWORD')
DB_PORT = os.environ.get('NOPAQUE_DB_PORT', DEFAULT_DB_PORT)
DB_USERNAME = os.environ.get('NOPAQUE_DB_USERNAME')
SQLALCHEMY_DATABASE_URI = 'postgresql://{}:{}@{}:{}/{}'.format(
DB_USERNAME, DB_PASSWORD, DB_HOST, DB_PORT, DB_NAME)
SQLALCHEMY_RECORD_QUERIES = True
SQLALCHEMY_TRACK_MODIFICATIONS = False
''' ### nopaque ### '''
NOPAQUE_ADMIN = os.environ.get('NOPAQUE_ADMIN')
NOPAQUE_CONTACT = os.environ.get('NOPAQUE_CONTACT')
NOPAQUE_MAIL_SENDER = os.environ.get('NOPAQUE_MAIL_SENDER')
NOPAQUE_MAIL_SUBJECT_PREFIX = '[nopaque]'
NOPAQUE_PROTOCOL = os.environ.get('NOPAQUE_PROTOCOL')
NOPAQUE_STORAGE = os.environ.get('NOPAQUE_STORAGE')
''' ### Email ### '''
MAIL_DEFAULT_SENDER = os.environ.get('NOPAQUE_SMTP_DEFAULT_SENDER')
MAIL_PASSWORD = os.environ.get('NOPAQUE_SMTP_PASSWORD')
MAIL_PORT = os.environ.get('NOPAQUE_SMTP_PORT')
MAIL_SERVER = os.environ.get('NOPAQUE_SMTP_SERVER')
MAIL_USERNAME = os.environ.get('NOPAQUE_SMTP_USERNAME')
MAIL_USE_SSL = os.environ.get('NOPAQUE_SMTP_USE_SSL',
DEFAULT_SMTP_USE_SSL).lower() == 'true'
MAIL_USE_TLS = os.environ.get('NOPAQUE_SMTP_USE_TLS',
DEFAULT_SMTP_USE_TLS).lower() == 'true'
os.makedirs('logs', exist_ok=True)
logging.basicConfig(filename='logs/nopaque.log',
format='[%(asctime)s] %(levelname)s in '
'%(pathname)s:%(lineno)d - %(message)s',
datefmt='%Y-%m-%d %H:%M:%S', filemode='w')
''' ### Security enhancements ### '''
if NOPAQUE_PROTOCOL == 'https':
''' ### Flask ### '''
SESSION_COOKIE_SECURE = True
''' ### Flask-Login ### '''
''' ### General ### '''
ADMIN_EMAIL_ADRESS = os.environ.get('NOPAQUE_ADMIN_EMAIL_ADRESS')
CONTACT_EMAIL_ADRESS = os.environ.get('NOPAQUE_CONTACT_EMAIL_ADRESS')
DATA_DIR = os.environ.get('NOPAQUE_DATA_DIR', DEFAULT_DATA_DIR)
DEBUG = os.environ.get('NOPAQUE_DEBUG', DEFAULT_DEBUG).lower() == 'true'
NUM_PROXIES = int(os.environ.get('NOPAQUE_NUM_PROXIES',
DEFAULT_NUM_PROXIES))
PROTOCOL = os.environ.get('NOPAQUE_PROTOCOL', DEFAULT_PROTOCOL)
RESSOURCES_PER_PAGE = int(os.environ.get('NOPAQUE_RESSOURCES_PER_PAGE',
DEFAULT_RESSOURCES_PER_PAGE))
SECRET_KEY = os.environ.get('NOPAQUE_SECRET_KEY', DEFAULT_SECRET_KEY)
USERS_PER_PAGE = int(os.environ.get('NOPAQUE_USERS_PER_PAGE',
DEFAULT_USERS_PER_PAGE))
if PROTOCOL == 'https':
REMEMBER_COOKIE_HTTPONLY = True
REMEMBER_COOKIE_SECURE = True
SESSION_COOKIE_SECURE = True
@staticmethod
def init_app(app):
proxy_fix_kwargs = {'x_for': 1, 'x_host': 1, 'x_port': 1, 'x_proto': 1}
app.wsgi_app = ProxyFix(app.wsgi_app, **proxy_fix_kwargs)
''' ### Logging ### '''
LOG_DATE_FORMAT = os.environ.get('NOPAQUE_LOG_DATE_FORMAT',
DEFAULT_LOG_DATE_FORMAT)
LOG_FILE = os.environ.get('NOPAQUE_LOG_FILE', DEFAULT_LOG_FILE)
LOG_FORMAT = os.environ.get('NOPAQUE_LOG_FORMAT', DEFAULT_LOG_FORMAT)
LOG_LEVEL = os.environ.get('NOPAQUE_LOG_LEVEL', DEFAULT_LOG_LEVEL)
''' ### Message queue ### '''
MQ_HOST = os.environ.get('NOPAQUE_MQ_HOST')
MQ_PORT = os.environ.get('NOPAQUE_MQ_PORT')
MQ_TYPE = os.environ.get('NOPAQUE_MQ_TYPE')
SOCKETIO_MESSAGE_QUEUE_URI = \
'{}://{}:{}/'.format(MQ_TYPE, MQ_HOST, MQ_PORT)
class DevelopmentConfig(Config):
''' ### Flask ### '''
DEBUG = True
''' ### nopaque ### '''
NOPAQUE_LOG_LEVEL = os.environ.get('NOPAQUE_LOG_LEVEL') or 'DEBUG'
logging.basicConfig(level=NOPAQUE_LOG_LEVEL)
class TestingConfig(Config):
''' ### Flask ### '''
TESTING = True
''' ### Flask-SQLAlchemy ### '''
SQLALCHEMY_DATABASE_URI = 'sqlite://'
''' ### Flask-WTF ### '''
WTF_CSRF_ENABLED = False
class ProductionConfig(Config):
''' ### nopaque ### '''
NOPAQUE_LOG_LEVEL = os.environ.get('NOPAQUE_LOG_LEVEL') or 'ERROR'
logging.basicConfig(level=NOPAQUE_LOG_LEVEL)
config = {
'development': DevelopmentConfig,
'testing': TestingConfig,
'production': ProductionConfig,
'default': DevelopmentConfig,
}
def init_app(self, app):
# Configure logging according to the corresponding (LOG_*) config
# entries
logging.basicConfig(datefmt=self.LOG_DATE_FORMAT,
filename=self.LOG_FILE,
format=self.LOG_FORMAT,
level=self.LOG_LEVEL)
# Apply the ProxyFix middleware if nopaque is running behind reverse
# proxies. (NUM_PROXIES indicates the number of reverse proxies running
# in front of nopaque)
if self.NUM_PROXIES > 0:
app.wsgi_app = ProxyFix(app.wsgi_app,
x_for=self.NUM_PROXIES,
x_host=self.NUM_PROXIES,
x_port=self.NUM_PROXIES,
x_proto=self.NUM_PROXIES)

16
web/docker-entrypoint.sh
View File

@ -1,16 +0,0 @@
#!/bin/bash
echo "Waiting for db..."
wait-for-it db:5432 --strict --timeout=0
echo "Waiting for redis..."
wait-for-it redis:6379 --strict --timeout=0
source venv/bin/activate
if [ $# -eq 0 ]; then
flask deploy
python nopaque.py
elif [ $1 == "flask" ]; then
flask ${@:2}
else
echo "$0 [flask [options]]"
fi

3
web/nopaque.py
View File

@ -5,9 +5,8 @@ from app.models import (Corpus, CorpusFile, Job, JobInput, JobResult,
NotificationData, NotificationEmailData, QueryResult,
Role, User)
from flask_migrate import Migrate, upgrade
import os
app = create_app(os.getenv('FLASK_CONFIG') or 'default')
app = create_app()
migrate = Migrate(app, db, compare_type=True)

3
web/tests/test_basics.py
View File

@ -17,6 +17,3 @@ class BasicsTestCase(unittest.TestCase):
def test_app_exists(self):
self.assertFalse(current_app is None)
def test_app_is_testing(self):
self.assertTrue(current_app.config['TESTING'])