sporada/nopaque
1
0
Fork 0
mirror of https://gitlab.ub.uni-bielefeld.de/sfb1288inf/nopaque.git synced 2024-12-25 02:44:18 +00:00
Code Issues Projects Releases Wiki Activity

Merge branch 'daemon-rework'

Browse Source
This commit is contained in:
Patrick Jentsch 2021-01-14 15:24:20 +01:00
commit 8552cbd20c
105 changed files with 3410 additions and 2759 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

153
.env.tpl
View File

@ -9,122 +9,134 @@
# NOTE: Use `.` as <project-root-dir> # NOTE: Use `.` as <project-root-dir>
# HOST_MQ_DIR= # HOST_MQ_DIR=
# Example: 999 # Example: 1000
# HINT: Use this bash command `getent group docker | cut -d: -f3` # HINT: Use this bash command `id -u`
HOST_DOCKER_GID= HOST_UID=
# Example: 1000 # Example: 1000
# HINT: Use this bash command `id -g` # HINT: Use this bash command `id -g`
HOST_GID= HOST_GID=
# DEFAULT: ./nopaqued.log # Example: 999
# NOTES: Use `.` as <project-root-dir>, # HINT: Use this bash command `getent group docker | cut -d: -f3`
# This file must be present on container startup HOST_DOCKER_GID=
# HOST_NOPAQUE_DAEMON_LOG_FILE=
# DEFAULT: ./nopaque.log # DEFAULT: ./nopaque.log
# NOTES: Use `.` as <project-root-dir>, # NOTES: Use `.` as <project-root-dir>,
# This file must be present on container startup # This file must be present on container startup
# HOST_NOPAQUE_LOG_FILE= # HOST_LOG_FILE=
# Example: 1000
# HINT: Use this bash command `id -u`
HOST_UID=
################################################################################ ################################################################################
# Cookies # # Flask #
# https://flask.palletsprojects.com/en/1.1.x/config/ #
################################################################################ ################################################################################
# CHOOSE ONE: False, True # CHOOSE ONE: http, https
# DEFAULT: False # DEFAULT: http
# HINT: Set to true if you redirect http to https # PREFERRED_URL_SCHEME=
# NOPAQUE_REMEMBER_COOKIE_SECURE=
# DEFAULT: hard to guess string
# HINT: Use this bash command `python -c "import uuid; print(uuid.uuid4().hex)"`
# SECRET_KEY=
# Example: nopaque.example.com/nopaque.example.com:5000
# HINT: If your instance is publicly available on a different Port then 80/443,
# you will have to add this to the server name
SERVER_NAME=
# CHOOSE ONE: False, True # CHOOSE ONE: False, True
# DEFAULT: False # DEFAULT: False
# HINT: Set to true if you redirect http to https # HINT: Set to true if you redirect http to https
# NOPAQUE_SESSION_COOKIE_SECURE= # SESSION_COOKIE_SECURE=
################################################################################ ################################################################################
# Database # # Flask-Login #
# DATABASE_URI blueprint: # # https://flask-login.readthedocs.io/en/latest/ #
# - dialect[+driver]://username:password@host[:port]/database #
# - sqlite is not supported #
# - values in square brackets are optional #
################################################################################ ################################################################################
# DEFAULT: postgresql://nopaque:nopaque@db/nopaque # CHOOSE ONE: False, True
# NOPAQUE_DATABASE_URL= # DEFAULT: False
# HINT: Set to true if you redirect http to https
# DEFAULT: postgresql://nopaque:nopaque@db/nopaque_dev # REMEMBER_COOKIE_SECURE=
# NOPAQUE_DEV_DATABASE_URL=
# DEFAULT: postgresql://nopaque:nopaque@db/nopaque_test
# NOPAQUE_TEST_DATABASE_URL=
################################################################################ ################################################################################
# Email # # Flask-Mail #
# https://pythonhosted.org/Flask-Mail/ #
################################################################################ ################################################################################
# EXAMPLE: nopaque Admin <nopaque@example.com> # EXAMPLE: nopaque Admin <nopaque@example.com>
NOPAQUE_SMTP_DEFAULT_SENDER= MAIL_DEFAULT_SENDER=
NOPAQUE_SMTP_PASSWORD= MAIL_PASSWORD=
# EXAMPLE: smtp.example.com # EXAMPLE: smtp.example.com
NOPAQUE_SMTP_SERVER= MAIL_SERVER=
# EXAMPLE: 587 # EXAMPLE: 587
NOPAQUE_SMTP_PORT= MAIL_PORT=
# CHOOSE ONE: False, True # CHOOSE ONE: False, True
# DEFAULT: False # DEFAULT: False
# NOPAQUE_SMTP_USE_SSL= # MAIL_USE_SSL=
# CHOOSE ONE: False, True # CHOOSE ONE: False, True
# DEFAULT: False # DEFAULT: False
# NOPAQUE_SMTP_USE_TLS= # MAIL_USE_TLS=
# EXAMPLE: nopaque@example.com # EXAMPLE: nopaque@example.com
NOPAQUE_SMTP_USERNAME= MAIL_USERNAME=
################################################################################ ################################################################################
# General # # Flask-SQLAlchemy #
# https://flask-sqlalchemy.palletsprojects.com/en/2.x/config/ #
################################################################################ ################################################################################
# DEFAULT with development config: postgresql://nopaque:nopaque@db/nopaque_dev
# DEFAULT with production config: postgresql://nopaque:nopaque@db/nopaque
# DEFAULT with testing config: postgresql://nopaque:nopaque@db/nopaque_test
# SQLALCHEMY_DATABASE_URI=
################################################################################
# nopaque #
################################################################################
# If an account is registered with this email adress gets automatically
# assigned the administrator role.
# EXAMPLE: admin.nopaque@example.com # EXAMPLE: admin.nopaque@example.com
NOPAQUE_ADMIN_EMAIL_ADRESS= NOPAQUE_ADMIN=
# DEFAULT: development # DEFAULT: development
# CHOOSE ONE: development, production, testing # CHOOSE ONE: development, production, testing
# NOPAQUE_CONFIG= # NOPAQUE_CONFIG=
# This email adress is used for the contact button in the nopaque footer. If
# not set, no contact button is displayed.
# DEFAULT: None # DEFAULT: None
# EXAMPLE: contact.nopaque@example.com # EXAMPLE: contact.nopaque@example.com
# NOPAQUE_CONTACT_EMAIL_ADRESS= # NOPAQUE_CONTACT=
# DEFAULT: /mnt/nopaque # DEFAULT: /mnt/nopaque
# NOTE: This must be a network share and it must be available on all Docker Swarm nodes # NOTE: This must be a network share and it must be available on all Docker
# Swarm nodes
# NOPAQUE_DATA_DIR= # NOPAQUE_DATA_DIR=
# DEFAULT: localhost # CHOOSE ONE: False, True
# NOPAQUE_DOMAIN= # DEFAULT: True
# NOPAQUE_DAEMON_ENABLED=
# CHOOSE ONE: http, https # The hostname or IP address for the server to listen on.
# DEFAULT: http # DEFAULT: 0.0.0.0
# NOPAQUE_PROTOCOL= # NOTES: To use a domain locally, add any names that should route to the app to your hosts file.
# If nopaque is running in a Docker container, you propably want to use the default value.
# NOPAQUE_HOST=
# DEFAULT: hard to guess string # The port number for the server to listen on.
# HINT: Use this bash command `python -c "import uuid; print(uuid.uuid4().hex)"` # DEFAULT: 5000
# NOPAQUE_SECRET_KEY= # NOTE: If nopaque is running in a Docker container, you propably want to use the default value.
# NOPAQUE_PORT=
# transport://[userid:password]@hostname[:port]/[virtual_host]
################################################################################ NOPAQUE_SOCKETIO_MESSAGE_QUEUE_URI=
# Logging #
################################################################################
# DEFAULT: /home/nopaqued/nopaqued.log ~ /home/nopaqued/nopaqued.log
# NOTE: Use `.` as <nopaqued-root-dir>
# NOPAQUE_DAEMON_LOG_FILE=
# DEFAULT: %Y-%m-%d %H:%M:%S # DEFAULT: %Y-%m-%d %H:%M:%S
# NOPAQUE_LOG_DATE_FORMAT= # NOPAQUE_LOG_DATE_FORMAT=
@ -140,37 +152,22 @@ NOPAQUE_ADMIN_EMAIL_ADRESS=
# CHOOSE ONE: CRITICAL, ERROR, WARNING, INFO, DEBUG # CHOOSE ONE: CRITICAL, ERROR, WARNING, INFO, DEBUG
# NOPAQUE_LOG_LEVEL= # NOPAQUE_LOG_LEVEL=
################################################################################
# Message queue #
# MESSAGE_QUEUE_URI blueprint: #
# - transport://[userid:password]@hostname[:port]/[virtual_host] #
# - values in square brackets are optional #
################################################################################
# DEFAULT: None
# HINT: A message queue is not required when using a single server process
# NOPAQUE_SOCKETIO_MESSAGE_QUEUE_URI=
################################################################################
# Proxy fix #
################################################################################
# DEFAULT: 0 # DEFAULT: 0
# Number of values to trust for X-Forwarded-For # Number of values to trust for X-Forwarded-For
# NOPAQUE_NUM_PROXIES_X_FOR= # NOPAQUE_PROXY_FIX_X_FOR=
# DEFAULT: 0 # DEFAULT: 0
# Number of values to trust for X-Forwarded-Host # Number of values to trust for X-Forwarded-Host
# NOPAQUE_NUM_PROXIES_X_HOST= # NOPAQUE_PROXY_FIX_X_HOST=
# DEFAULT: 0 # DEFAULT: 0
# Number of values to trust for X-Forwarded-Port # Number of values to trust for X-Forwarded-Port
# NOPAQUE_NUM_PROXIES_X_PORT= # NOPAQUE_PROXY_FIX_X_PORT=
# DEFAULT: 0 # DEFAULT: 0
# Number of values to trust for X-Forwarded-Prefix # Number of values to trust for X-Forwarded-Prefix
# NOPAQUE_NUM_PROXIES_X_PREFIX= # NOPAQUE_PROXY_FIX_X_PREFIX=
# DEFAULT: 0 # DEFAULT: 0
# Number of values to trust for X-Forwarded-Proto # Number of values to trust for X-Forwarded-Proto
# NOPAQUE_NUM_PROXIES_X_PROTO= # NOPAQUE_PROXY_FIX_X_PROTO=

8
README.md
View File

@ -56,6 +56,12 @@ username@hostname:~$ docker-compose build
``` bash ``` bash
# Create log files # Create log files
touch nopaque.log nopaqued.log touch nopaque.log nopaqued.log
# For background execution add the -d flag and to scale the app, add --scale web=<NUM-INSTANCES> # For background execution add the -d flag
username@hostname:~$ docker-compose up username@hostname:~$ docker-compose up
# To scale your app use
username@hostname:~$ docker-compose -f docker-compose.yml \
-f docker-compose.override.yml
-f docker-compose.scale.yml
up
-d --no-recreate --scale nopaque=<NUM_INSTANCES>
``` ```

6
daemon/.dockerignore
View File

@ -1,6 +0,0 @@
# Docker related files
Dockerfile
.dockerignore
# Packages
__pycache__

32
daemon/Dockerfile
View File

@ -1,32 +0,0 @@
FROM python:3.6.12-slim-buster
LABEL authors="Patrick Jentsch <p.jentsch@uni-bielefeld.de>, Stephan Porada <sporada@uni-bielefeld.de>"
ARG DOCKER_GID
ARG GID
ARG UID
ENV LANG=C.UTF-8
RUN apt-get update \
&& apt-get install --no-install-recommends --yes \
build-essential \
libpq-dev \
&& rm -r /var/lib/apt/lists/*
RUN groupadd --gid ${DOCKER_GID} --system docker \
&& groupadd --gid ${GID} --system nopaqued \
&& useradd --create-home --gid ${GID} --groups ${DOCKER_GID} --no-log-init --system --uid ${UID} nopaqued
USER nopaqued
WORKDIR /home/nopaqued
COPY --chown=nopaqued:nopaqued [".", "."]
RUN python -m venv venv \
&& venv/bin/pip install --requirement requirements.txt
ENTRYPOINT ["./boot.sh"]

31
daemon/app/__init__.py
View File

@ -1,31 +0,0 @@
from config import config
from sqlalchemy import create_engine
from sqlalchemy.orm import scoped_session, sessionmaker
from time import sleep
import docker
import os
configuration = config[os.environ.get('NOPAQUE_CONFIG', 'development')]
configuration.init()
docker_client = docker.from_env()
engine = create_engine(configuration.SQLALCHEMY_DATABASE_URI)
Session = scoped_session(sessionmaker(bind=engine))
def run():
from .tasks.check_corpora import check_corpora
check_corpora_thread = check_corpora()
from .tasks.check_jobs import check_jobs
check_jobs_thread = check_jobs()
from .tasks.notify import notify
notify_thread = notify()
while True:
if not check_corpora_thread.is_alive():
check_corpora_thread = check_corpora()
if not check_jobs_thread.is_alive():
check_jobs_thread = check_jobs()
if not notify_thread.is_alive():
notify_thread = notify()
sleep(3)

14
daemon/app/decorators.py
View File

@ -1,14 +0,0 @@
from functools import wraps
from threading import Thread
def background(f):
'''
' This decorator executes a function in a Thread.
'''
@wraps(f)
def wrapped(*args, **kwargs):
thread = Thread(target=f, args=args, kwargs=kwargs)
thread.start()
return thread
return wrapped

52
daemon/app/models.py
View File

@ -1,52 +0,0 @@
from sqlalchemy.ext.automap import automap_base
from sqlalchemy.orm import relationship
from . import engine
Base = automap_base()
# Classes for database models
class Corpus(Base):
__tablename__ = 'corpora'
files = relationship('CorpusFile', collection_class=set)
class CorpusFile(Base):
__tablename__ = 'corpus_files'
class Job(Base):
__tablename__ = 'jobs'
inputs = relationship('JobInput', collection_class=set)
results = relationship('JobResult', collection_class=set)
notification_data = relationship('NotificationData', collection_class=list)
notification_email_data = relationship('NotificationEmailData',
collection_class=list)
class JobInput(Base):
__tablename__ = 'job_results'
class JobResult(Base):
__tablename__ = 'job_results'
class NotificationData(Base):
__tablename__ = 'notification_data'
job = relationship('Job', collection_class=set)
class NotificationEmailData(Base):
__tablename__ = 'notification_email_data'
job = relationship('Job', collection_class=set)
class User(Base):
__tablename__ = 'users'
jobs = relationship('Job', collection_class=set)
corpora = relationship('Corpus', collection_class=set)
Base.prepare(engine, reflect=True)

0
daemon/app/tasks/__init__.py
View File

140
daemon/app/tasks/check_corpora.py
View File

@ -1,140 +0,0 @@
from .. import configuration as config
from .. import docker_client, Session
from ..decorators import background
from ..models import Corpus
import docker
import logging
import os
import shutil
@background
def check_corpora():
session = Session()
corpora = session.query(Corpus).all()
for corpus in filter(lambda corpus: corpus.status == 'submitted', corpora):
__create_build_corpus_service(corpus)
for corpus in filter(lambda corpus: (corpus.status == 'queued'
or corpus.status == 'running'),
corpora):
__checkout_build_corpus_service(corpus)
for corpus in filter(lambda corpus: corpus.status == 'start analysis',
corpora):
__create_cqpserver_container(corpus)
for corpus in filter(lambda corpus: corpus.status == 'stop analysis',
corpora):
__remove_cqpserver_container(corpus)
session.commit()
Session.remove()
def __create_build_corpus_service(corpus):
corpus_dir = os.path.join(config.DATA_DIR,
str(corpus.user_id),
'corpora',
str(corpus.id))
corpus_data_dir = os.path.join(corpus_dir, 'data')
corpus_file = os.path.join(corpus_dir, 'merged', 'corpus.vrt')
corpus_registry_dir = os.path.join(corpus_dir, 'registry')
if os.path.exists(corpus_data_dir):
shutil.rmtree(corpus_data_dir)
if os.path.exists(corpus_registry_dir):
shutil.rmtree(corpus_registry_dir)
os.mkdir(corpus_data_dir)
os.mkdir(corpus_registry_dir)
service_args = {'command': 'docker-entrypoint.sh build-corpus',
'constraints': ['node.role==worker'],
'labels': {'origin': 'nopaque',
'type': 'corpus.prepare',
'corpus_id': str(corpus.id)},
'mounts': [corpus_file + ':/root/files/corpus.vrt:ro',
corpus_data_dir + ':/corpora/data:rw',
corpus_registry_dir + ':/usr/local/share/cwb/registry:rw'],
'name': 'build-corpus_{}'.format(corpus.id),
'restart_policy': docker.types.RestartPolicy()}
service_image = \
'gitlab.ub.uni-bielefeld.de:4567/sfb1288inf/cqpserver:latest'
try:
service = docker_client.services.get(service_args['name'])
except docker.errors.NotFound:
pass
except docker.errors.DockerException:
return
else:
service.remove()
try:
docker_client.services.create(service_image, **service_args)
except docker.errors.DockerException:
corpus.status = 'failed'
else:
corpus.status = 'queued'
def __checkout_build_corpus_service(corpus):
service_name = 'build-corpus_{}'.format(corpus.id)
try:
service = docker_client.services.get(service_name)
except docker.errors.NotFound:
logging.error('__checkout_build_corpus_service({}):'.format(corpus.id)
+ ' The service does not exist.'
+ ' (stauts: {} -> failed)'.format(corpus.status))
corpus.status = 'failed'
return
except docker.errors.DockerException:
return
service_tasks = service.tasks()
if not service_tasks:
return
task_state = service_tasks[0].get('Status').get('State')
if corpus.status == 'queued' and task_state != 'pending':
corpus.status = 'running'
elif corpus.status == 'running' and task_state == 'complete':
service.remove()
corpus.status = 'prepared'
elif corpus.status == 'running' and task_state == 'failed':
service.remove()
corpus.status = task_state
def __create_cqpserver_container(corpus):
corpus_dir = os.path.join(config.DATA_DIR,
str(corpus.user_id),
'corpora',
str(corpus.id))
corpus_data_dir = os.path.join(corpus_dir, 'data')
corpus_registry_dir = os.path.join(corpus_dir, 'registry')
container_args = {'command': 'cqpserver',
'detach': True,
'volumes': [corpus_data_dir + ':/corpora/data:rw',
corpus_registry_dir + ':/usr/local/share/cwb/registry:rw'],
'name': 'cqpserver_{}'.format(corpus.id),
'network': 'nopaque_default'}
container_image = \
'gitlab.ub.uni-bielefeld.de:4567/sfb1288inf/cqpserver:latest'
try:
container = docker_client.containers.get(container_args['name'])
except docker.errors.NotFound:
pass
except docker.errors.DockerException:
return
else:
container.remove(force=True)
try:
docker_client.containers.run(container_image, **container_args)
except docker.errors.DockerException:
return
else:
corpus.status = 'analysing'
def __remove_cqpserver_container(corpus):
container_name = 'cqpserver_{}'.format(corpus.id)
try:
container = docker_client.containers.get(container_name)
except docker.errors.NotFound:
pass
except docker.errors.DockerException:
return
else:
container.remove(force=True)
corpus.status = 'prepared'

147
daemon/app/tasks/check_jobs.py
View File

@ -1,147 +0,0 @@
from datetime import datetime
from .. import configuration as config
from .. import docker_client, Session
from ..decorators import background
from ..models import Job, JobResult, NotificationData, NotificationEmailData
import docker
import logging
import json
import os
@background
def check_jobs():
session = Session()
jobs = session.query(Job).all()
for job in filter(lambda job: job.status == 'submitted', jobs):
__create_job_service(job)
for job in filter(lambda job: job.status == 'queued', jobs):
__checkout_job_service(job, session)
__add_notification_data(job, 'queued', session)
for job in filter(lambda job: job.status == 'running', jobs):
__checkout_job_service(job, session)
__add_notification_data(job, 'running', session)
for job in filter(lambda job: job.status == 'complete', jobs):
__add_notification_data(job, 'complete', session)
for job in filter(lambda job: job.status == 'failed', jobs):
__add_notification_data(job, 'failed', session)
for job in filter(lambda job: job.status == 'canceling', jobs):
__remove_job_service(job)
session.commit()
Session.remove()
def __add_notification_data(job, notified_on_status, session):
# checks if user wants any notifications at all
if (job.user.setting_job_status_mail_notifications == 'none'):
return
# checks if user wants only notification on completed jobs
elif (job.user.setting_job_status_mail_notifications == 'end'
and notified_on_status != 'complete'):
return
else:
# check if a job already has associated NotificationData
notification_exists = len(job.notification_data)
# create notification_data for current job if there is none
if (notification_exists == 0):
notification_data = NotificationData(job_id=job.id)
session.add(notification_data)
# If no commit job will have no NotificationData
session.commit()
if (job.notification_data[0].notified_on != notified_on_status):
notification_email_data = NotificationEmailData(job_id=job.id)
notification_email_data.notify_status = notified_on_status
notification_email_data.creation_date = datetime.utcnow()
job.notification_data[0].notified_on = notified_on_status
session.add(notification_email_data)
def __create_job_service(job):
job_dir = os.path.join(config.DATA_DIR,
str(job.user_id),
'jobs',
str(job.id))
cmd = '{} -i /files -o /files/output'.format(job.service)
if job.service == 'file-setup':
cmd += ' -f {}'.format(job.secure_filename)
cmd += ' --log-dir /files'
cmd += ' --zip [{}]_{}'.format(job.service, job.secure_filename)
cmd += ' ' + ' '.join(json.loads(job.service_args))
service_args = {'command': cmd,
'constraints': ['node.role==worker'],
'labels': {'origin': 'nopaque',
'type': 'service.{}'.format(job.service),
'job_id': str(job.id)},
'mounts': [job_dir + ':/files:rw'],
'name': 'job_{}'.format(job.id),
'resources': docker.types.Resources(
cpu_reservation=job.n_cores * (10 ** 9),
mem_reservation=job.mem_mb * (10 ** 6)),
'restart_policy': docker.types.RestartPolicy()}
service_image = ('gitlab.ub.uni-bielefeld.de:4567/sfb1288inf/'
+ job.service + ':' + job.service_version)
try:
service = docker_client.services.get(service_args['name'])
except docker.errors.NotFound:
pass
except docker.errors.DockerException:
return
else:
service.remove()
try:
docker_client.services.create(service_image, **service_args)
except docker.errors.DockerException:
job.status = 'failed'
else:
job.status = 'queued'
def __checkout_job_service(job, session):
service_name = 'job_{}'.format(job.id)
try:
service = docker_client.services.get(service_name)
except docker.errors.NotFound:
logging.error('__checkout_job_service({}): '.format(job.id)
+ 'The service does not exist. '
+ '(status: {} -> failed)'.format(job.status))
job.status = 'failed'
return
except docker.errors.DockerException:
return
service_tasks = service.tasks()
if not service_tasks:
return
task_state = service_tasks[0].get('Status').get('State')
if job.status == 'queued' and task_state != 'pending':
job.status = 'running'
elif (job.status == 'running'
and (task_state == 'complete' or task_state == 'failed')):
service.remove()
job.end_date = datetime.utcnow()
job.status = task_state
if task_state == 'complete':
results_dir = os.path.join(config.DATA_DIR,
str(job.user_id),
'jobs',
str(job.id),
'output')
results = filter(lambda x: x.endswith('.zip'),
os.listdir(results_dir))
for result in results:
job_result = JobResult(dir=results_dir,
filename=result,
job_id=job.id)
session.add(job_result)
def __remove_job_service(job):
service_name = 'job_{}'.format(job.id)
try:
service = docker_client.services.get(service_name)
except docker.errors.NotFound:
job.status = 'canceled'
except docker.errors.DockerException:
return
else:
service.update(mounts=None)
service.remove()

0
daemon/app/tasks/libnotify/__init__.py
View File

28
daemon/app/tasks/libnotify/notification.py
View File

@ -1,28 +0,0 @@
from email.message import EmailMessage
class Notification(EmailMessage):
"""docstring for Email."""
def set_notification_content(self,
subject_template,
subject_template_values_dict,
body_txt_template_path,
body_html_template_path,
body_template_values_dict):
# Create subject with subject_template_values_dict
self['subject'] = subject_template.format(
**subject_template_values_dict)
# Open template files and insert values from body_template_values_dict
with open(body_txt_template_path) as nfile:
self.body = nfile.read().format(**body_template_values_dict)
with open(body_html_template_path) as nfile:
self.html = nfile.read().format(**body_template_values_dict)
# Set txt of email
self.set_content(self.body)
# Set html alternative
self.add_alternative(self.html, subtype='html')
def set_addresses(self, sender, recipient):
self['From'] = sender
self['to'] = recipient

16
daemon/app/tasks/libnotify/service.py
View File

@ -1,16 +0,0 @@
class NotificationService:
"""This is a nopaque notifcation service object."""
def __init__(self, smtp):
# Bool to show if the mail server stoped sending mails due to exceeding
# its sending limit
self.mail_limit_exceeded = False
# Holds due to an error unsent email notifications
self.not_sent = {}
self.smtp = smtp
def send(self, email):
self.smtp.send_message(email)
def quit(self):
self.smtp.quit()

15
daemon/app/tasks/libnotify/templates/notification.html
View File

@ -1,15 +0,0 @@
<html>
<body>
<p>Dear <b>{username}</b>,</p>
<p>The status of your Job/Corpus({id}) with the title <b>"{title}"</b> has changed!</p>
<p>It is now <b>{status}</b>!</p>
<p>Time of this status update was: <b>{time} UTC</b></p>
<p>You can access your Job/Corpus here: <a href="{url}">{url}</a>
</p>
<p>Kind regards!<br>
Your nopaque team</p>
</body>
</html>

10
daemon/app/tasks/libnotify/templates/notification.txt
View File

@ -1,10 +0,0 @@
Dear {username},
The status of your Job/Corpus({id}) with the title "{title}" has changed!
It is now {status}!
Time of this status update was: {time} UTC
You can access your Job/Corpus here: {url}
Kind regards!
Your nopaque team

111
daemon/app/tasks/notify.py
View File

@ -1,111 +0,0 @@
from sqlalchemy import asc
from .libnotify.notification import Notification
from .libnotify.service import NotificationService
from .. import configuration as config
from .. import Session
from ..decorators import background
from ..models import NotificationEmailData
import logging
import os
import smtplib
ROOT_DIR = os.path.abspath(os.path.dirname(__file__))
@background
def notify():
session = Session()
if config.SMTP_USE_SSL:
smtp = smtplib.SMTP_SSL(host=config.SMTP_SERVER, port=config.SMTP_PORT)
else:
smtp = smtplib.SMTP(host=config.SMTP_SERVER, port=config.SMTP_PORT)
if config.SMTP_USE_TLS:
smtp.starttls()
try:
smtp.login(config.SMTP_USERNAME, config.SMTP_PASSWORD)
except smtplib.SMTPHeloError:
logging.warning('The server didn’t reply properly to the HELO '
'greeting.')
return
except smtplib.SMTPAuthenticationError as e:
logging.warning('The server didn’t accept the username/password '
'combination.')
logging.warning(e)
return
except smtplib.SMTPNotSupportedError:
logging.warning('The AUTH command is not supported by the server.')
return
except smtplib.SMTPException:
logging.warning('No suitable authentication method was found.')
return
notification_service = NotificationService(smtp)
# create notifications (content, recipient etc.)
notifications = __create_mail_notifications(notification_service, session)
# only login and send mails if there are any notifications
if (len(notifications) > 0):
# combine new and unsent notifications
notifications.update(notification_service.not_sent)
# send all notifications
__send_mail_notifications(notifications, notification_service)
# remove unsent notifications because they have been sent now
# but only if mail limit has not been exceeded
if (notification_service.mail_limit_exceeded is not True):
notification_service.not_sent = {}
smtp.quit()
Session.remove()
# Email notification functions
def __create_mail_notifications(notification_service, session):
notification_email_data = session.query(NotificationEmailData).order_by(asc(NotificationEmailData.creation_date)).all() # noqa
notifications = {}
for data in notification_email_data:
notification = Notification()
notification.set_addresses(config.SMTP_DEFAULT_SENDER,
data.job.user.email)
subject_template = ('[nopaque] Status update for your Job/Corpora: '
'{title}!')
subject_template_values_dict = {'title': data.job.title}
url = '{}://{}/{}/{}'.format(config.PROTOCOL,
config.DOMAIN,
'jobs',
data.job.id)
body_template_values_dict = {'username': data.job.user.username,
'id': data.job.id,
'title': data.job.title,
'status': data.notify_status,
'time': data.creation_date,
'url': url}
txt_tmplt = os.path.join(ROOT_DIR,
'libnotify/templates/notification.txt')
html_tmplt = os.path.join(ROOT_DIR,
'libnotify/templates/notification.html')
notification.set_notification_content(subject_template,
subject_template_values_dict,
txt_tmplt,
html_tmplt,
body_template_values_dict)
notifications[data.job.id] = notification
# Using a dictionary for notifications avoids sending multiple mails
# if the status of a job changes in a few seconds. The user will not
# get swamped with mails for queued, running and complete if those
# happen in in a few seconds. Only the last update will be sent.
# This depends on the sleep time interval though.
session.delete(data)
session.commit()
return notifications
def __send_mail_notifications(notifications, notification_service):
for key, notification in notifications.items():
try:
notification_service.send(notification)
notification_service.mail_limit_exceeded = False
except Exception:
# Adds notifications to unsent if mail server exceded limit for
# consecutive mail sending
logging.warning('limit')
notification_service.not_sent[key] = notification
notification_service.mail_limit_exceeded = True
notification_service.not_sent.update(notifications)

3
daemon/boot.sh
View File

@ -1,3 +0,0 @@
#!/bin/bash
source venv/bin/activate
python nopaqued.py

71
daemon/config.py
View File

@ -1,71 +0,0 @@
import logging
import os
ROOT_DIR = os.path.abspath(os.path.dirname(__file__))
class Config:
''' # Email # '''
SMTP_DEFAULT_SENDER = os.environ.get('NOPAQUE_SMTP_DEFAULT_SENDER')
SMTP_PASSWORD = os.environ.get('NOPAQUE_SMTP_PASSWORD')
SMTP_PORT = int(os.environ.get('NOPAQUE_SMTP_PORT'))
SMTP_SERVER = os.environ.get('NOPAQUE_SMTP_SERVER')
SMTP_USERNAME = os.environ.get('NOPAQUE_SMTP_USERNAME')
SMTP_USE_SSL = os.environ.get(
'NOPAQUE_SMTP_USE_SSL', 'false').lower() == 'true'
SMTP_USE_TLS = os.environ.get(
'NOPAQUE_SMTP_USE_TLS', 'false').lower() == 'true'
''' # General # '''
DATA_DIR = os.environ.get('NOPAQUE_DATA_DIR', '/mnt/nopaque')
DOMAIN = os.environ.get('NOPAQUE_DOMAIN', 'localhost')
PROTOCOL = os.environ.get('NOPAQUE_PROTOCOL', 'http')
SECRET_KEY = os.environ.get('NOPAQUE_SECRET_KEY', 'hard to guess string')
''' # Logging # '''
LOG_DATE_FORMAT = os.environ.get('NOPAQUE_LOG_DATE_FORMAT',
'%Y-%m-%d %H:%M:%S')
LOG_FILE = os.environ.get('NOPAQUED_LOG_FILE',
os.path.join(ROOT_DIR, 'nopaqued.log'))
LOG_FORMAT = os.environ.get(
'NOPAQUE_LOG_FORMAT',
'[%(asctime)s] %(levelname)s in '
'%(pathname)s (function: %(funcName)s, line: %(lineno)d): %(message)s'
)
LOG_LEVEL = os.environ.get('NOPAQUE_LOG_LEVEL', 'WARNING')
@classmethod
def init(cls):
# Set up logging according to the corresponding (LOG_*) variables
logging.basicConfig(datefmt=cls.LOG_DATE_FORMAT,
filename=cls.LOG_FILE,
format=cls.LOG_FORMAT,
level=cls.LOG_LEVEL)
class DevelopmentConfig(Config):
''' # Database # '''
SQLALCHEMY_DATABASE_URI = os.environ.get(
'NOPAQUE_DEV_DATABASE_URL',
'sqlite:///' + os.path.join(ROOT_DIR, 'data-dev.sqlite')
)
class ProductionConfig(Config):
''' # Database # '''
SQLALCHEMY_DATABASE_URI = os.environ.get(
'NOPAQUE_DATABASE_URL',
'sqlite:///' + os.path.join(ROOT_DIR, 'data.sqlite')
)
class TestingConfig(Config):
''' # Database # '''
SQLALCHEMY_DATABASE_URI = os.environ.get(
'NOPAQUE_TEST_DATABASE_URL', 'sqlite://')
config = {'development': DevelopmentConfig,
'production': ProductionConfig,
'testing': TestingConfig}

13
daemon/nopaqued.py
View File

@ -1,13 +0,0 @@
from dotenv import load_dotenv
from app import run
import os
# Load environment variables
DOTENV_FILE = os.path.join(os.path.dirname(__file__), '.env')
if os.path.exists(DOTENV_FILE):
load_dotenv(DOTENV_FILE)
if __name__ == '__main__':
run()

4
daemon/requirements.txt
View File

@ -1,4 +0,0 @@
docker
psycopg2
python-dotenv
SQLAlchemy

8
docker-compose.development.yml
View File

@ -13,11 +13,3 @@ services:
- "./web/nopaque.py:/home/nopaque/nopaque.py" - "./web/nopaque.py:/home/nopaque/nopaque.py"
- "./web/requirements.txt:/home/nopaque/requirements.txt" - "./web/requirements.txt:/home/nopaque/requirements.txt"
- "./web/tests:/home/nopaque/tests" - "./web/tests:/home/nopaque/tests"
nopaqued:
volumes:
# Mount code as volumes
- "./daemon/app:/home/nopaqued/app"
- "./daemon/boot.sh:/home/nopaqued/boot.sh"
- "./daemon/config.py:/home/nopaqued/config.py"
- "./daemon/nopaqued.py:/home/nopaqued/nopaqued.py"
- "./daemon/requirements.txt:/home/nopaqued/requirements.txt"

6
docker-compose.scale.yml Normal file
View File

@ -0,0 +1,6 @@
version: "3.5"
services:
nopaque:
environment:
- NOPAQUE_DAEMON_ENABLED=False

4
docker-compose.traefik.yml
View File

@ -18,13 +18,13 @@ services:
- "traefik.http.middlewares.nopaque-header.headers.customrequestheaders.X-Forwarded-Proto=http" - "traefik.http.middlewares.nopaque-header.headers.customrequestheaders.X-Forwarded-Proto=http"
- "traefik.http.routers.nopaque.entrypoints=web" - "traefik.http.routers.nopaque.entrypoints=web"
- "traefik.http.routers.nopaque.middlewares=nopaque-header, redirect-to-https@file" - "traefik.http.routers.nopaque.middlewares=nopaque-header, redirect-to-https@file"
- "traefik.http.routers.nopaque.rule=Host(`<DOMAIN>`)" - "traefik.http.routers.nopaque.rule=Host(`${SERVER_NAME}`)"
### </http> ### ### </http> ###
### <https> ### ### <https> ###
- "traefik.http.middlewares.nopaque-secure-header.headers.customrequestheaders.X-Forwarded-Proto=https" - "traefik.http.middlewares.nopaque-secure-header.headers.customrequestheaders.X-Forwarded-Proto=https"
- "traefik.http.routers.nopaque-secure.entrypoints=web-secure" - "traefik.http.routers.nopaque-secure.entrypoints=web-secure"
- "traefik.http.routers.nopaque-secure.middlewares=hsts-header@file, nopaque-secure-header" - "traefik.http.routers.nopaque-secure.middlewares=hsts-header@file, nopaque-secure-header"
- "traefik.http.routers.nopaque-secure.rule=Host(`<DOMAIN>`)" - "traefik.http.routers.nopaque-secure.rule=Host(`${SERVER_NAME}`)"
- "traefik.http.routers.nopaque-secure.tls.certresolver=<CERTRESOLVER>" - "traefik.http.routers.nopaque-secure.tls.certresolver=<CERTRESOLVER>"
- "traefik.http.routers.nopaque-secure.tls.options=intermediate@file" - "traefik.http.routers.nopaque-secure.tls.options=intermediate@file"
### </https> ### ### </https> ###

43
docker-compose.yml
View File

@ -1,9 +1,23 @@
version: "3.5" version: "3.5"
services: services:
db:
env_file: db.env
image: postgres:11
restart: unless-stopped
volumes:
- "${HOST_DB_DIR:-./db}:/var/lib/postgresql/data"
mq:
image: redis:6
restart: unless-stopped
volumes:
- "${HOST_MQ_DIR:-./mq}:/data"
nopaque: nopaque:
build: build:
args: args:
DOCKER_GID: ${HOST_DOCKER_GID}
GID: ${HOST_GID} GID: ${HOST_GID}
UID: ${HOST_UID} UID: ${HOST_UID}
context: ./web context: ./web
@ -13,34 +27,7 @@ services:
env_file: .env env_file: .env
image: nopaque:latest image: nopaque:latest
restart: unless-stopped restart: unless-stopped
volumes:
- "${NOPAQUE_DATA_DIR:-/mnt/nopaque}:${NOPAQUE_DATA_DIR:-/mnt/nopaque}"
- "${HOST_NOPAQUE_LOG_FILE-./nopaque.log}:${NOPAQUE_LOG_FILE:-/home/nopaque/nopaque.log}"
nopaqued:
build:
args:
DOCKER_GID: ${HOST_DOCKER_GID}
GID: ${HOST_GID}
UID: ${HOST_UID}
context: ./daemon
depends_on:
- db
- nopaque
env_file: .env
image: nopaqued:latest
restart: unless-stopped
volumes: volumes:
- "/var/run/docker.sock:/var/run/docker.sock" - "/var/run/docker.sock:/var/run/docker.sock"
- "${NOPAQUE_DATA_DIR:-/mnt/nopaque}:${NOPAQUE_DATA_DIR:-/mnt/nopaque}" - "${NOPAQUE_DATA_DIR:-/mnt/nopaque}:${NOPAQUE_DATA_DIR:-/mnt/nopaque}"
- "${HOST_NOPAQUE_DAEMON_LOG_FILE-./nopaqued.log}:${NOPAQUE_DAEMON_LOG_FILE:-/home/nopaqued/nopaqued.log}" - "${HOST_NOPAQUE_LOG_FILE-./nopaque.log}:${NOPAQUE_LOG_FILE:-/home/nopaque/nopaque.log}"
db:
env_file: db.env
image: postgres:11
restart: unless-stopped
volumes:
- "${HOST_DB_DIR:-./db}:/var/lib/postgresql/data"
mq:
image: redis:6
restart: unless-stopped
volumes:
- "${HOST_MQ_DIR:-./mq}:/data"

1
web/.flaskenv Normal file
View File

@ -0,0 +1 @@
FLASK_APP=nopaque.py

10
web/Dockerfile
View File

@ -1,12 +1,12 @@
FROM python:3.6.12-slim-buster FROM python:3.9.0-slim-buster
LABEL authors="Patrick Jentsch <p.jentsch@uni-bielefeld.de>, Stephan Porada <sporada@uni-bielefeld.de>" LABEL authors="Patrick Jentsch <p.jentsch@uni-bielefeld.de>, Stephan Porada <sporada@uni-bielefeld.de>"
ARG DOCKER_GID
ARG UID ARG UID
ARG GID ARG GID
ENV FLASK_APP=nopaque.py
ENV LANG=C.UTF-8 ENV LANG=C.UTF-8
@ -17,12 +17,12 @@ RUN apt-get update \
&& apt-get install --no-install-recommends --yes \ && apt-get install --no-install-recommends --yes \
build-essential \ build-essential \
libpq-dev \ libpq-dev \
wait-for-it \
&& rm -r /var/lib/apt/lists/* && rm -r /var/lib/apt/lists/*
RUN groupadd --gid ${GID} --system nopaque \ RUN groupadd --gid ${DOCKER_GID} --system docker \
&& useradd --create-home --gid ${GID} --no-log-init --system --uid ${UID} nopaque && groupadd --gid ${GID} --system nopaque \
&& useradd --create-home --gid ${GID} --groups ${DOCKER_GID} --no-log-init --system --uid ${UID} nopaque
USER nopaque USER nopaque
WORKDIR /home/nopaque WORKDIR /home/nopaque

3
web/app/__init__.py
View File

@ -26,7 +26,7 @@ def create_app(config_name):
mail.init_app(app) mail.init_app(app)
paranoid.init_app(app) paranoid.init_app(app)
socketio.init_app( socketio.init_app(
app, message_queue=config[config_name].SOCKETIO_MESSAGE_QUEUE_URI) app, message_queue=app.config['NOPAQUE_SOCKETIO_MESSAGE_QUEUE_URI'])
with app.app_context(): with app.app_context():
from . import events from . import events
@ -38,6 +38,7 @@ def create_app(config_name):
from .main import main as main_blueprint from .main import main as main_blueprint
from .services import services as services_blueprint from .services import services as services_blueprint
from .settings import settings as settings_blueprint from .settings import settings as settings_blueprint
app.register_blueprint(admin_blueprint, url_prefix='/admin') app.register_blueprint(admin_blueprint, url_prefix='/admin')
app.register_blueprint(auth_blueprint, url_prefix='/auth') app.register_blueprint(auth_blueprint, url_prefix='/auth')
app.register_blueprint(corpora_blueprint, url_prefix='/corpora') app.register_blueprint(corpora_blueprint, url_prefix='/corpora')

2
web/app/admin/__init__.py
View File

@ -2,4 +2,4 @@ from flask import Blueprint
admin = Blueprint('admin', __name__) admin = Blueprint('admin', __name__)
from . import views # noqa from . import views

1
web/app/admin/forms.py
View File

@ -12,4 +12,3 @@ class EditGeneralSettingsAdminForm(EditGeneralSettingsForm):
super().__init__(*args, user=user, **kwargs) super().__init__(*args, user=user, **kwargs)
self.role.choices = [(role.id, role.name) self.role.choices = [(role.id, role.name)
for role in Role.query.order_by(Role.name).all()] for role in Role.query.order_by(Role.name).all()]
self.user = user

36
web/app/admin/views.py
View File

@ -1,5 +1,5 @@
from flask import flash, redirect, render_template, url_for from flask import flash, redirect, render_template, url_for
from flask_login import current_user, login_required from flask_login import login_required
from . import admin from . import admin
from .forms import EditGeneralSettingsAdminForm from .forms import EditGeneralSettingsAdminForm
from .. import db from .. import db
@ -8,17 +8,19 @@ from ..models import Role, User
from ..settings import tasks as settings_tasks from ..settings import tasks as settings_tasks
@admin.route('/')
@login_required
@admin_required
def index():
return redirect(url_for('.users'))
@admin.route('/users') @admin.route('/users')
@login_required @login_required
@admin_required @admin_required
def users(): def users():
users = User.query.all() # users = [user.to_dict() for user in User.query.all()]
users = [dict(username=u.username, users = {user.id: user.to_dict() for user in User.query.all()}
email=u.email,
role_id=u.role_id,
confirmed=u.confirmed,
id=u.id)
for u in users]
return render_template('admin/users.html.j2', title='Users', users=users) return render_template('admin/users.html.j2', title='Users', users=users)
@ -35,15 +37,14 @@ def user(user_id):
@admin_required @admin_required
def delete_user(user_id): def delete_user(user_id):
settings_tasks.delete_user(user_id) settings_tasks.delete_user(user_id)
flash('User has been deleted!') flash('User has been marked for deletion!')
return redirect(url_for('.users')) return redirect(url_for('.users'))
@admin.route('/users/<int:user_id>/edit_general_settings', @admin.route('/users/<int:user_id>/edit', methods=['GET', 'POST']) # noqa
methods=['GET', 'POST'])
@login_required @login_required
@admin_required @admin_required
def edit_general_settings(user_id): def edit_user(user_id):
user = User.query.get_or_404(user_id) user = User.query.get_or_404(user_id)
form = EditGeneralSettingsAdminForm(user=user) form = EditGeneralSettingsAdminForm(user=user)
if form.validate_on_submit(): if form.validate_on_submit():
@ -52,16 +53,13 @@ def edit_general_settings(user_id):
user.username = form.username.data user.username = form.username.data
user.confirmed = form.confirmed.data user.confirmed = form.confirmed.data
user.role = Role.query.get(form.role.data) user.role = Role.query.get(form.role.data)
db.session.add(user)
db.session.commit() db.session.commit()
flash('The profile has been updated.') flash('Settings have been updated.')
return redirect(url_for('admin.edit_general_settings', user_id=user.id)) return redirect(url_for('.edit_user', user_id=user.id))
form.confirmed.data = user.confirmed form.confirmed.data = user.confirmed
form.dark_mode.data = user.setting_dark_mode form.dark_mode.data = user.setting_dark_mode
form.email.data = user.email form.email.data = user.email
form.role.data = user.role_id form.role.data = user.role_id
form.username.data = user.username form.username.data = user.username
return render_template('admin/edit_general_settings.html.j2', return render_template('admin/edit_user.html.j2', form=form,
form=form, title='Edit user', user=user)
title='General settings',
user=user)

2
web/app/auth/__init__.py
View File

@ -2,4 +2,4 @@ from flask import Blueprint
auth = Blueprint('auth', __name__) auth = Blueprint('auth', __name__)
from . import views # noqa from . import views

2
web/app/auth/forms.py
View File

@ -18,7 +18,7 @@ class RegistrationForm(FlaskForm):
username = StringField( username = StringField(
'Username', 'Username',
validators=[DataRequired(), Length(1, 64), validators=[DataRequired(), Length(1, 64),
Regexp(current_app.config['ALLOWED_USERNAME_REGEX'], Regexp(current_app.config['NOPAQUE_USERNAME_REGEX'],
message='Usernames must have only letters, numbers,' message='Usernames must have only letters, numbers,'
' dots or underscores')] ' dots or underscores')]
) )

107
web/app/auth/views.py
View File

@ -1,5 +1,5 @@
from flask import (current_app, flash, redirect, render_template, request, from datetime import datetime
url_for) from flask import abort, flash, redirect, render_template, request, url_for
from flask_login import current_user, login_user, login_required, logout_user from flask_login import current_user, login_user, login_required, logout_user
from . import auth from . import auth
from .forms import (LoginForm, ResetPasswordForm, ResetPasswordRequestForm, from .forms import (LoginForm, ResetPasswordForm, ResetPasswordRequestForm,
@ -7,8 +7,8 @@ from .forms import (LoginForm, ResetPasswordForm, ResetPasswordRequestForm,
from .. import db from .. import db
from ..email import create_message, send from ..email import create_message, send
from ..models import User from ..models import User
import logging
import os import os
import shutil
@auth.before_app_request @auth.before_app_request
@ -18,11 +18,12 @@ def before_request():
unconfirmed view if user is unconfirmed. unconfirmed view if user is unconfirmed.
""" """
if current_user.is_authenticated: if current_user.is_authenticated:
current_user.ping() current_user.last_seen = datetime.utcnow()
if not current_user.confirmed \ db.session.commit()
and request.endpoint \ if (not current_user.confirmed
and request.blueprint != 'auth' \ and request.endpoint
and request.endpoint != 'static': and request.blueprint != 'auth'
and request.endpoint != 'static'):
return redirect(url_for('auth.unconfirmed')) return redirect(url_for('auth.unconfirmed'))
@ -30,20 +31,19 @@ def before_request():
def login(): def login():
if current_user.is_authenticated: if current_user.is_authenticated:
return redirect(url_for('main.dashboard')) return redirect(url_for('main.dashboard'))
login_form = LoginForm(prefix='login-form') form = LoginForm(prefix='login-form')
if login_form.validate_on_submit(): if form.validate_on_submit():
user = User.query.filter_by(username=login_form.user.data).first() user = User.query.filter_by(username=form.user.data).first()
if user is None: if user is None:
user = User.query.filter_by(email=login_form.user.data).first() user = User.query.filter_by(email=form.user.data.lower()).first()
if user is not None and user.verify_password(login_form.password.data): if user is not None and user.verify_password(form.password.data):
login_user(user, login_form.remember_me.data) login_user(user, form.remember_me.data)
next = request.args.get('next') next = request.args.get('next')
if next is None or not next.startswith('/'): if next is None or not next.startswith('/'):
next = url_for('main.dashboard') next = url_for('main.dashboard')
return redirect(next) return redirect(next)
flash('Invalid email/username or password.') flash('Invalid email/username or password.')
return render_template('auth/login.html.j2', login_form=login_form, return render_template('auth/login.html.j2', form=form, title='Log in')
title='Log in')
@auth.route('/logout') @auth.route('/logout')
@ -58,26 +58,28 @@ def logout():
def register(): def register():
if current_user.is_authenticated: if current_user.is_authenticated:
return redirect(url_for('main.dashboard')) return redirect(url_for('main.dashboard'))
registration_form = RegistrationForm(prefix='registration-form') form = RegistrationForm(prefix='registration-form')
if registration_form.validate_on_submit(): if form.validate_on_submit():
user = User(email=registration_form.email.data.lower(), user = User(email=form.email.data.lower(),
password=registration_form.password.data, password=form.password.data,
username=registration_form.username.data) username=form.username.data)
db.session.add(user) db.session.add(user)
db.session.commit() db.session.commit()
user_dir = os.path.join(current_app.config['DATA_DIR'], try:
str(user.id)) os.makedirs(user.path)
if os.path.exists(user_dir): except OSError:
shutil.rmtree(user_dir) logging.error('Make dir {} led to an OSError!'.format(user.path))
os.mkdir(user_dir) db.session.delete(user)
token = user.generate_confirmation_token() db.session.commit()
msg = create_message(user.email, 'Confirm Your Account', abort(500)
'auth/email/confirm', token=token, user=user) else:
send(msg) token = user.generate_confirmation_token()
flash('A confirmation email has been sent to you by email.') msg = create_message(user.email, 'Confirm Your Account',
return redirect(url_for('auth.login')) 'auth/email/confirm', token=token, user=user)
return render_template('auth/register.html.j2', send(msg)
registration_form=registration_form, flash('A confirmation email has been sent to you by email.')
return redirect(url_for('.login'))
return render_template('auth/register.html.j2', form=form,
title='Register') title='Register')
@ -92,7 +94,7 @@ def confirm(token):
return redirect(url_for('main.dashboard')) return redirect(url_for('main.dashboard'))
else: else:
flash('The confirmation link is invalid or has expired.') flash('The confirmation link is invalid or has expired.')
return redirect(url_for('auth.unconfirmed')) return redirect(url_for('.unconfirmed'))
@auth.route('/unconfirmed') @auth.route('/unconfirmed')
@ -119,39 +121,32 @@ def resend_confirmation():
def reset_password_request(): def reset_password_request():
if current_user.is_authenticated: if current_user.is_authenticated:
return redirect(url_for('main.dashboard')) return redirect(url_for('main.dashboard'))
reset_password_request_form = ResetPasswordRequestForm( form = ResetPasswordRequestForm(prefix='reset-password-request-form')
prefix='reset-password-request-form') if form.validate_on_submit():
if reset_password_request_form.validate_on_submit(): user = User.query.filter_by(email=form.email.data.lower()).first()
submitted_email = reset_password_request_form.email.data if user is not None:
user = User.query.filter_by(email=submitted_email.lower()).first()
if user:
token = user.generate_reset_token() token = user.generate_reset_token()
msg = create_message(user.email, 'Reset Your Password', msg = create_message(user.email, 'Reset Your Password',
'auth/email/reset_password', token=token, 'auth/email/reset_password', token=token,
user=user) user=user)
send(msg) send(msg)
flash('An email with instructions to reset your password has been ' flash('An email with instructions to reset your password has been sent to you.') # noqa
'sent to you.') return redirect(url_for('.login'))
return redirect(url_for('auth.login')) return render_template('auth/reset_password_request.html.j2', form=form,
return render_template( title='Password Reset')
'auth/reset_password_request.html.j2',
reset_password_request_form=reset_password_request_form,
title='Password Reset')
@auth.route('/reset/<token>', methods=['GET', 'POST']) @auth.route('/reset/<token>', methods=['GET', 'POST'])
def reset_password(token): def reset_password(token):
if current_user.is_authenticated: if current_user.is_authenticated:
return redirect(url_for('main.dashboard')) return redirect(url_for('main.dashboard'))
reset_password_form = ResetPasswordForm(prefix='reset-password-form') form = ResetPasswordForm(prefix='reset-password-form')
if reset_password_form.validate_on_submit(): if form.validate_on_submit():
if User.reset_password(token, reset_password_form.password.data): if User.reset_password(token, form.password.data):
db.session.commit() db.session.commit()
flash('Your password has been updated.') flash('Your password has been updated.')
return redirect(url_for('auth.login')) return redirect(url_for('.login'))
else: else:
return redirect(url_for('main.index')) return redirect(url_for('main.index'))
return render_template('auth/reset_password.html.j2', return render_template('auth/reset_password.html.j2', form=form,
reset_password_form=reset_password_form, title='Password Reset', token=token)
title='Password Reset',
token=token)

34
web/app/corpora/events.py
View File

@ -24,27 +24,29 @@ corpus_analysis_sessions = {}
corpus_analysis_clients = {} corpus_analysis_clients = {}
@socketio.on('corpus_create_zip') @socketio.on('export_corpus')
@socketio_login_required @socketio_login_required
def corpus_create_zip(corpus_id): def export_corpus(corpus_id):
corpus = Corpus.query.get_or_404(corpus_id) # TODO: This should not be get_or_404 here - Socket.IO != HTTP request
corpus = Corpus.query.get(corpus_id)
if corpus is None:
response = {'code': 404, 'msg': 'Not found'}
socketio.emit('export_corpus', response, room=request.sid)
return
if corpus.status not in ['prepared', 'start analysis', 'stop analysis']:
response = {'code': 412, 'msg': 'Precondition Failed'}
socketio.emit('export_corpus', response, room=request.sid)
return
# delete old corpus archive if it exists/has been build before # delete old corpus archive if it exists/has been build before
if corpus.archive_file is not None: if corpus.archive_file is not None and os.path.isfile(corpus.archive_file):
if (os.path.isfile(corpus.archive_file)): os.remove(corpus.archive_file)
os.remove(corpus.archive_file)
root_dir = os.path.join(current_app.config['DATA_DIR'],
str(current_user.id),
'corpora')
base_dir = os.path.join(root_dir, str(corpus.id))
zip_name = corpus.title zip_name = corpus.title
zip_path = os.path.join(root_dir, zip_name) zip_path = os.path.join(current_user.path, 'corpora', zip_name)
corpus.archive_file = os.path.join(base_dir, zip_name) + '.zip' corpus.archive_file = os.path.join(corpus.path, zip_name) + '.zip'
db.session.commit() db.session.commit()
shutil.make_archive(zip_path, shutil.make_archive(zip_path, 'zip', corpus.path)
'zip',
base_dir)
shutil.move(zip_path + '.zip', corpus.archive_file) shutil.move(zip_path + '.zip', corpus.archive_file)
socketio.emit('corpus_zip_created', room=request.sid) socketio.emit('export_corpus_' + str(corpus.id), room=request.sid)
@socketio.on('corpus_analysis_init') @socketio.on('corpus_analysis_init')

400
web/app/corpora/views.py
View File

@ -1,4 +1,4 @@
from flask import (abort, current_app, flash, make_response, redirect, request, from flask import (abort, flash, make_response, redirect, request,
render_template, url_for, send_from_directory) render_template, url_for, send_from_directory)
from flask_login import current_user, login_required from flask_login import current_user, login_required
from . import corpora from . import corpora
@ -11,6 +11,7 @@ from jsonschema import validate
from .. import db from .. import db
from ..models import Corpus, CorpusFile, QueryResult from ..models import Corpus, CorpusFile, QueryResult
import json import json
import logging
import os import os
import shutil import shutil
import glob import glob
@ -22,106 +23,92 @@ from .import_corpus import check_zip_contents
@corpora.route('/add', methods=['GET', 'POST']) @corpora.route('/add', methods=['GET', 'POST'])
@login_required @login_required
def add_corpus(): def add_corpus():
add_corpus_form = AddCorpusForm() form = AddCorpusForm()
if add_corpus_form.validate_on_submit(): if form.validate_on_submit():
corpus = Corpus(creator=current_user, corpus = Corpus(creator=current_user,
description=add_corpus_form.description.data, description=form.description.data,
status='unprepared', title=add_corpus_form.title.data) title=form.title.data)
db.session.add(corpus) db.session.add(corpus)
db.session.commit() db.session.commit()
dir = os.path.join(current_app.config['DATA_DIR'],
str(corpus.user_id), 'corpora', str(corpus.id))
try: try:
os.makedirs(dir) os.makedirs(corpus.path)
except OSError: except OSError:
flash('[ERROR]: Could not add corpus!', 'corpus') logging.error('Make dir {} led to an OSError!'.format(corpus.path))
corpus.delete() db.session.delete(corpus)
else: db.session.commit()
url = url_for('corpora.corpus', corpus_id=corpus.id) abort(500)
flash('[<a href="{}">{}</a>] added'.format(url, corpus.title), flash('Corpus "{}" added!'.format(corpus.title), 'corpus')
'corpus') return redirect(url_for('.corpus', corpus_id=corpus.id))
return redirect(url_for('corpora.corpus', corpus_id=corpus.id)) return render_template('corpora/add_corpus.html.j2', form=form,
return render_template('corpora/add_corpus.html.j2',
add_corpus_form=add_corpus_form,
title='Add corpus') title='Add corpus')
@corpora.route('/import', methods=['GET', 'POST']) @corpora.route('/import', methods=['GET', 'POST'])
@login_required @login_required
def import_corpus(): def import_corpus():
import_corpus_form = ImportCorpusForm() form = ImportCorpusForm()
if import_corpus_form.is_submitted(): if form.is_submitted():
if not import_corpus_form.validate(): if not form.validate():
return make_response(import_corpus_form.errors, 400) return make_response(form.errors, 400)
corpus = Corpus(creator=current_user, corpus = Corpus(creator=current_user,
description=import_corpus_form.description.data, description=form.description.data,
status='unprepared', title=form.title.data)
title=import_corpus_form.title.data)
db.session.add(corpus) db.session.add(corpus)
db.session.commit() db.session.commit()
dir = os.path.join(current_app.config['DATA_DIR'],
str(corpus.user_id), 'corpora', str(corpus.id))
try: try:
os.makedirs(dir) os.makedirs(corpus.path)
except OSError: except OSError:
flash('[ERROR]: Could not import corpus!', 'corpus') logging.error('Make dir {} led to an OSError!'.format(corpus.path))
corpus.delete() db.session.delete(corpus)
db.session.commit()
flash('Internal Server Error', 'error')
return make_response(
{'redirect_url': url_for('.import_corpus')}, 500)
# Upload zip
archive_file = os.path.join(corpus.path, form.file.data.filename)
form.file.data.save(archive_file)
# Some checks to verify it is a valid exported corpus
with ZipFile(archive_file, 'r') as zip:
contents = zip.namelist()
if set(check_zip_contents).issubset(contents):
# Unzip
shutil.unpack_archive(archive_file, corpus.path)
# Register vrt files to corpus
vrts = glob.glob(corpus.path + '/*.vrt')
for file in vrts:
element_tree = ET.parse(file)
text_node = element_tree.find('text')
corpus_file = CorpusFile(
address=text_node.get('address', 'NULL'),
author=text_node.get('author', 'NULL'),
booktitle=text_node.get('booktitle', 'NULL'),
chapter=text_node.get('chapter', 'NULL'),
corpus=corpus,
editor=text_node.get('editor', 'NULL'),
filename=os.path.basename(file),
institution=text_node.get('institution', 'NULL'),
journal=text_node.get('journal', 'NULL'),
pages=text_node.get('pages', 'NULL'),
publisher=text_node.get('publisher', 'NULL'),
publishing_year=text_node.get('publishing_year', ''),
school=text_node.get('school', 'NULL'),
title=text_node.get('title', 'NULL')
)
db.session.add(corpus_file)
# finish import and redirect to imported corpus
corpus.status = 'prepared'
db.session.commit()
os.remove(archive_file)
flash('Corpus "{}" imported!'.format(corpus.title), 'corpus')
return make_response(
{'redirect_url': url_for('.corpus', corpus_id=corpus.id)}, 201)
else: else:
# Upload zip # If imported zip is not valid delete corpus and give feedback
archive_file = os.path.join(current_app.config['DATA_DIR'], dir, flash('Can not import corpus "{}" not imported: Invalid archive file!', 'error') # noqa
import_corpus_form.file.data.filename) tasks.delete_corpus(corpus.id)
corpus_dir = os.path.dirname(archive_file) return make_response(
import_corpus_form.file.data.save(archive_file) {'redirect_url': url_for('.import_corpus')}, 201)
# Some checks to verify it is a valid exported corpus return render_template('corpora/import_corpus.html.j2', form=form,
with ZipFile(archive_file, 'r') as zip:
contents = zip.namelist()
if set(check_zip_contents).issubset(contents):
# Unzip
shutil.unpack_archive(archive_file, corpus_dir)
# Register vrt files to corpus
vrts = glob.glob(corpus_dir + '/*.vrt')
for file in vrts:
element_tree = ET.parse(file)
text_node = element_tree.find('text')
corpus_file = CorpusFile(
address=text_node.get('address', 'NULL'),
author=text_node.get('author', 'NULL'),
booktitle=text_node.get('booktitle', 'NULL'),
chapter=text_node.get('chapter', 'NULL'),
corpus=corpus,
dir=dir,
editor=text_node.get('editor', 'NULL'),
filename=os.path.basename(file),
institution=text_node.get('institution', 'NULL'),
journal=text_node.get('journal', 'NULL'),
pages=text_node.get('pages', 'NULL'),
publisher=text_node.get('publisher', 'NULL'),
publishing_year=text_node.get('publishing_year', ''),
school=text_node.get('school', 'NULL'),
title=text_node.get('title', 'NULL'))
db.session.add(corpus_file)
# finish import and got to imported corpus
url = url_for('corpora.corpus', corpus_id=corpus.id)
corpus.status = 'prepared'
db.session.commit()
os.remove(archive_file)
flash('[<a href="{}">{}</a>] imported'.format(url,
corpus.title),
'corpus')
return make_response(
{'redirect_url': url_for('corpora.corpus',
corpus_id=corpus.id)},
201)
else:
# If imported zip is not valid delete corpus and give feedback
corpus.delete()
db.session.commit()
flash('Imported corpus is not valid.', 'error')
return make_response(
{'redirect_url': url_for('corpora.import_corpus')},
201)
return render_template('corpora/import_corpus.html.j2',
import_corpus_form=import_corpus_form,
title='Import Corpus') title='Import Corpus')
@ -131,31 +118,22 @@ def corpus(corpus_id):
corpus = Corpus.query.get_or_404(corpus_id) corpus = Corpus.query.get_or_404(corpus_id)
if not (corpus.creator == current_user or current_user.is_administrator()): if not (corpus.creator == current_user or current_user.is_administrator()):
abort(403) abort(403)
corpus_files = [dict(filename=corpus_file.filename, corpus_files = [corpus_file.to_dict() for corpus_file in corpus.files]
author=corpus_file.author, return render_template('corpora/corpus.html.j2', corpus=corpus,
title=corpus_file.title, corpus_files=corpus_files, title='Corpus')
publishing_year=corpus_file.publishing_year,
corpus_id=corpus.id,
id=corpus_file.id)
for corpus_file in corpus.files]
return render_template('corpora/corpus.html.j2',
corpus=corpus,
corpus_files=corpus_files,
title='Corpus')
@corpora.route('/<int:corpus_id>/export') @corpora.route('/<int:corpus_id>/download')
@login_required @login_required
def export_corpus(corpus_id): def download_corpus(corpus_id):
corpus = Corpus.query.get_or_404(corpus_id) corpus = Corpus.query.get_or_404(corpus_id)
if not (corpus.creator == current_user or current_user.is_administrator()): if not (corpus.creator == current_user or current_user.is_administrator()):
abort(403) abort(403)
# TODO: Check what happens here
dir = os.path.dirname(corpus.archive_file) dir = os.path.dirname(corpus.archive_file)
filename = os.path.basename(corpus.archive_file) filename = os.path.basename(corpus.archive_file)
return send_from_directory(directory=dir, return send_from_directory(as_attachment=True, directory=dir,
filename=filename, filename=filename, mimetype='zip')
mimetype='zip',
as_attachment=True)
@corpora.route('/<int:corpus_id>/analyse') @corpora.route('/<int:corpus_id>/analyse')
@ -168,7 +146,8 @@ def analyse_corpus(corpus_id):
display_options_form = DisplayOptionsForm( display_options_form = DisplayOptionsForm(
prefix='display-options-form', prefix='display-options-form',
result_context=request.args.get('context', 20), result_context=request.args.get('context', 20),
results_per_page=request.args.get('results_per_page', 30)) results_per_page=request.args.get('results_per_page', 30)
)
query_form = QueryForm(prefix='query-form', query_form = QueryForm(prefix='query-form',
query=request.args.get('query')) query=request.args.get('query'))
query_download_form = QueryDownloadForm(prefix='query-download-form') query_download_form = QueryDownloadForm(prefix='query-download-form')
@ -177,12 +156,12 @@ def analyse_corpus(corpus_id):
return render_template( return render_template(
'corpora/analyse_corpus.html.j2', 'corpora/analyse_corpus.html.j2',
corpus=corpus, corpus=corpus,
corpus_id=corpus_id,
display_options_form=display_options_form, display_options_form=display_options_form,
inspect_display_options_form=inspect_display_options_form,
query_form=query_form, query_form=query_form,
query_download_form=query_download_form, query_download_form=query_download_form,
inspect_display_options_form=inspect_display_options_form, title='Corpus analysis'
title='Corpus analysis') )
@corpora.route('/<int:corpus_id>/delete') @corpora.route('/<int:corpus_id>/delete')
@ -191,8 +170,8 @@ def delete_corpus(corpus_id):
corpus = Corpus.query.get_or_404(corpus_id) corpus = Corpus.query.get_or_404(corpus_id)
if not (corpus.creator == current_user or current_user.is_administrator()): if not (corpus.creator == current_user or current_user.is_administrator()):
abort(403) abort(403)
flash('Corpus "{}" marked for deletion!'.format(corpus.title), 'corpus')
tasks.delete_corpus(corpus_id) tasks.delete_corpus(corpus_id)
flash('Corpus deleted!', 'corpus')
return redirect(url_for('main.dashboard')) return redirect(url_for('main.dashboard'))
@ -202,43 +181,33 @@ def add_corpus_file(corpus_id):
corpus = Corpus.query.get_or_404(corpus_id) corpus = Corpus.query.get_or_404(corpus_id)
if not (corpus.creator == current_user or current_user.is_administrator()): if not (corpus.creator == current_user or current_user.is_administrator()):
abort(403) abort(403)
add_corpus_file_form = AddCorpusFileForm(corpus, form = AddCorpusFileForm(corpus, prefix='add-corpus-file-form')
prefix='add-corpus-file-form') if form.is_submitted():
if add_corpus_file_form.is_submitted(): if not form.validate():
if not add_corpus_file_form.validate(): return make_response(form.errors, 400)
return make_response(add_corpus_file_form.errors, 400)
# Save the file # Save the file
dir = os.path.join(str(corpus.user_id), 'corpora', str(corpus.id)) form.file.data.save(os.path.join(corpus.path, form.file.data.filename))
add_corpus_file_form.file.data.save( corpus_file = CorpusFile(address=form.address.data,
os.path.join(current_app.config['DATA_DIR'], dir, author=form.author.data,
add_corpus_file_form.file.data.filename)) booktitle=form.booktitle.data,
corpus_file = CorpusFile( chapter=form.chapter.data,
address=add_corpus_file_form.address.data, corpus=corpus,
author=add_corpus_file_form.author.data, editor=form.editor.data,
booktitle=add_corpus_file_form.booktitle.data, filename=form.file.data.filename,
chapter=add_corpus_file_form.chapter.data, institution=form.institution.data,
corpus=corpus, journal=form.journal.data,
dir=dir, pages=form.pages.data,
editor=add_corpus_file_form.editor.data, publisher=form.publisher.data,
filename=add_corpus_file_form.file.data.filename, publishing_year=form.publishing_year.data,
institution=add_corpus_file_form.institution.data, school=form.school.data,
journal=add_corpus_file_form.journal.data, title=form.title.data)
pages=add_corpus_file_form.pages.data,
publisher=add_corpus_file_form.publisher.data,
publishing_year=add_corpus_file_form.publishing_year.data,
school=add_corpus_file_form.school.data,
title=add_corpus_file_form.title.data)
db.session.add(corpus_file) db.session.add(corpus_file)
corpus.status = 'unprepared' corpus.status = 'unprepared'
db.session.commit() db.session.commit()
flash('Corpus file added!', 'corpus') flash('Corpus file "{}" added!'.format(corpus_file.filename), 'corpus')
return make_response( return make_response({'redirect_url': url_for('.corpus', corpus_id=corpus.id)}, 201) # noqa
{'redirect_url': url_for('corpora.corpus', corpus_id=corpus.id)}, return render_template('corpora/add_corpus_file.html.j2', corpus=corpus,
201) form=form, title='Add corpus file')
return render_template('corpora/add_corpus_file.html.j2',
corpus=corpus,
add_corpus_file_form=add_corpus_file_form,
title='Add corpus file')
@corpora.route('/<int:corpus_id>/files/<int:corpus_file_id>/delete') @corpora.route('/<int:corpus_id>/files/<int:corpus_file_id>/delete')
@ -250,9 +219,9 @@ def delete_corpus_file(corpus_id, corpus_file_id):
if not (corpus_file.corpus.creator == current_user if not (corpus_file.corpus.creator == current_user
or current_user.is_administrator()): or current_user.is_administrator()):
abort(403) abort(403)
flash('Corpus file "{}" marked for deletion!'.format(corpus_file.filename), 'corpus') # noqa
tasks.delete_corpus_file(corpus_file_id) tasks.delete_corpus_file(corpus_file_id)
flash('Corpus file deleted!', 'corpus') return redirect(url_for('.corpus', corpus_id=corpus_id))
return redirect(url_for('corpora.corpus', corpus_id=corpus_id))
@corpora.route('/<int:corpus_id>/files/<int:corpus_file_id>/download') @corpora.route('/<int:corpus_id>/files/<int:corpus_file_id>/download')
@ -264,9 +233,8 @@ def download_corpus_file(corpus_id, corpus_file_id):
if not (corpus_file.corpus.creator == current_user if not (corpus_file.corpus.creator == current_user
or current_user.is_administrator()): or current_user.is_administrator()):
abort(403) abort(403)
dir = os.path.join(current_app.config['DATA_DIR'], return send_from_directory(as_attachment=True,
corpus_file.dir) directory=os.path.dirname(corpus_file.path),
return send_from_directory(as_attachment=True, directory=dir,
filename=corpus_file.filename) filename=corpus_file.filename)
@ -275,47 +243,44 @@ def download_corpus_file(corpus_id, corpus_file_id):
@login_required @login_required
def corpus_file(corpus_id, corpus_file_id): def corpus_file(corpus_id, corpus_file_id):
corpus = Corpus.query.get_or_404(corpus_id) corpus = Corpus.query.get_or_404(corpus_id)
corpus_file = CorpusFile.query.get_or_404(corpus_file_id) if not (corpus.creator == current_user or current_user.is_administrator()):
if not corpus_file.corpus_id == corpus_id:
abort(404)
if not (corpus_file.corpus.creator == current_user
or current_user.is_administrator()):
abort(403) abort(403)
edit_corpus_file_form = EditCorpusFileForm(prefix='edit-corpus-file-form') corpus_file = CorpusFile.query.get_or_404(corpus_file_id)
if edit_corpus_file_form.validate_on_submit(): if corpus_file.corpus != corpus:
corpus_file.address = edit_corpus_file_form.address.data abort(404)
corpus_file.author = edit_corpus_file_form.author.data form = EditCorpusFileForm(prefix='edit-corpus-file-form')
corpus_file.booktitle = edit_corpus_file_form.booktitle.data if form.validate_on_submit():
corpus_file.chapter = edit_corpus_file_form.chapter.data corpus_file.address = form.address.data
corpus_file.editor = edit_corpus_file_form.editor.data corpus_file.author = form.author.data
corpus_file.institution = edit_corpus_file_form.institution.data corpus_file.booktitle = form.booktitle.data
corpus_file.journal = edit_corpus_file_form.journal.data corpus_file.chapter = form.chapter.data
corpus_file.pages = edit_corpus_file_form.pages.data corpus_file.editor = form.editor.data
corpus_file.publisher = edit_corpus_file_form.publisher.data corpus_file.institution = form.institution.data
corpus_file.publishing_year = \ corpus_file.journal = form.journal.data
edit_corpus_file_form.publishing_year.data corpus_file.pages = form.pages.data
corpus_file.school = edit_corpus_file_form.school.data corpus_file.publisher = form.publisher.data
corpus_file.title = edit_corpus_file_form.title.data corpus_file.publishing_year = form.publishing_year.data
corpus_file.school = form.school.data
corpus_file.title = form.title.data
corpus.status = 'unprepared' corpus.status = 'unprepared'
db.session.commit() db.session.commit()
flash('Corpus file edited!', 'corpus') flash('Corpus file "{}" edited!'.format(corpus_file.filename), 'corpus') # noqa
return redirect(url_for('corpora.corpus', corpus_id=corpus_id)) return redirect(url_for('.corpus', corpus_id=corpus_id))
# If no form is submitted or valid, fill out fields with current values # If no form is submitted or valid, fill out fields with current values
edit_corpus_file_form.address.data = corpus_file.address form.address.data = corpus_file.address
edit_corpus_file_form.author.data = corpus_file.author form.author.data = corpus_file.author
edit_corpus_file_form.booktitle.data = corpus_file.booktitle form.booktitle.data = corpus_file.booktitle
edit_corpus_file_form.chapter.data = corpus_file.chapter form.chapter.data = corpus_file.chapter
edit_corpus_file_form.editor.data = corpus_file.editor form.editor.data = corpus_file.editor
edit_corpus_file_form.institution.data = corpus_file.institution form.institution.data = corpus_file.institution
edit_corpus_file_form.journal.data = corpus_file.journal form.journal.data = corpus_file.journal
edit_corpus_file_form.pages.data = corpus_file.pages form.pages.data = corpus_file.pages
edit_corpus_file_form.publisher.data = corpus_file.publisher form.publisher.data = corpus_file.publisher
edit_corpus_file_form.publishing_year.data = corpus_file.publishing_year form.publishing_year.data = corpus_file.publishing_year
edit_corpus_file_form.school.data = corpus_file.school form.school.data = corpus_file.school
edit_corpus_file_form.title.data = corpus_file.title form.title.data = corpus_file.title
return render_template('corpora/corpus_file.html.j2', return render_template('corpora/corpus_file.html.j2', corpus=corpus,
corpus_file=corpus_file, corpus=corpus, corpus_file=corpus_file, form=form,
edit_corpus_file_form=edit_corpus_file_form,
title='Edit corpus file') title='Edit corpus file')
@ -327,10 +292,10 @@ def prepare_corpus(corpus_id):
abort(403) abort(403)
if corpus.files.all(): if corpus.files.all():
tasks.build_corpus(corpus_id) tasks.build_corpus(corpus_id)
flash('Building Corpus...', 'corpus') flash('Corpus "{}" has been marked to get build!'.format(corpus.title), 'corpus') # noqa
else: else:
flash('Can not build corpus, please add corpus file(s).', 'corpus') flash('Can not build corpus "{}": No corpus file(s)!'.format(corpus.title), 'error') # noqa
return redirect(url_for('corpora.corpus', corpus_id=corpus_id)) return redirect(url_for('.corpus', corpus_id=corpus_id))
# Following are view functions to add, view etc. exported results. # Following are view functions to add, view etc. exported results.
@ -340,35 +305,29 @@ def add_query_result():
''' '''
View to import a result as a json file. View to import a result as a json file.
''' '''
add_query_result_form = AddQueryResultForm(prefix='add-query-result-form') form = AddQueryResultForm(prefix='add-query-result-form')
if add_query_result_form.is_submitted(): if form.is_submitted():
if not add_query_result_form.validate(): if not form.validate():
return make_response(add_query_result_form.errors, 400) return make_response(form.errors, 400)
query_result = QueryResult( query_result = QueryResult(creator=current_user,
creator=current_user, description=form.description.data,
description=add_query_result_form.description.data, filename=form.file.data.filename,
filename=add_query_result_form.file.data.filename, title=form.title.data)
title=add_query_result_form.title.data
)
db.session.add(query_result) db.session.add(query_result)
db.session.commit() db.session.commit()
# create paths to save the uploaded json file
query_result_dir = os.path.join(current_app.config['DATA_DIR'],
str(current_user.id),
'query_results',
str(query_result.id))
try: try:
os.makedirs(query_result_dir) os.makedirs(query_result.path)
except Exception: except OSError:
logging.error('Make dir {} led to an OSError!'.format(query_result.path)) # noqa
db.session.delete(query_result) db.session.delete(query_result)
db.session.commit() db.session.commit()
flash('Internal Server Error', 'error') flash('Internal Server Error', 'error')
redirect_url = url_for('corpora.add_query_result') return make_response(
return make_response({'redirect_url': redirect_url}, 500) {'redirect_url': url_for('.add_query_result')}, 500)
# save the uploaded file # save the uploaded file
query_result_file_path = os.path.join(query_result_dir, query_result_file_path = os.path.join(query_result.path,
query_result.filename) query_result.filename)
add_query_result_form.file.data.save(query_result_file_path) form.file.data.save(query_result_file_path)
# parse json from file # parse json from file
with open(query_result_file_path, 'r') as file: with open(query_result_file_path, 'r') as file:
query_result_file_content = json.load(file) query_result_file_content = json.load(file)
@ -381,19 +340,16 @@ def add_query_result():
except Exception: except Exception:
tasks.delete_query_result(query_result.id) tasks.delete_query_result(query_result.id)
flash('Uploaded file is invalid', 'result') flash('Uploaded file is invalid', 'result')
redirect_url = url_for('corpora.add_query_result') return make_response(
return make_response({'redirect_url': redirect_url}, 201) {'redirect_url': url_for('.add_query_result')}, 201)
query_result_file_content.pop('matches') query_result_file_content.pop('matches')
query_result_file_content.pop('cpos_lookup') query_result_file_content.pop('cpos_lookup')
query_result.query_metadata = query_result_file_content query_result.query_metadata = query_result_file_content
db.session.commit() db.session.commit()
flash('Query result added!', 'result') flash('Query result added!', 'result')
redirect_url = url_for('corpora.query_result', return make_response({'redirect_url': url_for('.query_result', query_result_id=query_result.id)}, 201) # noqa
query_result_id=query_result.id)
return make_response({'redirect_url': redirect_url}, 201)
return render_template('corpora/query_results/add_query_result.html.j2', return render_template('corpora/query_results/add_query_result.html.j2',
add_query_result_form=add_query_result_form, form=form, title='Add query result')
title='Add query result')
@corpora.route('/result/<int:query_result_id>') @corpora.route('/result/<int:query_result_id>')
@ -404,8 +360,7 @@ def query_result(query_result_id):
or current_user.is_administrator()): or current_user.is_administrator()):
abort(403) abort(403)
return render_template('corpora/query_results/query_result.html.j2', return render_template('corpora/query_results/query_result.html.j2',
query_result=query_result, query_result=query_result, title='Query result')
title='Query result')
@corpora.route('/result/<int:query_result_id>/inspect') @corpora.route('/result/<int:query_result_id>/inspect')
@ -427,14 +382,7 @@ def inspect_query_result(query_result_id):
inspect_display_options_form = InspectDisplayOptionsForm( inspect_display_options_form = InspectDisplayOptionsForm(
prefix='inspect-display-options-form' prefix='inspect-display-options-form'
) )
query_result_file_path = os.path.join( with open(query_result.path, 'r') as query_result_file:
current_app.config['DATA_DIR'],
str(current_user.id),
'query_results',
str(query_result.id),
query_result.filename
)
with open(query_result_file_path, 'r') as query_result_file:
query_result_file_content = json.load(query_result_file) query_result_file_content = json.load(query_result_file)
return render_template('corpora/query_results/inspect.html.j2', return render_template('corpora/query_results/inspect.html.j2',
query_result=query_result, query_result=query_result,
@ -452,8 +400,8 @@ def delete_query_result(query_result_id):
if not (query_result.creator == current_user if not (query_result.creator == current_user
or current_user.is_administrator()): or current_user.is_administrator()):
abort(403) abort(403)
flash('Query result "{}" has been marked for deletion!'.format(query_result), 'result') # noqa
tasks.delete_query_result(query_result_id) tasks.delete_query_result(query_result_id)
flash('Query result deleted!', 'result')
return redirect(url_for('services.service', service="corpus_analysis")) return redirect(url_for('services.service', service="corpus_analysis"))
@ -464,10 +412,6 @@ def download_query_result(query_result_id):
if not (query_result.creator == current_user if not (query_result.creator == current_user
or current_user.is_administrator()): or current_user.is_administrator()):
abort(403) abort(403)
query_result_dir = os.path.join(current_app.config['DATA_DIR'],
str(current_user.id),
'query_results',
str(query_result.id))
return send_from_directory(as_attachment=True, return send_from_directory(as_attachment=True,
directory=query_result_dir, directory=os.path.dirname(query_result.path),
filename=query_result.filename) filename=query_result.filename)

4
web/app/decorators.py
View File

@ -38,7 +38,7 @@ def socketio_admin_required(f):
if current_user.is_administrator: if current_user.is_administrator:
return f(*args, **kwargs) return f(*args, **kwargs)
else: else:
response = {'code': 401, 'desc': 'Unauthorized'} response = {'code': 401, 'msg': 'Unauthorized'}
socketio.emit(request.event['message'], response, room=request.sid) socketio.emit(request.event['message'], response, room=request.sid)
return wrapped return wrapped
@ -49,6 +49,6 @@ def socketio_login_required(f):
if current_user.is_authenticated: if current_user.is_authenticated:
return f(*args, **kwargs) return f(*args, **kwargs)
else: else:
response = {'code': 401, 'desc': 'Unauthorized'} response = {'code': 401, 'msg': 'Unauthorized'}
socketio.emit(request.event['message'], response, room=request.sid) socketio.emit(request.event['message'], response, room=request.sid)
return wrapped return wrapped

4
web/app/email.py
View File

@ -1,11 +1,11 @@
from flask import render_template from flask import current_app, render_template
from flask_mail import Message from flask_mail import Message
from . import mail from . import mail
from .decorators import background from .decorators import background
def create_message(recipient, subject, template, **kwargs): def create_message(recipient, subject, template, **kwargs):
msg = Message('[nopaque] {}'.format(subject), recipients=[recipient]) msg = Message('{} {}'.format(current_app.config['NOPAQUE_MAIL_SUBJECT_PREFIX'], subject), recipients=[recipient]) # noqa
msg.body = render_template('{}.txt.j2'.format(template), **kwargs) msg.body = render_template('{}.txt.j2'.format(template), **kwargs)
msg.html = render_template('{}.html.j2'.format(template), **kwargs) msg.html = render_template('{}.html.j2'.format(template), **kwargs)
return msg return msg

40
web/app/events.py
View File

@ -33,38 +33,24 @@ def disconnect():
connected_sessions.remove(request.sid) connected_sessions.remove(request.sid)
@socketio.on('user_data_stream_init') @socketio.on('start_user_session')
@socketio_login_required @socketio_login_required
def user_data_stream_init(): def start_user_session(user_id):
socketio.start_background_task(user_data_stream, if not (current_user.id == user_id or current_user.is_administrator):
return
socketio.start_background_task(user_session,
current_app._get_current_object(), current_app._get_current_object(),
current_user.id, request.sid) user_id, request.sid)
@socketio.on('foreign_user_data_stream_init') def user_session(app, user_id, session_id):
@socketio_login_required
@socketio_admin_required
def foreign_user_data_stream_init(user_id):
socketio.start_background_task(user_data_stream,
current_app._get_current_object(),
user_id, request.sid, foreign=True)
def user_data_stream(app, user_id, session_id, foreign=False):
''' '''
' Sends initial corpus and job lists to the client. Afterwards it checks ' Sends initial user data to the client. Afterwards it checks every 3s if
' every 3 seconds if changes to the initial values appeared. If changes are ' changes to the initial values appeared. If changes are detected, a
' detected, a RFC 6902 compliant JSON patch gets send. ' RFC 6902 compliant JSON patch gets send.
'
' NOTE: The initial values are send as a init events.
' The JSON patches are send as update events.
''' '''
if foreign: init_event = 'user_{}_init'.format(user_id)
init_event = 'foreign_user_data_stream_init' patch_event = 'user_{}_patch'.format(user_id)
update_event = 'foreign_user_data_stream_update'
else:
init_event = 'user_data_stream_init'
update_event = 'user_data_stream_update'
with app.app_context(): with app.app_context():
# Gather current values from database. # Gather current values from database.
user = User.query.get(user_id) user = User.query.get(user_id)
@ -80,7 +66,7 @@ def user_data_stream(app, user_id, session_id, foreign=False):
new_user_dict) new_user_dict)
# In case there are patches, send them to the client. # In case there are patches, send them to the client.
if user_patch: if user_patch:
socketio.emit(update_event, user_patch.to_string(), socketio.emit(patch_event, user_patch.to_string(),
room=session_id) room=session_id)
# Set new values as references for the next iteration. # Set new values as references for the next iteration.
user_dict = new_user_dict user_dict = new_user_dict

2
web/app/jobs/__init__.py
View File

@ -2,4 +2,4 @@ from flask import Blueprint
jobs = Blueprint('jobs', __name__) jobs = Blueprint('jobs', __name__)
from . import views # noqa from . import views

37
web/app/jobs/views.py
View File

@ -1,4 +1,4 @@
from flask import (abort, current_app, flash, redirect, render_template, from flask import (abort, flash, redirect, render_template,
send_from_directory, url_for) send_from_directory, url_for)
from flask_login import current_user, login_required from flask_login import current_user, login_required
from . import jobs from . import jobs
@ -14,13 +14,8 @@ def job(job_id):
job = Job.query.get_or_404(job_id) job = Job.query.get_or_404(job_id)
if not (job.creator == current_user or current_user.is_administrator()): if not (job.creator == current_user or current_user.is_administrator()):
abort(403) abort(403)
job_inputs = [dict(filename=input.filename, job_inputs = [job_input.to_dict() for job_input in job.inputs]
id=input.id, return render_template('jobs/job.html.j2', job=job, job_inputs=job_inputs,
job_id=job.id)
for input in job.inputs]
return render_template('jobs/job.html.j2',
job=job,
job_inputs=job_inputs,
title='Job') title='Job')
@ -31,22 +26,19 @@ def delete_job(job_id):
if not (job.creator == current_user or current_user.is_administrator()): if not (job.creator == current_user or current_user.is_administrator()):
abort(403) abort(403)
tasks.delete_job(job_id) tasks.delete_job(job_id)
flash('Job has been deleted!', 'job') flash('Job has been marked for deletion!', 'job')
return redirect(url_for('main.dashboard')) return redirect(url_for('main.dashboard'))
@jobs.route('/<int:job_id>/inputs/<int:job_input_id>/download') @jobs.route('/<int:job_id>/inputs/<int:job_input_id>/download')
@login_required @login_required
def download_job_input(job_id, job_input_id): def download_job_input(job_id, job_input_id):
job_input = JobInput.query.get_or_404(job_input_id) job_input = JobInput.query.filter(JobInput.job_id == job_id, JobInput.id == job_input_id).first_or_404() # noqa
if not job_input.job_id == job_id:
abort(404)
if not (job_input.job.creator == current_user if not (job_input.job.creator == current_user
or current_user.is_administrator()): or current_user.is_administrator()):
abort(403) abort(403)
dir = os.path.join(current_app.config['DATA_DIR'], return send_from_directory(as_attachment=True,
job_input.dir) directory=os.path.dirname(job_input.path),
return send_from_directory(as_attachment=True, directory=dir,
filename=job_input.filename) filename=job_input.filename)
@ -56,23 +48,20 @@ def download_job_input(job_id, job_input_id):
def restart(job_id): def restart(job_id):
job = Job.query.get_or_404(job_id) job = Job.query.get_or_404(job_id)
if job.status != 'failed': if job.status != 'failed':
flash('Could not restart job: status is not "failed"', 'error') flash('Can not restart job "{}": Status is not "failed"'.format(job.title), 'error') # noqa
else: else:
tasks.restart_job(job_id) tasks.restart_job(job_id)
flash('Job has been restarted!', 'job') flash('Job "{}" has been marked to get restarted!'.format(job.title), 'job') # noqa
return redirect(url_for('jobs.job', job_id=job_id)) return redirect(url_for('.job', job_id=job_id))
@jobs.route('/<int:job_id>/results/<int:job_result_id>/download') @jobs.route('/<int:job_id>/results/<int:job_result_id>/download')
@login_required @login_required
def download_job_result(job_id, job_result_id): def download_job_result(job_id, job_result_id):
job_result = JobResult.query.get_or_404(job_result_id) job_result = JobResult.query.filter(JobResult.job_id == job_id, JobResult.id == job_result_id).first_or_404() # noqa
if not job_result.job_id == job_id:
abort(404)
if not (job_result.job.creator == current_user if not (job_result.job.creator == current_user
or current_user.is_administrator()): or current_user.is_administrator()):
abort(403) abort(403)
dir = os.path.join(current_app.config['DATA_DIR'], return send_from_directory(as_attachment=True,
job_result.dir) directory=os.path.dirname(job_result.path),
return send_from_directory(as_attachment=True, directory=dir,
filename=job_result.filename) filename=job_result.filename)

2
web/app/main/__init__.py
View File

@ -2,4 +2,4 @@ from flask import Blueprint
main = Blueprint('main', __name__) main = Blueprint('main', __name__)
from . import views # noqa from . import views

25
web/app/main/views.py
View File

@ -7,17 +7,16 @@ from ..models import User
@main.route('/', methods=['GET', 'POST']) @main.route('/', methods=['GET', 'POST'])
def index(): def index():
login_form = LoginForm(prefix='login-form') form = LoginForm(prefix='login-form')
if login_form.validate_on_submit(): if form.validate_on_submit():
user = User.query.filter_by(username=login_form.user.data).first() user = User.query.filter_by(username=form.user.data).first()
if user is None: if user is None:
user = User.query.filter_by(email=login_form.user.data).first() user = User.query.filter_by(email=form.user.data.lower()).first()
if user is not None and user.verify_password(login_form.password.data): if user is not None and user.verify_password(form.password.data):
login_user(user, login_form.remember_me.data) login_user(user, form.remember_me.data)
return redirect(url_for('main.dashboard')) return redirect(url_for('.dashboard'))
flash('Invalid email/username or password.') flash('Invalid email/username or password.')
return render_template('main/index.html.j2', login_form=login_form, return render_template('main/index.html.j2', form=form, title='nopaque')
title='nopaque')
@main.route('/about_and_faq') @main.route('/about_and_faq')
@ -31,7 +30,6 @@ def dashboard():
return render_template('main/dashboard.html.j2', title='Dashboard') return render_template('main/dashboard.html.j2', title='Dashboard')
@main.route('/news') @main.route('/news')
def news(): def news():
return render_template('main/news.html.j2', title='News') return render_template('main/news.html.j2', title='News')
@ -40,12 +38,9 @@ def news():
@main.route('/privacy_policy') @main.route('/privacy_policy')
def privacy_policy(): def privacy_policy():
return render_template('main/privacy_policy.html.j2', return render_template('main/privacy_policy.html.j2',
title=('Information on the processing of personal' title='Privacy statement (GDPR)')
' data for the nopaque platform (GDPR)'))
@main.route('/terms_of_use') @main.route('/terms_of_use')
def terms_of_use(): def terms_of_use():
return render_template('main/terms_of_use.html.j2', return render_template('main/terms_of_use.html.j2', title='Terms of Use')
title='General Terms of Use of the platform '
'nopaque')

290
web/app/models.py
View File

@ -1,12 +1,12 @@
from datetime import datetime from datetime import datetime
from flask import current_app from flask import current_app, url_for
from flask_login import UserMixin, AnonymousUserMixin from flask_login import UserMixin, AnonymousUserMixin
from itsdangerous import BadSignature, TimedJSONWebSignatureSerializer from itsdangerous import BadSignature, TimedJSONWebSignatureSerializer
from time import sleep from time import sleep
from werkzeug.security import generate_password_hash, check_password_hash from werkzeug.security import generate_password_hash, check_password_hash
from werkzeug.utils import secure_filename
import xml.etree.ElementTree as ET import xml.etree.ElementTree as ET
from . import db, login_manager from . import db, login_manager
import logging
import os import os
import shutil import shutil
@ -35,7 +35,7 @@ class Role(db.Model):
# Fields # Fields
default = db.Column(db.Boolean, default=False, index=True) default = db.Column(db.Boolean, default=False, index=True)
name = db.Column(db.String(64), unique=True) name = db.Column(db.String(64), unique=True)
permissions = db.Column(db.BigInteger) permissions = db.Column(db.Integer)
# Relationships # Relationships
users = db.relationship('User', backref='role', lazy='dynamic') users = db.relationship('User', backref='role', lazy='dynamic')
@ -54,7 +54,7 @@ class Role(db.Model):
''' '''
String representation of the Role. For human readability. String representation of the Role. For human readability.
''' '''
return '<Role {role_name}>'.format(role_name=self.name) return '<Role {}>'.format(self.name)
def add_permission(self, perm): def add_permission(self, perm):
''' '''
@ -138,6 +138,19 @@ class User(UserMixin, db.Model):
cascade='save-update, merge, delete', cascade='save-update, merge, delete',
lazy='dynamic') lazy='dynamic')
@property
def path(self):
return os.path.join(current_app.config['NOPAQUE_DATA_DIR'],
str(self.id))
@property
def password(self):
raise AttributeError('password is not a readable attribute')
@password.setter
def password(self, password):
self.password_hash = generate_password_hash(password)
def to_dict(self): def to_dict(self):
return {'id': self.id, return {'id': self.id,
'role_id': self.role_id, 'role_id': self.role_id,
@ -145,28 +158,29 @@ class User(UserMixin, db.Model):
'email': self.email, 'email': self.email,
'last_seen': self.last_seen.timestamp(), 'last_seen': self.last_seen.timestamp(),
'member_since': self.member_since.timestamp(), 'member_since': self.member_since.timestamp(),
'username': self.username,
'settings': {'dark_mode': self.setting_dark_mode, 'settings': {'dark_mode': self.setting_dark_mode,
'job_status_mail_notifications': 'job_status_mail_notifications':
self.setting_job_status_mail_notifications, self.setting_job_status_mail_notifications,
'job_status_site_notifications': 'job_status_site_notifications':
self.setting_job_status_site_notifications}, self.setting_job_status_site_notifications},
'username': self.username,
'corpora': {corpus.id: corpus.to_dict() 'corpora': {corpus.id: corpus.to_dict()
for corpus in self.corpora}, for corpus in self.corpora},
'jobs': {job.id: job.to_dict() for job in self.jobs}, 'jobs': {job.id: job.to_dict() for job in self.jobs},
'query_results': {query_result.id: query_result.to_dict() 'query_results': {query_result.id: query_result.to_dict()
for query_result in self.query_results}} for query_result in self.query_results},
'role': self.role.to_dict()}
def __repr__(self): def __repr__(self):
''' '''
String representation of the User. For human readability. String representation of the User. For human readability.
''' '''
return '<User {username}>'.format(username=self.username) return '<User {}>'.format(self.username)
def __init__(self, **kwargs): def __init__(self, **kwargs):
super(User, self).__init__(**kwargs) super(User, self).__init__(**kwargs)
if self.role is None: if self.role is None:
if self.email == current_app.config['ADMIN_EMAIL_ADRESS']: if self.email == current_app.config['NOPAQUE_ADMIN']:
self.role = Role.query.filter_by(name='Administrator').first() self.role = Role.query.filter_by(name='Administrator').first()
if self.role is None: if self.role is None:
self.role = Role.query.filter_by(default=True).first() self.role = Role.query.filter_by(default=True).first()
@ -219,14 +233,6 @@ class User(UserMixin, db.Model):
db.session.add(user) db.session.add(user)
return True return True
@property
def password(self):
raise AttributeError('password is not a readable attribute')
@password.setter
def password(self, password):
self.password_hash = generate_password_hash(password)
def verify_password(self, password): def verify_password(self, password):
return check_password_hash(self.password_hash, password) return check_password_hash(self.password_hash, password)
@ -243,17 +249,11 @@ class User(UserMixin, db.Model):
''' '''
return self.can(Permission.ADMIN) return self.can(Permission.ADMIN)
def ping(self):
self.last_seen = datetime.utcnow()
db.session.add(self)
def delete(self): def delete(self):
''' '''
Delete the user and its corpora and jobs from database and filesystem. Delete the user and its corpora and jobs from database and filesystem.
''' '''
user_dir = os.path.join(current_app.config['DATA_DIR'], shutil.rmtree(self.path, ignore_errors=True)
str(self.id))
shutil.rmtree(user_dir, ignore_errors=True)
db.session.delete(self) db.session.delete(self)
@ -279,17 +279,32 @@ class JobInput(db.Model):
# Foreign keys # Foreign keys
job_id = db.Column(db.Integer, db.ForeignKey('jobs.id')) job_id = db.Column(db.Integer, db.ForeignKey('jobs.id'))
# Fields # Fields
dir = db.Column(db.String(255))
filename = db.Column(db.String(255)) filename = db.Column(db.String(255))
@property
def download_url(self):
return url_for('jobs.download_job_input', job_id=self.job_id,
job_input_id=self.id)
@property
def path(self):
return os.path.join(self.job.path, self.filename)
@property
def url(self):
return url_for('jobs.job', job_id=self.job_id,
_anchor='job-{}-input-{}'.format(self.job_id, self.id))
def __repr__(self): def __repr__(self):
''' '''
String representation of the JobInput. For human readability. String representation of the JobInput. For human readability.
''' '''
return '<JobInput {filename}>'.format(filename=self.filename) return '<JobInput {}>'.format(self.filename)
def to_dict(self): def to_dict(self):
return {'id': self.id, return {'download_url': self.download_url,
'url': self.url,
'id': self.id,
'job_id': self.job_id, 'job_id': self.job_id,
'filename': self.filename} 'filename': self.filename}
@ -304,17 +319,32 @@ class JobResult(db.Model):
# Foreign keys # Foreign keys
job_id = db.Column(db.Integer, db.ForeignKey('jobs.id')) job_id = db.Column(db.Integer, db.ForeignKey('jobs.id'))
# Fields # Fields
dir = db.Column(db.String(255))
filename = db.Column(db.String(255)) filename = db.Column(db.String(255))
@property
def download_url(self):
return url_for('jobs.download_job_result', job_id=self.job_id,
job_result_id=self.id)
@property
def path(self):
return os.path.join(self.job.path, 'output', self.filename)
@property
def url(self):
return url_for('jobs.job', job_id=self.job_id,
_anchor='job-{}-result-{}'.format(self.job_id, self.id))
def __repr__(self): def __repr__(self):
''' '''
String representation of the JobResult. For human readability. String representation of the JobResult. For human readability.
''' '''
return '<JobResult {filename}>'.format(filename=self.filename) return '<JobResult {}>'.format(self.filename)
def to_dict(self): def to_dict(self):
return {'id': self.id, return {'download_url': self.download_url,
'url': self.url,
'id': self.id,
'job_id': self.job_id, 'job_id': self.job_id,
'filename': self.filename} 'filename': self.filename}
@ -334,7 +364,6 @@ class Job(db.Model):
end_date = db.Column(db.DateTime()) end_date = db.Column(db.DateTime())
mem_mb = db.Column(db.Integer) mem_mb = db.Column(db.Integer)
n_cores = db.Column(db.Integer) n_cores = db.Column(db.Integer)
secure_filename = db.Column(db.String(32))
service = db.Column(db.String(64)) service = db.Column(db.String(64))
''' '''
' Service specific arguments as string list. ' Service specific arguments as string list.
@ -349,25 +378,20 @@ class Job(db.Model):
cascade='save-update, merge, delete') cascade='save-update, merge, delete')
results = db.relationship('JobResult', backref='job', lazy='dynamic', results = db.relationship('JobResult', backref='job', lazy='dynamic',
cascade='save-update, merge, delete') cascade='save-update, merge, delete')
notification_data = db.relationship('NotificationData',
cascade='save-update, merge, delete', @property
uselist=False, def path(self):
back_populates='job') # One-to-One relationship return os.path.join(self.creator.path, 'jobs', str(self.id))
notification_email_data = db.relationship('NotificationEmailData',
cascade='save-update, merge, delete', @property
back_populates='job') def url(self):
return url_for('jobs.job', job_id=self.id)
def __repr__(self): def __repr__(self):
''' '''
String representation of the Job. For human readability. String representation of the Job. For human readability.
''' '''
return '<Job {job_title}>'.format(job_title=self.title) return '<Job {}>'.format(self.title)
def create_secure_filename(self):
'''
Takes the job.title string nad cratesa a secure filename from this.
'''
self.secure_filename = secure_filename(self.title)
def delete(self): def delete(self):
''' '''
@ -383,11 +407,7 @@ class Job(db.Model):
db.session.commit() db.session.commit()
sleep(1) sleep(1)
db.session.refresh(self) db.session.refresh(self)
job_dir = os.path.join(current_app.config['DATA_DIR'], shutil.rmtree(self.path, ignore_errors=True)
str(self.user_id),
'jobs',
str(self.id))
shutil.rmtree(job_dir, ignore_errors=True)
db.session.delete(self) db.session.delete(self)
def restart(self): def restart(self):
@ -397,89 +417,27 @@ class Job(db.Model):
if self.status != 'failed': if self.status != 'failed':
raise Exception('Could not restart job: status is not "failed"') raise Exception('Could not restart job: status is not "failed"')
job_dir = os.path.join(current_app.config['DATA_DIR'], shutil.rmtree(os.path.join(self.path, 'output'), ignore_errors=True)
str(self.user_id), shutil.rmtree(os.path.join(self.path, 'pyflow.data'), ignore_errors=True) # noqa
'jobs',
str(self.id))
shutil.rmtree(os.path.join(job_dir, 'output'), ignore_errors=True)
shutil.rmtree(os.path.join(job_dir, 'pyflow.data'), ignore_errors=True)
self.end_date = None self.end_date = None
self.status = 'submitted' self.status = 'submitted'
def to_dict(self): def to_dict(self):
return {'id': self.id, return {'url': self.url,
'id': self.id,
'user_id': self.user_id, 'user_id': self.user_id,
'creation_date': self.creation_date.timestamp(), 'creation_date': self.creation_date.timestamp(),
'description': self.description, 'description': self.description,
'end_date': (self.end_date.timestamp() if self.end_date else 'end_date': (self.end_date.timestamp() if self.end_date else
None), None),
'inputs': {input.id: input.to_dict() for input in self.inputs},
'mem_mb': self.mem_mb,
'n_cores': self.n_cores,
'results': {result.id: result.to_dict()
for result in self.results},
'service': self.service, 'service': self.service,
'service_args': self.service_args, 'service_args': self.service_args,
'service_version': self.service_version, 'service_version': self.service_version,
'status': self.status, 'status': self.status,
'title': self.title} 'title': self.title,
'inputs': {input.id: input.to_dict() for input in self.inputs},
'results': {result.id: result.to_dict()
class NotificationData(db.Model): for result in self.results}}
'''
Class to define notification data used for sending a notification mail with
nopaque_notify.
'''
__tablename__ = 'notification_data'
# Primary key
id = db.Column(db.Integer, primary_key=True)
# Foreign Key
job_id = db.Column(db.Integer, db.ForeignKey('jobs.id'))
# relationships
job = db.relationship('Job', back_populates='notification_data')
# Fields
notified_on = db.Column(db.String(16), default=None)
def __repr__(self):
'''
String representation of the NotificationData. For human readability.
'''
return '<NotificationData {id}>'.format(id=self.id)
def to_dict(self):
return {'id': self.id,
'job_id': self.job_id,
'job': self.job,
'notified': self.notified}
class NotificationEmailData(db.Model):
'''
Class to define data that will be used to send a corresponding Notification
via email.
'''
__tablename__ = 'notification_email_data'
# Primary Key
id = db.Column(db.Integer, primary_key=True)
# Foreign Key
job_id = db.Column(db.Integer, db.ForeignKey('jobs.id'))
# relationships
job = db.relationship('Job', back_populates='notification_email_data')
notify_status = db.Column(db.String(16), default=None)
creation_date = db.Column(db.DateTime(), default=datetime.utcnow)
def __repr__(self):
'''
String representation of the NotificationEmailData. For human readability.
'''
return '<NotificationData {id}>'.format(id=self.id)
def to_dict(self):
return {'id': self.id,
'job_id': self.job_id,
'job': self.job,
'notify_status': self.notify_status,
'creation_date': self.creation_date}
class CorpusFile(db.Model): class CorpusFile(db.Model):
@ -496,7 +454,6 @@ class CorpusFile(db.Model):
author = db.Column(db.String(255)) author = db.Column(db.String(255))
booktitle = db.Column(db.String(255)) booktitle = db.Column(db.String(255))
chapter = db.Column(db.String(255)) chapter = db.Column(db.String(255))
dir = db.Column(db.String(255))
editor = db.Column(db.String(255)) editor = db.Column(db.String(255))
filename = db.Column(db.String(255)) filename = db.Column(db.String(255))
institution = db.Column(db.String(255)) institution = db.Column(db.String(255))
@ -507,21 +464,33 @@ class CorpusFile(db.Model):
school = db.Column(db.String(255)) school = db.Column(db.String(255))
title = db.Column(db.String(255)) title = db.Column(db.String(255))
@property
def download_url(self):
return url_for('corpora.download_corpus_file',
corpus_id=self.corpus_id, corpus_file_id=self.id)
@property
def path(self):
return os.path.join(self.corpus.path, self.filename)
@property
def url(self):
return url_for('corpora.corpus_file', corpus_id=self.corpus_id,
corpus_file_id=self.id)
def delete(self): def delete(self):
corpus_file_path = os.path.join(current_app.config['DATA_DIR'],
str(self.corpus.user_id),
'corpora',
str(self.corpus_id),
self.filename)
try: try:
os.remove(corpus_file_path) os.remove(self.path)
except OSError: except OSError:
logging.error('Removing {} led to an OSError!'.format(self.path))
pass pass
db.session.delete(self) db.session.delete(self)
self.corpus.status = 'unprepared' self.corpus.status = 'unprepared'
def to_dict(self): def to_dict(self):
return {'id': self.id, return {'download_url': self.download_url,
'url': self.url,
'id': self.id,
'corpus_id': self.corpus_id, 'corpus_id': self.corpus_id,
'address': self.address, 'address': self.address,
'author': self.author, 'author': self.author,
@ -553,37 +522,48 @@ class Corpus(db.Model):
description = db.Column(db.String(255)) description = db.Column(db.String(255))
last_edited_date = db.Column(db.DateTime(), default=datetime.utcnow) last_edited_date = db.Column(db.DateTime(), default=datetime.utcnow)
max_nr_of_tokens = db.Column(db.BigInteger, default=2147483647) max_nr_of_tokens = db.Column(db.BigInteger, default=2147483647)
status = db.Column(db.String(16)) status = db.Column(db.String(16), default='unprepared')
title = db.Column(db.String(32)) title = db.Column(db.String(32))
archive_file = db.Column(db.String(255)) archive_file = db.Column(db.String(255))
# Relationships # Relationships
files = db.relationship('CorpusFile', backref='corpus', lazy='dynamic', files = db.relationship('CorpusFile', backref='corpus', lazy='dynamic',
cascade='save-update, merge, delete') cascade='save-update, merge, delete')
@property
def analysis_url(self):
return url_for('corpora.analyse_corpus', corpus_id=self.id)
@property
def path(self):
return os.path.join(self.creator.path, 'corpora', str(self.id))
@property
def url(self):
return url_for('corpora.corpus', corpus_id=self.id)
def to_dict(self): def to_dict(self):
return {'id': self.id, return {'analysis_url': self.analysis_url,
'url': self.url,
'id': self.id,
'user_id': self.user_id, 'user_id': self.user_id,
'creation_date': self.creation_date.timestamp(), 'creation_date': self.creation_date.timestamp(),
'current_nr_of_tokens': self.current_nr_of_tokens,
'description': self.description, 'description': self.description,
'status': self.status, 'status': self.status,
'last_edited_date': self.last_edited_date.timestamp(), 'last_edited_date': self.last_edited_date.timestamp(),
'max_nr_of_tokens': self.max_nr_of_tokens,
'title': self.title, 'title': self.title,
'files': {file.id: file.to_dict() for file in self.files}} 'files': {file.id: file.to_dict() for file in self.files}}
def build(self): def build(self):
corpus_dir = os.path.join(current_app.config['DATA_DIR'], output_dir = os.path.join(self.path, 'merged')
str(self.user_id),
'corpora',
str(self.id))
output_dir = os.path.join(corpus_dir, 'merged')
shutil.rmtree(output_dir, ignore_errors=True) shutil.rmtree(output_dir, ignore_errors=True)
os.mkdir(output_dir) os.mkdir(output_dir)
master_element_tree = ET.ElementTree( master_element_tree = ET.ElementTree(
ET.fromstring('<corpus>\n</corpus>') ET.fromstring('<corpus>\n</corpus>')
) )
for corpus_file in self.files: for corpus_file in self.files:
corpus_file_path = os.path.join(corpus_dir, corpus_file.filename) element_tree = ET.parse(corpus_file.path)
element_tree = ET.parse(corpus_file_path)
text_node = element_tree.find('text') text_node = element_tree.find('text')
text_node.set('address', corpus_file.address or "NULL") text_node.set('address', corpus_file.address or "NULL")
text_node.set('author', corpus_file.author) text_node.set('author', corpus_file.author)
@ -597,7 +577,7 @@ class Corpus(db.Model):
text_node.set('publishing_year', str(corpus_file.publishing_year)) text_node.set('publishing_year', str(corpus_file.publishing_year))
text_node.set('school', corpus_file.school or "NULL") text_node.set('school', corpus_file.school or "NULL")
text_node.set('title', corpus_file.title) text_node.set('title', corpus_file.title)
element_tree.write(corpus_file_path) element_tree.write(corpus_file.path)
master_element_tree.getroot().insert(1, text_node) master_element_tree.getroot().insert(1, text_node)
output_file = os.path.join(output_dir, 'corpus.vrt') output_file = os.path.join(output_dir, 'corpus.vrt')
master_element_tree.write(output_file, master_element_tree.write(output_file,
@ -607,18 +587,14 @@ class Corpus(db.Model):
self.status = 'submitted' self.status = 'submitted'
def delete(self): def delete(self):
corpus_dir = os.path.join(current_app.config['DATA_DIR'], shutil.rmtree(self.path, ignore_errors=True)
str(self.user_id),
'corpora',
str(self.id))
shutil.rmtree(corpus_dir, ignore_errors=True)
db.session.delete(self) db.session.delete(self)
def __repr__(self): def __repr__(self):
''' '''
String representation of the corpus. For human readability. String representation of the corpus. For human readability.
''' '''
return '<Corpus {corpus_title}>'.format(corpus_title=self.title) return '<Corpus {}>'.format(self.title)
class QueryResult(db.Model): class QueryResult(db.Model):
@ -636,25 +612,39 @@ class QueryResult(db.Model):
query_metadata = db.Column(db.JSON()) query_metadata = db.Column(db.JSON())
title = db.Column(db.String(32)) title = db.Column(db.String(32))
@property
def download_url(self):
return url_for('corpora.download_query_result',
query_result_id=self.id)
@property
def path(self):
return os.path.join(
self.creator.path, 'query_results', str(self.id), self.filename)
@property
def url(self):
return url_for('corpora.query_result', query_result_id=self.id)
def delete(self): def delete(self):
query_result_dir = os.path.join(current_app.config['DATA_DIR'], shutil.rmtree(self.path, ignore_errors=True)
str(self.user_id),
'query_results',
str(self.id))
shutil.rmtree(query_result_dir, ignore_errors=True)
db.session.delete(self) db.session.delete(self)
def to_dict(self): def to_dict(self):
return {'id': self.id, return {'download_url': self.download_url,
'url': self.url,
'id': self.id,
'user_id': self.user_id, 'user_id': self.user_id,
'corpus_title': self.query_metadata['corpus_name'],
'description': self.description, 'description': self.description,
'filename': self.filename, 'filename': self.filename,
'query': self.query_metadata['query'],
'query_metadata': self.query_metadata, 'query_metadata': self.query_metadata,
'title': self.title} 'title': self.title}
def __repr__(self): def __repr__(self):
''' '''
String representation of the CorpusAnalysisResult. For human readability. String representation of the QueryResult. For human readability.
''' '''
return '<QueryResult {}>'.format(self.title) return '<QueryResult {}>'.format(self.title)

150
web/app/query_results/views.py
View File

@ -1,150 +0,0 @@
from . import query_results
from . import tasks
from .. import db
from ..corpora.forms import DisplayOptionsForm, InspectDisplayOptionsForm
from ..models import QueryResult
from .forms import AddQueryResultForm
from flask import (abort, current_app, flash, make_response, redirect,
render_template, request, send_from_directory, url_for)
from flask_login import current_user, login_required
import json
import os
from jsonschema import validate
@query_results.route('/add', methods=['GET', 'POST'])
@login_required
def add_query_result():
'''
View to import a result as a json file.
'''
add_query_result_form = AddQueryResultForm(prefix='add-query-result-form')
if add_query_result_form.is_submitted():
if not add_query_result_form.validate():
return make_response(add_query_result_form.errors, 400)
query_result = QueryResult(
creator=current_user,
description=add_query_result_form.description.data,
filename=add_query_result_form.file.data.filename,
title=add_query_result_form.title.data
)
db.session.add(query_result)
db.session.commit()
# create paths to save the uploaded json file
query_result_dir = os.path.join(current_app.config['DATA_DIR'],
str(current_user.id),
'query_results',
str(query_result.id))
try:
os.makedirs(query_result_dir)
except Exception:
db.session.delete(query_result)
db.session.commit()
flash('Internal Server Error', 'error')
redirect_url = url_for('query_results.add_query_result')
return make_response({'redirect_url': redirect_url}, 500)
# save the uploaded file
query_result_file_path = os.path.join(query_result_dir,
query_result.filename)
add_query_result_form.file.data.save(query_result_file_path)
# parse json from file
with open(query_result_file_path, 'r') as file:
query_result_file_content = json.load(file)
# parse json schema
with open('app/static/json_schema/nopaque_cqi_py_results_schema.json', 'r') as file: # noqa
schema = json.load(file)
try:
# validate imported json file
validate(instance=query_result_file_content, schema=schema)
except Exception:
tasks.delete_query_result(query_result.id)
flash('Uploaded file is invalid', 'result')
redirect_url = url_for('query_results.add_query_result')
return make_response({'redirect_url': redirect_url}, 201)
query_result_file_content.pop('matches')
query_result_file_content.pop('cpos_lookup')
query_result.query_metadata = query_result_file_content
db.session.commit()
flash('Query result added!', 'result')
redirect_url = url_for('query_results.query_result',
query_result_id=query_result.id)
return make_response({'redirect_url': redirect_url}, 201)
return render_template('corpora/query_results/add_query_result.html.j2',
add_query_result_form=add_query_result_form,
title='Add query result')
@query_results.route('/<int:query_result_id>')
@login_required
def query_result(query_result_id):
query_result = QueryResult.query.get_or_404(query_result_id)
if not (query_result.creator == current_user
or current_user.is_administrator()):
abort(403)
return render_template('corpora/query_results/query_result.html.j2',
query_result=query_result,
title='Query result')
@query_results.route('/<int:query_result_id>/inspect')
@login_required
def inspect_query_result(query_result_id):
'''
View to inspect imported result file in a corpus analysis like interface
'''
query_result = QueryResult.query.get_or_404(query_result_id)
query_metadata = query_result.query_metadata
if not (query_result.creator == current_user
or current_user.is_administrator()):
abort(403)
display_options_form = DisplayOptionsForm(
prefix='display-options-form',
results_per_page=request.args.get('results_per_page', 30),
result_context=request.args.get('context', 20)
)
inspect_display_options_form = InspectDisplayOptionsForm(
prefix='inspect-display-options-form'
)
query_result_file_path = os.path.join(
current_app.config['DATA_DIR'],
str(current_user.id),
'query_results',
str(query_result.id),
query_result.filename
)
with open(query_result_file_path, 'r') as query_result_file:
query_result_file_content = json.load(query_result_file)
return render_template('corpora/query_results/inspect.html.j2',
display_options_form=display_options_form,
inspect_display_options_form=inspect_display_options_form,
query_result_file_content=query_result_file_content,
query_metadata=query_metadata,
title='Inspect query result')
@query_results.route('/<int:query_result_id>/delete')
@login_required
def delete_query_result(query_result_id):
query_result = QueryResult.query.get_or_404(query_result_id)
if not (query_result.creator == current_user
or current_user.is_administrator()):
abort(403)
tasks.delete_query_result(query_result_id)
flash('Query result deleted!', 'result')
return redirect(url_for('services.service', service="corpus_analysis"))
@query_results.route('/<int:query_result_id>/download')
@login_required
def download_query_result(query_result_id):
query_result = QueryResult.query.get_or_404(query_result_id)
if not (query_result.creator == current_user
or current_user.is_administrator()):
abort(403)
query_result_dir = os.path.join(current_app.config['DATA_DIR'],
str(current_user.id),
'query_results',
str(query_result.id))
return send_from_directory(as_attachment=True,
directory=query_result_dir,
filename=query_result.filename)

2
web/app/services/__init__.py
View File

@ -2,4 +2,4 @@ from flask import Blueprint
services = Blueprint('services', __name__) services = Blueprint('services', __name__)
from . import views # noqa from . import views

63
web/app/services/views.py
View File

@ -1,5 +1,4 @@
from flask import (abort, current_app, flash, make_response, render_template, from flask import abort, flash, make_response, render_template, url_for
url_for)
from flask_login import current_user, login_required from flask_login import current_user, login_required
from werkzeug.utils import secure_filename from werkzeug.utils import secure_filename
from . import services from . import services
@ -7,19 +6,20 @@ from .. import db
from ..jobs.forms import AddFileSetupJobForm, AddNLPJobForm, AddOCRJobForm from ..jobs.forms import AddFileSetupJobForm, AddNLPJobForm, AddOCRJobForm
from ..models import Job, JobInput from ..models import Job, JobInput
import json import json
import logging
import os import os
SERVICES = {'corpus_analysis': {'name': 'Corpus analysis'}, SERVICES = {'corpus_analysis': {'name': 'Corpus analysis'},
'file-setup': {'name': 'File setup', 'file-setup': {'name': 'File setup',
'resources': {'mem_mb': 4096, 'n_cores': 4}, 'resources': {'mem_mb': 4096, 'n_cores': 4},
'add_job_form': AddFileSetupJobForm}, 'form': AddFileSetupJobForm},
'nlp': {'name': 'Natural Language Processing', 'nlp': {'name': 'Natural Language Processing',
'resources': {'mem_mb': 4096, 'n_cores': 2}, 'resources': {'mem_mb': 4096, 'n_cores': 2},
'add_job_form': AddNLPJobForm}, 'form': AddNLPJobForm},
'ocr': {'name': 'Optical Character Recognition', 'ocr': {'name': 'Optical Character Recognition',
'resources': {'mem_mb': 8192, 'n_cores': 4}, 'resources': {'mem_mb': 8192, 'n_cores': 4},
'add_job_form': AddOCRJobForm}} 'form': AddOCRJobForm}}
@services.route('/<service>', methods=['GET', 'POST']) @services.route('/<service>', methods=['GET', 'POST'])
@ -30,54 +30,47 @@ def service(service):
if service == 'corpus_analysis': if service == 'corpus_analysis':
return render_template('services/{}.html.j2'.format(service), return render_template('services/{}.html.j2'.format(service),
title=SERVICES[service]['name']) title=SERVICES[service]['name'])
add_job_form = SERVICES[service]['add_job_form'](prefix='add-job-form') form = SERVICES[service]['form'](prefix='add-job-form')
if add_job_form.is_submitted(): if form.is_submitted():
if not add_job_form.validate(): if not form.validate():
return make_response(add_job_form.errors, 400) return make_response(form.errors, 400)
service_args = [] service_args = []
if service == 'nlp': if service == 'nlp':
service_args.append('-l {}'.format(add_job_form.language.data)) service_args.append('-l {}'.format(form.language.data))
if add_job_form.check_encoding.data: if form.check_encoding.data:
service_args.append('--check-encoding') service_args.append('--check-encoding')
if service == 'ocr': if service == 'ocr':
service_args.append('-l {}'.format(add_job_form.language.data)) service_args.append('-l {}'.format(form.language.data))
if add_job_form.binarization.data: if form.binarization.data:
service_args.append('--binarize') service_args.append('--binarize')
job = Job(creator=current_user, job = Job(creator=current_user,
description=add_job_form.description.data, description=form.description.data,
mem_mb=SERVICES[service]['resources']['mem_mb'], mem_mb=SERVICES[service]['resources']['mem_mb'],
n_cores=SERVICES[service]['resources']['n_cores'], n_cores=SERVICES[service]['resources']['n_cores'],
service=service, service_args=json.dumps(service_args), service=service, service_args=json.dumps(service_args),
service_version=add_job_form.version.data, service_version=form.version.data,
status='preparing', title=add_job_form.title.data) status='preparing', title=form.title.data)
if job.service != 'corpus_analysis':
job.create_secure_filename()
db.session.add(job) db.session.add(job)
db.session.commit() db.session.commit()
relative_dir = os.path.join(str(job.user_id), 'jobs', str(job.id))
absolut_dir = os.path.join(current_app.config['DATA_DIR'],
relative_dir)
try: try:
os.makedirs(absolut_dir) os.makedirs(job.path)
except OSError: except OSError:
job.delete() logging.error('Make dir {} led to an OSError!'.format(job.path))
flash('Internal Server Error', 'job') db.session.delete(job)
return make_response({'redirect_url': url_for('services.service', db.session.commit()
service=service)}, flash('Internal Server Error', 'error')
500) return make_response(
{'redirect_url': url_for('.service', service=service)}, 500)
else: else:
for file in add_job_form.files.data: for file in form.files.data:
filename = secure_filename(file.filename) filename = secure_filename(file.filename)
file.save(os.path.join(absolut_dir, filename)) job_input = JobInput(filename=filename, job=job)
job_input = JobInput(dir=relative_dir, filename=filename, file.save(job_input.path)
job=job)
db.session.add(job_input) db.session.add(job_input)
job.status = 'submitted' job.status = 'submitted'
db.session.commit() db.session.commit()
url = url_for('jobs.job', job_id=job.id) flash('Job "{}" added'.format(job.title), 'job')
flash('[<a href="{}">{}</a>] added'.format(url, job.title), 'job')
return make_response( return make_response(
{'redirect_url': url_for('jobs.job', job_id=job.id)}, 201) {'redirect_url': url_for('jobs.job', job_id=job.id)}, 201)
return render_template('services/{}.html.j2'.format(service), return render_template('services/{}.html.j2'.format(service),
title=SERVICES[service]['name'], form=form, title=SERVICES[service]['name'])
add_job_form=add_job_form)

2
web/app/settings/forms.py
View File

@ -35,7 +35,7 @@ class EditGeneralSettingsForm(FlaskForm):
'Benutzername', 'Benutzername',
validators=[DataRequired(), validators=[DataRequired(),
Length(1, 64), Length(1, 64),
Regexp(current_app.config['ALLOWED_USERNAME_REGEX'], Regexp(current_app.config['NOPAQUE_USERNAME_REGEX'],
message='Usernames must have only letters, numbers,' message='Usernames must have only letters, numbers,'
' dots or underscores')] ' dots or underscores')]
) )

19
web/app/settings/views.py
View File

@ -1,13 +1,9 @@
from flask import current_app, flash, redirect, render_template, url_for from flask import flash, redirect, render_template, url_for
from flask_login import current_user, login_required, logout_user from flask_login import current_user, login_required, logout_user
from . import settings, tasks from . import settings, tasks
from .forms import (ChangePasswordForm, EditGeneralSettingsForm, from .forms import (ChangePasswordForm, EditGeneralSettingsForm,
EditNotificationSettingsForm) EditNotificationSettingsForm)
from .. import db from .. import db
from ..decorators import admin_required
from ..models import Role, User
import os
import uuid
@settings.route('/') @settings.route('/')
@ -26,8 +22,7 @@ def change_password():
flash('Your password has been updated.') flash('Your password has been updated.')
return redirect(url_for('.change_password')) return redirect(url_for('.change_password'))
return render_template('settings/change_password.html.j2', return render_template('settings/change_password.html.j2',
form=form, form=form, title='Change password')
title='Change password')
@settings.route('/edit_general_settings', methods=['GET', 'POST']) @settings.route('/edit_general_settings', methods=['GET', 'POST'])
@ -40,12 +35,12 @@ def edit_general_settings():
current_user.username = form.username.data current_user.username = form.username.data
db.session.commit() db.session.commit()
flash('Your changes have been saved.') flash('Your changes have been saved.')
return redirect(url_for('.edit_general_settings'))
form.dark_mode.data = current_user.setting_dark_mode form.dark_mode.data = current_user.setting_dark_mode
form.email.data = current_user.email form.email.data = current_user.email
form.username.data = current_user.username form.username.data = current_user.username
return render_template('settings/edit_general_settings.html.j2', return render_template('settings/edit_general_settings.html.j2',
form=form, form=form, title='General settings')
title='General settings')
@settings.route('/edit_notification_settings', methods=['GET', 'POST']) @settings.route('/edit_notification_settings', methods=['GET', 'POST'])
@ -59,13 +54,13 @@ def edit_notification_settings():
form.job_status_site_notifications.data form.job_status_site_notifications.data
db.session.commit() db.session.commit()
flash('Your changes have been saved.') flash('Your changes have been saved.')
return redirect(url_for('.edit_notification_settings'))
form.job_status_mail_notifications.data = \ form.job_status_mail_notifications.data = \
current_user.setting_job_status_mail_notifications current_user.setting_job_status_mail_notifications
form.job_status_site_notifications.data = \ form.job_status_site_notifications.data = \
current_user.setting_job_status_site_notifications current_user.setting_job_status_site_notifications
return render_template('settings/edit_notification_settings.html.j2', return render_template('settings/edit_notification_settings.html.j2',
form=form, form=form, title='Notification settings')
title='Notification settings')
@settings.route('/delete') @settings.route('/delete')
@ -76,5 +71,5 @@ def delete():
""" """
tasks.delete_user(current_user.id) tasks.delete_user(current_user.id)
logout_user() logout_user()
flash('Your account has been deleted!') flash('Your account has been marked for deletion!')
return redirect(url_for('main.index')) return redirect(url_for('main.index'))

4
web/app/static/css/nopaque.css
View File

@ -8,6 +8,10 @@ main {
margin-top: 48px; margin-top: 48px;
} }
table.ressource-list tr {
cursor: pointer;
}
.parallax-container .parallax { .parallax-container .parallax {
z-index: auto; z-index: auto;
} }

4
web/app/static/js/list.min.js vendored
View File

File diff suppressed because one or more lines are too long

4
web/app/static/js/modules/corpus_analysis/client/Client.js
View File

@ -136,7 +136,7 @@ class Client {
tmp_first_cpos.push(results.data.matches[dataIndex].c[0]); tmp_first_cpos.push(results.data.matches[dataIndex].c[0]);
tmp_last_cpos.push(results.data.matches[dataIndex].c[1]); tmp_last_cpos.push(results.data.matches[dataIndex].c[1]);
} }
nopaque.socket.emit('corpus_analysis_get_match_with_full_context', this.socket.emit('corpus_analysis_get_match_with_full_context',
{type: resultsType, {type: resultsType,
data_indexes: dataIndexes, data_indexes: dataIndexes,
first_cpos: tmp_first_cpos, first_cpos: tmp_first_cpos,
@ -279,4 +279,4 @@ export {
Client, Client,
ClientEventListener, ClientEventListener,
ListenerCallback, ListenerCallback,
}; };

239
web/app/static/js/nopaque.js
View File

@ -1,96 +1,138 @@
class AppClient {
constructor(currentUserId) {
this.socket = io({transports: ['websocket']});
this.users = {};
this.users.self = this.loadUser(currentUserId);
}
loadUser(userId) {
let user = new User();
this.users[userId] = user;
this.socket.on(`user_${userId}_init`, msg => user.init(JSON.parse(msg)));
this.socket.on(`user_${userId}_patch`, msg => user.patch(JSON.parse(msg)));
this.socket.emit('start_user_session', userId);
return user;
}
}
class User {
constructor() {
this.data = undefined;
this.eventListeners = {
corporaInit: [],
corporaPatch: [],
jobsInit: [],
jobsPatch: [],
queryResultsInit: [],
queryResultsPatch: []
};
}
init(data) {
this.data = data;
let listener;
for (listener of this.eventListeners.corporaInit) {
listener(this.data.corpora);
}
for (listener of this.eventListeners.jobsInit) {
listener(this.data.jobs);
}
for (listener of this.eventListeners.queryResultsInit) {
listener(this.data.query_results);
}
}
patch(patch) {
this.data = jsonpatch.apply_patch(this.data, patch);
let corporaPatch = patch.filter(operation => operation.path.startsWith("/corpora"));
let jobsPatch = patch.filter(operation => operation.path.startsWith("/jobs"));
let queryResultsPatch = patch.filter(operation => operation.path.startsWith("/query_results"));
for (let listener of this.eventListeners.corporaPatch) {
if (corporaPatch.length > 0) {listener(corporaPatch);}
}
for (let listener of this.eventListeners.jobsPatch) {
if (jobsPatch.length > 0) {listener(jobsPatch);}
}
for (let listener of this.eventListeners.queryResultsPatch) {
if (queryResultsPatch.length > 0) {listener(queryResultsPatch);}
}
for (let operation of jobsPatch) {
if (operation.op !== 'replace') {continue;}
// Matches the only path that should be handled here: /jobs/{jobId}/status
if (/^\/jobs\/(\d+)\/status$/.test(operation.path)) {
let [match, jobId] = operation.path.match(/^\/jobs\/(\d+)\/status$/);
if (this.data.settings.job_status_site_notifications === "end" && !["complete", "failed"].includes(operation.value)) {continue;}
nopaque.flash(`[<a href="/jobs/${jobId}">${this.data.jobs[jobId].title}</a>] New status: ${operation.value}`, "job");
}
}
}
addEventListener(type, listener) {
switch (type) {
case 'corporaInit':
this.eventListeners.corporaInit.push(listener);
if (this.data !== undefined) {listener(this.data.corpora);}
break;
case 'corporaPatch':
this.eventListeners.corporaPatch.push(listener);
break;
case 'jobsInit':
this.eventListeners.jobsInit.push(listener);
if (this.data !== undefined) {listener(this.data.jobs);}
break;
case 'jobsPatch':
this.eventListeners.jobsPatch.push(listener);
break;
case 'queryResultsInit':
this.eventListeners.queryResultsInit.push(listener);
if (this.data !== undefined) {listener(this.data.query_results);}
break;
case 'queryResultsPatch':
this.eventListeners.queryResultsPatch.push(listener);
break;
default:
console.error(`Unknown event type: ${type}`);
}
}
}
/* /*
* The nopaque object is used as a namespace for nopaque specific functions and * The nopaque object is used as a namespace for nopaque specific functions and
* variables. * variables.
*/ */
var nopaque = {}; var nopaque = {};
// User data nopaque.flash = function(message, category) {
nopaque.user = {}; let toast;
nopaque.user.settings = {}; let toastActionElement;
nopaque.user.settings.darkMode = undefined;
nopaque.corporaSubscribers = [];
nopaque.jobsSubscribers = [];
nopaque.queryResultsSubscribers = [];
// Foreign user (user inspected with admin credentials) data switch (category) {
nopaque.foreignUser = {}; case "corpus":
nopaque.foreignUser.isAuthenticated = undefined; message = `<i class="left material-icons">book</i>${message}`;
nopaque.foreignUser.settings = {}; break;
nopaque.foreignUser.settings.darkMode = undefined; case "error":
nopaque.foreignCorporaSubscribers = []; message = `<i class="left material-icons red-text">error</i>${message}`;
nopaque.foreignJobsSubscribers = []; break;
nopaque.foreignQueryResultsSubscribers = []; case "job":
message = `<i class="left material-icons">work</i>${message}`;
break;
default:
message = `<i class="left material-icons">notifications</i>${message}`;
}
// nopaque functions toast = M.toast({html: `<span>${message}</span>
nopaque.socket = io({transports: ['websocket']}); <button data-action="close" class="btn-flat toast-action white-text">
// Add event handlers <i class="material-icons">close</i>
nopaque.socket.on("user_data_stream_init", function(msg) { </button>`});
nopaque.user = JSON.parse(msg); toastActionElement = toast.el.querySelector('.toast-action[data-action="close"]');
for (let subscriber of nopaque.corporaSubscribers) { toastActionElement.addEventListener('click', () => {toast.dismiss();});
subscriber._init(nopaque.user.corpora); };
}
for (let subscriber of nopaque.jobsSubscribers) {
subscriber._init(nopaque.user.jobs);
}
for (let subscriber of nopaque.queryResultsSubscribers) {
subscriber._init(nopaque.user.query_results);
}
});
nopaque.socket.on("user_data_stream_update", function(msg) {
var patch;
patch = JSON.parse(msg);
nopaque.user = jsonpatch.apply_patch(nopaque.user, patch);
corpora_patch = patch.filter(operation => operation.path.startsWith("/corpora"));
jobs_patch = patch.filter(operation => operation.path.startsWith("/jobs"));
query_results_patch = patch.filter(operation => operation.path.startsWith("/query_results"));
for (let subscriber of nopaque.corporaSubscribers) {
subscriber._update(corpora_patch);
}
for (let subscriber of nopaque.jobsSubscribers) {
subscriber._update(jobs_patch);
}
for (let subscriber of nopaque.queryResultsSubscribers) {
subscriber._update(query_results_patch);
}
if (["all", "end"].includes(nopaque.user.settings.job_status_site_notifications)) {
for (operation of jobs_patch) {
/* "/jobs/{jobId}/..." -> ["{jobId}", ...] */
pathArray = operation.path.split("/").slice(2);
if (operation.op === "replace" && pathArray[1] === "status") {
if (nopaque.user.settings.job_status_site_notifications === "end" && !["complete", "failed"].includes(operation.value)) {continue;}
nopaque.flash(`[<a href="/jobs/${pathArray[0]}">${nopaque.user.jobs[pathArray[0]].title}</a>] New status: ${operation.value}`, "job");
}
}
}
});
nopaque.socket.on("foreign_user_data_stream_init", function(msg) {
nopaque.foreignUser = JSON.parse(msg);
for (let subscriber of nopaque.foreignCorporaSubscribers) {
subscriber._init(nopaque.foreignUser.corpora);
}
for (let subscriber of nopaque.foreignJobsSubscribers) {
subscriber._init(nopaque.foreignUser.jobs);
}
for (let subscriber of nopaque.foreignQueryResultsSubscribers) {
subscriber._init(nopaque.foreignUser.query_results);
}
});
nopaque.socket.on("foreign_user_data_stream_update", function(msg) {
var patch;
patch = JSON.parse(msg);
nopaque.foreignUser = jsonpatch.apply_patch(nopaque.foreignUser, patch);
corpora_patch = patch.filter(operation => operation.path.startsWith("/corpora"));
jobs_patch = patch.filter(operation => operation.path.startsWith("/jobs"));
query_results_patch = patch.filter(operation => operation.path.startsWith("/query_results"));
for (let subscriber of nopaque.foreignCorporaSubscribers) {subscriber._update(corpora_patch);}
for (let subscriber of nopaque.foreignJobsSubscribers) {subscriber._update(jobs_patch);}
for (let subscriber of nopaque.foreignQueryResultsSubscribers) {subscriber._update(query_results_patch);}
});
nopaque.Forms = {}; nopaque.Forms = {};
nopaque.Forms.init = function() { nopaque.Forms.init = function() {
@ -163,30 +205,3 @@ nopaque.Forms.init = function() {
} }
} }
} }
nopaque.flash = function(message, category) {
let toast;
let toastActionElement;
switch (category) {
case "corpus":
message = `<i class="left material-icons">book</i>${message}`;
break;
case "error":
message = `<i class="left material-icons red-text">error</i>${message}`;
break;
case "job":
message = `<i class="left material-icons">work</i>${message}`;
break;
default:
message = `<i class="left material-icons">notifications</i>${message}`;
}
toast = M.toast({html: `<span>${message}</span>
<button data-action="close" class="btn-flat toast-action white-text">
<i class="material-icons">close</i>
</button>`});
toastActionElement = toast.el.querySelector('.toast-action[data-action="close"]');
toastActionElement.addEventListener('click', () => {toast.dismiss();});
}

644
web/app/static/js/nopaque.lists.js
View File

@ -1,432 +1,258 @@
class RessourceList extends List { class RessourceList {
constructor(idOrElement, subscriberList, type, options) { /* A wrapper class for the list.js list.
if (!type || !["Corpus", "CorpusFile", "Job", "JobInput", "QueryResult", "User"].includes(type)) { * This class is not meant to be used directly, instead it should be used as
throw "Unknown Type!"; * a template for concrete ressource list implementations.
*/
constructor(listElement, options = {}) {
if (listElement.dataset.userId) {
if (listElement.dataset.userId in nopaque.appClient.users) {
this.user = nopaque.appClient.users[listElement.dataset.userId];
} else {
console.error(`User not found: ${listElement.dataset.userId}`);
return;
}
} else {
this.user = nopaque.appClient.users.self;
}
this.list = new List(listElement, {...RessourceList.options, ...options});
this.valueNames = ['id'];
for (let element of this.list.valueNames) {
switch (typeof element) {
case 'object':
if (element.hasOwnProperty('name')) {this.valueNames.push(element.name);}
break;
case 'string':
this.valueNames.push(element);
break;
default:
console.error(`Unknown value name definition: ${element}`);
}
} }
super(idOrElement, {...RessourceList.options['common'],
...RessourceList.options[type],
...(options ? options : {})});
if (subscriberList) {subscriberList.push(this);}
this.type = type;
} }
init(ressources) {
_init(ressources) { this.list.clear();
this.clear(); this.add(Object.values(ressources));
this._add(Object.values(ressources)); this.list.sort('id', {order: 'desc'});
this.sort("creation_date", {order: "desc"});
} }
patch(patch) {
/*
* It's not possible to generalize a patch Handler for all type of
* ressources. So this method is meant to be an interface.
*/
console.error('patch method not implemented!');
}
_update(patch) { add(values) {
let item, pathArray; let ressources = Array.isArray(values) ? values : [values];
// Discard ressource values, that are not defined to be used in the list.
ressources = ressources.map(ressource => {
let cleanedRessource = {};
for (let [valueName, value] of Object.entries(ressource)) {
if (this.valueNames.includes(valueName)) {cleanedRessource[valueName] = value;}
}
return cleanedRessource;
});
// Set a callback function ('() => {return;}') to force List.js perform the
// add method asynchronous: https://listjs.com/api/#add
this.list.add(ressources, () => {return;});
}
remove(id) {
this.list.remove('id', id);
}
replace(id, valueName, newValue) {
if (this.valueNames.includes(valueName)) {
let item = this.list.get('id', id)[0];
item.values({[valueName]: newValue});
}
}
}
RessourceList.options = {page: 5, pagination: [{innerWindow: 4, outerWindow: 1}]};
class CorpusList extends RessourceList {
constructor(listElement, options = {}) {
super(listElement, {...CorpusList.options, ...options});
this.user.addEventListener('corporaInit', corpora => this.init(corpora));
this.user.addEventListener('corporaPatch', patch => this.patch(patch));
listElement.addEventListener('click', (event) => {this.onclick(event)});
}
onclick(event) {
let corpusId = event.target.closest('tr').dataset.id;
let actionButtonElement = event.target.closest('.action-button');
let action = actionButtonElement === null ? 'view' : actionButtonElement.dataset.action;
switch (action) {
case 'analyse':
window.location.href = nopaque.user.corpora[corpusId].analysis_url;
case 'delete':
let deleteModalHTML = `<div class="modal">
<div class="modal-content">
<h4>Confirm corpus deletion</h4>
<p>Do you really want to delete the corpus <b>${nopaque.user.corpora[corpusId].title}</b>? All files will be permanently deleted!</p>
</div>
<div class="modal-footer">
<a href="#!" class="btn modal-close waves-effect waves-light">Cancel</a>
<a class="btn modal-close red waves-effect waves-light" href="${nopaque.user.corpora[corpusId].url}/delete"><i class="material-icons left">delete</i>Delete</a>
</div>
</div>`;
let deleteModalParentElement = document.querySelector('main');
deleteModalParentElement.insertAdjacentHTML('beforeend', deleteModalHTML);
let deleteModalElement = deleteModalParentElement.lastChild;
let deleteModal = M.Modal.init(deleteModalElement, {onCloseEnd: () => {deleteModal.destroy(); deleteModalElement.remove();}});
deleteModal.open();
break;
case 'view':
// TODO: handle unprepared corpora
window.location.href = nopaque.user.corpora[corpusId].url;
break;
default:
console.error(`Unknown action: ${action}`);
break;
}
}
patch(patch) {
for (let operation of patch) { for (let operation of patch) {
/* "/{ressourceName}/{ressourceId}/..." -> ["{ressourceId}", "..."] */
pathArray = operation.path.split("/").slice(2);
switch(operation.op) { switch(operation.op) {
case "add": case 'add':
if (pathArray.includes("results")) {break;} // Matches the only paths that should be handled here: /corpora/{corpusId}
this._add([operation.value]); if (/^\/corpora\/(\d+)$/.test(operation.path)) {this.add(operation.value);}
break; break;
case "remove": case 'remove':
this.remove("id", pathArray[0]); // See case 'add' ;)
break; if (/^\/corpora\/(\d+)$/.test(operation.path)) {
case "replace": let [match, id] = operation.path.match(/^\/corpora\/(\d+)$/);
item = this.get("id", pathArray[0])[0]; this.remove(corpusId);
switch(pathArray[1]) {
case "status":
item.values({status: operation.value,
"analyse-link": ["analysing", "prepared", "start analysis"].includes(operation.value) ? `/corpora/${pathArray[0]}/analyse` : ""});
break;
default:
break;
} }
break;
case 'replace':
// Matches the only paths that should be handled here: /corpora/{corpusId}/{status || description || title}
if (/^\/corpora\/(\d+)\/(status|description|title)$/.test(operation.path)) {
let [match, id, valueName] = operation.path.match(/^\/corpora\/(\d+)\/(status|description|title)$/);
this.replace(id, valueName, operation.value);
}
break;
default: default:
break; break;
} }
} }
} }
}
CorpusList.options = {
item: `<tr>
<td><a class="btn-floating disabled"><i class="material-icons">book</i></a></td>
<td><b class="title"></b><br><i class="description"></i></td>
<td><span class="badge new status" data-badge-caption=""></span></td>
<td class="right-align">
<a class="action-button btn-floating red tooltipped waves-effect waves-light" data-action="delete" data-position="top" data-tooltip="Delete"><i class="material-icons">delete</i></a>
<a class="action-button btn-floating tooltipped waves-effect waves-light" data-action="analyse" data-position="top" data-tooltip="Analyse"><i class="material-icons">search</i></a>
<a class="action-button btn-floating tooltipped waves-effect waves-light" data-action="view" data-position="top" data-tooltip="View"><i class="material-icons">send</i></a>
</td>
</tr>`,
valueNames: [{data: ['id']}, {name: 'status', attr: 'data-status'}, 'description', 'title']
};
_add(values, callback) {
this.add(values.map(x => RessourceList.dataMappers[this.type](x)), callback); class JobList extends RessourceList {
// Initialize modal and tooltipped elements in list constructor(listElement, options = {}) {
M.AutoInit(this.listContainer); super(listElement, {...JobList.options, ...options});
this.user.addEventListener('jobsInit', jobs => this.init(jobs));
this.user.addEventListener('jobsPatch', patch => this.patch(patch));
listElement.addEventListener('click', (event) => {this.onclick(event)});
}
onclick(event) {
let jobId = event.target.closest('tr').dataset.id;
let actionButtonElement = event.target.closest('.action-button');
let action = actionButtonElement === null ? 'view' : actionButtonElement.dataset.action;
switch (action) {
case 'delete':
let deleteModalHTML = `<div class="modal">
<div class="modal-content">
<h4>Confirm job deletion</h4>
<p>Do you really want to delete the job <b>${this.user.data.jobs[jobId].title}</b>? All files will be permanently deleted!</p>
</div>
<div class="modal-footer">
<a href="#!" class="btn modal-close waves-effect waves-light">Cancel</a>
<a class="btn modal-close red waves-effect waves-light" href="${this.user.data.jobs[jobId].url}/delete"><i class="material-icons left">delete</i>Delete</a>
</div>
</div>`;
let deleteModalParentElement = document.querySelector('main');
deleteModalParentElement.insertAdjacentHTML('beforeend', deleteModalHTML);
let deleteModalElement = deleteModalParentElement.lastChild;
let deleteModal = M.Modal.init(deleteModalElement, {onCloseEnd: () => {deleteModal.destroy(); deleteModalElement.remove();}});
deleteModal.open();
break;
case 'view':
window.location.href = this.user.data.jobs[jobId].url;
break;
default:
console.error(`Unknown action: "${action}"`);
break;
}
}
patch(patch) {
for (let operation of patch) {
switch(operation.op) {
case 'add':
// Matches the only paths that should be handled here: /jobs/{jobId}
if (/^\/jobs\/(\d+)$/.test(operation.path)) {this.add(operation.value);}
break;
case 'remove':
// See case add ;)
if (/^\/jobs\/(\d+)$/.test(operation.path)) {
let [match, id] = operation.path.match(/^\/jobs\/(\d+)$/);
this.remove(jobId);
}
break;
case 'replace':
// Matches the only paths that should be handled here: /jobs/{jobId}/{service || status || description || title}
if (/^\/jobs\/(\d+)\/(service|status|description|title)$/.test(operation.path)) {
let [match, id, valueName] = operation.path.match(/^\/jobs\/(\d+)\/(service|status|description|title)$/);
this.replace(id, valueName, operation.value);
}
break;
default:
break;
}
}
} }
} }
JobList.options = {
item: `<tr>
<td><a class="btn-floating disabled"><i class="material-icons service"></i></a></td>
<td><b class="title"></b><br><i class="description"></i></td>
RessourceList.dataMappers = { <td><span class="badge new status" data-badge-caption=""></span></td>
// A data mapper describes entitys rendered per row. One key value pair holds <td class="right-align">
// the data to be rendered in the list.js table. Key has to correspond <a class="action-button btn-floating red tooltipped waves-effect waves-light" data-action="delete" data-position="top" data-tooltip="Delete"><i class="material-icons">delete</i></a>
// with the ValueNames defined below in RessourceList.options ValueNames. <a class="action-button btn-floating tooltipped waves-effect waves-light" data-action="view" data-position="top" data-tooltip="View"><i class="material-icons">send</i></a>
// Links are declared with double ticks(") around them. The key for links </td>
// have to correspond with the class of an <a> element in the </tr>`,
// RessourceList.options item blueprint. valueNames: [{data: ['id']}, {name: 'service', attr: 'data-service'}, {name: 'status', attr: 'data-status'}, 'description', 'title']
/* ### Corpus mapper ### */
Corpus: corpus => ({
creation_date: corpus.creation_date,
description: corpus.description,
id: corpus.id,
link: `/corpora/${corpus.id}`,
status: corpus.status,
title: corpus.title,
title1: corpus.title,
"analyse-link": ["analysing", "prepared", "start analysis"].includes(corpus.status) ? `/corpora/${corpus.id}/analyse` : "",
"delete-link": `/corpora/${corpus.id}/delete`,
"delete-modal": `delete-corpus-${corpus.id}-modal`,
"delete-modal-trigger": `delete-corpus-${corpus.id}-modal`,
}),
/* ### CorpusFile mapper ### TODO: replace delete-modal with delete-onclick */
CorpusFile: corpus_file => ({
author: corpus_file.author,
filename: corpus_file.filename,
link: `${corpus_file.corpus_id}/files/${corpus_file.id}`,
title: corpus_file.title,
title1: corpus_file.title,
"delete-link": `/corpora/${corpus_file.corpus_id}/files/${corpus_file.id}/delete`,
"delete-modal": `delete-corpus-file-${corpus_file.id}-modal`,
"delete-modal-trigger": `delete-corpus-file-${corpus_file.id}-modal`,
"download-link": `${corpus_file.corpus_id}/files/${corpus_file.id}/download`,
}),
/* ### Job mapper ### */
Job: job => ({
creation_date: job.creation_date,
description: job.description,
id: job.id,
link: `/jobs/${job.id}`,
service: job.service,
status: job.status,
title: job.title,
title1: job.title,
"delete-link": `/jobs/${job.id}/delete`,
"delete-modal": `delete-job-${job.id}-modal`,
"delete-modal-trigger": `delete-job-${job.id}-modal`,
}),
/* ### JobInput mapper ### */
JobInput: job_input => ({
filename: job_input.filename,
id: job_input.job_id,
"download-link": `${job_input.job_id}/inputs/${job_input.id}/download`
}),
/* ### QueryResult mapper ### */
QueryResult: query_result => ({
corpus_name: query_result.query_metadata.corpus_name,
description: query_result.description,
id: query_result.id,
link: `/corpora/result/${query_result.id}`,
query: query_result.query_metadata.query,
title: query_result.title,
"delete-link": `/corpora/result/${query_result.id}/delete`,
"delete-modal": `delete-query-result-${query_result.id}-modal`,
"delete-modal-trigger": `delete-query-result-${query_result.id}-modal`,
"inspect-link": `/corpora/result/${query_result.id}/inspect`,
}),
/* ### User mapper ### */
User: user => ({
confirmed: user.confirmed,
email: user.email,
id: user.id,
link: `users/${user.id}`,
role_id: user.role_id,
username: user.username,
username2: user.username,
"delete-link": `/admin/users/${user.id}/delete`,
"delete-modal": `delete-user-${user.id}-modal`,
"delete-modal-trigger": `delete-user-${user.id}-modal`,
}),
}; };
RessourceList.options = { class QueryResultList extends RessourceList {
// common list.js options for 5 rows per page etc. constructor(listElement, options = {}) {
common: { super(listElement, {...QueryResultList.options, ...options});
page: 5, this.user.addEventListener('queryResultsInit', queryResults => this.init(queryResults));
pagination: [ this.user.addEventListener('queryResultsPatch', patch => this.init(patch));
{ }
name: "paginationTop", }
paginationClass: "paginationTop", QueryResultList.options = {
innerWindow: 4, item: `<tr>
outerWindow: 1 <td><b class="title"></b><br><i class="description"></i><br></td>
}, <td><span class="corpus_title"></span><br><span class="query"></span></td>
{ <td class="right-align">
paginationClass: "paginationBottom", <a class="action-button btn-floating red tooltipped waves-effect waves-light" data-action="delete" data-position="top" data-tooltip="Delete"><i class="material-icons">delete</i></a>
innerWindow: 4, <a class="action-button btn-floating tooltipped waves-effect waves-light" data-action="view" data-position="top" data-tooltip="View"><i class="material-icons">send</i></a>
outerWindow: 1, <a class="action-button btn-floating tooltipped waves-effect waves-light" data-action="analyse" data-position="top" data-tooltip="Analyse"><i class="material-icons">search</i></a>
}, </td>
], </tr>`,
}, valueNames: [{data: ['id']}, 'corpus_title', 'description', 'query', 'title']
// extended list.js options for 10 rows per page etc.
extended: {
page: 10,
pagination: [
{
name: "paginationTop",
paginationClass: "paginationTop",
innerWindow: 8,
outerWindow: 1
},
{
paginationClass: "paginationBottom",
innerWindow: 8,
outerWindow: 1,
},
],
},
/* Type specific List.js options. Usually only "item" and "valueNames" gets
* defined here but it is possible to define other List.js options.
* item: https://listjs.com/api/#item
* valueNames: https://listjs.com/api/#valueNames
*/
Corpus: {
item: `<tr>
<td>
<a class="btn-floating disabled">
<i class="material-icons service">book</i>
</a>
</td>
<td>
<b class="title"></b><br>
<i class="description"></i>
</td>
<td>
<span class="badge new status" data-badge-caption=""></span>
</td>
<td>
<div class="right-align">
<a class="btn-floating modal-trigger red tooltipped waves-effect waves-light delete-modal-trigger" data-position="top" data-tooltip="Delete">
<i class="material-icons">delete</i>
</a>
<a class="btn-floating tooltipped waves-effect waves-light link" data-position="top" data-tooltip="Edit">
<i class="material-icons">edit</i>
</a>
<a class="btn-floating tooltipped waves-effect waves-light analyse-link" data-position="top" data-tooltip="Analyse">
<i class="material-icons">search</i>
</a>
</div>
<div class="modal delete-modal">
<div class="modal-content">
<h4>Confirm corpus deletion</h4>
<p>Do you really want to delete the corpus <b class="title1"></b>? All files will be permanently deleted!</p>
</div>
<div class="modal-footer">
<a href="#!" class="btn modal-close waves-effect waves-light">Cancel</a>
<a class="btn modal-close red waves-effect waves-light delete-link"><i class="material-icons left">delete</i>Delete</a>
</div>
</div>
</td>
</tr>`,
valueNames: [
"creation_date",
"description",
"title",
"title1",
{data: ["id"]},
{name: "analyse-link", attr: "href"},
{name: "delete-link", attr: "href"},
{name: "delete-modal-trigger", attr: "data-target"},
{name: "delete-modal", attr: "id"},
{name: "link", attr: "href"},
{name: "status", attr: "data-status"},
]
},
CorpusFile: {
item: `<tr>
<td class="filename" style="word-break: break-word;"></td>
<td class="author" style="word-break: break-word;"></td>
<td class="title" style="word-break: break-word;"></td>
<td>
<div class="right-align">
<a class="btn-floating modal-trigger red tooltipped waves-effect waves-light delete-modal-trigger" data-position="top" data-tooltip="Delete">
<i class="material-icons">delete</i>
</a>
<a class="btn-floating tooltipped waves-effect waves-light download-link" data-position="top" data-tooltip="Download">
<i class="material-icons">file_download</i>
</a>
<a class="btn-floating tooltipped waves-effect waves-light link" data-position="top" data-tooltip="Edit">
<i class="material-icons">edit</i>
</a>
</div>
<div class="modal delete-modal">
<div class="modal-content">
<h4>Confirm corpus file deletion</h4>
<p>Do you really want to delete the corpus file <b class="title1"></b>? It be permanently deleted!</p>
</div>
<div class="modal-footer">
<a href="#!" class="btn modal-close waves-effect waves-light">Cancel</a>
<a class="btn modal-close red waves-effect waves-light delete-link"><i class="material-icons left">delete</i>Delete</a>
</div>
</div>
</td>
</tr>`,
valueNames: [
"author",
"filename",
"title",
"title1",
{name: "delete-link", attr: "href"},
{name: "delete-modal-trigger", attr: "data-target"},
{name: "delete-modal", attr: "id"},
{name: "download-link", attr: "href"},
{name: "link", attr: "href"},
],
},
Job: {
item: `<tr>
<td>
<a class="btn-floating disabled">
<i class="material-icons service"></i>
</a>
</td>
<td>
<b class="title"></b><br>
<i class="description"></i>
</td>
<td>
<span class="badge new status" data-badge-caption=""></span>
</td>
<td>
<div class="right-align">
<a class="btn-floating modal-trigger red tooltipped waves-effect waves-light delete-modal-trigger" data-position="top" data-tooltip="Delete">
<i class="material-icons">delete</i>
</a>
<a class="btn-floating tooltipped waves-effect waves-light link" data-position="top" data-tooltip="Go to job">
<i class="material-icons">send</i>
</a>
</div>
<div class="modal delete-modal">
<div class="modal-content">
<h4>Confirm job deletion</h4>
<p>Do you really want to delete the job <b class="title1"></b>? All files will be permanently deleted!</p>
</div>
<div class="modal-footer">
<a href="#!" class="btn modal-close waves-effect waves-light">Cancel</a>
<a class="btn modal-close red waves-effect waves-light delete-link"><i class="material-icons left">delete</i>Delete</a>
</div>
</div>
</td>
</tr>`,
valueNames: [
"creation_date",
"description",
"title",
"title1",
{data: ["id"]},
{name: "delete-link", attr: "href"},
{name: "delete-modal-trigger", attr: "data-target"},
{name: "delete-modal", attr: "id"},
{name: "link", attr: "href"},
{name: "service", attr: "data-service"},
{name: "status", attr: "data-status"},
],
},
JobInput: {
item : `<tr>
<td class="filename"></td>
<td class="right-align">
<a class="btn-floating tooltipped waves-effect waves-light download-link" data-position="top" data-tooltip="Download">
<i class="material-icons">file_download</i>
</a>
</td>
</tr>`,
valueNames: [
"filename",
"id",
{name: "download-link", attr: "href"},
],
},
QueryResult: {
item: `<tr>
<td>
<b class="title"></b><br>
<i class="description"></i><br>
</td>
<td>
<span class="corpus_name"></span><br>
<span class="query"></span>
</td>
<td>
<div class="right-align">
<a class="btn-floating modal-trigger red tooltipped waves-effect waves-light delete-modal-trigger" data-position="top" data-tooltip="Delete">
<i class="material-icons">delete</i>
</a>
<a class="btn-floating tooltipped waves-effect waves-light link" data-position="top" data-tooltip="Info">
<i class="material-icons">info</i>
</a>
<a class="btn-floating tooltipped waves-effect waves-light inspect-link" data-position="top" data-tooltip="Analyse">
<i class="material-icons">search</i>
</a>
</div>
<div class="modal delete-modal">
<div class="modal-content">
<h4>Confirm query result deletion</h4>
<p>Do you really want to delete the query result <b class="title1"></b>? It will be permanently deleted!</p>
</div>
<div class="modal-footer">
<a href="#!" class="btn modal-close waves-effect waves-light">Cancel</a>
<a class="btn modal-close red waves-effect waves-light delete-link"><i class="material-icons left">delete</i>Delete</a>
</div>
</div>
</td>
</tr>`,
valueNames: [
"corpus_name",
"description",
"query",
"title",
"title2",
{data: ["id"]},
{name: "delete-link", attr: "href"},
{name: "delete-modal-trigger", attr: "data-target"},
{name: "delete-modal", attr: "id"},
{name: "inspect-link", attr: "href"},
{name: "link", attr: "href"},
],
},
User: {
item: `<tr>
<td class="username"></td>
<td class="email"></td>
<td class="role_id"></td>
<td class="confirmed"></td>
<td class="id"></td>
<td>
<div class="right-align">
<a class="btn-floating modal-trigger red tooltipped waves-effect waves-light delete-modal-trigger" data-position="top" data-tooltip="Delete">
<i class="material-icons">delete</i>
</a>
<a class="btn-floating tooltipped waves-effect waves-light link" data-position="top" data-tooltip="Go to user">
<i class="material-icons">send</i>
</a>
</div>
<div class="modal delete-modal">
<div class="modal-content">
<h4>Confirm corpus deletion</h4>
<p>Do you really want to delete the job <b class="title1"></b>? All files will be permanently deleted!</p>
</div>
<div class="modal-footer">
<a href="#!" class="btn modal-close waves-effect waves-light">Cancel</a>
<a class="btn modal-close red waves-effect waves-light delete-link"><i class="material-icons left">delete</i>Delete</a>
</div>
</div>
</td>
</tr>`,
valueNames: [
"username",
"username2",
"email",
"role_id",
"confirmed",
"id",
{name: "link", attr: "href"},
{name: "delete-link", attr: "href"},
{name: "delete-modal-trigger", attr: "data-target"},
{name: "delete-modal", attr: "id"},
],
},
}; };
export { RessourceList, };

420
web/app/static/js/nopaque.lists.js.bak Normal file
View File

@ -0,0 +1,420 @@
class RessourceList extends List {
constructor(idOrElement, subscriberList, type, options) {
if (!type || !["Corpus", "CorpusFile", "Job", "JobInput", "QueryResult", "User"].includes(type)) {
throw "Unknown Type!";
}
super(idOrElement, {...RessourceList.options['common'],
...RessourceList.options[type],
...(options ? options : {})});
if (subscriberList) {subscriberList.push(this);}
this.type = type;
}
_init(ressources) {
this.clear();
this._add(Object.values(ressources));
this.sort("id", {order: "desc"});
}
_update(patch) {
let item, pathArray;
for (let operation of patch) {
/* "/{ressourceName}/{ressourceId}/..." -> ["{ressourceId}", "..."] */
pathArray = operation.path.split("/").slice(2);
switch(operation.op) {
case "add":
if (pathArray.includes("results")) {break;}
this._add([operation.value]);
break;
case "remove":
this.remove("id", pathArray[0]);
break;
case "replace":
item = this.get("id", pathArray[0])[0];
switch(pathArray[1]) {
case "status":
item.values({status: operation.value,
"analyse-link": ["analysing", "prepared", "start analysis"].includes(operation.value) ? `/corpora/${pathArray[0]}/analyse` : ""});
break;
default:
break;
}
default:
break;
}
}
}
_add(values, callback) {
this.add(values.map(x => RessourceList.dataMappers[this.type](x)), callback);
// Initialize modal and tooltipped elements in list
M.AutoInit(this.listContainer);
}
}
RessourceList.dataMappers = {
// A data mapper describes entitys rendered per row. One key value pair holds
// the data to be rendered in the list.js table. Key has to correspond
// with the ValueNames defined below in RessourceList.options ValueNames.
// Links are declared with double ticks(") around them. The key for links
// have to correspond with the class of an <a> element in the
// RessourceList.options item blueprint.
/* ### Corpus mapper ### */
Corpus: corpus => ({
creation_date: corpus.creation_date,
description: corpus.description,
id: corpus.id,
link: `/corpora/${corpus.id}`,
status: corpus.status,
title: corpus.title,
title1: corpus.title,
"analyse-link": ["analysing", "prepared", "start analysis"].includes(corpus.status) ? `/corpora/${corpus.id}/analyse` : "",
"delete-link": `/corpora/${corpus.id}/delete`,
"delete-modal": `delete-corpus-${corpus.id}-modal`,
"delete-modal-trigger": `delete-corpus-${corpus.id}-modal`,
}),
/* ### CorpusFile mapper ### TODO: replace delete-modal with delete-onclick */
CorpusFile: corpus_file => ({
author: corpus_file.author,
filename: corpus_file.filename,
id: corpus_file.id,
link: `${corpus_file.corpus_id}/files/${corpus_file.id}`,
"publishing-year": corpus_file.publishing_year,
title: corpus_file.title,
title1: corpus_file.title,
"delete-link": `/corpora/${corpus_file.corpus_id}/files/${corpus_file.id}/delete`,
"delete-modal": `delete-corpus-file-${corpus_file.id}-modal`,
"delete-modal-trigger": `delete-corpus-file-${corpus_file.id}-modal`,
"download-link": `${corpus_file.corpus_id}/files/${corpus_file.id}/download`,
}),
/* ### Job mapper ### */
Job: job => ({
creation_date: job.creation_date,
description: job.description,
id: job.id,
link: `/jobs/${job.id}`,
service: job.service.name,
status: job.status,
title: job.title,
title1: job.title,
"delete-link": `/jobs/${job.id}/delete`,
"delete-modal": `delete-job-${job.id}-modal`,
"delete-modal-trigger": `delete-job-${job.id}-modal`,
}),
/* ### JobInput mapper ### */
JobInput: job_input => ({
filename: job_input.filename,
id: job_input.job_id,
"download-link": `${job_input.job_id}/inputs/${job_input.id}/download`
}),
/* ### QueryResult mapper ### */
QueryResult: query_result => ({
corpus_name: query_result.query_metadata.corpus_name,
description: query_result.description,
id: query_result.id,
link: `/corpora/result/${query_result.id}`,
query: query_result.query_metadata.query,
title: query_result.title,
"delete-link": `/corpora/result/${query_result.id}/delete`,
"delete-modal": `delete-query-result-${query_result.id}-modal`,
"delete-modal-trigger": `delete-query-result-${query_result.id}-modal`,
"inspect-link": `/corpora/result/${query_result.id}/inspect`,
}),
/* ### User mapper ### */
User: user => ({
confirmed: user.confirmed,
email: user.email,
id: user.id,
link: `users/${user.id}`,
role: user.role.name,
username: user.username,
username2: user.username,
"delete-link": `/admin/users/${user.id}/delete`,
"delete-modal": `delete-user-${user.id}-modal`,
"delete-modal-trigger": `delete-user-${user.id}-modal`,
}),
};
RessourceList.options = {
// common list.js options for 5 rows per page etc.
common: {page: 5, pagination: [{innerWindow: 4, outerWindow: 1}]},
// extended list.js options for 10 rows per page etc.
extended: {
page: 10,
pagination: [
{
name: "paginationTop",
paginationClass: "paginationTop",
innerWindow: 8,
outerWindow: 1
},
{
paginationClass: "paginationBottom",
innerWindow: 8,
outerWindow: 1,
},
],
},
/* Type specific List.js options. Usually only "item" and "valueNames" gets
* defined here but it is possible to define other List.js options.
* item: https://listjs.com/api/#item
* valueNames: https://listjs.com/api/#valueNames
*/
Corpus: {
item: `<tr>
<td>
<a class="btn-floating disabled">
<i class="material-icons service">book</i>
</a>
</td>
<td>
<b class="title"></b><br>
<i class="description"></i>
</td>
<td>
<span class="badge new status" data-badge-caption=""></span>
</td>
<td>
<div class="right-align">
<a class="btn-floating modal-trigger red tooltipped waves-effect waves-light delete-modal-trigger" data-position="top" data-tooltip="Delete">
<i class="material-icons">delete</i>
</a>
<a class="btn-floating tooltipped waves-effect waves-light link" data-position="top" data-tooltip="Edit">
<i class="material-icons">edit</i>
</a>
<a class="btn-floating tooltipped waves-effect waves-light analyse-link" data-position="top" data-tooltip="Analyse">
<i class="material-icons">search</i>
</a>
</div>
<div class="modal delete-modal">
<div class="modal-content">
<h4>Confirm corpus deletion</h4>
<p>Do you really want to delete the corpus <b class="title1"></b>? All files will be permanently deleted!</p>
</div>
<div class="modal-footer">
<a href="#!" class="btn modal-close waves-effect waves-light">Cancel</a>
<a class="btn modal-close red waves-effect waves-light delete-link"><i class="material-icons left">delete</i>Delete</a>
</div>
</div>
</td>
</tr>`,
valueNames: [
"creation_date",
"description",
"title",
"title1",
{data: ["id"]},
{name: "analyse-link", attr: "href"},
{name: "delete-link", attr: "href"},
{name: "delete-modal-trigger", attr: "data-target"},
{name: "delete-modal", attr: "id"},
{name: "link", attr: "href"},
{name: "status", attr: "data-status"},
]
},
CorpusFile: {
item: `<tr>
<td class="filename" style="word-break: break-word;"></td>
<td class="author" style="word-break: break-word;"></td>
<td class="title" style="word-break: break-word;"></td>
<td class="publishing-year" style="word-break: break-word;"></td>
<td>
<div class="right-align">
<a class="btn-floating modal-trigger red tooltipped waves-effect waves-light delete-modal-trigger" data-position="top" data-tooltip="Delete">
<i class="material-icons">delete</i>
</a>
<a class="btn-floating tooltipped waves-effect waves-light download-link" data-position="top" data-tooltip="Download">
<i class="material-icons">file_download</i>
</a>
<a class="btn-floating tooltipped waves-effect waves-light link" data-position="top" data-tooltip="Edit">
<i class="material-icons">edit</i>
</a>
</div>
<div class="modal delete-modal">
<div class="modal-content">
<h4>Confirm corpus file deletion</h4>
<p>Do you really want to delete the corpus file <b class="title1"></b>? It be permanently deleted!</p>
</div>
<div class="modal-footer">
<a href="#!" class="btn modal-close waves-effect waves-light">Cancel</a>
<a class="btn modal-close red waves-effect waves-light delete-link"><i class="material-icons left">delete</i>Delete</a>
</div>
</div>
</td>
</tr>`,
valueNames: [
"author",
"filename",
"publishing-year",
"title",
"title1",
{data: ["id"]},
{name: "delete-link", attr: "href"},
{name: "delete-modal-trigger", attr: "data-target"},
{name: "delete-modal", attr: "id"},
{name: "download-link", attr: "href"},
{name: "link", attr: "href"},
],
},
Job: {
item: `<tr>
<td>
<a class="btn-floating disabled">
<i class="material-icons service"></i>
</a>
</td>
<td>
<b class="title"></b><br>
<i class="description"></i>
</td>
<td>
<span class="badge new status" data-badge-caption=""></span>
</td>
<td>
<div class="right-align">
<a class="btn-floating modal-trigger red tooltipped waves-effect waves-light delete-modal-trigger" data-position="top" data-tooltip="Delete">
<i class="material-icons">delete</i>
</a>
<a class="btn-floating tooltipped waves-effect waves-light link" data-position="top" data-tooltip="Go to job">
<i class="material-icons">send</i>
</a>
</div>
<div class="modal delete-modal">
<div class="modal-content">
<h4>Confirm job deletion</h4>
<p>Do you really want to delete the job <b class="title1"></b>? All files will be permanently deleted!</p>
</div>
<div class="modal-footer">
<a href="#!" class="btn modal-close waves-effect waves-light">Cancel</a>
<a class="btn modal-close red waves-effect waves-light delete-link"><i class="material-icons left">delete</i>Delete</a>
</div>
</div>
</td>
</tr>`,
valueNames: [
"creation_date",
"description",
"title",
"title1",
{data: ["id"]},
{name: "delete-link", attr: "href"},
{name: "delete-modal-trigger", attr: "data-target"},
{name: "delete-modal", attr: "id"},
{name: "link", attr: "href"},
{name: "service", attr: "data-service"},
{name: "status", attr: "data-status"},
],
},
JobInput: {
item : `<tr>
<td class="filename"></td>
<td class="right-align">
<a class="btn-floating tooltipped waves-effect waves-light download-link" data-position="top" data-tooltip="Download">
<i class="material-icons">file_download</i>
</a>
</td>
</tr>`,
valueNames: [
"filename",
"id",
{name: "download-link", attr: "href"},
],
},
QueryResult: {
item: `<tr>
<td>
<b class="title"></b><br>
<i class="description"></i><br>
</td>
<td>
<span class="corpus_name"></span><br>
<span class="query"></span>
</td>
<td>
<div class="right-align">
<a class="btn-floating modal-trigger red tooltipped waves-effect waves-light delete-modal-trigger" data-position="top" data-tooltip="Delete">
<i class="material-icons">delete</i>
</a>
<a class="btn-floating tooltipped waves-effect waves-light link" data-position="top" data-tooltip="Info">
<i class="material-icons">info</i>
</a>
<a class="btn-floating tooltipped waves-effect waves-light inspect-link" data-position="top" data-tooltip="Analyse">
<i class="material-icons">search</i>
</a>
</div>
<div class="modal delete-modal">
<div class="modal-content">
<h4>Confirm query result deletion</h4>
<p>Do you really want to delete the query result <b class="title1"></b>? It will be permanently deleted!</p>
</div>
<div class="modal-footer">
<a href="#!" class="btn modal-close waves-effect waves-light">Cancel</a>
<a class="btn modal-close red waves-effect waves-light delete-link"><i class="material-icons left">delete</i>Delete</a>
</div>
</div>
</td>
</tr>`,
valueNames: [
"corpus_name",
"description",
"query",
"title",
"title2",
{data: ["id"]},
{name: "delete-link", attr: "href"},
{name: "delete-modal-trigger", attr: "data-target"},
{name: "delete-modal", attr: "id"},
{name: "inspect-link", attr: "href"},
{name: "link", attr: "href"},
],
},
User: {
item: `<tr>
<td class="id"></td>
<td class="username"></td>
<td class="email"></td>
<td class="role"></td>
<td>
<div class="right-align">
<a class="btn-floating modal-trigger red tooltipped waves-effect waves-light delete-modal-trigger" data-position="top" data-tooltip="Delete">
<i class="material-icons">delete</i>
</a>
<a class="btn-floating tooltipped waves-effect waves-light link" data-position="top" data-tooltip="Go to user">
<i class="material-icons">send</i>
</a>
</div>
<div class="modal delete-modal">
<div class="modal-content">
<h4>Confirm corpus deletion</h4>
<p>Do you really want to delete the job <b class="title1"></b>? All files will be permanently deleted!</p>
</div>
<div class="modal-footer">
<a href="#!" class="btn modal-close waves-effect waves-light">Cancel</a>
<a class="btn modal-close red waves-effect waves-light delete-link"><i class="material-icons left">delete</i>Delete</a>
</div>
</div>
</td>
</tr>`,
valueNames: [
"username",
"username2",
"email",
"role",
"id",
{name: "link", attr: "href"},
{name: "delete-link", attr: "href"},
{name: "delete-modal-trigger", attr: "data-target"},
{name: "delete-modal", attr: "id"},
],
},
};
export { RessourceList, };

104
web/app/static/js/nopaque/displays/CorpusDisplay.js Normal file
View File

@ -0,0 +1,104 @@
class CorpusDisplay extends RessourceDisplay {
constructor(displayElement) {
super(displayElement);
this.corpus = undefined;
this.user.eventListeners.corpus.addEventListener((eventType, payload) => this.eventHandler(eventType, payload), displayElement.dataset.corpusId);
}
init(corpus) {
this.corpus = corpus;
for (let exportCorpusTriggerElement of this.displayElement.querySelectorAll('.export-corpus-trigger')) {exportCorpusTriggerElement.addEventListener('click', () => this.requestCorpusExport());}
nopaque.appClient.socket.on(`export_corpus_${this.corpus.id}`, () => this.downloadCorpus());
this.setCreationDate(this.corpus.creation_date);
this.setDescription(this.corpus.description);
this.setLastEditedDate(this.corpus.last_edited_date);
this.setStatus(this.corpus.status);
this.setTitle(this.corpus.title);
this.setTokenRatio(this.corpus.current_nr_of_tokens, this.corpus.max_nr_of_tokens);
}
patch(patch) {
let re;
for (let operation of patch) {
switch(operation.op) {
case 'replace':
// Matches: /jobs/{this.job.id}/status
re = new RegExp('^/corpora/' + this.corpus.id + '/last_edited_date');
if (re.test(operation.path)) {this.setLastEditedDate(operation.value); break;}
// Matches: /jobs/{this.job.id}/status
re = new RegExp('^/corpora/' + this.corpus.id + '/status$');
if (re.test(operation.path)) {this.setStatus(operation.value); break;}
break;
default:
break;
}
}
}
requestCorpusExport() {
nopaque.appClient.socket.emit('export_corpus', this.corpus.id);
nopaque.flash('Preparing your corpus export...', 'corpus');
for (let exportCorpusTriggerElement of this.displayElement.querySelectorAll('.export-corpus-trigger')) {exportCorpusTriggerElement.classList.toggle('disabled', true);}
}
downloadCorpus() {
nopaque.flash('Corpus export is done. Your corpus download is ready!', 'corpus');
for (let exportCorpusTriggerElement of this.displayElement.querySelectorAll('.export-corpus-trigger')) {exportCorpusTriggerElement.classList.toggle('disabled', false);}
// Little trick to call the download view after ziping has finished
let fakeBtn = document.createElement('a');
fakeBtn.href = `/corpora/${this.corpus.id}/download`;
fakeBtn.click();
}
setTitle(title) {
for (let element of this.displayElement.querySelectorAll('.corpus-title')) {this.setElement(element, title);}
}
setTokenRatio(currentNrOfTokens, maxNrOfTokens) {
let tokenRatio = `${currentNrOfTokens}/${maxNrOfTokens}`;
for (let element of this.displayElement.querySelectorAll('.corpus-token-ratio')) {this.setElement(element, tokenRatio);}
}
setDescription(description) {
for (let element of this.displayElement.querySelectorAll('.corpus-description')) {this.setElement(element, description);}
}
setStatus(status) {
for (let element of this.displayElement.querySelectorAll('.analyse-corpus-trigger')) {
if (['analysing', 'prepared', 'start analysis'].includes(status)) {
element.classList.remove('disabled');
} else {
element.classList.add('disabled');
}
}
for (let element of this.displayElement.querySelectorAll('.build-corpus-trigger')) {
if (status === 'unprepared' && Object.values(this.corpus.files).length > 0) {
element.classList.remove('disabled');
} else {
element.classList.add('disabled');
}
}
for (let element of this.displayElement.querySelectorAll('.corpus-status')) {this.setElement(element, status);}
for (let exportCorpusTriggerElement of this.displayElement.querySelectorAll('.export-corpus-trigger')) {
exportCorpusTriggerElement.classList.toggle('disabled', !['prepared', 'start analysis', 'stop analysis'].includes(status));
}
for (let element of this.displayElement.querySelectorAll('.status')) {element.dataset.status = status;}
for (let element of this.displayElement.querySelectorAll('.status-spinner')) {
if (['submitted', 'queued', 'running', 'canceling', 'start analysis', 'stop analysis'].includes(status)) {
element.classList.remove('hide');
} else {
element.classList.add('hide');
}
}
}
setCreationDate(creationDateTimestamp) {
let creationDate = new Date(creationDateTimestamp * 1000).toLocaleString("en-US");
for (let element of this.displayElement.querySelectorAll('.corpus-creation-date')) {this.setElement(element, creationDate);}
}
setLastEditedDate(endDateTimestamp) {
let endDate = new Date(endDateTimestamp * 1000).toLocaleString("en-US");
for (let element of this.displayElement.querySelectorAll('.corpus-end-date')) {this.setElement(element, endDate);}
}
}

88
web/app/static/js/nopaque/displays/JobDisplay.js Normal file
View File

@ -0,0 +1,88 @@
class JobDisplay extends RessourceDisplay {
constructor(displayElement) {
super(displayElement);
this.job = undefined;
this.user.eventListeners.job.addEventListener((eventType, payload) => this.eventHandler(eventType, payload), displayElement.dataset.jobId);
}
init(job) {
this.job = job;
this.setCreationDate(this.job.creation_date);
this.setEndDate(this.job.creation_date);
this.setDescription(this.job.description);
this.setService(this.job.service);
this.setServiceArgs(this.job.service_args);
this.setServiceVersion(this.job.service_version);
this.setStatus(this.job.status);
this.setTitle(this.job.title);
}
patch(patch) {
let re;
for (let operation of patch) {
switch(operation.op) {
case 'replace':
// Matches: /jobs/{this.job.id}/status
re = new RegExp('^/jobs/' + this.job.id + '/end_date');
if (re.test(operation.path)) {this.setEndDate(operation.value); break;}
// Matches: /jobs/{this.job.id}/status
re = new RegExp('^/jobs/' + this.job.id + '/status$');
if (re.test(operation.path)) {this.setStatus(operation.value); break;}
break;
default:
break;
}
}
}
setTitle(title) {
for (let element of this.displayElement.querySelectorAll('.job-title')) {this.setElement(element, title);}
}
setDescription(description) {
for (let element of this.displayElement.querySelectorAll('.job-description')) {this.setElement(element, description);}
}
setStatus(status) {
for (let element of this.displayElement.querySelectorAll('.job-status')) {
this.setElement(element, status);
}
for (let element of this.displayElement.querySelectorAll('.status')) {element.dataset.status = status;}
for (let element of this.displayElement.querySelectorAll('.status-spinner')) {
if (['complete', 'failed'].includes(status)) {
element.classList.add('hide');
} else {
element.classList.remove('hide');
}
}
for (let element of this.displayElement.querySelectorAll('.restart-job-trigger')) {
if (['complete', 'failed'].includes(status)) {
element.classList.remove('hide');
} else {
element.classList.add('hide');
}
}
}
setCreationDate(creationDateTimestamp) {
let creationDate = new Date(creationDateTimestamp * 1000).toLocaleString("en-US");
for (let element of this.displayElement.querySelectorAll('.job-creation-date')) {this.setElement(element, creationDate);}
}
setEndDate(endDateTimestamp) {
let endDate = new Date(endDateTimestamp * 1000).toLocaleString("en-US");
for (let element of this.displayElement.querySelectorAll('.job-end-date')) {this.setElement(element, endDate);}
}
setService(service) {
for (let element of this.displayElement.querySelectorAll('.job-service')) {this.setElement(element, service);}
}
setServiceArgs(serviceArgs) {
for (let element of this.displayElement.querySelectorAll('.job-service-args')) {this.setElement(element, serviceArgs);}
}
setServiceVersion(serviceVersion) {
for (let element of this.displayElement.querySelectorAll('.job-service-version')) {this.setElement(element, serviceVersion);}
}
}

45
web/app/static/js/nopaque/displays/RessourceDisplay.js Normal file
View File

@ -0,0 +1,45 @@
class RessourceDisplay {
constructor(displayElement) {
if (displayElement.dataset.userId) {
if (displayElement.dataset.userId in nopaque.appClient.users) {
this.user = nopaque.appClient.users[displayElement.dataset.userId];
} else {
console.error(`User not found: ${displayElement.dataset.userId}`);
return;
}
} else {
this.user = nopaque.appClient.users.self;
}
this.displayElement = displayElement;
}
eventHandler(eventType, payload) {
switch (eventType) {
case 'init':
this.init(payload);
break;
case 'patch':
this.patch(payload);
break;
default:
console.log(`Unknown event type: ${eventType}`);
break;
}
}
init() {console.error('init method not implemented!');}
patch() {console.error('patch method not implemented!');}
setElement(element, value) {
switch (element.tagName) {
case 'INPUT':
element.value = value;
M.updateTextFields();
break;
default:
element.innerText = value;
break;
}
}
}

98
web/app/static/js/nopaque/lists/CorpusFileList.js Normal file
View File

@ -0,0 +1,98 @@
class CorpusFileList extends RessourceList {
constructor(listElement, options = {}) {
super(listElement, {...CorpusFileList.options, ...options});
this.corpus = undefined;
this.user.eventListeners.corpus.addEventListener((eventType, payload) => this.eventHandler(eventType, payload), listElement.dataset.corpusId);
}
init(corpus) {
this.corpus = corpus;
super.init(this.corpus.files);
}
onclick(event) {
let ressourceElement = event.target.closest('tr');
if (ressourceElement === null) {return;}
let corpusFileId = ressourceElement.dataset.id;
let actionButtonElement = event.target.closest('.action-button');
if (actionButtonElement === null) {return;}
let action = actionButtonElement === null ? 'view' : actionButtonElement.dataset.action;
switch (action) {
case 'delete':
let deleteModalHTML = `<div class="modal">
<div class="modal-content">
<h4>Confirm corpus deletion</h4>
<p>Do you really want to delete the corpus file <b>${this.corpus.files[corpusFileId].filename}</b>? It will be permanently deleted!</p>
</div>
<div class="modal-footer">
<a href="#!" class="btn modal-close waves-effect waves-light">Cancel</a>
<a class="btn modal-close red waves-effect waves-light" href="${this.corpus.files[corpusFileId].url}/delete"><i class="material-icons left">delete</i>Delete</a>
</div>
</div>`;
let deleteModalParentElement = document.querySelector('main');
deleteModalParentElement.insertAdjacentHTML('beforeend', deleteModalHTML);
let deleteModalElement = deleteModalParentElement.lastChild;
let deleteModal = M.Modal.init(deleteModalElement, {onCloseEnd: () => {deleteModal.destroy(); deleteModalElement.remove();}});
deleteModal.open();
break;
case 'download':
window.location.href = this.corpus.files[corpusFileId].download_url;
break;
case 'view':
if (corpusFileId !== '-1') {window.location.href = this.corpus.files[corpusFileId].url;}
break;
default:
console.error(`Unknown action: "${action}"`);
break;
}
}
patch(patch) {
let id, match, re, valueName;
for (let operation of patch) {
switch(operation.op) {
case 'add':
// Matches the only paths that should be handled here: /corpora/{this.corpus.id}/files/{corpusFileId}
re = new RegExp('^/corpora/' + this.corpus.id + '/files/(\\d+)$');
if (re.test(operation.path)) {this.add(operation.value);}
break;
case 'remove':
// See case add ;)
re = new RegExp('^/corpora/' + this.corpus.id + '/files/(\\d+)$');
if (re.test(operation.path)) {
[match, id] = operation.path.match(re);
this.remove(id);
}
break;
case 'replace':
// Matches the only paths that should be handled here: /corpora/{corpusId}/{status || description || title}
re = new RegExp('^/corpora/' + this.corpus.id + '/files/(\\d+)/(author|filename|publishing_year|title)$');
if (re.test(operation.path)) {
[match, id, valueName] = operation.path.match(re);
this.replace(id, valueName, operation.value);
}
break;
default:
break;
}
}
}
preprocessRessource(corpusFile) {
return {id: corpusFile.id, author: corpusFile.author, filename: corpusFile.filename, publishing_year: corpusFile.publishing_year, title: corpusFile.title};
}
}
CorpusFileList.options = {
item: `<tr>
<td><span class="filename"></span></td>
<td><span class="author"></span></td>
<td><span class="title"></span></td>
<td><span class="publishing_year"></span></td>
<td class="right-align">
<a class="action-button btn-floating red tooltipped waves-effect waves-light" data-action="delete" data-position="top" data-tooltip="Delete"><i class="material-icons">delete</i></a>
<a class="action-button btn-floating tooltipped waves-effect waves-light" data-action="download" data-position="top" data-tooltip="View"><i class="material-icons">file_download</i></a>
<a class="action-button btn-floating tooltipped waves-effect waves-light" data-action="view" data-position="top" data-tooltip="View"><i class="material-icons">send</i></a>
</td>
</tr>`,
valueNames: [{data: ['id']}, 'author', 'filename', 'publishing_year', 'title']
};

95
web/app/static/js/nopaque/lists/CorpusList.js Normal file
View File

@ -0,0 +1,95 @@
class CorpusList extends RessourceList {
constructor(listElement, options = {}) {
super(listElement, {...CorpusList.options, ...options});
this.corpora = undefined;
this.user.eventListeners.corpus.addEventListener((eventType, payload) => this.eventHandler(eventType, payload));
}
init(corpora) {
this.corpora = corpora;
super.init(corpora);
}
onclick(event) {
let ressourceElement = event.target.closest('tr');
if (ressourceElement === null) {return;}
let corpusId = ressourceElement.dataset.id;
let actionButtonElement = event.target.closest('.action-button');
let action = (actionButtonElement === null) ? 'view' : actionButtonElement.dataset.action;
switch (action) {
case 'delete':
let deleteModalHTML = `<div class="modal">
<div class="modal-content">
<h4>Confirm corpus deletion</h4>
<p>Do you really want to delete the corpus <b>${this.corpora[corpusId].title}</b>? All files will be permanently deleted!</p>
</div>
<div class="modal-footer">
<a href="#!" class="btn modal-close waves-effect waves-light">Cancel</a>
<a class="btn modal-close red waves-effect waves-light" href="${this.corpora[corpusId].url}/delete"><i class="material-icons left">delete</i>Delete</a>
</div>
</div>`;
let deleteModalParentElement = document.querySelector('main');
deleteModalParentElement.insertAdjacentHTML('beforeend', deleteModalHTML);
let deleteModalElement = deleteModalParentElement.lastChild;
let deleteModal = M.Modal.init(deleteModalElement, {onCloseEnd: () => {deleteModal.destroy(); deleteModalElement.remove();}});
deleteModal.open();
break;
case 'view':
if (corpusId !== '-1') {window.location.href = this.corpora[corpusId].url;}
break;
default:
console.error(`Unknown action: ${action}`);
break;
}
}
patch(patch) {
let id, match, re, valueName;
for (let operation of patch) {
switch(operation.op) {
case 'add':
// Matches the only paths that should be handled here: /corpora/{corpusId}
re = /^\/corpora\/(\d+)$/;
if (re.test(operation.path)) {this.add(operation.value);}
break;
case 'remove':
// See case 'add' ;)
re = /^\/corpora\/(\d+)$/;
if (re.test(operation.path)) {
[match, id] = operation.path.match(re);
this.remove(id);
}
break;
case 'replace':
// Matches the only paths that should be handled here: /corpora/{corpusId}/{status || description || title}
re = /^\/corpora\/(\d+)\/(status|description|title)$/;
if (re.test(operation.path)) {
[match, id, valueName] = operation.path.match(re);
this.replace(id, valueName, operation.value);
}
break;
default:
break;
}
}
}
preprocessRessource(corpus) {
return {id: corpus.id,
status: corpus.status,
description: corpus.description,
title: corpus.title};
}
}
CorpusList.options = {
item: `<tr>
<td><a class="btn-floating disabled"><i class="material-icons">book</i></a></td>
<td><b class="title"></b><br><i class="description"></i></td>
<td><span class="badge new status" data-badge-caption=""></span></td>
<td class="right-align">
<a class="action-button btn-floating red tooltipped waves-effect waves-light" data-action="delete" data-position="top" data-tooltip="Delete"><i class="material-icons">delete</i></a>
<a class="action-button btn-floating tooltipped waves-effect waves-light" data-action="view" data-position="top" data-tooltip="View"><i class="material-icons">send</i></a>
</td>
</tr>`,
valueNames: [{data: ['id']}, {name: 'status', attr: 'data-status'}, 'description', 'title']
};

42
web/app/static/js/nopaque/lists/JobInputList.js Normal file
View File

@ -0,0 +1,42 @@
class JobInputList extends RessourceList {
constructor(listElement, options = {}) {
super(listElement, {...JobInputList.options, ...options});
this.job = undefined;
this.user.eventListeners.job.addEventListener((eventType, payload) => this.eventHandler(eventType, payload), listElement.dataset.jobId);
}
init(job) {
this.job = job;
super.init(this.job.inputs);
}
onclick(event) {
let ressourceElement = event.target.closest('tr');
if (ressourceElement === null) {return;}
let jobInputId = ressourceElement.dataset.id;
let actionButtonElement = event.target.closest('.action-button');
if (actionButtonElement === null) {return;}
let action = actionButtonElement.dataset.action;
switch (action) {
case 'download':
window.location.href = this.job.inputs[jobInputId].download_url;
break;
default:
console.error(`Unknown action: "${action}"`);
break;
}
}
preprocessRessource(jobInput) {
return {id: jobInput.id, filename: jobInput.filename};
}
}
JobInputList.options = {
item: `<tr>
<td><span class="filename"></span></td>
<td class="right-align">
<a class="action-button btn-floating tooltipped waves-effect waves-light" data-action="download" data-position="top" data-tooltip="View"><i class="material-icons">file_download</i></a>
</td>
</tr>`,
valueNames: [{data: ['id']}, 'filename']
};

96
web/app/static/js/nopaque/lists/JobList.js Normal file
View File

@ -0,0 +1,96 @@
class JobList extends RessourceList {
constructor(listElement, options = {}) {
super(listElement, {...JobList.options, ...options});
this.jobs = undefined;
this.user.eventListeners.job.addEventListener((eventType, payload) => this.eventHandler(eventType, payload));
}
init(jobs) {
this.jobs = jobs;
super.init(jobs);
}
onclick(event) {
let ressourceElement = event.target.closest('tr');
if (ressourceElement === null) {return;}
let jobId = ressourceElement.dataset.id;
let actionButtonElement = event.target.closest('.action-button');
let action = actionButtonElement === null ? 'view' : actionButtonElement.dataset.action;
switch (action) {
case 'delete':
let deleteModalHTML = `<div class="modal">
<div class="modal-content">
<h4>Confirm job deletion</h4>
<p>Do you really want to delete the job <b>${this.user.data.jobs[jobId].title}</b>? All files will be permanently deleted!</p>
</div>
<div class="modal-footer">
<a href="#!" class="btn modal-close waves-effect waves-light">Cancel</a>
<a class="btn modal-close red waves-effect waves-light" href="${this.user.data.jobs[jobId].url}/delete"><i class="material-icons left">delete</i>Delete</a>
</div>
</div>`;
let deleteModalParentElement = document.querySelector('main');
deleteModalParentElement.insertAdjacentHTML('beforeend', deleteModalHTML);
let deleteModalElement = deleteModalParentElement.lastChild;
let deleteModal = M.Modal.init(deleteModalElement, {onCloseEnd: () => {deleteModal.destroy(); deleteModalElement.remove();}});
deleteModal.open();
break;
case 'view':
if (jobId !== '-1') {window.location.href = this.user.data.jobs[jobId].url;}
break;
default:
console.error(`Unknown action: "${action}"`);
break;
}
}
patch(patch) {
let id, match, re, valueName;
for (let operation of patch) {
switch(operation.op) {
case 'add':
// Matches the only paths that should be handled here: /jobs/{jobId}
re = /^\/jobs\/(\d+)$/;
if (re.test(operation.path)) {this.add(operation.value);}
break;
case 'remove':
// See case add ;)
re = /^\/jobs\/(\d+)$/;
if (re.test(operation.path)) {
[match, id] = operation.path.match(re);
this.remove(id);
}
break;
case 'replace':
// Matches the only paths that should be handled here: /jobs/{jobId}/{service || status || description || title}
re = /^\/jobs\/(\d+)\/(service|status|description|title)$/;
if (re.test(operation.path)) {
[match, id, valueName] = operation.path.match(re);
this.replace(id, valueName, operation.value);
}
break;
default:
break;
}
}
}
preprocessRessource(job) {
return {id: job.id,
service: job.service,
status: job.status,
description: job.description,
title: job.title};
}
}
JobList.options = {
item: `<tr>
<td><a class="btn-floating disabled"><i class="material-icons service"></i></a></td>
<td><b class="title"></b><br><i class="description"></i></td>
<td><span class="badge new status" data-badge-caption=""></span></td>
<td class="right-align">
<a class="action-button btn-floating red tooltipped waves-effect waves-light" data-action="delete" data-position="top" data-tooltip="Delete"><i class="material-icons">delete</i></a>
<a class="action-button btn-floating tooltipped waves-effect waves-light" data-action="view" data-position="top" data-tooltip="View"><i class="material-icons">send</i></a>
</td>
</tr>`,
valueNames: [{data: ['id']}, {name: 'service', attr: 'data-service'}, {name: 'status', attr: 'data-status'}, 'description', 'title']
};

72
web/app/static/js/nopaque/lists/JobResultList.js Normal file
View File

@ -0,0 +1,72 @@
class JobResultList extends RessourceList {
constructor(listElement, options = {}) {
super(listElement, {...JobResultList.options, ...options});
this.job = undefined;
this.user.eventListeners.job.addEventListener((eventType, payload) => this.eventHandler(eventType, payload), listElement.dataset.jobId);
}
init(job) {
this.job = job;
super.init(this.job.results);
}
onclick(event) {
let ressourceElement = event.target.closest('tr');
if (ressourceElement === null) {return;}
let jobResultId = ressourceElement.dataset.id;
let actionButtonElement = event.target.closest('.action-button');
if (actionButtonElement === null) {return;}
let action = actionButtonElement.dataset.action;
switch (action) {
case 'download':
window.location.href = this.job.results[jobResultId].download_url;
break;
default:
console.error(`Unknown action: "${action}"`);
break;
}
}
patch(patch) {
let re;
for (let operation of patch) {
switch(operation.op) {
case 'add':
// Matches the only paths that should be handled here: /jobs/{this.job.id}/results/{jobResultId}
re = new RegExp('^/jobs/' + this.job.id + '/results/(\\d+)$');
if (re.test(operation.path)) {this.add(operation.value);}
break;
default:
break;
}
}
}
preprocessRessource(jobResult) {
let description;
if (jobResult.filename.endsWith('.pdf.zip')) {
description = 'PDF files with text layer';
} else if (jobResult.filename.endsWith('.txt.zip')) {
description = 'Raw text files';
} else if (jobResult.filename.endsWith('.vrt.zip')) {
description = 'VRT compliant files including the NLP data';
} else if (jobResult.filename.endsWith('.xml.zip')) {
description = 'TEI compliant files';
} else if (jobResult.filename.endsWith('.poco.zip')) {
description = 'HOCR and image files for post correction (PoCo)';
} else {
description = 'All result files created during this job';
}
return {id: jobResult.id, description: description, filename: jobResult.filename};
}
}
JobResultList.options = {
item: `<tr>
<td><span class="description"></span></td>
<td><span class="filename"></span></td>
<td class="right-align">
<a class="action-button btn-floating tooltipped waves-effect waves-light" data-action="download" data-position="top" data-tooltip="View"><i class="material-icons">file_download</i></a>
</td>
</tr>`,
valueNames: [{data: ['id']}, 'description', 'filename']
};

95
web/app/static/js/nopaque/lists/QueryResultList.js Normal file
View File

@ -0,0 +1,95 @@
class QueryResultList extends RessourceList {
constructor(listElement, options = {}) {
super(listElement, {...QueryResultList.options, ...options});
this.queryResults = undefined;
this.user.eventListeners.queryResult.addEventListener((eventType, payload) => this.eventHandler(eventType, payload));
}
init(queryResults) {
this.queryResults = queryResults;
super.init(queryResults);
}
onclick(event) {
let ressourceElement = event.target.closest('tr');
if (ressourceElement === null) {return;}
let queryResultId = ressourceElement.dataset.id;
let actionButtonElement = event.target.closest('.action-button');
let action = actionButtonElement === null ? 'view' : actionButtonElement.dataset.action;
switch (action) {
case 'delete':
let deleteModalHTML = `<div class="modal">
<div class="modal-content">
<h4>Confirm query result deletion</h4>
<p>Do you really want to delete the query result <b>${this.user.data.query_results[queryResultId].title}</b>? It will be permanently deleted!</p>
</div>
<div class="modal-footer">
<a href="#!" class="btn modal-close waves-effect waves-light">Cancel</a>
<a class="btn modal-close red waves-effect waves-light" href="${this.user.data.query_results[queryResultId].url}/delete"><i class="material-icons left">delete</i>Delete</a>
</div>
</div>`;
let deleteModalParentElement = document.querySelector('main');
deleteModalParentElement.insertAdjacentHTML('beforeend', deleteModalHTML);
let deleteModalElement = deleteModalParentElement.lastChild;
let deleteModal = M.Modal.init(deleteModalElement, {onCloseEnd: () => {deleteModal.destroy(); deleteModalElement.remove();}});
deleteModal.open();
break;
case 'view':
if (queryResultId !== '-1') {window.location.href = this.user.data.query_results[queryResultId].url;}
break;
default:
console.error(`Unknown action: "${action}"`);
break;
}
}
patch(patch) {
let id, match, re, valueName;
for (let operation of patch) {
switch(operation.op) {
case 'add':
// Matches the only paths that should be handled here: /jobs/{jobId}
re = /^\/query_results\/(\d+)$/;
if (re.test(operation.path)) {this.add(operation.value);}
break;
case 'remove':
// See case add ;)
re = /^\/query_results\/(\d+)$/;
if (re.test(operation.path)) {
[match, id] = operation.path.match(re);
this.remove(id);
}
break;
case 'replace':
// Matches the only paths that should be handled here: /jobs/{jobId}/{service || status || description || title}
re = /^\/query_results\/(\d+)\/(corpus_title|description|query|title)$/;
if (re.test(operation.path)) {
[match, id, valueName] = operation.path.match(re);
this.replace(id, valueName, operation.value);
}
break;
default:
break;
}
}
}
preprocessRessource(queryResult) {
return {id: queryResult.id,
corpus_title: queryResult.corpus_title,
description: queryResult.description,
query: queryResult.query,
title: queryResult.title};
}
}
QueryResultList.options = {
item: `<tr>
<td><b class="title"></b><br><i class="description"></i><br></td>
<td><span class="corpus_title"></span><br><span class="query"></span></td>
<td class="right-align">
<a class="action-button btn-floating red tooltipped waves-effect waves-light" data-action="delete" data-position="top" data-tooltip="Delete"><i class="material-icons">delete</i></a>
<a class="action-button btn-floating tooltipped waves-effect waves-light" data-action="view" data-position="top" data-tooltip="View"><i class="material-icons">send</i></a>
</td>
</tr>`,
valueNames: [{data: ['id']}, 'corpus_title', 'description', 'query', 'title']
};

98
web/app/static/js/nopaque/lists/RessourceList.js Normal file
View File

@ -0,0 +1,98 @@
class RessourceList {
/* A wrapper class for the list.js list.
* This class is not meant to be used directly, instead it should be used as
* a base class for concrete ressource list implementations.
*/
constructor(listElement, options = {}) {
if (listElement.dataset.userId) {
if (listElement.dataset.userId in nopaque.appClient.users) {
this.user = nopaque.appClient.users[listElement.dataset.userId];
} else {
console.error(`User not found: ${listElement.dataset.userId}`);
return;
}
} else {
this.user = nopaque.appClient.users.self;
}
this.list = new List(listElement, {...RessourceList.options, ...options});
this.list.list.innerHTML = `<tr>
<td class="row" colspan="100%">
<div class="col s12">&nbsp;</div>
<div class="col s3 m2 xl1">
<div class="preloader-wrapper active">
<div class="spinner-layer spinner-green-only">
<div class="circle-clipper left">
<div class="circle"></div>
</div>
<div class="gap-patch">
<div class="circle"></div>
</div>
<div class="circle-clipper right">
<div class="circle"></div>
</div>
</div>
</div>
</div>
<div class="col s9 m6 xl5">
<span class="card-title">Waiting for data...</span>
<p>This list is not initialized yet.</p>
</div>
</td>
</tr>`;
if (typeof this.onclick === 'function') {this.list.list.addEventListener('click', event => this.onclick(event));}
}
eventHandler(eventType, payload) {
switch (eventType) {
case 'init':
this.init(payload);
break;
case 'patch':
this.patch(payload);
break;
default:
console.error(`Unknown event type: ${eventType}`);
break;
}
}
init(ressources) {
this.list.clear();
this.add(Object.values(ressources));
this.list.sort('id', {order: 'desc'});
let emptyListElementHTML = `<tr class="show-if-only-child" data-id="-1">
<td colspan="100%">
<span class="card-title"><i class="left material-icons" style="font-size: inherit;">file_download</i>Nothing here...</span>
<p>No ressource available.</p>
</td>
</tr>`;
this.list.list.insertAdjacentHTML('afterbegin', emptyListElementHTML);
}
patch(patch) {
/*
* It's not possible to generalize a patch Handler for all type of
* ressources. So this method is meant to be an interface.
*/
console.error('patch method not implemented!');
}
add(values) {
let ressources = Array.isArray(values) ? values : [values];
if (typeof this.preprocessRessource === 'function') {
ressources = ressources.map(ressource => this.preprocessRessource(ressource));
}
// Set a callback function ('() => {return;}') to force List.js perform the
// add method asynchronous: https://listjs.com/api/#add
this.list.add(ressources, () => {return;});
}
remove(id) {
this.list.remove('id', id);
}
replace(id, valueName, newValue) {
this.list.get('id', id)[0].values({[valueName]: newValue});
}
}
RessourceList.options = {page: 5, pagination: [{innerWindow: 4, outerWindow: 1}]};

71
web/app/static/js/nopaque/lists/Userlist.js Normal file
View File

@ -0,0 +1,71 @@
class UserList extends RessourceList {
constructor(listElement, options = {}) {
super(listElement, {...UserList.options, ...options});
users = undefined;
}
init(users) {
this.users = users;
super.init(users);
}
onclick(event) {
let ressourceElement = event.target.closest('tr');
if (ressourceElement === null) {return;}
let userId = ressourceElement.dataset.id;
let actionButtonElement = event.target.closest('.action-button');
let action = (actionButtonElement === null) ? 'view' : actionButtonElement.dataset.action;
switch (action) {
case 'delete':
let deleteModalHTML = `<div class="modal">
<div class="modal-content">
<h4>Confirm user deletion</h4>
<p>Do you really want to delete the corpus <b>${this.users[userId].username}</b>? All files will be permanently deleted!</p>
</div>
<div class="modal-footer">
<a href="#!" class="btn modal-close waves-effect waves-light">Cancel</a>
<a class="btn modal-close red waves-effect waves-light" href="/admin/users/${userId}/delete"><i class="material-icons left">delete</i>Delete</a>
</div>
</div>`;
let deleteModalParentElement = document.querySelector('main');
deleteModalParentElement.insertAdjacentHTML('beforeend', deleteModalHTML);
let deleteModalElement = deleteModalParentElement.lastChild;
let deleteModal = M.Modal.init(deleteModalElement, {onCloseEnd: () => {deleteModal.destroy(); deleteModalElement.remove();}});
deleteModal.open();
break;
case 'edit':
window.location.href = `/admin/users/${userId}/edit`;
break;
case 'view':
if (userId !== '-1') {window.location.href = `/admin/users/${userId}`;}
break;
default:
console.error(`Unknown action: ${action}`);
break;
}
}
preprocessRessource(user) {
return {id: user.id,
id_: user.id,
username: user.username,
email: user.email,
last_seen: new Date(user.last_seen * 1000).toLocaleString("en-US"),
role: user.role.name};
}
}
UserList.options = {
item: `<tr>
<td><span class="id_"></span></td>
<td><span class="username"></span></td>
<td><span class="email"></span></td>
<td><span class="last_seen"></span></td>
<td><span class="role"></span></td>
<td class="right-align">
<a class="action-button btn-floating red tooltipped waves-effect waves-light" data-action="delete" data-position="top" data-tooltip="Delete"><i class="material-icons">delete</i></a>
<a class="action-button btn-floating tooltipped waves-effect waves-light" data-action="edit" data-position="top" data-tooltip="Edit"><i class="material-icons">edit</i></a>
<a class="action-button btn-floating tooltipped waves-effect waves-light" data-action="view" data-position="top" data-tooltip="View"><i class="material-icons">send</i></a>
</td>
</tr>`,
valueNames: [{data: ['id']}, 'id_', 'username', 'email', 'last_seen', 'role']
};

235
web/app/static/js/nopaque/main.js Normal file
View File

@ -0,0 +1,235 @@
class AppClient {
constructor(currentUserId) {
this.socket = io({transports: ['websocket']});
this.users = {};
this.users.self = this.loadUser(currentUserId);
}
loadUser(userId) {
if (userId in this.users) {return this.users[userId];}
let user = new User();
this.users[userId] = user;
this.socket.on(`user_${userId}_init`, msg => user.init(JSON.parse(msg)));
this.socket.on(`user_${userId}_patch`, msg => user.patch(JSON.parse(msg)));
this.socket.emit('start_user_session', userId);
return user;
}
}
class User {
constructor() {
this.data = undefined;
this.eventListeners = {
corpus: {
addEventListener(listener, corpusId='*') {
if (corpusId in this) {this[corpusId].push(listener);} else {this[corpusId] = [listener];}
}
},
job: {
addEventListener(listener, jobId='*') {
if (jobId in this) {this[jobId].push(listener);} else {this[jobId] = [listener];}
}
},
queryResult: {
addEventListener(listener, queryResultId='*') {
if (queryResultId in this) {this[queryResultId].push(listener);} else {this[queryResultId] = [listener];}
}
}
};
}
init(data) {
this.data = data;
for (let [corpusId, eventListeners] of Object.entries(this.eventListeners.corpus)) {
if (corpusId === '*') {
for (let eventListener of eventListeners) {eventListener('init', this.data.corpora);}
} else {
if (corpusId in this.data.corpora) {
for (let eventListener of eventListeners) {eventListener('init', this.data.corpora[corpusId]);}
}
}
}
for (let [jobId, eventListeners] of Object.entries(this.eventListeners.job)) {
if (jobId === '*') {
for (let eventListener of eventListeners) {eventListener('init', this.data.jobs);}
} else {
if (jobId in this.data.jobs) {
for (let eventListener of eventListeners) {eventListener('init', this.data.jobs[jobId]);}
}
}
}
for (let [queryResultId, eventListeners] of Object.entries(this.eventListeners.queryResult)) {
if (queryResultId === '*') {
for (let eventListener of eventListeners) {eventListener('init', this.data.query_results);}
} else {
if (queryResultId in this.data.query_results) {
for (let eventListener of eventListeners) {eventListener('init', this.data.query_results[queryResultId]);}
}
}
}
}
patch(patch) {
this.data = jsonpatch.apply_patch(this.data, patch);
let corporaPatch = patch.filter(operation => operation.path.startsWith("/corpora"));
if (corporaPatch.length > 0) {
for (let [corpusId, eventListeners] of Object.entries(this.eventListeners.corpus)) {
if (corpusId === '*') {
for (let eventListener of eventListeners) {eventListener('patch', corporaPatch);}
} else {
let corpusPatch = corporaPatch.filter(operation => operation.path.startsWith(`/corpora/${corpusId}`));
if (corpusPatch.length > 0) {
for (let eventListener of eventListeners) {eventListener('patch', corpusPatch);}
}
}
}
}
let jobsPatch = patch.filter(operation => operation.path.startsWith("/jobs"));
if (jobsPatch.length > 0) {
for (let [jobId, eventListeners] of Object.entries(this.eventListeners.job)) {
if (jobId === '*') {
for (let eventListener of eventListeners) {eventListener('patch', jobsPatch);}
} else {
let jobPatch = jobsPatch.filter(operation => operation.path.startsWith(`/jobs/${jobId}`));
if (jobPatch.length > 0) {
for (let eventListener of eventListeners) {eventListener('patch', jobPatch);}
}
}
}
}
let queryResultsPatch = patch.filter(operation => operation.path.startsWith("/query_results"));
if (queryResultsPatch.length > 0) {
for (let [queryResultId, eventListeners] of Object.entries(this.eventListeners.queryResult)) {
if (queryResultId === '*') {
for (let eventListener of eventListeners) {eventListener('patch', queryResultsPatch);}
} else {
let queryResultPatch = queryResultsPatch.filter(operation => operation.path.startsWith(`/query_results/${queryResultId}`));
if (queryResultPatch.length > 0) {
for (let eventListener of eventListeners) {eventListener('patch', queryResultPatch);}
}
}
}
}
for (let operation of jobsPatch) {
if (operation.op !== 'replace') {continue;}
// Matches the only path that should be handled here: /jobs/{jobId}/status
if (/^\/jobs\/(\d+)\/status$/.test(operation.path)) {
let [match, jobId] = operation.path.match(/^\/jobs\/(\d+)\/status$/);
if (this.data.settings.job_status_site_notifications === "end" && !['complete', 'failed'].includes(operation.value)) {continue;}
nopaque.flash(`[<a href="/jobs/${jobId}">${this.data.jobs[jobId].title}</a>] New status: ${operation.value}`, 'job');
}
}
}
}
/*
* The nopaque object is used as a namespace for nopaque specific functions and
* variables.
*/
var nopaque = {};
nopaque.flash = function(message, category) {
let toast;
let toastActionElement;
switch (category) {
case "corpus":
message = `<i class="left material-icons">book</i>${message}`;
break;
case "error":
message = `<i class="left material-icons red-text">error</i>${message}`;
break;
case "job":
message = `<i class="left material-icons">work</i>${message}`;
break;
default:
message = `<i class="left material-icons">notifications</i>${message}`;
}
toast = M.toast({html: `<span>${message}</span>
<button data-action="close" class="btn-flat toast-action white-text">
<i class="material-icons">close</i>
</button>`});
toastActionElement = toast.el.querySelector('.toast-action[data-action="close"]');
toastActionElement.addEventListener('click', () => {toast.dismiss();});
};
nopaque.Forms = {};
nopaque.Forms.init = function() {
var abortRequestElement, parentElement, progressElement, progressModal,
progressModalElement, request, submitElement;
for (let form of document.querySelectorAll(".nopaque-submit-form")) {
submitElement = form.querySelector('button[type="submit"]');
submitElement.addEventListener("click", function() {
for (let selectElement of form.querySelectorAll('select')) {
if (selectElement.value === "") {
parentElement = selectElement.closest(".input-field");
parentElement.querySelector(".select-dropdown").classList.add("invalid");
for (let helperTextElement of parentElement.querySelectorAll(".helper-text")) {
helperTextElement.remove();
}
parentElement.insertAdjacentHTML("beforeend", `<span class="helper-text red-text">Please select an option.</span>`);
}
}
});
request = new XMLHttpRequest();
if (form.dataset.hasOwnProperty("progressModal")) {
progressModalElement = document.getElementById(form.dataset.progressModal);
progressModal = M.Modal.getInstance(progressModalElement);
progressModal.options.dismissible = false;
abortRequestElement = progressModalElement.querySelector(".abort-request");
abortRequestElement.addEventListener("click", function() {request.abort();});
progressElement = progressModalElement.querySelector(".determinate");
}
form.addEventListener("submit", function(event) {
event.preventDefault();
var formData;
formData = new FormData(form);
// Initialize progress modal
if (progressModalElement) {
progressElement.style.width = "0%";
progressModal.open();
}
request.open("POST", window.location.href);
request.send(formData);
});
request.addEventListener("load", function(event) {
var fieldElement;
if (request.status === 201) {
window.location.href = JSON.parse(this.responseText).redirect_url;
}
if (request.status === 400) {
for (let [field, errors] of Object.entries(JSON.parse(this.responseText))) {
fieldElement = form.querySelector(`input[name$="${field}"]`).closest(".input-field");
for (let error of errors) {
fieldElement.insertAdjacentHTML("beforeend", `<span class="helper-text red-text">${error}</span>`);
}
}
if (progressModalElement) {
progressModal.close();
}
}
if (request.status === 500) {
location.reload();
}
});
if (progressModalElement) {
request.upload.addEventListener("progress", function(event) {
progressElement.style.width = Math.floor(100 * event.loaded / event.total).toString() + "%";
});
}
}
}

7
web/app/static/js/socket.io.min.js vendored Normal file
View File

File diff suppressed because one or more lines are too long

1
web/app/static/js/socket.io.min.js.map Normal file
View File

File diff suppressed because one or more lines are too long

9
web/app/static/js/socket.io.slim.js
View File

File diff suppressed because one or more lines are too long

1
web/app/static/js/socket.io.slim.js.map
View File

File diff suppressed because one or more lines are too long

31
web/app/tasks/__init__.py Normal file
View File

@ -0,0 +1,31 @@
from .. import db
from ..models import Corpus, Job
import docker
docker_client = docker.from_env()
from . import corpus_utils, job_utils # noqa
def check_corpora():
corpora = Corpus.query.all()
for corpus in filter(lambda corpus: corpus.status == 'submitted', corpora):
corpus_utils.create_build_corpus_service(corpus)
for corpus in filter(lambda corpus: corpus.status in ['queued', 'running'], corpora): # noqa
corpus_utils.checkout_build_corpus_service(corpus)
for corpus in filter(lambda corpus: corpus.status == 'start analysis', corpora): # noqa
corpus_utils.create_cqpserver_container(corpus)
for corpus in filter(lambda corpus: corpus.status == 'stop analysis', corpora): # noqa
corpus_utils.remove_cqpserver_container(corpus)
db.session.commit()
def check_jobs():
jobs = Job.query.all()
for job in filter(lambda job: job.status == 'submitted', jobs):
job_utils.create_job_service(job)
for job in filter(lambda job: job.status in ['queued', 'running'], jobs):
job_utils.checkout_job_service(job)
for job in filter(lambda job: job.status == 'canceling', jobs):
job_utils.remove_job_service(job)
db.session.commit()

174
web/app/tasks/corpus_utils.py Normal file
View File

@ -0,0 +1,174 @@
from . import docker_client
import docker
import logging
import os
import shutil
def create_build_corpus_service(corpus):
corpus_data_dir = os.path.join(corpus.path, 'data')
shutil.rmtree(corpus_data_dir, ignore_errors=True)
os.mkdir(corpus_data_dir)
corpus_registry_dir = os.path.join(corpus.path, 'registry')
shutil.rmtree(corpus_registry_dir, ignore_errors=True)
os.mkdir(corpus_registry_dir)
corpus_file = os.path.join(corpus.path, 'merged', 'corpus.vrt')
service_kwargs = {
'command': 'docker-entrypoint.sh build-corpus',
'constraints': ['node.role==worker'],
'labels': {'origin': 'nopaque',
'type': 'corpus.build',
'corpus_id': str(corpus.id)},
'mounts': [corpus_file + ':/root/files/corpus.vrt:ro',
corpus_data_dir + ':/corpora/data:rw',
corpus_registry_dir + ':/usr/local/share/cwb/registry:rw'],
'name': 'build-corpus_{}'.format(corpus.id),
'restart_policy': docker.types.RestartPolicy()
}
service_image = \
'gitlab.ub.uni-bielefeld.de:4567/sfb1288inf/cqpserver:latest'
try:
docker_client.services.create(service_image, **service_kwargs)
except docker.errors.APIError as e:
logging.error(
'Create "{}" service raised '.format(service_kwargs['name'])
+ '"docker.errors.APIError" The server returned an error. '
+ 'Details: {}'.format(e)
)
else:
corpus.status = 'queued'
def checkout_build_corpus_service(corpus):
service_name = 'build-corpus_{}'.format(corpus.id)
try:
service = docker_client.services.get(service_name)
except docker.errors.NotFound:
logging.error(
'Get "{}" service raised '.format(service_name)
+ '"docker.errors.NotFound" The service does not exist. '
+ '(corpus.status: {} -> failed)'.format(corpus.status)
)
corpus.status = 'failed'
except docker.errors.APIError as e:
logging.error(
'Get "{}" service raised '.format(service_name)
+ '"docker.errors.APIError" The server returned an error. '
+ 'Details: {}'.format(e)
)
except docker.errors.InvalidVersion:
logging.error(
'Get "{}" service raised '.format(service_name)
+ '"docker.errors.InvalidVersion" One of the arguments is '
+ 'not supported with the current API version.'
)
else:
service_tasks = service.tasks()
if not service_tasks:
return
task_state = service_tasks[0].get('Status').get('State')
if corpus.status == 'queued' and task_state != 'pending':
corpus.status = 'running'
elif (corpus.status == 'running'
and task_state in ['complete', 'failed']):
try:
service.remove()
except docker.errors.APIError as e:
logging.error(
'Remove "{}" service raised '.format(service_name)
+ '"docker.errors.APIError" The server returned an error. '
+ 'Details: {}'.format(e)
)
return
else:
corpus.status = 'prepared' if task_state == 'complete' \
else 'failed'
def create_cqpserver_container(corpus):
corpus_data_dir = os.path.join(corpus.path, 'data')
corpus_registry_dir = os.path.join(corpus.path, 'registry')
container_kwargs = {
'command': 'cqpserver',
'detach': True,
'volumes': [corpus_data_dir + ':/corpora/data:rw',
corpus_registry_dir + ':/usr/local/share/cwb/registry:rw'],
'name': 'cqpserver_{}'.format(corpus.id),
'network': 'nopaque_default'
}
container_image = \
'gitlab.ub.uni-bielefeld.de:4567/sfb1288inf/cqpserver:latest'
# Check if a cqpserver container already exists. If this is the case,
# remove it and create a new one
try:
container = docker_client.containers.get(container_kwargs['name'])
except docker.errors.NotFound:
pass
except docker.errors.APIError as e:
logging.error(
'Get "{}" container raised '.format(container_kwargs['name'])
+ '"docker.errors.APIError" The server returned an error. '
+ 'Details: {}'.format(e)
)
return
else:
try:
container.remove(force=True)
except docker.errors.APIError as e:
logging.error(
'Remove "{}" container raised '.format(container_kwargs['name']) # noqa
+ '"docker.errors.APIError" The server returned an error. '
+ 'Details: {}'.format(e)
)
return
try:
docker_client.containers.run(container_image, **container_kwargs)
except docker.errors.ContainerError:
# This case should not occur, because detach is True.
logging.error(
'Run "{}" container raised '.format(container_kwargs['name'])
+ '"docker.errors.ContainerError" The container exits with a '
+ 'non-zero exit code and detach is False.'
)
corpus.status = 'failed'
except docker.errors.ImageNotFound:
logging.error(
'Run "{}" container raised '.format(container_kwargs['name'])
+ '"docker.errors.ImageNotFound" The specified image does not '
+ 'exist.'
)
corpus.status = 'failed'
except docker.errors.APIError as e:
logging.error(
'Run "{}" container raised '.format(container_kwargs['name'])
+ '"docker.errors.APIError" The server returned an error. '
+ 'Details: {}'.format(e)
)
else:
corpus.status = 'analysing'
def remove_cqpserver_container(corpus):
container_name = 'cqpserver_{}'.format(corpus.id)
try:
container = docker_client.containers.get(container_name)
except docker.errors.NotFound:
pass
except docker.errors.APIError as e:
logging.error(
'Get "{}" container raised '.format(container_name)
+ '"docker.errors.APIError" The server returned an error. '
+ 'Details: {}'.format(e)
)
return
else:
try:
container.remove(force=True)
except docker.errors.APIError as e:
logging.error(
'Remove "{}" container raised '.format(container_name)
+ '"docker.errors.APIError" The server returned an error. '
+ 'Details: {}'.format(e)
)
return
corpus.status = 'prepared'

152
web/app/tasks/job_utils.py Normal file
View File

@ -0,0 +1,152 @@
from datetime import datetime
from werkzeug.utils import secure_filename
from . import docker_client
from .. import db, mail
from ..email import create_message
from ..models import JobResult
import docker
import logging
import json
import os
def create_job_service(job):
cmd = '{} -i /files -o /files/output'.format(job.service)
if job.service == 'file-setup':
cmd += ' -f {}'.format(secure_filename(job.title))
cmd += ' --log-dir /files'
cmd += ' --zip [{}]_{}'.format(job.service, secure_filename(job.title))
cmd += ' ' + ' '.join(json.loads(job.service_args))
service_kwargs = {'command': cmd,
'constraints': ['node.role==worker'],
'labels': {'origin': 'nopaque',
'type': 'service.{}'.format(job.service),
'job_id': str(job.id)},
'mounts': [job.path + ':/files:rw'],
'name': 'job_{}'.format(job.id),
'resources': docker.types.Resources(
cpu_reservation=job.n_cores * (10 ** 9),
mem_reservation=job.mem_mb * (10 ** 6)
),
'restart_policy': docker.types.RestartPolicy()}
service_image = 'gitlab.ub.uni-bielefeld.de:4567/sfb1288inf/{}:{}'.format(
job.service, job.service_version)
try:
docker_client.services.create(service_image, **service_kwargs)
except docker.errors.APIError as e:
logging.error(
'Create "{}" service raised '.format(service_kwargs['name'])
+ '"docker.errors.APIError" The server returned an error. '
+ 'Details: {}'.format(e)
)
return
else:
job.status = 'queued'
finally:
send_notification(job)
def checkout_job_service(job):
service_name = 'job_{}'.format(job.id)
try:
service = docker_client.services.get(service_name)
except docker.errors.NotFound:
logging.error('Get "{}" service raised '.format(service_name)
+ '"docker.errors.NotFound" The service does not exist. '
+ '(job.status: {} -> failed)'.format(job.status))
job.status = 'failed'
except docker.errors.APIError as e:
logging.error(
'Get "{}" service raised '.format(service_name)
+ '"docker.errors.APIError" The server returned an error. '
+ 'Details: {}'.format(e)
)
return
except docker.errors.InvalidVersion:
logging.error(
'Get "{}" service raised '.format(service_name)
+ '"docker.errors.InvalidVersion" One of the arguments is '
+ 'not supported with the current API version.'
)
return
else:
service_tasks = service.tasks()
if not service_tasks:
return
task_state = service_tasks[0].get('Status').get('State')
if job.status == 'queued' and task_state != 'pending':
job.status = 'running'
elif job.status == 'running' and task_state in ['complete', 'failed']:
try:
service.remove()
except docker.errors.APIError as e:
logging.error(
'Remove "{}" service raised '.format(service_name)
+ '"docker.errors.APIError" The server returned an error. '
+ 'Details: {}'.format(e)
)
return
else:
if task_state == 'complete':
job_results_dir = os.path.join(job.path, 'output')
job_results = filter(lambda x: x.endswith('.zip'),
os.listdir(job_results_dir))
for job_result in job_results:
job_result = JobResult(filename=job_result, job=job)
db.session.add(job_result)
job.end_date = datetime.utcnow()
job.status = task_state
finally:
send_notification(job)
def remove_job_service(job):
service_name = 'job_{}'.format(job.id)
try:
service = docker_client.services.get(service_name)
except docker.errors.NotFound:
job.status = 'canceled'
except docker.errors.APIError as e:
logging.error(
'Get "{}" service raised '.format(service_name)
+ '"docker.errors.APIError" The server returned an error. '
+ 'Details: {}'.format(e)
)
return
except docker.errors.InvalidVersion:
logging.error(
'Get "{}" service raised '.format(service_name)
+ '"docker.errors.InvalidVersion" One of the arguments is '
+ 'not supported with the current API version.'
)
return
else:
try:
service.update(mounts=None)
except docker.errors.APIError as e:
logging.error(
'Update "{}" service raised '.format(service_name)
+ '"docker.errors.APIError" The server returned an error. '
+ 'Details: {}'.format(e)
)
return
try:
service.remove()
except docker.errors.APIError as e:
logging.error(
'Remove "{}" service raised '.format(service_name)
+ '"docker.errors.APIError" The server returned an error. '
+ 'Details: {}'.format(e)
)
def send_notification(job):
if job.creator.setting_job_status_mail_notifications == 'none':
return
if (job.creator.setting_job_status_mail_notifications == 'end'
and job.status not in ['complete', 'failed']):
return
msg = create_message(job.creator.email,
'Status update for your Job "{}"'.format(job.title),
'tasks/email/notification', job=job)
mail.send(msg)

19
web/app/templates/admin/_breadcrumbs.html.j2 Normal file
View File

@ -0,0 +1,19 @@
<ul class="tabs tabs-transparent">
<li class="tab"><a href="{{ url_for('main.index') }}" target="_self"><i class="material-icons">home</i></a></li>
<li class="tab disabled"><i class="material-icons">navigate_next</i></li>
<li class="tab"><a href="{{ url_for('.index') }}" target="_self">Administration</a></li>
<li class="tab disabled"><i class="material-icons">navigate_next</i></li>
{% if request.path == url_for('.users') %}
<li class="tab"><a class="active" href="{{ url_for('.users') }}" target="_self">Users</a></li>
{% elif request.path == url_for('.user', user_id=user.id) %}
<li class="tab"><a href="{{ url_for('.users') }}" target="_self">Users</a></li>
<li class="tab disabled"><i class="material-icons">navigate_next</i></li>
<li class="tab"><a class="active" href="{{ url_for('.user', user_id=user.id) }}" target="_self">{{ user.username }}</a></li>
{% elif request.path == url_for('.edit_user', user_id=user.id) %}
<li class="tab"><a href="{{ url_for('.users') }}" target="_self">Users</a></li>
<li class="tab disabled"><i class="material-icons">navigate_next</i></li>
<li class="tab"><a href="{{ url_for('.user', user_id=user.id) }}" target="_self">{{ user.username }}</a></li>
<li class="tab disabled"><i class="material-icons">navigate_next</i></li>
<li class="tab"><a class="active" href="{{ url_for('.edit_user', user_id=user.id) }}" target="_self">Edit</a></li>
{% endif %}
</ul>

4
web/app/templates/admin/edit_general_settings.html.j2 → web/app/templates/admin/edit_user.html.j2
View File

@ -1,6 +1,10 @@
{% extends "nopaque.html.j2" %} {% extends "nopaque.html.j2" %}
{% import 'materialize/wtf.html.j2' as wtf %} {% import 'materialize/wtf.html.j2' as wtf %}
{% block nav_content %}
{% include 'admin/_breadcrumbs.html.j2' %}
{% endblock nav_content %}
{% block page_content %} {% block page_content %}
<div class="container"> <div class="container">
<div class="row"> <div class="row">

33
web/app/templates/admin/user.html.j2
View File

@ -1,5 +1,9 @@
{% extends "nopaque.html.j2" %} {% extends "nopaque.html.j2" %}
{% block nav_content %}
{% include 'admin/_breadcrumbs.html.j2' %}
{% endblock nav_content %}
{% block page_content %} {% block page_content %}
<div class="container"> <div class="container">
<div class="row"> <div class="row">
@ -30,23 +34,22 @@
</ul> </ul>
</div> </div>
<div class="card-action right-align"> <div class="card-action right-align">
<a href="{{ url_for('.edit_general_settings', user_id=user.id) }}" class="waves-effect waves-light btn"><i class="material-icons left">edit</i>Edit</a> <a href="{{ url_for('.edit_user', user_id=user.id) }}" class="waves-effect waves-light btn"><i class="material-icons left">edit</i>Edit</a>
<a data-target="delete-user-modal" class="waves-effect waves-light btn red modal-trigger"><i class="material-icons left">delete</i>Delete</a> <a data-target="delete-user-modal" class="waves-effect waves-light btn red modal-trigger"><i class="material-icons left">delete</i>Delete</a>
</div> </div>
</div> </div>
</div> </div>
<div class="col s12 l6"> <div class="col s12 l6" id="corpora" data-user-id="{{ user.id }}">
<h3>Corpora</h3> <h3>Corpora</h3>
<div class="card"> <div class="card">
<div class="card-content" id="corpora"> <div class="card-content">
<div class="input-field"> <div class="input-field">
<i class="material-icons prefix">search</i> <i class="material-icons prefix">search</i>
<input id="search-corpus" class="search" type="search"></input> <input id="search-corpus" class="search" type="search"></input>
<label for="search-corpus">Search corpus</label> <label for="search-corpus">Search corpus</label>
</div> </div>
<ul class="pagination paginationTop"></ul> <table class="highlight ressource-list">
<table class="highlight">
<thead> <thead>
<tr> <tr>
<th></th> <th></th>
@ -60,22 +63,21 @@
</thead> </thead>
<tbody class="list"></tbody> <tbody class="list"></tbody>
</table> </table>
<ul class="pagination paginationBottom"></ul> <ul class="pagination"></ul>
</div> </div>
</div> </div>
</div> </div>
<div class="col s12 l6"> <div class="col s12 l6" id="jobs" data-user-id="{{ user.id }}">
<h3>Jobs</h3> <h3>Jobs</h3>
<div class="card"> <div class="card">
<div class="card-content" id="jobs"> <div class="card-content">
<div class="input-field"> <div class="input-field">
<i class="material-icons prefix">search</i> <i class="material-icons prefix">search</i>
<input id="search-job" class="search" type="search"></input> <input id="search-job" class="search" type="search"></input>
<label for="search-job">Search job</label> <label for="search-job">Search job</label>
</div> </div>
<ul class="pagination paginationTop"></ul> <table class="highlight ressource-list">
<table class="highlight">
<thead> <thead>
<tr> <tr>
<th><span class="sort" data-sort="service">Service</span></th> <th><span class="sort" data-sort="service">Service</span></th>
@ -89,7 +91,7 @@
</thead> </thead>
<tbody class="list"></tbody> <tbody class="list"></tbody>
</table> </table>
<ul class="pagination paginationBottom"></ul> <ul class="pagination"></ul>
</div> </div>
</div> </div>
</div> </div>
@ -111,10 +113,9 @@
{% block scripts %} {% block scripts %}
{{ super() }} {{ super() }}
<script type="module"> <script>
import {RessourceList} from '{{ url_for('static', filename='js/nopaque.lists.js') }}'; nopaque.appClient.loadUser({{ user.id }});
let corpusList = new RessourceList("corpora", nopaque.foreignCorporaSubscribers, "Corpus"); let corpusList = new CorpusList(document.querySelector('#corpora'));
let jobList = new RessourceList("jobs", nopaque.foreignJobsSubscribers, "Job"); let jobList = new JobList(document.querySelector('#jobs'));
nopaque.socket.emit("foreign_user_data_stream_init", {{ user.id }});
</script> </script>
{% endblock scripts %} {% endblock scripts %}

31
web/app/templates/admin/users.html.j2
View File

@ -1,5 +1,9 @@
{% extends "nopaque.html.j2" %} {% extends "nopaque.html.j2" %}
{% block nav_content %}
{% include 'admin/_breadcrumbs.html.j2' %}
{% endblock nav_content %}
{% block page_content %} {% block page_content %}
<div class="container"> <div class="container">
<div class="row"> <div class="row">
@ -7,30 +11,28 @@
<h1 id="title">{{ title }}</h1> <h1 id="title">{{ title }}</h1>
</div> </div>
<div class="col s12"> <div class="col s12" id="users">
<div class="card"> <div class="card">
<div class="card-content" id="users"> <div class="card-content">
<div class="input-field"> <div class="input-field">
<i class="material-icons prefix">search</i> <i class="material-icons prefix">search</i>
<input id="search-user" class="search" type="text"></input> <input id="search-user" class="search" type="text"></input>
<label for="search-user">Search user</label> <label for="search-user">Search user</label>
</div> </div>
<ul class="pagination paginationTop"></ul> <table class="highlight ressource-list">
<table class="highlight responsive-table">
<thead> <thead>
<tr> <tr>
<th class="sort" data-sort="id">Id</th>
<th class="sort" data-sort="username">Username</th> <th class="sort" data-sort="username">Username</th>
<th class="sort" data-sort="email">Email</th> <th class="sort" data-sort="email">Email</th>
<th class="sort" data-sort="role_id">Role</th> <th class="sort" data-sort="last_seen">Last seen</th>
<th class="sort" data-sort="confirmed">Confirmed Status</th> <th class="sort" data-sort="role">Role</th>
<th class="sort" data-sort="id">Id</th> <th></th>
<th>{# Actions #}</th>
</tr> </tr>
</thead> </thead>
<tbody class="list"> <tbody class="list"></tbody>
</tbody>
</table> </table>
<ul class="pagination paginationBottom"></ul> <ul class="pagination"></ul>
</div> </div>
</div> </div>
</div> </div>
@ -40,9 +42,8 @@
{% block scripts %} {% block scripts %}
{{ super() }} {{ super() }}
<script type="module"> <script>
import {RessourceList} from '{{ url_for('static', filename='js/nopaque.lists.js') }}'; let userList = new UserList(document.querySelector('#users'), {page: 10});
let userList = new RessourceList('users', null, "User", RessourceList.options.extended); userList.init({{ users|tojson }});
userList._add({{ users|tojson}});
</script> </script>
{% endblock scripts %} {% endblock scripts %}

10
web/app/templates/auth/login.html.j2
View File

@ -35,20 +35,20 @@
<div class="card medium"> <div class="card medium">
<form method="POST"> <form method="POST">
<div class="card-content"> <div class="card-content">
{{ login_form.hidden_tag() }} {{ form.hidden_tag() }}
{{ wtf.render_field(login_form.user, material_icon='person') }} {{ wtf.render_field(form.user, material_icon='person') }}
{{ wtf.render_field(login_form.password, material_icon='vpn_key') }} {{ wtf.render_field(form.password, material_icon='vpn_key') }}
<div class="row" style="margin-bottom: 0;"> <div class="row" style="margin-bottom: 0;">
<div class="col s6 left-align"> <div class="col s6 left-align">
<a href="{{ url_for('.reset_password_request') }}">Forgot your password?</a> <a href="{{ url_for('.reset_password_request') }}">Forgot your password?</a>
</div> </div>
<div class="col s6 right-align"> <div class="col s6 right-align">
{{ wtf.render_field(login_form.remember_me) }} {{ wtf.render_field(form.remember_me) }}
</div> </div>
</div> </div>
</div> </div>
<div class="card-action right-align"> <div class="card-action right-align">
{{ wtf.render_field(login_form.submit, material_icon='send') }} {{ wtf.render_field(form.submit, material_icon='send') }}
</div> </div>
</form> </form>
</div> </div>

12
web/app/templates/auth/register.html.j2
View File

@ -34,14 +34,14 @@
<div class="card medium"> <div class="card medium">
<form method="POST"> <form method="POST">
<div class="card-content"> <div class="card-content">
{{ registration_form.hidden_tag() }} {{ form.hidden_tag() }}
{{ wtf.render_field(registration_form.username, data_length='64', material_icon='person') }} {{ wtf.render_field(form.username, data_length='64', material_icon='person') }}
{{ wtf.render_field(registration_form.password, data_length='128', material_icon='vpn_key') }} {{ wtf.render_field(form.password, data_length='128', material_icon='vpn_key') }}
{{ wtf.render_field(registration_form.password_confirmation, data_length='128', material_icon='vpn_key') }} {{ wtf.render_field(form.password_confirmation, data_length='128', material_icon='vpn_key') }}
{{ wtf.render_field(registration_form.email, class_='validate', material_icon='email', type='email') }} {{ wtf.render_field(form.email, class_='validate', material_icon='email', type='email') }}
</div> </div>
<div class="card-action right-align"> <div class="card-action right-align">
{{ wtf.render_field(registration_form.submit, material_icon='send') }} {{ wtf.render_field(form.submit, material_icon='send') }}
</div> </div>
</form> </form>
</div> </div>

8
web/app/templates/auth/reset_password.html.j2
View File

@ -20,12 +20,12 @@
<div class="card"> <div class="card">
<form method="POST"> <form method="POST">
<div class="card-content"> <div class="card-content">
{{ reset_password_form.hidden_tag() }} {{ form.hidden_tag() }}
{{ wtf.render_field(reset_password_form.password, data_length='128') }} {{ wtf.render_field(form.password, data_length='128') }}
{{ wtf.render_field(reset_password_form.password_confirmation, data_length='128') }} {{ wtf.render_field(form.password_confirmation, data_length='128') }}
</div> </div>
<div class="card-action right-align"> <div class="card-action right-align">
{{ wtf.render_field(reset_password_form.submit, material_icon='send') }} {{ wtf.render_field(form.submit, material_icon='send') }}
</div> </div>
</form> </form>
</div> </div>

6
web/app/templates/auth/reset_password_request.html.j2
View File

@ -20,11 +20,11 @@
<div class="card"> <div class="card">
<form method="POST"> <form method="POST">
<div class="card-content"> <div class="card-content">
{{ reset_password_request_form.hidden_tag() }} {{ form.hidden_tag() }}
{{ wtf.render_field(reset_password_request_form.email, class_='validate', material_icon='email', type='email') }} {{ wtf.render_field(form.email, class_='validate', material_icon='email', type='email') }}
</div> </div>
<div class="card-action right-align"> <div class="card-action right-align">
{{ wtf.render_field(reset_password_request_form.submit, material_icon='send') }} {{ wtf.render_field(form.submit, material_icon='send') }}
</div> </div>
</form> </form>
</div> </div>

8
web/app/templates/corpora/add_corpus.html.j2
View File

@ -27,18 +27,18 @@
<div class="card"> <div class="card">
<form method="POST"> <form method="POST">
<div class="card-content"> <div class="card-content">
{{ add_corpus_form.hidden_tag() }} {{ form.hidden_tag() }}
<div class="row"> <div class="row">
<div class="col s12 m4"> <div class="col s12 m4">
{{ wtf.render_field(add_corpus_form.title, data_length='32', material_icon='title') }} {{ wtf.render_field(form.title, data_length='32', material_icon='title') }}
</div> </div>
<div class="col s12 m8"> <div class="col s12 m8">
{{ wtf.render_field(add_corpus_form.description, data_length='255', material_icon='description') }} {{ wtf.render_field(form.description, data_length='255', material_icon='description') }}
</div> </div>
</div> </div>
</div> </div>
<div class="card-action right-align"> <div class="card-action right-align">
{{ wtf.render_field(add_corpus_form.submit, material_icon='send') }} {{ wtf.render_field(form.submit, material_icon='send') }}
</div> </div>
</form> </form>
</div> </div>

14
web/app/templates/corpora/add_corpus_file.html.j2
View File

@ -27,24 +27,24 @@
<form class="nopaque-submit-form" data-progress-modal="progress-modal"> <form class="nopaque-submit-form" data-progress-modal="progress-modal">
<div class="card"> <div class="card">
<div class="card-content"> <div class="card-content">
{{ add_corpus_file_form.hidden_tag() }} {{ form.hidden_tag() }}
<div class="row"> <div class="row">
<div class="col s12 m4"> <div class="col s12 m4">
{{ wtf.render_field(add_corpus_file_form.author, data_length='255', material_icon='person') }} {{ wtf.render_field(form.author, data_length='255', material_icon='person') }}
</div> </div>
<div class="col s12 m4"> <div class="col s12 m4">
{{ wtf.render_field(add_corpus_file_form.title, data_length='255', material_icon='title') }} {{ wtf.render_field(form.title, data_length='255', material_icon='title') }}
</div> </div>
<div class="col s12 m4"> <div class="col s12 m4">
{{ wtf.render_field(add_corpus_file_form.publishing_year, material_icon='access_time') }} {{ wtf.render_field(form.publishing_year, material_icon='access_time') }}
</div> </div>
<div class="col s12"> <div class="col s12">
{{ wtf.render_field(add_corpus_file_form.file, accept='.vrt', placeholder='Choose your .vrt file') }} {{ wtf.render_field(form.file, accept='.vrt', placeholder='Choose your .vrt file') }}
</div> </div>
</div> </div>
</div> </div>
<div class="card-action right-align"> <div class="card-action right-align">
{{ wtf.render_field(add_corpus_file_form.submit, material_icon='send') }} {{ wtf.render_field(form.submit, material_icon='send') }}
</div> </div>
</div> </div>
<br> <br>
@ -52,7 +52,7 @@
<li> <li>
<div class="collapsible-header"><i class="material-icons">add</i>Add additional metadata</div> <div class="collapsible-header"><i class="material-icons">add</i>Add additional metadata</div>
<div class="collapsible-body"> <div class="collapsible-body">
{% for field in add_corpus_file_form {% for field in form
if field.short_name not in ['author', 'csrf_token', 'file', 'publishing_year', 'submit', 'title'] %} if field.short_name not in ['author', 'csrf_token', 'file', 'publishing_year', 'submit', 'title'] %}
{{ wtf.render_field(field, data_length='255', material_icon=field.label.text[0:1]) }} {{ wtf.render_field(field, data_length='255', material_icon=field.label.text[0:1]) }}
{% endfor %} {% endfor %}

4
web/app/templates/corpora/analyse_corpus.html.j2
View File

@ -155,9 +155,9 @@ import {
*/ */
document.addEventListener("DOMContentLoaded", () => { document.addEventListener("DOMContentLoaded", () => {
// Initialize the client for server client communication in dynamic mode // Initialize the client for server client communication in dynamic mode
let corpusId = {{ corpus_id }} let corpusId = {{ corpus.id }}
const client = new Client({'corpusId': corpusId, const client = new Client({'corpusId': corpusId,
'socket': nopaque.socket, 'socket': nopaque.appClient.socket,
'logging': true, 'logging': true,
'dynamicMode': true}); 'dynamicMode': true});
/** /**

253
web/app/templates/corpora/corpus.html.j2
View File

@ -2,107 +2,115 @@
{% from '_colors.html.j2' import colors %} {% from '_colors.html.j2' import colors %}
{% set scheme_primary_color = colors.corpus_analysis_darken %} {% set scheme_primary_color = colors.corpus_analysis_darken %}
{% set scheme_secondary_color = colors.corpus_analysis %} {% set scheme_secondary_color = colors.corpus_analysis_lighten %}
{% block main_attribs %} style="background-color: {{ scheme_secondary_color }};"{% endblock main_attribs %}
{% block nav_content %} {% block nav_content %}
{% include 'corpora/_breadcrumbs.html.j2' %} {% include 'corpora/_breadcrumbs.html.j2' %}
{% endblock nav_content %} {% endblock nav_content %}
{% block main_attribs %} class="corpus-analysis-color lighten"{% endblock main_attribs %}
{% block page_content %} {% block page_content %}
<div class="container"> <div class="container">
<div class="row"> <div class="row">
<div class="col s12"> <div class="col s12" data-corpus-id="{{ corpus.id }}" data-user-id="{{ corpus.creator.id }}" id="corpus-display">
<h1 id="title">{{ corpus.title }}</h1> <div class="row">
<p id="description">{{ corpus.description }}</p> <div class="col s8 m9 l10">
</div> <h1 id="title"><span class="corpus-title"></span></h1>
</div>
<div class="col s12 m4"> <div class="col s4 m3 l2 right-align">
<span class="chip status white-text hide" id="status"></span> <p>&nbsp;</p>
<div class="active preloader-wrapper small hide status-spinner" id="progress-indicator"> <p>&nbsp;</p>
<div class="spinner-layer spinner-blue-only"> <span class="chip status white-text"></span>
<div class="circle-clipper left"> <div class="active preloader-wrapper small status-spinner">
<div class="circle"></div> <div class="spinner-layer spinner-blue-only">
</div> <div class="circle-clipper left">
<div class="gap-patch"> <div class="circle"></div>
<div class="circle"></div> </div>
</div> <div class="gap-patch">
<div class="circle-clipper right"> <div class="circle"></div>
<div class="circle"></div> </div>
<div class="circle-clipper right">
<div class="circle"></div>
</div>
</div>
</div> </div>
</div> </div>
</div> </div>
</div>
<div class="col s12 m8">
<div class="card"> <div class="card">
<div class="card-content"> <div class="card-content" style="border-top: 10px solid {{ scheme_primary_color }}">
<span class="card-title">Chronometrics</span>
<div class="row"> <div class="row">
<div class="col s12 m6"> <div class="col s12">
<div class="input-field"> <div class="input-field">
<input disabled value="{{ corpus.creation_date.strftime('%d/%m/%Y, %H:%M:%S %p') }}" id="creation-date" type="text" class="validate"> <input class="corpus-description" disabled id="corpus-description" type="text">
<label for="creation-date">Creation date</label> <label for="corpus-description">Description</label>
</div> </div>
</div> </div>
<div class="col s12 m6"> <div class="col s12 m6">
<div class="input-field"> <div class="input-field">
<input disabled value="{{ corpus.last_edited_date.strftime('%d/%m/%Y, %H:%M:%S %p') }}" id="last_edited_date" type="text" class="validate"> <input class="corpus-creation-date validate" disabled id="corpus-creation-date" type="text">
<label for="creation-date">Last edited</label> <label for="corpus-creation-date">Creation date</label>
</div> </div>
</div> </div>
<div class="col s12 m6"> <div class="col s12 m6">
<div class="input-field"> <div class="input-field">
<input disabled value="{{ corpus.current_nr_of_tokens }} / {{ corpus.max_nr_of_tokens }}" id="nr_of_tokens" type="text" class="validate"> <input class="corpus-last-edited-date validate" disabled id="corpus-last-edited-date" type="text">
<label for="creation-date">Nr. of tokens used <label for="corpus-last-edited-date">Last edited</label>
<i class="material-icons tooltipped" data-position="bottom" data-tooltip="Current number of tokens in this corpus. Updates after every analyze session.">help</i> </div>
</label> </div>
<div class="col s12 m6">
<div class="input-field">
<input class="corpus-token-ratio validate" disabled id="corpus-token-ratio" type="text">
<label for="corpus-token-ratio">Nr. of tokens used <sup><i class="material-icons tooltipped tiny" data-position="bottom" data-tooltip="Current number of tokens in this corpus. Updates after every analyze session.">help</i></sup></label>
</div> </div>
</div> </div>
</div> </div>
</div> </div>
<div class="card-action right-align"> <div class="card-action right-align">
<a href="{{ url_for('corpora.analyse_corpus', corpus_id=corpus.id) }}" class="btn disabled hide waves-effect waves-light" id="analyze"><i class="material-icons left">search</i>Analyze</a> <a class="analyse-corpus-trigger btn disabled waves-effect waves-light" href="{{ url_for('corpora.analyse_corpus', corpus_id=corpus.id) }}"><i class="material-icons left">search</i>Analyze</a>
<a href="{{ url_for('corpora.prepare_corpus', corpus_id=corpus.id) }}" class="btn disabled hide waves-effect waves-light" id="build"><i class="material-icons left">build</i>Build</a> <a class="btn build-corpus-trigger disabled waves-effect waves-light" href="{{ url_for('corpora.prepare_corpus', corpus_id=corpus.id) }}"><i class="material-icons left">build</i>Build</a>
<a class="btn hide waves-effect waves-light download" id="corpus_create_zip"><i class="material-icons left">import_export</i>Export Corpus</a> <a class="btn disabled export-corpus-trigger waves-effect waves-light"><i class="material-icons left">import_export</i>Export</a>
<a data-target="delete-corpus-modal" class="btn modal-trigger red waves-effect waves-light"><i class="material-icons left">delete</i>Delete</a> <a class="btn modal-trigger red waves-effect waves-light" data-target="delete-corpus-modal"><i class="material-icons left">delete</i>Delete</a>
</div>
</div>
<div id="delete-corpus-modal" class="modal">
<div class="modal-content">
<h4>Confirm corpus deletion</h4>
<p>Do you really want to delete the corpus <span class="corpus-title"></span>? All files will be permanently deleted!</p>
</div>
<div class="modal-footer">
<a class="btn modal-close waves-effect waves-light" href="#!">Cancel</a>
<a class="btn modal-close red waves-effect waves-light" href="{{ url_for('corpora.delete_corpus', corpus_id=corpus.id) }}"><i class="material-icons left">delete</i>Delete</a>
</div> </div>
</div> </div>
</div> </div>
<div class="col s12"></div> <div class="col s12" id="corpus-files" data-corpus-id="{{ corpus.id }}" data-user-id="{{ corpus.creator.id }}">
<div class="col s12">
<div class="card"> <div class="card">
<div class="card-content" id="corpus-files" style="overflow: hidden;"> <div class="card-content">
<span class="card-title" id="files">Corpus files</span> <span class="card-title" id="files">Corpus files</span>
<div class="input-field"> <div class="input-field">
<i class="material-icons prefix">search</i> <i class="material-icons prefix">search</i>
<input id="search-results" class="search" type="search"></input> <input class="search" id="search-corpus-files" type="search"></input>
<label for="search-results">Search results</label> <label for="search-corpus-files">Search corpus files</label>
</div> </div>
<ul class="pagination paginationTop"></ul>
<table class="highlight responsive-table"> <table class="highlight responsive-table">
<thead> <thead>
<tr> <tr>
<th class="sort" data-sort="filename">Filename</th> <th class="sort" data-sort="filename">Filename</th>
<th class="sort" data-sort="author">Author</th> <th class="sort" data-sort="author">Author</th>
<th class="sort" data-sort="title">Title</th> <th class="sort" data-sort="title">Title</th>
<th>{# Actions #}</th> <th class="sort" data-sort="publishing-year">Publishing year</th>
<th></th>
</tr> </tr>
</thead> </thead>
<tbody class="list"> <tbody class="list"></tbody>
{% if corpus_files|length == 0 %}
<tr class="show-if-only-child">
<td colspan="5">
<span class="card-title"><i class="material-icons left">book</i>Nothing here...</span>
<p>Corpus is empty. Add texts using the option below.</p>
</td>
</tr>
{% endif %}
</table> </table>
<ul class="pagination paginationBottom"></ul> <ul class="pagination"></ul>
</div> </div>
<div class="card-action right-align"> <div class="card-action right-align">
<a href="{{ url_for('corpora.add_corpus_file', corpus_id=corpus.id) }}" class="btn waves-effect waves-light"><i class="material-icons left">add</i>Add corpus file</a> <a href="{{ url_for('corpora.add_corpus_file', corpus_id=corpus.id) }}" class="btn waves-effect waves-light"><i class="material-icons left">add</i>Add corpus file</a>
@ -111,140 +119,13 @@
</div> </div>
</div> </div>
</div> </div>
<!-- Modals -->
<div id="delete-corpus-modal" class="modal">
<div class="modal-content">
<h4>Confirm corpus deletion</h4>
<p>Do you really want to delete the corpus {{corpus.title}}? All files will be permanently deleted!</p>
</div>
<div class="modal-footer">
<a href="#!" class="btn modal-close waves-effect waves-light">Cancel</a>
<a href="{{ url_for('corpora.delete_corpus', corpus_id=corpus.id) }}" class="btn modal-close red waves-effect waves-light"><i class="material-icons left">delete</i>Delete</a>
</div>
</div>
{% endblock page_content %} {% endblock page_content %}
{% block scripts %} {% block scripts %}
{{ super() }} {{ super() }}
<script type="module"> <script>
import { nopaque.appClient.loadUser({{ corpus.creator.id }});
RessourceList let corpusDisplay = new CorpusDisplay(document.querySelector('#corpus-display'));
} from '../../static/js/nopaque.lists.js'; let corpusFileList = new CorpusFileList(document.querySelector('#corpus-files'));
class InformationUpdater {
constructor(corpusId, foreignCorpusFlag) {
this.corpusId = corpusId;
this.foreignCorpusFlag = foreignCorpusFlag;
if (this.foreignCorpusFlag) {
nopaque.foreignCorporaSubscribers.push(this);
} else {
nopaque.corporaSubscribers.push(this);
}
}
_init() {
let corpus;
corpus = (this.foreignCorpusFlag ? nopaque.foreignUser.corpora[this.corpusId]
: nopaque.user.corpora[this.corpusId]);
// Status
this.setStatus(corpus.status);
}
_update(patch) {
let pathArray;
for (let operation of patch) {
/* "/corpora/{corpusId}/valueName" -> ["{corpusId}", ...] */
pathArray = operation.path.split("/").slice(2);
if (pathArray[0] != this.corpusId) {continue;}
switch(operation.op) {
case "add":
location.reload();
break;
case "delete":
location.reload();
break;
case "replace":
if (pathArray[1] === "status") {
this.setStatus(operation.value);
}
break;
default:
break;
}
}
}
setStatus(status) {
let analyzeElement, buildElement, numFiles, progressIndicatorElement, statusElement;
numFiles = Object.keys((this.foreignCorpusFlag ? nopaque.foreignUser.corpora[this.corpusId] : nopaque.user.corpora[this.corpusId]).files).length;
progressIndicatorElement = document.getElementById("progress-indicator");
if (["queued", "running", "start analysis", "stop analysis"].includes(status)) {
progressIndicatorElement.classList.remove("hide");
} else {
progressIndicatorElement.classList.add("hide");
}
statusElement = document.getElementById("status");
statusElement.dataset.status = status;
statusElement.classList.remove("hide");
analyzeElement = document.getElementById("analyze");
if (["analysing", "prepared", "start analysis"].includes(status)) {
analyzeElement.classList.remove("disabled", "hide");
} else {
analyzeElement.classList.add("disabled", "hide");
}
buildElement = document.getElementById("build");
if (status === "unprepared" && numFiles > 0) {
buildElement.classList.remove("disabled", "hide");
} else {
buildElement.classList.add("disabled", "hide");
}
let downloadBtn = document.querySelector('#corpus_create_zip');
if (status === "prepared") {
downloadBtn.classList.toggle('hide', false);
} else {
downloadBtn.classList.toggle('hide', true);
}
}
}
{% if corpus.creator == current_user %}
var informationUpdater = new InformationUpdater({{ corpus.id }}, false);
{% else %}
var informationUpdater = new InformationUpdater({{ corpus.id }}, true);
nopaque.socket.emit("foreign_user_data_stream_init", {{ corpus.user_id }});
{% endif %}
let corpusFilesList = new RessourceList("corpus-files", null, "CorpusFile");
corpusFilesList._add({{ corpus_files|tojson|safe }});
// Events to handle full corpus download
let downloadBtn = document.querySelector('#corpus_create_zip');
downloadBtn.addEventListener('click', () => {
nopaque.flash('Compressing your corpus', 'corpus')
nopaque.socket.emit('corpus_create_zip', {{ corpus.id }});
downloadBtn.classList.toggle('disabled', true);
});
document.addEventListener('DOMContentLoaded', () => {
nopaque.socket.on('corpus_zip_created', () => {
nopaque.flash('Downloading your corpus', 'corpus');
downloadBtn.classList.toggle('disabled', false);
// Little trick to call the download view after ziping has finished
let fakeBtn = document.createElement('a');
fakeBtn.href = '{{ url_for('corpora.export_corpus',
corpus_id=corpus.id) }}';
fakeBtn.click();
});
});
</script> </script>
{% endblock scripts %} {% endblock scripts %}

12
web/app/templates/corpora/corpus_file.html.j2
View File

@ -20,23 +20,23 @@
<div class="col s12"> <div class="col s12">
<form method="POST"> <form method="POST">
{{ edit_corpus_file_form.hidden_tag() }} {{ form.hidden_tag() }}
<div class="card"> <div class="card">
<div class="card-content"> <div class="card-content">
<div class="row"> <div class="row">
<div class="col s12 m4"> <div class="col s12 m4">
{{ wtf.render_field(edit_corpus_file_form.author, data_length='255', material_icon='person') }} {{ wtf.render_field(form.author, data_length='255', material_icon='person') }}
</div> </div>
<div class="col s12 m4"> <div class="col s12 m4">
{{ wtf.render_field(edit_corpus_file_form.title, data_length='255', material_icon='title') }} {{ wtf.render_field(form.title, data_length='255', material_icon='title') }}
</div> </div>
<div class="col s12 m4"> <div class="col s12 m4">
{{ wtf.render_field(edit_corpus_file_form.publishing_year, material_icon='access_time') }} {{ wtf.render_field(form.publishing_year, material_icon='access_time') }}
</div> </div>
</div> </div>
</div> </div>
<div class="card-action right-align"> <div class="card-action right-align">
{{ wtf.render_field(edit_corpus_file_form.submit, material_icon='send') }} {{ wtf.render_field(form.submit, material_icon='send') }}
</div> </div>
</div> </div>
<br> <br>
@ -44,7 +44,7 @@
<li> <li>
<div class="collapsible-header"><i class="material-icons">edit</i>Edit additional metadata</div> <div class="collapsible-header"><i class="material-icons">edit</i>Edit additional metadata</div>
<div class="collapsible-body"> <div class="collapsible-body">
{% for field in edit_corpus_file_form {% for field in form
if field.short_name not in ['author', 'csrf_token', 'publishing_year', 'submit', 'title'] %} if field.short_name not in ['author', 'csrf_token', 'publishing_year', 'submit', 'title'] %}
{{ wtf.render_field(field, data_length='255', material_icon=field.label.text[0:1]) }} {{ wtf.render_field(field, data_length='255', material_icon=field.label.text[0:1]) }}
{% endfor %} {% endfor %}

12
web/app/templates/corpora/import_corpus.html.j2
View File

@ -1,4 +1,4 @@
{% extends "nopaque.html.j2" %} {% extends "nopaque.html.j2" %}
{% from '_colors.html.j2' import colors %} {% from '_colors.html.j2' import colors %}
{% import 'materialize/wtf.html.j2' as wtf %} {% import 'materialize/wtf.html.j2' as wtf %}
@ -27,23 +27,23 @@
<form class="nopaque-submit-form" data-progress-modal="progress-modal"> <form class="nopaque-submit-form" data-progress-modal="progress-modal">
<div class="card"> <div class="card">
<div class="card-content"> <div class="card-content">
{{ import_corpus_form.hidden_tag() }} {{ form.hidden_tag() }}
<div class="row"> <div class="row">
<div class="col s12 m4"> <div class="col s12 m4">
{{ wtf.render_field(import_corpus_form.title, data_length='32', material_icon='title') }} {{ wtf.render_field(form.title, data_length='32', material_icon='title') }}
</div> </div>
<div class="col s12 m8"> <div class="col s12 m8">
{{ wtf.render_field(import_corpus_form.description, data_length='255', material_icon='description') }} {{ wtf.render_field(form.description, data_length='255', material_icon='description') }}
</div> </div>
</div> </div>
<div class="row"> <div class="row">
<div class="col s12"> <div class="col s12">
{{ wtf.render_field(import_corpus_form.file, accept='.zip', placeholder='Choose your exported .zip file') }} {{ wtf.render_field(form.file, accept='.zip', placeholder='Choose your exported .zip file') }}
</div> </div>
</div> </div>
</div> </div>
<div class="card-action right-align"> <div class="card-action right-align">
{{ wtf.render_field(import_corpus_form.submit, material_icon='send') }} {{ wtf.render_field(form.submit, material_icon='send') }}
</div> </div>
</form> </form>
</div> </div>

10
web/app/templates/corpora/query_results/add_query_result.html.j2
View File

@ -27,21 +27,21 @@
<form class="nopaque-submit-form" data-progress-modal="progress-modal"> <form class="nopaque-submit-form" data-progress-modal="progress-modal">
<div class="card"> <div class="card">
<div class="card-content"> <div class="card-content">
{{ add_query_result_form.hidden_tag() }} {{ form.hidden_tag() }}
<div class="row"> <div class="row">
<div class="col s12 m4"> <div class="col s12 m4">
{{ wtf.render_field(add_query_result_form.title, data_length='32', material_icon='title') }} {{ wtf.render_field(form.title, data_length='32', material_icon='title') }}
</div> </div>
<div class="col s12 m8"> <div class="col s12 m8">
{{ wtf.render_field(add_query_result_form.description, data_length='255', material_icon='description') }} {{ wtf.render_field(form.description, data_length='255', material_icon='description') }}
</div> </div>
<div class="col s12"> <div class="col s12">
{{ wtf.render_field(add_query_result_form.file, accept='.json', placeholder='Choose your .json file') }} {{ wtf.render_field(form.file, accept='.json', placeholder='Choose your .json file') }}
</div> </div>
</div> </div>
</div> </div>
<div class="card-action right-align"> <div class="card-action right-align">
{{ wtf.render_field(add_query_result_form.submit, material_icon='send') }} {{ wtf.render_field(form.submit, material_icon='send') }}
</div> </div>
</div> </div>
</form> </form>

320
web/app/templates/jobs/job.html.j2
View File

@ -2,137 +2,155 @@
{% from '_colors.html.j2' import colors %} {% from '_colors.html.j2' import colors %}
{% if job.service == 'file-setup' %} {% if job.service == 'file-setup' %}
{% set border_color = colors.file_setup_darken %} {% set scheme_primary_color = colors.file_setup_darken %}
{% set main_class = 'file-setup-color lighten' %} {% set scheme_secondary_color = colors.file_setup_lighten %}
{% set scheme_color = colors.file_setup_darken %}
{% elif job.service == 'nlp' %} {% elif job.service == 'nlp' %}
{% set border_color = colors.nlp_darken %} {% set scheme_primary_color = colors.nlp_darken %}
{% set main_class = 'nlp-color lighten' %} {% set scheme_secondary_color = colors.nlp_lighten %}
{% set scheme_color = colors.nlp_darken %}
{% elif job.service == 'ocr' %} {% elif job.service == 'ocr' %}
{% set border_color = colors.ocr_darken %} {% set scheme_primary_color = colors.ocr_darken %}
{% set main_class = 'ocr-color lighten' %} {% set scheme_secondary_color = colors.ocr_lighten %}
{% set scheme_color = colors.ocr_darken %}
{% endif %} {% endif %}
{% block main_attribs %} style="background-color: {{ scheme_secondary_color }};"{% endblock main_attribs %}
{% block nav_content %} {% block nav_content %}
{% include 'jobs/_breadcrumbs.html.j2' %} {% include 'jobs/_breadcrumbs.html.j2' %}
{% endblock nav_content %} {% endblock nav_content %}
{% block main_attribs %} class="{{ main_class }}"{% endblock main_attribs %}
{% block page_content %} {% block page_content %}
<div class="container"> <div class="container">
<div class="row"> <div class="row">
<div class="col s12"> <div class="col s12" data-job-id="{{ job.id }}" data-user-id="{{ job.creator.id }}" id="job-display">
<h1>[{{ job.service }}] {{ job.title }}</h1> <div class="row">
</div> <div class="col s8 m9 l10">
<h1 id="title">[<span class="job-service"></span>] <span class="job-title"></span></h1>
</div>
<div class="col s4 m3 l2 right-align">
<p>&nbsp;</p>
<p>&nbsp;</p>
<span class="chip status white-text"></span>
<div class="active preloader-wrapper small status-spinner">
<div class="spinner-layer spinner-blue-only">
<div class="circle-clipper left">
<div class="circle"></div>
</div>
<div class="gap-patch">
<div class="circle"></div>
</div>
<div class="circle-clipper right">
<div class="circle"></div>
</div>
</div>
</div>
</div>
</div>
<div class="col s12"> <div class="card" style="border-top: 10px solid {{ scheme_primary_color }}">
<div class="card" style="border-top: 10px solid {{border_color}}">
<div class="card-content"> <div class="card-content">
<div class="row"> <div class="row">
<div class="col s8 m9 l10">
<span class="card-title title">{{ job.title }}</span>
</div>
<div class="col s4 m3 l2 right-align">
<span class="chip status white-text"></span>
<div class="active preloader-wrapper small status-spinner" id="progress-indicator">
<div class="spinner-layer spinner-blue-only">
<div class="circle-clipper left">
<div class="circle"></div>
</div>
<div class="gap-patch">
<div class="circle"></div>
</div>
<div class="circle-clipper right">
<div class="circle"></div>
</div>
</div>
</div>
</div>
<div class="col s12"> <div class="col s12">
<p class="description">{{ job.description }}</p>
</div>
<div class="col s12">&nbsp;</div>
<div class="col s12 m6">
<div class="input-field"> <div class="input-field">
<input disabled id="creation-date" type="text" value="{{ job.creation_date.strftime('%d/%m/%Y, %H:%M:%S %p') }}"> <input class="job-description" disabled id="job-description" type="text">
<label for="creation-date">Creation date</label> <label for="job-description">Description</label>
</div> </div>
</div> </div>
<div class="col s12 m6"> <div class="col s12 m6">
<div class="input-field"> <div class="input-field">
<input class="end-date" disabled id="end-date" type="text" value=""> <input class="job-creation-date" disabled id="job-creation-date" type="text">
<label for="end-date">End date</label> <label for="job-creation-date">Creation date</label>
</div>
</div>
<div class="col s12 m6">
<div class="input-field">
<input class="job-end-date" disabled id="job-end-date" type="text">
<label for="job-end-date">End date</label>
</div> </div>
</div> </div>
<div class="col s12 m4"> <div class="col s12 m4">
<div class="input-field"> <div class="input-field">
<input disabled id="service" type="text" value="{{ job.service }}"> <input class="job-service" disabled id="job-service" type="text">
<label for="service">Service</label> <label for="job-service">Service</label>
</div> </div>
</div> </div>
<div class="col s12 m4"> <div class="col s12 m4">
<div class="input-field"> <div class="input-field">
<input disabled id="service-args" type="text" value="{{ job.service_args|e }}"> <input class="job-service-args" disabled id="job-service-args" type="text">
<label for="service-args">Service arguments</label> <label for="job-service-args">Service arguments</label>
</div> </div>
</div> </div>
<div class="col s12 m4"> <div class="col s12 m4">
<div class="input-field"> <div class="input-field">
<input disabled id="service-version" type="text" value="{{ job.service_version }}"> <input class="job-service-version" disabled id="job-service-version" type="text">
<label for="service-version">Service version</label> <label for="job-service-version">Service version</label>
</div> </div>
</div> </div>
</div> </div>
</div> </div>
<div class="card-action right-align"> <div class="card-action right-align">
{% if current_user.is_administrator() and job.status == 'failed' %} {% if current_user.is_administrator() %}
<a href="{{ url_for('jobs.restart', job_id=job.id) }}" class="btn waves-effect waves-light"><i class="material-icons left">repeat</i>Restart</a> <a class="btn hide modal-trigger restart-job-trigger waves-effect waves-light" data-target="restart-job-modal"><i class="material-icons left">repeat</i>Restart</a>
{% endif %} {% endif %}
<!-- <a href="#" class="btn disabled waves-effect waves-light"><i class="material-icons left">settings</i>Export Parameters</a> --> <!-- <a href="#" class="btn disabled waves-effect waves-light"><i class="material-icons left">settings</i>Export Parameters</a> -->
<a data-target="delete-job-modal" class="waves-effect waves-light btn red modal-trigger"><i class="material-icons left">delete</i>Delete</a> <a class="btn modal-trigger red waves-effect waves-light" data-target="delete-job-modal"><i class="material-icons left">delete</i>Delete</a>
</div> </div>
</div> </div>
<div id="delete-job-modal" class="modal">
<div class="modal-content">
<h4>Confirm deletion</h4>
<p>Do you really want to delete the job <span class="job-title"></span>? All associated files will be permanently deleted.</p>
</div>
<div class="modal-footer">
<a href="#!" class="btn modal-close waves-effect waves-light">Cancel</a>
<a class="btn modal-close red waves-effect waves-light" href="{{ url_for('jobs.delete_job', job_id=job.id) }}"><i class="material-icons left">delete</i>Delete</a>
</div>
</div>
{% if current_user.is_administrator() %}
<div id="restart-job-modal" class="modal">
<div class="modal-content">
<h4>Confirm restart</h4>
<p>Do you really want to restart the job <span class="job-title"></span>? All log and result files will be permanently deleted.</p>
</div>
<div class="modal-footer">
<a href="#!" class="btn modal-close waves-effect waves-light">Cancel</a>
<a class="btn modal-close red waves-effect waves-light" href="{{ url_for('jobs.restart', job_id=job.id) }}"><i class="material-icons left">restart</i>Restart</a>
</div>
</div>
{% endif %}
</div> </div>
<div class="col s12"> <div class="col s12" id="job-inputs" data-job-id="{{ job.id }}" data-user-id="{{ job.creator.id }}">
<div class="card"> <div class="card">
<div class="card-content" id="inputs"> <div class="card-content">
<div class="row"> <div class="row">
<div class="col s12 m2"> <div class="col s12 m2">
<span class="card-title"><i class="left material-icons" style="font-size: inherit;">input</i>Inputs</span> <span class="card-title"><i class="left material-icons" style="font-size: inherit;">input</i>Inputs</span>
<p>Original input files.</p> <p>Original input files.</p>
</div> </div>
<div class="col s12 m10"> <div class="col s12 m10">
<ul class="pagination paginationTop"></ul>
<table class="highlight responsive-table"> <table class="highlight responsive-table">
<thead> <thead>
<tr> <tr>
<th class="sort" data-sort="filename">Filename</th> <th class="sort" data-sort="filename">Filename</th>
<th>{# Actions #}</th> <th></th>
</tr> </tr>
</thead> </thead>
<tbody class="list"> <tbody class="list"></tbody>
</tbody>
</table> </table>
<ul class="pagination paginationBottom"></ul> <ul class="pagination"></ul>
</div> </div>
</div> </div>
</div> </div>
</div> </div>
</div> </div>
<div class="col s12"> <div class="col s12" id="job-results" data-job-id="{{ job.id }}" data-user-id="{{ job.creator.id }}">
<div class="card"> <div class="card">
<div class="card-content"> <div class="card-content">
<div class="row"> <div class="row">
@ -144,24 +162,14 @@
<table class="highlight responsive-table"> <table class="highlight responsive-table">
<thead> <thead>
<tr> <tr>
<th>Result Type</th> <th>Description</th>
<th>Archive Name</th> <th>Filename</th>
<th>{# Actions #}</th> <th></th>
</tr> </tr>
</thead> </thead>
<tbody class="results"> <tbody class="list"></tbody>
<tr class="show-if-only-child">
<td colspan="3">
<span class="card-title">
<i class="left material-icons" style="font-size: inherit;">file_download</i>Nothing here...
</span>
<p>
No results available (yet). Is the job already completed?
</p>
</td>
</tr>
</tbody>
</table> </table>
<ul class="pagination"></ul>
</div> </div>
</div> </div>
</div> </div>
@ -169,158 +177,14 @@
</div> </div>
</div> </div>
</div> </div>
<!-- Modals -->
<div id="delete-job-modal" class="modal">
<div class="modal-content">
<h4>Confirm deletion</h4>
<p>Do you really want to delete the job {{ job.title }}? All associated files will be permanently deleted.</p>
</div>
<div class="modal-footer">
<a href="#!" class="btn modal-close waves-effect waves-light">Cancel</a>
<a class="btn modal-close red waves-effect waves-light" href="{{ url_for('jobs.delete_job', job_id=job.id) }}"><i class="material-icons left">delete</i>Delete</a>
</div>
</div>
{% endblock page_content %} {% endblock page_content %}
{% block scripts %} {% block scripts %}
{{ super() }} {{ super() }}
<script type="module"> <script>
import {RessourceList} from '../../static/js/nopaque.lists.js'; nopaque.appClient.loadUser({{ job.creator.id }});
class InformationUpdater { let jobDisplay = new JobDisplay(document.querySelector('#job-display'));
constructor(jobId, foreignJobFlag) { let jobInputList = new JobInputList(document.querySelector('#job-inputs'));
this.jobId = jobId; let jobResultList = new JobResultList(document.querySelector('#job-results'));
this.foreignJobFlag = foreignJobFlag;
if (this.foreignJobFlag) {
nopaque.foreignJobsSubscribers.push(this);
} else {
nopaque.jobsSubscribers.push(this);
}
}
_init() {
let job;
job = (this.foreignJobFlag ? nopaque.foreignUser.jobs[this.jobId]
: nopaque.user.jobs[this.jobId]);
// Results
this.addResults(job.results);
// End date
this.setEndDate(job.end_date);
// Status
this.setStatus(job.status);
}
_update(patch) {
let pathArray;
for (let operation of patch) {
/* "/jobs/{jobId}/..." -> ["{jobId}", ...] */
pathArray = operation.path.split("/").slice(2);
if (pathArray[0] != this.jobId) {continue;}
switch(operation.op) {
case "add":
if (pathArray[1] === "results") {
this.addResults([operation.value]);
}
break;
case "delete":
location.reload();
break;
case "replace":
if (pathArray[1] === "end_date") {
this.setEndDate(operation.value);
} else if (pathArray[1] === "status") {
this.setStatus(operation.value);
}
break;
default:
break;
}
}
}
addResults(results) {
let resultsArray, resultsElements, resultsHTML, resultType;
resultsArray = Object.values(results);
resultsArray.sort(function (a, b) {
if (a.filename < b.filename) {return -1;}
if (a.filename > b.filename) {return 1;}
return 0;
});
resultsHTML = ``;
for (let result of resultsArray) {
if (result.filename.endsWith(".pdf.zip")) {
resultType = "PDF file with text layer";
} else if (result.filename.endsWith(".txt.zip")) {
resultType = "Raw text files";
} else if (result.filename.endsWith(".vrt.zip")) {
resultType = "VRT(XML dialect) files holding the NLP data";
} else if (result.filename.endsWith(".xml.zip")) {
resultType = "XML files";
} else if (result.filename.endsWith(".poco.zip")) {
resultType = "HCOR und image files needed for Post correction(PoCo)";
} else {
resultType = "All result files created during this job";
}
resultsHTML += `
<tr>
<td>${resultType}</td>
<td>${result.filename}</td>
<td class="right-align">
<a class="btn-floating tooltipped waves-effect waves-light"
download href="/jobs/${result.job_id}/results/${result.id}/download"
data-position="top"
data-tooltip="Download">
<i class="material-icons">file_download</i>
</a>
</td>
</tr>
`;
};
resultsHTML += `
</tbody>
</table>
`;
resultsElements = document.querySelectorAll(".results");
for (let resultsElement of resultsElements) {
resultsElement.innerHTML += resultsHTML;
}
}
setEndDate(timestamp) {
let endDate;
if (timestamp === null) {
endDate = "N.a.";
} else {
endDate = new Date(timestamp * 1000).toLocaleString("en-US");
}
document.getElementById("end-date").value = endDate;
M.updateTextFields();
}
setStatus(status) {
let progressIndicator, statusElements;
if (status === "complete" || status === "failed") {
progressIndicator = document.getElementById("progress-indicator");
progressIndicator.classList.add("hide");
}
statusElements = document.querySelectorAll(".status");
for (let statusElement of statusElements) {
statusElement.dataset.status = status;
}
}
}
{% if job.creator == current_user %}
var informationUpdater = new InformationUpdater({{ job.id }}, false);
{% else %}
var informationUpdater = new InformationUpdater({{ job.id }}, true);
nopaque.socket.emit("foreign_user_data_stream_init", {{ job.user_id }});
{% endif %}
let jobInputsList = new RessourceList("inputs", null, "JobInput");
jobInputsList._add({{ job_inputs|tojson|safe }});
</script> </script>
{% endblock scripts %} {% endblock scripts %}

27
web/app/templates/main/dashboard.html.j2
View File

@ -29,8 +29,7 @@
<input id="search-corpus" class="search" type="search"></input> <input id="search-corpus" class="search" type="search"></input>
<label for="search-corpus">Search corpus</label> <label for="search-corpus">Search corpus</label>
</div> </div>
<ul class="pagination paginationTop"></ul> <table class="highlight ressource-list">
<table class="highlight">
<thead> <thead>
<tr> <tr>
<th></th> <th></th>
@ -44,7 +43,7 @@
</thead> </thead>
<tbody class="list"></tbody> <tbody class="list"></tbody>
</table> </table>
<ul class="pagination paginationBottom"></ul> <ul class="pagination"></ul>
</div> </div>
<div class="card-action right-align"> <div class="card-action right-align">
<a class="waves-effect waves-light btn" href="{{ url_for('corpora.import_corpus') }}"><i class="material-icons right">import_export</i>Import Corpus</a> <a class="waves-effect waves-light btn" href="{{ url_for('corpora.import_corpus') }}"><i class="material-icons right">import_export</i>Import Corpus</a>
@ -60,8 +59,7 @@
<input id="search-query-results" class="search" type="search"></input> <input id="search-query-results" class="search" type="search"></input>
<label for="search-query-results">Search query result</label> <label for="search-query-results">Search query result</label>
</div> </div>
<ul class="pagination paginationTop"></ul> <table class="highlight ressource-list">
<table class="highlight responsive-table">
<thead> <thead>
<tr> <tr>
<th> <th>
@ -72,7 +70,7 @@
<span class="sort" data-sort="corpus">Corpus</span> and<br> <span class="sort" data-sort="corpus">Corpus</span> and<br>
<span class="sort" data-sort="query">Query</span> <span class="sort" data-sort="query">Query</span>
</th> </th>
<th>{# Actions #}</th> <th></th>
</tr> </tr>
</thead> </thead>
<tbody class="list"> <tbody class="list">
@ -84,7 +82,7 @@
</tr> </tr>
</tbody> </tbody>
</table> </table>
<ul class="pagination paginationBottom"></ul> <ul class="pagination"></ul>
</div> </div>
<div class="card-action right-align"> <div class="card-action right-align">
<a class="waves-effect waves-light btn" href="{{ url_for('corpora.add_query_result') }}">Add query result<i class="material-icons right">file_upload</i></a> <a class="waves-effect waves-light btn" href="{{ url_for('corpora.add_query_result') }}">Add query result<i class="material-icons right">file_upload</i></a>
@ -104,8 +102,7 @@
<input id="search-job" class="search" type="search"></input> <input id="search-job" class="search" type="search"></input>
<label for="search-job">Search job</label> <label for="search-job">Search job</label>
</div> </div>
<ul class="pagination paginationTop"></ul> <table class="highlight ressource-list">
<table class="highlight">
<thead> <thead>
<tr> <tr>
<th><span class="sort" data-sort="service">Service</span></th> <th><span class="sort" data-sort="service">Service</span></th>
@ -119,12 +116,13 @@
</thead> </thead>
<tbody class="list"></tbody> <tbody class="list"></tbody>
</table> </table>
<ul class="pagination paginationBottom"></ul> <ul class="pagination"></ul>
</div> </div>
<div class="card-action right-align"> <div class="card-action right-align">
<p><a class="modal-trigger waves-effect waves-light btn" href="#" data-target="new-job-modal"><i class="material-icons left">add</i>New job</a></p> <p><a class="modal-trigger waves-effect waves-light btn" href="#" data-target="new-job-modal"><i class="material-icons left">add</i>New job</a></p>
</div> </div>
</div> </div>
<div id="new-job-modal" class="modal"> <div id="new-job-modal" class="modal">
<div class="modal-content"> <div class="modal-content">
<h4>Select a service</h4> <h4>Select a service</h4>
@ -178,10 +176,9 @@
{% block scripts %} {% block scripts %}
{{ super() }} {{ super() }}
<script type="module"> <script>
import {RessourceList} from '../../static/js/nopaque.lists.js'; let corpusList = new CorpusList(document.querySelector('#corpora'));
let corpusList = new RessourceList("corpora", nopaque.corporaSubscribers, "Corpus"); let jobList = new JobList(document.querySelector('#jobs'));
let jobList = new RessourceList("jobs", nopaque.jobsSubscribers, "Job"); let queryResultList = new QueryResultList(document.querySelector('#query-results'));
let queryResultList = new RessourceList("query-results", nopaque.queryResultsSubscribers, "QueryResult");
</script> </script>
{% endblock scripts %} {% endblock scripts %}

10
web/app/templates/main/index.html.j2
View File

@ -159,20 +159,20 @@
<form method="POST"> <form method="POST">
<div class="card-content"> <div class="card-content">
<span class="card-title">Log in</span> <span class="card-title">Log in</span>
{{ login_form.hidden_tag() }} {{ form.hidden_tag() }}
{{ wtf.render_field(login_form.user, material_icon='person') }} {{ wtf.render_field(form.user, material_icon='person') }}
{{ wtf.render_field(login_form.password, material_icon='vpn_key') }} {{ wtf.render_field(form.password, material_icon='vpn_key') }}
<div class="row" style="margin-bottom: 0;"> <div class="row" style="margin-bottom: 0;">
<div class="col s6 left-align"> <div class="col s6 left-align">
<a href="{{ url_for('auth.reset_password_request') }}">Forgot your password?</a> <a href="{{ url_for('auth.reset_password_request') }}">Forgot your password?</a>
</div> </div>
<div class="col s6 right-align"> <div class="col s6 right-align">
{{ wtf.render_field(login_form.remember_me) }} {{ wtf.render_field(form.remember_me) }}
</div> </div>
</div> </div>
</div> </div>
<div class="card-action right-align"> <div class="card-action right-align">
{{ wtf.render_field(login_form.submit, material_icon='send') }} {{ wtf.render_field(form.submit, material_icon='send') }}
</div> </div>
</form> </form>
</div> </div>

2
web/app/templates/modals/analysis_init.html.j2
View File

@ -5,7 +5,7 @@
<p>If the loading takes to long or an error occured, <p>If the loading takes to long or an error occured,
<a onclick="window.location.reload()" href="#">click here</a> <a onclick="window.location.reload()" href="#">click here</a>
to refresh your session or to refresh your session or
<a href="{{ url_for('corpora.corpus', corpus_id=corpus_id) }}">go back</a>! <a href="{{ url_for('corpora.corpus', corpus_id=corpus.id) }}">go back</a>!
</p> </p>
<div id="analysis-init-progress" class="progress"> <div id="analysis-init-progress" class="progress">
<div class="indeterminate"></div> <div class="indeterminate"></div>

45
web/app/templates/nopaque.html.j2
View File

@ -13,7 +13,7 @@
{{ super() }} {{ super() }}
{% endblock metas %} {% endblock metas %}
{% block title %}{{title}}{% endblock title %} {% block title %}{{ title }}{% endblock title %}
{% block styles %} {% block styles %}
{{ super() }} {{ super() }}
@ -150,7 +150,7 @@
{% if current_user.is_administrator() %} {% if current_user.is_administrator() %}
<li><div class="divider"></div></li> <li><div class="divider"></div></li>
<li><a class="subheader">Administration</a></li> <li><a class="subheader">Administration</a></li>
<li><a href="{{ url_for('admin.users') }}"><i class="material-icons">build</i>Administration tools</a></li> <li><a href="{{ url_for('admin.index') }}"><i class="material-icons">build</i>Administration</a></li>
{% endif %} {% endif %}
</ul> </ul>
{% endblock sidenav %} {% endblock sidenav %}
@ -231,9 +231,9 @@
</div> </div>
<div class="col s12 m9 right-align"> <div class="col s12 m9 right-align">
<a class="btn-small blue waves-effect waves-light" href="{{ url_for('main.about_and_faq') }}"><i class="left material-icons">info_outline</i>About and faq</a> <a class="btn-small blue waves-effect waves-light" href="{{ url_for('main.about_and_faq') }}"><i class="left material-icons">info_outline</i>About and faq</a>
{% if config.CONTACT_EMAIL_ADRESS %} {% if config.NOPAQUE_CONTACT %}
<a class="btn-small pink waves-effect waves-light" href="mailto:{{ config.CONTACT_EMAIL_ADRESS }}?subject=[nopaque] Contact"><i class="left material-icons">rate_review</i>Contact</a> <a class="btn-small pink waves-effect waves-light" href="mailto:{{ config.NOPAQUE_CONTACT }}?subject={{ config.NOPAQUE_MAIL_SUBJECT_PREFIX }} Contact"><i class="left material-icons">rate_review</i>Contact</a>
<a class="btn-small green waves-effect waves-light" href="mailto:{{ config.CONTACT_EMAIL_ADRESS }}?subject=[nopaque] Feedback"><i class="left material-icons">feedback</i>Feedback</a> <a class="btn-small green waves-effect waves-light" href="mailto:{{ config.NOPAQUE_CONTACT }}?subject={{ config.NOPAQUE_MAIL_SUBJECT_PREFIX }} Feedback"><i class="left material-icons">feedback</i>Feedback</a>
{% endif %} {% endif %}
<a class="btn-small orange waves-effect waves-light" href="https://gitlab.ub.uni-bielefeld.de/sfb1288inf/nopaque"><i class="left material-icons">code</i>GitLab</a> <a class="btn-small orange waves-effect waves-light" href="https://gitlab.ub.uni-bielefeld.de/sfb1288inf/nopaque"><i class="left material-icons">code</i>GitLab</a>
</div> </div>
@ -244,28 +244,39 @@
{% block scripts %} {% block scripts %}
{{ super() }} {{ super() }}
{% if current_user.setting_dark_mode %}
<script src="{{ url_for('static', filename='js/darkreader.js') }}"></script> <script src="{{ url_for('static', filename='js/darkreader.js') }}"></script>
<script>
DarkReader.enable({brightness: 150, contrast: 100, sepia: 0});
</script>
{% endif %}
<script src="{{ url_for('static', filename='js/jsonpatch.min.js') }}"></script> <script src="{{ url_for('static', filename='js/jsonpatch.min.js') }}"></script>
<script src="{{ url_for('static', filename='js/list.min.js') }}"></script> <script src="{{ url_for('static', filename='js/list.min.js') }}"></script>
<script src="{{ url_for('static', filename='js/socket.io.slim.js') }}"></script> <script src="{{ url_for('static', filename='js/socket.io.min.js') }}"></script>
<script src="{{ url_for('static', filename='js/nopaque.js') }}"></script> <script src="{{ url_for('static', filename='js/nopaque/main.js') }}"></script>
<script src="{{ url_for('static', filename='js/nopaque/displays/RessourceDisplay.js') }}"></script>
<script src="{{ url_for('static', filename='js/nopaque/displays/CorpusDisplay.js') }}"></script>
<script src="{{ url_for('static', filename='js/nopaque/displays/JobDisplay.js') }}"></script>
<script src="{{ url_for('static', filename='js/nopaque/lists/RessourceList.js') }}"></script>
<script src="{{ url_for('static', filename='js/nopaque/lists/CorpusList.js') }}"></script>
<script src="{{ url_for('static', filename='js/nopaque/lists/CorpusFileList.js') }}"></script>
<script src="{{ url_for('static', filename='js/nopaque/lists/JobList.js') }}"></script>
<script src="{{ url_for('static', filename='js/nopaque/lists/JobInputList.js') }}"></script>
<script src="{{ url_for('static', filename='js/nopaque/lists/JobResultList.js') }}"></script>
<script src="{{ url_for('static', filename='js/nopaque/lists/QueryResultList.js') }}"></script>
<script src="{{ url_for('static', filename='js/nopaque/lists/UserList.js') }}"></script>
<script> <script>
{% if current_user.setting_dark_mode %}
DarkReader.enable({brightness: 150, contrast: 100, sepia: 0});
{% endif %}
// Disable all option elements with no value // Disable all option elements with no value
for (let optionElement of document.querySelectorAll('option[value=""]')) { for (let optionElement of document.querySelectorAll('option[value=""]')) {optionElement.disabled = true;}
optionElement.disabled = true;
}
M.AutoInit(); M.AutoInit();
M.CharacterCounter.init(document.querySelectorAll('input[data-length][type="email"], input[data-length][type="password"], input[data-length][type="text"], textarea[data-length]')); M.CharacterCounter.init(document.querySelectorAll('input[data-length][type="email"], input[data-length][type="password"], input[data-length][type="text"], textarea[data-length]'));
M.Dropdown.init(document.querySelectorAll('#nav-more-dropdown-trigger'), {alignment: 'right', constrainWidth: false, coverTrigger: false}); M.Dropdown.init(document.querySelectorAll('#nav-more-dropdown-trigger'), {alignment: 'right', constrainWidth: false, coverTrigger: false});
nopaque.Forms.init(); nopaque.Forms.init();
for (let flashedMessage of {{ get_flashed_messages(with_categories=True)|tojson }}) {nopaque.flash(flashedMessage[1], flashedMessage[0]);}
</script>
<script>
{% if current_user.is_authenticated %} {% if current_user.is_authenticated %}
nopaque.socket.emit('user_data_stream_init'); nopaque.appClient = new AppClient({{ current_user.id }});
{% endif %} {% endif %}
for (let flashedMessage of {{ get_flashed_messages(with_categories=True)|tojson }}) {
nopaque.flash(flashedMessage[1], flashedMessage[0]);
}
</script> </script>
{% endblock scripts %} {% endblock scripts %}

36
web/app/templates/services/corpus_analysis.html.j2
View File

@ -29,17 +29,16 @@
<p>Nopaque lets you create and upload as many text corpora as you want. It makes use of CQP Query Language, which allows for complex search requests with the aid of metadata and NLP tags. The results can either be displayed as text or abstract visualizations.</p> <p>Nopaque lets you create and upload as many text corpora as you want. It makes use of CQP Query Language, which allows for complex search requests with the aid of metadata and NLP tags. The results can either be displayed as text or abstract visualizations.</p>
</div> </div>
<div class="col s12"> <div class="col s12" id="corpora">
<h2>My Corpora</h2> <h2>My Corpora</h2>
<div class="card"> <div class="card">
<div class="card-content" id="corpora"> <div class="card-content">
<div class="input-field"> <div class="input-field">
<i class="material-icons prefix">search</i> <i class="material-icons prefix">search</i>
<input id="search-corpus" class="search" type="search"></input> <input id="search-corpus" class="search" type="search"></input>
<label for="search-corpus">Search corpus</label> <label for="search-corpus">Search corpus</label>
</div> </div>
<ul class="pagination paginationTop"></ul> <table class="highlight ressource-list">
<table>
<thead> <thead>
<tr> <tr>
<th></th> <th></th>
@ -53,7 +52,7 @@
</thead> </thead>
<tbody class="list"></tbody> <tbody class="list"></tbody>
</table> </table>
<ul class="pagination paginationBottom"></ul> <ul class="pagination"></ul>
</div> </div>
<div class="card-action right-align"> <div class="card-action right-align">
<a class="waves-effect waves-light btn" href="{{ url_for('corpora.import_corpus') }}"><i class="material-icons right">import_export</i>Import Corpus</a> <a class="waves-effect waves-light btn" href="{{ url_for('corpora.import_corpus') }}"><i class="material-icons right">import_export</i>Import Corpus</a>
@ -62,17 +61,16 @@
</div> </div>
</div> </div>
<div class="col s12"> <div class="col s12" id="query-results">
<h2>My query results</h2> <h2>My query results</h2>
<div class="card"> <div class="card">
<div class="card-content" id="query-results"> <div class="card-content">
<div class="input-field"> <div class="input-field">
<i class="material-icons prefix">search</i> <i class="material-icons prefix">search</i>
<input id="search-query-results" class="search" type="search"></input> <input id="search-query-results" class="search" type="search"></input>
<label for="search-query-results">Search query result</label> <label for="search-query-results">Search query result</label>
</div> </div>
<ul class="pagination paginationTop"></ul> <table class="highlight ressource-list">
<table class="highlight responsive-table">
<thead> <thead>
<tr> <tr>
<th> <th>
@ -83,19 +81,12 @@
<span class="sort" data-sort="corpus">Corpus</span> and<br> <span class="sort" data-sort="corpus">Corpus</span> and<br>
<span class="sort" data-sort="query">Query</span> <span class="sort" data-sort="query">Query</span>
</th> </th>
<th>{# Actions #}</th> <th></th>
</tr> </tr>
</thead> </thead>
<tbody class="list"> <tbody class="list"></tbody>
<tr class="show-if-only-child">
<td colspan="5">
<span class="card-title"><i class="material-icons left">folder</i>Nothing here...</span>
<p>No query results yet imported.</p>
</td>
</tr>
</tbody>
</table> </table>
<ul class="pagination paginationBottom"></ul> <ul class="pagination"></ul>
</div> </div>
<div class="card-action right-align"> <div class="card-action right-align">
<a class="btn waves-effect waves-light" href="{{ url_for('corpora.add_query_result') }}">Add query result<i class="material-icons right">file_upload</i></a> <a class="btn waves-effect waves-light" href="{{ url_for('corpora.add_query_result') }}">Add query result<i class="material-icons right">file_upload</i></a>
@ -108,9 +99,8 @@
{% block scripts %} {% block scripts %}
{{ super() }} {{ super() }}
<script type="module"> <script>
import {RessourceList} from '../../static/js/nopaque.lists.js'; let corpusList = new CorpusList(document.querySelector('#corpora'));
let corpusList = new RessourceList("corpora", nopaque.corporaSubscribers, "Corpus"); let queryResultList = new QueryResultList(document.querySelector('#query-results'));
let queryResultList = new RessourceList("query-results", nopaque.queryResultsSubscribers, "QueryResult");
</script> </script>
{% endblock scripts %} {% endblock scripts %}

12
web/app/templates/services/file-setup.html.j2
View File

@ -48,24 +48,24 @@
<div class="card"> <div class="card">
<form class="nopaque-submit-form" data-progress-modal="progress-modal"> <form class="nopaque-submit-form" data-progress-modal="progress-modal">
<div class="card-content"> <div class="card-content">
{{ add_job_form.hidden_tag() }} {{ form.hidden_tag() }}
<div class="row"> <div class="row">
<div class="col s12 l4"> <div class="col s12 l4">
{{ wtf.render_field(add_job_form.title, data_length='32', material_icon='title') }} {{ wtf.render_field(form.title, data_length='32', material_icon='title') }}
</div> </div>
<div class="col s12 l8"> <div class="col s12 l8">
{{ wtf.render_field(add_job_form.description, data_length='255', material_icon='description') }} {{ wtf.render_field(form.description, data_length='255', material_icon='description') }}
</div> </div>
<div class="col s12"> <div class="col s12">
{{ wtf.render_field(add_job_form.files, accept='image/jpeg, image/png, image/tiff', placeholder='Choose your .jpeg, .png or .tiff files') }} {{ wtf.render_field(form.files, accept='image/jpeg, image/png, image/tiff', placeholder='Choose your .jpeg, .png or .tiff files') }}
</div> </div>
<div class="col s12 hide"> <div class="col s12 hide">
{{ wtf.render_field(add_job_form.version, material_icon='apps') }} {{ wtf.render_field(form.version, material_icon='apps') }}
</div> </div>
</div> </div>
</div> </div>
<div class="card-action right-align"> <div class="card-action right-align">
{{ wtf.render_field(add_job_form.submit, material_icon='send') }} {{ wtf.render_field(form.submit, material_icon='send') }}
</div> </div>
</form> </form>
</div> </div>

18
web/app/templates/services/nlp.html.j2
View File

@ -66,34 +66,34 @@
<div class="card"> <div class="card">
<form class="nopaque-submit-form" data-progress-modal="progress-modal"> <form class="nopaque-submit-form" data-progress-modal="progress-modal">
<div class="card-content"> <div class="card-content">
{{ add_job_form.hidden_tag() }} {{ form.hidden_tag() }}
<div class="row"> <div class="row">
<div class="col s12 l4"> <div class="col s12 l4">
{{ wtf.render_field(add_job_form.title, data_length='32', material_icon='title') }} {{ wtf.render_field(form.title, data_length='32', material_icon='title') }}
</div> </div>
<div class="col s12 l8"> <div class="col s12 l8">
{{ wtf.render_field(add_job_form.description, data_length='255', material_icon='description') }} {{ wtf.render_field(form.description, data_length='255', material_icon='description') }}
</div> </div>
<div class="col s12 l5"> <div class="col s12 l5">
{{ wtf.render_field(add_job_form.files, accept='text/plain', placeholder='Choose your .txt files') }} {{ wtf.render_field(form.files, accept='text/plain', placeholder='Choose your .txt files') }}
</div> </div>
<div class="col s12 l4"> <div class="col s12 l4">
{{ wtf.render_field(add_job_form.language, material_icon='language') }} {{ wtf.render_field(form.language, material_icon='language') }}
</div> </div>
<div class="col s12 l3"> <div class="col s12 l3">
{{ wtf.render_field(add_job_form.version, material_icon='apps') }} {{ wtf.render_field(form.version, material_icon='apps') }}
</div> </div>
<div class="col s12"> <div class="col s12">
<span class="card-title">Preprocessing</span> <span class="card-title">Preprocessing</span>
</div> </div>
<div class="col s9"> <div class="col s9">
<p>{{ add_job_form.check_encoding.label.text }}</p> <p>{{ form.check_encoding.label.text }}</p>
<p class="light">If the input files are not created with the nopaque OCR service or you do not know if your text files are UTF-8 encoded, check this switch. We will try to automatically determine the right encoding for your texts to process them.</p> <p class="light">If the input files are not created with the nopaque OCR service or you do not know if your text files are UTF-8 encoded, check this switch. We will try to automatically determine the right encoding for your texts to process them.</p>
</div> </div>
<div class="col s3 right-align"> <div class="col s3 right-align">
<div class="switch"> <div class="switch">
<label> <label>
{{ add_job_form.check_encoding() }} {{ form.check_encoding() }}
<span class="lever"></span> <span class="lever"></span>
</label> </label>
</div> </div>
@ -107,7 +107,7 @@
</div> </div>
</div> </div>
<div class="card-action right-align"> <div class="card-action right-align">
{{ wtf.render_field(add_job_form.submit, material_icon='send') }} {{ wtf.render_field(form.submit, material_icon='send') }}
</div> </div>
</form> </form>
</div> </div>

18
web/app/templates/services/ocr.html.j2
View File

@ -48,34 +48,34 @@
<div class="card"> <div class="card">
<form class="nopaque-submit-form" data-progress-modal="progress-modal"> <form class="nopaque-submit-form" data-progress-modal="progress-modal">
<div class="card-content"> <div class="card-content">
{{ add_job_form.hidden_tag() }} {{ form.hidden_tag() }}
<div class="row"> <div class="row">
<div class="col s12 l4"> <div class="col s12 l4">
{{ wtf.render_field(add_job_form.title, data_length='32', material_icon='title') }} {{ wtf.render_field(form.title, data_length='32', material_icon='title') }}
</div> </div>
<div class="col s12 l8"> <div class="col s12 l8">
{{ wtf.render_field(add_job_form.description, data_length='255', material_icon='description') }} {{ wtf.render_field(form.description, data_length='255', material_icon='description') }}
</div> </div>
<div class="col s12 l5"> <div class="col s12 l5">
{{ wtf.render_field(add_job_form.files, accept='application/pdf', color=ocr_color_darken, placeholder='Choose your .pdf files') }} {{ wtf.render_field(form.files, accept='application/pdf', color=ocr_color_darken, placeholder='Choose your .pdf files') }}
</div> </div>
<div class="col s12 l4"> <div class="col s12 l4">
{{ wtf.render_field(add_job_form.language, material_icon='language') }} {{ wtf.render_field(form.language, material_icon='language') }}
</div> </div>
<div class="col s12 l3"> <div class="col s12 l3">
{{ wtf.render_field(add_job_form.version, material_icon='apps') }} {{ wtf.render_field(form.version, material_icon='apps') }}
</div> </div>
<div class="col s12"> <div class="col s12">
<span class="card-title">Preprocessing</span> <span class="card-title">Preprocessing</span>
</div> </div>
<div class="col s9"> <div class="col s9">
<p>{{ add_job_form.binarization.label.text }}</p> <p>{{ form.binarization.label.text }}</p>
<p class="light">Based on a brightness threshold pixels are converted into either black or white. It is useful to reduce noise in images. (<b>longer duration</b>)</p> <p class="light">Based on a brightness threshold pixels are converted into either black or white. It is useful to reduce noise in images. (<b>longer duration</b>)</p>
</div> </div>
<div class="col s3 right-align"> <div class="col s3 right-align">
<div class="switch"> <div class="switch">
<label> <label>
{{ add_job_form.binarization() }} {{ form.binarization() }}
<span class="lever"></span> <span class="lever"></span>
</label> </label>
</div> </div>
@ -134,7 +134,7 @@
</div> </div>
</div> </div>
<div class="card-action right-align"> <div class="card-action right-align">
{{ wtf.render_field(add_job_form.submit, color=ocr_color_darken, material_icon='send') }} {{ wtf.render_field(form.submit, color=ocr_color_darken, material_icon='send') }}
</div> </div>
</form> </form>
</div> </div>

8
web/app/templates/tasks/email/notification.html.j2 Normal file
View File

@ -0,0 +1,8 @@
<p>Dear <b>{{ job.creator.username }}</b>,</p>
<p>The status of your Job "<b>{{ job.title }}</b>" has changed!</p>
<p>It is now <b>{{ job.status }}</b>!</p>
<p>You can access your Job here: <a href="{{ url_for('jobs.job', job_id=job.id) }}">{{ url_for('jobs.job', job_id=job.id) }}</a></p>
<p>Kind regards!<br>Your nopaque team</p>

9
web/app/templates/tasks/email/notification.txt.j2 Normal file
View File

@ -0,0 +1,9 @@
Dear {{ job.creator.username }},
The status of your Job "{{ job.title }}" has changed!
It is now {{ job.status }}!
You can access your Job here: {{ url_for('jobs.job', job_id=job.id) }}
Kind regards!
Your nopaque team

18
web/boot.sh
View File

@ -1,20 +1,26 @@
#!/bin/bash #!/bin/bash
source venv/bin/activate source venv/bin/activate
export FLASK_APP=nopaque.py
if [[ "$#" -eq 0 ]]; then if [[ "${NOPAQUE_DAEMON_ENABLED:-True}" == "True" ]]; then
echo "INFO Starting nopaque daemon process..."
./nopaque-daemon.sh &
fi
if [[ "${#}" -eq 0 ]]; then
while true; do while true; do
flask deploy flask deploy
if [[ "$?" == "0" ]]; then if [[ "${?}" == "0" ]]; then
break break
fi fi
echo Deploy command failed, retrying in 5 secs... echo "Deploy command failed, retrying in 5 secs..."
sleep 5 sleep 5
done done
python nopaque.py python nopaque.py
elif [[ "$1" == "flask" ]]; then elif [[ "${1}" == "flask" ]]; then
exec ${@:1} exec ${@:1}
else else
echo "$0 [COMMAND]" echo "${0} [COMMAND]"
echo "" echo ""
echo "nopaque startup script" echo "nopaque startup script"
echo "" echo ""

Some files were not shown because too many files have changed in this diff Show More