sporada/nopaque
1
0
Fork 0
mirror of https://gitlab.ub.uni-bielefeld.de/sfb1288inf/nopaque.git synced 2024-11-14 16:55:42 +00:00
Code Issues Projects Releases Wiki Activity

Merge branch 'daemon-rework'

Browse Source
This commit is contained in:
Patrick Jentsch 2021-01-14 15:24:20 +01:00
commit 8552cbd20c
105 changed files with 3410 additions and 2759 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

153
.env.tpl
View File

@ -9,122 +9,134 @@
# NOTE: Use `.` as <project-root-dir>
# HOST_MQ_DIR=
# Example: 999
# HINT: Use this bash command `getent group docker | cut -d: -f3`
HOST_DOCKER_GID=
# Example: 1000
# HINT: Use this bash command `id -u`
HOST_UID=
# Example: 1000
# HINT: Use this bash command `id -g`
HOST_GID=
# DEFAULT: ./nopaqued.log
# NOTES: Use `.` as <project-root-dir>,
# This file must be present on container startup
# HOST_NOPAQUE_DAEMON_LOG_FILE=
# Example: 999
# HINT: Use this bash command `getent group docker | cut -d: -f3`
HOST_DOCKER_GID=
# DEFAULT: ./nopaque.log
# NOTES: Use `.` as <project-root-dir>,
# This file must be present on container startup
# HOST_NOPAQUE_LOG_FILE=
# Example: 1000
# HINT: Use this bash command `id -u`
HOST_UID=
# HOST_LOG_FILE=
################################################################################
# Cookies #
# Flask #
# https://flask.palletsprojects.com/en/1.1.x/config/ #
################################################################################
# CHOOSE ONE: False, True
# DEFAULT: False
# HINT: Set to true if you redirect http to https
# NOPAQUE_REMEMBER_COOKIE_SECURE=
# CHOOSE ONE: http, https
# DEFAULT: http
# PREFERRED_URL_SCHEME=
# DEFAULT: hard to guess string
# HINT: Use this bash command `python -c "import uuid; print(uuid.uuid4().hex)"`
# SECRET_KEY=
# Example: nopaque.example.com/nopaque.example.com:5000
# HINT: If your instance is publicly available on a different Port then 80/443,
# you will have to add this to the server name
SERVER_NAME=
# CHOOSE ONE: False, True
# DEFAULT: False
# HINT: Set to true if you redirect http to https
# NOPAQUE_SESSION_COOKIE_SECURE=
# SESSION_COOKIE_SECURE=
################################################################################
# Database #
# DATABASE_URI blueprint: #
# - dialect[+driver]://username:password@host[:port]/database #
# - sqlite is not supported #
# - values in square brackets are optional #
# Flask-Login #
# https://flask-login.readthedocs.io/en/latest/ #
################################################################################
# DEFAULT: postgresql://nopaque:nopaque@db/nopaque
# NOPAQUE_DATABASE_URL=
# DEFAULT: postgresql://nopaque:nopaque@db/nopaque_dev
# NOPAQUE_DEV_DATABASE_URL=
# DEFAULT: postgresql://nopaque:nopaque@db/nopaque_test
# NOPAQUE_TEST_DATABASE_URL=
# CHOOSE ONE: False, True
# DEFAULT: False
# HINT: Set to true if you redirect http to https
# REMEMBER_COOKIE_SECURE=
################################################################################
# Email #
# Flask-Mail #
# https://pythonhosted.org/Flask-Mail/ #
################################################################################
# EXAMPLE: nopaque Admin <nopaque@example.com>
NOPAQUE_SMTP_DEFAULT_SENDER=
MAIL_DEFAULT_SENDER=
NOPAQUE_SMTP_PASSWORD=
MAIL_PASSWORD=
# EXAMPLE: smtp.example.com
NOPAQUE_SMTP_SERVER=
MAIL_SERVER=
# EXAMPLE: 587
NOPAQUE_SMTP_PORT=
MAIL_PORT=
# CHOOSE ONE: False, True
# DEFAULT: False
# NOPAQUE_SMTP_USE_SSL=
# MAIL_USE_SSL=
# CHOOSE ONE: False, True
# DEFAULT: False
# NOPAQUE_SMTP_USE_TLS=
# MAIL_USE_TLS=
# EXAMPLE: nopaque@example.com
NOPAQUE_SMTP_USERNAME=
MAIL_USERNAME=
################################################################################
# General #
# Flask-SQLAlchemy #
# https://flask-sqlalchemy.palletsprojects.com/en/2.x/config/ #
################################################################################
# DEFAULT with development config: postgresql://nopaque:nopaque@db/nopaque_dev
# DEFAULT with production config: postgresql://nopaque:nopaque@db/nopaque
# DEFAULT with testing config: postgresql://nopaque:nopaque@db/nopaque_test
# SQLALCHEMY_DATABASE_URI=
################################################################################
# nopaque #
################################################################################
# If an account is registered with this email adress gets automatically
# assigned the administrator role.
# EXAMPLE: admin.nopaque@example.com
NOPAQUE_ADMIN_EMAIL_ADRESS=
NOPAQUE_ADMIN=
# DEFAULT: development
# CHOOSE ONE: development, production, testing
# NOPAQUE_CONFIG=
# This email adress is used for the contact button in the nopaque footer. If
# not set, no contact button is displayed.
# DEFAULT: None
# EXAMPLE: contact.nopaque@example.com
# NOPAQUE_CONTACT_EMAIL_ADRESS=
# NOPAQUE_CONTACT=
# DEFAULT: /mnt/nopaque
# NOTE: This must be a network share and it must be available on all Docker Swarm nodes
# NOTE: This must be a network share and it must be available on all Docker
# Swarm nodes
# NOPAQUE_DATA_DIR=
# DEFAULT: localhost
# NOPAQUE_DOMAIN=
# CHOOSE ONE: False, True
# DEFAULT: True
# NOPAQUE_DAEMON_ENABLED=
# CHOOSE ONE: http, https
# DEFAULT: http
# NOPAQUE_PROTOCOL=
# The hostname or IP address for the server to listen on.
# DEFAULT: 0.0.0.0
# NOTES: To use a domain locally, add any names that should route to the app to your hosts file.
# If nopaque is running in a Docker container, you propably want to use the default value.
# NOPAQUE_HOST=
# DEFAULT: hard to guess string
# HINT: Use this bash command `python -c "import uuid; print(uuid.uuid4().hex)"`
# NOPAQUE_SECRET_KEY=
# The port number for the server to listen on.
# DEFAULT: 5000
# NOTE: If nopaque is running in a Docker container, you propably want to use the default value.
# NOPAQUE_PORT=
################################################################################
# Logging #
################################################################################
# DEFAULT: /home/nopaqued/nopaqued.log ~ /home/nopaqued/nopaqued.log
# NOTE: Use `.` as <nopaqued-root-dir>
# NOPAQUE_DAEMON_LOG_FILE=
# transport://[userid:password]@hostname[:port]/[virtual_host]
NOPAQUE_SOCKETIO_MESSAGE_QUEUE_URI=
# DEFAULT: %Y-%m-%d %H:%M:%S
# NOPAQUE_LOG_DATE_FORMAT=
@ -140,37 +152,22 @@ NOPAQUE_ADMIN_EMAIL_ADRESS=
# CHOOSE ONE: CRITICAL, ERROR, WARNING, INFO, DEBUG
# NOPAQUE_LOG_LEVEL=
################################################################################
# Message queue #
# MESSAGE_QUEUE_URI blueprint: #
# - transport://[userid:password]@hostname[:port]/[virtual_host] #
# - values in square brackets are optional #
################################################################################
# DEFAULT: None
# HINT: A message queue is not required when using a single server process
# NOPAQUE_SOCKETIO_MESSAGE_QUEUE_URI=
################################################################################
# Proxy fix #
################################################################################
# DEFAULT: 0
# Number of values to trust for X-Forwarded-For
# NOPAQUE_NUM_PROXIES_X_FOR=
# NOPAQUE_PROXY_FIX_X_FOR=
# DEFAULT: 0
# Number of values to trust for X-Forwarded-Host
# NOPAQUE_NUM_PROXIES_X_HOST=
# NOPAQUE_PROXY_FIX_X_HOST=
# DEFAULT: 0
# Number of values to trust for X-Forwarded-Port
# NOPAQUE_NUM_PROXIES_X_PORT=
# NOPAQUE_PROXY_FIX_X_PORT=
# DEFAULT: 0
# Number of values to trust for X-Forwarded-Prefix
# NOPAQUE_NUM_PROXIES_X_PREFIX=
# NOPAQUE_PROXY_FIX_X_PREFIX=
# DEFAULT: 0
# Number of values to trust for X-Forwarded-Proto
# NOPAQUE_NUM_PROXIES_X_PROTO=
# NOPAQUE_PROXY_FIX_X_PROTO=

8
README.md
View File

@ -56,6 +56,12 @@ username@hostname:~$ docker-compose build
``` bash
# Create log files
touch nopaque.log nopaqued.log
# For background execution add the -d flag and to scale the app, add --scale web=<NUM-INSTANCES>
# For background execution add the -d flag
username@hostname:~$ docker-compose up
# To scale your app use
username@hostname:~$ docker-compose -f docker-compose.yml \
-f docker-compose.override.yml
-f docker-compose.scale.yml
up
-d --no-recreate --scale nopaque=<NUM_INSTANCES>
```

6
daemon/.dockerignore
View File

@ -1,6 +0,0 @@
# Docker related files
Dockerfile
.dockerignore
# Packages
__pycache__

32
daemon/Dockerfile
View File

@ -1,32 +0,0 @@
FROM python:3.6.12-slim-buster
LABEL authors="Patrick Jentsch <p.jentsch@uni-bielefeld.de>, Stephan Porada <sporada@uni-bielefeld.de>"
ARG DOCKER_GID
ARG GID
ARG UID
ENV LANG=C.UTF-8
RUN apt-get update \
&& apt-get install --no-install-recommends --yes \
build-essential \
libpq-dev \
&& rm -r /var/lib/apt/lists/*
RUN groupadd --gid ${DOCKER_GID} --system docker \
&& groupadd --gid ${GID} --system nopaqued \
&& useradd --create-home --gid ${GID} --groups ${DOCKER_GID} --no-log-init --system --uid ${UID} nopaqued
USER nopaqued
WORKDIR /home/nopaqued
COPY --chown=nopaqued:nopaqued [".", "."]
RUN python -m venv venv \
&& venv/bin/pip install --requirement requirements.txt
ENTRYPOINT ["./boot.sh"]

31
daemon/app/__init__.py
View File

@ -1,31 +0,0 @@
from config import config
from sqlalchemy import create_engine
from sqlalchemy.orm import scoped_session, sessionmaker
from time import sleep
import docker
import os
configuration = config[os.environ.get('NOPAQUE_CONFIG', 'development')]
configuration.init()
docker_client = docker.from_env()
engine = create_engine(configuration.SQLALCHEMY_DATABASE_URI)
Session = scoped_session(sessionmaker(bind=engine))
def run():
from .tasks.check_corpora import check_corpora
check_corpora_thread = check_corpora()
from .tasks.check_jobs import check_jobs
check_jobs_thread = check_jobs()
from .tasks.notify import notify
notify_thread = notify()
while True:
if not check_corpora_thread.is_alive():
check_corpora_thread = check_corpora()
if not check_jobs_thread.is_alive():
check_jobs_thread = check_jobs()
if not notify_thread.is_alive():
notify_thread = notify()
sleep(3)

14
daemon/app/decorators.py
View File

@ -1,14 +0,0 @@
from functools import wraps
from threading import Thread
def background(f):
'''
' This decorator executes a function in a Thread.
'''
@wraps(f)
def wrapped(*args, **kwargs):
thread = Thread(target=f, args=args, kwargs=kwargs)
thread.start()
return thread
return wrapped

52
daemon/app/models.py
View File

@ -1,52 +0,0 @@
from sqlalchemy.ext.automap import automap_base
from sqlalchemy.orm import relationship
from . import engine
Base = automap_base()
# Classes for database models
class Corpus(Base):
__tablename__ = 'corpora'
files = relationship('CorpusFile', collection_class=set)
class CorpusFile(Base):
__tablename__ = 'corpus_files'
class Job(Base):
__tablename__ = 'jobs'
inputs = relationship('JobInput', collection_class=set)
results = relationship('JobResult', collection_class=set)
notification_data = relationship('NotificationData', collection_class=list)
notification_email_data = relationship('NotificationEmailData',
collection_class=list)
class JobInput(Base):
__tablename__ = 'job_results'
class JobResult(Base):
__tablename__ = 'job_results'
class NotificationData(Base):
__tablename__ = 'notification_data'
job = relationship('Job', collection_class=set)
class NotificationEmailData(Base):
__tablename__ = 'notification_email_data'
job = relationship('Job', collection_class=set)
class User(Base):
__tablename__ = 'users'
jobs = relationship('Job', collection_class=set)
corpora = relationship('Corpus', collection_class=set)
Base.prepare(engine, reflect=True)

0
daemon/app/tasks/__init__.py
View File

140
daemon/app/tasks/check_corpora.py
View File

@ -1,140 +0,0 @@
from .. import configuration as config
from .. import docker_client, Session
from ..decorators import background
from ..models import Corpus
import docker
import logging
import os
import shutil
@background
def check_corpora():
session = Session()
corpora = session.query(Corpus).all()
for corpus in filter(lambda corpus: corpus.status == 'submitted', corpora):
__create_build_corpus_service(corpus)
for corpus in filter(lambda corpus: (corpus.status == 'queued'
or corpus.status == 'running'),
corpora):
__checkout_build_corpus_service(corpus)
for corpus in filter(lambda corpus: corpus.status == 'start analysis',
corpora):
__create_cqpserver_container(corpus)
for corpus in filter(lambda corpus: corpus.status == 'stop analysis',
corpora):
__remove_cqpserver_container(corpus)
session.commit()
Session.remove()
def __create_build_corpus_service(corpus):
corpus_dir = os.path.join(config.DATA_DIR,
str(corpus.user_id),
'corpora',
str(corpus.id))
corpus_data_dir = os.path.join(corpus_dir, 'data')
corpus_file = os.path.join(corpus_dir, 'merged', 'corpus.vrt')
corpus_registry_dir = os.path.join(corpus_dir, 'registry')
if os.path.exists(corpus_data_dir):
shutil.rmtree(corpus_data_dir)
if os.path.exists(corpus_registry_dir):
shutil.rmtree(corpus_registry_dir)
os.mkdir(corpus_data_dir)
os.mkdir(corpus_registry_dir)
service_args = {'command': 'docker-entrypoint.sh build-corpus',
'constraints': ['node.role==worker'],
'labels': {'origin': 'nopaque',
'type': 'corpus.prepare',
'corpus_id': str(corpus.id)},
'mounts': [corpus_file + ':/root/files/corpus.vrt:ro',
corpus_data_dir + ':/corpora/data:rw',
corpus_registry_dir + ':/usr/local/share/cwb/registry:rw'],
'name': 'build-corpus_{}'.format(corpus.id),
'restart_policy': docker.types.RestartPolicy()}
service_image = \
'gitlab.ub.uni-bielefeld.de:4567/sfb1288inf/cqpserver:latest'
try:
service = docker_client.services.get(service_args['name'])
except docker.errors.NotFound:
pass
except docker.errors.DockerException:
return
else:
service.remove()
try:
docker_client.services.create(service_image, **service_args)
except docker.errors.DockerException:
corpus.status = 'failed'
else:
corpus.status = 'queued'
def __checkout_build_corpus_service(corpus):
service_name = 'build-corpus_{}'.format(corpus.id)
try:
service = docker_client.services.get(service_name)
except docker.errors.NotFound:
logging.error('__checkout_build_corpus_service({}):'.format(corpus.id)
+ ' The service does not exist.'
+ ' (stauts: {} -> failed)'.format(corpus.status))
corpus.status = 'failed'
return
except docker.errors.DockerException:
return
service_tasks = service.tasks()
if not service_tasks:
return
task_state = service_tasks[0].get('Status').get('State')
if corpus.status == 'queued' and task_state != 'pending':
corpus.status = 'running'
elif corpus.status == 'running' and task_state == 'complete':
service.remove()
corpus.status = 'prepared'
elif corpus.status == 'running' and task_state == 'failed':
service.remove()
corpus.status = task_state
def __create_cqpserver_container(corpus):
corpus_dir = os.path.join(config.DATA_DIR,
str(corpus.user_id),
'corpora',
str(corpus.id))
corpus_data_dir = os.path.join(corpus_dir, 'data')
corpus_registry_dir = os.path.join(corpus_dir, 'registry')
container_args = {'command': 'cqpserver',
'detach': True,
'volumes': [corpus_data_dir + ':/corpora/data:rw',
corpus_registry_dir + ':/usr/local/share/cwb/registry:rw'],
'name': 'cqpserver_{}'.format(corpus.id),
'network': 'nopaque_default'}
container_image = \
'gitlab.ub.uni-bielefeld.de:4567/sfb1288inf/cqpserver:latest'
try:
container = docker_client.containers.get(container_args['name'])
except docker.errors.NotFound:
pass
except docker.errors.DockerException:
return
else:
container.remove(force=True)
try:
docker_client.containers.run(container_image, **container_args)
except docker.errors.DockerException:
return
else:
corpus.status = 'analysing'
def __remove_cqpserver_container(corpus):
container_name = 'cqpserver_{}'.format(corpus.id)
try:
container = docker_client.containers.get(container_name)
except docker.errors.NotFound:
pass
except docker.errors.DockerException:
return
else:
container.remove(force=True)
corpus.status = 'prepared'

147
daemon/app/tasks/check_jobs.py
View File

@ -1,147 +0,0 @@
from datetime import datetime
from .. import configuration as config
from .. import docker_client, Session
from ..decorators import background
from ..models import Job, JobResult, NotificationData, NotificationEmailData
import docker
import logging
import json
import os
@background
def check_jobs():
session = Session()
jobs = session.query(Job).all()
for job in filter(lambda job: job.status == 'submitted', jobs):
__create_job_service(job)
for job in filter(lambda job: job.status == 'queued', jobs):
__checkout_job_service(job, session)
__add_notification_data(job, 'queued', session)
for job in filter(lambda job: job.status == 'running', jobs):
__checkout_job_service(job, session)
__add_notification_data(job, 'running', session)
for job in filter(lambda job: job.status == 'complete', jobs):
__add_notification_data(job, 'complete', session)
for job in filter(lambda job: job.status == 'failed', jobs):
__add_notification_data(job, 'failed', session)
for job in filter(lambda job: job.status == 'canceling', jobs):
__remove_job_service(job)
session.commit()
Session.remove()
def __add_notification_data(job, notified_on_status, session):
# checks if user wants any notifications at all
if (job.user.setting_job_status_mail_notifications == 'none'):
return
# checks if user wants only notification on completed jobs
elif (job.user.setting_job_status_mail_notifications == 'end'
and notified_on_status != 'complete'):
return
else:
# check if a job already has associated NotificationData
notification_exists = len(job.notification_data)
# create notification_data for current job if there is none
if (notification_exists == 0):
notification_data = NotificationData(job_id=job.id)
session.add(notification_data)
# If no commit job will have no NotificationData
session.commit()
if (job.notification_data[0].notified_on != notified_on_status):
notification_email_data = NotificationEmailData(job_id=job.id)
notification_email_data.notify_status = notified_on_status
notification_email_data.creation_date = datetime.utcnow()
job.notification_data[0].notified_on = notified_on_status
session.add(notification_email_data)
def __create_job_service(job):
job_dir = os.path.join(config.DATA_DIR,
str(job.user_id),
'jobs',
str(job.id))
cmd = '{} -i /files -o /files/output'.format(job.service)
if job.service == 'file-setup':
cmd += ' -f {}'.format(job.secure_filename)
cmd += ' --log-dir /files'
cmd += ' --zip [{}]_{}'.format(job.service, job.secure_filename)
cmd += ' ' + ' '.join(json.loads(job.service_args))
service_args = {'command': cmd,
'constraints': ['node.role==worker'],
'labels': {'origin': 'nopaque',
'type': 'service.{}'.format(job.service),
'job_id': str(job.id)},
'mounts': [job_dir + ':/files:rw'],
'name': 'job_{}'.format(job.id),
'resources': docker.types.Resources(
cpu_reservation=job.n_cores * (10 ** 9),
mem_reservation=job.mem_mb * (10 ** 6)),
'restart_policy': docker.types.RestartPolicy()}
service_image = ('gitlab.ub.uni-bielefeld.de:4567/sfb1288inf/'
+ job.service + ':' + job.service_version)
try:
service = docker_client.services.get(service_args['name'])
except docker.errors.NotFound:
pass
except docker.errors.DockerException:
return
else:
service.remove()
try:
docker_client.services.create(service_image, **service_args)
except docker.errors.DockerException:
job.status = 'failed'
else:
job.status = 'queued'
def __checkout_job_service(job, session):
service_name = 'job_{}'.format(job.id)
try:
service = docker_client.services.get(service_name)
except docker.errors.NotFound:
logging.error('__checkout_job_service({}): '.format(job.id)
+ 'The service does not exist. '
+ '(status: {} -> failed)'.format(job.status))
job.status = 'failed'
return
except docker.errors.DockerException:
return
service_tasks = service.tasks()
if not service_tasks:
return
task_state = service_tasks[0].get('Status').get('State')
if job.status == 'queued' and task_state != 'pending':
job.status = 'running'
elif (job.status == 'running'
and (task_state == 'complete' or task_state == 'failed')):
service.remove()
job.end_date = datetime.utcnow()
job.status = task_state
if task_state == 'complete':
results_dir = os.path.join(config.DATA_DIR,
str(job.user_id),
'jobs',
str(job.id),
'output')
results = filter(lambda x: x.endswith('.zip'),
os.listdir(results_dir))
for result in results:
job_result = JobResult(dir=results_dir,
filename=result,
job_id=job.id)
session.add(job_result)
def __remove_job_service(job):
service_name = 'job_{}'.format(job.id)
try:
service = docker_client.services.get(service_name)
except docker.errors.NotFound:
job.status = 'canceled'
except docker.errors.DockerException:
return
else:
service.update(mounts=None)
service.remove()

0
daemon/app/tasks/libnotify/__init__.py
View File

28
daemon/app/tasks/libnotify/notification.py
View File

@ -1,28 +0,0 @@
from email.message import EmailMessage
class Notification(EmailMessage):
"""docstring for Email."""
def set_notification_content(self,
subject_template,
subject_template_values_dict,
body_txt_template_path,
body_html_template_path,
body_template_values_dict):
# Create subject with subject_template_values_dict
self['subject'] = subject_template.format(
**subject_template_values_dict)
# Open template files and insert values from body_template_values_dict
with open(body_txt_template_path) as nfile:
self.body = nfile.read().format(**body_template_values_dict)
with open(body_html_template_path) as nfile:
self.html = nfile.read().format(**body_template_values_dict)
# Set txt of email
self.set_content(self.body)
# Set html alternative
self.add_alternative(self.html, subtype='html')
def set_addresses(self, sender, recipient):
self['From'] = sender
self['to'] = recipient

16
daemon/app/tasks/libnotify/service.py
View File

@ -1,16 +0,0 @@
class NotificationService:
"""This is a nopaque notifcation service object."""
def __init__(self, smtp):
# Bool to show if the mail server stoped sending mails due to exceeding
# its sending limit
self.mail_limit_exceeded = False
# Holds due to an error unsent email notifications
self.not_sent = {}
self.smtp = smtp
def send(self, email):
self.smtp.send_message(email)
def quit(self):
self.smtp.quit()

15
daemon/app/tasks/libnotify/templates/notification.html
View File

@ -1,15 +0,0 @@
<html>
<body>
<p>Dear <b>{username}</b>,</p>
<p>The status of your Job/Corpus({id}) with the title <b>"{title}"</b> has changed!</p>
<p>It is now <b>{status}</b>!</p>
<p>Time of this status update was: <b>{time} UTC</b></p>
<p>You can access your Job/Corpus here: <a href="{url}">{url}</a>
</p>
<p>Kind regards!<br>
Your nopaque team</p>
</body>
</html>

10
daemon/app/tasks/libnotify/templates/notification.txt
View File

@ -1,10 +0,0 @@
Dear {username},
The status of your Job/Corpus({id}) with the title "{title}" has changed!
It is now {status}!
Time of this status update was: {time} UTC
You can access your Job/Corpus here: {url}
Kind regards!
Your nopaque team

111
daemon/app/tasks/notify.py
View File

@ -1,111 +0,0 @@
from sqlalchemy import asc
from .libnotify.notification import Notification
from .libnotify.service import NotificationService
from .. import configuration as config
from .. import Session
from ..decorators import background
from ..models import NotificationEmailData
import logging
import os
import smtplib
ROOT_DIR = os.path.abspath(os.path.dirname(__file__))
@background
def notify():
session = Session()
if config.SMTP_USE_SSL:
smtp = smtplib.SMTP_SSL(host=config.SMTP_SERVER, port=config.SMTP_PORT)
else:
smtp = smtplib.SMTP(host=config.SMTP_SERVER, port=config.SMTP_PORT)
if config.SMTP_USE_TLS:
smtp.starttls()
try:
smtp.login(config.SMTP_USERNAME, config.SMTP_PASSWORD)
except smtplib.SMTPHeloError:
logging.warning('The server didn’t reply properly to the HELO '
'greeting.')
return
except smtplib.SMTPAuthenticationError as e:
logging.warning('The server didn’t accept the username/password '
'combination.')
logging.warning(e)
return
except smtplib.SMTPNotSupportedError:
logging.warning('The AUTH command is not supported by the server.')
return
except smtplib.SMTPException:
logging.warning('No suitable authentication method was found.')
return
notification_service = NotificationService(smtp)
# create notifications (content, recipient etc.)
notifications = __create_mail_notifications(notification_service, session)
# only login and send mails if there are any notifications
if (len(notifications) > 0):
# combine new and unsent notifications
notifications.update(notification_service.not_sent)
# send all notifications
__send_mail_notifications(notifications, notification_service)
# remove unsent notifications because they have been sent now
# but only if mail limit has not been exceeded
if (notification_service.mail_limit_exceeded is not True):
notification_service.not_sent = {}
smtp.quit()
Session.remove()
# Email notification functions
def __create_mail_notifications(notification_service, session):
notification_email_data = session.query(NotificationEmailData).order_by(asc(NotificationEmailData.creation_date)).all() # noqa
notifications = {}
for data in notification_email_data:
notification = Notification()
notification.set_addresses(config.SMTP_DEFAULT_SENDER,
data.job.user.email)
subject_template = ('[nopaque] Status update for your Job/Corpora: '
'{title}!')
subject_template_values_dict = {'title': data.job.title}
url = '{}://{}/{}/{}'.format(config.PROTOCOL,
config.DOMAIN,
'jobs',
data.job.id)
body_template_values_dict = {'username': data.job.user.username,
'id': data.job.id,
'title': data.job.title,
'status': data.notify_status,
'time': data.creation_date,
'url': url}
txt_tmplt = os.path.join(ROOT_DIR,
'libnotify/templates/notification.txt')
html_tmplt = os.path.join(ROOT_DIR,
'libnotify/templates/notification.html')
notification.set_notification_content(subject_template,
subject_template_values_dict,
txt_tmplt,
html_tmplt,
body_template_values_dict)
notifications[data.job.id] = notification
# Using a dictionary for notifications avoids sending multiple mails
# if the status of a job changes in a few seconds. The user will not
# get swamped with mails for queued, running and complete if those
# happen in in a few seconds. Only the last update will be sent.
# This depends on the sleep time interval though.
session.delete(data)
session.commit()
return notifications
def __send_mail_notifications(notifications, notification_service):
for key, notification in notifications.items():
try:
notification_service.send(notification)
notification_service.mail_limit_exceeded = False
except Exception:
# Adds notifications to unsent if mail server exceded limit for
# consecutive mail sending
logging.warning('limit')
notification_service.not_sent[key] = notification
notification_service.mail_limit_exceeded = True
notification_service.not_sent.update(notifications)

3
daemon/boot.sh
View File

@ -1,3 +0,0 @@
#!/bin/bash
source venv/bin/activate
python nopaqued.py

71
daemon/config.py
View File

@ -1,71 +0,0 @@
import logging
import os
ROOT_DIR = os.path.abspath(os.path.dirname(__file__))
class Config:
''' # Email # '''
SMTP_DEFAULT_SENDER = os.environ.get('NOPAQUE_SMTP_DEFAULT_SENDER')
SMTP_PASSWORD = os.environ.get('NOPAQUE_SMTP_PASSWORD')
SMTP_PORT = int(os.environ.get('NOPAQUE_SMTP_PORT'))
SMTP_SERVER = os.environ.get('NOPAQUE_SMTP_SERVER')
SMTP_USERNAME = os.environ.get('NOPAQUE_SMTP_USERNAME')
SMTP_USE_SSL = os.environ.get(
'NOPAQUE_SMTP_USE_SSL', 'false').lower() == 'true'
SMTP_USE_TLS = os.environ.get(
'NOPAQUE_SMTP_USE_TLS', 'false').lower() == 'true'
''' # General # '''
DATA_DIR = os.environ.get('NOPAQUE_DATA_DIR', '/mnt/nopaque')
DOMAIN = os.environ.get('NOPAQUE_DOMAIN', 'localhost')
PROTOCOL = os.environ.get('NOPAQUE_PROTOCOL', 'http')
SECRET_KEY = os.environ.get('NOPAQUE_SECRET_KEY', 'hard to guess string')
''' # Logging # '''
LOG_DATE_FORMAT = os.environ.get('NOPAQUE_LOG_DATE_FORMAT',
'%Y-%m-%d %H:%M:%S')
LOG_FILE = os.environ.get('NOPAQUED_LOG_FILE',
os.path.join(ROOT_DIR, 'nopaqued.log'))
LOG_FORMAT = os.environ.get(
'NOPAQUE_LOG_FORMAT',
'[%(asctime)s] %(levelname)s in '
'%(pathname)s (function: %(funcName)s, line: %(lineno)d): %(message)s'
)
LOG_LEVEL = os.environ.get('NOPAQUE_LOG_LEVEL', 'WARNING')
@classmethod
def init(cls):
# Set up logging according to the corresponding (LOG_*) variables
logging.basicConfig(datefmt=cls.LOG_DATE_FORMAT,
filename=cls.LOG_FILE,
format=cls.LOG_FORMAT,
level=cls.LOG_LEVEL)
class DevelopmentConfig(Config):
''' # Database # '''
SQLALCHEMY_DATABASE_URI = os.environ.get(
'NOPAQUE_DEV_DATABASE_URL',
'sqlite:///' + os.path.join(ROOT_DIR, 'data-dev.sqlite')
)
class ProductionConfig(Config):
''' # Database # '''
SQLALCHEMY_DATABASE_URI = os.environ.get(
'NOPAQUE_DATABASE_URL',
'sqlite:///' + os.path.join(ROOT_DIR, 'data.sqlite')
)
class TestingConfig(Config):
''' # Database # '''
SQLALCHEMY_DATABASE_URI = os.environ.get(
'NOPAQUE_TEST_DATABASE_URL', 'sqlite://')
config = {'development': DevelopmentConfig,
'production': ProductionConfig,
'testing': TestingConfig}

13
daemon/nopaqued.py
View File

@ -1,13 +0,0 @@
from dotenv import load_dotenv
from app import run
import os
# Load environment variables
DOTENV_FILE = os.path.join(os.path.dirname(__file__), '.env')
if os.path.exists(DOTENV_FILE):
load_dotenv(DOTENV_FILE)
if __name__ == '__main__':
run()

4
daemon/requirements.txt
View File

@ -1,4 +0,0 @@
docker
psycopg2
python-dotenv
SQLAlchemy

8
docker-compose.development.yml
View File

@ -13,11 +13,3 @@ services:
- "./web/nopaque.py:/home/nopaque/nopaque.py"
- "./web/requirements.txt:/home/nopaque/requirements.txt"
- "./web/tests:/home/nopaque/tests"
nopaqued:
volumes:
# Mount code as volumes
- "./daemon/app:/home/nopaqued/app"
- "./daemon/boot.sh:/home/nopaqued/boot.sh"
- "./daemon/config.py:/home/nopaqued/config.py"
- "./daemon/nopaqued.py:/home/nopaqued/nopaqued.py"
- "./daemon/requirements.txt:/home/nopaqued/requirements.txt"

6
docker-compose.scale.yml Normal file
View File

@ -0,0 +1,6 @@
version: "3.5"
services:
nopaque:
environment:
- NOPAQUE_DAEMON_ENABLED=False

4
docker-compose.traefik.yml
View File

@ -18,13 +18,13 @@ services:
- "traefik.http.middlewares.nopaque-header.headers.customrequestheaders.X-Forwarded-Proto=http"
- "traefik.http.routers.nopaque.entrypoints=web"
- "traefik.http.routers.nopaque.middlewares=nopaque-header, redirect-to-https@file"
- "traefik.http.routers.nopaque.rule=Host(`<DOMAIN>`)"
- "traefik.http.routers.nopaque.rule=Host(`${SERVER_NAME}`)"
### </http> ###
### <https> ###
- "traefik.http.middlewares.nopaque-secure-header.headers.customrequestheaders.X-Forwarded-Proto=https"
- "traefik.http.routers.nopaque-secure.entrypoints=web-secure"
- "traefik.http.routers.nopaque-secure.middlewares=hsts-header@file, nopaque-secure-header"
- "traefik.http.routers.nopaque-secure.rule=Host(`<DOMAIN>`)"
- "traefik.http.routers.nopaque-secure.rule=Host(`${SERVER_NAME}`)"
- "traefik.http.routers.nopaque-secure.tls.certresolver=<CERTRESOLVER>"
- "traefik.http.routers.nopaque-secure.tls.options=intermediate@file"
### </https> ###

43
docker-compose.yml
View File

@ -1,9 +1,23 @@
version: "3.5"
services:
db:
env_file: db.env
image: postgres:11
restart: unless-stopped
volumes:
- "${HOST_DB_DIR:-./db}:/var/lib/postgresql/data"
mq:
image: redis:6
restart: unless-stopped
volumes:
- "${HOST_MQ_DIR:-./mq}:/data"
nopaque:
build:
args:
DOCKER_GID: ${HOST_DOCKER_GID}
GID: ${HOST_GID}
UID: ${HOST_UID}
context: ./web
@ -13,34 +27,7 @@ services:
env_file: .env
image: nopaque:latest
restart: unless-stopped
volumes:
- "${NOPAQUE_DATA_DIR:-/mnt/nopaque}:${NOPAQUE_DATA_DIR:-/mnt/nopaque}"
- "${HOST_NOPAQUE_LOG_FILE-./nopaque.log}:${NOPAQUE_LOG_FILE:-/home/nopaque/nopaque.log}"
nopaqued:
build:
args:
DOCKER_GID: ${HOST_DOCKER_GID}
GID: ${HOST_GID}
UID: ${HOST_UID}
context: ./daemon
depends_on:
- db
- nopaque
env_file: .env
image: nopaqued:latest
restart: unless-stopped
volumes:
- "/var/run/docker.sock:/var/run/docker.sock"
- "${NOPAQUE_DATA_DIR:-/mnt/nopaque}:${NOPAQUE_DATA_DIR:-/mnt/nopaque}"
- "${HOST_NOPAQUE_DAEMON_LOG_FILE-./nopaqued.log}:${NOPAQUE_DAEMON_LOG_FILE:-/home/nopaqued/nopaqued.log}"
db:
env_file: db.env
image: postgres:11
restart: unless-stopped
volumes:
- "${HOST_DB_DIR:-./db}:/var/lib/postgresql/data"
mq:
image: redis:6
restart: unless-stopped
volumes:
- "${HOST_MQ_DIR:-./mq}:/data"
- "${HOST_NOPAQUE_LOG_FILE-./nopaque.log}:${NOPAQUE_LOG_FILE:-/home/nopaque/nopaque.log}"

1
web/.flaskenv Normal file
View File

@ -0,0 +1 @@
FLASK_APP=nopaque.py

10
web/Dockerfile
View File

@ -1,12 +1,12 @@
FROM python:3.6.12-slim-buster
FROM python:3.9.0-slim-buster
LABEL authors="Patrick Jentsch <p.jentsch@uni-bielefeld.de>, Stephan Porada <sporada@uni-bielefeld.de>"
ARG DOCKER_GID
ARG UID
ARG GID
ENV FLASK_APP=nopaque.py
ENV LANG=C.UTF-8
@ -17,12 +17,12 @@ RUN apt-get update \
&& apt-get install --no-install-recommends --yes \
build-essential \
libpq-dev \
wait-for-it \
&& rm -r /var/lib/apt/lists/*
RUN groupadd --gid ${GID} --system nopaque \
&& useradd --create-home --gid ${GID} --no-log-init --system --uid ${UID} nopaque
RUN groupadd --gid ${DOCKER_GID} --system docker \
&& groupadd --gid ${GID} --system nopaque \
&& useradd --create-home --gid ${GID} --groups ${DOCKER_GID} --no-log-init --system --uid ${UID} nopaque
USER nopaque
WORKDIR /home/nopaque

3
web/app/__init__.py
View File

@ -26,7 +26,7 @@ def create_app(config_name):
mail.init_app(app)
paranoid.init_app(app)
socketio.init_app(
app, message_queue=config[config_name].SOCKETIO_MESSAGE_QUEUE_URI)
app, message_queue=app.config['NOPAQUE_SOCKETIO_MESSAGE_QUEUE_URI'])
with app.app_context():
from . import events
@ -38,6 +38,7 @@ def create_app(config_name):
from .main import main as main_blueprint
from .services import services as services_blueprint
from .settings import settings as settings_blueprint
app.register_blueprint(admin_blueprint, url_prefix='/admin')
app.register_blueprint(auth_blueprint, url_prefix='/auth')
app.register_blueprint(corpora_blueprint, url_prefix='/corpora')

2
web/app/admin/__init__.py
View File

@ -2,4 +2,4 @@ from flask import Blueprint
admin = Blueprint('admin', __name__)
from . import views # noqa
from . import views

1
web/app/admin/forms.py
View File

@ -12,4 +12,3 @@ class EditGeneralSettingsAdminForm(EditGeneralSettingsForm):
super().__init__(*args, user=user, **kwargs)
self.role.choices = [(role.id, role.name)
for role in Role.query.order_by(Role.name).all()]
self.user = user

36
web/app/admin/views.py
View File

@ -1,5 +1,5 @@
from flask import flash, redirect, render_template, url_for
from flask_login import current_user, login_required
from flask_login import login_required
from . import admin
from .forms import EditGeneralSettingsAdminForm
from .. import db
@ -8,17 +8,19 @@ from ..models import Role, User
from ..settings import tasks as settings_tasks
@admin.route('/')
@login_required
@admin_required
def index():
return redirect(url_for('.users'))
@admin.route('/users')
@login_required
@admin_required
def users():
users = User.query.all()
users = [dict(username=u.username,
email=u.email,
role_id=u.role_id,
confirmed=u.confirmed,
id=u.id)
for u in users]
# users = [user.to_dict() for user in User.query.all()]
users = {user.id: user.to_dict() for user in User.query.all()}
return render_template('admin/users.html.j2', title='Users', users=users)
@ -35,15 +37,14 @@ def user(user_id):
@admin_required
def delete_user(user_id):
settings_tasks.delete_user(user_id)
flash('User has been deleted!')
flash('User has been marked for deletion!')
return redirect(url_for('.users'))
@admin.route('/users/<int:user_id>/edit_general_settings',
methods=['GET', 'POST'])
@admin.route('/users/<int:user_id>/edit', methods=['GET', 'POST']) # noqa
@login_required
@admin_required
def edit_general_settings(user_id):
def edit_user(user_id):
user = User.query.get_or_404(user_id)
form = EditGeneralSettingsAdminForm(user=user)
if form.validate_on_submit():
@ -52,16 +53,13 @@ def edit_general_settings(user_id):
user.username = form.username.data
user.confirmed = form.confirmed.data
user.role = Role.query.get(form.role.data)
db.session.add(user)
db.session.commit()
flash('The profile has been updated.')
return redirect(url_for('admin.edit_general_settings', user_id=user.id))
flash('Settings have been updated.')
return redirect(url_for('.edit_user', user_id=user.id))
form.confirmed.data = user.confirmed
form.dark_mode.data = user.setting_dark_mode
form.email.data = user.email
form.role.data = user.role_id
form.username.data = user.username
return render_template('admin/edit_general_settings.html.j2',
form=form,
title='General settings',
user=user)
return render_template('admin/edit_user.html.j2', form=form,
title='Edit user', user=user)

2
web/app/auth/__init__.py
View File

@ -2,4 +2,4 @@ from flask import Blueprint
auth = Blueprint('auth', __name__)
from . import views # noqa
from . import views

2
web/app/auth/forms.py
View File

@ -18,7 +18,7 @@ class RegistrationForm(FlaskForm):
username = StringField(
'Username',
validators=[DataRequired(), Length(1, 64),
Regexp(current_app.config['ALLOWED_USERNAME_REGEX'],
Regexp(current_app.config['NOPAQUE_USERNAME_REGEX'],
message='Usernames must have only letters, numbers,'
' dots or underscores')]
)

107
web/app/auth/views.py
View File

@ -1,5 +1,5 @@
from flask import (current_app, flash, redirect, render_template, request,
url_for)
from datetime import datetime
from flask import abort, flash, redirect, render_template, request, url_for
from flask_login import current_user, login_user, login_required, logout_user
from . import auth
from .forms import (LoginForm, ResetPasswordForm, ResetPasswordRequestForm,
@ -7,8 +7,8 @@ from .forms import (LoginForm, ResetPasswordForm, ResetPasswordRequestForm,
from .. import db
from ..email import create_message, send
from ..models import User
import logging
import os
import shutil
@auth.before_app_request
@ -18,11 +18,12 @@ def before_request():
unconfirmed view if user is unconfirmed.
"""
if current_user.is_authenticated:
current_user.ping()
if not current_user.confirmed \
and request.endpoint \
and request.blueprint != 'auth' \
and request.endpoint != 'static':
current_user.last_seen = datetime.utcnow()
db.session.commit()
if (not current_user.confirmed
and request.endpoint
and request.blueprint != 'auth'
and request.endpoint != 'static'):
return redirect(url_for('auth.unconfirmed'))
@ -30,20 +31,19 @@ def before_request():
def login():
if current_user.is_authenticated:
return redirect(url_for('main.dashboard'))
login_form = LoginForm(prefix='login-form')
if login_form.validate_on_submit():
user = User.query.filter_by(username=login_form.user.data).first()
form = LoginForm(prefix='login-form')
if form.validate_on_submit():
user = User.query.filter_by(username=form.user.data).first()
if user is None:
user = User.query.filter_by(email=login_form.user.data).first()
if user is not None and user.verify_password(login_form.password.data):
login_user(user, login_form.remember_me.data)
user = User.query.filter_by(email=form.user.data.lower()).first()
if user is not None and user.verify_password(form.password.data):
login_user(user, form.remember_me.data)
next = request.args.get('next')
if next is None or not next.startswith('/'):
next = url_for('main.dashboard')
return redirect(next)
flash('Invalid email/username or password.')
return render_template('auth/login.html.j2', login_form=login_form,
title='Log in')
return render_template('auth/login.html.j2', form=form, title='Log in')
@auth.route('/logout')
@ -58,26 +58,28 @@ def logout():
def register():
if current_user.is_authenticated:
return redirect(url_for('main.dashboard'))
registration_form = RegistrationForm(prefix='registration-form')
if registration_form.validate_on_submit():
user = User(email=registration_form.email.data.lower(),
password=registration_form.password.data,
username=registration_form.username.data)
form = RegistrationForm(prefix='registration-form')
if form.validate_on_submit():
user = User(email=form.email.data.lower(),
password=form.password.data,
username=form.username.data)
db.session.add(user)
db.session.commit()
user_dir = os.path.join(current_app.config['DATA_DIR'],
str(user.id))
if os.path.exists(user_dir):
shutil.rmtree(user_dir)
os.mkdir(user_dir)
token = user.generate_confirmation_token()
msg = create_message(user.email, 'Confirm Your Account',
'auth/email/confirm', token=token, user=user)
send(msg)
flash('A confirmation email has been sent to you by email.')
return redirect(url_for('auth.login'))
return render_template('auth/register.html.j2',
registration_form=registration_form,
try:
os.makedirs(user.path)
except OSError:
logging.error('Make dir {} led to an OSError!'.format(user.path))
db.session.delete(user)
db.session.commit()
abort(500)
else:
token = user.generate_confirmation_token()
msg = create_message(user.email, 'Confirm Your Account',
'auth/email/confirm', token=token, user=user)
send(msg)
flash('A confirmation email has been sent to you by email.')
return redirect(url_for('.login'))
return render_template('auth/register.html.j2', form=form,
title='Register')
@ -92,7 +94,7 @@ def confirm(token):
return redirect(url_for('main.dashboard'))
else:
flash('The confirmation link is invalid or has expired.')
return redirect(url_for('auth.unconfirmed'))
return redirect(url_for('.unconfirmed'))
@auth.route('/unconfirmed')
@ -119,39 +121,32 @@ def resend_confirmation():
def reset_password_request():
if current_user.is_authenticated:
return redirect(url_for('main.dashboard'))
reset_password_request_form = ResetPasswordRequestForm(
prefix='reset-password-request-form')
if reset_password_request_form.validate_on_submit():
submitted_email = reset_password_request_form.email.data
user = User.query.filter_by(email=submitted_email.lower()).first()
if user:
form = ResetPasswordRequestForm(prefix='reset-password-request-form')
if form.validate_on_submit():
user = User.query.filter_by(email=form.email.data.lower()).first()
if user is not None:
token = user.generate_reset_token()
msg = create_message(user.email, 'Reset Your Password',
'auth/email/reset_password', token=token,
user=user)
send(msg)
flash('An email with instructions to reset your password has been '
'sent to you.')
return redirect(url_for('auth.login'))
return render_template(
'auth/reset_password_request.html.j2',
reset_password_request_form=reset_password_request_form,
title='Password Reset')
flash('An email with instructions to reset your password has been sent to you.') # noqa
return redirect(url_for('.login'))
return render_template('auth/reset_password_request.html.j2', form=form,
title='Password Reset')
@auth.route('/reset/<token>', methods=['GET', 'POST'])
def reset_password(token):
if current_user.is_authenticated:
return redirect(url_for('main.dashboard'))
reset_password_form = ResetPasswordForm(prefix='reset-password-form')
if reset_password_form.validate_on_submit():
if User.reset_password(token, reset_password_form.password.data):
form = ResetPasswordForm(prefix='reset-password-form')
if form.validate_on_submit():
if User.reset_password(token, form.password.data):
db.session.commit()
flash('Your password has been updated.')
return redirect(url_for('auth.login'))
return redirect(url_for('.login'))
else:
return redirect(url_for('main.index'))
return render_template('auth/reset_password.html.j2',
reset_password_form=reset_password_form,
title='Password Reset',
token=token)
return render_template('auth/reset_password.html.j2', form=form,
title='Password Reset', token=token)

34
web/app/corpora/events.py
View File

@ -24,27 +24,29 @@ corpus_analysis_sessions = {}
corpus_analysis_clients = {}
@socketio.on('corpus_create_zip')
@socketio.on('export_corpus')
@socketio_login_required
def corpus_create_zip(corpus_id):
corpus = Corpus.query.get_or_404(corpus_id)
def export_corpus(corpus_id):
# TODO: This should not be get_or_404 here - Socket.IO != HTTP request
corpus = Corpus.query.get(corpus_id)
if corpus is None:
response = {'code': 404, 'msg': 'Not found'}
socketio.emit('export_corpus', response, room=request.sid)
return
if corpus.status not in ['prepared', 'start analysis', 'stop analysis']:
response = {'code': 412, 'msg': 'Precondition Failed'}
socketio.emit('export_corpus', response, room=request.sid)
return
# delete old corpus archive if it exists/has been build before
if corpus.archive_file is not None:
if (os.path.isfile(corpus.archive_file)):
os.remove(corpus.archive_file)
root_dir = os.path.join(current_app.config['DATA_DIR'],
str(current_user.id),
'corpora')
base_dir = os.path.join(root_dir, str(corpus.id))
if corpus.archive_file is not None and os.path.isfile(corpus.archive_file):
os.remove(corpus.archive_file)
zip_name = corpus.title
zip_path = os.path.join(root_dir, zip_name)
corpus.archive_file = os.path.join(base_dir, zip_name) + '.zip'
zip_path = os.path.join(current_user.path, 'corpora', zip_name)
corpus.archive_file = os.path.join(corpus.path, zip_name) + '.zip'
db.session.commit()
shutil.make_archive(zip_path,
'zip',
base_dir)
shutil.make_archive(zip_path, 'zip', corpus.path)
shutil.move(zip_path + '.zip', corpus.archive_file)
socketio.emit('corpus_zip_created', room=request.sid)
socketio.emit('export_corpus_' + str(corpus.id), room=request.sid)
@socketio.on('corpus_analysis_init')

400
web/app/corpora/views.py
View File

@ -1,4 +1,4 @@
from flask import (abort, current_app, flash, make_response, redirect, request,
from flask import (abort, flash, make_response, redirect, request,
render_template, url_for, send_from_directory)
from flask_login import current_user, login_required
from . import corpora
@ -11,6 +11,7 @@ from jsonschema import validate
from .. import db
from ..models import Corpus, CorpusFile, QueryResult
import json
import logging
import os
import shutil
import glob
@ -22,106 +23,92 @@ from .import_corpus import check_zip_contents
@corpora.route('/add', methods=['GET', 'POST'])
@login_required
def add_corpus():
add_corpus_form = AddCorpusForm()
if add_corpus_form.validate_on_submit():
form = AddCorpusForm()
if form.validate_on_submit():
corpus = Corpus(creator=current_user,
description=add_corpus_form.description.data,
status='unprepared', title=add_corpus_form.title.data)
description=form.description.data,
title=form.title.data)
db.session.add(corpus)
db.session.commit()
dir = os.path.join(current_app.config['DATA_DIR'],
str(corpus.user_id), 'corpora', str(corpus.id))
try:
os.makedirs(dir)
os.makedirs(corpus.path)
except OSError:
flash('[ERROR]: Could not add corpus!', 'corpus')
corpus.delete()
else:
url = url_for('corpora.corpus', corpus_id=corpus.id)
flash('[<a href="{}">{}</a>] added'.format(url, corpus.title),
'corpus')
return redirect(url_for('corpora.corpus', corpus_id=corpus.id))
return render_template('corpora/add_corpus.html.j2',
add_corpus_form=add_corpus_form,
logging.error('Make dir {} led to an OSError!'.format(corpus.path))
db.session.delete(corpus)
db.session.commit()
abort(500)
flash('Corpus "{}" added!'.format(corpus.title), 'corpus')
return redirect(url_for('.corpus', corpus_id=corpus.id))
return render_template('corpora/add_corpus.html.j2', form=form,
title='Add corpus')
@corpora.route('/import', methods=['GET', 'POST'])
@login_required
def import_corpus():
import_corpus_form = ImportCorpusForm()
if import_corpus_form.is_submitted():
if not import_corpus_form.validate():
return make_response(import_corpus_form.errors, 400)
form = ImportCorpusForm()
if form.is_submitted():
if not form.validate():
return make_response(form.errors, 400)
corpus = Corpus(creator=current_user,
description=import_corpus_form.description.data,
status='unprepared',
title=import_corpus_form.title.data)
description=form.description.data,
title=form.title.data)
db.session.add(corpus)
db.session.commit()
dir = os.path.join(current_app.config['DATA_DIR'],
str(corpus.user_id), 'corpora', str(corpus.id))
try:
os.makedirs(dir)
os.makedirs(corpus.path)
except OSError:
flash('[ERROR]: Could not import corpus!', 'corpus')
corpus.delete()
logging.error('Make dir {} led to an OSError!'.format(corpus.path))
db.session.delete(corpus)
db.session.commit()
flash('Internal Server Error', 'error')
return make_response(
{'redirect_url': url_for('.import_corpus')}, 500)
# Upload zip
archive_file = os.path.join(corpus.path, form.file.data.filename)
form.file.data.save(archive_file)
# Some checks to verify it is a valid exported corpus
with ZipFile(archive_file, 'r') as zip:
contents = zip.namelist()
if set(check_zip_contents).issubset(contents):
# Unzip
shutil.unpack_archive(archive_file, corpus.path)
# Register vrt files to corpus
vrts = glob.glob(corpus.path + '/*.vrt')
for file in vrts:
element_tree = ET.parse(file)
text_node = element_tree.find('text')
corpus_file = CorpusFile(
address=text_node.get('address', 'NULL'),
author=text_node.get('author', 'NULL'),
booktitle=text_node.get('booktitle', 'NULL'),
chapter=text_node.get('chapter', 'NULL'),
corpus=corpus,
editor=text_node.get('editor', 'NULL'),
filename=os.path.basename(file),
institution=text_node.get('institution', 'NULL'),
journal=text_node.get('journal', 'NULL'),
pages=text_node.get('pages', 'NULL'),
publisher=text_node.get('publisher', 'NULL'),
publishing_year=text_node.get('publishing_year', ''),
school=text_node.get('school', 'NULL'),
title=text_node.get('title', 'NULL')
)
db.session.add(corpus_file)
# finish import and redirect to imported corpus
corpus.status = 'prepared'
db.session.commit()
os.remove(archive_file)
flash('Corpus "{}" imported!'.format(corpus.title), 'corpus')
return make_response(
{'redirect_url': url_for('.corpus', corpus_id=corpus.id)}, 201)
else:
# Upload zip
archive_file = os.path.join(current_app.config['DATA_DIR'], dir,
import_corpus_form.file.data.filename)
corpus_dir = os.path.dirname(archive_file)
import_corpus_form.file.data.save(archive_file)
# Some checks to verify it is a valid exported corpus
with ZipFile(archive_file, 'r') as zip:
contents = zip.namelist()
if set(check_zip_contents).issubset(contents):
# Unzip
shutil.unpack_archive(archive_file, corpus_dir)
# Register vrt files to corpus
vrts = glob.glob(corpus_dir + '/*.vrt')
for file in vrts:
element_tree = ET.parse(file)
text_node = element_tree.find('text')
corpus_file = CorpusFile(
address=text_node.get('address', 'NULL'),
author=text_node.get('author', 'NULL'),
booktitle=text_node.get('booktitle', 'NULL'),
chapter=text_node.get('chapter', 'NULL'),
corpus=corpus,
dir=dir,
editor=text_node.get('editor', 'NULL'),
filename=os.path.basename(file),
institution=text_node.get('institution', 'NULL'),
journal=text_node.get('journal', 'NULL'),
pages=text_node.get('pages', 'NULL'),
publisher=text_node.get('publisher', 'NULL'),
publishing_year=text_node.get('publishing_year', ''),
school=text_node.get('school', 'NULL'),
title=text_node.get('title', 'NULL'))
db.session.add(corpus_file)
# finish import and got to imported corpus
url = url_for('corpora.corpus', corpus_id=corpus.id)
corpus.status = 'prepared'
db.session.commit()
os.remove(archive_file)
flash('[<a href="{}">{}</a>] imported'.format(url,
corpus.title),
'corpus')
return make_response(
{'redirect_url': url_for('corpora.corpus',
corpus_id=corpus.id)},
201)
else:
# If imported zip is not valid delete corpus and give feedback
corpus.delete()
db.session.commit()
flash('Imported corpus is not valid.', 'error')
return make_response(
{'redirect_url': url_for('corpora.import_corpus')},
201)
return render_template('corpora/import_corpus.html.j2',
import_corpus_form=import_corpus_form,
# If imported zip is not valid delete corpus and give feedback
flash('Can not import corpus "{}" not imported: Invalid archive file!', 'error') # noqa
tasks.delete_corpus(corpus.id)
return make_response(
{'redirect_url': url_for('.import_corpus')}, 201)
return render_template('corpora/import_corpus.html.j2', form=form,
title='Import Corpus')
@ -131,31 +118,22 @@ def corpus(corpus_id):
corpus = Corpus.query.get_or_404(corpus_id)
if not (corpus.creator == current_user or current_user.is_administrator()):
abort(403)
corpus_files = [dict(filename=corpus_file.filename,
author=corpus_file.author,
title=corpus_file.title,
publishing_year=corpus_file.publishing_year,
corpus_id=corpus.id,
id=corpus_file.id)
for corpus_file in corpus.files]
return render_template('corpora/corpus.html.j2',
corpus=corpus,
corpus_files=corpus_files,
title='Corpus')
corpus_files = [corpus_file.to_dict() for corpus_file in corpus.files]
return render_template('corpora/corpus.html.j2', corpus=corpus,
corpus_files=corpus_files, title='Corpus')
@corpora.route('/<int:corpus_id>/export')
@corpora.route('/<int:corpus_id>/download')
@login_required
def export_corpus(corpus_id):
def download_corpus(corpus_id):
corpus = Corpus.query.get_or_404(corpus_id)
if not (corpus.creator == current_user or current_user.is_administrator()):
abort(403)
# TODO: Check what happens here
dir = os.path.dirname(corpus.archive_file)
filename = os.path.basename(corpus.archive_file)
return send_from_directory(directory=dir,
filename=filename,
mimetype='zip',
as_attachment=True)
return send_from_directory(as_attachment=True, directory=dir,
filename=filename, mimetype='zip')
@corpora.route('/<int:corpus_id>/analyse')
@ -168,7 +146,8 @@ def analyse_corpus(corpus_id):
display_options_form = DisplayOptionsForm(
prefix='display-options-form',
result_context=request.args.get('context', 20),
results_per_page=request.args.get('results_per_page', 30))
results_per_page=request.args.get('results_per_page', 30)
)
query_form = QueryForm(prefix='query-form',
query=request.args.get('query'))
query_download_form = QueryDownloadForm(prefix='query-download-form')
@ -177,12 +156,12 @@ def analyse_corpus(corpus_id):
return render_template(
'corpora/analyse_corpus.html.j2',
corpus=corpus,
corpus_id=corpus_id,
display_options_form=display_options_form,
inspect_display_options_form=inspect_display_options_form,
query_form=query_form,
query_download_form=query_download_form,
inspect_display_options_form=inspect_display_options_form,
title='Corpus analysis')
title='Corpus analysis'
)
@corpora.route('/<int:corpus_id>/delete')
@ -191,8 +170,8 @@ def delete_corpus(corpus_id):
corpus = Corpus.query.get_or_404(corpus_id)
if not (corpus.creator == current_user or current_user.is_administrator()):
abort(403)
flash('Corpus "{}" marked for deletion!'.format(corpus.title), 'corpus')
tasks.delete_corpus(corpus_id)
flash('Corpus deleted!', 'corpus')
return redirect(url_for('main.dashboard'))
@ -202,43 +181,33 @@ def add_corpus_file(corpus_id):
corpus = Corpus.query.get_or_404(corpus_id)
if not (corpus.creator == current_user or current_user.is_administrator()):
abort(403)
add_corpus_file_form = AddCorpusFileForm(corpus,
prefix='add-corpus-file-form')
if add_corpus_file_form.is_submitted():
if not add_corpus_file_form.validate():
return make_response(add_corpus_file_form.errors, 400)
form = AddCorpusFileForm(corpus, prefix='add-corpus-file-form')
if form.is_submitted():
if not form.validate():
return make_response(form.errors, 400)
# Save the file
dir = os.path.join(str(corpus.user_id), 'corpora', str(corpus.id))
add_corpus_file_form.file.data.save(
os.path.join(current_app.config['DATA_DIR'], dir,
add_corpus_file_form.file.data.filename))
corpus_file = CorpusFile(
address=add_corpus_file_form.address.data,
author=add_corpus_file_form.author.data,
booktitle=add_corpus_file_form.booktitle.data,
chapter=add_corpus_file_form.chapter.data,
corpus=corpus,
dir=dir,
editor=add_corpus_file_form.editor.data,
filename=add_corpus_file_form.file.data.filename,
institution=add_corpus_file_form.institution.data,
journal=add_corpus_file_form.journal.data,
pages=add_corpus_file_form.pages.data,
publisher=add_corpus_file_form.publisher.data,
publishing_year=add_corpus_file_form.publishing_year.data,
school=add_corpus_file_form.school.data,
title=add_corpus_file_form.title.data)
form.file.data.save(os.path.join(corpus.path, form.file.data.filename))
corpus_file = CorpusFile(address=form.address.data,
author=form.author.data,
booktitle=form.booktitle.data,
chapter=form.chapter.data,
corpus=corpus,
editor=form.editor.data,
filename=form.file.data.filename,
institution=form.institution.data,
journal=form.journal.data,
pages=form.pages.data,
publisher=form.publisher.data,
publishing_year=form.publishing_year.data,
school=form.school.data,
title=form.title.data)
db.session.add(corpus_file)
corpus.status = 'unprepared'
db.session.commit()
flash('Corpus file added!', 'corpus')
return make_response(
{'redirect_url': url_for('corpora.corpus', corpus_id=corpus.id)},
201)
return render_template('corpora/add_corpus_file.html.j2',
corpus=corpus,
add_corpus_file_form=add_corpus_file_form,
title='Add corpus file')
flash('Corpus file "{}" added!'.format(corpus_file.filename), 'corpus')
return make_response({'redirect_url': url_for('.corpus', corpus_id=corpus.id)}, 201) # noqa
return render_template('corpora/add_corpus_file.html.j2', corpus=corpus,
form=form, title='Add corpus file')
@corpora.route('/<int:corpus_id>/files/<int:corpus_file_id>/delete')
@ -250,9 +219,9 @@ def delete_corpus_file(corpus_id, corpus_file_id):
if not (corpus_file.corpus.creator == current_user
or current_user.is_administrator()):
abort(403)
flash('Corpus file "{}" marked for deletion!'.format(corpus_file.filename), 'corpus') # noqa
tasks.delete_corpus_file(corpus_file_id)
flash('Corpus file deleted!', 'corpus')
return redirect(url_for('corpora.corpus', corpus_id=corpus_id))
return redirect(url_for('.corpus', corpus_id=corpus_id))
@corpora.route('/<int:corpus_id>/files/<int:corpus_file_id>/download')
@ -264,9 +233,8 @@ def download_corpus_file(corpus_id, corpus_file_id):
if not (corpus_file.corpus.creator == current_user
or current_user.is_administrator()):
abort(403)
dir = os.path.join(current_app.config['DATA_DIR'],
corpus_file.dir)
return send_from_directory(as_attachment=True, directory=dir,
return send_from_directory(as_attachment=True,
directory=os.path.dirname(corpus_file.path),
filename=corpus_file.filename)
@ -275,47 +243,44 @@ def download_corpus_file(corpus_id, corpus_file_id):
@login_required
def corpus_file(corpus_id, corpus_file_id):
corpus = Corpus.query.get_or_404(corpus_id)
corpus_file = CorpusFile.query.get_or_404(corpus_file_id)
if not corpus_file.corpus_id == corpus_id:
abort(404)
if not (corpus_file.corpus.creator == current_user
or current_user.is_administrator()):
if not (corpus.creator == current_user or current_user.is_administrator()):
abort(403)
edit_corpus_file_form = EditCorpusFileForm(prefix='edit-corpus-file-form')
if edit_corpus_file_form.validate_on_submit():
corpus_file.address = edit_corpus_file_form.address.data
corpus_file.author = edit_corpus_file_form.author.data
corpus_file.booktitle = edit_corpus_file_form.booktitle.data
corpus_file.chapter = edit_corpus_file_form.chapter.data
corpus_file.editor = edit_corpus_file_form.editor.data
corpus_file.institution = edit_corpus_file_form.institution.data
corpus_file.journal = edit_corpus_file_form.journal.data
corpus_file.pages = edit_corpus_file_form.pages.data
corpus_file.publisher = edit_corpus_file_form.publisher.data
corpus_file.publishing_year = \
edit_corpus_file_form.publishing_year.data
corpus_file.school = edit_corpus_file_form.school.data
corpus_file.title = edit_corpus_file_form.title.data
corpus_file = CorpusFile.query.get_or_404(corpus_file_id)
if corpus_file.corpus != corpus:
abort(404)
form = EditCorpusFileForm(prefix='edit-corpus-file-form')
if form.validate_on_submit():
corpus_file.address = form.address.data
corpus_file.author = form.author.data
corpus_file.booktitle = form.booktitle.data
corpus_file.chapter = form.chapter.data
corpus_file.editor = form.editor.data
corpus_file.institution = form.institution.data
corpus_file.journal = form.journal.data
corpus_file.pages = form.pages.data
corpus_file.publisher = form.publisher.data
corpus_file.publishing_year = form.publishing_year.data
corpus_file.school = form.school.data
corpus_file.title = form.title.data
corpus.status = 'unprepared'
db.session.commit()
flash('Corpus file edited!', 'corpus')
return redirect(url_for('corpora.corpus', corpus_id=corpus_id))
flash('Corpus file "{}" edited!'.format(corpus_file.filename), 'corpus') # noqa
return redirect(url_for('.corpus', corpus_id=corpus_id))
# If no form is submitted or valid, fill out fields with current values
edit_corpus_file_form.address.data = corpus_file.address
edit_corpus_file_form.author.data = corpus_file.author
edit_corpus_file_form.booktitle.data = corpus_file.booktitle
edit_corpus_file_form.chapter.data = corpus_file.chapter
edit_corpus_file_form.editor.data = corpus_file.editor
edit_corpus_file_form.institution.data = corpus_file.institution
edit_corpus_file_form.journal.data = corpus_file.journal
edit_corpus_file_form.pages.data = corpus_file.pages
edit_corpus_file_form.publisher.data = corpus_file.publisher
edit_corpus_file_form.publishing_year.data = corpus_file.publishing_year
edit_corpus_file_form.school.data = corpus_file.school
edit_corpus_file_form.title.data = corpus_file.title
return render_template('corpora/corpus_file.html.j2',
corpus_file=corpus_file, corpus=corpus,
edit_corpus_file_form=edit_corpus_file_form,
form.address.data = corpus_file.address
form.author.data = corpus_file.author
form.booktitle.data = corpus_file.booktitle
form.chapter.data = corpus_file.chapter
form.editor.data = corpus_file.editor
form.institution.data = corpus_file.institution
form.journal.data = corpus_file.journal
form.pages.data = corpus_file.pages
form.publisher.data = corpus_file.publisher
form.publishing_year.data = corpus_file.publishing_year
form.school.data = corpus_file.school
form.title.data = corpus_file.title
return render_template('corpora/corpus_file.html.j2', corpus=corpus,
corpus_file=corpus_file, form=form,
title='Edit corpus file')
@ -327,10 +292,10 @@ def prepare_corpus(corpus_id):
abort(403)
if corpus.files.all():
tasks.build_corpus(corpus_id)
flash('Building Corpus...', 'corpus')
flash('Corpus "{}" has been marked to get build!'.format(corpus.title), 'corpus') # noqa
else:
flash('Can not build corpus, please add corpus file(s).', 'corpus')
return redirect(url_for('corpora.corpus', corpus_id=corpus_id))
flash('Can not build corpus "{}": No corpus file(s)!'.format(corpus.title), 'error') # noqa
return redirect(url_for('.corpus', corpus_id=corpus_id))
# Following are view functions to add, view etc. exported results.
@ -340,35 +305,29 @@ def add_query_result():
'''
View to import a result as a json file.
'''
add_query_result_form = AddQueryResultForm(prefix='add-query-result-form')
if add_query_result_form.is_submitted():
if not add_query_result_form.validate():
return make_response(add_query_result_form.errors, 400)
query_result = QueryResult(
creator=current_user,
description=add_query_result_form.description.data,
filename=add_query_result_form.file.data.filename,
title=add_query_result_form.title.data
)
form = AddQueryResultForm(prefix='add-query-result-form')
if form.is_submitted():
if not form.validate():
return make_response(form.errors, 400)
query_result = QueryResult(creator=current_user,
description=form.description.data,
filename=form.file.data.filename,
title=form.title.data)
db.session.add(query_result)
db.session.commit()
# create paths to save the uploaded json file
query_result_dir = os.path.join(current_app.config['DATA_DIR'],
str(current_user.id),
'query_results',
str(query_result.id))
try:
os.makedirs(query_result_dir)
except Exception:
os.makedirs(query_result.path)
except OSError:
logging.error('Make dir {} led to an OSError!'.format(query_result.path)) # noqa
db.session.delete(query_result)
db.session.commit()
flash('Internal Server Error', 'error')
redirect_url = url_for('corpora.add_query_result')
return make_response({'redirect_url': redirect_url}, 500)
return make_response(
{'redirect_url': url_for('.add_query_result')}, 500)
# save the uploaded file
query_result_file_path = os.path.join(query_result_dir,
query_result_file_path = os.path.join(query_result.path,
query_result.filename)
add_query_result_form.file.data.save(query_result_file_path)
form.file.data.save(query_result_file_path)
# parse json from file
with open(query_result_file_path, 'r') as file:
query_result_file_content = json.load(file)
@ -381,19 +340,16 @@ def add_query_result():
except Exception:
tasks.delete_query_result(query_result.id)
flash('Uploaded file is invalid', 'result')
redirect_url = url_for('corpora.add_query_result')
return make_response({'redirect_url': redirect_url}, 201)
return make_response(
{'redirect_url': url_for('.add_query_result')}, 201)
query_result_file_content.pop('matches')
query_result_file_content.pop('cpos_lookup')
query_result.query_metadata = query_result_file_content
db.session.commit()
flash('Query result added!', 'result')
redirect_url = url_for('corpora.query_result',
query_result_id=query_result.id)
return make_response({'redirect_url': redirect_url}, 201)
return make_response({'redirect_url': url_for('.query_result', query_result_id=query_result.id)}, 201) # noqa
return render_template('corpora/query_results/add_query_result.html.j2',
add_query_result_form=add_query_result_form,
title='Add query result')
form=form, title='Add query result')
@corpora.route('/result/<int:query_result_id>')
@ -404,8 +360,7 @@ def query_result(query_result_id):
or current_user.is_administrator()):
abort(403)
return render_template('corpora/query_results/query_result.html.j2',
query_result=query_result,
title='Query result')
query_result=query_result, title='Query result')
@corpora.route('/result/<int:query_result_id>/inspect')
@ -427,14 +382,7 @@ def inspect_query_result(query_result_id):
inspect_display_options_form = InspectDisplayOptionsForm(
prefix='inspect-display-options-form'
)
query_result_file_path = os.path.join(
current_app.config['DATA_DIR'],
str(current_user.id),
'query_results',
str(query_result.id),
query_result.filename
)
with open(query_result_file_path, 'r') as query_result_file:
with open(query_result.path, 'r') as query_result_file:
query_result_file_content = json.load(query_result_file)
return render_template('corpora/query_results/inspect.html.j2',
query_result=query_result,
@ -452,8 +400,8 @@ def delete_query_result(query_result_id):
if not (query_result.creator == current_user
or current_user.is_administrator()):
abort(403)
flash('Query result "{}" has been marked for deletion!'.format(query_result), 'result') # noqa
tasks.delete_query_result(query_result_id)
flash('Query result deleted!', 'result')
return redirect(url_for('services.service', service="corpus_analysis"))
@ -464,10 +412,6 @@ def download_query_result(query_result_id):
if not (query_result.creator == current_user
or current_user.is_administrator()):
abort(403)
query_result_dir = os.path.join(current_app.config['DATA_DIR'],
str(current_user.id),
'query_results',
str(query_result.id))
return send_from_directory(as_attachment=True,
directory=query_result_dir,
directory=os.path.dirname(query_result.path),
filename=query_result.filename)

4
web/app/decorators.py
View File

@ -38,7 +38,7 @@ def socketio_admin_required(f):
if current_user.is_administrator:
return f(*args, **kwargs)
else:
response = {'code': 401, 'desc': 'Unauthorized'}
response = {'code': 401, 'msg': 'Unauthorized'}
socketio.emit(request.event['message'], response, room=request.sid)
return wrapped
@ -49,6 +49,6 @@ def socketio_login_required(f):
if current_user.is_authenticated:
return f(*args, **kwargs)
else:
response = {'code': 401, 'desc': 'Unauthorized'}
response = {'code': 401, 'msg': 'Unauthorized'}
socketio.emit(request.event['message'], response, room=request.sid)
return wrapped

4
web/app/email.py
View File

@ -1,11 +1,11 @@
from flask import render_template
from flask import current_app, render_template
from flask_mail import Message
from . import mail
from .decorators import background
def create_message(recipient, subject, template, **kwargs):
msg = Message('[nopaque] {}'.format(subject), recipients=[recipient])
msg = Message('{} {}'.format(current_app.config['NOPAQUE_MAIL_SUBJECT_PREFIX'], subject), recipients=[recipient]) # noqa
msg.body = render_template('{}.txt.j2'.format(template), **kwargs)
msg.html = render_template('{}.html.j2'.format(template), **kwargs)
return msg

40
web/app/events.py
View File

@ -33,38 +33,24 @@ def disconnect():
connected_sessions.remove(request.sid)
@socketio.on('user_data_stream_init')
@socketio.on('start_user_session')
@socketio_login_required
def user_data_stream_init():
socketio.start_background_task(user_data_stream,
def start_user_session(user_id):
if not (current_user.id == user_id or current_user.is_administrator):
return
socketio.start_background_task(user_session,
current_app._get_current_object(),
current_user.id, request.sid)
user_id, request.sid)
@socketio.on('foreign_user_data_stream_init')
@socketio_login_required
@socketio_admin_required
def foreign_user_data_stream_init(user_id):
socketio.start_background_task(user_data_stream,
current_app._get_current_object(),
user_id, request.sid, foreign=True)
def user_data_stream(app, user_id, session_id, foreign=False):
def user_session(app, user_id, session_id):
'''
' Sends initial corpus and job lists to the client. Afterwards it checks
' every 3 seconds if changes to the initial values appeared. If changes are
' detected, a RFC 6902 compliant JSON patch gets send.
'
' NOTE: The initial values are send as a init events.
' The JSON patches are send as update events.
' Sends initial user data to the client. Afterwards it checks every 3s if
' changes to the initial values appeared. If changes are detected, a
' RFC 6902 compliant JSON patch gets send.
'''
if foreign:
init_event = 'foreign_user_data_stream_init'
update_event = 'foreign_user_data_stream_update'
else:
init_event = 'user_data_stream_init'
update_event = 'user_data_stream_update'
init_event = 'user_{}_init'.format(user_id)
patch_event = 'user_{}_patch'.format(user_id)
with app.app_context():
# Gather current values from database.
user = User.query.get(user_id)
@ -80,7 +66,7 @@ def user_data_stream(app, user_id, session_id, foreign=False):
new_user_dict)
# In case there are patches, send them to the client.
if user_patch:
socketio.emit(update_event, user_patch.to_string(),
socketio.emit(patch_event, user_patch.to_string(),
room=session_id)
# Set new values as references for the next iteration.
user_dict = new_user_dict

2
web/app/jobs/__init__.py
View File

@ -2,4 +2,4 @@ from flask import Blueprint
jobs = Blueprint('jobs', __name__)
from . import views # noqa
from . import views

37
web/app/jobs/views.py
View File

@ -1,4 +1,4 @@
from flask import (abort, current_app, flash, redirect, render_template,
from flask import (abort, flash, redirect, render_template,
send_from_directory, url_for)
from flask_login import current_user, login_required
from . import jobs
@ -14,13 +14,8 @@ def job(job_id):
job = Job.query.get_or_404(job_id)
if not (job.creator == current_user or current_user.is_administrator()):
abort(403)
job_inputs = [dict(filename=input.filename,
id=input.id,
job_id=job.id)
for input in job.inputs]
return render_template('jobs/job.html.j2',
job=job,
job_inputs=job_inputs,
job_inputs = [job_input.to_dict() for job_input in job.inputs]
return render_template('jobs/job.html.j2', job=job, job_inputs=job_inputs,
title='Job')
@ -31,22 +26,19 @@ def delete_job(job_id):
if not (job.creator == current_user or current_user.is_administrator()):
abort(403)
tasks.delete_job(job_id)
flash('Job has been deleted!', 'job')
flash('Job has been marked for deletion!', 'job')
return redirect(url_for('main.dashboard'))
@jobs.route('/<int:job_id>/inputs/<int:job_input_id>/download')
@login_required
def download_job_input(job_id, job_input_id):
job_input = JobInput.query.get_or_404(job_input_id)
if not job_input.job_id == job_id:
abort(404)
job_input = JobInput.query.filter(JobInput.job_id == job_id, JobInput.id == job_input_id).first_or_404() # noqa
if not (job_input.job.creator == current_user
or current_user.is_administrator()):
abort(403)
dir = os.path.join(current_app.config['DATA_DIR'],
job_input.dir)
return send_from_directory(as_attachment=True, directory=dir,
return send_from_directory(as_attachment=True,
directory=os.path.dirname(job_input.path),
filename=job_input.filename)
@ -56,23 +48,20 @@ def download_job_input(job_id, job_input_id):
def restart(job_id):
job = Job.query.get_or_404(job_id)
if job.status != 'failed':
flash('Could not restart job: status is not "failed"', 'error')
flash('Can not restart job "{}": Status is not "failed"'.format(job.title), 'error') # noqa
else:
tasks.restart_job(job_id)
flash('Job has been restarted!', 'job')
return redirect(url_for('jobs.job', job_id=job_id))
flash('Job "{}" has been marked to get restarted!'.format(job.title), 'job') # noqa
return redirect(url_for('.job', job_id=job_id))
@jobs.route('/<int:job_id>/results/<int:job_result_id>/download')
@login_required
def download_job_result(job_id, job_result_id):
job_result = JobResult.query.get_or_404(job_result_id)
if not job_result.job_id == job_id:
abort(404)
job_result = JobResult.query.filter(JobResult.job_id == job_id, JobResult.id == job_result_id).first_or_404() # noqa
if not (job_result.job.creator == current_user
or current_user.is_administrator()):
abort(403)
dir = os.path.join(current_app.config['DATA_DIR'],
job_result.dir)
return send_from_directory(as_attachment=True, directory=dir,
return send_from_directory(as_attachment=True,
directory=os.path.dirname(job_result.path),
filename=job_result.filename)

2
web/app/main/__init__.py
View File

@ -2,4 +2,4 @@ from flask import Blueprint
main = Blueprint('main', __name__)
from . import views # noqa
from . import views

25
web/app/main/views.py
View File

@ -7,17 +7,16 @@ from ..models import User
@main.route('/', methods=['GET', 'POST'])
def index():
login_form = LoginForm(prefix='login-form')
if login_form.validate_on_submit():
user = User.query.filter_by(username=login_form.user.data).first()
form = LoginForm(prefix='login-form')
if form.validate_on_submit():
user = User.query.filter_by(username=form.user.data).first()
if user is None:
user = User.query.filter_by(email=login_form.user.data).first()
if user is not None and user.verify_password(login_form.password.data):
login_user(user, login_form.remember_me.data)
return redirect(url_for('main.dashboard'))
user = User.query.filter_by(email=form.user.data.lower()).first()
if user is not None and user.verify_password(form.password.data):
login_user(user, form.remember_me.data)
return redirect(url_for('.dashboard'))
flash('Invalid email/username or password.')
return render_template('main/index.html.j2', login_form=login_form,
title='nopaque')
return render_template('main/index.html.j2', form=form, title='nopaque')
@main.route('/about_and_faq')
@ -31,7 +30,6 @@ def dashboard():
return render_template('main/dashboard.html.j2', title='Dashboard')
@main.route('/news')
def news():
return render_template('main/news.html.j2', title='News')
@ -40,12 +38,9 @@ def news():
@main.route('/privacy_policy')
def privacy_policy():
return render_template('main/privacy_policy.html.j2',
title=('Information on the processing of personal'
' data for the nopaque platform (GDPR)'))
title='Privacy statement (GDPR)')
@main.route('/terms_of_use')
def terms_of_use():
return render_template('main/terms_of_use.html.j2',
title='General Terms of Use of the platform '
'nopaque')
return render_template('main/terms_of_use.html.j2', title='Terms of Use')

290
web/app/models.py
View File

@ -1,12 +1,12 @@
from datetime import datetime
from flask import current_app
from flask import current_app, url_for
from flask_login import UserMixin, AnonymousUserMixin
from itsdangerous import BadSignature, TimedJSONWebSignatureSerializer
from time import sleep
from werkzeug.security import generate_password_hash, check_password_hash
from werkzeug.utils import secure_filename
import xml.etree.ElementTree as ET
from . import db, login_manager
import logging
import os
import shutil
@ -35,7 +35,7 @@ class Role(db.Model):
# Fields
default = db.Column(db.Boolean, default=False, index=True)
name = db.Column(db.String(64), unique=True)
permissions = db.Column(db.BigInteger)
permissions = db.Column(db.Integer)
# Relationships
users = db.relationship('User', backref='role', lazy='dynamic')
@ -54,7 +54,7 @@ class Role(db.Model):
'''
String representation of the Role. For human readability.
'''
return '<Role {role_name}>'.format(role_name=self.name)
return '<Role {}>'.format(self.name)
def add_permission(self, perm):
'''
@ -138,6 +138,19 @@ class User(UserMixin, db.Model):
cascade='save-update, merge, delete',
lazy='dynamic')
@property
def path(self):
return os.path.join(current_app.config['NOPAQUE_DATA_DIR'],
str(self.id))
@property
def password(self):
raise AttributeError('password is not a readable attribute')
@password.setter
def password(self, password):
self.password_hash = generate_password_hash(password)
def to_dict(self):
return {'id': self.id,
'role_id': self.role_id,
@ -145,28 +158,29 @@ class User(UserMixin, db.Model):
'email': self.email,
'last_seen': self.last_seen.timestamp(),
'member_since': self.member_since.timestamp(),
'username': self.username,
'settings': {'dark_mode': self.setting_dark_mode,
'job_status_mail_notifications':
self.setting_job_status_mail_notifications,
'job_status_site_notifications':
self.setting_job_status_site_notifications},
'username': self.username,
'corpora': {corpus.id: corpus.to_dict()
for corpus in self.corpora},
'jobs': {job.id: job.to_dict() for job in self.jobs},
'query_results': {query_result.id: query_result.to_dict()
for query_result in self.query_results}}
for query_result in self.query_results},
'role': self.role.to_dict()}
def __repr__(self):
'''
String representation of the User. For human readability.
'''
return '<User {username}>'.format(username=self.username)
return '<User {}>'.format(self.username)
def __init__(self, **kwargs):
super(User, self).__init__(**kwargs)
if self.role is None:
if self.email == current_app.config['ADMIN_EMAIL_ADRESS']:
if self.email == current_app.config['NOPAQUE_ADMIN']:
self.role = Role.query.filter_by(name='Administrator').first()
if self.role is None:
self.role = Role.query.filter_by(default=True).first()
@ -219,14 +233,6 @@ class User(UserMixin, db.Model):
db.session.add(user)
return True
@property
def password(self):
raise AttributeError('password is not a readable attribute')
@password.setter
def password(self, password):
self.password_hash = generate_password_hash(password)
def verify_password(self, password):
return check_password_hash(self.password_hash, password)
@ -243,17 +249,11 @@ class User(UserMixin, db.Model):
'''
return self.can(Permission.ADMIN)
def ping(self):
self.last_seen = datetime.utcnow()
db.session.add(self)
def delete(self):
'''
Delete the user and its corpora and jobs from database and filesystem.
'''
user_dir = os.path.join(current_app.config['DATA_DIR'],
str(self.id))
shutil.rmtree(user_dir, ignore_errors=True)
shutil.rmtree(self.path, ignore_errors=True)
db.session.delete(self)
@ -279,17 +279,32 @@ class JobInput(db.Model):
# Foreign keys
job_id = db.Column(db.Integer, db.ForeignKey('jobs.id'))
# Fields
dir = db.Column(db.String(255))
filename = db.Column(db.String(255))
@property
def download_url(self):
return url_for('jobs.download_job_input', job_id=self.job_id,
job_input_id=self.id)
@property
def path(self):
return os.path.join(self.job.path, self.filename)
@property
def url(self):
return url_for('jobs.job', job_id=self.job_id,
_anchor='job-{}-input-{}'.format(self.job_id, self.id))
def __repr__(self):
'''
String representation of the JobInput. For human readability.
'''
return '<JobInput {filename}>'.format(filename=self.filename)
return '<JobInput {}>'.format(self.filename)
def to_dict(self):
return {'id': self.id,
return {'download_url': self.download_url,
'url': self.url,
'id': self.id,
'job_id': self.job_id,
'filename': self.filename}
@ -304,17 +319,32 @@ class JobResult(db.Model):
# Foreign keys
job_id = db.Column(db.Integer, db.ForeignKey('jobs.id'))
# Fields
dir = db.Column(db.String(255))
filename = db.Column(db.String(255))
@property
def download_url(self):
return url_for('jobs.download_job_result', job_id=self.job_id,
job_result_id=self.id)
@property
def path(self):
return os.path.join(self.job.path, 'output', self.filename)
@property
def url(self):
return url_for('jobs.job', job_id=self.job_id,
_anchor='job-{}-result-{}'.format(self.job_id, self.id))
def __repr__(self):
'''
String representation of the JobResult. For human readability.
'''
return '<JobResult {filename}>'.format(filename=self.filename)
return '<JobResult {}>'.format(self.filename)
def to_dict(self):
return {'id': self.id,
return {'download_url': self.download_url,
'url': self.url,
'id': self.id,
'job_id': self.job_id,
'filename': self.filename}
@ -334,7 +364,6 @@ class Job(db.Model):
end_date = db.Column(db.DateTime())
mem_mb = db.Column(db.Integer)
n_cores = db.Column(db.Integer)
secure_filename = db.Column(db.String(32))
service = db.Column(db.String(64))
'''
' Service specific arguments as string list.
@ -349,25 +378,20 @@ class Job(db.Model):
cascade='save-update, merge, delete')
results = db.relationship('JobResult', backref='job', lazy='dynamic',
cascade='save-update, merge, delete')
notification_data = db.relationship('NotificationData',
cascade='save-update, merge, delete',
uselist=False,
back_populates='job') # One-to-One relationship
notification_email_data = db.relationship('NotificationEmailData',
cascade='save-update, merge, delete',
back_populates='job')
@property
def path(self):
return os.path.join(self.creator.path, 'jobs', str(self.id))
@property
def url(self):
return url_for('jobs.job', job_id=self.id)
def __repr__(self):
'''
String representation of the Job. For human readability.
'''
return '<Job {job_title}>'.format(job_title=self.title)
def create_secure_filename(self):
'''
Takes the job.title string nad cratesa a secure filename from this.
'''
self.secure_filename = secure_filename(self.title)
return '<Job {}>'.format(self.title)
def delete(self):
'''
@ -383,11 +407,7 @@ class Job(db.Model):
db.session.commit()
sleep(1)
db.session.refresh(self)
job_dir = os.path.join(current_app.config['DATA_DIR'],
str(self.user_id),
'jobs',
str(self.id))
shutil.rmtree(job_dir, ignore_errors=True)
shutil.rmtree(self.path, ignore_errors=True)
db.session.delete(self)
def restart(self):
@ -397,89 +417,27 @@ class Job(db.Model):
if self.status != 'failed':
raise Exception('Could not restart job: status is not "failed"')
job_dir = os.path.join(current_app.config['DATA_DIR'],
str(self.user_id),
'jobs',
str(self.id))
shutil.rmtree(os.path.join(job_dir, 'output'), ignore_errors=True)
shutil.rmtree(os.path.join(job_dir, 'pyflow.data'), ignore_errors=True)
shutil.rmtree(os.path.join(self.path, 'output'), ignore_errors=True)
shutil.rmtree(os.path.join(self.path, 'pyflow.data'), ignore_errors=True) # noqa
self.end_date = None
self.status = 'submitted'
def to_dict(self):
return {'id': self.id,
return {'url': self.url,
'id': self.id,
'user_id': self.user_id,
'creation_date': self.creation_date.timestamp(),
'description': self.description,
'end_date': (self.end_date.timestamp() if self.end_date else
None),
'inputs': {input.id: input.to_dict() for input in self.inputs},
'mem_mb': self.mem_mb,
'n_cores': self.n_cores,
'results': {result.id: result.to_dict()
for result in self.results},
'service': self.service,
'service_args': self.service_args,
'service_version': self.service_version,
'status': self.status,
'title': self.title}
class NotificationData(db.Model):
'''
Class to define notification data used for sending a notification mail with
nopaque_notify.
'''
__tablename__ = 'notification_data'
# Primary key
id = db.Column(db.Integer, primary_key=True)
# Foreign Key
job_id = db.Column(db.Integer, db.ForeignKey('jobs.id'))
# relationships
job = db.relationship('Job', back_populates='notification_data')
# Fields
notified_on = db.Column(db.String(16), default=None)
def __repr__(self):
'''
String representation of the NotificationData. For human readability.
'''
return '<NotificationData {id}>'.format(id=self.id)
def to_dict(self):
return {'id': self.id,
'job_id': self.job_id,
'job': self.job,
'notified': self.notified}
class NotificationEmailData(db.Model):
'''
Class to define data that will be used to send a corresponding Notification
via email.
'''
__tablename__ = 'notification_email_data'
# Primary Key
id = db.Column(db.Integer, primary_key=True)
# Foreign Key
job_id = db.Column(db.Integer, db.ForeignKey('jobs.id'))
# relationships
job = db.relationship('Job', back_populates='notification_email_data')
notify_status = db.Column(db.String(16), default=None)
creation_date = db.Column(db.DateTime(), default=datetime.utcnow)
def __repr__(self):
'''
String representation of the NotificationEmailData. For human readability.
'''
return '<NotificationData {id}>'.format(id=self.id)
def to_dict(self):
return {'id': self.id,
'job_id': self.job_id,
'job': self.job,
'notify_status': self.notify_status,
'creation_date': self.creation_date}
'title': self.title,
'inputs': {input.id: input.to_dict() for input in self.inputs},
'results': {result.id: result.to_dict()
for result in self.results}}
class CorpusFile(db.Model):
@ -496,7 +454,6 @@ class CorpusFile(db.Model):
author = db.Column(db.String(255))
booktitle = db.Column(db.String(255))
chapter = db.Column(db.String(255))
dir = db.Column(db.String(255))
editor = db.Column(db.String(255))
filename = db.Column(db.String(255))
institution = db.Column(db.String(255))
@ -507,21 +464,33 @@ class CorpusFile(db.Model):
school = db.Column(db.String(255))
title = db.Column(db.String(255))
@property
def download_url(self):
return url_for('corpora.download_corpus_file',
corpus_id=self.corpus_id, corpus_file_id=self.id)
@property
def path(self):
return os.path.join(self.corpus.path, self.filename)
@property
def url(self):
return url_for('corpora.corpus_file', corpus_id=self.corpus_id,
corpus_file_id=self.id)
def delete(self):
corpus_file_path = os.path.join(current_app.config['DATA_DIR'],
str(self.corpus.user_id),
'corpora',
str(self.corpus_id),
self.filename)
try:
os.remove(corpus_file_path)
os.remove(self.path)
except OSError:
logging.error('Removing {} led to an OSError!'.format(self.path))
pass
db.session.delete(self)
self.corpus.status = 'unprepared'
def to_dict(self):
return {'id': self.id,
return {'download_url': self.download_url,
'url': self.url,
'id': self.id,
'corpus_id': self.corpus_id,
'address': self.address,
'author': self.author,
@ -553,37 +522,48 @@ class Corpus(db.Model):
description = db.Column(db.String(255))
last_edited_date = db.Column(db.DateTime(), default=datetime.utcnow)
max_nr_of_tokens = db.Column(db.BigInteger, default=2147483647)
status = db.Column(db.String(16))
status = db.Column(db.String(16), default='unprepared')
title = db.Column(db.String(32))
archive_file = db.Column(db.String(255))
# Relationships
files = db.relationship('CorpusFile', backref='corpus', lazy='dynamic',
cascade='save-update, merge, delete')
@property
def analysis_url(self):
return url_for('corpora.analyse_corpus', corpus_id=self.id)
@property
def path(self):
return os.path.join(self.creator.path, 'corpora', str(self.id))
@property
def url(self):
return url_for('corpora.corpus', corpus_id=self.id)
def to_dict(self):
return {'id': self.id,
return {'analysis_url': self.analysis_url,
'url': self.url,
'id': self.id,
'user_id': self.user_id,
'creation_date': self.creation_date.timestamp(),
'current_nr_of_tokens': self.current_nr_of_tokens,
'description': self.description,
'status': self.status,
'last_edited_date': self.last_edited_date.timestamp(),
'max_nr_of_tokens': self.max_nr_of_tokens,
'title': self.title,
'files': {file.id: file.to_dict() for file in self.files}}
def build(self):
corpus_dir = os.path.join(current_app.config['DATA_DIR'],
str(self.user_id),
'corpora',
str(self.id))
output_dir = os.path.join(corpus_dir, 'merged')
output_dir = os.path.join(self.path, 'merged')
shutil.rmtree(output_dir, ignore_errors=True)
os.mkdir(output_dir)
master_element_tree = ET.ElementTree(
ET.fromstring('<corpus>\n</corpus>')
)
for corpus_file in self.files:
corpus_file_path = os.path.join(corpus_dir, corpus_file.filename)
element_tree = ET.parse(corpus_file_path)
element_tree = ET.parse(corpus_file.path)
text_node = element_tree.find('text')
text_node.set('address', corpus_file.address or "NULL")
text_node.set('author', corpus_file.author)
@ -597,7 +577,7 @@ class Corpus(db.Model):
text_node.set('publishing_year', str(corpus_file.publishing_year))
text_node.set('school', corpus_file.school or "NULL")
text_node.set('title', corpus_file.title)
element_tree.write(corpus_file_path)
element_tree.write(corpus_file.path)
master_element_tree.getroot().insert(1, text_node)
output_file = os.path.join(output_dir, 'corpus.vrt')
master_element_tree.write(output_file,
@ -607,18 +587,14 @@ class Corpus(db.Model):
self.status = 'submitted'
def delete(self):
corpus_dir = os.path.join(current_app.config['DATA_DIR'],
str(self.user_id),
'corpora',
str(self.id))
shutil.rmtree(corpus_dir, ignore_errors=True)
shutil.rmtree(self.path, ignore_errors=True)
db.session.delete(self)
def __repr__(self):
'''
String representation of the corpus. For human readability.
'''
return '<Corpus {corpus_title}>'.format(corpus_title=self.title)
return '<Corpus {}>'.format(self.title)
class QueryResult(db.Model):
@ -636,25 +612,39 @@ class QueryResult(db.Model):
query_metadata = db.Column(db.JSON())
title = db.Column(db.String(32))
@property
def download_url(self):
return url_for('corpora.download_query_result',
query_result_id=self.id)
@property
def path(self):
return os.path.join(
self.creator.path, 'query_results', str(self.id), self.filename)
@property
def url(self):
return url_for('corpora.query_result', query_result_id=self.id)
def delete(self):
query_result_dir = os.path.join(current_app.config['DATA_DIR'],
str(self.user_id),
'query_results',
str(self.id))
shutil.rmtree(query_result_dir, ignore_errors=True)
shutil.rmtree(self.path, ignore_errors=True)
db.session.delete(self)
def to_dict(self):
return {'id': self.id,
return {'download_url': self.download_url,
'url': self.url,
'id': self.id,
'user_id': self.user_id,
'corpus_title': self.query_metadata['corpus_name'],
'description': self.description,
'filename': self.filename,
'query': self.query_metadata['query'],
'query_metadata': self.query_metadata,
'title': self.title}
def __repr__(self):
'''
String representation of the CorpusAnalysisResult. For human readability.
String representation of the QueryResult. For human readability.
'''
return '<QueryResult {}>'.format(self.title)

150
web/app/query_results/views.py
View File

@ -1,150 +0,0 @@
from . import query_results
from . import tasks
from .. import db
from ..corpora.forms import DisplayOptionsForm, InspectDisplayOptionsForm
from ..models import QueryResult
from .forms import AddQueryResultForm
from flask import (abort, current_app, flash, make_response, redirect,
render_template, request, send_from_directory, url_for)
from flask_login import current_user, login_required
import json
import os
from jsonschema import validate
@query_results.route('/add', methods=['GET', 'POST'])
@login_required
def add_query_result():
'''
View to import a result as a json file.
'''
add_query_result_form = AddQueryResultForm(prefix='add-query-result-form')
if add_query_result_form.is_submitted():
if not add_query_result_form.validate():
return make_response(add_query_result_form.errors, 400)
query_result = QueryResult(
creator=current_user,
description=add_query_result_form.description.data,
filename=add_query_result_form.file.data.filename,
title=add_query_result_form.title.data
)
db.session.add(query_result)
db.session.commit()
# create paths to save the uploaded json file
query_result_dir = os.path.join(current_app.config['DATA_DIR'],
str(current_user.id),
'query_results',
str(query_result.id))
try:
os.makedirs(query_result_dir)
except Exception:
db.session.delete(query_result)
db.session.commit()
flash('Internal Server Error', 'error')
redirect_url = url_for('query_results.add_query_result')
return make_response({'redirect_url': redirect_url}, 500)
# save the uploaded file
query_result_file_path = os.path.join(query_result_dir,
query_result.filename)
add_query_result_form.file.data.save(query_result_file_path)
# parse json from file
with open(query_result_file_path, 'r') as file:
query_result_file_content = json.load(file)
# parse json schema
with open('app/static/json_schema/nopaque_cqi_py_results_schema.json', 'r') as file: # noqa
schema = json.load(file)
try:
# validate imported json file
validate(instance=query_result_file_content, schema=schema)
except Exception:
tasks.delete_query_result(query_result.id)
flash('Uploaded file is invalid', 'result')
redirect_url = url_for('query_results.add_query_result')
return make_response({'redirect_url': redirect_url}, 201)
query_result_file_content.pop('matches')
query_result_file_content.pop('cpos_lookup')
query_result.query_metadata = query_result_file_content
db.session.commit()
flash('Query result added!', 'result')
redirect_url = url_for('query_results.query_result',
query_result_id=query_result.id)
return make_response({'redirect_url': redirect_url}, 201)
return render_template('corpora/query_results/add_query_result.html.j2',
add_query_result_form=add_query_result_form,
title='Add query result')
@query_results.route('/<int:query_result_id>')
@login_required
def query_result(query_result_id):
query_result = QueryResult.query.get_or_404(query_result_id)
if not (query_result.creator == current_user
or current_user.is_administrator()):
abort(403)
return render_template('corpora/query_results/query_result.html.j2',
query_result=query_result,
title='Query result')
@query_results.route('/<int:query_result_id>/inspect')
@login_required
def inspect_query_result(query_result_id):
'''
View to inspect imported result file in a corpus analysis like interface
'''
query_result = QueryResult.query.get_or_404(query_result_id)
query_metadata = query_result.query_metadata
if not (query_result.creator == current_user
or current_user.is_administrator()):
abort(403)
display_options_form = DisplayOptionsForm(
prefix='display-options-form',
results_per_page=request.args.get('results_per_page', 30),
result_context=request.args.get('context', 20)
)
inspect_display_options_form = InspectDisplayOptionsForm(
prefix='inspect-display-options-form'
)
query_result_file_path = os.path.join(
current_app.config['DATA_DIR'],
str(current_user.id),
'query_results',
str(query_result.id),
query_result.filename
)
with open(query_result_file_path, 'r') as query_result_file:
query_result_file_content = json.load(query_result_file)
return render_template('corpora/query_results/inspect.html.j2',
display_options_form=display_options_form,
inspect_display_options_form=inspect_display_options_form,
query_result_file_content=query_result_file_content,
query_metadata=query_metadata,
title='Inspect query result')
@query_results.route('/<int:query_result_id>/delete')
@login_required
def delete_query_result(query_result_id):
query_result = QueryResult.query.get_or_404(query_result_id)
if not (query_result.creator == current_user
or current_user.is_administrator()):
abort(403)
tasks.delete_query_result(query_result_id)
flash('Query result deleted!', 'result')
return redirect(url_for('services.service', service="corpus_analysis"))
@query_results.route('/<int:query_result_id>/download')
@login_required
def download_query_result(query_result_id):
query_result = QueryResult.query.get_or_404(query_result_id)
if not (query_result.creator == current_user
or current_user.is_administrator()):
abort(403)
query_result_dir = os.path.join(current_app.config['DATA_DIR'],
str(current_user.id),
'query_results',
str(query_result.id))
return send_from_directory(as_attachment=True,
directory=query_result_dir,
filename=query_result.filename)

2
web/app/services/__init__.py
View File

@ -2,4 +2,4 @@ from flask import Blueprint
services = Blueprint('services', __name__)
from . import views # noqa
from . import views

63
web/app/services/views.py
View File

@ -1,5 +1,4 @@
from flask import (abort, current_app, flash, make_response, render_template,
url_for)
from flask import abort, flash, make_response, render_template, url_for
from flask_login import current_user, login_required
from werkzeug.utils import secure_filename
from . import services
@ -7,19 +6,20 @@ from .. import db
from ..jobs.forms import AddFileSetupJobForm, AddNLPJobForm, AddOCRJobForm
from ..models import Job, JobInput
import json
import logging
import os
SERVICES = {'corpus_analysis': {'name': 'Corpus analysis'},
'file-setup': {'name': 'File setup',
'resources': {'mem_mb': 4096, 'n_cores': 4},
'add_job_form': AddFileSetupJobForm},
'form': AddFileSetupJobForm},
'nlp': {'name': 'Natural Language Processing',
'resources': {'mem_mb': 4096, 'n_cores': 2},
'add_job_form': AddNLPJobForm},
'form': AddNLPJobForm},
'ocr': {'name': 'Optical Character Recognition',
'resources': {'mem_mb': 8192, 'n_cores': 4},
'add_job_form': AddOCRJobForm}}
'form': AddOCRJobForm}}
@services.route('/<service>', methods=['GET', 'POST'])
@ -30,54 +30,47 @@ def service(service):
if service == 'corpus_analysis':
return render_template('services/{}.html.j2'.format(service),
title=SERVICES[service]['name'])
add_job_form = SERVICES[service]['add_job_form'](prefix='add-job-form')
if add_job_form.is_submitted():
if not add_job_form.validate():
return make_response(add_job_form.errors, 400)
form = SERVICES[service]['form'](prefix='add-job-form')
if form.is_submitted():
if not form.validate():
return make_response(form.errors, 400)
service_args = []
if service == 'nlp':
service_args.append('-l {}'.format(add_job_form.language.data))
if add_job_form.check_encoding.data:
service_args.append('-l {}'.format(form.language.data))
if form.check_encoding.data:
service_args.append('--check-encoding')
if service == 'ocr':
service_args.append('-l {}'.format(add_job_form.language.data))
if add_job_form.binarization.data:
service_args.append('-l {}'.format(form.language.data))
if form.binarization.data:
service_args.append('--binarize')
job = Job(creator=current_user,
description=add_job_form.description.data,
description=form.description.data,
mem_mb=SERVICES[service]['resources']['mem_mb'],
n_cores=SERVICES[service]['resources']['n_cores'],
service=service, service_args=json.dumps(service_args),
service_version=add_job_form.version.data,
status='preparing', title=add_job_form.title.data)
if job.service != 'corpus_analysis':
job.create_secure_filename()
service_version=form.version.data,
status='preparing', title=form.title.data)
db.session.add(job)
db.session.commit()
relative_dir = os.path.join(str(job.user_id), 'jobs', str(job.id))
absolut_dir = os.path.join(current_app.config['DATA_DIR'],
relative_dir)
try:
os.makedirs(absolut_dir)
os.makedirs(job.path)
except OSError:
job.delete()
flash('Internal Server Error', 'job')
return make_response({'redirect_url': url_for('services.service',
service=service)},
500)
logging.error('Make dir {} led to an OSError!'.format(job.path))
db.session.delete(job)
db.session.commit()
flash('Internal Server Error', 'error')
return make_response(
{'redirect_url': url_for('.service', service=service)}, 500)
else:
for file in add_job_form.files.data:
for file in form.files.data:
filename = secure_filename(file.filename)
file.save(os.path.join(absolut_dir, filename))
job_input = JobInput(dir=relative_dir, filename=filename,
job=job)
job_input = JobInput(filename=filename, job=job)
file.save(job_input.path)
db.session.add(job_input)
job.status = 'submitted'
db.session.commit()
url = url_for('jobs.job', job_id=job.id)
flash('[<a href="{}">{}</a>] added'.format(url, job.title), 'job')
flash('Job "{}" added'.format(job.title), 'job')
return make_response(
{'redirect_url': url_for('jobs.job', job_id=job.id)}, 201)
return render_template('services/{}.html.j2'.format(service),
title=SERVICES[service]['name'],
add_job_form=add_job_form)
form=form, title=SERVICES[service]['name'])

2
web/app/settings/forms.py
View File

@ -35,7 +35,7 @@ class EditGeneralSettingsForm(FlaskForm):
'Benutzername',
validators=[DataRequired(),
Length(1, 64),
Regexp(current_app.config['ALLOWED_USERNAME_REGEX'],
Regexp(current_app.config['NOPAQUE_USERNAME_REGEX'],
message='Usernames must have only letters, numbers,'
' dots or underscores')]
)

19
web/app/settings/views.py
View File

@ -1,13 +1,9 @@
from flask import current_app, flash, redirect, render_template, url_for
from flask import flash, redirect, render_template, url_for
from flask_login import current_user, login_required, logout_user
from . import settings, tasks
from .forms import (ChangePasswordForm, EditGeneralSettingsForm,
EditNotificationSettingsForm)
from .. import db
from ..decorators import admin_required
from ..models import Role, User
import os
import uuid
@settings.route('/')
@ -26,8 +22,7 @@ def change_password():
flash('Your password has been updated.')
return redirect(url_for('.change_password'))
return render_template('settings/change_password.html.j2',
form=form,
title='Change password')
form=form, title='Change password')
@settings.route('/edit_general_settings', methods=['GET', 'POST'])
@ -40,12 +35,12 @@ def edit_general_settings():
current_user.username = form.username.data
db.session.commit()
flash('Your changes have been saved.')
return redirect(url_for('.edit_general_settings'))
form.dark_mode.data = current_user.setting_dark_mode
form.email.data = current_user.email
form.username.data = current_user.username
return render_template('settings/edit_general_settings.html.j2',
form=form,
title='General settings')
form=form, title='General settings')
@settings.route('/edit_notification_settings', methods=['GET', 'POST'])
@ -59,13 +54,13 @@ def edit_notification_settings():
form.job_status_site_notifications.data
db.session.commit()
flash('Your changes have been saved.')
return redirect(url_for('.edit_notification_settings'))
form.job_status_mail_notifications.data = \
current_user.setting_job_status_mail_notifications
form.job_status_site_notifications.data = \
current_user.setting_job_status_site_notifications
return render_template('settings/edit_notification_settings.html.j2',
form=form,
title='Notification settings')
form=form, title='Notification settings')
@settings.route('/delete')
@ -76,5 +71,5 @@ def delete():
"""
tasks.delete_user(current_user.id)
logout_user()
flash('Your account has been deleted!')
flash('Your account has been marked for deletion!')
return redirect(url_for('main.index'))

4
web/app/static/css/nopaque.css
View File

@ -8,6 +8,10 @@ main {
margin-top: 48px;
}
table.ressource-list tr {
cursor: pointer;
}
.parallax-container .parallax {
z-index: auto;
}

4
web/app/static/js/list.min.js vendored
View File

File diff suppressed because one or more lines are too long

4
web/app/static/js/modules/corpus_analysis/client/Client.js
View File

@ -136,7 +136,7 @@ class Client {
tmp_first_cpos.push(results.data.matches[dataIndex].c[0]);
tmp_last_cpos.push(results.data.matches[dataIndex].c[1]);
}
nopaque.socket.emit('corpus_analysis_get_match_with_full_context',
this.socket.emit('corpus_analysis_get_match_with_full_context',
{type: resultsType,
data_indexes: dataIndexes,
first_cpos: tmp_first_cpos,
@ -279,4 +279,4 @@ export {
Client,
ClientEventListener,
ListenerCallback,
};
};

239
web/app/static/js/nopaque.js
View File

@ -1,96 +1,138 @@
class AppClient {
constructor(currentUserId) {
this.socket = io({transports: ['websocket']});
this.users = {};
this.users.self = this.loadUser(currentUserId);
}
loadUser(userId) {
let user = new User();
this.users[userId] = user;
this.socket.on(`user_${userId}_init`, msg => user.init(JSON.parse(msg)));
this.socket.on(`user_${userId}_patch`, msg => user.patch(JSON.parse(msg)));
this.socket.emit('start_user_session', userId);
return user;
}
}
class User {
constructor() {
this.data = undefined;
this.eventListeners = {
corporaInit: [],
corporaPatch: [],
jobsInit: [],
jobsPatch: [],
queryResultsInit: [],
queryResultsPatch: []
};
}
init(data) {
this.data = data;
let listener;
for (listener of this.eventListeners.corporaInit) {
listener(this.data.corpora);
}
for (listener of this.eventListeners.jobsInit) {
listener(this.data.jobs);
}
for (listener of this.eventListeners.queryResultsInit) {
listener(this.data.query_results);
}
}
patch(patch) {
this.data = jsonpatch.apply_patch(this.data, patch);
let corporaPatch = patch.filter(operation => operation.path.startsWith("/corpora"));
let jobsPatch = patch.filter(operation => operation.path.startsWith("/jobs"));
let queryResultsPatch = patch.filter(operation => operation.path.startsWith("/query_results"));
for (let listener of this.eventListeners.corporaPatch) {
if (corporaPatch.length > 0) {listener(corporaPatch);}
}
for (let listener of this.eventListeners.jobsPatch) {
if (jobsPatch.length > 0) {listener(jobsPatch);}
}
for (let listener of this.eventListeners.queryResultsPatch) {
if (queryResultsPatch.length > 0) {listener(queryResultsPatch);}
}
for (let operation of jobsPatch) {
if (operation.op !== 'replace') {continue;}
// Matches the only path that should be handled here: /jobs/{jobId}/status
if (/^\/jobs\/(\d+)\/status$/.test(operation.path)) {
let [match, jobId] = operation.path.match(/^\/jobs\/(\d+)\/status$/);
if (this.data.settings.job_status_site_notifications === "end" && !["complete", "failed"].includes(operation.value)) {continue;}
nopaque.flash(`[<a href="/jobs/${jobId}">${this.data.jobs[jobId].title}</a>] New status: ${operation.value}`, "job");
}
}
}
addEventListener(type, listener) {
switch (type) {
case 'corporaInit':
this.eventListeners.corporaInit.push(listener);
if (this.data !== undefined) {listener(this.data.corpora);}
break;
case 'corporaPatch':
this.eventListeners.corporaPatch.push(listener);
break;
case 'jobsInit':
this.eventListeners.jobsInit.push(listener);
if (this.data !== undefined) {listener(this.data.jobs);}
break;
case 'jobsPatch':
this.eventListeners.jobsPatch.push(listener);
break;
case 'queryResultsInit':
this.eventListeners.queryResultsInit.push(listener);
if (this.data !== undefined) {listener(this.data.query_results);}
break;
case 'queryResultsPatch':
this.eventListeners.queryResultsPatch.push(listener);
break;
default:
console.error(`Unknown event type: ${type}`);
}
}
}
/*
* The nopaque object is used as a namespace for nopaque specific functions and
* variables.
*/
var nopaque = {};
// User data
nopaque.user = {};
nopaque.user.settings = {};
nopaque.user.settings.darkMode = undefined;
nopaque.corporaSubscribers = [];
nopaque.jobsSubscribers = [];
nopaque.queryResultsSubscribers = [];
nopaque.flash = function(message, category) {
let toast;
let toastActionElement;
// Foreign user (user inspected with admin credentials) data
nopaque.foreignUser = {};
nopaque.foreignUser.isAuthenticated = undefined;
nopaque.foreignUser.settings = {};
nopaque.foreignUser.settings.darkMode = undefined;
nopaque.foreignCorporaSubscribers = [];
nopaque.foreignJobsSubscribers = [];
nopaque.foreignQueryResultsSubscribers = [];
switch (category) {
case "corpus":
message = `<i class="left material-icons">book</i>${message}`;
break;
case "error":
message = `<i class="left material-icons red-text">error</i>${message}`;
break;
case "job":
message = `<i class="left material-icons">work</i>${message}`;
break;
default:
message = `<i class="left material-icons">notifications</i>${message}`;
}
// nopaque functions
nopaque.socket = io({transports: ['websocket']});
// Add event handlers
nopaque.socket.on("user_data_stream_init", function(msg) {
nopaque.user = JSON.parse(msg);
for (let subscriber of nopaque.corporaSubscribers) {
subscriber._init(nopaque.user.corpora);
}
for (let subscriber of nopaque.jobsSubscribers) {
subscriber._init(nopaque.user.jobs);
}
for (let subscriber of nopaque.queryResultsSubscribers) {
subscriber._init(nopaque.user.query_results);
}
});
nopaque.socket.on("user_data_stream_update", function(msg) {
var patch;
patch = JSON.parse(msg);
nopaque.user = jsonpatch.apply_patch(nopaque.user, patch);
corpora_patch = patch.filter(operation => operation.path.startsWith("/corpora"));
jobs_patch = patch.filter(operation => operation.path.startsWith("/jobs"));
query_results_patch = patch.filter(operation => operation.path.startsWith("/query_results"));
for (let subscriber of nopaque.corporaSubscribers) {
subscriber._update(corpora_patch);
}
for (let subscriber of nopaque.jobsSubscribers) {
subscriber._update(jobs_patch);
}
for (let subscriber of nopaque.queryResultsSubscribers) {
subscriber._update(query_results_patch);
}
if (["all", "end"].includes(nopaque.user.settings.job_status_site_notifications)) {
for (operation of jobs_patch) {
/* "/jobs/{jobId}/..." -> ["{jobId}", ...] */
pathArray = operation.path.split("/").slice(2);
if (operation.op === "replace" && pathArray[1] === "status") {
if (nopaque.user.settings.job_status_site_notifications === "end" && !["complete", "failed"].includes(operation.value)) {continue;}
nopaque.flash(`[<a href="/jobs/${pathArray[0]}">${nopaque.user.jobs[pathArray[0]].title}</a>] New status: ${operation.value}`, "job");
}
}
}
});
nopaque.socket.on("foreign_user_data_stream_init", function(msg) {
nopaque.foreignUser = JSON.parse(msg);
for (let subscriber of nopaque.foreignCorporaSubscribers) {
subscriber._init(nopaque.foreignUser.corpora);
}
for (let subscriber of nopaque.foreignJobsSubscribers) {
subscriber._init(nopaque.foreignUser.jobs);
}
for (let subscriber of nopaque.foreignQueryResultsSubscribers) {
subscriber._init(nopaque.foreignUser.query_results);
}
});
nopaque.socket.on("foreign_user_data_stream_update", function(msg) {
var patch;
patch = JSON.parse(msg);
nopaque.foreignUser = jsonpatch.apply_patch(nopaque.foreignUser, patch);
corpora_patch = patch.filter(operation => operation.path.startsWith("/corpora"));
jobs_patch = patch.filter(operation => operation.path.startsWith("/jobs"));
query_results_patch = patch.filter(operation => operation.path.startsWith("/query_results"));
for (let subscriber of nopaque.foreignCorporaSubscribers) {subscriber._update(corpora_patch);}
for (let subscriber of nopaque.foreignJobsSubscribers) {subscriber._update(jobs_patch);}
for (let subscriber of nopaque.foreignQueryResultsSubscribers) {subscriber._update(query_results_patch);}
});
toast = M.toast({html: `<span>${message}</span>
<button data-action="close" class="btn-flat toast-action white-text">
<i class="material-icons">close</i>
</button>`});
toastActionElement = toast.el.querySelector('.toast-action[data-action="close"]');
toastActionElement.addEventListener('click', () => {toast.dismiss();});
};
nopaque.Forms = {};
nopaque.Forms.init = function() {
@ -163,30 +205,3 @@ nopaque.Forms.init = function() {
}
}
}
nopaque.flash = function(message, category) {
let toast;
let toastActionElement;
switch (category) {
case "corpus":
message = `<i class="left material-icons">book</i>${message}`;
break;
case "error":
message = `<i class="left material-icons red-text">error</i>${message}`;
break;
case "job":
message = `<i class="left material-icons">work</i>${message}`;
break;
default:
message = `<i class="left material-icons">notifications</i>${message}`;
}
toast = M.toast({html: `<span>${message}</span>
<button data-action="close" class="btn-flat toast-action white-text">
<i class="material-icons">close</i>
</button>`});
toastActionElement = toast.el.querySelector('.toast-action[data-action="close"]');
toastActionElement.addEventListener('click', () => {toast.dismiss();});
}

644
web/app/static/js/nopaque.lists.js
View File

@ -1,432 +1,258 @@
class RessourceList extends List {
constructor(idOrElement, subscriberList, type, options) {
if (!type || !["Corpus", "CorpusFile", "Job", "JobInput", "QueryResult", "User"].includes(type)) {
throw "Unknown Type!";
class RessourceList {
/* A wrapper class for the list.js list.
* This class is not meant to be used directly, instead it should be used as
* a template for concrete ressource list implementations.
*/
constructor(listElement, options = {}) {
if (listElement.dataset.userId) {
if (listElement.dataset.userId in nopaque.appClient.users) {
this.user = nopaque.appClient.users[listElement.dataset.userId];
} else {
console.error(`User not found: ${listElement.dataset.userId}`);
return;
}
} else {
this.user = nopaque.appClient.users.self;
}
this.list = new List(listElement, {...RessourceList.options, ...options});
this.valueNames = ['id'];
for (let element of this.list.valueNames) {
switch (typeof element) {
case 'object':
if (element.hasOwnProperty('name')) {this.valueNames.push(element.name);}
break;
case 'string':
this.valueNames.push(element);
break;
default:
console.error(`Unknown value name definition: ${element}`);
}
}
super(idOrElement, {...RessourceList.options['common'],
...RessourceList.options[type],
...(options ? options : {})});
if (subscriberList) {subscriberList.push(this);}
this.type = type;
}
_init(ressources) {
this.clear();
this._add(Object.values(ressources));
this.sort("creation_date", {order: "desc"});
init(ressources) {
this.list.clear();
this.add(Object.values(ressources));
this.list.sort('id', {order: 'desc'});
}
patch(patch) {
/*
* It's not possible to generalize a patch Handler for all type of
* ressources. So this method is meant to be an interface.
*/
console.error('patch method not implemented!');
}
_update(patch) {
let item, pathArray;
add(values) {
let ressources = Array.isArray(values) ? values : [values];
// Discard ressource values, that are not defined to be used in the list.
ressources = ressources.map(ressource => {
let cleanedRessource = {};
for (let [valueName, value] of Object.entries(ressource)) {
if (this.valueNames.includes(valueName)) {cleanedRessource[valueName] = value;}
}
return cleanedRessource;
});
// Set a callback function ('() => {return;}') to force List.js perform the
// add method asynchronous: https://listjs.com/api/#add
this.list.add(ressources, () => {return;});
}
remove(id) {
this.list.remove('id', id);
}
replace(id, valueName, newValue) {
if (this.valueNames.includes(valueName)) {
let item = this.list.get('id', id)[0];
item.values({[valueName]: newValue});
}
}
}
RessourceList.options = {page: 5, pagination: [{innerWindow: 4, outerWindow: 1}]};
class CorpusList extends RessourceList {
constructor(listElement, options = {}) {
super(listElement, {...CorpusList.options, ...options});
this.user.addEventListener('corporaInit', corpora => this.init(corpora));
this.user.addEventListener('corporaPatch', patch => this.patch(patch));
listElement.addEventListener('click', (event) => {this.onclick(event)});
}
onclick(event) {
let corpusId = event.target.closest('tr').dataset.id;
let actionButtonElement = event.target.closest('.action-button');
let action = actionButtonElement === null ? 'view' : actionButtonElement.dataset.action;
switch (action) {
case 'analyse':
window.location.href = nopaque.user.corpora[corpusId].analysis_url;
case 'delete':
let deleteModalHTML = `<div class="modal">
<div class="modal-content">
<h4>Confirm corpus deletion</h4>
<p>Do you really want to delete the corpus <b>${nopaque.user.corpora[corpusId].title}</b>? All files will be permanently deleted!</p>
</div>
<div class="modal-footer">
<a href="#!" class="btn modal-close waves-effect waves-light">Cancel</a>
<a class="btn modal-close red waves-effect waves-light" href="${nopaque.user.corpora[corpusId].url}/delete"><i class="material-icons left">delete</i>Delete</a>
</div>
</div>`;
let deleteModalParentElement = document.querySelector('main');
deleteModalParentElement.insertAdjacentHTML('beforeend', deleteModalHTML);
let deleteModalElement = deleteModalParentElement.lastChild;
let deleteModal = M.Modal.init(deleteModalElement, {onCloseEnd: () => {deleteModal.destroy(); deleteModalElement.remove();}});
deleteModal.open();
break;
case 'view':
// TODO: handle unprepared corpora
window.location.href = nopaque.user.corpora[corpusId].url;
break;
default:
console.error(`Unknown action: ${action}`);
break;
}
}
patch(patch) {
for (let operation of patch) {
/* "/{ressourceName}/{ressourceId}/..." -> ["{ressourceId}", "..."] */
pathArray = operation.path.split("/").slice(2);
switch(operation.op) {
case "add":
if (pathArray.includes("results")) {break;}
this._add([operation.value]);
case 'add':
// Matches the only paths that should be handled here: /corpora/{corpusId}
if (/^\/corpora\/(\d+)$/.test(operation.path)) {this.add(operation.value);}
break;
case "remove":
this.remove("id", pathArray[0]);
break;
case "replace":
item = this.get("id", pathArray[0])[0];
switch(pathArray[1]) {
case "status":
item.values({status: operation.value,
"analyse-link": ["analysing", "prepared", "start analysis"].includes(operation.value) ? `/corpora/${pathArray[0]}/analyse` : ""});
break;
default:
break;
case 'remove':
// See case 'add' ;)
if (/^\/corpora\/(\d+)$/.test(operation.path)) {
let [match, id] = operation.path.match(/^\/corpora\/(\d+)$/);
this.remove(corpusId);
}
break;
case 'replace':
// Matches the only paths that should be handled here: /corpora/{corpusId}/{status || description || title}
if (/^\/corpora\/(\d+)\/(status|description|title)$/.test(operation.path)) {
let [match, id, valueName] = operation.path.match(/^\/corpora\/(\d+)\/(status|description|title)$/);
this.replace(id, valueName, operation.value);
}
break;
default:
break;
}
}
}
}
CorpusList.options = {
item: `<tr>
<td><a class="btn-floating disabled"><i class="material-icons">book</i></a></td>
<td><b class="title"></b><br><i class="description"></i></td>
<td><span class="badge new status" data-badge-caption=""></span></td>
<td class="right-align">
<a class="action-button btn-floating red tooltipped waves-effect waves-light" data-action="delete" data-position="top" data-tooltip="Delete"><i class="material-icons">delete</i></a>
<a class="action-button btn-floating tooltipped waves-effect waves-light" data-action="analyse" data-position="top" data-tooltip="Analyse"><i class="material-icons">search</i></a>
<a class="action-button btn-floating tooltipped waves-effect waves-light" data-action="view" data-position="top" data-tooltip="View"><i class="material-icons">send</i></a>
</td>
</tr>`,
valueNames: [{data: ['id']}, {name: 'status', attr: 'data-status'}, 'description', 'title']
};
_add(values, callback) {
this.add(values.map(x => RessourceList.dataMappers[this.type](x)), callback);
// Initialize modal and tooltipped elements in list
M.AutoInit(this.listContainer);
class JobList extends RessourceList {
constructor(listElement, options = {}) {
super(listElement, {...JobList.options, ...options});
this.user.addEventListener('jobsInit', jobs => this.init(jobs));
this.user.addEventListener('jobsPatch', patch => this.patch(patch));
listElement.addEventListener('click', (event) => {this.onclick(event)});
}
onclick(event) {
let jobId = event.target.closest('tr').dataset.id;
let actionButtonElement = event.target.closest('.action-button');
let action = actionButtonElement === null ? 'view' : actionButtonElement.dataset.action;
switch (action) {
case 'delete':
let deleteModalHTML = `<div class="modal">
<div class="modal-content">
<h4>Confirm job deletion</h4>
<p>Do you really want to delete the job <b>${this.user.data.jobs[jobId].title}</b>? All files will be permanently deleted!</p>
</div>
<div class="modal-footer">
<a href="#!" class="btn modal-close waves-effect waves-light">Cancel</a>
<a class="btn modal-close red waves-effect waves-light" href="${this.user.data.jobs[jobId].url}/delete"><i class="material-icons left">delete</i>Delete</a>
</div>
</div>`;
let deleteModalParentElement = document.querySelector('main');
deleteModalParentElement.insertAdjacentHTML('beforeend', deleteModalHTML);
let deleteModalElement = deleteModalParentElement.lastChild;
let deleteModal = M.Modal.init(deleteModalElement, {onCloseEnd: () => {deleteModal.destroy(); deleteModalElement.remove();}});
deleteModal.open();
break;
case 'view':
window.location.href = this.user.data.jobs[jobId].url;
break;
default:
console.error(`Unknown action: "${action}"`);
break;
}
}
patch(patch) {
for (let operation of patch) {
switch(operation.op) {
case 'add':
// Matches the only paths that should be handled here: /jobs/{jobId}
if (/^\/jobs\/(\d+)$/.test(operation.path)) {this.add(operation.value);}
break;
case 'remove':
// See case add ;)
if (/^\/jobs\/(\d+)$/.test(operation.path)) {
let [match, id] = operation.path.match(/^\/jobs\/(\d+)$/);
this.remove(jobId);
}
break;
case 'replace':
// Matches the only paths that should be handled here: /jobs/{jobId}/{service || status || description || title}
if (/^\/jobs\/(\d+)\/(service|status|description|title)$/.test(operation.path)) {
let [match, id, valueName] = operation.path.match(/^\/jobs\/(\d+)\/(service|status|description|title)$/);
this.replace(id, valueName, operation.value);
}
break;
default:
break;
}
}
}
}
RessourceList.dataMappers = {
// A data mapper describes entitys rendered per row. One key value pair holds
// the data to be rendered in the list.js table. Key has to correspond
// with the ValueNames defined below in RessourceList.options ValueNames.
// Links are declared with double ticks(") around them. The key for links
// have to correspond with the class of an <a> element in the
// RessourceList.options item blueprint.
/* ### Corpus mapper ### */
Corpus: corpus => ({
creation_date: corpus.creation_date,
description: corpus.description,
id: corpus.id,
link: `/corpora/${corpus.id}`,
status: corpus.status,
title: corpus.title,
title1: corpus.title,
"analyse-link": ["analysing", "prepared", "start analysis"].includes(corpus.status) ? `/corpora/${corpus.id}/analyse` : "",
"delete-link": `/corpora/${corpus.id}/delete`,
"delete-modal": `delete-corpus-${corpus.id}-modal`,
"delete-modal-trigger": `delete-corpus-${corpus.id}-modal`,
}),
/* ### CorpusFile mapper ### TODO: replace delete-modal with delete-onclick */
CorpusFile: corpus_file => ({
author: corpus_file.author,
filename: corpus_file.filename,
link: `${corpus_file.corpus_id}/files/${corpus_file.id}`,
title: corpus_file.title,
title1: corpus_file.title,
"delete-link": `/corpora/${corpus_file.corpus_id}/files/${corpus_file.id}/delete`,
"delete-modal": `delete-corpus-file-${corpus_file.id}-modal`,
"delete-modal-trigger": `delete-corpus-file-${corpus_file.id}-modal`,
"download-link": `${corpus_file.corpus_id}/files/${corpus_file.id}/download`,
}),
/* ### Job mapper ### */
Job: job => ({
creation_date: job.creation_date,
description: job.description,
id: job.id,
link: `/jobs/${job.id}`,
service: job.service,
status: job.status,
title: job.title,
title1: job.title,
"delete-link": `/jobs/${job.id}/delete`,
"delete-modal": `delete-job-${job.id}-modal`,
"delete-modal-trigger": `delete-job-${job.id}-modal`,
}),
/* ### JobInput mapper ### */
JobInput: job_input => ({
filename: job_input.filename,
id: job_input.job_id,
"download-link": `${job_input.job_id}/inputs/${job_input.id}/download`
}),
/* ### QueryResult mapper ### */
QueryResult: query_result => ({
corpus_name: query_result.query_metadata.corpus_name,
description: query_result.description,
id: query_result.id,
link: `/corpora/result/${query_result.id}`,
query: query_result.query_metadata.query,
title: query_result.title,
"delete-link": `/corpora/result/${query_result.id}/delete`,
"delete-modal": `delete-query-result-${query_result.id}-modal`,
"delete-modal-trigger": `delete-query-result-${query_result.id}-modal`,
"inspect-link": `/corpora/result/${query_result.id}/inspect`,
}),
/* ### User mapper ### */
User: user => ({
confirmed: user.confirmed,
email: user.email,
id: user.id,
link: `users/${user.id}`,
role_id: user.role_id,
username: user.username,
username2: user.username,
"delete-link": `/admin/users/${user.id}/delete`,
"delete-modal": `delete-user-${user.id}-modal`,
"delete-modal-trigger": `delete-user-${user.id}-modal`,
}),
JobList.options = {
item: `<tr>
<td><a class="btn-floating disabled"><i class="material-icons service"></i></a></td>
<td><b class="title"></b><br><i class="description"></i></td>
<td><span class="badge new status" data-badge-caption=""></span></td>
<td class="right-align">
<a class="action-button btn-floating red tooltipped waves-effect waves-light" data-action="delete" data-position="top" data-tooltip="Delete"><i class="material-icons">delete</i></a>
<a class="action-button btn-floating tooltipped waves-effect waves-light" data-action="view" data-position="top" data-tooltip="View"><i class="material-icons">send</i></a>
</td>
</tr>`,
valueNames: [{data: ['id']}, {name: 'service', attr: 'data-service'}, {name: 'status', attr: 'data-status'}, 'description', 'title']
};
RessourceList.options = {
// common list.js options for 5 rows per page etc.
common: {
page: 5,
pagination: [
{
name: "paginationTop",
paginationClass: "paginationTop",
innerWindow: 4,
outerWindow: 1
},
{
paginationClass: "paginationBottom",
innerWindow: 4,
outerWindow: 1,
},
],
},
// extended list.js options for 10 rows per page etc.
extended: {
page: 10,
pagination: [
{
name: "paginationTop",
paginationClass: "paginationTop",
innerWindow: 8,
outerWindow: 1
},
{
paginationClass: "paginationBottom",
innerWindow: 8,
outerWindow: 1,
},
],
},
/* Type specific List.js options. Usually only "item" and "valueNames" gets
* defined here but it is possible to define other List.js options.
* item: https://listjs.com/api/#item
* valueNames: https://listjs.com/api/#valueNames
*/
Corpus: {
item: `<tr>
<td>
<a class="btn-floating disabled">
<i class="material-icons service">book</i>
</a>
</td>
<td>
<b class="title"></b><br>
<i class="description"></i>
</td>
<td>
<span class="badge new status" data-badge-caption=""></span>
</td>
<td>
<div class="right-align">
<a class="btn-floating modal-trigger red tooltipped waves-effect waves-light delete-modal-trigger" data-position="top" data-tooltip="Delete">
<i class="material-icons">delete</i>
</a>
<a class="btn-floating tooltipped waves-effect waves-light link" data-position="top" data-tooltip="Edit">
<i class="material-icons">edit</i>
</a>
<a class="btn-floating tooltipped waves-effect waves-light analyse-link" data-position="top" data-tooltip="Analyse">
<i class="material-icons">search</i>
</a>
</div>
<div class="modal delete-modal">
<div class="modal-content">
<h4>Confirm corpus deletion</h4>
<p>Do you really want to delete the corpus <b class="title1"></b>? All files will be permanently deleted!</p>
</div>
<div class="modal-footer">
<a href="#!" class="btn modal-close waves-effect waves-light">Cancel</a>
<a class="btn modal-close red waves-effect waves-light delete-link"><i class="material-icons left">delete</i>Delete</a>
</div>
</div>
</td>
</tr>`,
valueNames: [
"creation_date",
"description",
"title",
"title1",
{data: ["id"]},
{name: "analyse-link", attr: "href"},
{name: "delete-link", attr: "href"},
{name: "delete-modal-trigger", attr: "data-target"},
{name: "delete-modal", attr: "id"},
{name: "link", attr: "href"},
{name: "status", attr: "data-status"},
]
},
CorpusFile: {
item: `<tr>
<td class="filename" style="word-break: break-word;"></td>
<td class="author" style="word-break: break-word;"></td>
<td class="title" style="word-break: break-word;"></td>
<td>
<div class="right-align">
<a class="btn-floating modal-trigger red tooltipped waves-effect waves-light delete-modal-trigger" data-position="top" data-tooltip="Delete">
<i class="material-icons">delete</i>
</a>
<a class="btn-floating tooltipped waves-effect waves-light download-link" data-position="top" data-tooltip="Download">
<i class="material-icons">file_download</i>
</a>
<a class="btn-floating tooltipped waves-effect waves-light link" data-position="top" data-tooltip="Edit">
<i class="material-icons">edit</i>
</a>
</div>
<div class="modal delete-modal">
<div class="modal-content">
<h4>Confirm corpus file deletion</h4>
<p>Do you really want to delete the corpus file <b class="title1"></b>? It be permanently deleted!</p>
</div>
<div class="modal-footer">
<a href="#!" class="btn modal-close waves-effect waves-light">Cancel</a>
<a class="btn modal-close red waves-effect waves-light delete-link"><i class="material-icons left">delete</i>Delete</a>
</div>
</div>
</td>
</tr>`,
valueNames: [
"author",
"filename",
"title",
"title1",
{name: "delete-link", attr: "href"},
{name: "delete-modal-trigger", attr: "data-target"},
{name: "delete-modal", attr: "id"},
{name: "download-link", attr: "href"},
{name: "link", attr: "href"},
],
},
Job: {
item: `<tr>
<td>
<a class="btn-floating disabled">
<i class="material-icons service"></i>
</a>
</td>
<td>
<b class="title"></b><br>
<i class="description"></i>
</td>
<td>
<span class="badge new status" data-badge-caption=""></span>
</td>
<td>
<div class="right-align">
<a class="btn-floating modal-trigger red tooltipped waves-effect waves-light delete-modal-trigger" data-position="top" data-tooltip="Delete">
<i class="material-icons">delete</i>
</a>
<a class="btn-floating tooltipped waves-effect waves-light link" data-position="top" data-tooltip="Go to job">
<i class="material-icons">send</i>
</a>
</div>
<div class="modal delete-modal">
<div class="modal-content">
<h4>Confirm job deletion</h4>
<p>Do you really want to delete the job <b class="title1"></b>? All files will be permanently deleted!</p>
</div>
<div class="modal-footer">
<a href="#!" class="btn modal-close waves-effect waves-light">Cancel</a>
<a class="btn modal-close red waves-effect waves-light delete-link"><i class="material-icons left">delete</i>Delete</a>
</div>
</div>
</td>
</tr>`,
valueNames: [
"creation_date",
"description",
"title",
"title1",
{data: ["id"]},
{name: "delete-link", attr: "href"},
{name: "delete-modal-trigger", attr: "data-target"},
{name: "delete-modal", attr: "id"},
{name: "link", attr: "href"},
{name: "service", attr: "data-service"},
{name: "status", attr: "data-status"},
],
},
JobInput: {
item : `<tr>
<td class="filename"></td>
<td class="right-align">
<a class="btn-floating tooltipped waves-effect waves-light download-link" data-position="top" data-tooltip="Download">
<i class="material-icons">file_download</i>
</a>
</td>
</tr>`,
valueNames: [
"filename",
"id",
{name: "download-link", attr: "href"},
],
},
QueryResult: {
item: `<tr>
<td>
<b class="title"></b><br>
<i class="description"></i><br>
</td>
<td>
<span class="corpus_name"></span><br>
<span class="query"></span>
</td>
<td>
<div class="right-align">
<a class="btn-floating modal-trigger red tooltipped waves-effect waves-light delete-modal-trigger" data-position="top" data-tooltip="Delete">
<i class="material-icons">delete</i>
</a>
<a class="btn-floating tooltipped waves-effect waves-light link" data-position="top" data-tooltip="Info">
<i class="material-icons">info</i>
</a>
<a class="btn-floating tooltipped waves-effect waves-light inspect-link" data-position="top" data-tooltip="Analyse">
<i class="material-icons">search</i>
</a>
</div>
<div class="modal delete-modal">
<div class="modal-content">
<h4>Confirm query result deletion</h4>
<p>Do you really want to delete the query result <b class="title1"></b>? It will be permanently deleted!</p>
</div>
<div class="modal-footer">
<a href="#!" class="btn modal-close waves-effect waves-light">Cancel</a>
<a class="btn modal-close red waves-effect waves-light delete-link"><i class="material-icons left">delete</i>Delete</a>
</div>
</div>
</td>
</tr>`,
valueNames: [
"corpus_name",
"description",
"query",
"title",
"title2",
{data: ["id"]},
{name: "delete-link", attr: "href"},
{name: "delete-modal-trigger", attr: "data-target"},
{name: "delete-modal", attr: "id"},
{name: "inspect-link", attr: "href"},
{name: "link", attr: "href"},
],
},
User: {
item: `<tr>
<td class="username"></td>
<td class="email"></td>
<td class="role_id"></td>
<td class="confirmed"></td>
<td class="id"></td>
<td>
<div class="right-align">
<a class="btn-floating modal-trigger red tooltipped waves-effect waves-light delete-modal-trigger" data-position="top" data-tooltip="Delete">
<i class="material-icons">delete</i>
</a>
<a class="btn-floating tooltipped waves-effect waves-light link" data-position="top" data-tooltip="Go to user">
<i class="material-icons">send</i>
</a>
</div>
<div class="modal delete-modal">
<div class="modal-content">
<h4>Confirm corpus deletion</h4>
<p>Do you really want to delete the job <b class="title1"></b>? All files will be permanently deleted!</p>
</div>
<div class="modal-footer">
<a href="#!" class="btn modal-close waves-effect waves-light">Cancel</a>
<a class="btn modal-close red waves-effect waves-light delete-link"><i class="material-icons left">delete</i>Delete</a>
</div>
</div>
</td>
</tr>`,
valueNames: [
"username",
"username2",
"email",
"role_id",
"confirmed",
"id",
{name: "link", attr: "href"},
{name: "delete-link", attr: "href"},
{name: "delete-modal-trigger", attr: "data-target"},
{name: "delete-modal", attr: "id"},
],
},
class QueryResultList extends RessourceList {
constructor(listElement, options = {}) {
super(listElement, {...QueryResultList.options, ...options});
this.user.addEventListener('queryResultsInit', queryResults => this.init(queryResults));
this.user.addEventListener('queryResultsPatch', patch => this.init(patch));
}
}
QueryResultList.options = {
item: `<tr>
<td><b class="title"></b><br><i class="description"></i><br></td>
<td><span class="corpus_title"></span><br><span class="query"></span></td>
<td class="right-align">
<a class="action-button btn-floating red tooltipped waves-effect waves-light" data-action="delete" data-position="top" data-tooltip="Delete"><i class="material-icons">delete</i></a>
<a class="action-button btn-floating tooltipped waves-effect waves-light" data-action="view" data-position="top" data-tooltip="View"><i class="material-icons">send</i></a>
<a class="action-button btn-floating tooltipped waves-effect waves-light" data-action="analyse" data-position="top" data-tooltip="Analyse"><i class="material-icons">search</i></a>
</td>
</tr>`,
valueNames: [{data: ['id']}, 'corpus_title', 'description', 'query', 'title']
};
export { RessourceList, };

420
web/app/static/js/nopaque.lists.js.bak Normal file
View File

@ -0,0 +1,420 @@
class RessourceList extends List {
constructor(idOrElement, subscriberList, type, options) {
if (!type || !["Corpus", "CorpusFile", "Job", "JobInput", "QueryResult", "User"].includes(type)) {
throw "Unknown Type!";
}
super(idOrElement, {...RessourceList.options['common'],
...RessourceList.options[type],
...(options ? options : {})});
if (subscriberList) {subscriberList.push(this);}
this.type = type;
}
_init(ressources) {
this.clear();
this._add(Object.values(ressources));
this.sort("id", {order: "desc"});
}
_update(patch) {
let item, pathArray;
for (let operation of patch) {
/* "/{ressourceName}/{ressourceId}/..." -> ["{ressourceId}", "..."] */
pathArray = operation.path.split("/").slice(2);
switch(operation.op) {
case "add":
if (pathArray.includes("results")) {break;}
this._add([operation.value]);
break;
case "remove":
this.remove("id", pathArray[0]);
break;
case "replace":
item = this.get("id", pathArray[0])[0];
switch(pathArray[1]) {
case "status":
item.values({status: operation.value,
"analyse-link": ["analysing", "prepared", "start analysis"].includes(operation.value) ? `/corpora/${pathArray[0]}/analyse` : ""});
break;
default:
break;
}
default:
break;
}
}
}
_add(values, callback) {
this.add(values.map(x => RessourceList.dataMappers[this.type](x)), callback);
// Initialize modal and tooltipped elements in list
M.AutoInit(this.listContainer);
}
}
RessourceList.dataMappers = {
// A data mapper describes entitys rendered per row. One key value pair holds
// the data to be rendered in the list.js table. Key has to correspond
// with the ValueNames defined below in RessourceList.options ValueNames.
// Links are declared with double ticks(") around them. The key for links
// have to correspond with the class of an <a> element in the
// RessourceList.options item blueprint.
/* ### Corpus mapper ### */
Corpus: corpus => ({
creation_date: corpus.creation_date,
description: corpus.description,
id: corpus.id,
link: `/corpora/${corpus.id}`,
status: corpus.status,
title: corpus.title,
title1: corpus.title,
"analyse-link": ["analysing", "prepared", "start analysis"].includes(corpus.status) ? `/corpora/${corpus.id}/analyse` : "",
"delete-link": `/corpora/${corpus.id}/delete`,
"delete-modal": `delete-corpus-${corpus.id}-modal`,
"delete-modal-trigger": `delete-corpus-${corpus.id}-modal`,
}),
/* ### CorpusFile mapper ### TODO: replace delete-modal with delete-onclick */
CorpusFile: corpus_file => ({
author: corpus_file.author,
filename: corpus_file.filename,
id: corpus_file.id,
link: `${corpus_file.corpus_id}/files/${corpus_file.id}`,
"publishing-year": corpus_file.publishing_year,
title: corpus_file.title,
title1: corpus_file.title,
"delete-link": `/corpora/${corpus_file.corpus_id}/files/${corpus_file.id}/delete`,
"delete-modal": `delete-corpus-file-${corpus_file.id}-modal`,
"delete-modal-trigger": `delete-corpus-file-${corpus_file.id}-modal`,
"download-link": `${corpus_file.corpus_id}/files/${corpus_file.id}/download`,
}),
/* ### Job mapper ### */
Job: job => ({
creation_date: job.creation_date,
description: job.description,
id: job.id,
link: `/jobs/${job.id}`,
service: job.service.name,
status: job.status,
title: job.title,
title1: job.title,
"delete-link": `/jobs/${job.id}/delete`,
"delete-modal": `delete-job-${job.id}-modal`,
"delete-modal-trigger": `delete-job-${job.id}-modal`,
}),
/* ### JobInput mapper ### */
JobInput: job_input => ({
filename: job_input.filename,
id: job_input.job_id,
"download-link": `${job_input.job_id}/inputs/${job_input.id}/download`
}),
/* ### QueryResult mapper ### */
QueryResult: query_result => ({
corpus_name: query_result.query_metadata.corpus_name,
description: query_result.description,
id: query_result.id,
link: `/corpora/result/${query_result.id}`,
query: query_result.query_metadata.query,
title: query_result.title,
"delete-link": `/corpora/result/${query_result.id}/delete`,
"delete-modal": `delete-query-result-${query_result.id}-modal`,
"delete-modal-trigger": `delete-query-result-${query_result.id}-modal`,
"inspect-link": `/corpora/result/${query_result.id}/inspect`,
}),
/* ### User mapper ### */
User: user => ({
confirmed: user.confirmed,
email: user.email,
id: user.id,
link: `users/${user.id}`,
role: user.role.name,
username: user.username,
username2: user.username,
"delete-link": `/admin/users/${user.id}/delete`,
"delete-modal": `delete-user-${user.id}-modal`,
"delete-modal-trigger": `delete-user-${user.id}-modal`,
}),
};
RessourceList.options = {
// common list.js options for 5 rows per page etc.
common: {page: 5, pagination: [{innerWindow: 4, outerWindow: 1}]},
// extended list.js options for 10 rows per page etc.
extended: {
page: 10,
pagination: [
{
name: "paginationTop",
paginationClass: "paginationTop",
innerWindow: 8,
outerWindow: 1
},
{
paginationClass: "paginationBottom",
innerWindow: 8,
outerWindow: 1,
},
],
},
/* Type specific List.js options. Usually only "item" and "valueNames" gets
* defined here but it is possible to define other List.js options.
* item: https://listjs.com/api/#item
* valueNames: https://listjs.com/api/#valueNames
*/
Corpus: {
item: `<tr>
<td>
<a class="btn-floating disabled">
<i class="material-icons service">book</i>
</a>
</td>
<td>
<b class="title"></b><br>
<i class="description"></i>
</td>
<td>
<span class="badge new status" data-badge-caption=""></span>
</td>
<td>
<div class="right-align">
<a class="btn-floating modal-trigger red tooltipped waves-effect waves-light delete-modal-trigger" data-position="top" data-tooltip="Delete">
<i class="material-icons">delete</i>
</a>
<a class="btn-floating tooltipped waves-effect waves-light link" data-position="top" data-tooltip="Edit">
<i class="material-icons">edit</i>
</a>
<a class="btn-floating tooltipped waves-effect waves-light analyse-link" data-position="top" data-tooltip="Analyse">
<i class="material-icons">search</i>
</a>
</div>
<div class="modal delete-modal">
<div class="modal-content">
<h4>Confirm corpus deletion</h4>
<p>Do you really want to delete the corpus <b class="title1"></b>? All files will be permanently deleted!</p>
</div>
<div class="modal-footer">
<a href="#!" class="btn modal-close waves-effect waves-light">Cancel</a>
<a class="btn modal-close red waves-effect waves-light delete-link"><i class="material-icons left">delete</i>Delete</a>
</div>
</div>
</td>
</tr>`,
valueNames: [
"creation_date",
"description",
"title",
"title1",
{data: ["id"]},
{name: "analyse-link", attr: "href"},
{name: "delete-link", attr: "href"},
{name: "delete-modal-trigger", attr: "data-target"},
{name: "delete-modal", attr: "id"},
{name: "link", attr: "href"},
{name: "status", attr: "data-status"},
]
},
CorpusFile: {
item: `<tr>
<td class="filename" style="word-break: break-word;"></td>
<td class="author" style="word-break: break-word;"></td>
<td class="title" style="word-break: break-word;"></td>
<td class="publishing-year" style="word-break: break-word;"></td>
<td>
<div class="right-align">
<a class="btn-floating modal-trigger red tooltipped waves-effect waves-light delete-modal-trigger" data-position="top" data-tooltip="Delete">
<i class="material-icons">delete</i>
</a>
<a class="btn-floating tooltipped waves-effect waves-light download-link" data-position="top" data-tooltip="Download">
<i class="material-icons">file_download</i>
</a>
<a class="btn-floating tooltipped waves-effect waves-light link" data-position="top" data-tooltip="Edit">
<i class="material-icons">edit</i>
</a>
</div>
<div class="modal delete-modal">
<div class="modal-content">
<h4>Confirm corpus file deletion</h4>
<p>Do you really want to delete the corpus file <b class="title1"></b>? It be permanently deleted!</p>
</div>
<div class="modal-footer">
<a href="#!" class="btn modal-close waves-effect waves-light">Cancel</a>
<a class="btn modal-close red waves-effect waves-light delete-link"><i class="material-icons left">delete</i>Delete</a>
</div>
</div>
</td>
</tr>`,
valueNames: [
"author",
"filename",
"publishing-year",
"title",
"title1",
{data: ["id"]},
{name: "delete-link", attr: "href"},
{name: "delete-modal-trigger", attr: "data-target"},
{name: "delete-modal", attr: "id"},
{name: "download-link", attr: "href"},
{name: "link", attr: "href"},
],
},
Job: {
item: `<tr>
<td>
<a class="btn-floating disabled">
<i class="material-icons service"></i>
</a>
</td>
<td>
<b class="title"></b><br>
<i class="description"></i>
</td>
<td>
<span class="badge new status" data-badge-caption=""></span>
</td>
<td>
<div class="right-align">
<a class="btn-floating modal-trigger red tooltipped waves-effect waves-light delete-modal-trigger" data-position="top" data-tooltip="Delete">
<i class="material-icons">delete</i>
</a>
<a class="btn-floating tooltipped waves-effect waves-light link" data-position="top" data-tooltip="Go to job">
<i class="material-icons">send</i>
</a>
</div>
<div class="modal delete-modal">
<div class="modal-content">
<h4>Confirm job deletion</h4>
<p>Do you really want to delete the job <b class="title1"></b>? All files will be permanently deleted!</p>
</div>
<div class="modal-footer">
<a href="#!" class="btn modal-close waves-effect waves-light">Cancel</a>
<a class="btn modal-close red waves-effect waves-light delete-link"><i class="material-icons left">delete</i>Delete</a>
</div>
</div>
</td>
</tr>`,
valueNames: [
"creation_date",
"description",
"title",
"title1",
{data: ["id"]},
{name: "delete-link", attr: "href"},
{name: "delete-modal-trigger", attr: "data-target"},
{name: "delete-modal", attr: "id"},
{name: "link", attr: "href"},
{name: "service", attr: "data-service"},
{name: "status", attr: "data-status"},
],
},
JobInput: {
item : `<tr>
<td class="filename"></td>
<td class="right-align">
<a class="btn-floating tooltipped waves-effect waves-light download-link" data-position="top" data-tooltip="Download">
<i class="material-icons">file_download</i>
</a>
</td>
</tr>`,
valueNames: [
"filename",
"id",
{name: "download-link", attr: "href"},
],
},
QueryResult: {
item: `<tr>
<td>
<b class="title"></b><br>
<i class="description"></i><br>
</td>
<td>
<span class="corpus_name"></span><br>
<span class="query"></span>
</td>
<td>
<div class="right-align">
<a class="btn-floating modal-trigger red tooltipped waves-effect waves-light delete-modal-trigger" data-position="top" data-tooltip="Delete">
<i class="material-icons">delete</i>
</a>
<a class="btn-floating tooltipped waves-effect waves-light link" data-position="top" data-tooltip="Info">
<i class="material-icons">info</i>
</a>
<a class="btn-floating tooltipped waves-effect waves-light inspect-link" data-position="top" data-tooltip="Analyse">
<i class="material-icons">search</i>
</a>
</div>
<div class="modal delete-modal">
<div class="modal-content">
<h4>Confirm query result deletion</h4>
<p>Do you really want to delete the query result <b class="title1"></b>? It will be permanently deleted!</p>
</div>
<div class="modal-footer">
<a href="#!" class="btn modal-close waves-effect waves-light">Cancel</a>
<a class="btn modal-close red waves-effect waves-light delete-link"><i class="material-icons left">delete</i>Delete</a>
</div>
</div>
</td>
</tr>`,
valueNames: [
"corpus_name",
"description",
"query",
"title",
"title2",
{data: ["id"]},
{name: "delete-link", attr: "href"},
{name: "delete-modal-trigger", attr: "data-target"},
{name: "delete-modal", attr: "id"},
{name: "inspect-link", attr: "href"},
{name: "link", attr: "href"},
],
},
User: {
item: `<tr>
<td class="id"></td>
<td class="username"></td>
<td class="email"></td>
<td class="role"></td>
<td>
<div class="right-align">
<a class="btn-floating modal-trigger red tooltipped waves-effect waves-light delete-modal-trigger" data-position="top" data-tooltip="Delete">
<i class="material-icons">delete</i>
</a>
<a class="btn-floating tooltipped waves-effect waves-light link" data-position="top" data-tooltip="Go to user">
<i class="material-icons">send</i>
</a>
</div>
<div class="modal delete-modal">
<div class="modal-content">
<h4>Confirm corpus deletion</h4>
<p>Do you really want to delete the job <b class="title1"></b>? All files will be permanently deleted!</p>
</div>
<div class="modal-footer">
<a href="#!" class="btn modal-close waves-effect waves-light">Cancel</a>
<a class="btn modal-close red waves-effect waves-light delete-link"><i class="material-icons left">delete</i>Delete</a>
</div>
</div>
</td>
</tr>`,
valueNames: [
"username",
"username2",
"email",
"role",
"id",
{name: "link", attr: "href"},
{name: "delete-link", attr: "href"},
{name: "delete-modal-trigger", attr: "data-target"},
{name: "delete-modal", attr: "id"},
],
},
};
export { RessourceList, };

104
web/app/static/js/nopaque/displays/CorpusDisplay.js Normal file
View File

@ -0,0 +1,104 @@
class CorpusDisplay extends RessourceDisplay {
constructor(displayElement) {
super(displayElement);
this.corpus = undefined;
this.user.eventListeners.corpus.addEventListener((eventType, payload) => this.eventHandler(eventType, payload), displayElement.dataset.corpusId);
}
init(corpus) {
this.corpus = corpus;
for (let exportCorpusTriggerElement of this.displayElement.querySelectorAll('.export-corpus-trigger')) {exportCorpusTriggerElement.addEventListener('click', () => this.requestCorpusExport());}
nopaque.appClient.socket.on(`export_corpus_${this.corpus.id}`, () => this.downloadCorpus());
this.setCreationDate(this.corpus.creation_date);
this.setDescription(this.corpus.description);
this.setLastEditedDate(this.corpus.last_edited_date);
this.setStatus(this.corpus.status);
this.setTitle(this.corpus.title);
this.setTokenRatio(this.corpus.current_nr_of_tokens, this.corpus.max_nr_of_tokens);
}
patch(patch) {
let re;
for (let operation of patch) {
switch(operation.op) {
case 'replace':
// Matches: /jobs/{this.job.id}/status
re = new RegExp('^/corpora/' + this.corpus.id + '/last_edited_date');
if (re.test(operation.path)) {this.setLastEditedDate(operation.value); break;}
// Matches: /jobs/{this.job.id}/status
re = new RegExp('^/corpora/' + this.corpus.id + '/status$');
if (re.test(operation.path)) {this.setStatus(operation.value); break;}
break;
default:
break;
}
}
}
requestCorpusExport() {
nopaque.appClient.socket.emit('export_corpus', this.corpus.id);
nopaque.flash('Preparing your corpus export...', 'corpus');
for (let exportCorpusTriggerElement of this.displayElement.querySelectorAll('.export-corpus-trigger')) {exportCorpusTriggerElement.classList.toggle('disabled', true);}
}
downloadCorpus() {
nopaque.flash('Corpus export is done. Your corpus download is ready!', 'corpus');
for (let exportCorpusTriggerElement of this.displayElement.querySelectorAll('.export-corpus-trigger')) {exportCorpusTriggerElement.classList.toggle('disabled', false);}
// Little trick to call the download view after ziping has finished
let fakeBtn = document.createElement('a');
fakeBtn.href = `/corpora/${this.corpus.id}/download`;
fakeBtn.click();
}
setTitle(title) {
for (let element of this.displayElement.querySelectorAll('.corpus-title')) {this.setElement(element, title);}
}
setTokenRatio(currentNrOfTokens, maxNrOfTokens) {
let tokenRatio = `${currentNrOfTokens}/${maxNrOfTokens}`;
for (let element of this.displayElement.querySelectorAll('.corpus-token-ratio')) {this.setElement(element, tokenRatio);}
}
setDescription(description) {
for (let element of this.displayElement.querySelectorAll('.corpus-description')) {this.setElement(element, description);}
}
setStatus(status) {
for (let element of this.displayElement.querySelectorAll('.analyse-corpus-trigger')) {
if (['analysing', 'prepared', 'start analysis'].includes(status)) {
element.classList.remove('disabled');
} else {
element.classList.add('disabled');
}
}
for (let element of this.displayElement.querySelectorAll('.build-corpus-trigger')) {
if (status === 'unprepared' && Object.values(this.corpus.files).length > 0) {
element.classList.remove('disabled');
} else {
element.classList.add('disabled');
}
}
for (let element of this.displayElement.querySelectorAll('.corpus-status')) {this.setElement(element, status);}
for (let exportCorpusTriggerElement of this.displayElement.querySelectorAll('.export-corpus-trigger')) {
exportCorpusTriggerElement.classList.toggle('disabled', !['prepared', 'start analysis', 'stop analysis'].includes(status));
}
for (let element of this.displayElement.querySelectorAll('.status')) {element.dataset.status = status;}
for (let element of this.displayElement.querySelectorAll('.status-spinner')) {
if (['submitted', 'queued', 'running', 'canceling', 'start analysis', 'stop analysis'].includes(status)) {
element.classList.remove('hide');
} else {
element.classList.add('hide');
}
}
}
setCreationDate(creationDateTimestamp) {
let creationDate = new Date(creationDateTimestamp * 1000).toLocaleString("en-US");
for (let element of this.displayElement.querySelectorAll('.corpus-creation-date')) {this.setElement(element, creationDate);}
}
setLastEditedDate(endDateTimestamp) {
let endDate = new Date(endDateTimestamp * 1000).toLocaleString("en-US");
for (let element of this.displayElement.querySelectorAll('.corpus-end-date')) {this.setElement(element, endDate);}
}
}

88
web/app/static/js/nopaque/displays/JobDisplay.js Normal file
View File

@ -0,0 +1,88 @@
class JobDisplay extends RessourceDisplay {
constructor(displayElement) {
super(displayElement);
this.job = undefined;
this.user.eventListeners.job.addEventListener((eventType, payload) => this.eventHandler(eventType, payload), displayElement.dataset.jobId);
}
init(job) {
this.job = job;
this.setCreationDate(this.job.creation_date);
this.setEndDate(this.job.creation_date);
this.setDescription(this.job.description);
this.setService(this.job.service);
this.setServiceArgs(this.job.service_args);
this.setServiceVersion(this.job.service_version);
this.setStatus(this.job.status);
this.setTitle(this.job.title);
}
patch(patch) {
let re;
for (let operation of patch) {
switch(operation.op) {
case 'replace':
// Matches: /jobs/{this.job.id}/status
re = new RegExp('^/jobs/' + this.job.id + '/end_date');
if (re.test(operation.path)) {this.setEndDate(operation.value); break;}
// Matches: /jobs/{this.job.id}/status
re = new RegExp('^/jobs/' + this.job.id + '/status$');
if (re.test(operation.path)) {this.setStatus(operation.value); break;}
break;
default:
break;
}
}
}
setTitle(title) {
for (let element of this.displayElement.querySelectorAll('.job-title')) {this.setElement(element, title);}
}
setDescription(description) {
for (let element of this.displayElement.querySelectorAll('.job-description')) {this.setElement(element, description);}
}
setStatus(status) {
for (let element of this.displayElement.querySelectorAll('.job-status')) {
this.setElement(element, status);
}
for (let element of this.displayElement.querySelectorAll('.status')) {element.dataset.status = status;}
for (let element of this.displayElement.querySelectorAll('.status-spinner')) {
if (['complete', 'failed'].includes(status)) {
element.classList.add('hide');
} else {
element.classList.remove('hide');
}
}
for (let element of this.displayElement.querySelectorAll('.restart-job-trigger')) {
if (['complete', 'failed'].includes(status)) {
element.classList.remove('hide');
} else {
element.classList.add('hide');
}
}
}
setCreationDate(creationDateTimestamp) {
let creationDate = new Date(creationDateTimestamp * 1000).toLocaleString("en-US");
for (let element of this.displayElement.querySelectorAll('.job-creation-date')) {this.setElement(element, creationDate);}
}
setEndDate(endDateTimestamp) {
let endDate = new Date(endDateTimestamp * 1000).toLocaleString("en-US");
for (let element of this.displayElement.querySelectorAll('.job-end-date')) {this.setElement(element, endDate);}
}
setService(service) {
for (let element of this.displayElement.querySelectorAll('.job-service')) {this.setElement(element, service);}
}
setServiceArgs(serviceArgs) {
for (let element of this.displayElement.querySelectorAll('.job-service-args')) {this.setElement(element, serviceArgs);}
}
setServiceVersion(serviceVersion) {
for (let element of this.displayElement.querySelectorAll('.job-service-version')) {this.setElement(element, serviceVersion);}
}
}

45
web/app/static/js/nopaque/displays/RessourceDisplay.js Normal file
View File

@ -0,0 +1,45 @@
class RessourceDisplay {
constructor(displayElement) {
if (displayElement.dataset.userId) {
if (displayElement.dataset.userId in nopaque.appClient.users) {
this.user = nopaque.appClient.users[displayElement.dataset.userId];
} else {
console.error(`User not found: ${displayElement.dataset.userId}`);
return;
}
} else {
this.user = nopaque.appClient.users.self;
}
this.displayElement = displayElement;
}
eventHandler(eventType, payload) {
switch (eventType) {
case 'init':
this.init(payload);
break;
case 'patch':
this.patch(payload);
break;
default:
console.log(`Unknown event type: ${eventType}`);
break;
}
}
init() {console.error('init method not implemented!');}
patch() {console.error('patch method not implemented!');}
setElement(element, value) {
switch (element.tagName) {
case 'INPUT':
element.value = value;
M.updateTextFields();
break;
default:
element.innerText = value;
break;
}
}
}

98
web/app/static/js/nopaque/lists/CorpusFileList.js Normal file
View File

@ -0,0 +1,98 @@
class CorpusFileList extends RessourceList {
constructor(listElement, options = {}) {
super(listElement, {...CorpusFileList.options, ...options});
this.corpus = undefined;
this.user.eventListeners.corpus.addEventListener((eventType, payload) => this.eventHandler(eventType, payload), listElement.dataset.corpusId);
}
init(corpus) {
this.corpus = corpus;
super.init(this.corpus.files);
}
onclick(event) {
let ressourceElement = event.target.closest('tr');
if (ressourceElement === null) {return;}
let corpusFileId = ressourceElement.dataset.id;
let actionButtonElement = event.target.closest('.action-button');
if (actionButtonElement === null) {return;}
let action = actionButtonElement === null ? 'view' : actionButtonElement.dataset.action;
switch (action) {
case 'delete':
let deleteModalHTML = `<div class="modal">
<div class="modal-content">
<h4>Confirm corpus deletion</h4>
<p>Do you really want to delete the corpus file <b>${this.corpus.files[corpusFileId].filename}</b>? It will be permanently deleted!</p>
</div>
<div class="modal-footer">
<a href="#!" class="btn modal-close waves-effect waves-light">Cancel</a>
<a class="btn modal-close red waves-effect waves-light" href="${this.corpus.files[corpusFileId].url}/delete"><i class="material-icons left">delete</i>Delete</a>
</div>
</div>`;
let deleteModalParentElement = document.querySelector('main');
deleteModalParentElement.insertAdjacentHTML('beforeend', deleteModalHTML);
let deleteModalElement = deleteModalParentElement.lastChild;
let deleteModal = M.Modal.init(deleteModalElement, {onCloseEnd: () => {deleteModal.destroy(); deleteModalElement.remove();}});
deleteModal.open();
break;
case 'download':
window.location.href = this.corpus.files[corpusFileId].download_url;
break;
case 'view':
if (corpusFileId !== '-1') {window.location.href = this.corpus.files[corpusFileId].url;}
break;
default:
console.error(`Unknown action: "${action}"`);
break;
}
}
patch(patch) {
let id, match, re, valueName;
for (let operation of patch) {
switch(operation.op) {
case 'add':
// Matches the only paths that should be handled here: /corpora/{this.corpus.id}/files/{corpusFileId}
re = new RegExp('^/corpora/' + this.corpus.id + '/files/(\\d+)$');
if (re.test(operation.path)) {this.add(operation.value);}
break;
case 'remove':
// See case add ;)
re = new RegExp('^/corpora/' + this.corpus.id + '/files/(\\d+)$');
if (re.test(operation.path)) {
[match, id] = operation.path.match(re);
this.remove(id);
}
break;
case 'replace':
// Matches the only paths that should be handled here: /corpora/{corpusId}/{status || description || title}
re = new RegExp('^/corpora/' + this.corpus.id + '/files/(\\d+)/(author|filename|publishing_year|title)$');
if (re.test(operation.path)) {
[match, id, valueName] = operation.path.match(re);
this.replace(id, valueName, operation.value);
}
break;
default:
break;
}
}
}
preprocessRessource(corpusFile) {
return {id: corpusFile.id, author: corpusFile.author, filename: corpusFile.filename, publishing_year: corpusFile.publishing_year, title: corpusFile.title};
}
}
CorpusFileList.options = {
item: `<tr>
<td><span class="filename"></span></td>
<td><span class="author"></span></td>
<td><span class="title"></span></td>
<td><span class="publishing_year"></span></td>
<td class="right-align">
<a class="action-button btn-floating red tooltipped waves-effect waves-light" data-action="delete" data-position="top" data-tooltip="Delete"><i class="material-icons">delete</i></a>
<a class="action-button btn-floating tooltipped waves-effect waves-light" data-action="download" data-position="top" data-tooltip="View"><i class="material-icons">file_download</i></a>
<a class="action-button btn-floating tooltipped waves-effect waves-light" data-action="view" data-position="top" data-tooltip="View"><i class="material-icons">send</i></a>
</td>
</tr>`,
valueNames: [{data: ['id']}, 'author', 'filename', 'publishing_year', 'title']
};

95
web/app/static/js/nopaque/lists/CorpusList.js Normal file
View File

@ -0,0 +1,95 @@
class CorpusList extends RessourceList {
constructor(listElement, options = {}) {
super(listElement, {...CorpusList.options, ...options});
this.corpora = undefined;
this.user.eventListeners.corpus.addEventListener((eventType, payload) => this.eventHandler(eventType, payload));
}
init(corpora) {
this.corpora = corpora;
super.init(corpora);
}
onclick(event) {
let ressourceElement = event.target.closest('tr');
if (ressourceElement === null) {return;}
let corpusId = ressourceElement.dataset.id;
let actionButtonElement = event.target.closest('.action-button');
let action = (actionButtonElement === null) ? 'view' : actionButtonElement.dataset.action;
switch (action) {
case 'delete':
let deleteModalHTML = `<div class="modal">
<div class="modal-content">
<h4>Confirm corpus deletion</h4>
<p>Do you really want to delete the corpus <b>${this.corpora[corpusId].title}</b>? All files will be permanently deleted!</p>
</div>
<div class="modal-footer">
<a href="#!" class="btn modal-close waves-effect waves-light">Cancel</a>
<a class="btn modal-close red waves-effect waves-light" href="${this.corpora[corpusId].url}/delete"><i class="material-icons left">delete</i>Delete</a>
</div>
</div>`;
let deleteModalParentElement = document.querySelector('main');
deleteModalParentElement.insertAdjacentHTML('beforeend', deleteModalHTML);
let deleteModalElement = deleteModalParentElement.lastChild;
let deleteModal = M.Modal.init(deleteModalElement, {onCloseEnd: () => {deleteModal.destroy(); deleteModalElement.remove();}});
deleteModal.open();
break;
case 'view':
if (corpusId !== '-1') {window.location.href = this.corpora[corpusId].url;}
break;
default:
console.error(`Unknown action: ${action}`);
break;
}
}
patch(patch) {
let id, match, re, valueName;
for (let operation of patch) {
switch(operation.op) {
case 'add':
// Matches the only paths that should be handled here: /corpora/{corpusId}
re = /^\/corpora\/(\d+)$/;
if (re.test(operation.path)) {this.add(operation.value);}
break;
case 'remove':
// See case 'add' ;)
re = /^\/corpora\/(\d+)$/;
if (re.test(operation.path)) {
[match, id] = operation.path.match(re);
this.remove(id);
}
break;
case 'replace':
// Matches the only paths that should be handled here: /corpora/{corpusId}/{status || description || title}
re = /^\/corpora\/(\d+)\/(status|description|title)$/;
if (re.test(operation.path)) {
[match, id, valueName] = operation.path.match(re);
this.replace(id, valueName, operation.value);
}
break;
default:
break;
}
}
}
preprocessRessource(corpus) {
return {id: corpus.id,
status: corpus.status,
description: corpus.description,
title: corpus.title};
}
}
CorpusList.options = {
item: `<tr>
<td><a class="btn-floating disabled"><i class="material-icons">book</i></a></td>
<td><b class="title"></b><br><i class="description"></i></td>
<td><span class="badge new status" data-badge-caption=""></span></td>
<td class="right-align">
<a class="action-button btn-floating red tooltipped waves-effect waves-light" data-action="delete" data-position="top" data-tooltip="Delete"><i class="material-icons">delete</i></a>
<a class="action-button btn-floating tooltipped waves-effect waves-light" data-action="view" data-position="top" data-tooltip="View"><i class="material-icons">send</i></a>
</td>
</tr>`,
valueNames: [{data: ['id']}, {name: 'status', attr: 'data-status'}, 'description', 'title']
};

42
web/app/static/js/nopaque/lists/JobInputList.js Normal file
View File

@ -0,0 +1,42 @@
class JobInputList extends RessourceList {
constructor(listElement, options = {}) {
super(listElement, {...JobInputList.options, ...options});
this.job = undefined;
this.user.eventListeners.job.addEventListener((eventType, payload) => this.eventHandler(eventType, payload), listElement.dataset.jobId);
}
init(job) {
this.job = job;
super.init(this.job.inputs);
}
onclick(event) {
let ressourceElement = event.target.closest('tr');
if (ressourceElement === null) {return;}
let jobInputId = ressourceElement.dataset.id;
let actionButtonElement = event.target.closest('.action-button');
if (actionButtonElement === null) {return;}
let action = actionButtonElement.dataset.action;
switch (action) {
case 'download':
window.location.href = this.job.inputs[jobInputId].download_url;
break;
default:
console.error(`Unknown action: "${action}"`);
break;
}
}
preprocessRessource(jobInput) {
return {id: jobInput.id, filename: jobInput.filename};
}
}
JobInputList.options = {
item: `<tr>
<td><span class="filename"></span></td>
<td class="right-align">
<a class="action-button btn-floating tooltipped waves-effect waves-light" data-action="download" data-position="top" data-tooltip="View"><i class="material-icons">file_download</i></a>
</td>
</tr>`,
valueNames: [{data: ['id']}, 'filename']
};

96
web/app/static/js/nopaque/lists/JobList.js Normal file
View File

@ -0,0 +1,96 @@
class JobList extends RessourceList {
constructor(listElement, options = {}) {
super(listElement, {...JobList.options, ...options});
this.jobs = undefined;
this.user.eventListeners.job.addEventListener((eventType, payload) => this.eventHandler(eventType, payload));
}
init(jobs) {
this.jobs = jobs;
super.init(jobs);
}
onclick(event) {
let ressourceElement = event.target.closest('tr');
if (ressourceElement === null) {return;}
let jobId = ressourceElement.dataset.id;
let actionButtonElement = event.target.closest('.action-button');
let action = actionButtonElement === null ? 'view' : actionButtonElement.dataset.action;
switch (action) {
case 'delete':
let deleteModalHTML = `<div class="modal">
<div class="modal-content">
<h4>Confirm job deletion</h4>
<p>Do you really want to delete the job <b>${this.user.data.jobs[jobId].title}</b>? All files will be permanently deleted!</p>
</div>
<div class="modal-footer">
<a href="#!" class="btn modal-close waves-effect waves-light">Cancel</a>
<a class="btn modal-close red waves-effect waves-light" href="${this.user.data.jobs[jobId].url}/delete"><i class="material-icons left">delete</i>Delete</a>
</div>
</div>`;
let deleteModalParentElement = document.querySelector('main');
deleteModalParentElement.insertAdjacentHTML('beforeend', deleteModalHTML);
let deleteModalElement = deleteModalParentElement.lastChild;
let deleteModal = M.Modal.init(deleteModalElement, {onCloseEnd: () => {deleteModal.destroy(); deleteModalElement.remove();}});
deleteModal.open();
break;
case 'view':
if (jobId !== '-1') {window.location.href = this.user.data.jobs[jobId].url;}
break;
default:
console.error(`Unknown action: "${action}"`);
break;
}
}
patch(patch) {
let id, match, re, valueName;
for (let operation of patch) {
switch(operation.op) {
case 'add':
// Matches the only paths that should be handled here: /jobs/{jobId}
re = /^\/jobs\/(\d+)$/;
if (re.test(operation.path)) {this.add(operation.value);}
break;
case 'remove':
// See case add ;)
re = /^\/jobs\/(\d+)$/;
if (re.test(operation.path)) {
[match, id] = operation.path.match(re);
this.remove(id);
}
break;
case 'replace':
// Matches the only paths that should be handled here: /jobs/{jobId}/{service || status || description || title}
re = /^\/jobs\/(\d+)\/(service|status|description|title)$/;
if (re.test(operation.path)) {
[match, id, valueName] = operation.path.match(re);
this.replace(id, valueName, operation.value);
}
break;
default:
break;
}
}
}
preprocessRessource(job) {
return {id: job.id,
service: job.service,
status: job.status,
description: job.description,
title: job.title};
}
}
JobList.options = {
item: `<tr>
<td><a class="btn-floating disabled"><i class="material-icons service"></i></a></td>
<td><b class="title"></b><br><i class="description"></i></td>
<td><span class="badge new status" data-badge-caption=""></span></td>
<td class="right-align">
<a class="action-button btn-floating red tooltipped waves-effect waves-light" data-action="delete" data-position="top" data-tooltip="Delete"><i class="material-icons">delete</i></a>
<a class="action-button btn-floating tooltipped waves-effect waves-light" data-action="view" data-position="top" data-tooltip="View"><i class="material-icons">send</i></a>
</td>
</tr>`,
valueNames: [{data: ['id']}, {name: 'service', attr: 'data-service'}, {name: 'status', attr: 'data-status'}, 'description', 'title']
};

72
web/app/static/js/nopaque/lists/JobResultList.js Normal file
View File

@ -0,0 +1,72 @@
class JobResultList extends RessourceList {
constructor(listElement, options = {}) {
super(listElement, {...JobResultList.options, ...options});
this.job = undefined;
this.user.eventListeners.job.addEventListener((eventType, payload) => this.eventHandler(eventType, payload), listElement.dataset.jobId);
}
init(job) {
this.job = job;
super.init(this.job.results);
}
onclick(event) {
let ressourceElement = event.target.closest('tr');
if (ressourceElement === null) {return;}
let jobResultId = ressourceElement.dataset.id;
let actionButtonElement = event.target.closest('.action-button');
if (actionButtonElement === null) {return;}
let action = actionButtonElement.dataset.action;
switch (action) {
case 'download':
window.location.href = this.job.results[jobResultId].download_url;
break;
default:
console.error(`Unknown action: "${action}"`);
break;
}
}
patch(patch) {
let re;
for (let operation of patch) {
switch(operation.op) {
case 'add':
// Matches the only paths that should be handled here: /jobs/{this.job.id}/results/{jobResultId}
re = new RegExp('^/jobs/' + this.job.id + '/results/(\\d+)$');
if (re.test(operation.path)) {this.add(operation.value);}
break;
default:
break;
}
}
}
preprocessRessource(jobResult) {
let description;
if (jobResult.filename.endsWith('.pdf.zip')) {
description = 'PDF files with text layer';
} else if (jobResult.filename.endsWith('.txt.zip')) {
description = 'Raw text files';
} else if (jobResult.filename.endsWith('.vrt.zip')) {
description = 'VRT compliant files including the NLP data';
} else if (jobResult.filename.endsWith('.xml.zip')) {
description = 'TEI compliant files';
} else if (jobResult.filename.endsWith('.poco.zip')) {
description = 'HOCR and image files for post correction (PoCo)';
} else {
description = 'All result files created during this job';
}
return {id: jobResult.id, description: description, filename: jobResult.filename};
}
}
JobResultList.options = {
item: `<tr>
<td><span class="description"></span></td>
<td><span class="filename"></span></td>
<td class="right-align">
<a class="action-button btn-floating tooltipped waves-effect waves-light" data-action="download" data-position="top" data-tooltip="View"><i class="material-icons">file_download</i></a>
</td>
</tr>`,
valueNames: [{data: ['id']}, 'description', 'filename']
};

95
web/app/static/js/nopaque/lists/QueryResultList.js Normal file
View File

@ -0,0 +1,95 @@
class QueryResultList extends RessourceList {
constructor(listElement, options = {}) {
super(listElement, {...QueryResultList.options, ...options});
this.queryResults = undefined;
this.user.eventListeners.queryResult.addEventListener((eventType, payload) => this.eventHandler(eventType, payload));
}
init(queryResults) {
this.queryResults = queryResults;
super.init(queryResults);
}
onclick(event) {
let ressourceElement = event.target.closest('tr');
if (ressourceElement === null) {return;}
let queryResultId = ressourceElement.dataset.id;
let actionButtonElement = event.target.closest('.action-button');
let action = actionButtonElement === null ? 'view' : actionButtonElement.dataset.action;
switch (action) {
case 'delete':
let deleteModalHTML = `<div class="modal">
<div class="modal-content">
<h4>Confirm query result deletion</h4>
<p>Do you really want to delete the query result <b>${this.user.data.query_results[queryResultId].title}</b>? It will be permanently deleted!</p>
</div>
<div class="modal-footer">
<a href="#!" class="btn modal-close waves-effect waves-light">Cancel</a>
<a class="btn modal-close red waves-effect waves-light" href="${this.user.data.query_results[queryResultId].url}/delete"><i class="material-icons left">delete</i>Delete</a>
</div>
</div>`;
let deleteModalParentElement = document.querySelector('main');
deleteModalParentElement.insertAdjacentHTML('beforeend', deleteModalHTML);
let deleteModalElement = deleteModalParentElement.lastChild;
let deleteModal = M.Modal.init(deleteModalElement, {onCloseEnd: () => {deleteModal.destroy(); deleteModalElement.remove();}});
deleteModal.open();
break;
case 'view':
if (queryResultId !== '-1') {window.location.href = this.user.data.query_results[queryResultId].url;}
break;
default:
console.error(`Unknown action: "${action}"`);
break;
}
}
patch(patch) {
let id, match, re, valueName;
for (let operation of patch) {
switch(operation.op) {
case 'add':
// Matches the only paths that should be handled here: /jobs/{jobId}
re = /^\/query_results\/(\d+)$/;
if (re.test(operation.path)) {this.add(operation.value);}
break;
case 'remove':
// See case add ;)
re = /^\/query_results\/(\d+)$/;
if (re.test(operation.path)) {
[match, id] = operation.path.match(re);
this.remove(id);
}
break;
case 'replace':
// Matches the only paths that should be handled here: /jobs/{jobId}/{service || status || description || title}
re = /^\/query_results\/(\d+)\/(corpus_title|description|query|title)$/;
if (re.test(operation.path)) {
[match, id, valueName] = operation.path.match(re);
this.replace(id, valueName, operation.value);
}
break;
default:
break;
}
}
}
preprocessRessource(queryResult) {
return {id: queryResult.id,
corpus_title: queryResult.corpus_title,
description: queryResult.description,
query: queryResult.query,
title: queryResult.title};
}
}
QueryResultList.options = {
item: `<tr>
<td><b class="title"></b><br><i class="description"></i><br></td>
<td><span class="corpus_title"></span><br><span class="query"></span></td>
<td class="right-align">
<a class="action-button btn-floating red tooltipped waves-effect waves-light" data-action="delete" data-position="top" data-tooltip="Delete"><i class="material-icons">delete</i></a>
<a class="action-button btn-floating tooltipped waves-effect waves-light" data-action="view" data-position="top" data-tooltip="View"><i class="material-icons">send</i></a>
</td>
</tr>`,
valueNames: [{data: ['id']}, 'corpus_title', 'description', 'query', 'title']
};

98
web/app/static/js/nopaque/lists/RessourceList.js Normal file
View File

@ -0,0 +1,98 @@
class RessourceList {
/* A wrapper class for the list.js list.
* This class is not meant to be used directly, instead it should be used as
* a base class for concrete ressource list implementations.
*/
constructor(listElement, options = {}) {
if (listElement.dataset.userId) {
if (listElement.dataset.userId in nopaque.appClient.users) {
this.user = nopaque.appClient.users[listElement.dataset.userId];
} else {
console.error(`User not found: ${listElement.dataset.userId}`);
return;
}
} else {
this.user = nopaque.appClient.users.self;
}
this.list = new List(listElement, {...RessourceList.options, ...options});
this.list.list.innerHTML = `<tr>
<td class="row" colspan="100%">
<div class="col s12">&nbsp;</div>
<div class="col s3 m2 xl1">
<div class="preloader-wrapper active">
<div class="spinner-layer spinner-green-only">
<div class="circle-clipper left">
<div class="circle"></div>
</div>
<div class="gap-patch">
<div class="circle"></div>
</div>
<div class="circle-clipper right">
<div class="circle"></div>
</div>
</div>
</div>
</div>
<div class="col s9 m6 xl5">
<span class="card-title">Waiting for data...</span>
<p>This list is not initialized yet.</p>
</div>
</td>
</tr>`;
if (typeof this.onclick === 'function') {this.list.list.addEventListener('click', event => this.onclick(event));}
}
eventHandler(eventType, payload) {
switch (eventType) {
case 'init':
this.init(payload);
break;
case 'patch':
this.patch(payload);
break;
default:
console.error(`Unknown event type: ${eventType}`);
break;
}
}
init(ressources) {
this.list.clear();
this.add(Object.values(ressources));
this.list.sort('id', {order: 'desc'});
let emptyListElementHTML = `<tr class="show-if-only-child" data-id="-1">
<td colspan="100%">
<span class="card-title"><i class="left material-icons" style="font-size: inherit;">file_download</i>Nothing here...</span>
<p>No ressource available.</p>
</td>
</tr>`;
this.list.list.insertAdjacentHTML('afterbegin', emptyListElementHTML);
}
patch(patch) {
/*
* It's not possible to generalize a patch Handler for all type of
* ressources. So this method is meant to be an interface.
*/
console.error('patch method not implemented!');
}
add(values) {
let ressources = Array.isArray(values) ? values : [values];
if (typeof this.preprocessRessource === 'function') {
ressources = ressources.map(ressource => this.preprocessRessource(ressource));
}
// Set a callback function ('() => {return;}') to force List.js perform the
// add method asynchronous: https://listjs.com/api/#add
this.list.add(ressources, () => {return;});
}
remove(id) {
this.list.remove('id', id);
}
replace(id, valueName, newValue) {
this.list.get('id', id)[0].values({[valueName]: newValue});
}
}
RessourceList.options = {page: 5, pagination: [{innerWindow: 4, outerWindow: 1}]};

71
web/app/static/js/nopaque/lists/Userlist.js Normal file
View File

@ -0,0 +1,71 @@
class UserList extends RessourceList {
constructor(listElement, options = {}) {
super(listElement, {...UserList.options, ...options});
users = undefined;
}
init(users) {
this.users = users;
super.init(users);
}
onclick(event) {
let ressourceElement = event.target.closest('tr');
if (ressourceElement === null) {return;}
let userId = ressourceElement.dataset.id;
let actionButtonElement = event.target.closest('.action-button');
let action = (actionButtonElement === null) ? 'view' : actionButtonElement.dataset.action;
switch (action) {
case 'delete':
let deleteModalHTML = `<div class="modal">
<div class="modal-content">
<h4>Confirm user deletion</h4>
<p>Do you really want to delete the corpus <b>${this.users[userId].username}</b>? All files will be permanently deleted!</p>
</div>
<div class="modal-footer">
<a href="#!" class="btn modal-close waves-effect waves-light">Cancel</a>
<a class="btn modal-close red waves-effect waves-light" href="/admin/users/${userId}/delete"><i class="material-icons left">delete</i>Delete</a>
</div>
</div>`;
let deleteModalParentElement = document.querySelector('main');
deleteModalParentElement.insertAdjacentHTML('beforeend', deleteModalHTML);
let deleteModalElement = deleteModalParentElement.lastChild;
let deleteModal = M.Modal.init(deleteModalElement, {onCloseEnd: () => {deleteModal.destroy(); deleteModalElement.remove();}});
deleteModal.open();
break;
case 'edit':
window.location.href = `/admin/users/${userId}/edit`;
break;
case 'view':
if (userId !== '-1') {window.location.href = `/admin/users/${userId}`;}
break;
default:
console.error(`Unknown action: ${action}`);
break;
}
}
preprocessRessource(user) {
return {id: user.id,
id_: user.id,
username: user.username,
email: user.email,
last_seen: new Date(user.last_seen * 1000).toLocaleString("en-US"),
role: user.role.name};
}
}
UserList.options = {
item: `<tr>
<td><span class="id_"></span></td>
<td><span class="username"></span></td>
<td><span class="email"></span></td>
<td><span class="last_seen"></span></td>
<td><span class="role"></span></td>
<td class="right-align">
<a class="action-button btn-floating red tooltipped waves-effect waves-light" data-action="delete" data-position="top" data-tooltip="Delete"><i class="material-icons">delete</i></a>
<a class="action-button btn-floating tooltipped waves-effect waves-light" data-action="edit" data-position="top" data-tooltip="Edit"><i class="material-icons">edit</i></a>
<a class="action-button btn-floating tooltipped waves-effect waves-light" data-action="view" data-position="top" data-tooltip="View"><i class="material-icons">send</i></a>
</td>
</tr>`,
valueNames: [{data: ['id']}, 'id_', 'username', 'email', 'last_seen', 'role']
};

235
web/app/static/js/nopaque/main.js Normal file
View File

@ -0,0 +1,235 @@
class AppClient {
constructor(currentUserId) {
this.socket = io({transports: ['websocket']});
this.users = {};
this.users.self = this.loadUser(currentUserId);
}
loadUser(userId) {
if (userId in this.users) {return this.users[userId];}
let user = new User();
this.users[userId] = user;
this.socket.on(`user_${userId}_init`, msg => user.init(JSON.parse(msg)));
this.socket.on(`user_${userId}_patch`, msg => user.patch(JSON.parse(msg)));
this.socket.emit('start_user_session', userId);
return user;
}
}
class User {
constructor() {
this.data = undefined;
this.eventListeners = {
corpus: {
addEventListener(listener, corpusId='*') {
if (corpusId in this) {this[corpusId].push(listener);} else {this[corpusId] = [listener];}
}
},
job: {
addEventListener(listener, jobId='*') {
if (jobId in this) {this[jobId].push(listener);} else {this[jobId] = [listener];}
}
},
queryResult: {
addEventListener(listener, queryResultId='*') {
if (queryResultId in this) {this[queryResultId].push(listener);} else {this[queryResultId] = [listener];}
}
}
};
}
init(data) {
this.data = data;
for (let [corpusId, eventListeners] of Object.entries(this.eventListeners.corpus)) {
if (corpusId === '*') {
for (let eventListener of eventListeners) {eventListener('init', this.data.corpora);}
} else {
if (corpusId in this.data.corpora) {
for (let eventListener of eventListeners) {eventListener('init', this.data.corpora[corpusId]);}
}
}
}
for (let [jobId, eventListeners] of Object.entries(this.eventListeners.job)) {
if (jobId === '*') {
for (let eventListener of eventListeners) {eventListener('init', this.data.jobs);}
} else {
if (jobId in this.data.jobs) {
for (let eventListener of eventListeners) {eventListener('init', this.data.jobs[jobId]);}
}
}
}
for (let [queryResultId, eventListeners] of Object.entries(this.eventListeners.queryResult)) {
if (queryResultId === '*') {
for (let eventListener of eventListeners) {eventListener('init', this.data.query_results);}
} else {
if (queryResultId in this.data.query_results) {
for (let eventListener of eventListeners) {eventListener('init', this.data.query_results[queryResultId]);}
}
}
}
}
patch(patch) {
this.data = jsonpatch.apply_patch(this.data, patch);
let corporaPatch = patch.filter(operation => operation.path.startsWith("/corpora"));
if (corporaPatch.length > 0) {
for (let [corpusId, eventListeners] of Object.entries(this.eventListeners.corpus)) {
if (corpusId === '*') {
for (let eventListener of eventListeners) {eventListener('patch', corporaPatch);}
} else {
let corpusPatch = corporaPatch.filter(operation => operation.path.startsWith(`/corpora/${corpusId}`));
if (corpusPatch.length > 0) {
for (let eventListener of eventListeners) {eventListener('patch', corpusPatch);}
}
}
}
}
let jobsPatch = patch.filter(operation => operation.path.startsWith("/jobs"));
if (jobsPatch.length > 0) {
for (let [jobId, eventListeners] of Object.entries(this.eventListeners.job)) {
if (jobId === '*') {
for (let eventListener of eventListeners) {eventListener('patch', jobsPatch);}
} else {
let jobPatch = jobsPatch.filter(operation => operation.path.startsWith(`/jobs/${jobId}`));
if (jobPatch.length > 0) {
for (let eventListener of eventListeners) {eventListener('patch', jobPatch);}
}
}
}
}
let queryResultsPatch = patch.filter(operation => operation.path.startsWith("/query_results"));
if (queryResultsPatch.length > 0) {
for (let [queryResultId, eventListeners] of Object.entries(this.eventListeners.queryResult)) {
if (queryResultId === '*') {
for (let eventListener of eventListeners) {eventListener('patch', queryResultsPatch);}
} else {
let queryResultPatch = queryResultsPatch.filter(operation => operation.path.startsWith(`/query_results/${queryResultId}`));
if (queryResultPatch.length > 0) {
for (let eventListener of eventListeners) {eventListener('patch', queryResultPatch);}
}
}
}
}
for (let operation of jobsPatch) {
if (operation.op !== 'replace') {continue;}
// Matches the only path that should be handled here: /jobs/{jobId}/status
if (/^\/jobs\/(\d+)\/status$/.test(operation.path)) {
let [match, jobId] = operation.path.match(/^\/jobs\/(\d+)\/status$/);
if (this.data.settings.job_status_site_notifications === "end" && !['complete', 'failed'].includes(operation.value)) {continue;}
nopaque.flash(`[<a href="/jobs/${jobId}">${this.data.jobs[jobId].title}</a>] New status: ${operation.value}`, 'job');
}
}
}
}
/*
* The nopaque object is used as a namespace for nopaque specific functions and
* variables.
*/
var nopaque = {};
nopaque.flash = function(message, category) {
let toast;
let toastActionElement;
switch (category) {
case "corpus":
message = `<i class="left material-icons">book</i>${message}`;
break;
case "error":
message = `<i class="left material-icons red-text">error</i>${message}`;
break;
case "job":
message = `<i class="left material-icons">work</i>${message}`;
break;
default:
message = `<i class="left material-icons">notifications</i>${message}`;
}
toast = M.toast({html: `<span>${message}</span>
<button data-action="close" class="btn-flat toast-action white-text">
<i class="material-icons">close</i>
</button>`});
toastActionElement = toast.el.querySelector('.toast-action[data-action="close"]');
toastActionElement.addEventListener('click', () => {toast.dismiss();});
};
nopaque.Forms = {};
nopaque.Forms.init = function() {
var abortRequestElement, parentElement, progressElement, progressModal,
progressModalElement, request, submitElement;
for (let form of document.querySelectorAll(".nopaque-submit-form")) {
submitElement = form.querySelector('button[type="submit"]');
submitElement.addEventListener("click", function() {
for (let selectElement of form.querySelectorAll('select')) {
if (selectElement.value === "") {
parentElement = selectElement.closest(".input-field");
parentElement.querySelector(".select-dropdown").classList.add("invalid");
for (let helperTextElement of parentElement.querySelectorAll(".helper-text")) {
helperTextElement.remove();
}
parentElement.insertAdjacentHTML("beforeend", `<span class="helper-text red-text">Please select an option.</span>`);
}
}
});
request = new XMLHttpRequest();
if (form.dataset.hasOwnProperty("progressModal")) {
progressModalElement = document.getElementById(form.dataset.progressModal);
progressModal = M.Modal.getInstance(progressModalElement);
progressModal.options.dismissible = false;
abortRequestElement = progressModalElement.querySelector(".abort-request");
abortRequestElement.addEventListener("click", function() {request.abort();});
progressElement = progressModalElement.querySelector(".determinate");
}
form.addEventListener("submit", function(event) {
event.preventDefault();
var formData;
formData = new FormData(form);
// Initialize progress modal
if (progressModalElement) {
progressElement.style.width = "0%";
progressModal.open();
}
request.open("POST", window.location.href);
request.send(formData);
});
request.addEventListener("load", function(event) {
var fieldElement;
if (request.status === 201) {
window.location.href = JSON.parse(this.responseText).redirect_url;
}
if (request.status === 400) {
for (let [field, errors] of Object.entries(JSON.parse(this.responseText))) {
fieldElement = form.querySelector(`input[name$="${field}"]`).closest(".input-field");
for (let error of errors) {
fieldElement.insertAdjacentHTML("beforeend", `<span class="helper-text red-text">${error}</span>`);
}
}
if (progressModalElement) {
progressModal.close();
}
}
if (request.status === 500) {
location.reload();
}
});
if (progressModalElement) {
request.upload.addEventListener("progress", function(event) {
progressElement.style.width = Math.floor(100 * event.loaded / event.total).toString() + "%";
});
}
}
}

7
web/app/static/js/socket.io.min.js vendored Normal file
View File

File diff suppressed because one or more lines are too long

1
web/app/static/js/socket.io.min.js.map Normal file
View File

File diff suppressed because one or more lines are too long

9
web/app/static/js/socket.io.slim.js
View File

File diff suppressed because one or more lines are too long

1
web/app/static/js/socket.io.slim.js.map
View File

File diff suppressed because one or more lines are too long

31
web/app/tasks/__init__.py Normal file
View File

@ -0,0 +1,31 @@
from .. import db
from ..models import Corpus, Job
import docker
docker_client = docker.from_env()
from . import corpus_utils, job_utils # noqa
def check_corpora():
corpora = Corpus.query.all()
for corpus in filter(lambda corpus: corpus.status == 'submitted', corpora):
corpus_utils.create_build_corpus_service(corpus)
for corpus in filter(lambda corpus: corpus.status in ['queued', 'running'], corpora): # noqa
corpus_utils.checkout_build_corpus_service(corpus)
for corpus in filter(lambda corpus: corpus.status == 'start analysis', corpora): # noqa
corpus_utils.create_cqpserver_container(corpus)
for corpus in filter(lambda corpus: corpus.status == 'stop analysis', corpora): # noqa
corpus_utils.remove_cqpserver_container(corpus)
db.session.commit()
def check_jobs():
jobs = Job.query.all()
for job in filter(lambda job: job.status == 'submitted', jobs):
job_utils.create_job_service(job)
for job in filter(lambda job: job.status in ['queued', 'running'], jobs):
job_utils.checkout_job_service(job)
for job in filter(lambda job: job.status == 'canceling', jobs):
job_utils.remove_job_service(job)
db.session.commit()

174
web/app/tasks/corpus_utils.py Normal file
View File

@ -0,0 +1,174 @@
from . import docker_client
import docker
import logging
import os
import shutil
def create_build_corpus_service(corpus):
corpus_data_dir = os.path.join(corpus.path, 'data')
shutil.rmtree(corpus_data_dir, ignore_errors=True)
os.mkdir(corpus_data_dir)
corpus_registry_dir = os.path.join(corpus.path, 'registry')
shutil.rmtree(corpus_registry_dir, ignore_errors=True)
os.mkdir(corpus_registry_dir)
corpus_file = os.path.join(corpus.path, 'merged', 'corpus.vrt')
service_kwargs = {
'command': 'docker-entrypoint.sh build-corpus',
'constraints': ['node.role==worker'],
'labels': {'origin': 'nopaque',
'type': 'corpus.build',
'corpus_id': str(corpus.id)},
'mounts': [corpus_file + ':/root/files/corpus.vrt:ro',
corpus_data_dir + ':/corpora/data:rw',
corpus_registry_dir + ':/usr/local/share/cwb/registry:rw'],
'name': 'build-corpus_{}'.format(corpus.id),
'restart_policy': docker.types.RestartPolicy()
}
service_image = \
'gitlab.ub.uni-bielefeld.de:4567/sfb1288inf/cqpserver:latest'
try:
docker_client.services.create(service_image, **service_kwargs)
except docker.errors.APIError as e:
logging.error(
'Create "{}" service raised '.format(service_kwargs['name'])
+ '"docker.errors.APIError" The server returned an error. '
+ 'Details: {}'.format(e)
)
else:
corpus.status = 'queued'
def checkout_build_corpus_service(corpus):
service_name = 'build-corpus_{}'.format(corpus.id)
try:
service = docker_client.services.get(service_name)
except docker.errors.NotFound:
logging.error(
'Get "{}" service raised '.format(service_name)
+ '"docker.errors.NotFound" The service does not exist. '
+ '(corpus.status: {} -> failed)'.format(corpus.status)
)
corpus.status = 'failed'
except docker.errors.APIError as e:
logging.error(
'Get "{}" service raised '.format(service_name)
+ '"docker.errors.APIError" The server returned an error. '
+ 'Details: {}'.format(e)
)
except docker.errors.InvalidVersion:
logging.error(
'Get "{}" service raised '.format(service_name)
+ '"docker.errors.InvalidVersion" One of the arguments is '
+ 'not supported with the current API version.'
)
else:
service_tasks = service.tasks()
if not service_tasks:
return
task_state = service_tasks[0].get('Status').get('State')
if corpus.status == 'queued' and task_state != 'pending':
corpus.status = 'running'
elif (corpus.status == 'running'
and task_state in ['complete', 'failed']):
try:
service.remove()
except docker.errors.APIError as e:
logging.error(
'Remove "{}" service raised '.format(service_name)
+ '"docker.errors.APIError" The server returned an error. '
+ 'Details: {}'.format(e)
)
return
else:
corpus.status = 'prepared' if task_state == 'complete' \
else 'failed'
def create_cqpserver_container(corpus):
corpus_data_dir = os.path.join(corpus.path, 'data')
corpus_registry_dir = os.path.join(corpus.path, 'registry')
container_kwargs = {
'command': 'cqpserver',
'detach': True,
'volumes': [corpus_data_dir + ':/corpora/data:rw',
corpus_registry_dir + ':/usr/local/share/cwb/registry:rw'],
'name': 'cqpserver_{}'.format(corpus.id),
'network': 'nopaque_default'
}
container_image = \
'gitlab.ub.uni-bielefeld.de:4567/sfb1288inf/cqpserver:latest'
# Check if a cqpserver container already exists. If this is the case,
# remove it and create a new one
try:
container = docker_client.containers.get(container_kwargs['name'])
except docker.errors.NotFound:
pass
except docker.errors.APIError as e:
logging.error(
'Get "{}" container raised '.format(container_kwargs['name'])
+ '"docker.errors.APIError" The server returned an error. '
+ 'Details: {}'.format(e)
)
return
else:
try:
container.remove(force=True)
except docker.errors.APIError as e:
logging.error(
'Remove "{}" container raised '.format(container_kwargs['name']) # noqa
+ '"docker.errors.APIError" The server returned an error. '
+ 'Details: {}'.format(e)
)
return
try:
docker_client.containers.run(container_image, **container_kwargs)
except docker.errors.ContainerError:
# This case should not occur, because detach is True.
logging.error(
'Run "{}" container raised '.format(container_kwargs['name'])
+ '"docker.errors.ContainerError" The container exits with a '
+ 'non-zero exit code and detach is False.'
)
corpus.status = 'failed'
except docker.errors.ImageNotFound:
logging.error(
'Run "{}" container raised '.format(container_kwargs['name'])
+ '"docker.errors.ImageNotFound" The specified image does not '
+ 'exist.'
)
corpus.status = 'failed'
except docker.errors.APIError as e:
logging.error(
'Run "{}" container raised '.format(container_kwargs['name'])
+ '"docker.errors.APIError" The server returned an error. '
+ 'Details: {}'.format(e)
)
else:
corpus.status = 'analysing'
def remove_cqpserver_container(corpus):
container_name = 'cqpserver_{}'.format(corpus.id)
try:
container = docker_client.containers.get(container_name)
except docker.errors.NotFound:
pass
except docker.errors.APIError as e:
logging.error(
'Get "{}" container raised '.format(container_name)
+ '"docker.errors.APIError" The server returned an error. '
+ 'Details: {}'.format(e)
)
return
else:
try:
container.remove(force=True)
except docker.errors.APIError as e:
logging.error(
'Remove "{}" container raised '.format(container_name)
+ '"docker.errors.APIError" The server returned an error. '
+ 'Details: {}'.format(e)
)
return
corpus.status = 'prepared'

152
web/app/tasks/job_utils.py Normal file
View File

@ -0,0 +1,152 @@
from datetime import datetime
from werkzeug.utils import secure_filename
from . import docker_client
from .. import db, mail
from ..email import create_message
from ..models import JobResult
import docker
import logging
import json
import os
def create_job_service(job):
cmd = '{} -i /files -o /files/output'.format(job.service)
if job.service == 'file-setup':
cmd += ' -f {}'.format(secure_filename(job.title))
cmd += ' --log-dir /files'
cmd += ' --zip [{}]_{}'.format(job.service, secure_filename(job.title))
cmd += ' ' + ' '.join(json.loads(job.service_args))
service_kwargs = {'command': cmd,
'constraints': ['node.role==worker'],
'labels': {'origin': 'nopaque',
'type': 'service.{}'.format(job.service),
'job_id': str(job.id)},
'mounts': [job.path + ':/files:rw'],
'name': 'job_{}'.format(job.id),
'resources': docker.types.Resources(
cpu_reservation=job.n_cores * (10 ** 9),
mem_reservation=job.mem_mb * (10 ** 6)
),
'restart_policy': docker.types.RestartPolicy()}
service_image = 'gitlab.ub.uni-bielefeld.de:4567/sfb1288inf/{}:{}'.format(
job.service, job.service_version)
try:
docker_client.services.create(service_image, **service_kwargs)
except docker.errors.APIError as e:
logging.error(
'Create "{}" service raised '.format(service_kwargs['name'])
+ '"docker.errors.APIError" The server returned an error. '
+ 'Details: {}'.format(e)
)
return
else:
job.status = 'queued'
finally:
send_notification(job)
def checkout_job_service(job):
service_name = 'job_{}'.format(job.id)
try:
service = docker_client.services.get(service_name)
except docker.errors.NotFound:
logging.error('Get "{}" service raised '.format(service_name)
+ '"docker.errors.NotFound" The service does not exist. '
+ '(job.status: {} -> failed)'.format(job.status))
job.status = 'failed'
except docker.errors.APIError as e:
logging.error(
'Get "{}" service raised '.format(service_name)
+ '"docker.errors.APIError" The server returned an error. '
+ 'Details: {}'.format(e)
)
return
except docker.errors.InvalidVersion:
logging.error(
'Get "{}" service raised '.format(service_name)
+ '"docker.errors.InvalidVersion" One of the arguments is '
+ 'not supported with the current API version.'
)
return
else:
service_tasks = service.tasks()
if not service_tasks:
return
task_state = service_tasks[0].get('Status').get('State')
if job.status == 'queued' and task_state != 'pending':
job.status = 'running'
elif job.status == 'running' and task_state in ['complete', 'failed']:
try:
service.remove()
except docker.errors.APIError as e:
logging.error(
'Remove "{}" service raised '.format(service_name)
+ '"docker.errors.APIError" The server returned an error. '
+ 'Details: {}'.format(e)
)
return
else:
if task_state == 'complete':
job_results_dir = os.path.join(job.path, 'output')
job_results = filter(lambda x: x.endswith('.zip'),
os.listdir(job_results_dir))
for job_result in job_results:
job_result = JobResult(filename=job_result, job=job)
db.session.add(job_result)
job.end_date = datetime.utcnow()
job.status = task_state
finally:
send_notification(job)
def remove_job_service(job):
service_name = 'job_{}'.format(job.id)
try:
service = docker_client.services.get(service_name)
except docker.errors.NotFound:
job.status = 'canceled'
except docker.errors.APIError as e:
logging.error(
'Get "{}" service raised '.format(service_name)
+ '"docker.errors.APIError" The server returned an error. '
+ 'Details: {}'.format(e)
)
return
except docker.errors.InvalidVersion:
logging.error(
'Get "{}" service raised '.format(service_name)
+ '"docker.errors.InvalidVersion" One of the arguments is '
+ 'not supported with the current API version.'
)
return
else:
try:
service.update(mounts=None)
except docker.errors.APIError as e:
logging.error(
'Update "{}" service raised '.format(service_name)
+ '"docker.errors.APIError" The server returned an error. '
+ 'Details: {}'.format(e)
)
return
try:
service.remove()
except docker.errors.APIError as e:
logging.error(
'Remove "{}" service raised '.format(service_name)
+ '"docker.errors.APIError" The server returned an error. '
+ 'Details: {}'.format(e)
)
def send_notification(job):
if job.creator.setting_job_status_mail_notifications == 'none':
return
if (job.creator.setting_job_status_mail_notifications == 'end'
and job.status not in ['complete', 'failed']):
return
msg = create_message(job.creator.email,
'Status update for your Job "{}"'.format(job.title),
'tasks/email/notification', job=job)
mail.send(msg)

19
web/app/templates/admin/_breadcrumbs.html.j2 Normal file
View File

@ -0,0 +1,19 @@
<ul class="tabs tabs-transparent">
<li class="tab"><a href="{{ url_for('main.index') }}" target="_self"><i class="material-icons">home</i></a></li>
<li class="tab disabled"><i class="material-icons">navigate_next</i></li>
<li class="tab"><a href="{{ url_for('.index') }}" target="_self">Administration</a></li>
<li class="tab disabled"><i class="material-icons">navigate_next</i></li>
{% if request.path == url_for('.users') %}
<li class="tab"><a class="active" href="{{ url_for('.users') }}" target="_self">Users</a></li>
{% elif request.path == url_for('.user', user_id=user.id) %}
<li class="tab"><a href="{{ url_for('.users') }}" target="_self">Users</a></li>
<li class="tab disabled"><i class="material-icons">navigate_next</i></li>
<li class="tab"><a class="active" href="{{ url_for('.user', user_id=user.id) }}" target="_self">{{ user.username }}</a></li>
{% elif request.path == url_for('.edit_user', user_id=user.id) %}
<li class="tab"><a href="{{ url_for('.users') }}" target="_self">Users</a></li>
<li class="tab disabled"><i class="material-icons">navigate_next</i></li>
<li class="tab"><a href="{{ url_for('.user', user_id=user.id) }}" target="_self">{{ user.username }}</a></li>
<li class="tab disabled"><i class="material-icons">navigate_next</i></li>
<li class="tab"><a class="active" href="{{ url_for('.edit_user', user_id=user.id) }}" target="_self">Edit</a></li>
{% endif %}
</ul>

4
web/app/templates/admin/edit_general_settings.html.j2 → web/app/templates/admin/edit_user.html.j2
View File

@ -1,6 +1,10 @@
{% extends "nopaque.html.j2" %}
{% import 'materialize/wtf.html.j2' as wtf %}
{% block nav_content %}
{% include 'admin/_breadcrumbs.html.j2' %}
{% endblock nav_content %}
{% block page_content %}
<div class="container">
<div class="row">

33
web/app/templates/admin/user.html.j2
View File

@ -1,5 +1,9 @@
{% extends "nopaque.html.j2" %}
{% block nav_content %}
{% include 'admin/_breadcrumbs.html.j2' %}
{% endblock nav_content %}
{% block page_content %}
<div class="container">
<div class="row">
@ -30,23 +34,22 @@
</ul>
</div>
<div class="card-action right-align">
<a href="{{ url_for('.edit_general_settings', user_id=user.id) }}" class="waves-effect waves-light btn"><i class="material-icons left">edit</i>Edit</a>
<a href="{{ url_for('.edit_user', user_id=user.id) }}" class="waves-effect waves-light btn"><i class="material-icons left">edit</i>Edit</a>
<a data-target="delete-user-modal" class="waves-effect waves-light btn red modal-trigger"><i class="material-icons left">delete</i>Delete</a>
</div>
</div>
</div>
<div class="col s12 l6">
<div class="col s12 l6" id="corpora" data-user-id="{{ user.id }}">
<h3>Corpora</h3>
<div class="card">
<div class="card-content" id="corpora">
<div class="card-content">
<div class="input-field">
<i class="material-icons prefix">search</i>
<input id="search-corpus" class="search" type="search"></input>
<label for="search-corpus">Search corpus</label>
</div>
<ul class="pagination paginationTop"></ul>
<table class="highlight">
<table class="highlight ressource-list">
<thead>
<tr>
<th></th>
@ -60,22 +63,21 @@
</thead>
<tbody class="list"></tbody>
</table>
<ul class="pagination paginationBottom"></ul>
<ul class="pagination"></ul>
</div>
</div>
</div>
<div class="col s12 l6">
<div class="col s12 l6" id="jobs" data-user-id="{{ user.id }}">
<h3>Jobs</h3>
<div class="card">
<div class="card-content" id="jobs">
<div class="card-content">
<div class="input-field">
<i class="material-icons prefix">search</i>
<input id="search-job" class="search" type="search"></input>
<label for="search-job">Search job</label>
</div>
<ul class="pagination paginationTop"></ul>
<table class="highlight">
<table class="highlight ressource-list">
<thead>
<tr>
<th><span class="sort" data-sort="service">Service</span></th>
@ -89,7 +91,7 @@
</thead>
<tbody class="list"></tbody>
</table>
<ul class="pagination paginationBottom"></ul>
<ul class="pagination"></ul>
</div>
</div>
</div>
@ -111,10 +113,9 @@
{% block scripts %}
{{ super() }}
<script type="module">
import {RessourceList} from '{{ url_for('static', filename='js/nopaque.lists.js') }}';
let corpusList = new RessourceList("corpora", nopaque.foreignCorporaSubscribers, "Corpus");
let jobList = new RessourceList("jobs", nopaque.foreignJobsSubscribers, "Job");
nopaque.socket.emit("foreign_user_data_stream_init", {{ user.id }});
<script>
nopaque.appClient.loadUser({{ user.id }});
let corpusList = new CorpusList(document.querySelector('#corpora'));
let jobList = new JobList(document.querySelector('#jobs'));
</script>
{% endblock scripts %}

31
web/app/templates/admin/users.html.j2
View File

@ -1,5 +1,9 @@
{% extends "nopaque.html.j2" %}
{% block nav_content %}
{% include 'admin/_breadcrumbs.html.j2' %}
{% endblock nav_content %}
{% block page_content %}
<div class="container">
<div class="row">
@ -7,30 +11,28 @@
<h1 id="title">{{ title }}</h1>
</div>
<div class="col s12">
<div class="col s12" id="users">
<div class="card">
<div class="card-content" id="users">
<div class="card-content">
<div class="input-field">
<i class="material-icons prefix">search</i>
<input id="search-user" class="search" type="text"></input>
<label for="search-user">Search user</label>
</div>
<ul class="pagination paginationTop"></ul>
<table class="highlight responsive-table">
<table class="highlight ressource-list">
<thead>
<tr>
<th class="sort" data-sort="id">Id</th>
<th class="sort" data-sort="username">Username</th>
<th class="sort" data-sort="email">Email</th>
<th class="sort" data-sort="role_id">Role</th>
<th class="sort" data-sort="confirmed">Confirmed Status</th>
<th class="sort" data-sort="id">Id</th>
<th>{# Actions #}</th>
<th class="sort" data-sort="last_seen">Last seen</th>
<th class="sort" data-sort="role">Role</th>
<th></th>
</tr>
</thead>
<tbody class="list">
</tbody>
<tbody class="list"></tbody>
</table>
<ul class="pagination paginationBottom"></ul>
<ul class="pagination"></ul>
</div>
</div>
</div>
@ -40,9 +42,8 @@
{% block scripts %}
{{ super() }}
<script type="module">
import {RessourceList} from '{{ url_for('static', filename='js/nopaque.lists.js') }}';
let userList = new RessourceList('users', null, "User", RessourceList.options.extended);
userList._add({{ users|tojson}});
<script>
let userList = new UserList(document.querySelector('#users'), {page: 10});
userList.init({{ users|tojson }});
</script>
{% endblock scripts %}

10
web/app/templates/auth/login.html.j2
View File

@ -35,20 +35,20 @@
<div class="card medium">
<form method="POST">
<div class="card-content">
{{ login_form.hidden_tag() }}
{{ wtf.render_field(login_form.user, material_icon='person') }}
{{ wtf.render_field(login_form.password, material_icon='vpn_key') }}
{{ form.hidden_tag() }}
{{ wtf.render_field(form.user, material_icon='person') }}
{{ wtf.render_field(form.password, material_icon='vpn_key') }}
<div class="row" style="margin-bottom: 0;">
<div class="col s6 left-align">
<a href="{{ url_for('.reset_password_request') }}">Forgot your password?</a>
</div>
<div class="col s6 right-align">
{{ wtf.render_field(login_form.remember_me) }}
{{ wtf.render_field(form.remember_me) }}
</div>
</div>
</div>
<div class="card-action right-align">
{{ wtf.render_field(login_form.submit, material_icon='send') }}
{{ wtf.render_field(form.submit, material_icon='send') }}
</div>
</form>
</div>

12
web/app/templates/auth/register.html.j2
View File

@ -34,14 +34,14 @@
<div class="card medium">
<form method="POST">
<div class="card-content">
{{ registration_form.hidden_tag() }}
{{ wtf.render_field(registration_form.username, data_length='64', material_icon='person') }}
{{ wtf.render_field(registration_form.password, data_length='128', material_icon='vpn_key') }}
{{ wtf.render_field(registration_form.password_confirmation, data_length='128', material_icon='vpn_key') }}
{{ wtf.render_field(registration_form.email, class_='validate', material_icon='email', type='email') }}
{{ form.hidden_tag() }}
{{ wtf.render_field(form.username, data_length='64', material_icon='person') }}
{{ wtf.render_field(form.password, data_length='128', material_icon='vpn_key') }}
{{ wtf.render_field(form.password_confirmation, data_length='128', material_icon='vpn_key') }}
{{ wtf.render_field(form.email, class_='validate', material_icon='email', type='email') }}
</div>
<div class="card-action right-align">
{{ wtf.render_field(registration_form.submit, material_icon='send') }}
{{ wtf.render_field(form.submit, material_icon='send') }}
</div>
</form>
</div>

8
web/app/templates/auth/reset_password.html.j2
View File

@ -20,12 +20,12 @@
<div class="card">
<form method="POST">
<div class="card-content">
{{ reset_password_form.hidden_tag() }}
{{ wtf.render_field(reset_password_form.password, data_length='128') }}
{{ wtf.render_field(reset_password_form.password_confirmation, data_length='128') }}
{{ form.hidden_tag() }}
{{ wtf.render_field(form.password, data_length='128') }}
{{ wtf.render_field(form.password_confirmation, data_length='128') }}
</div>
<div class="card-action right-align">
{{ wtf.render_field(reset_password_form.submit, material_icon='send') }}
{{ wtf.render_field(form.submit, material_icon='send') }}
</div>
</form>
</div>

6
web/app/templates/auth/reset_password_request.html.j2
View File

@ -20,11 +20,11 @@
<div class="card">
<form method="POST">
<div class="card-content">
{{ reset_password_request_form.hidden_tag() }}
{{ wtf.render_field(reset_password_request_form.email, class_='validate', material_icon='email', type='email') }}
{{ form.hidden_tag() }}
{{ wtf.render_field(form.email, class_='validate', material_icon='email', type='email') }}
</div>
<div class="card-action right-align">
{{ wtf.render_field(reset_password_request_form.submit, material_icon='send') }}
{{ wtf.render_field(form.submit, material_icon='send') }}
</div>
</form>
</div>

8
web/app/templates/corpora/add_corpus.html.j2
View File

@ -27,18 +27,18 @@
<div class="card">
<form method="POST">
<div class="card-content">
{{ add_corpus_form.hidden_tag() }}
{{ form.hidden_tag() }}
<div class="row">
<div class="col s12 m4">
{{ wtf.render_field(add_corpus_form.title, data_length='32', material_icon='title') }}
{{ wtf.render_field(form.title, data_length='32', material_icon='title') }}
</div>
<div class="col s12 m8">
{{ wtf.render_field(add_corpus_form.description, data_length='255', material_icon='description') }}
{{ wtf.render_field(form.description, data_length='255', material_icon='description') }}
</div>
</div>
</div>
<div class="card-action right-align">
{{ wtf.render_field(add_corpus_form.submit, material_icon='send') }}
{{ wtf.render_field(form.submit, material_icon='send') }}
</div>
</form>
</div>

14
web/app/templates/corpora/add_corpus_file.html.j2
View File

@ -27,24 +27,24 @@
<form class="nopaque-submit-form" data-progress-modal="progress-modal">
<div class="card">
<div class="card-content">
{{ add_corpus_file_form.hidden_tag() }}
{{ form.hidden_tag() }}
<div class="row">
<div class="col s12 m4">
{{ wtf.render_field(add_corpus_file_form.author, data_length='255', material_icon='person') }}
{{ wtf.render_field(form.author, data_length='255', material_icon='person') }}
</div>
<div class="col s12 m4">
{{ wtf.render_field(add_corpus_file_form.title, data_length='255', material_icon='title') }}
{{ wtf.render_field(form.title, data_length='255', material_icon='title') }}
</div>
<div class="col s12 m4">
{{ wtf.render_field(add_corpus_file_form.publishing_year, material_icon='access_time') }}
{{ wtf.render_field(form.publishing_year, material_icon='access_time') }}
</div>
<div class="col s12">
{{ wtf.render_field(add_corpus_file_form.file, accept='.vrt', placeholder='Choose your .vrt file') }}
{{ wtf.render_field(form.file, accept='.vrt', placeholder='Choose your .vrt file') }}
</div>
</div>
</div>
<div class="card-action right-align">
{{ wtf.render_field(add_corpus_file_form.submit, material_icon='send') }}
{{ wtf.render_field(form.submit, material_icon='send') }}
</div>
</div>
<br>
@ -52,7 +52,7 @@
<li>
<div class="collapsible-header"><i class="material-icons">add</i>Add additional metadata</div>
<div class="collapsible-body">
{% for field in add_corpus_file_form
{% for field in form
if field.short_name not in ['author', 'csrf_token', 'file', 'publishing_year', 'submit', 'title'] %}
{{ wtf.render_field(field, data_length='255', material_icon=field.label.text[0:1]) }}
{% endfor %}

4
web/app/templates/corpora/analyse_corpus.html.j2
View File

@ -155,9 +155,9 @@ import {
*/
document.addEventListener("DOMContentLoaded", () => {
// Initialize the client for server client communication in dynamic mode
let corpusId = {{ corpus_id }}
let corpusId = {{ corpus.id }}
const client = new Client({'corpusId': corpusId,
'socket': nopaque.socket,
'socket': nopaque.appClient.socket,
'logging': true,
'dynamicMode': true});
/**

253
web/app/templates/corpora/corpus.html.j2
View File

@ -2,107 +2,115 @@
{% from '_colors.html.j2' import colors %}
{% set scheme_primary_color = colors.corpus_analysis_darken %}
{% set scheme_secondary_color = colors.corpus_analysis %}
{% set scheme_secondary_color = colors.corpus_analysis_lighten %}
{% block main_attribs %} style="background-color: {{ scheme_secondary_color }};"{% endblock main_attribs %}
{% block nav_content %}
{% include 'corpora/_breadcrumbs.html.j2' %}
{% endblock nav_content %}
{% block main_attribs %} class="corpus-analysis-color lighten"{% endblock main_attribs %}
{% block page_content %}
<div class="container">
<div class="row">
<div class="col s12">
<h1 id="title">{{ corpus.title }}</h1>
<p id="description">{{ corpus.description }}</p>
</div>
<div class="col s12 m4">
<span class="chip status white-text hide" id="status"></span>
<div class="active preloader-wrapper small hide status-spinner" id="progress-indicator">
<div class="spinner-layer spinner-blue-only">
<div class="circle-clipper left">
<div class="circle"></div>
</div>
<div class="gap-patch">
<div class="circle"></div>
</div>
<div class="circle-clipper right">
<div class="circle"></div>
<div class="col s12" data-corpus-id="{{ corpus.id }}" data-user-id="{{ corpus.creator.id }}" id="corpus-display">
<div class="row">
<div class="col s8 m9 l10">
<h1 id="title"><span class="corpus-title"></span></h1>
</div>
<div class="col s4 m3 l2 right-align">
<p>&nbsp;</p>
<p>&nbsp;</p>
<span class="chip status white-text"></span>
<div class="active preloader-wrapper small status-spinner">
<div class="spinner-layer spinner-blue-only">
<div class="circle-clipper left">
<div class="circle"></div>
</div>
<div class="gap-patch">
<div class="circle"></div>
</div>
<div class="circle-clipper right">
<div class="circle"></div>
</div>
</div>
</div>
</div>
</div>
</div>
<div class="col s12 m8">
<div class="card">
<div class="card-content">
<span class="card-title">Chronometrics</span>
<div class="card-content" style="border-top: 10px solid {{ scheme_primary_color }}">
<div class="row">
<div class="col s12 m6">
<div class="col s12">
<div class="input-field">
<input disabled value="{{ corpus.creation_date.strftime('%d/%m/%Y, %H:%M:%S %p') }}" id="creation-date" type="text" class="validate">
<label for="creation-date">Creation date</label>
<input class="corpus-description" disabled id="corpus-description" type="text">
<label for="corpus-description">Description</label>
</div>
</div>
<div class="col s12 m6">
<div class="input-field">
<input disabled value="{{ corpus.last_edited_date.strftime('%d/%m/%Y, %H:%M:%S %p') }}" id="last_edited_date" type="text" class="validate">
<label for="creation-date">Last edited</label>
<input class="corpus-creation-date validate" disabled id="corpus-creation-date" type="text">
<label for="corpus-creation-date">Creation date</label>
</div>
</div>
<div class="col s12 m6">
<div class="input-field">
<input disabled value="{{ corpus.current_nr_of_tokens }} / {{ corpus.max_nr_of_tokens }}" id="nr_of_tokens" type="text" class="validate">
<label for="creation-date">Nr. of tokens used
<i class="material-icons tooltipped" data-position="bottom" data-tooltip="Current number of tokens in this corpus. Updates after every analyze session.">help</i>
</label>
<input class="corpus-last-edited-date validate" disabled id="corpus-last-edited-date" type="text">
<label for="corpus-last-edited-date">Last edited</label>
</div>
</div>
<div class="col s12 m6">
<div class="input-field">
<input class="corpus-token-ratio validate" disabled id="corpus-token-ratio" type="text">
<label for="corpus-token-ratio">Nr. of tokens used <sup><i class="material-icons tooltipped tiny" data-position="bottom" data-tooltip="Current number of tokens in this corpus. Updates after every analyze session.">help</i></sup></label>
</div>
</div>
</div>
</div>
<div class="card-action right-align">
<a href="{{ url_for('corpora.analyse_corpus', corpus_id=corpus.id) }}" class="btn disabled hide waves-effect waves-light" id="analyze"><i class="material-icons left">search</i>Analyze</a>
<a href="{{ url_for('corpora.prepare_corpus', corpus_id=corpus.id) }}" class="btn disabled hide waves-effect waves-light" id="build"><i class="material-icons left">build</i>Build</a>
<a class="btn hide waves-effect waves-light download" id="corpus_create_zip"><i class="material-icons left">import_export</i>Export Corpus</a>
<a data-target="delete-corpus-modal" class="btn modal-trigger red waves-effect waves-light"><i class="material-icons left">delete</i>Delete</a>
<a class="analyse-corpus-trigger btn disabled waves-effect waves-light" href="{{ url_for('corpora.analyse_corpus', corpus_id=corpus.id) }}"><i class="material-icons left">search</i>Analyze</a>
<a class="btn build-corpus-trigger disabled waves-effect waves-light" href="{{ url_for('corpora.prepare_corpus', corpus_id=corpus.id) }}"><i class="material-icons left">build</i>Build</a>
<a class="btn disabled export-corpus-trigger waves-effect waves-light"><i class="material-icons left">import_export</i>Export</a>
<a class="btn modal-trigger red waves-effect waves-light" data-target="delete-corpus-modal"><i class="material-icons left">delete</i>Delete</a>
</div>
</div>
<div id="delete-corpus-modal" class="modal">
<div class="modal-content">
<h4>Confirm corpus deletion</h4>
<p>Do you really want to delete the corpus <span class="corpus-title"></span>? All files will be permanently deleted!</p>
</div>
<div class="modal-footer">
<a class="btn modal-close waves-effect waves-light" href="#!">Cancel</a>
<a class="btn modal-close red waves-effect waves-light" href="{{ url_for('corpora.delete_corpus', corpus_id=corpus.id) }}"><i class="material-icons left">delete</i>Delete</a>
</div>
</div>
</div>
<div class="col s12"></div>
<div class="col s12">
<div class="col s12" id="corpus-files" data-corpus-id="{{ corpus.id }}" data-user-id="{{ corpus.creator.id }}">
<div class="card">
<div class="card-content" id="corpus-files" style="overflow: hidden;">
<div class="card-content">
<span class="card-title" id="files">Corpus files</span>
<div class="input-field">
<i class="material-icons prefix">search</i>
<input id="search-results" class="search" type="search"></input>
<label for="search-results">Search results</label>
<input class="search" id="search-corpus-files" type="search"></input>
<label for="search-corpus-files">Search corpus files</label>
</div>
<ul class="pagination paginationTop"></ul>
<table class="highlight responsive-table">
<thead>
<tr>
<th class="sort" data-sort="filename">Filename</th>
<th class="sort" data-sort="author">Author</th>
<th class="sort" data-sort="title">Title</th>
<th>{# Actions #}</th>
<th class="sort" data-sort="publishing-year">Publishing year</th>
<th></th>
</tr>
</thead>
<tbody class="list">
{% if corpus_files|length == 0 %}
<tr class="show-if-only-child">
<td colspan="5">
<span class="card-title"><i class="material-icons left">book</i>Nothing here...</span>
<p>Corpus is empty. Add texts using the option below.</p>
</td>
</tr>
{% endif %}
<tbody class="list"></tbody>
</table>
<ul class="pagination paginationBottom"></ul>
<ul class="pagination"></ul>
</div>
<div class="card-action right-align">
<a href="{{ url_for('corpora.add_corpus_file', corpus_id=corpus.id) }}" class="btn waves-effect waves-light"><i class="material-icons left">add</i>Add corpus file</a>
@ -111,140 +119,13 @@
</div>
</div>
</div>
<!-- Modals -->
<div id="delete-corpus-modal" class="modal">
<div class="modal-content">
<h4>Confirm corpus deletion</h4>
<p>Do you really want to delete the corpus {{corpus.title}}? All files will be permanently deleted!</p>
</div>
<div class="modal-footer">
<a href="#!" class="btn modal-close waves-effect waves-light">Cancel</a>
<a href="{{ url_for('corpora.delete_corpus', corpus_id=corpus.id) }}" class="btn modal-close red waves-effect waves-light"><i class="material-icons left">delete</i>Delete</a>
</div>
</div>
{% endblock page_content %}
{% block scripts %}
{{ super() }}
<script type="module">
import {
RessourceList
} from '../../static/js/nopaque.lists.js';
class InformationUpdater {
constructor(corpusId, foreignCorpusFlag) {
this.corpusId = corpusId;
this.foreignCorpusFlag = foreignCorpusFlag;
if (this.foreignCorpusFlag) {
nopaque.foreignCorporaSubscribers.push(this);
} else {
nopaque.corporaSubscribers.push(this);
}
}
_init() {
let corpus;
corpus = (this.foreignCorpusFlag ? nopaque.foreignUser.corpora[this.corpusId]
: nopaque.user.corpora[this.corpusId]);
// Status
this.setStatus(corpus.status);
}
_update(patch) {
let pathArray;
for (let operation of patch) {
/* "/corpora/{corpusId}/valueName" -> ["{corpusId}", ...] */
pathArray = operation.path.split("/").slice(2);
if (pathArray[0] != this.corpusId) {continue;}
switch(operation.op) {
case "add":
location.reload();
break;
case "delete":
location.reload();
break;
case "replace":
if (pathArray[1] === "status") {
this.setStatus(operation.value);
}
break;
default:
break;
}
}
}
setStatus(status) {
let analyzeElement, buildElement, numFiles, progressIndicatorElement, statusElement;
numFiles = Object.keys((this.foreignCorpusFlag ? nopaque.foreignUser.corpora[this.corpusId] : nopaque.user.corpora[this.corpusId]).files).length;
progressIndicatorElement = document.getElementById("progress-indicator");
if (["queued", "running", "start analysis", "stop analysis"].includes(status)) {
progressIndicatorElement.classList.remove("hide");
} else {
progressIndicatorElement.classList.add("hide");
}
statusElement = document.getElementById("status");
statusElement.dataset.status = status;
statusElement.classList.remove("hide");
analyzeElement = document.getElementById("analyze");
if (["analysing", "prepared", "start analysis"].includes(status)) {
analyzeElement.classList.remove("disabled", "hide");
} else {
analyzeElement.classList.add("disabled", "hide");
}
buildElement = document.getElementById("build");
if (status === "unprepared" && numFiles > 0) {
buildElement.classList.remove("disabled", "hide");
} else {
buildElement.classList.add("disabled", "hide");
}
let downloadBtn = document.querySelector('#corpus_create_zip');
if (status === "prepared") {
downloadBtn.classList.toggle('hide', false);
} else {
downloadBtn.classList.toggle('hide', true);
}
}
}
{% if corpus.creator == current_user %}
var informationUpdater = new InformationUpdater({{ corpus.id }}, false);
{% else %}
var informationUpdater = new InformationUpdater({{ corpus.id }}, true);
nopaque.socket.emit("foreign_user_data_stream_init", {{ corpus.user_id }});
{% endif %}
let corpusFilesList = new RessourceList("corpus-files", null, "CorpusFile");
corpusFilesList._add({{ corpus_files|tojson|safe }});
// Events to handle full corpus download
let downloadBtn = document.querySelector('#corpus_create_zip');
downloadBtn.addEventListener('click', () => {
nopaque.flash('Compressing your corpus', 'corpus')
nopaque.socket.emit('corpus_create_zip', {{ corpus.id }});
downloadBtn.classList.toggle('disabled', true);
});
document.addEventListener('DOMContentLoaded', () => {
nopaque.socket.on('corpus_zip_created', () => {
nopaque.flash('Downloading your corpus', 'corpus');
downloadBtn.classList.toggle('disabled', false);
// Little trick to call the download view after ziping has finished
let fakeBtn = document.createElement('a');
fakeBtn.href = '{{ url_for('corpora.export_corpus',
corpus_id=corpus.id) }}';
fakeBtn.click();
});
});
<script>
nopaque.appClient.loadUser({{ corpus.creator.id }});
let corpusDisplay = new CorpusDisplay(document.querySelector('#corpus-display'));
let corpusFileList = new CorpusFileList(document.querySelector('#corpus-files'));
</script>
{% endblock scripts %}

12
web/app/templates/corpora/corpus_file.html.j2
View File

@ -20,23 +20,23 @@
<div class="col s12">
<form method="POST">
{{ edit_corpus_file_form.hidden_tag() }}
{{ form.hidden_tag() }}
<div class="card">
<div class="card-content">
<div class="row">
<div class="col s12 m4">
{{ wtf.render_field(edit_corpus_file_form.author, data_length='255', material_icon='person') }}
{{ wtf.render_field(form.author, data_length='255', material_icon='person') }}
</div>
<div class="col s12 m4">
{{ wtf.render_field(edit_corpus_file_form.title, data_length='255', material_icon='title') }}
{{ wtf.render_field(form.title, data_length='255', material_icon='title') }}
</div>
<div class="col s12 m4">
{{ wtf.render_field(edit_corpus_file_form.publishing_year, material_icon='access_time') }}
{{ wtf.render_field(form.publishing_year, material_icon='access_time') }}
</div>
</div>
</div>
<div class="card-action right-align">
{{ wtf.render_field(edit_corpus_file_form.submit, material_icon='send') }}
{{ wtf.render_field(form.submit, material_icon='send') }}
</div>
</div>
<br>
@ -44,7 +44,7 @@
<li>
<div class="collapsible-header"><i class="material-icons">edit</i>Edit additional metadata</div>
<div class="collapsible-body">
{% for field in edit_corpus_file_form
{% for field in form
if field.short_name not in ['author', 'csrf_token', 'publishing_year', 'submit', 'title'] %}
{{ wtf.render_field(field, data_length='255', material_icon=field.label.text[0:1]) }}
{% endfor %}

12
web/app/templates/corpora/import_corpus.html.j2
View File

@ -1,4 +1,4 @@
{% extends "nopaque.html.j2" %}
{% extends "nopaque.html.j2" %}
{% from '_colors.html.j2' import colors %}
{% import 'materialize/wtf.html.j2' as wtf %}
@ -27,23 +27,23 @@
<form class="nopaque-submit-form" data-progress-modal="progress-modal">
<div class="card">
<div class="card-content">
{{ import_corpus_form.hidden_tag() }}
{{ form.hidden_tag() }}
<div class="row">
<div class="col s12 m4">
{{ wtf.render_field(import_corpus_form.title, data_length='32', material_icon='title') }}
{{ wtf.render_field(form.title, data_length='32', material_icon='title') }}
</div>
<div class="col s12 m8">
{{ wtf.render_field(import_corpus_form.description, data_length='255', material_icon='description') }}
{{ wtf.render_field(form.description, data_length='255', material_icon='description') }}
</div>
</div>
<div class="row">
<div class="col s12">
{{ wtf.render_field(import_corpus_form.file, accept='.zip', placeholder='Choose your exported .zip file') }}
{{ wtf.render_field(form.file, accept='.zip', placeholder='Choose your exported .zip file') }}
</div>
</div>
</div>
<div class="card-action right-align">
{{ wtf.render_field(import_corpus_form.submit, material_icon='send') }}
{{ wtf.render_field(form.submit, material_icon='send') }}
</div>
</form>
</div>

10
web/app/templates/corpora/query_results/add_query_result.html.j2
View File

@ -27,21 +27,21 @@
<form class="nopaque-submit-form" data-progress-modal="progress-modal">
<div class="card">
<div class="card-content">
{{ add_query_result_form.hidden_tag() }}
{{ form.hidden_tag() }}
<div class="row">
<div class="col s12 m4">
{{ wtf.render_field(add_query_result_form.title, data_length='32', material_icon='title') }}
{{ wtf.render_field(form.title, data_length='32', material_icon='title') }}
</div>
<div class="col s12 m8">
{{ wtf.render_field(add_query_result_form.description, data_length='255', material_icon='description') }}
{{ wtf.render_field(form.description, data_length='255', material_icon='description') }}
</div>
<div class="col s12">
{{ wtf.render_field(add_query_result_form.file, accept='.json', placeholder='Choose your .json file') }}
{{ wtf.render_field(form.file, accept='.json', placeholder='Choose your .json file') }}
</div>
</div>
</div>
<div class="card-action right-align">
{{ wtf.render_field(add_query_result_form.submit, material_icon='send') }}
{{ wtf.render_field(form.submit, material_icon='send') }}
</div>
</div>
</form>

320
web/app/templates/jobs/job.html.j2
View File

@ -2,137 +2,155 @@
{% from '_colors.html.j2' import colors %}
{% if job.service == 'file-setup' %}
{% set border_color = colors.file_setup_darken %}
{% set main_class = 'file-setup-color lighten' %}
{% set scheme_color = colors.file_setup_darken %}
{% set scheme_primary_color = colors.file_setup_darken %}
{% set scheme_secondary_color = colors.file_setup_lighten %}
{% elif job.service == 'nlp' %}
{% set border_color = colors.nlp_darken %}
{% set main_class = 'nlp-color lighten' %}
{% set scheme_color = colors.nlp_darken %}
{% set scheme_primary_color = colors.nlp_darken %}
{% set scheme_secondary_color = colors.nlp_lighten %}
{% elif job.service == 'ocr' %}
{% set border_color = colors.ocr_darken %}
{% set main_class = 'ocr-color lighten' %}
{% set scheme_color = colors.ocr_darken %}
{% set scheme_primary_color = colors.ocr_darken %}
{% set scheme_secondary_color = colors.ocr_lighten %}
{% endif %}
{% block main_attribs %} style="background-color: {{ scheme_secondary_color }};"{% endblock main_attribs %}
{% block nav_content %}
{% include 'jobs/_breadcrumbs.html.j2' %}
{% endblock nav_content %}
{% block main_attribs %} class="{{ main_class }}"{% endblock main_attribs %}
{% block page_content %}
<div class="container">
<div class="row">
<div class="col s12">
<h1>[{{ job.service }}] {{ job.title }}</h1>
</div>
<div class="col s12" data-job-id="{{ job.id }}" data-user-id="{{ job.creator.id }}" id="job-display">
<div class="row">
<div class="col s8 m9 l10">
<h1 id="title">[<span class="job-service"></span>] <span class="job-title"></span></h1>
</div>
<div class="col s4 m3 l2 right-align">
<p>&nbsp;</p>
<p>&nbsp;</p>
<span class="chip status white-text"></span>
<div class="active preloader-wrapper small status-spinner">
<div class="spinner-layer spinner-blue-only">
<div class="circle-clipper left">
<div class="circle"></div>
</div>
<div class="gap-patch">
<div class="circle"></div>
</div>
<div class="circle-clipper right">
<div class="circle"></div>
</div>
</div>
</div>
</div>
</div>
<div class="col s12">
<div class="card" style="border-top: 10px solid {{border_color}}">
<div class="card" style="border-top: 10px solid {{ scheme_primary_color }}">
<div class="card-content">
<div class="row">
<div class="col s8 m9 l10">
<span class="card-title title">{{ job.title }}</span>
</div>
<div class="col s4 m3 l2 right-align">
<span class="chip status white-text"></span>
<div class="active preloader-wrapper small status-spinner" id="progress-indicator">
<div class="spinner-layer spinner-blue-only">
<div class="circle-clipper left">
<div class="circle"></div>
</div>
<div class="gap-patch">
<div class="circle"></div>
</div>
<div class="circle-clipper right">
<div class="circle"></div>
</div>
</div>
</div>
</div>
<div class="col s12">
<p class="description">{{ job.description }}</p>
</div>
<div class="col s12">&nbsp;</div>
<div class="col s12 m6">
<div class="input-field">
<input disabled id="creation-date" type="text" value="{{ job.creation_date.strftime('%d/%m/%Y, %H:%M:%S %p') }}">
<label for="creation-date">Creation date</label>
<input class="job-description" disabled id="job-description" type="text">
<label for="job-description">Description</label>
</div>
</div>
<div class="col s12 m6">
<div class="input-field">
<input class="end-date" disabled id="end-date" type="text" value="">
<label for="end-date">End date</label>
<input class="job-creation-date" disabled id="job-creation-date" type="text">
<label for="job-creation-date">Creation date</label>
</div>
</div>
<div class="col s12 m6">
<div class="input-field">
<input class="job-end-date" disabled id="job-end-date" type="text">
<label for="job-end-date">End date</label>
</div>
</div>
<div class="col s12 m4">
<div class="input-field">
<input disabled id="service" type="text" value="{{ job.service }}">
<label for="service">Service</label>
<input class="job-service" disabled id="job-service" type="text">
<label for="job-service">Service</label>
</div>
</div>
<div class="col s12 m4">
<div class="input-field">
<input disabled id="service-args" type="text" value="{{ job.service_args|e }}">
<label for="service-args">Service arguments</label>
<input class="job-service-args" disabled id="job-service-args" type="text">
<label for="job-service-args">Service arguments</label>
</div>
</div>
<div class="col s12 m4">
<div class="input-field">
<input disabled id="service-version" type="text" value="{{ job.service_version }}">
<label for="service-version">Service version</label>
<input class="job-service-version" disabled id="job-service-version" type="text">
<label for="job-service-version">Service version</label>
</div>
</div>
</div>
</div>
<div class="card-action right-align">
{% if current_user.is_administrator() and job.status == 'failed' %}
<a href="{{ url_for('jobs.restart', job_id=job.id) }}" class="btn waves-effect waves-light"><i class="material-icons left">repeat</i>Restart</a>
{% if current_user.is_administrator() %}
<a class="btn hide modal-trigger restart-job-trigger waves-effect waves-light" data-target="restart-job-modal"><i class="material-icons left">repeat</i>Restart</a>
{% endif %}
<!-- <a href="#" class="btn disabled waves-effect waves-light"><i class="material-icons left">settings</i>Export Parameters</a> -->
<a data-target="delete-job-modal" class="waves-effect waves-light btn red modal-trigger"><i class="material-icons left">delete</i>Delete</a>
<a class="btn modal-trigger red waves-effect waves-light" data-target="delete-job-modal"><i class="material-icons left">delete</i>Delete</a>
</div>
</div>
<div id="delete-job-modal" class="modal">
<div class="modal-content">
<h4>Confirm deletion</h4>
<p>Do you really want to delete the job <span class="job-title"></span>? All associated files will be permanently deleted.</p>
</div>
<div class="modal-footer">
<a href="#!" class="btn modal-close waves-effect waves-light">Cancel</a>
<a class="btn modal-close red waves-effect waves-light" href="{{ url_for('jobs.delete_job', job_id=job.id) }}"><i class="material-icons left">delete</i>Delete</a>
</div>
</div>
{% if current_user.is_administrator() %}
<div id="restart-job-modal" class="modal">
<div class="modal-content">
<h4>Confirm restart</h4>
<p>Do you really want to restart the job <span class="job-title"></span>? All log and result files will be permanently deleted.</p>
</div>
<div class="modal-footer">
<a href="#!" class="btn modal-close waves-effect waves-light">Cancel</a>
<a class="btn modal-close red waves-effect waves-light" href="{{ url_for('jobs.restart', job_id=job.id) }}"><i class="material-icons left">restart</i>Restart</a>
</div>
</div>
{% endif %}
</div>
<div class="col s12">
<div class="col s12" id="job-inputs" data-job-id="{{ job.id }}" data-user-id="{{ job.creator.id }}">
<div class="card">
<div class="card-content" id="inputs">
<div class="card-content">
<div class="row">
<div class="col s12 m2">
<span class="card-title"><i class="left material-icons" style="font-size: inherit;">input</i>Inputs</span>
<p>Original input files.</p>
</div>
<div class="col s12 m10">
<ul class="pagination paginationTop"></ul>
<table class="highlight responsive-table">
<thead>
<tr>
<th class="sort" data-sort="filename">Filename</th>
<th>{# Actions #}</th>
<th></th>
</tr>
</thead>
<tbody class="list">
</tbody>
<tbody class="list"></tbody>
</table>
<ul class="pagination paginationBottom"></ul>
<ul class="pagination"></ul>
</div>
</div>
</div>
</div>
</div>
<div class="col s12">
<div class="col s12" id="job-results" data-job-id="{{ job.id }}" data-user-id="{{ job.creator.id }}">
<div class="card">
<div class="card-content">
<div class="row">
@ -144,24 +162,14 @@
<table class="highlight responsive-table">
<thead>
<tr>
<th>Result Type</th>
<th>Archive Name</th>
<th>{# Actions #}</th>
<th>Description</th>
<th>Filename</th>
<th></th>
</tr>
</thead>
<tbody class="results">
<tr class="show-if-only-child">
<td colspan="3">
<span class="card-title">
<i class="left material-icons" style="font-size: inherit;">file_download</i>Nothing here...
</span>
<p>
No results available (yet). Is the job already completed?
</p>
</td>
</tr>
</tbody>
<tbody class="list"></tbody>
</table>
<ul class="pagination"></ul>
</div>
</div>
</div>
@ -169,158 +177,14 @@
</div>
</div>
</div>
<!-- Modals -->
<div id="delete-job-modal" class="modal">
<div class="modal-content">
<h4>Confirm deletion</h4>
<p>Do you really want to delete the job {{ job.title }}? All associated files will be permanently deleted.</p>
</div>
<div class="modal-footer">
<a href="#!" class="btn modal-close waves-effect waves-light">Cancel</a>
<a class="btn modal-close red waves-effect waves-light" href="{{ url_for('jobs.delete_job', job_id=job.id) }}"><i class="material-icons left">delete</i>Delete</a>
</div>
</div>
{% endblock page_content %}
{% block scripts %}
{{ super() }}
<script type="module">
import {RessourceList} from '../../static/js/nopaque.lists.js';
class InformationUpdater {
constructor(jobId, foreignJobFlag) {
this.jobId = jobId;
this.foreignJobFlag = foreignJobFlag;
if (this.foreignJobFlag) {
nopaque.foreignJobsSubscribers.push(this);
} else {
nopaque.jobsSubscribers.push(this);
}
}
_init() {
let job;
job = (this.foreignJobFlag ? nopaque.foreignUser.jobs[this.jobId]
: nopaque.user.jobs[this.jobId]);
// Results
this.addResults(job.results);
// End date
this.setEndDate(job.end_date);
// Status
this.setStatus(job.status);
}
_update(patch) {
let pathArray;
for (let operation of patch) {
/* "/jobs/{jobId}/..." -> ["{jobId}", ...] */
pathArray = operation.path.split("/").slice(2);
if (pathArray[0] != this.jobId) {continue;}
switch(operation.op) {
case "add":
if (pathArray[1] === "results") {
this.addResults([operation.value]);
}
break;
case "delete":
location.reload();
break;
case "replace":
if (pathArray[1] === "end_date") {
this.setEndDate(operation.value);
} else if (pathArray[1] === "status") {
this.setStatus(operation.value);
}
break;
default:
break;
}
}
}
addResults(results) {
let resultsArray, resultsElements, resultsHTML, resultType;
resultsArray = Object.values(results);
resultsArray.sort(function (a, b) {
if (a.filename < b.filename) {return -1;}
if (a.filename > b.filename) {return 1;}
return 0;
});
resultsHTML = ``;
for (let result of resultsArray) {
if (result.filename.endsWith(".pdf.zip")) {
resultType = "PDF file with text layer";
} else if (result.filename.endsWith(".txt.zip")) {
resultType = "Raw text files";
} else if (result.filename.endsWith(".vrt.zip")) {
resultType = "VRT(XML dialect) files holding the NLP data";
} else if (result.filename.endsWith(".xml.zip")) {
resultType = "XML files";
} else if (result.filename.endsWith(".poco.zip")) {
resultType = "HCOR und image files needed for Post correction(PoCo)";
} else {
resultType = "All result files created during this job";
}
resultsHTML += `
<tr>
<td>${resultType}</td>
<td>${result.filename}</td>
<td class="right-align">
<a class="btn-floating tooltipped waves-effect waves-light"
download href="/jobs/${result.job_id}/results/${result.id}/download"
data-position="top"
data-tooltip="Download">
<i class="material-icons">file_download</i>
</a>
</td>
</tr>
`;
};
resultsHTML += `
</tbody>
</table>
`;
resultsElements = document.querySelectorAll(".results");
for (let resultsElement of resultsElements) {
resultsElement.innerHTML += resultsHTML;
}
}
setEndDate(timestamp) {
let endDate;
if (timestamp === null) {
endDate = "N.a.";
} else {
endDate = new Date(timestamp * 1000).toLocaleString("en-US");
}
document.getElementById("end-date").value = endDate;
M.updateTextFields();
}
setStatus(status) {
let progressIndicator, statusElements;
if (status === "complete" || status === "failed") {
progressIndicator = document.getElementById("progress-indicator");
progressIndicator.classList.add("hide");
}
statusElements = document.querySelectorAll(".status");
for (let statusElement of statusElements) {
statusElement.dataset.status = status;
}
}
}
{% if job.creator == current_user %}
var informationUpdater = new InformationUpdater({{ job.id }}, false);
{% else %}
var informationUpdater = new InformationUpdater({{ job.id }}, true);
nopaque.socket.emit("foreign_user_data_stream_init", {{ job.user_id }});
{% endif %}
let jobInputsList = new RessourceList("inputs", null, "JobInput");
jobInputsList._add({{ job_inputs|tojson|safe }});
<script>
nopaque.appClient.loadUser({{ job.creator.id }});
let jobDisplay = new JobDisplay(document.querySelector('#job-display'));
let jobInputList = new JobInputList(document.querySelector('#job-inputs'));
let jobResultList = new JobResultList(document.querySelector('#job-results'));
</script>
{% endblock scripts %}

27
web/app/templates/main/dashboard.html.j2
View File

@ -29,8 +29,7 @@
<input id="search-corpus" class="search" type="search"></input>
<label for="search-corpus">Search corpus</label>
</div>
<ul class="pagination paginationTop"></ul>
<table class="highlight">
<table class="highlight ressource-list">
<thead>
<tr>
<th></th>
@ -44,7 +43,7 @@
</thead>
<tbody class="list"></tbody>
</table>
<ul class="pagination paginationBottom"></ul>
<ul class="pagination"></ul>
</div>
<div class="card-action right-align">
<a class="waves-effect waves-light btn" href="{{ url_for('corpora.import_corpus') }}"><i class="material-icons right">import_export</i>Import Corpus</a>
@ -60,8 +59,7 @@
<input id="search-query-results" class="search" type="search"></input>
<label for="search-query-results">Search query result</label>
</div>
<ul class="pagination paginationTop"></ul>
<table class="highlight responsive-table">
<table class="highlight ressource-list">
<thead>
<tr>
<th>
@ -72,7 +70,7 @@
<span class="sort" data-sort="corpus">Corpus</span> and<br>
<span class="sort" data-sort="query">Query</span>
</th>
<th>{# Actions #}</th>
<th></th>
</tr>
</thead>
<tbody class="list">
@ -84,7 +82,7 @@
</tr>
</tbody>
</table>
<ul class="pagination paginationBottom"></ul>
<ul class="pagination"></ul>
</div>
<div class="card-action right-align">
<a class="waves-effect waves-light btn" href="{{ url_for('corpora.add_query_result') }}">Add query result<i class="material-icons right">file_upload</i></a>
@ -104,8 +102,7 @@
<input id="search-job" class="search" type="search"></input>
<label for="search-job">Search job</label>
</div>
<ul class="pagination paginationTop"></ul>
<table class="highlight">
<table class="highlight ressource-list">
<thead>
<tr>
<th><span class="sort" data-sort="service">Service</span></th>
@ -119,12 +116,13 @@
</thead>
<tbody class="list"></tbody>
</table>
<ul class="pagination paginationBottom"></ul>
<ul class="pagination"></ul>
</div>
<div class="card-action right-align">
<p><a class="modal-trigger waves-effect waves-light btn" href="#" data-target="new-job-modal"><i class="material-icons left">add</i>New job</a></p>
</div>
</div>
<div id="new-job-modal" class="modal">
<div class="modal-content">
<h4>Select a service</h4>
@ -178,10 +176,9 @@
{% block scripts %}
{{ super() }}
<script type="module">
import {RessourceList} from '../../static/js/nopaque.lists.js';
let corpusList = new RessourceList("corpora", nopaque.corporaSubscribers, "Corpus");
let jobList = new RessourceList("jobs", nopaque.jobsSubscribers, "Job");
let queryResultList = new RessourceList("query-results", nopaque.queryResultsSubscribers, "QueryResult");
<script>
let corpusList = new CorpusList(document.querySelector('#corpora'));
let jobList = new JobList(document.querySelector('#jobs'));
let queryResultList = new QueryResultList(document.querySelector('#query-results'));
</script>
{% endblock scripts %}

10
web/app/templates/main/index.html.j2
View File

@ -159,20 +159,20 @@
<form method="POST">
<div class="card-content">
<span class="card-title">Log in</span>
{{ login_form.hidden_tag() }}
{{ wtf.render_field(login_form.user, material_icon='person') }}
{{ wtf.render_field(login_form.password, material_icon='vpn_key') }}
{{ form.hidden_tag() }}
{{ wtf.render_field(form.user, material_icon='person') }}
{{ wtf.render_field(form.password, material_icon='vpn_key') }}
<div class="row" style="margin-bottom: 0;">
<div class="col s6 left-align">
<a href="{{ url_for('auth.reset_password_request') }}">Forgot your password?</a>
</div>
<div class="col s6 right-align">
{{ wtf.render_field(login_form.remember_me) }}
{{ wtf.render_field(form.remember_me) }}
</div>
</div>
</div>
<div class="card-action right-align">
{{ wtf.render_field(login_form.submit, material_icon='send') }}
{{ wtf.render_field(form.submit, material_icon='send') }}
</div>
</form>
</div>

2
web/app/templates/modals/analysis_init.html.j2
View File

@ -5,7 +5,7 @@
<p>If the loading takes to long or an error occured,
<a onclick="window.location.reload()" href="#">click here</a>
to refresh your session or
<a href="{{ url_for('corpora.corpus', corpus_id=corpus_id) }}">go back</a>!
<a href="{{ url_for('corpora.corpus', corpus_id=corpus.id) }}">go back</a>!
</p>
<div id="analysis-init-progress" class="progress">
<div class="indeterminate"></div>

45
web/app/templates/nopaque.html.j2
View File

@ -13,7 +13,7 @@
{{ super() }}
{% endblock metas %}
{% block title %}{{title}}{% endblock title %}
{% block title %}{{ title }}{% endblock title %}
{% block styles %}
{{ super() }}
@ -150,7 +150,7 @@
{% if current_user.is_administrator() %}
<li><div class="divider"></div></li>
<li><a class="subheader">Administration</a></li>
<li><a href="{{ url_for('admin.users') }}"><i class="material-icons">build</i>Administration tools</a></li>
<li><a href="{{ url_for('admin.index') }}"><i class="material-icons">build</i>Administration</a></li>
{% endif %}
</ul>
{% endblock sidenav %}
@ -231,9 +231,9 @@
</div>
<div class="col s12 m9 right-align">
<a class="btn-small blue waves-effect waves-light" href="{{ url_for('main.about_and_faq') }}"><i class="left material-icons">info_outline</i>About and faq</a>
{% if config.CONTACT_EMAIL_ADRESS %}
<a class="btn-small pink waves-effect waves-light" href="mailto:{{ config.CONTACT_EMAIL_ADRESS }}?subject=[nopaque] Contact"><i class="left material-icons">rate_review</i>Contact</a>
<a class="btn-small green waves-effect waves-light" href="mailto:{{ config.CONTACT_EMAIL_ADRESS }}?subject=[nopaque] Feedback"><i class="left material-icons">feedback</i>Feedback</a>
{% if config.NOPAQUE_CONTACT %}
<a class="btn-small pink waves-effect waves-light" href="mailto:{{ config.NOPAQUE_CONTACT }}?subject={{ config.NOPAQUE_MAIL_SUBJECT_PREFIX }} Contact"><i class="left material-icons">rate_review</i>Contact</a>
<a class="btn-small green waves-effect waves-light" href="mailto:{{ config.NOPAQUE_CONTACT }}?subject={{ config.NOPAQUE_MAIL_SUBJECT_PREFIX }} Feedback"><i class="left material-icons">feedback</i>Feedback</a>
{% endif %}
<a class="btn-small orange waves-effect waves-light" href="https://gitlab.ub.uni-bielefeld.de/sfb1288inf/nopaque"><i class="left material-icons">code</i>GitLab</a>
</div>
@ -244,28 +244,39 @@
{% block scripts %}
{{ super() }}
{% if current_user.setting_dark_mode %}
<script src="{{ url_for('static', filename='js/darkreader.js') }}"></script>
<script>
DarkReader.enable({brightness: 150, contrast: 100, sepia: 0});
</script>
{% endif %}
<script src="{{ url_for('static', filename='js/jsonpatch.min.js') }}"></script>
<script src="{{ url_for('static', filename='js/list.min.js') }}"></script>
<script src="{{ url_for('static', filename='js/socket.io.slim.js') }}"></script>
<script src="{{ url_for('static', filename='js/nopaque.js') }}"></script>
<script src="{{ url_for('static', filename='js/socket.io.min.js') }}"></script>
<script src="{{ url_for('static', filename='js/nopaque/main.js') }}"></script>
<script src="{{ url_for('static', filename='js/nopaque/displays/RessourceDisplay.js') }}"></script>
<script src="{{ url_for('static', filename='js/nopaque/displays/CorpusDisplay.js') }}"></script>
<script src="{{ url_for('static', filename='js/nopaque/displays/JobDisplay.js') }}"></script>
<script src="{{ url_for('static', filename='js/nopaque/lists/RessourceList.js') }}"></script>
<script src="{{ url_for('static', filename='js/nopaque/lists/CorpusList.js') }}"></script>
<script src="{{ url_for('static', filename='js/nopaque/lists/CorpusFileList.js') }}"></script>
<script src="{{ url_for('static', filename='js/nopaque/lists/JobList.js') }}"></script>
<script src="{{ url_for('static', filename='js/nopaque/lists/JobInputList.js') }}"></script>
<script src="{{ url_for('static', filename='js/nopaque/lists/JobResultList.js') }}"></script>
<script src="{{ url_for('static', filename='js/nopaque/lists/QueryResultList.js') }}"></script>
<script src="{{ url_for('static', filename='js/nopaque/lists/UserList.js') }}"></script>
<script>
{% if current_user.setting_dark_mode %}
DarkReader.enable({brightness: 150, contrast: 100, sepia: 0});
{% endif %}
// Disable all option elements with no value
for (let optionElement of document.querySelectorAll('option[value=""]')) {
optionElement.disabled = true;
}
for (let optionElement of document.querySelectorAll('option[value=""]')) {optionElement.disabled = true;}
M.AutoInit();
M.CharacterCounter.init(document.querySelectorAll('input[data-length][type="email"], input[data-length][type="password"], input[data-length][type="text"], textarea[data-length]'));
M.Dropdown.init(document.querySelectorAll('#nav-more-dropdown-trigger'), {alignment: 'right', constrainWidth: false, coverTrigger: false});
nopaque.Forms.init();
for (let flashedMessage of {{ get_flashed_messages(with_categories=True)|tojson }}) {nopaque.flash(flashedMessage[1], flashedMessage[0]);}
</script>
<script>
{% if current_user.is_authenticated %}
nopaque.socket.emit('user_data_stream_init');
nopaque.appClient = new AppClient({{ current_user.id }});
{% endif %}
for (let flashedMessage of {{ get_flashed_messages(with_categories=True)|tojson }}) {
nopaque.flash(flashedMessage[1], flashedMessage[0]);
}
</script>
{% endblock scripts %}

36
web/app/templates/services/corpus_analysis.html.j2
View File

@ -29,17 +29,16 @@
<p>Nopaque lets you create and upload as many text corpora as you want. It makes use of CQP Query Language, which allows for complex search requests with the aid of metadata and NLP tags. The results can either be displayed as text or abstract visualizations.</p>
</div>
<div class="col s12">
<div class="col s12" id="corpora">
<h2>My Corpora</h2>
<div class="card">
<div class="card-content" id="corpora">
<div class="card-content">
<div class="input-field">
<i class="material-icons prefix">search</i>
<input id="search-corpus" class="search" type="search"></input>
<label for="search-corpus">Search corpus</label>
</div>
<ul class="pagination paginationTop"></ul>
<table>
<table class="highlight ressource-list">
<thead>
<tr>
<th></th>
@ -53,7 +52,7 @@
</thead>
<tbody class="list"></tbody>
</table>
<ul class="pagination paginationBottom"></ul>
<ul class="pagination"></ul>
</div>
<div class="card-action right-align">
<a class="waves-effect waves-light btn" href="{{ url_for('corpora.import_corpus') }}"><i class="material-icons right">import_export</i>Import Corpus</a>
@ -62,17 +61,16 @@
</div>
</div>
<div class="col s12">
<div class="col s12" id="query-results">
<h2>My query results</h2>
<div class="card">
<div class="card-content" id="query-results">
<div class="card-content">
<div class="input-field">
<i class="material-icons prefix">search</i>
<input id="search-query-results" class="search" type="search"></input>
<label for="search-query-results">Search query result</label>
</div>
<ul class="pagination paginationTop"></ul>
<table class="highlight responsive-table">
<table class="highlight ressource-list">
<thead>
<tr>
<th>
@ -83,19 +81,12 @@
<span class="sort" data-sort="corpus">Corpus</span> and<br>
<span class="sort" data-sort="query">Query</span>
</th>
<th>{# Actions #}</th>
<th></th>
</tr>
</thead>
<tbody class="list">
<tr class="show-if-only-child">
<td colspan="5">
<span class="card-title"><i class="material-icons left">folder</i>Nothing here...</span>
<p>No query results yet imported.</p>
</td>
</tr>
</tbody>
<tbody class="list"></tbody>
</table>
<ul class="pagination paginationBottom"></ul>
<ul class="pagination"></ul>
</div>
<div class="card-action right-align">
<a class="btn waves-effect waves-light" href="{{ url_for('corpora.add_query_result') }}">Add query result<i class="material-icons right">file_upload</i></a>
@ -108,9 +99,8 @@
{% block scripts %}
{{ super() }}
<script type="module">
import {RessourceList} from '../../static/js/nopaque.lists.js';
let corpusList = new RessourceList("corpora", nopaque.corporaSubscribers, "Corpus");
let queryResultList = new RessourceList("query-results", nopaque.queryResultsSubscribers, "QueryResult");
<script>
let corpusList = new CorpusList(document.querySelector('#corpora'));
let queryResultList = new QueryResultList(document.querySelector('#query-results'));
</script>
{% endblock scripts %}

12
web/app/templates/services/file-setup.html.j2
View File

@ -48,24 +48,24 @@
<div class="card">
<form class="nopaque-submit-form" data-progress-modal="progress-modal">
<div class="card-content">
{{ add_job_form.hidden_tag() }}
{{ form.hidden_tag() }}
<div class="row">
<div class="col s12 l4">
{{ wtf.render_field(add_job_form.title, data_length='32', material_icon='title') }}
{{ wtf.render_field(form.title, data_length='32', material_icon='title') }}
</div>
<div class="col s12 l8">
{{ wtf.render_field(add_job_form.description, data_length='255', material_icon='description') }}
{{ wtf.render_field(form.description, data_length='255', material_icon='description') }}
</div>
<div class="col s12">
{{ wtf.render_field(add_job_form.files, accept='image/jpeg, image/png, image/tiff', placeholder='Choose your .jpeg, .png or .tiff files') }}
{{ wtf.render_field(form.files, accept='image/jpeg, image/png, image/tiff', placeholder='Choose your .jpeg, .png or .tiff files') }}
</div>
<div class="col s12 hide">
{{ wtf.render_field(add_job_form.version, material_icon='apps') }}
{{ wtf.render_field(form.version, material_icon='apps') }}
</div>
</div>
</div>
<div class="card-action right-align">
{{ wtf.render_field(add_job_form.submit, material_icon='send') }}
{{ wtf.render_field(form.submit, material_icon='send') }}
</div>
</form>
</div>

18
web/app/templates/services/nlp.html.j2
View File

@ -66,34 +66,34 @@
<div class="card">
<form class="nopaque-submit-form" data-progress-modal="progress-modal">
<div class="card-content">
{{ add_job_form.hidden_tag() }}
{{ form.hidden_tag() }}
<div class="row">
<div class="col s12 l4">
{{ wtf.render_field(add_job_form.title, data_length='32', material_icon='title') }}
{{ wtf.render_field(form.title, data_length='32', material_icon='title') }}
</div>
<div class="col s12 l8">
{{ wtf.render_field(add_job_form.description, data_length='255', material_icon='description') }}
{{ wtf.render_field(form.description, data_length='255', material_icon='description') }}
</div>
<div class="col s12 l5">
{{ wtf.render_field(add_job_form.files, accept='text/plain', placeholder='Choose your .txt files') }}
{{ wtf.render_field(form.files, accept='text/plain', placeholder='Choose your .txt files') }}
</div>
<div class="col s12 l4">
{{ wtf.render_field(add_job_form.language, material_icon='language') }}
{{ wtf.render_field(form.language, material_icon='language') }}
</div>
<div class="col s12 l3">
{{ wtf.render_field(add_job_form.version, material_icon='apps') }}
{{ wtf.render_field(form.version, material_icon='apps') }}
</div>
<div class="col s12">
<span class="card-title">Preprocessing</span>
</div>
<div class="col s9">
<p>{{ add_job_form.check_encoding.label.text }}</p>
<p>{{ form.check_encoding.label.text }}</p>
<p class="light">If the input files are not created with the nopaque OCR service or you do not know if your text files are UTF-8 encoded, check this switch. We will try to automatically determine the right encoding for your texts to process them.</p>
</div>
<div class="col s3 right-align">
<div class="switch">
<label>
{{ add_job_form.check_encoding() }}
{{ form.check_encoding() }}
<span class="lever"></span>
</label>
</div>
@ -107,7 +107,7 @@
</div>
</div>
<div class="card-action right-align">
{{ wtf.render_field(add_job_form.submit, material_icon='send') }}
{{ wtf.render_field(form.submit, material_icon='send') }}
</div>
</form>
</div>

18
web/app/templates/services/ocr.html.j2
View File

@ -48,34 +48,34 @@
<div class="card">
<form class="nopaque-submit-form" data-progress-modal="progress-modal">
<div class="card-content">
{{ add_job_form.hidden_tag() }}
{{ form.hidden_tag() }}
<div class="row">
<div class="col s12 l4">
{{ wtf.render_field(add_job_form.title, data_length='32', material_icon='title') }}
{{ wtf.render_field(form.title, data_length='32', material_icon='title') }}
</div>
<div class="col s12 l8">
{{ wtf.render_field(add_job_form.description, data_length='255', material_icon='description') }}
{{ wtf.render_field(form.description, data_length='255', material_icon='description') }}
</div>
<div class="col s12 l5">
{{ wtf.render_field(add_job_form.files, accept='application/pdf', color=ocr_color_darken, placeholder='Choose your .pdf files') }}
{{ wtf.render_field(form.files, accept='application/pdf', color=ocr_color_darken, placeholder='Choose your .pdf files') }}
</div>
<div class="col s12 l4">
{{ wtf.render_field(add_job_form.language, material_icon='language') }}
{{ wtf.render_field(form.language, material_icon='language') }}
</div>
<div class="col s12 l3">
{{ wtf.render_field(add_job_form.version, material_icon='apps') }}
{{ wtf.render_field(form.version, material_icon='apps') }}
</div>
<div class="col s12">
<span class="card-title">Preprocessing</span>
</div>
<div class="col s9">
<p>{{ add_job_form.binarization.label.text }}</p>
<p>{{ form.binarization.label.text }}</p>
<p class="light">Based on a brightness threshold pixels are converted into either black or white. It is useful to reduce noise in images. (<b>longer duration</b>)</p>
</div>
<div class="col s3 right-align">
<div class="switch">
<label>
{{ add_job_form.binarization() }}
{{ form.binarization() }}
<span class="lever"></span>
</label>
</div>
@ -134,7 +134,7 @@
</div>
</div>
<div class="card-action right-align">
{{ wtf.render_field(add_job_form.submit, color=ocr_color_darken, material_icon='send') }}
{{ wtf.render_field(form.submit, color=ocr_color_darken, material_icon='send') }}
</div>
</form>
</div>

8
web/app/templates/tasks/email/notification.html.j2 Normal file
View File

@ -0,0 +1,8 @@
<p>Dear <b>{{ job.creator.username }}</b>,</p>
<p>The status of your Job "<b>{{ job.title }}</b>" has changed!</p>
<p>It is now <b>{{ job.status }}</b>!</p>
<p>You can access your Job here: <a href="{{ url_for('jobs.job', job_id=job.id) }}">{{ url_for('jobs.job', job_id=job.id) }}</a></p>
<p>Kind regards!<br>Your nopaque team</p>

9
web/app/templates/tasks/email/notification.txt.j2 Normal file
View File

@ -0,0 +1,9 @@
Dear {{ job.creator.username }},
The status of your Job "{{ job.title }}" has changed!
It is now {{ job.status }}!
You can access your Job here: {{ url_for('jobs.job', job_id=job.id) }}
Kind regards!
Your nopaque team

18
web/boot.sh
View File

@ -1,20 +1,26 @@
#!/bin/bash
source venv/bin/activate
export FLASK_APP=nopaque.py
if [[ "$#" -eq 0 ]]; then
if [[ "${NOPAQUE_DAEMON_ENABLED:-True}" == "True" ]]; then
echo "INFO Starting nopaque daemon process..."
./nopaque-daemon.sh &
fi
if [[ "${#}" -eq 0 ]]; then
while true; do
flask deploy
if [[ "$?" == "0" ]]; then
if [[ "${?}" == "0" ]]; then
break
fi
echo Deploy command failed, retrying in 5 secs...
echo "Deploy command failed, retrying in 5 secs..."
sleep 5
done
python nopaque.py
elif [[ "$1" == "flask" ]]; then
elif [[ "${1}" == "flask" ]]; then
exec ${@:1}
else
echo "$0 [COMMAND]"
echo "${0} [COMMAND]"
echo ""
echo "nopaque startup script"
echo ""

Some files were not shown because too many files have changed in this diff Show More