Merge branch 'development'

This commit is contained in:
Patrick Jentsch 2022-11-18 16:02:52 +01:00
commit bc92fd249f
50 changed files with 1862 additions and 875 deletions

View File

@ -1,29 +1,12 @@
**/__pycache__
**/.venv
**/.classpath
**/.dockerignore
**/.env
**/.git
**/.gitignore
**/.project
**/.settings
**/.toolstarget
**/.vs
**/.vscode
**/*.*proj.user
**/*.dbmdl
**/*.jfm
**/bin
**/charts
**/docker-compose*
**/compose*
**/Dockerfile*
**/node_modules
**/npm-debug.log
**/obj
**/secrets.dev.yaml
**/values.dev.yaml
README.md
data
# Exclude everything
*
# Include what we need
!app
!migrations
!tests
!.flaskenv
!boot.sh
!config.py
!nopaque.py
!requirements.txt

186
.gitignore vendored
View File

@ -1,42 +1,168 @@
# nopaque specifics
app/static/gen/
data/
docker-compose.override.yml
logs/
!logs/dummy
*.env
# Byte-compiled / optimized / DLL files
__pycache__/
*.py[cod]
*$py.class
# C extensions
*.so
# Flask-Assets files
.webassets-cache
app/static/gen
# Distribution / packaging
.Python
build/
develop-eggs/
dist/
downloads/
eggs/
.eggs/
lib/
lib64/
parts/
sdist/
var/
wheels/
share/python-wheels/
*.egg-info/
.installed.cfg
*.egg
MANIFEST
# Docker related files
docker-compose.override.yml
data/**
# Environment files
*.env
# PyInstaller
# Usually these files are written by a python script from a template
# before PyInstaller builds the exe, so as to inject date/other infos into it.
*.manifest
*.spec
# Installer logs
pip-log.txt
pip-delete-this-directory.txt
# Logs in log folder
logs/*
!logs/dummy
# Unit test / coverage reports
htmlcov/
.tox/
.nox/
.coverage
.coverage.*
.cache
nosetests.xml
coverage.xml
*.cover
*.py,cover
.hypothesis/
.pytest_cache/
cover/
# Packages
*.egg
*.egg-info
dist
build
eggs
parts
bin
var
sdist
develop-eggs
.installed.cfg
lib
lib64
__pycache__
# Translations
*.mo
*.pot
# Virtual environment
venv
.idea
# Django stuff:
*.log
local_settings.py
db.sqlite3
db.sqlite3-journal
# Flask stuff:
instance/
.webassets-cache
# Scrapy stuff:
.scrapy
# Sphinx documentation
docs/_build/
# PyBuilder
.pybuilder/
target/
# Jupyter Notebook
.ipynb_checkpoints
# IPython
profile_default/
ipython_config.py
# pyenv
# For a library or package, you might want to ignore these files since the code is
# intended to run in multiple environments; otherwise, check them in:
# .python-version
# pipenv
# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
# However, in case of collaboration, if having platform-specific dependencies or dependencies
# having no cross-platform support, pipenv may install dependencies that don't work, or not
# install all needed dependencies.
#Pipfile.lock
# poetry
# Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control.
# This is especially recommended for binary packages to ensure reproducibility, and is more
# commonly ignored for libraries.
# https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control
#poetry.lock
# pdm
# Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control.
#pdm.lock
# pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it
# in version control.
# https://pdm.fming.dev/#use-with-ide
.pdm.toml
# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm
__pypackages__/
# Celery stuff
celerybeat-schedule
celerybeat.pid
# SageMath parsed files
*.sage.py
# Environments
.env
.venv
env/
venv/
ENV/
env.bak/
venv.bak/
# Spyder project settings
.spyderproject
.spyproject
# Rope project settings
.ropeproject
# mkdocs documentation
/site
# mypy
.mypy_cache/
.dmypy.json
dmypy.json
# Pyre type checker
.pyre/
# pytype static type analyzer
.pytype/
# Cython debug symbols
cython_debug/
# PyCharm
# JetBrains specific template is maintained in a separate JetBrains.gitignore that can
# be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore
# and can be added to the global gitignore or merged into this file. For a more nuclear
# option (not recommended) you can uncomment the following to ignore the entire idea folder.
#.idea/

View File

@ -1,44 +1,50 @@
FROM python:3.8.10-slim-buster
LABEL authors="Patrick Jentsch <p.jentsch@uni-bielefeld.de>"
ARG DOCKER_GID
ARG UID
ARG GID
ENV FLASK_APP nopaque.py
ENV LANG=C.UTF-8
ENV PYTHONDONTWRITEBYTECODE=1
ENV PYTHONUNBUFFERED=1
RUN apt-get update \
&& apt-get install --no-install-recommends --yes \
build-essential \
libpq-dev \
&& rm -r /var/lib/apt/lists/*
RUN groupadd --gid ${DOCKER_GID} --system docker \
&& groupadd --gid ${GID} --system nopaque \
&& useradd --create-home --gid ${GID} --groups ${DOCKER_GID} --no-log-init --system --uid ${UID} nopaque
USER nopaque
WORKDIR /home/nopaque
COPY --chown=nopaque:nopaque requirements.txt ./
RUN python -m venv venv \
&& venv/bin/pip install --requirement requirements.txt
COPY --chown=nopaque:nopaque app app
COPY --chown=nopaque:nopaque migrations migrations
COPY --chown=nopaque:nopaque tests tests
COPY --chown=nopaque:nopaque boot.sh config.py nopaque.py ./
# run-time configuration
EXPOSE 5000
ENTRYPOINT ["./boot.sh"]
FROM python:3.9.15-slim-bullseye
LABEL authors="Patrick Jentsch <p.jentsch@uni-bielefeld.de>"
ARG DOCKER_GID
ARG UID
ARG GID
ENV LANG="C.UTF-8"
ENV PYTHONDONTWRITEBYTECODE="1"
ENV PYTHONUNBUFFERED="1"
RUN apt-get update \
&& apt-get install --no-install-recommends --yes \
build-essential \
libpq-dev \
&& rm --recursive /var/lib/apt/lists/*
RUN groupadd --gid "${DOCKER_GID}" docker \
&& groupadd --gid "${GID}" nopaque \
&& useradd --create-home --gid nopaque --groups "${DOCKER_GID}" --no-log-init --uid "${UID}" nopaque
USER nopaque
WORKDIR /home/nopaque
ENV PYTHON3_VENV_PATH="/home/nopaque/venv"
RUN python3 -m venv "${PYTHON3_VENV_PATH}"
ENV PATH="${PYTHON3_VENV_PATH}/bin:${PATH}"
COPY --chown=nopaque:nopaque requirements.txt .
RUN python3 -m pip install --requirement requirements.txt \
&& rm requirements.txt
COPY --chown=nopaque:nopaque app app
COPY --chown=nopaque:nopaque migrations migrations
COPY --chown=nopaque:nopaque tests tests
COPY --chown=nopaque:nopaque .flaskenv boot.sh config.py nopaque.py ./
EXPOSE 5000
ENTRYPOINT ["./boot.sh"]

View File

@ -1,10 +1,221 @@
- title: 'de_core_news_md-3.4.0'
- title: 'Catalan'
description: 'Catalan pipeline optimized for CPU. Components: tok2vec, morphologizer, parser, senter, ner, attribute_ruler, lemmatizer.'
url: 'https://github.com/explosion/spacy-models/releases/download/ca_core_news_md-3.2.0/ca_core_news_md-3.2.0.tar.gz'
publisher: 'Explosion'
publisher_url: 'https://github.com/explosion'
publishing_url: 'https://github.com/explosion/spacy-models/releases/tag/ca_core_news_md-3.2.0'
publishing_year: 2021
pipeline_name: 'ca_core_news_md'
version: '3.2.0'
compatible_service_versions:
- '0.1.0'
- title: 'German'
description: 'German pipeline optimized for CPU. Components: tok2vec, tagger, morphologizer, parser, senter, ner, attribute_ruler, lemmatizer.'
url: 'https://github.com/explosion/spacy-models/releases/download/de_core_news_md-3.2.0/de_core_news_md-3.2.0.tar.gz'
publisher: 'Explosion'
publisher_url: 'https://github.com/explosion'
publishing_url: 'https://github.com/explosion/spacy-models/releases/tag/de_core_news_md-3.2.0'
publishing_year: 2021
pipeline_name: 'de_core_news_md'
version: '3.2.0'
compatible_service_versions:
- '0.1.0'
- title: 'Greek'
description: 'Greek pipeline optimized for CPU. Components: tok2vec, morphologizer, parser, senter, ner, attribute_ruler, lemmatizer.'
url: 'https://github.com/explosion/spacy-models/releases/download/el_core_news_md-3.2.0/el_core_news_md-3.2.0.tar.gz'
publisher: 'Explosion'
publisher_url: 'https://github.com/explosion'
publishing_url: 'https://github.com/explosion/spacy-models/releases/tag/el_core_news_md-3.2.0'
publishing_year: 2021
pipeline_name: 'el_core_news_md'
version: '3.2.0'
compatible_service_versions:
- '0.1.0'
- title: 'English'
description: 'English pipeline optimized for CPU. Components: tok2vec, tagger, parser, senter, ner, attribute_ruler, lemmatizer.'
url: 'https://github.com/explosion/spacy-models/releases/download/en_core_web_md-3.2.0/en_core_web_md-3.2.0.tar.gz'
publisher: 'Explosion'
publisher_url: 'https://github.com/explosion'
publishing_url: 'https://github.com/explosion/spacy-models/releases/tag/en_core_web_md-3.2.0'
publishing_year: 2021
pipeline_name: 'en_core_web_md'
version: '3.2.0'
compatible_service_versions:
- '0.1.0'
- title: 'Spanish'
description: 'Spanish pipeline optimized for CPU. Components: tok2vec, morphologizer, parser, senter, ner, attribute_ruler, lemmatizer.'
url: 'https://github.com/explosion/spacy-models/releases/download/es_core_news_md-3.2.0/es_core_news_md-3.2.0.tar.gz'
publisher: 'Explosion'
publisher_url: 'https://github.com/explosion'
publishing_url: 'https://github.com/explosion/spacy-models/releases/tag/es_core_news_md-3.2.0'
publishing_year: 2021
pipeline_name: 'es_core_news_md'
version: '3.2.0'
compatible_service_versions:
- '0.1.0'
- title: 'French'
description: 'French pipeline optimized for CPU. Components: tok2vec, morphologizer, parser, senter, ner, attribute_ruler, lemmatizer.'
url: 'https://github.com/explosion/spacy-models/releases/download/fr_core_news_md-3.2.0/fr_core_news_md-3.2.0.tar.gz'
publisher: 'Explosion'
publisher_url: 'https://github.com/explosion'
publishing_url: 'https://github.com/explosion/spacy-models/releases/tag/fr_core_news_md-3.2.0'
publishing_year: 2021
pipeline_name: 'fr_core_news_md'
version: '3.2.0'
compatible_service_versions:
- '0.1.0'
- title: 'Italian'
description: 'Italian pipeline optimized for CPU. Components: tok2vec, morphologizer, parser, senter, ner, attribute_ruler, lemmatizer.'
url: 'https://github.com/explosion/spacy-models/releases/download/it_core_news_md-3.2.0/it_core_news_md-3.2.0.tar.gz'
publisher: 'Explosion'
publisher_url: 'https://github.com/explosion'
publishing_url: 'https://github.com/explosion/spacy-models/releases/tag/it_core_news_md-3.2.0'
publishing_year: 2021
pipeline_name: 'it_core_news_md'
version: '3.2.0'
compatible_service_versions:
- '0.1.0'
- title: 'Polish'
description: 'Polish pipeline optimized for CPU. Components: tok2vec, morphologizer, parser, senter, ner, attribute_ruler, lemmatizer.'
url: 'https://github.com/explosion/spacy-models/releases/download/pl_core_news_md-3.2.0/pl_core_news_md-3.2.0.tar.gz'
publisher: 'Explosion'
publisher_url: 'https://github.com/explosion'
publishing_url: 'https://github.com/explosion/spacy-models/releases/tag/pl_core_news_md-3.2.0'
publishing_year: 2021
pipeline_name: 'pl_core_news_md'
version: '3.2.0'
compatible_service_versions:
- '0.1.0'
- title: 'Russian'
description: 'Russian pipeline optimized for CPU. Components: tok2vec, morphologizer, parser, senter, ner, attribute_ruler, lemmatizer.'
url: 'https://github.com/explosion/spacy-models/releases/download/ru_core_news_md-3.2.0/ru_core_news_md-3.2.0.tar.gz'
publisher: 'Explosion'
publisher_url: 'https://github.com/explosion'
publishing_url: 'https://github.com/explosion/spacy-models/releases/tag/ru_core_news_md-3.2.0'
publishing_year: 2021
pipeline_name: 'ru_core_news_md'
version: '3.2.0'
compatible_service_versions:
- '0.1.0'
- title: 'Chinese'
description: 'Chinese pipeline optimized for CPU. Components: tok2vec, tagger, parser, senter, ner, attribute_ruler.'
url: 'https://github.com/explosion/spacy-models/releases/download/zh_core_web_md-3.2.0/zh_core_web_md-3.2.0.tar.gz'
publisher: 'Explosion'
publisher_url: 'https://github.com/explosion'
publishing_url: 'https://github.com/explosion/spacy-models/releases/tag/zh_core_web_md-3.2.0'
publishing_year: 2021
pipeline_name: 'zh_core_web_md'
version: '3.2.0'
compatible_service_versions:
- '0.1.0'
- title: 'Catalan'
description: 'Catalan pipeline optimized for CPU. Components: tok2vec, morphologizer, parser, senter, ner, attribute_ruler, lemmatizer.'
url: 'https://github.com/explosion/spacy-models/releases/download/ca_core_news_md-3.4.0/ca_core_news_md-3.4.0.tar.gz'
publisher: 'Explosion'
publisher_url: 'https://github.com/explosion'
publishing_url: 'https://github.com/explosion/spacy-models/releases/tag/ca_core_news_md-3.4.0'
publishing_year: 2022
pipeline_name: 'ca_core_news_md'
version: '3.4.0'
compatible_service_versions:
- '0.1.1'
- title: 'German'
description: 'German pipeline optimized for CPU. Components: tok2vec, tagger, morphologizer, parser, lemmatizer (trainable_lemmatizer), senter, ner.'
url: 'https://github.com/explosion/spacy-models/releases/download/de_core_news_md-3.4.0/de_core_news_md-3.4.0.tar.gz'
publisher: 'Explosion'
publisher_url: 'https://github.com/explosion'
publishing_url: 'https://github.com/explosion/spacy-models/releases/tag/de_core_news_md-3.4.0'
publishing_year: 2022
pipeline_name: 'de_core_news_md'
version: '3.4.0'
compatible_service_versions:
- '0.1.0'
- '0.1.1'
- title: 'Greek'
description: 'Greek pipeline optimized for CPU. Components: tok2vec, morphologizer, parser, lemmatizer (trainable_lemmatizer), senter, ner, attribute_ruler.'
url: 'https://github.com/explosion/spacy-models/releases/download/el_core_news_md-3.4.0/el_core_news_md-3.4.0.tar.gz'
publisher: 'Explosion'
publisher_url: 'https://github.com/explosion'
publishing_url: 'https://github.com/explosion/spacy-models/releases/tag/el_core_news_md-3.4.0'
publishing_year: 2022
pipeline_name: 'el_core_news_md'
version: '3.4.0'
compatible_service_versions:
- '0.1.1'
- title: 'English'
description: 'English pipeline optimized for CPU. Components: tok2vec, tagger, parser, senter, ner, attribute_ruler, lemmatizer.'
url: 'https://github.com/explosion/spacy-models/releases/download/en_core_web_md-3.4.1/en_core_web_md-3.4.1.tar.gz'
publisher: 'Explosion'
publisher_url: 'https://github.com/explosion'
publishing_url: 'https://github.com/explosion/spacy-models/releases/tag/en_core_web_md-3.4.1'
publishing_year: 2022
pipeline_name: 'en_core_web_md'
version: '3.4.1'
compatible_service_versions:
- '0.1.1'
- title: 'Spanish'
description: 'Spanish pipeline optimized for CPU. Components: tok2vec, morphologizer, parser, senter, ner, attribute_ruler, lemmatizer.'
url: 'https://github.com/explosion/spacy-models/releases/download/es_core_news_md-3.4.0/es_core_news_md-3.4.0.tar.gz'
publisher: 'Explosion'
publisher_url: 'https://github.com/explosion'
publishing_url: 'https://github.com/explosion/spacy-models/releases/tag/es_core_news_md-3.4.0'
publishing_year: 2022
pipeline_name: 'es_core_news_md'
version: '3.4.0'
compatible_service_versions:
- '0.1.1'
- title: 'French'
description: 'French pipeline optimized for CPU. Components: tok2vec, morphologizer, parser, senter, ner, attribute_ruler, lemmatizer.'
url: 'https://github.com/explosion/spacy-models/releases/download/fr_core_news_md-3.4.0/fr_core_news_md-3.4.0.tar.gz'
publisher: 'Explosion'
publisher_url: 'https://github.com/explosion'
publishing_url: 'https://github.com/explosion/spacy-models/releases/tag/fr_core_news_md-3.4.0'
publishing_year: 2022
pipeline_name: 'fr_core_news_md'
version: '3.4.0'
compatible_service_versions:
- '0.1.1'
- title: 'Italian'
description: 'Italian pipeline optimized for CPU. Components: tok2vec, morphologizer, tagger, parser, lemmatizer (trainable_lemmatizer), senter, ner'
url: 'https://github.com/explosion/spacy-models/releases/download/it_core_news_md-3.4.0/it_core_news_md-3.4.0.tar.gz'
publisher: 'Explosion'
publisher_url: 'https://github.com/explosion'
publishing_url: 'https://github.com/explosion/spacy-models/releases/tag/it_core_news_md-3.4.0'
publishing_year: 2022
pipeline_name: 'it_core_news_md'
version: '3.4.0'
compatible_service_versions:
- '0.1.1'
- title: 'Polish'
description: 'Polish pipeline optimized for CPU. Components: tok2vec, morphologizer, parser, lemmatizer (trainable_lemmatizer), tagger, senter, ner.'
url: 'https://github.com/explosion/spacy-models/releases/download/pl_core_news_md-3.4.0/pl_core_news_md-3.4.0.tar.gz'
publisher: 'Explosion'
publisher_url: 'https://github.com/explosion'
publishing_url: 'https://github.com/explosion/spacy-models/releases/tag/pl_core_news_md-3.4.0'
publishing_year: 2022
pipeline_name: 'pl_core_news_md'
version: '3.4.0'
compatible_service_versions:
- '0.1.1'
- title: 'Russian'
description: 'Russian pipeline optimized for CPU. Components: tok2vec, morphologizer, parser, senter, ner, attribute_ruler, lemmatizer.'
url: 'https://github.com/explosion/spacy-models/releases/download/ru_core_news_md-3.4.0/ru_core_news_md-3.4.0.tar.gz'
publisher: 'Explosion'
publisher_url: 'https://github.com/explosion'
publishing_url: 'https://github.com/explosion/spacy-models/releases/tag/ru_core_news_md-3.4.0'
publishing_year: 2022
pipeline_name: 'ru_core_news_md'
version: '3.4.0'
compatible_service_versions:
- '0.1.1'
- title: 'Chinese'
description: 'Chinese pipeline optimized for CPU. Components: tok2vec, tagger, parser, senter, ner, attribute_ruler.'
url: 'https://github.com/explosion/spacy-models/releases/download/zh_core_web_md-3.4.0/zh_core_web_md-3.4.0.tar.gz'
publisher: 'Explosion'
publisher_url: 'https://github.com/explosion'
publishing_url: 'https://github.com/explosion/spacy-models/releases/tag/zh_core_web_md-3.4.0'
publishing_year: 2022
pipeline_name: 'zh_core_web_md'
version: '3.4.0'
compatible_service_versions:
- '0.1.1'

View File

@ -6,7 +6,6 @@ from app.decorators import admin_required
from app.models import Role, User, UserSettingJobStatusMailNotificationLevel
from app.settings.forms import (
EditGeneralSettingsForm,
EditInterfaceSettingsForm,
EditNotificationSettingsForm
)
from . import bp
@ -49,15 +48,14 @@ def user(user_id):
def edit_user(user_id):
user = User.query.get_or_404(user_id)
admin_edit_user_form = AdminEditUserForm(
obj=user,
prefix='admin-edit-user-form'
)
edit_general_settings_form = EditGeneralSettingsForm(
user,
obj=user,
prefix='edit-general-settings-form'
)
edit_interface_settings_form = EditInterfaceSettingsForm(
prefix='edit-interface-settings-form'
)
edit_notification_settings_form = EditNotificationSettingsForm(
prefix='edit-notification-settings-form'
)
@ -76,12 +74,6 @@ def edit_user(user_id):
db.session.commit()
flash('Your changes have been saved')
return redirect(url_for('.edit_user', user_id=user.id))
if (edit_interface_settings_form.submit.data
and edit_interface_settings_form.validate()):
user.setting_dark_mode = edit_interface_settings_form.dark_mode.data
db.session.commit()
flash('Your changes have been saved')
return redirect(url_for('.edit_user', user_id=user.id))
if (edit_notification_settings_form.submit.data
and edit_notification_settings_form.validate()):
user.setting_job_status_mail_notification_level = \
@ -91,15 +83,11 @@ def edit_user(user_id):
db.session.commit()
flash('Your changes have been saved')
return redirect(url_for('.edit_user', user_id=user.id))
admin_edit_user_form.prefill(user)
edit_general_settings_form.prefill(user)
edit_interface_settings_form.prefill(user)
edit_notification_settings_form.prefill(user)
return render_template(
'admin/edit_user.html.j2',
admin_edit_user_form=admin_edit_user_form,
edit_general_settings_form=edit_general_settings_form,
edit_interface_settings_form=edit_interface_settings_form,
edit_notification_settings_form=edit_notification_settings_form,
title='Edit user',
user=user

View File

@ -150,7 +150,6 @@ class UserSchema(ma.SQLAlchemySchema):
last_seen = ma.auto_field(dump_only=True)
password = ma.String(load_only=True)
last_seen = ma.auto_field(dump_only=True)
setting_dark_mode = ma.auto_field()
setting_job_status_mail_notification_level = ma.String(
validate=validate.OneOf(list(UserSettingJobStatusMailNotificationLevel.__members__.keys()))
)

View File

@ -1,3 +1,4 @@
from flask import current_app
from flask_wtf import FlaskForm
from flask_wtf.file import FileField, FileRequired
from wtforms import (
@ -5,13 +6,14 @@ from wtforms import (
StringField,
SubmitField,
SelectMultipleField,
IntegerField
IntegerField,
ValidationError
)
from wtforms.validators import InputRequired, Length
from app.services import SERVICES
class TesseractOCRModelContributionForm(FlaskForm):
class ContributionBaseForm(FlaskForm):
title = StringField(
'Title',
validators=[InputRequired(), Length(max=64)]
@ -24,9 +26,6 @@ class TesseractOCRModelContributionForm(FlaskForm):
'Version',
validators=[InputRequired(), Length(max=16)]
)
compatible_service_versions = SelectMultipleField(
'Compatible service versions'
)
publisher = StringField(
'Publisher',
validators=[InputRequired(), Length(max=128)]
@ -43,11 +42,23 @@ class TesseractOCRModelContributionForm(FlaskForm):
'Publishing year',
validators=[InputRequired()]
)
shared = BooleanField('Shared', validators=[InputRequired()])
model_file = FileField('File',validators=[FileRequired()])
compatible_service_versions = SelectMultipleField(
'Compatible service versions'
)
submit = SubmitField()
class CreateTesseractOCRPipelineModelForm(ContributionBaseForm):
tesseract_model_file = FileField(
'File',
validators=[FileRequired()]
)
def validate_tesseract_model_file(self, field):
current_app.logger.warning(field.data.filename)
if not field.data.filename.lower().endswith('.traineddata'):
raise ValidationError('traineddata files only!')
def __init__(self, *args, **kwargs):
service_manifest = SERVICES['tesseract-ocr-pipeline']
super().__init__(*args, **kwargs)
@ -56,3 +67,57 @@ class TesseractOCRModelContributionForm(FlaskForm):
(x, x) for x in service_manifest['versions'].keys()
]
self.compatible_service_versions.default = ''
class CreateSpaCyNLPPipelineModelForm(ContributionBaseForm):
spacy_model_file = FileField(
'File',
validators=[FileRequired()]
)
pipeline_name = StringField(
'Pipeline name',
validators=[InputRequired(), Length(max=64)]
)
def validate_spacy_model_file(self, field):
current_app.logger.warning(field.data.filename)
if not field.data.filename.lower().endswith('.tar.gz'):
raise ValidationError('.tar.gz files only!')
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
service_manifest = SERVICES['spacy-nlp-pipeline']
self.compatible_service_versions.choices = [('', 'Choose your option')]
self.compatible_service_versions.choices += [
(x, x) for x in service_manifest['versions'].keys()
]
self.compatible_service_versions.default = ''
class EditContributionBaseForm(ContributionBaseForm):
pass
class EditTesseractOCRPipelineModelForm(EditContributionBaseForm):
def __init__(self, *args, **kwargs):
service_manifest = SERVICES['tesseract-ocr-pipeline']
super().__init__(*args, **kwargs)
self.compatible_service_versions.choices = [('', 'Choose your option')]
self.compatible_service_versions.choices += [
(x, x) for x in service_manifest['versions'].keys()
]
self.compatible_service_versions.default = ''
class EditSpaCyNLPPipelineModelForm(EditContributionBaseForm):
pipeline_name = StringField(
'Pipeline name',
validators=[InputRequired(), Length(max=64)]
)
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
service_manifest = SERVICES['spacy-nlp-pipeline']
self.compatible_service_versions.choices = [('', 'Choose your option')]
self.compatible_service_versions.choices += [
(x, x) for x in service_manifest['versions'].keys()
]
self.compatible_service_versions.default = ''

View File

@ -1,54 +1,233 @@
from flask import abort, flash, Markup, render_template, url_for
from flask_login import login_required
from flask import (
abort,
current_app,
flash,
Markup,
redirect,
render_template,
url_for
)
from flask_login import login_required, current_user
from threading import Thread
from app import db
from app.decorators import permission_required
from app.models import TesseractOCRPipelineModel, Permission
from app.decorators import permission_required
from app.models import (
Permission,
SpaCyNLPPipelineModel,
TesseractOCRPipelineModel
)
from . import bp
from .forms import TesseractOCRModelContributionForm
from .forms import (
CreateSpaCyNLPPipelineModelForm,
CreateTesseractOCRPipelineModelForm,
EditSpaCyNLPPipelineModelForm,
EditTesseractOCRPipelineModelForm
)
@bp.before_request
@login_required
@permission_required(Permission.CONTRIBUTE)
def before_request():
pass
@bp.route('')
@bp.route('/')
def contributions():
pass
@bp.route('/tesseract-ocr-pipeline-models', methods=['GET', 'POST'])
def tesseract_ocr_pipeline_models():
form = TesseractOCRModelContributionForm(
prefix='contribute-tesseract-ocr-pipeline-model-form'
return render_template(
'contributions/contributions.html.j2',
title='Contributions'
)
@bp.route('/tesseract-ocr-pipeline-models')
def tesseract_ocr_pipeline_models():
return render_template(
'contributions/tesseract_ocr_pipeline_models.html.j2',
title='Tesseract OCR Pipeline Models'
)
@bp.route('/tesseract-ocr-pipeline-models/<hashid:tesseract_ocr_pipeline_model_id>', methods=['GET', 'POST'])
def tesseract_ocr_pipeline_model(tesseract_ocr_pipeline_model_id):
tesseract_ocr_pipeline_model = TesseractOCRPipelineModel.query.get_or_404(tesseract_ocr_pipeline_model_id)
form = EditTesseractOCRPipelineModelForm(
obj=tesseract_ocr_pipeline_model,
prefix='edit-tesseract-ocr-pipeline-model-form'
)
if form.validate_on_submit():
form.populate_obj(tesseract_ocr_pipeline_model)
if db.session.is_modified(tesseract_ocr_pipeline_model):
message = Markup(f'Tesseract OCR Pipeline model "<a href="{tesseract_ocr_pipeline_model.url}">{tesseract_ocr_pipeline_model.title}</a>" updated')
flash(message)
db.session.commit()
return redirect(url_for('.tesseract_ocr_pipeline_models'))
return render_template(
'contributions/tesseract_ocr_pipeline_model.html.j2',
form=form,
tesseract_ocr_pipeline_model=tesseract_ocr_pipeline_model,
title=f'{tesseract_ocr_pipeline_model.title} {tesseract_ocr_pipeline_model.version}'
)
@bp.route('/tesseract-ocr-pipeline-models/<hashid:tesseract_ocr_pipeline_model_id>', methods=['DELETE'])
def delete_tesseract_model(tesseract_ocr_pipeline_model_id):
def _delete_tesseract_ocr_pipeline_model(app, tesseract_ocr_pipeline_model_id):
with app.app_context():
tesseract_ocr_pipeline_model = TesseractOCRPipelineModel.query.get(tesseract_ocr_pipeline_model_id)
tesseract_ocr_pipeline_model.delete()
db.session.commit()
tesseract_ocr_pipeline_model = TesseractOCRPipelineModel.query.get_or_404(tesseract_ocr_pipeline_model_id)
if not (tesseract_ocr_pipeline_model.user == current_user or current_user.is_administrator()):
abort(403)
thread = Thread(
target=_delete_tesseract_ocr_pipeline_model,
args=(current_app._get_current_object(), tesseract_ocr_pipeline_model_id)
)
thread.start()
return {}, 202
@bp.route('/tesseract-ocr-pipeline-models/create', methods=['GET', 'POST'])
def create_tesseract_ocr_pipeline_model():
form = CreateTesseractOCRPipelineModelForm(prefix='create-tesseract-ocr-pipeline-model-form')
if form.is_submitted():
if not form.validate():
response = {'errors': form.errors}
return response, 400
try:
tesseract_ocr_model = TesseractOCRPipelineModel.create(
form.file.data,
tesseract_ocr_pipeline_model = TesseractOCRPipelineModel.create(
form.tesseract_model_file.data,
compatible_service_versions=form.compatible_service_versions.data,
description=form.description.data,
publisher=form.publisher.data,
publisher_url=form.publisher_url.data,
publishing_url=form.publishing_url.data,
publishing_year=form.publishing_year.data,
shared=form.shared.data,
shared=False,
title=form.title.data,
version=form.version.data
version=form.version.data,
user=current_user
)
except OSError:
abort(500)
db.session.commit()
message = Markup(f'Model "{tesseract_ocr_model.title}" created')
tesseract_ocr_pipeline_model_url = url_for(
'.tesseract_ocr_pipeline_model',
tesseract_ocr_pipeline_model_id=tesseract_ocr_pipeline_model.id
)
message = Markup(f'Tesseract OCR Pipeline model "<a href="{tesseract_ocr_pipeline_model_url}">{tesseract_ocr_pipeline_model.title}</a>" created')
flash(message)
return {}, 201, {'Location': url_for('contributions.contributions')}
return {}, 201, {'Location': tesseract_ocr_pipeline_model_url}
return render_template(
'contributions/contribute.html.j2',
'contributions/create_tesseract_ocr_pipeline_model.html.j2',
form=form,
title='Contribution'
title='Create Tesseract OCR Pipeline Model'
)
@bp.route('/tesseract-ocr-pipeline-models/<hashid:tesseract_ocr_pipeline_model_id>/toggle-public-status', methods=['POST'])
@permission_required(Permission.CONTRIBUTE)
def toggle_tesseract_ocr_pipeline_model_public_status(tesseract_ocr_pipeline_model_id):
tesseract_ocr_pipeline_model = TesseractOCRPipelineModel.query.get_or_404(tesseract_ocr_pipeline_model_id)
if not (tesseract_ocr_pipeline_model.user == current_user or current_user.is_administrator()):
abort(403)
tesseract_ocr_pipeline_model.shared = not tesseract_ocr_pipeline_model.shared
db.session.commit()
return {}, 201
@bp.route('/spacy-nlp-pipeline-models')
def spacy_nlp_pipeline_models():
return render_template(
'contributions/spacy_nlp_pipeline_models.html.j2',
title='SpaCy NLP Pipeline Models'
)
@bp.route('/spacy-nlp-pipeline-models/<hashid:spacy_nlp_pipeline_model_id>', methods=['GET', 'POST'])
def spacy_nlp_pipeline_model(spacy_nlp_pipeline_model_id):
spacy_nlp_pipeline_model = SpaCyNLPPipelineModel.query.get_or_404(spacy_nlp_pipeline_model_id)
form = EditSpaCyNLPPipelineModelForm(
obj=spacy_nlp_pipeline_model,
prefix='edit-spacy-nlp-pipeline-model-form'
)
if form.validate_on_submit():
form.populate_obj(spacy_nlp_pipeline_model)
if db.session.is_modified(spacy_nlp_pipeline_model):
message = Markup(f'SpaCy NLP Pipeline model "<a href="{spacy_nlp_pipeline_model.url}">{spacy_nlp_pipeline_model.title}</a>" updated')
flash(message)
db.session.commit()
return redirect(url_for('.spacy_nlp_pipeline_models'))
return render_template(
'contributions/spacy_nlp_pipeline_model.html.j2',
form=form,
spacy_nlp_pipeline_model=spacy_nlp_pipeline_model,
title=f'{spacy_nlp_pipeline_model.title} {spacy_nlp_pipeline_model.version}'
)
@bp.route('/spacy-nlp-pipeline-models/<hashid:spacy_nlp_pipeline_model_id>', methods=['DELETE'])
def delete_spacy_model(spacy_nlp_pipeline_model_id):
def _delete_spacy_model(app, spacy_nlp_pipeline_model_id):
with app.app_context():
spacy_nlp_pipeline_model = SpaCyNLPPipelineModel.query.get(spacy_nlp_pipeline_model_id)
spacy_nlp_pipeline_model.delete()
db.session.commit()
spacy_nlp_pipeline_model = SpaCyNLPPipelineModel.query.get_or_404(spacy_nlp_pipeline_model_id)
if not (spacy_nlp_pipeline_model.user == current_user or current_user.is_administrator()):
abort(403)
thread = Thread(
target=_delete_spacy_model,
args=(current_app._get_current_object(), spacy_nlp_pipeline_model_id)
)
thread.start()
return {}, 202
@bp.route('/spacy-nlp-pipeline-models/create', methods=['GET', 'POST'])
def create_spacy_nlp_pipeline_model():
form = CreateSpaCyNLPPipelineModelForm(prefix='create-spacy-nlp-pipeline-model-form')
if form.is_submitted():
if not form.validate():
response = {'errors': form.errors}
return response, 400
try:
spacy_nlp_pipeline_model = SpaCyNLPPipelineModel.create(
form.spacy_model_file.data,
compatible_service_versions=form.compatible_service_versions.data,
description=form.description.data,
pipeline_name=form.pipeline_name.data,
publisher=form.publisher.data,
publisher_url=form.publisher_url.data,
publishing_url=form.publishing_url.data,
publishing_year=form.publishing_year.data,
shared=False,
title=form.title.data,
version=form.version.data,
user=current_user
)
except OSError:
abort(500)
db.session.commit()
spacy_nlp_pipeline_model_url = url_for(
'.spacy_nlp_pipeline_model',
spacy_nlp_pipeline_model_id=spacy_nlp_pipeline_model.id
)
message = Markup(f'SpaCy NLP Pipeline model "<a href="{spacy_nlp_pipeline_model_url}">{spacy_nlp_pipeline_model.title}</a>" created')
flash(message)
return {}, 201, {'Location': spacy_nlp_pipeline_model_url}
return render_template(
'contributions/create_spacy_nlp_pipeline_model.html.j2',
form=form,
title='Create SpaCy NLP Pipeline Model'
)
@bp.route('/spacy-nlp-pipeline-models/<hashid:spacy_nlp_pipeline_model_id>/toggle-public-status', methods=['POST'])
@permission_required(Permission.CONTRIBUTE)
def toggle_spacy_nlp_pipeline_model_public_status(spacy_nlp_pipeline_model_id):
spacy_nlp_pipeline_model = SpaCyNLPPipelineModel.query.get_or_404(spacy_nlp_pipeline_model_id)
if not (spacy_nlp_pipeline_model.user == current_user or current_user.is_administrator()):
abort(403)
spacy_nlp_pipeline_model.shared = not spacy_nlp_pipeline_model.shared
db.session.commit()
return {}, 201

View File

@ -47,20 +47,7 @@ class CreateCorpusFileForm(CorpusFileBaseForm):
class EditCorpusFileForm(CorpusFileBaseForm):
def prefill(self, corpus_file):
''' Pre-fill the form with data of an exististing corpus file '''
self.address.data = corpus_file.address
self.author.data = corpus_file.author
self.booktitle.data = corpus_file.booktitle
self.chapter.data = corpus_file.chapter
self.editor.data = corpus_file.editor
self.institution.data = corpus_file.institution
self.journal.data = corpus_file.journal
self.pages.data = corpus_file.pages
self.publisher.data = corpus_file.publisher
self.publishing_year.data = corpus_file.publishing_year
self.school.data = corpus_file.school
self.title.data = corpus_file.title
pass
class ImportCorpusForm(FlaskForm):

View File

@ -1,21 +0,0 @@
from flask_wtf import FlaskForm
from werkzeug.utils import secure_filename
from wtforms import FileField, StringField, SubmitField, ValidationError
from wtforms.validators import DataRequired, Length
class AddQueryResultForm(FlaskForm):
'''
Form used to import one result json file.
'''
description = StringField('Description',
validators=[DataRequired(), Length(1, 255)])
file = FileField('File', validators=[DataRequired()])
title = StringField('Title', validators=[DataRequired(), Length(1, 32)])
submit = SubmitField()
def validate_file(self, field):
if not field.data.filename.lower().endswith('.json'):
raise ValidationError('File does not have an approved extension: '
'.json')
field.data.filename = secure_filename(field.data.filename)

View File

@ -1,135 +0,0 @@
from flask import (abort, current_app, flash, make_response, redirect, request,
render_template, url_for, send_from_directory)
from flask_login import current_user, login_required
from . import bp
from . import tasks
from .forms import (AddQueryResultForm, DisplayOptionsForm,
InspectDisplayOptionsForm)
from .. import db
from ..models import QueryResult
import json
import os
@bp.route('/result/add', methods=['GET', 'POST'])
@login_required
def add_query_result():
'''
View to import a result as a json file.
'''
abort(503)
form = AddQueryResultForm(prefix='add-query-result-form')
if form.is_submitted():
if not form.validate():
return make_response(form.errors, 400)
query_result = QueryResult(user=current_user,
description=form.description.data,
filename=form.file.data.filename,
title=form.title.data)
db.session.add(query_result)
db.session.flush()
db.session.refresh(query_result)
try:
os.makedirs(os.path.dirname(query_result.path))
except OSError:
current_app.logger.error(
f'Make dir {query_result.path} led to an OSError!')
db.session.rollback()
flash('Internal Server Error', 'error')
return make_response(
{'redirect_url': url_for('.add_query_result')}, 500)
# save the uploaded file
form.file.data.save(query_result.path)
# parse json from file
with open(query_result.path, 'r') as file:
query_result_file_content = json.load(file)
# parse json schema
# with open('app/static/json_schema/nopaque_cqi_py_results_schema.json', 'r') as file: # noqa
# schema = json.load(file)
# try:
# # validate imported json file
# validate(instance=query_result_file_content, schema=schema)
# except Exception:
# tasks.delete_query_result(query_result.id)
# flash('Uploaded file is invalid', 'result')
# return make_response(
# {'redirect_url': url_for('.add_query_result')}, 201)
query_result_file_content.pop('matches')
query_result_file_content.pop('cpos_lookup')
query_result.query_metadata = query_result_file_content
db.session.commit()
flash('Query result added', 'result')
return make_response({'redirect_url': url_for('.query_result', query_result_id=query_result.id)}, 201) # noqa
return render_template('corpora/query_results/add_query_result.html.j2',
form=form, title='Add query result')
@bp.route('/result/<hashid:query_result_id>')
@login_required
def query_result(query_result_id):
abort(503)
query_result = QueryResult.query.get_or_404(query_result_id)
if not (query_result.user == current_user
or current_user.is_administrator()):
abort(403)
return render_template('corpora/query_results/query_result.html.j2',
query_result=query_result, title='Query result')
@bp.route('/result/<hashid:query_result_id>/inspect')
@login_required
def inspect_query_result(query_result_id):
'''
View to inspect imported result file in a corpus analysis like interface
'''
abort(503)
query_result = QueryResult.query.get_or_404(query_result_id)
query_metadata = query_result.query_metadata
if not (query_result.user == current_user
or current_user.is_administrator()):
abort(403)
display_options_form = DisplayOptionsForm(
prefix='display-options-form',
results_per_page=request.args.get('results_per_page', 30),
result_context=request.args.get('context', 20)
)
inspect_display_options_form = InspectDisplayOptionsForm(
prefix='inspect-display-options-form'
)
with open(query_result.path, 'r') as query_result_file:
query_result_file_content = json.load(query_result_file)
return render_template(
'corpora/query_results/inspect.html.j2',
query_result=query_result,
display_options_form=display_options_form,
inspect_display_options_form=inspect_display_options_form, # noqa
query_result_file_content=query_result_file_content,
query_metadata=query_metadata,
title='Inspect query result'
)
@bp.route('/result/<hashid:query_result_id>/delete')
@login_required
def delete_query_result(query_result_id):
abort(503)
query_result = QueryResult.query.get_or_404(query_result_id)
if not (query_result.user == current_user
or current_user.is_administrator()):
abort(403)
flash(f'Query result "{query_result}" marked for deletion', 'result')
tasks.delete_query_result(query_result_id)
return redirect(url_for('services.service', service="corpus_analysis"))
@bp.route('/result/<hashid:query_result_id>/download')
@login_required
def download_query_result(query_result_id):
abort(503)
query_result = QueryResult.query.get_or_404(query_result_id)
if not (query_result.user == current_user
or current_user.is_administrator()):
abort(403)
return send_from_directory(as_attachment=True,
directory=os.path.dirname(query_result.path),
filename=query_result.filename)

View File

@ -1,13 +0,0 @@
from .. import db
from ..decorators import background
from ..models import QueryResult
@background
def delete_query_result(query_result_id, *args, **kwargs):
with kwargs['app'].app_context():
query_result = QueryResult.query.get(query_result_id)
if query_result is None:
raise Exception(f'QueryResult {query_result_id} not found')
query_result.delete()
db.session.commit()

View File

@ -176,52 +176,15 @@ def corpus_file(corpus_id, corpus_file_id):
abort(404)
if not (corpus_file.corpus.user == current_user or current_user.is_administrator()):
abort(403)
form = EditCorpusFileForm(prefix='edit-corpus-file-form')
form = EditCorpusFileForm(obj=corpus_file, prefix='edit-corpus-file-form')
if form.validate_on_submit():
has_changes = False
if corpus_file.address != form.address.data:
corpus_file.address = form.address.data
has_changes = True
if corpus_file.author != form.author.data:
corpus_file.author = form.author.data
has_changes = True
if corpus_file.booktitle != form.booktitle.data:
corpus_file.booktitle = form.booktitle.data
has_changes = True
if corpus_file.chapter != form.chapter.data:
corpus_file.chapter = form.chapter.data
has_changes = True
if corpus_file.editor != form.editor.data:
corpus_file.editor = form.editor.data
has_changes = True
if corpus_file.institution != form.institution.data:
corpus_file.institution = form.institution.data
has_changes = True
if corpus_file.journal != form.journal.data:
corpus_file.journal = form.journal.data
has_changes = True
if corpus_file.pages != form.pages.data:
corpus_file.pages = form.pages.data
has_changes = True
if corpus_file.publisher != form.publisher.data:
corpus_file.publisher = form.publisher.data
has_changes = True
if corpus_file.publishing_year != form.publishing_year.data:
corpus_file.publishing_year = form.publishing_year.data
has_changes = True
if corpus_file.school != form.school.data:
corpus_file.school = form.school.data
has_changes = True
if corpus_file.title != form.title.data:
corpus_file.title = form.title.data
has_changes = True
if has_changes:
form.populate_obj(corpus_file)
if db.session.is_modified(corpus_file):
corpus_file.corpus.status = CorpusStatus.UNPREPARED
db.session.commit()
message = Markup(f'Corpus file "<a href="{corpus_file.url}">{corpus_file.filename}</a>" updated')
flash(message, category='corpus')
db.session.commit()
message = Markup(f'Corpus file "<a href="{corpus_file.url}">{corpus_file.filename}</a>" updated')
flash(message, category='corpus')
return redirect(corpus_file.corpus.url)
form.prefill(corpus_file)
return render_template(
'corpora/corpus_file.html.j2',
corpus=corpus_file.corpus,

View File

@ -3,7 +3,8 @@ from app.models import (
Job,
JobResult,
JobStatus,
TesseractOCRPipelineModel
TesseractOCRPipelineModel,
SpaCyNLPPipelineModel
)
from datetime import datetime
from flask import current_app
@ -52,13 +53,21 @@ def _create_job_service(job):
command += f' --mem-mb {mem_mb}'
command += f' --n-cores {n_cores}'
if job.service == 'spacy-nlp-pipeline':
command += f' -m {job.service_args["model"]}'
model_id = hashids.decode(job.service_args['model'])
model = SpaCyNLPPipelineModel.query.get(model_id)
if model is None:
job.status = JobStatus.FAILED
return
command += f' -m {model.pipeline_name}'
if 'encoding_detection' in job.service_args and job.service_args['encoding_detection']:
command += ' --check-encoding'
elif job.service == 'tesseract-ocr-pipeline':
command += f' -m {job.service_args["model"]}'
if 'binarization' in job.service_args and job.service_args['binarization']:
command += ' --binarize'
if 'ocropus_nlbin_threshold' in job.service_args and job.service_args['ocropus_nlbin_threshold']:
value = job.service_args['ocropus_nlbin_threshold']
command += f' --ocropus-nlbin-threshold {value}'
elif job.service == 'transkribus-htr-pipeline':
transkribus_htr_pipeline_model_id = job.service_args['model']
command += f' -m {transkribus_htr_pipeline_model_id}'
@ -103,6 +112,16 @@ def _create_job_service(job):
models_mount_target = f'/usr/local/share/tessdata/{model.filename}'
models_mount = f'{models_mount_source}:{models_mount_target}:ro'
mounts.append(models_mount)
elif job.service == 'spacy-nlp-pipeline':
model_id = hashids.decode(job.service_args['model'])
model = SpaCyNLPPipelineModel.query.get(model_id)
if model is None:
job.status = JobStatus.FAILED
return
models_mount_source = model.path
models_mount_target = f'/usr/local/share/spacy/models/{model.filename}'
models_mount = f'{models_mount_source}:{models_mount_target}:ro'
mounts.append(models_mount)
''' ### Output mount ### '''
output_mount_source = os.path.join(job.path, 'results')
output_mount_target = '/output'

View File

@ -1,6 +1,5 @@
from datetime import datetime, timedelta
from enum import Enum, IntEnum
import re
from flask import current_app, url_for
from flask_hashids import HashidMixin
from flask_login import UserMixin
@ -263,7 +262,6 @@ class User(HashidMixin, UserMixin, db.Model):
password_hash = db.Column(db.String(128))
confirmed = db.Column(db.Boolean, default=False)
member_since = db.Column(db.DateTime(), default=datetime.utcnow)
setting_dark_mode = db.Column(db.Boolean, default=False)
setting_job_status_mail_notification_level = db.Column(
IntEnumColumn(UserSettingJobStatusMailNotificationLevel),
default=UserSettingJobStatusMailNotificationLevel.END
@ -500,7 +498,6 @@ class User(HashidMixin, UserMixin, db.Model):
'member_since': f'{self.member_since.isoformat()}Z',
'username': self.username,
'settings': {
'dark_mode': self.setting_dark_mode,
'job_status_mail_notification_level': \
self.setting_job_status_mail_notification_level.name
}
@ -520,6 +517,10 @@ class User(HashidMixin, UserMixin, db.Model):
x.hashid: x.to_json(relationships=True)
for x in self.tesseract_ocr_pipeline_models
}
_json['spacy_nlp_pipeline_models'] = {
x.hashid: x.to_json(relationships=True)
for x in self.spacy_nlp_pipeline_models
}
return _json
class TesseractOCRPipelineModel(FileMixin, HashidMixin, db.Model):
@ -548,6 +549,21 @@ class TesseractOCRPipelineModel(FileMixin, HashidMixin, db.Model):
str(self.id)
)
@property
def jsonpatch_path(self):
return f'{self.user.jsonpatch_path}/tesseract_ocr_pipeline_models/{self.hashid}'
@property
def url(self):
return url_for(
'contributions.tesseract_ocr_pipeline_model',
tesseract_ocr_pipeline_model_id=self.id
)
@property
def user_hashid(self):
return self.user.hashid
@staticmethod
def insert_defaults():
nopaque_user = User.query.filter_by(username='nopaque').first()
@ -603,6 +619,13 @@ class TesseractOCRPipelineModel(FileMixin, HashidMixin, db.Model):
pbar.close()
db.session.commit()
def delete(self):
try:
os.remove(self.path)
except OSError as e:
current_app.logger.error(e)
db.session.delete(self)
def to_json(self, backrefs=False, relationships=False):
_json = {
'id': self.hashid,
@ -614,6 +637,7 @@ class TesseractOCRPipelineModel(FileMixin, HashidMixin, db.Model):
'publishing_year': self.publishing_year,
'shared': self.shared,
'title': self.title,
'version': self.version,
**self.file_mixin_to_json()
}
if backrefs:
@ -636,6 +660,7 @@ class SpaCyNLPPipelineModel(FileMixin, HashidMixin, db.Model):
publisher_url = db.Column(db.String(512))
publishing_url = db.Column(db.String(512))
publishing_year = db.Column(db.Integer)
pipeline_name = db.Column(db.String(64))
shared = db.Column(db.Boolean, default=False)
# Backrefs: user: User
@ -647,6 +672,21 @@ class SpaCyNLPPipelineModel(FileMixin, HashidMixin, db.Model):
str(self.id)
)
@property
def jsonpatch_path(self):
return f'{self.user.jsonpatch_path}/spacy_nlp_pipeline_models/{self.hashid}'
@property
def url(self):
return url_for(
'contributions.spacy_nlp_pipeline_model',
spacy_nlp_pipeline_model_id=self.id
)
@property
def user_hashid(self):
return self.user.hashid
@staticmethod
def insert_defaults():
nopaque_user = User.query.filter_by(username='nopaque').first()
@ -668,6 +708,7 @@ class SpaCyNLPPipelineModel(FileMixin, HashidMixin, db.Model):
model.shared = True
model.title = m['title']
model.version = m['version']
model.pipeline_name = m['pipeline_name']
continue
model = SpaCyNLPPipelineModel(
compatible_service_versions=m['compatible_service_versions'],
@ -679,12 +720,13 @@ class SpaCyNLPPipelineModel(FileMixin, HashidMixin, db.Model):
shared=True,
title=m['title'],
user=nopaque_user,
version=m['version']
version=m['version'],
pipeline_name=m['pipeline_name']
)
db.session.add(model)
db.session.flush(objects=[model])
db.session.refresh(model)
model.filename = f'{model.id}.traineddata'
model.filename = m['url'].split('/')[-1]
r = requests.get(m['url'], stream=True)
pbar = tqdm(
desc=f'{model.title} ({model.filename})',
@ -701,6 +743,13 @@ class SpaCyNLPPipelineModel(FileMixin, HashidMixin, db.Model):
f.write(chunk)
pbar.close()
db.session.commit()
def delete(self):
try:
os.remove(self.path)
except OSError as e:
current_app.logger.error(e)
db.session.delete(self)
def to_json(self, backrefs=False, relationships=False):
_json = {
@ -711,8 +760,10 @@ class SpaCyNLPPipelineModel(FileMixin, HashidMixin, db.Model):
'publisher_url': self.publisher_url,
'publishing_url': self.publishing_url,
'publishing_year': self.publishing_year,
'pipeline_name': self.pipeline_name,
'shared': self.shared,
'title': self.title,
'version': self.version,
**self.file_mixin_to_json()
}
if backrefs:
@ -1023,11 +1074,8 @@ class CorpusFile(FileMixin, HashidMixin, db.Model):
def delete(self):
try:
os.remove(self.path)
except OSError:
current_app.logger.error(
f'Removing {self.path} led to an OSError!'
)
pass
except OSError as e:
current_app.logger.error(e)
db.session.delete(self)
self.corpus.status = CorpusStatus.UNPREPARED

View File

@ -1,58 +0,0 @@
class QueryResult(FileMixin, HashidMixin, db.Model):
__tablename__ = 'query_results'
# Primary key
id = db.Column(db.Integer, primary_key=True)
# Foreign keys
user_id = db.Column(db.Integer, db.ForeignKey('users.id'))
# Fields
description = db.Column(db.String(255))
query_metadata = db.Column(db.JSON())
title = db.Column(db.String(32))
# Backrefs: user: User
def __repr__(self):
'''
String representation of the QueryResult. For human readability.
'''
return f'<QueryResult {self.title}>'
@property
def download_url(self):
return url_for(
'corpora.download_query_result', query_result_id=self.id)
@property
def jsonpatch_path(self):
return f'{self.user.jsonpatch_path}/query_results/{self.hashid}'
@property
def path(self):
return os.path.join(
self.user.path, 'query_results', str(self.id), self.filename)
@property
def url(self):
return url_for('corpora.query_result', query_result_id=self.id)
@property
def user_hashid(self):
return self.user.hashid
def delete(self):
shutil.rmtree(self.path, ignore_errors=True)
db.session.delete(self)
def to_json(self, backrefs=False, relationships=False):
_json = {
'id': self.hashid,
'corpus_title': self.query_metadata['corpus_name'],
'description': self.description,
'filename': self.filename,
'query': self.query_metadata['query'],
'query_metadata': self.query_metadata,
'title': self.title,
**self.file_mixin_to_json(
backrefs=backrefs, relationships=relationships)
}
if backrefs:
_json['user'] = self.user.to_json(backrefs=True, relationships=False)

View File

@ -1,16 +1,12 @@
from flask_login import current_user
from flask_wtf import FlaskForm
from flask_wtf.file import FileField, FileRequired
from wtforms import (
BooleanField,
MultipleFileField,
SelectField,
StringField,
SubmitField,
ValidationError
)
from wtforms import (BooleanField, DecimalRangeField, MultipleFileField,
SelectField, StringField, SubmitField, ValidationError)
from wtforms.validators import InputRequired, Length
from app.models import TesseractOCRPipelineModel
from app.models import SpaCyNLPPipelineModel, TesseractOCRPipelineModel
from . import SERVICES
@ -49,13 +45,16 @@ class CreateTesseractOCRPipelineJobForm(CreateJobBaseForm):
binarization = BooleanField('Binarization')
pdf = FileField('File', validators=[FileRequired()])
model = SelectField('Model', validators=[InputRequired()])
ocropus_nlbin_threshold = DecimalRangeField(
render_kw={'min': 0, 'max': 1, 'step': 0.1, 'start': [0.5], 'disabled': True}
)
def validate_binarization(self, field):
service_info = SERVICES['tesseract-ocr-pipeline']['versions'][self.version.data]
if field.data:
if not('methods' in service_info and 'binarization' in service_info['methods']):
raise ValidationError('Binarization is not available')
def validate_pdf(self, field):
if field.data.mimetype != 'application/pdf':
raise ValidationError('PDF files only!')
@ -68,16 +67,20 @@ class CreateTesseractOCRPipelineJobForm(CreateJobBaseForm):
if self.binarization.render_kw is None:
self.binarization.render_kw = {}
self.binarization.render_kw['disabled'] = True
if self.ocropus_nlbin_threshold.render_kw is None:
self.ocropus_nlbin_threshold.render_kw = {}
self.ocropus_nlbin_threshold.render_kw['disabled'] = True
if 'methods' in service_info:
if 'binarization' in service_info['methods']:
if 'disabled' in self.binarization.render_kw:
del self.binarization.render_kw['disabled']
del self.binarization.render_kw['disabled']
if 'ocropus_nlbin_threshold' in service_info['methods']:
del self.ocropus_nlbin_threshold.render_kw['disabled']
models = [
x for x in TesseractOCRPipelineModel.query.filter().all()
x for x in TesseractOCRPipelineModel.query.order_by(TesseractOCRPipelineModel.title).all()
if version in x.compatible_service_versions and (x.shared == True or x.user == current_user)
]
self.model.choices = [('', 'Choose your option')]
self.model.choices += [(x.hashid, x.title) for x in models]
self.model.choices += [(x.hashid, f'{x.title} [{x.version}]') for x in models]
self.model.default = ''
self.version.choices = [(x, x) for x in service_manifest['versions']]
self.version.data = version
@ -113,8 +116,7 @@ class CreateTranskribusHTRPipelineJobForm(CreateJobBaseForm):
self.binarization.render_kw['disabled'] = True
if 'methods' in service_info:
if 'binarization' in service_info['methods']:
if 'disabled' in self.binarization.render_kw:
del self.binarization.render_kw['disabled']
del self.binarization.render_kw['disabled']
self.model.choices = [('', 'Choose your option')]
self.model.choices += [(x['modelId'], x['name']) for x in transkribus_htr_pipeline_models]
self.model.default = ''
@ -127,7 +129,7 @@ class CreateSpacyNLPPipelineJobForm(CreateJobBaseForm):
encoding_detection = BooleanField('Encoding detection', render_kw={'disabled': True})
txt = FileField('File', validators=[FileRequired()])
model = SelectField('Model', validators=[InputRequired()])
def validate_encoding_detection(self, field):
service_info = SERVICES['spacy-nlp-pipeline']['versions'][self.version.data]
if field.data:
@ -146,15 +148,19 @@ class CreateSpacyNLPPipelineJobForm(CreateJobBaseForm):
version = kwargs.pop('version', service_manifest['latest_version'])
super().__init__(*args, **kwargs)
service_info = service_manifest['versions'][version]
print(service_info)
if self.encoding_detection.render_kw is None:
self.encoding_detection.render_kw = {}
self.encoding_detection.render_kw['disabled'] = True
if 'methods' in service_info:
if 'encoding_detection' in service_info['methods']:
if 'disabled' in self.encoding_detection.render_kw:
del self.encoding_detection.render_kw['disabled']
del self.encoding_detection.render_kw['disabled']
models = [
x for x in SpaCyNLPPipelineModel.query.order_by(SpaCyNLPPipelineModel.title).all()
if version in x.compatible_service_versions and (x.shared == True or x.user == current_user)
]
self.model.choices = [('', 'Choose your option')]
self.model.choices += [(x, y) for x, y in service_info['models'].items()] # noqa
self.model.choices += [(x.hashid, f'{x.title} [{x.version}]') for x in models]
self.model.default = ''
self.version.choices = [(x, x) for x in service_manifest['versions']]
self.version.data = version

View File

@ -6,7 +6,8 @@ from app.models import (
Job,
JobInput,
JobStatus,
TesseractOCRPipelineModel
TesseractOCRPipelineModel,
SpaCyNLPPipelineModel
)
from . import bp, SERVICES
from .forms import (
@ -78,7 +79,8 @@ def tesseract_ocr_pipeline():
service=service_name,
service_args={
'binarization': form.binarization.data,
'model': hashids.decode(form.model.data)
'model': hashids.decode(form.model.data),
'ocropus_nlbin_threshold': float(form.ocropus_nlbin_threshold.data)
},
service_version=form.version.data,
user=current_user
@ -172,6 +174,7 @@ def spacy_nlp_pipeline():
if version not in service_manifest['versions']:
abort(404)
form = CreateSpacyNLPPipelineJobForm(prefix='create-job-form', version=version)
spacy_nlp_pipeline_models = SpaCyNLPPipelineModel.query.all()
if form.is_submitted():
if not form.validate():
response = {'errors': form.errors}
@ -202,6 +205,7 @@ def spacy_nlp_pipeline():
return render_template(
'services/spacy_nlp_pipeline.html.j2',
form=form,
spacy_nlp_pipeline_models=spacy_nlp_pipeline_models,
title=service_manifest['name']
)

View File

@ -1,6 +1,6 @@
# TODO: This could also be done via GitLab/GitHub APIs
file-setup-pipeline:
name: 'File setup pipeline'
name: 'File Setup Pipeline'
publisher: 'Bielefeld University - CRC 1288 - INF'
latest_version: '0.1.0'
versions:
@ -20,6 +20,7 @@ tesseract-ocr-pipeline:
0.1.1:
methods:
- 'binarization'
- 'ocropus_nlbin_threshold'
publishing_year: 2022
url: 'https://gitlab.ub.uni-bielefeld.de/sfb1288inf/tesseract-ocr-pipeline/-/releases/v0.1.1'
transkribus-htr-pipeline:
@ -38,23 +39,17 @@ transkribus-htr-pipeline:
publishing_year: 2022
url: 'https://gitlab.ub.uni-bielefeld.de/sfb1288inf/transkribus-htr-pipeline/-/releases/v0.1.1'
spacy-nlp-pipeline:
name: 'spaCy NLP Pipeline'
name: 'SpaCy NLP Pipeline'
publisher: 'Bielefeld University - CRC 1288 - INF'
latest_version: '0.1.0'
latest_version: '0.1.1'
versions:
0.1.0:
methods:
- 'encoding_detection'
models:
ca: 'Catalan'
de: 'German'
el: 'Greek'
en: 'English'
es: 'Spanish'
fr: 'French'
it: 'Italian'
pl: 'Polish'
ru: 'Russian'
zh: 'Chinese'
publishing_year: 2022
url: 'https://gitlab.ub.uni-bielefeld.de/sfb1288inf/spacy-nlp-pipeline/-/releases/v0.1.0'
0.1.1:
methods:
- 'encoding_detection'
publishing_year: 2022
url: 'https://gitlab.ub.uni-bielefeld.de/sfb1288inf/spacy-nlp-pipeline/-/releases/v0.1.1'

View File

@ -71,10 +71,6 @@ class EditGeneralSettingsForm(FlaskForm):
super().__init__(*args, **kwargs)
self.user = user
def prefill(self, user):
self.email.data = user.email
self.username.data = user.username
def validate_email(self, field):
if (field.data != self.user.email
and User.query.filter_by(email=field.data).first()):
@ -86,13 +82,6 @@ class EditGeneralSettingsForm(FlaskForm):
raise ValidationError('Username already in use')
class EditInterfaceSettingsForm(FlaskForm):
dark_mode = BooleanField('Dark mode')
submit = SubmitField()
def prefill(self, user):
self.dark_mode.data = user.setting_dark_mode
class EditNotificationSettingsForm(FlaskForm):
job_status_mail_notification_level = SelectField(
'Job status mail notification level',

View File

@ -6,7 +6,6 @@ from . import bp
from .forms import (
ChangePasswordForm,
EditGeneralSettingsForm,
EditInterfaceSettingsForm,
EditNotificationSettingsForm
)
@ -20,11 +19,9 @@ def settings():
)
edit_general_settings_form = EditGeneralSettingsForm(
current_user,
obj=current_user,
prefix='edit-general-settings-form'
)
edit_interface_settings_form = EditInterfaceSettingsForm(
prefix='edit-interface-settings-form'
)
edit_notification_settings_form = EditNotificationSettingsForm(
prefix='edit-notification-settings-form'
)
@ -41,13 +38,6 @@ def settings():
db.session.commit()
flash('Your changes have been saved')
return redirect(url_for('.settings'))
if (edit_interface_settings_form.submit.data
and edit_interface_settings_form.validate()):
current_user.setting_dark_mode = (
edit_interface_settings_form.dark_mode.data)
db.session.commit()
flash('Your changes have been saved')
return redirect(url_for('.settings'))
if (edit_notification_settings_form.submit.data
and edit_notification_settings_form.validate()):
current_user.setting_job_status_mail_notification_level = (
@ -58,14 +48,11 @@ def settings():
db.session.commit()
flash('Your changes have been saved')
return redirect(url_for('.settings'))
edit_general_settings_form.prefill(current_user)
edit_interface_settings_form.prefill(current_user)
edit_notification_settings_form.prefill(current_user)
return render_template(
'settings/settings.html.j2',
change_password_form=change_password_form,
edit_general_settings_form=edit_general_settings_form,
edit_interface_settings_form=edit_interface_settings_form,
edit_notification_settings_form=edit_notification_settings_form,
title='Settings'
)

View File

@ -0,0 +1,18 @@
class CreateContributionForm extends Form {
static autoInit() {
let createContributionFormElements = document.querySelectorAll('.create-contribution-form');
for (let createContributionFormElement of createContributionFormElements) {
new CreateContributionForm(createContributionFormElement);
}
}
constructor(formElement) {
super(formElement);
this.addEventListener('requestLoad', (event) => {
if (event.target.status === 201) {
window.location.href = event.target.getResponseHeader('Location');
}
});
}
}

View File

@ -1,5 +1,6 @@
class Form {
static autoInit() {
CreateContributionForm.autoInit();
CreateCorpusFileForm.autoInit();
CreateJobForm.autoInit();
}

View File

@ -1,134 +0,0 @@
class QueryResultList extends RessourceList {
static autoInit() {
for (let queryResultListElement of document.querySelectorAll('.query-result-list:not(.no-autoinit)')) {
new QueryResultList(queryResultListElement);
}
}
static options = {
item: `
<tr class="hoverable">
<td><b class="title"></b><br><i class="description"></i><br></td>
<td><span class="corpus-title"></span><br><span class="query"></span></td>
<td class="right-align">
<a class="action-button btn-floating red waves-effect waves-light" data-action="delete"><i class="material-icons">delete</i></a>
<a class="action-button btn-floating waves-effect waves-light" data-action="view"><i class="material-icons">send</i></a>
</td>
</tr>
`.trim(),
ressourceMapper: queryResult => {
return {
'id': queryResult.id,
'corpus-title': queryResult.corpus_title,
'creation-date': queryResult.creation_date,
'description': queryResult.description,
'query': queryResult.query,
'title': queryResult.title
};
},
sortArgs: ['creation-date', {order: 'desc'}],
valueNames: [
{data: ['id']},
{data: ['creation-date']},
'corpus-title',
'description',
'query',
'title'
]
};
constructor(listElement, options = {}) {
super(listElement, {...QueryResultList.options, ...options});
}
init(user) {
super._init(user.query_results);
}
onclick(event) {
let action;
let actionButtonElement;
let deleteModal;
let deleteModalElement;
let queryResultElement;
let queryResultId;
let tmp;
queryResultElement = event.target.closest('tr[data-id]');
if (queryResultElement === null) {return;}
queryResultId = queryResultElement.dataset.id;
actionButtonElement = event.target.closest('.action-button[data-action]');
action = actionButtonElement === null ? 'view' : actionButtonElement.dataset.action;
switch (action) {
case 'delete':
tmp = document.createElement('div');
tmp.innerHTML = `
<div class="modal">
<div class="modal-content">
<h4>Confirm query result deletion</h4>
<p>Do you really want to delete the query result <b>${app.users[this.userId].query_results[queryResultId].title}</b>? It will be permanently deleted!</p>
</div>
<div class="modal-footer">
<a href="#!" class="btn modal-close waves-effect waves-light">Cancel</a>
<a class="btn modal-close red waves-effect waves-light" href="/query_results/${queryResultId}/delete"><i class="material-icons left">delete</i>Delete</a>
</div>
</div>
`.trim();
deleteModalElement = document.querySelector('#modals').appendChild(tmp.firstChild);
deleteModal = M.Modal.init(
deleteModalElement,
{
onCloseEnd: () => {
deleteModal.destroy();
deleteModalElement.remove();
}
}
);
deleteModal.open();
break;
case 'view':
window.location.href = `/query_results/${queryResultId}`;
break;
default:
break;
}
}
onPATCH(patch) {
let filteredPatch;
let match;
let operation;
let queryResultId;
let re;
let valueName;
re = new RegExp(`^/users/${this.userId}/query_results/([A-Za-z0-9]*)`);
filteredPatch = patch.filter(operation => re.test(operation.path));
for (operation of filteredPatch) {
switch(operation.op) {
case 'add':
re = new RegExp(`^/users/${this.userId}/query_results/([A-Za-z0-9]*)$`);
if (re.test(operation.path)) {
this.add(operation.value);
}
break;
case 'remove':
re = new RegExp(`^/users/${this.userId}/query_results/([A-Za-z0-9]*)$`);
if (re.test(operation.path)) {
[match, queryResultId] = operation.path.match(re);
this.remove(queryResultId);
}
break;
case 'replace':
re = new RegExp(`^/users/${this.userId}/query_results/([A-Za-z0-9]*)/(corpus_title|description|query|title)$`);
if (re.test(operation.path)) {
[match, queryResultId, valueName] = operation.path.match(re);
this.replace(queryResultId, valueName.replace('_', '-'), operation.value);
}
break;
default:
break;
}
}
}
}

View File

@ -10,7 +10,8 @@ class RessourceList {
JobList.autoInit();
JobInputList.autoInit();
JobResultList.autoInit();
QueryResultList.autoInit();
SpaCyNLPPipelineModelList.autoInit();
TesseractOCRPipelineModelList.autoInit();
UserList.autoInit();
}

View File

@ -0,0 +1,135 @@
class SpaCyNLPPipelineModelList extends RessourceList {
static autoInit() {
for (let spaCyNLPPipelineModelListElement of document.querySelectorAll('.spacy-nlp-pipeline-model-list:not(.no-autoinit)')) {
new SpaCyNLPPipelineModelList(spaCyNLPPipelineModelListElement);
}
}
static options = {
initialHtmlGenerator: (id) => {
return `
<div class="input-field">
<i class="material-icons prefix">search</i>
<input id="${id}-search" class="search" type="search"></input>
<label for="${id}-search">Search SpaCy NLP Pipeline Model</label>
</div>
<table>
<thead>
<tr>
<th>Title and Description</th>
<th>Publisher</th>
<th></th>
</tr>
</thead>
<tbody class="list"></tbody>
</table>
<ul class="pagination"></ul>
`.trim();
},
item: `
<tr class="clickable hoverable">
<td><b><span class="title"></span> <span class="version"></span></b><br><i><span class="description"></span></i></td>
<td><a class="publisher-url"><span class="publisher"></span></a> (<span class="publishing-year"></span>)<br><a class="publishing-url"><span class="publishing-url-2"></span></a></td>
<td>
<div class="switch action-switch center-align" data-action="share-request">
<span class="share"></span>
<label>
<input type="checkbox" class="shared">
<span class="lever"></span>
shared
</label>
</div>
</td>
<td class="right-align">
<a class="action-button btn-floating red waves-effect waves-light" data-action="delete-request"><i class="material-icons">delete</i></a>
<a class="action-button btn-floating service-color darken waves-effect waves-light service-2" data-action="view"><i class="material-icons">send</i></a>
</td>
</tr>
`.trim(),
ressourceMapper: (spaCyNLPPipelineModel) => {
return {
'id': spaCyNLPPipelineModel.id,
'creation-date': spaCyNLPPipelineModel.creation_date,
'description': spaCyNLPPipelineModel.description,
'publisher': spaCyNLPPipelineModel.publisher,
'publisher-url': spaCyNLPPipelineModel.publisher_url,
'publishing-url': spaCyNLPPipelineModel.publishing_url,
'publishing-url-2': spaCyNLPPipelineModel.publishing_url,
'publishing-year': spaCyNLPPipelineModel.publishing_year,
'title': spaCyNLPPipelineModel.title,
'title-2': spaCyNLPPipelineModel.title,
'version': spaCyNLPPipelineModel.version,
'shared': spaCyNLPPipelineModel.shared ? 'True' : 'False'
};
},
sortArgs: ['creation-date', {order: 'desc'}],
valueNames: [
{data: ['id']},
{data: ['creation-date']},
{name: 'publisher-url', attr: 'href'},
{name: 'publishing-url', attr: 'href'},
'description',
'publisher',
'publishing-url-2',
'publishing-year',
'title',
'title-2',
'version',
{name: 'shared', attr: 'data-checked'}
]
};
constructor(listElement, options = {}) {
super(listElement, {...SpaCyNLPPipelineModelList.options, ...options});
this.listjs.list.addEventListener('change', (event) => {this.onChange(event)});
}
init (user) {
this._init(user.spacy_nlp_pipeline_models);
}
_init(ressources) {
super._init(ressources);
for (let uncheckedCheckbox of this.listjs.list.querySelectorAll('input[data-checked="True"]')) {
uncheckedCheckbox.setAttribute('checked', '');
}
}
onClick(event) {
if (event.target.closest('.action-switch')) {return;}
let actionButtonElement = event.target.closest('.action-button');
let action = actionButtonElement === null ? 'view' : actionButtonElement.dataset.action;
let spaCyNLPPipelineModelElement = event.target.closest('tr');
let spaCyNLPPipelineModelId = spaCyNLPPipelineModelElement.dataset.id;
switch (action) {
case 'delete-request': {
Utils.deleteSpaCyNLPPipelineModelRequest(this.userId, spaCyNLPPipelineModelId);
break;
}
case 'view': {
window.location.href = `/contributions/spacy-nlp-pipeline-models/${spaCyNLPPipelineModelId}`;
break;
}
default: {
break;
}
}
}
onChange(event) {
let actionSwitchElement = event.target.closest('.action-switch');
let action = actionSwitchElement.dataset.action;
let spaCyNLPPipelineModelElement = event.target.closest('tr');
let spaCyNLPPipelineModelId = spaCyNLPPipelineModelElement.dataset.id;
switch (action) {
case 'share-request': {
let shared = actionSwitchElement.querySelector('input').checked;
Utils.shareSpaCyNLPPipelineModelRequest(this.userId, spaCyNLPPipelineModelId, shared);
break;
}
default: {
break;
}
}
}
}

View File

@ -0,0 +1,135 @@
class TesseractOCRPipelineModelList extends RessourceList {
static autoInit() {
for (let tesseractOCRPipelineModelListElement of document.querySelectorAll('.tesseract-ocr-pipeline-model-list:not(.no-autoinit)')) {
new TesseractOCRPipelineModelList(tesseractOCRPipelineModelListElement);
}
}
static options = {
initialHtmlGenerator: (id) => {
return `
<div class="input-field">
<i class="material-icons prefix">search</i>
<input id="${id}-search" class="search" type="search"></input>
<label for="${id}-search">Search Tesseract OCR Pipeline Model</label>
</div>
<table>
<thead>
<tr>
<th>Title and Description</th>
<th>Publisher</th>
<th></th>
</tr>
</thead>
<tbody class="list"></tbody>
</table>
<ul class="pagination"></ul>
`.trim();
},
item: `
<tr class="clickable hoverable">
<td><b><span class="title"></span> <span class="version"></span></b><br><i><span class="description"></span></i></td>
<td><a class="publisher-url"><span class="publisher"></span></a> (<span class="publishing-year"></span>)<br><a class="publishing-url"><span class="publishing-url-2"></span></a></td>
<td>
<div class="switch action-switch center-align" data-action="share-request">
<span class="share"></span>
<label>
<input type="checkbox" class="shared">
<span class="lever"></span>
shared
</label>
</div>
</td>
<td class="right-align">
<a class="action-button btn-floating red waves-effect waves-light" data-action="delete-request"><i class="material-icons">delete</i></a>
<a class="action-button btn-floating service-color darken waves-effect waves-light service-2" data-action="view"><i class="material-icons">send</i></a>
</td>
</tr>
`.trim(),
ressourceMapper: (tesseractOCRPipelineModel) => {
return {
'id': tesseractOCRPipelineModel.id,
'creation-date': tesseractOCRPipelineModel.creation_date,
'description': tesseractOCRPipelineModel.description,
'publisher': tesseractOCRPipelineModel.publisher,
'publisher-url': tesseractOCRPipelineModel.publisher_url,
'publishing-url': tesseractOCRPipelineModel.publishing_url,
'publishing-url-2': tesseractOCRPipelineModel.publishing_url,
'publishing-year': tesseractOCRPipelineModel.publishing_year,
'title': tesseractOCRPipelineModel.title,
'title-2': tesseractOCRPipelineModel.title,
'version': tesseractOCRPipelineModel.version,
'shared': tesseractOCRPipelineModel.shared ? 'True' : 'False'
};
},
sortArgs: ['creation-date', {order: 'desc'}],
valueNames: [
{data: ['id']},
{data: ['creation-date']},
{name: 'publisher-url', attr: 'href'},
{name: 'publishing-url', attr: 'href'},
'description',
'publisher',
'publishing-url-2',
'publishing-year',
'title',
'title-2',
'version',
{name: 'shared', attr: 'data-checked'}
]
};
constructor(listElement, options = {}) {
super(listElement, {...TesseractOCRPipelineModelList.options, ...options});
this.listjs.list.addEventListener('change', (event) => {this.onChange(event)});
}
init (user) {
this._init(user.tesseract_ocr_pipeline_models);
}
_init(ressources) {
super._init(ressources);
for (let uncheckedCheckbox of this.listjs.list.querySelectorAll('input[data-checked="True"]')) {
uncheckedCheckbox.setAttribute('checked', '');
}
}
onClick(event) {
if (event.target.closest('.action-switch')) {return;}
let actionButtonElement = event.target.closest('.action-button');
let action = actionButtonElement === null ? 'view' : actionButtonElement.dataset.action;
let tesseractOCRPipelineModelElement = event.target.closest('tr');
let tesseractOCRPipelineModelId = tesseractOCRPipelineModelElement.dataset.id;
switch (action) {
case 'delete-request': {
Utils.deleteTesseractOCRPipelineModelRequest(this.userId, tesseractOCRPipelineModelId);
break;
}
case 'view': {
window.location.href = `/contributions/tesseract-ocr-pipeline-models/${tesseractOCRPipelineModelId}`;
break;
}
default: {
break;
}
}
}
onChange(event) {
let actionSwitchElement = event.target.closest('.action-switch');
let action = actionSwitchElement.dataset.action;
let tesseractOCRPipelineModelElement = event.target.closest('tr');
let tesseractOCRPipelineModelId = tesseractOCRPipelineModelElement.dataset.id;
switch (action) {
case 'share-request': {
let shared = actionSwitchElement.querySelector('input').checked;
Utils.shareTesseractOCRPipelineModelRequest(this.userId, tesseractOCRPipelineModelId, shared);
break;
}
default: {
break;
}
}
}
}

View File

@ -33,8 +33,8 @@ class Utils {
`
<div class="modal">
<div class="modal-content">
<h4>Confirm job deletion</h4>
<p>Do you really want to delete the job <b>${corpus.title}</b>? All files will be permanently deleted!</p>
<h4>Confirm Corpus deletion</h4>
<p>Do you really want to delete the Corpus <b>${corpus.title}</b>? All files will be permanently deleted!</p>
</div>
<div class="modal-footer">
<a class="action-button btn modal-close waves-effect waves-light" data-action="cancel">Cancel</a>
@ -84,8 +84,8 @@ class Utils {
`
<div class="modal">
<div class="modal-content">
<h4>Confirm job deletion</h4>
<p>Do you really want to delete the job <b>${corpusFile.title}</b>? All files will be permanently deleted!</p>
<h4>Confirm Corpus File deletion</h4>
<p>Do you really want to delete the Corpus File <b>${corpusFile.title}</b>? All files will be permanently deleted!</p>
</div>
<div class="modal-footer">
<a class="action-button btn modal-close waves-effect waves-light" data-action="cancel">Cancel</a>
@ -126,6 +126,102 @@ class Utils {
});
}
static deleteSpaCyNLPPipelineModelRequest(userId, spaCyNLPPipelineModelId) {
return new Promise((resolve, reject) => {
let spaCyNLPPipelineModel = app.data.users[userId].spacy_nlp_pipeline_models[spaCyNLPPipelineModelId];
let modalElement = Utils.elementFromString(
`
<div class="modal">
<div class="modal-content">
<h4>Confirm SpaCy NLP Pipeline Model deletion</h4>
<p>Do you really want to delete the SpaCy NLP Pipeline Model <b>${spaCyNLPPipelineModel.title}</b>? All files will be permanently deleted!</p>
</div>
<div class="modal-footer">
<a class="action-button btn modal-close waves-effect waves-light" data-action="cancel">Cancel</a>
<a class="action-button btn modal-close red waves-effect waves-light" data-action="confirm">Delete</a>
</div>
</div>
`
);
document.querySelector('#modals').appendChild(modalElement);
let modal = M.Modal.init(
modalElement,
{
dismissible: false,
onCloseEnd: () => {
modal.destroy();
modalElement.remove();
}
}
);
let confirmElement = modalElement.querySelector('.action-button[data-action="confirm"]');
confirmElement.addEventListener('click', (event) => {
let spaCyNLPPipelineModelTitle = spaCyNLPPipelineModel.title;
fetch(`/contributions/spacy-nlp-pipeline-models/${spaCyNLPPipelineModelId}`, {method: 'DELETE'})
.then(
(response) => {
app.flash(`SpaCy NLP Pipeline Model "${spaCyNLPPipelineModelTitle}" marked for deletion`);
resolve(response);
},
(response) => {
if (response.status === 403) {app.flash('Forbidden', 'error');}
if (response.status === 404) {app.flash('Not Found', 'error');}
reject(response);
}
);
});
modal.open();
});
}
static deleteTesseractOCRPipelineModelRequest(userId, tesseractOCRPipelineModelId) {
return new Promise((resolve, reject) => {
let tesseractOCRPipelineModel = app.data.users[userId].tesseract_ocr_pipeline_models[tesseractOCRPipelineModelId];
let modalElement = Utils.elementFromString(
`
<div class="modal">
<div class="modal-content">
<h4>Confirm Tesseract OCR Pipeline Model deletion</h4>
<p>Do you really want to delete the Tesseract OCR Pipeline Model <b>${tesseractOCRPipelineModel.title}</b>? All files will be permanently deleted!</p>
</div>
<div class="modal-footer">
<a class="action-button btn modal-close waves-effect waves-light" data-action="cancel">Cancel</a>
<a class="action-button btn modal-close red waves-effect waves-light" data-action="confirm">Delete</a>
</div>
</div>
`
);
document.querySelector('#modals').appendChild(modalElement);
let modal = M.Modal.init(
modalElement,
{
dismissible: false,
onCloseEnd: () => {
modal.destroy();
modalElement.remove();
}
}
);
let confirmElement = modalElement.querySelector('.action-button[data-action="confirm"]');
confirmElement.addEventListener('click', (event) => {
let tesseractOCRPipelineModelTitle = tesseractOCRPipelineModel.title;
fetch(`/contributions/tesseract-ocr-pipeline-models/${tesseractOCRPipelineModelId}`, {method: 'DELETE'})
.then(
(response) => {
app.flash(`Tesseract OCR Pipeline Model "${tesseractOCRPipelineModelTitle}" marked for deletion`);
resolve(response);
},
(response) => {
if (response.status === 403) {app.flash('Forbidden', 'error');}
if (response.status === 404) {app.flash('Not Found', 'error');}
reject(response);
}
);
});
modal.open();
});
}
static deleteJobRequest(userId, jobId) {
return new Promise((resolve, reject) => {
let job = app.data.users[userId].jobs[jobId];
@ -134,8 +230,8 @@ class Utils {
`
<div class="modal">
<div class="modal-content">
<h4>Confirm job deletion</h4>
<p>Do you really want to delete the job <b>${job.title}</b>? All files will be permanently deleted!</p>
<h4>Confirm Job deletion</h4>
<p>Do you really want to delete the Job <b>${job.title}</b>? All files will be permanently deleted!</p>
</div>
<div class="modal-footer">
<a class="action-button btn modal-close waves-effect waves-light" data-action="cancel">Cancel</a>
@ -231,8 +327,8 @@ class Utils {
`
<div class="modal">
<div class="modal-content">
<h4>Confirm job restart</h4>
<p>Do you really want to restart the job <b>${job.title}</b>? All log and result files will be permanently deleted.</p>
<h4>Confirm Job restart</h4>
<p>Do you really want to restart the Job <b>${job.title}</b>? All Job Results will be permanently deleted.</p>
</div>
<div class="modal-footer">
<a class="action-button btn modal-close waves-effect waves-light" data-action="cancel">Cancel</a>
@ -282,8 +378,8 @@ class Utils {
`
<div class="modal">
<div class="modal-content">
<h4>Confirm job deletion</h4>
<p>Do you really want to delete the user <b>${user.username}</b>? All files will be permanently deleted!</p>
<h4>Confirm User deletion</h4>
<p>Do you really want to delete the User <b>${user.username}</b>? All files will be permanently deleted!</p>
</div>
<div class="modal-footer">
<a class="action-button btn modal-close waves-effect waves-light" data-action="cancel">Cancel</a>
@ -323,4 +419,50 @@ class Utils {
modal.open();
});
}
static shareTesseractOCRPipelineModelRequest(userId, tesseractOCRPipelineModelId, shared) {
return new Promise((resolve, reject) => {
let tesseractOCRPipelineModel = app.data.users[userId].tesseract_ocr_pipeline_models[tesseractOCRPipelineModelId];
let msg = '';
if (shared) {
msg = `Model "${tesseractOCRPipelineModel.title}" is now public`;
} else {
msg = `Model "${tesseractOCRPipelineModel.title}" is now private`;
}
fetch(`/contributions/tesseract-ocr-pipeline-models/${tesseractOCRPipelineModel.id}/toggle-public-status`, {method: 'POST', headers: {Accept: 'application/json'}})
.then(
(response) => {
app.flash(msg);
resolve(response);
},
(response) => {
if (response.status === 403) {app.flash('Forbidden', 'error');}
reject(response);
}
);
});
}
static shareSpaCyNLPPipelineModelRequest(userId, spaCyNLPPipelineModelId, shared) {
return new Promise((resolve, reject) => {
let spaCyNLPPipelineModel = app.data.users[userId].spacy_nlp_pipeline_models[spaCyNLPPipelineModelId];
let msg = '';
if (shared) {
msg = `Model "${spaCyNLPPipelineModel.title}" is now public`;
} else {
msg = `Model "${spaCyNLPPipelineModel.title}" is now private`;
}
fetch(`/contributions/spacy-nlp-pipeline-models/${spaCyNLPPipelineModel.id}/toggle-public-status`, {method: 'POST', headers: {Accept: 'application/json'}})
.then(
(response) => {
app.flash(msg);
resolve(response);
},
(response) => {
if (response.status === 403) {app.flash('Forbidden', 'error');}
reject(response);
}
);
});
}
}

View File

@ -9,6 +9,7 @@
'js/Forms/Form.js',
'js/Forms/CreateCorpusFileForm.js',
'js/Forms/CreateJobForm.js',
'js/Forms/CreateContributionForm.js',
'js/CorpusAnalysis/CQiClient.js',
'js/CorpusAnalysis/CorpusAnalysisApp.js',
'js/CorpusAnalysis/CorpusAnalysisConcordance.js',
@ -23,7 +24,8 @@
'js/RessourceLists/JobList.js',
'js/RessourceLists/JobInputList.js',
'js/RessourceLists/JobResultList.js',
'js/RessourceLists/QueryResultList.js',
'js/RessourceLists/SpacyNLPPipelineModelList.js',
'js/RessourceLists/TesseractOCRPipelineModelList.js',
'js/RessourceLists/UserList.js'
%}
<script src="{{ ASSET_URL }}"></script>

View File

@ -12,6 +12,7 @@
<li><a href="{{ url_for('main.dashboard') }}"><i class="material-icons">dashboard</i>Dashboard</a></li>
<li><a href="{{ url_for('main.dashboard', _anchor='corpora') }}" style="padding-left: 47px;"><i class="nopaque-icons">I</i>My Corpora</a></li>
<li><a href="{{ url_for('main.dashboard', _anchor='jobs') }}" style="padding-left: 47px;"><i class="nopaque-icons">J</i>My Jobs</a></li>
<li><a href="{{ url_for('contributions.contributions') }}"><i class="material-icons">new_label</i>Contribute</a></li>
<li><div class="divider"></div></li>
<li><a class="subheader">Processes & Services</a></li>
<li class="service-color service-color-border border-darken" data-service="file-setup-pipeline" style="border-left: 10px solid; margin-top: 5px;"><a href="{{ url_for('services.file_setup_pipeline') }}"><i class="nopaque-icons service-icon" data-service="file-setup-pipeline"></i>File setup</a></li>
@ -34,7 +35,4 @@
{% if current_user.can(Permission.USE_API) %}
<li><a href="{{ url_for('apifairy.docs') }}"><i class="material-icons">api</i>API</a></li>
{% endif %}
{% if current_user.can(Permission.CONTRIBUTE) %}
<li><a href="{{ url_for('contributions.contributions') }}"><i class="material-icons">new_label</i>Contribute</a></li>
{% endif %}
</ul>

View File

@ -27,38 +27,6 @@
</div>
</form>
<form method="POST">
{{ edit_interface_settings_form.hidden_tag() }}
<div class="card">
<div class="card-content">
<span class="card-title">Interface settings</span>
<div class="row">
<div class="col s12"><p>&nbsp;</p></div>
<div class="col s1">
<p><i class="material-icons">brightness_3</i></p>
</div>
<div class="col s8">
<p>{{ edit_interface_settings_form.dark_mode.label.text }}</p>
<p class="light">Enable dark mode to ease your eyes.</p>
</div>
<div class="col s3 right-align">
<div class="switch">
<label>
{{ edit_interface_settings_form.dark_mode() }}
<span class="lever"></span>
</label>
</div>
</div>
</div>
</div>
<div class="card-action">
<div class="right-align">
{{ wtf.render_field(edit_interface_settings_form.submit, material_icon='send') }}
</div>
</div>
</div>
</form>
<form method="POST">
{{ edit_notification_settings_form.hidden_tag() }}
<div class="card">

View File

@ -5,12 +5,6 @@
{% block head %}
{{ super() }}
<link href="{{ url_for('static', filename='images/nopaque_-_favicon.png') }}" rel="icon">
{% if current_user.setting_dark_mode %}
<script src="https://cdnjs.cloudflare.com/ajax/libs/darkreader/4.9.40/darkreader.min.js" integrity="sha512-0Jbi9gWSyU5SvNS16za0aILl6l+MgM8N+TGlZxy4qPQEzqKoU9egh4h56Kz0xp2R+ZFPQMfeDn26Gh6cqu2WAg==" crossorigin="anonymous" referrerpolicy="no-referrer"></script>
<script>
DarkReader.enable();
</script>
{% endif %}
{% endblock head %}
{% block metas %}

View File

@ -0,0 +1,46 @@
{% set breadcrumbs %}
<li class="tab disabled"><i class="material-icons">navigate_next</i></li>
{% if request.path == url_for('.contributions') %}
<li class="tab"><a class="active" href="{{ url_for('.contributions') }}" target="_self">Contributions Overview</a></li>
{% elif request.path == url_for('.tesseract_ocr_pipeline_models')%}
<li class="tab"><a href="{{ url_for('.contributions') }}" target="_self">Contributions Overview</a></li>
<li class="tab disabled"><i class="material-icons">navigate_next</i></li>
<li class="tab"><a class="active" href="{{ url_for('.tesseract_ocr_pipeline_models') }}" target="_self">Tesseract OCR Pipeline Models</a></li>
{% elif request.path == url_for('.spacy_nlp_pipeline_models')%}
<li class="tab"><a href="{{ url_for('.contributions') }}" target="_self">Contributions Overview</a></li>
<li class="tab disabled"><i class="material-icons">navigate_next</i></li>
<li class="tab"><a class="active" href="{{ url_for('.spacy_nlp_pipeline_models') }}" target="_self">SpaCy NLP Pipeline Models</a></li>
{% elif request.path == url_for('.create_tesseract_ocr_pipeline_model') %}
<li class="tab"><a href="{{ url_for('.contributions') }}" target="_self">Contributions Overview</a></li>
<li class="tab disabled"><i class="material-icons">navigate_next</i></li>
<li class="tab"><a href="{{ url_for('.tesseract_ocr_pipeline_models') }}" target="_self">Tesseract OCR Pipeline Models</a></li>
<li class="tab disabled"><i class="material-icons">navigate_next</i></li>
<li class="tab"><a class="active" href="{{ url_for('.create_tesseract_ocr_pipeline_model') }}" target="_self">{{ title }}</a></li>
{% elif request.path == url_for('.create_spacy_nlp_pipeline_model') %}
<li class="tab"><a href="{{ url_for('.contributions') }}" target="_self">Contributions Overview</a></li>
<li class="tab disabled"><i class="material-icons">navigate_next</i></li>
<li class="tab"><a href="{{ url_for('.spacy_nlp_pipeline_models') }}" target="_self">SpaCy NLP Pipeline Models</a></li>
<li class="tab disabled"><i class="material-icons">navigate_next</i></li>
<li class="tab"><a class="active" href="{{ url_for('.create_spacy_nlp_pipeline_model') }}" target="_self">{{ title }}</a></li>
{% elif tesseract_ocr_pipeline_model and request.path == url_for('.tesseract_ocr_pipeline_model', tesseract_ocr_pipeline_model_id=tesseract_ocr_pipeline_model.id) %}
<li class="tab"><a href="{{ url_for('.contributions') }}" target="_self">Contributions Overview</a></li>
<li class="tab disabled"><i class="material-icons">navigate_next</i></li>
<li class="tab"><a href="{{ url_for('.tesseract_ocr_pipeline_models') }}" target="_self">Tesseract OCR Pipeline Models</a></li>
<li class="tab disabled"><i class="material-icons">navigate_next</i></li>
<li class="tab">
<a class="active" href="{{ url_for('.tesseract_ocr_pipeline_model', tesseract_ocr_pipeline_model_id=tesseract_ocr_pipeline_model.id) }}" target="_self">
{{ tesseract_ocr_pipeline_model.title }} {{ tesseract_ocr_pipeline_model.version }}
</a>
</li>
{% elif spacy_nlp_pipeline_model and request.path == url_for('.spacy_nlp_pipeline_model', spacy_nlp_pipeline_model_id=spacy_nlp_pipeline_model.id) %}
<li class="tab"><a href="{{ url_for('.contributions') }}" target="_self">Contributions Overview</a></li>
<li class="tab disabled"><i class="material-icons">navigate_next</i></li>
<li class="tab"><a href="{{ url_for('.spacy_nlp_pipeline_models') }}" target="_self">SpaCy NLP Pipeline Models</a></li>
<li class="tab disabled"><i class="material-icons">navigate_next</i></li>
<li class="tab">
<a class="active" href="{{ url_for('.spacy_nlp_pipeline_model', spacy_nlp_pipeline_model_id=spacy_nlp_pipeline_model.id) }}" target="_self">
{{ spacy_nlp_pipeline_model.title }} {{ spacy_nlp_pipeline_model.version }}
</a>
</li>
{% endif %}
{% endset %}

View File

@ -1,32 +0,0 @@
{% extends "base.html.j2" %}
{% import "materialize/wtf.html.j2" as wtf %}
{% block page_content %}
<div class="container">
<div class="row">
<div class="col s12 m8 offset-m2">
<h1 id="title">{{ title }}</h1>
<p>
In order to add a new model, please fill in the form below.
</p>
<form method="POST">
<div class="card-panel">
{{ form.hidden_tag() }}
{{ wtf.render_field(form.title) }}
{{ wtf.render_field(form.description) }}
{{ wtf.render_field(form.publisher) }}
{{ wtf.render_field(form.publisher_url) }}
{{ wtf.render_field(form.publishing_url) }}
{{ wtf.render_field(form.publishing_year) }}
{{ wtf.render_field(form.shared) }}
{{ wtf.render_field(form.version) }}
{{ wtf.render_field(form.compatible_service_versions) }}
{{ wtf.render_field(form.submit, class_='width-100', material_icon='send') }}
</div>
</form>
</div>
</div>
{% endblock page_content %}

View File

@ -0,0 +1,45 @@
{% extends "base.html.j2" %}
{% import "materialize/wtf.html.j2" as wtf %}
{% from "contributions/_breadcrumbs.html.j2" import breadcrumbs with context %}
{% block page_content %}
<div class="container">
<div class="row">
<div class="col s12">
<h1 id="title">{{ title }}</h1>
</div>
<div class="col s4">
<div class="card extension-selector hoverable service-color" data-service="tesseract-ocr-pipeline">
<a href="{{ url_for('.tesseract_ocr_pipeline_models') }}" style="position: absolute; width: 100%; height: 100%;"></a>
<div class="card-content">
<span class="card-title" data-service="tesseract-ocr-pipeline"><i class="nopaque-icons service-icon" data-service="tesseract-ocr-pipeline"></i>Tesseract OCR Pipeline Models</span>
<p>Here you can see and edit the models that you have created. You can also create new models.</p>
</div>
</div>
</div>
<div class="col s4">
<div class="card extension-selector hoverable service-color" data-service="spacy-nlp-pipeline">
<a href="{{ url_for('.spacy_nlp_pipeline_models') }}" style="position: absolute; width: 100%; height: 100%;"></a>
<div class="card-content">
<span class="card-title"><i class="nopaque-icons service-icon" data-service="spacy-nlp-pipeline"></i>SpaCy NLP Pipeline Models</span>
<p>Here you can see and edit the models that you have created. You can also create new models.</p>
</div>
</div>
</div>
<!--
<div class="col s4">
<div class="card extension-selector hoverable service-color" data-service="transkribus-htr-pipeline">
<a href="" style="position: absolute; width: 100%; height: 100%;"></a>
<div class="card-content">
<span class="card-title"><i class="nopaque-icons service-icon" data-service="transkribus-htr-pipeline"></i>Transkribus HTR Pipeline Models</span>
<p>Here you can see and edit the models that you have created. You can also create new models.</p>
</div>
</div>
</div>
-->
</div>
</div>
{% endblock page_content %}

View File

@ -0,0 +1,85 @@
{% extends "base.html.j2" %}
{% import "materialize/wtf.html.j2" as wtf %}
{% from "contributions/_breadcrumbs.html.j2" import breadcrumbs with context %}
{% block main_attribs %} class="service-scheme" data-service="spacy-nlp-pipeline"{% endblock main_attribs %}
{% block page_content %}
<div class="container">
<div class="row">
<div class="col s12">
<h1 id="title">{{ title }}</h1>
</div>
<div class="col s12 m3 push-m9">
<div class="center-align">
<p class="hide-on-small-only">&nbsp;</p>
<p class="hide-on-small-only">&nbsp;</p>
<a class="btn-floating btn-large btn-scale-x2 waves-effect waves-light">
<i class="nopaque-icons service-color darken service-icon" data-service="spacy-nlp-pipeline"></i>
</a>
</div>
</div>
<div class="col s12 m9 pull-m3">
<div class="card service-color-border border-darken" data-service="spacy-nlp-pipeline" style="border-top: 10px solid;">
<div class="card-content">
<div class="row">
<div class="col s12">
<div class="card-panel z-depth-0">
<span class="card-title"><i class="left material-icons">layers</i>SpaCy NLP Pipeline Model</span>
<p>You can create a new SpaCy NLP Pipeline Model using the form below. They will automatically appear in the list of usable models on the <a href="{{ url_for('services.spacy_nlp_pipeline') }}">SpaCy NLP Pipeline</a> service page.</p>
</div>
</div>
</div>
</div>
</div>
</div>
<div class="col s12">
<div class="card">
<form class="create-contribution-form" enctype="multipart/form-data" method="POST">
<div class="card-content">
{{ form.hidden_tag() }}
<div class="row">
<div class="col s12 l5">
{{ wtf.render_field(form.spacy_model_file, accept='.tar.gz', placeholder='Choose a .tar.gz file') }}
</div>
<div class="col s12 l7">
{{ wtf.render_field(form.title, material_icon='title') }}
</div>
<div class="col s12">
{{ wtf.render_field(form.description, material_icon='description') }}
</div>
<div class="col s12">
{{ wtf.render_field(form.pipeline_name, material_icon='emoji_objects') }}
</div>
<div class="col s12 l6">
{{ wtf.render_field(form.publisher, material_icon='account_balance') }}
</div>
<div class="col s12 l6">
{{ wtf.render_field(form.publishing_year, material_icon='calendar_month') }}
</div>
<div class="col s12">
{{ wtf.render_field(form.publisher_url, material_icon='link') }}
</div>
<div class="col s12">
{{ wtf.render_field(form.publishing_url, material_icon='link') }}
</div>
<div class="col s12 l10">
{{ wtf.render_field(form.version, material_icon='apps') }}
</div>
<div class="col s12 l6">
{{ wtf.render_field(form.compatible_service_versions) }}
</div>
</div>
</div>
<div class="card-action right-align">
{{ wtf.render_field(form.submit, material_icon='send') }}
</div>
</form>
</div>
</div>
</div>
</div>
{% endblock page_content %}

View File

@ -0,0 +1,116 @@
{% extends "base.html.j2" %}
{% import "materialize/wtf.html.j2" as wtf %}
{% from "contributions/_breadcrumbs.html.j2" import breadcrumbs with context %}
{% block main_attribs %} class="service-scheme" data-service="tesseract-ocr-pipeline"{% endblock main_attribs %}
{% block page_content %}
<div class="container">
<div class="row">
<div class="col s12">
<h1 id="title">{{ title }}</h1>
</div>
<div class="col s12 m3 push-m9">
<div class="center-align">
<p class="hide-on-small-only">&nbsp;</p>
<p class="hide-on-small-only">&nbsp;</p>
<a class="btn-floating btn-large btn-scale-x2 waves-effect waves-light">
<i class="nopaque-icons service-color darken service-icon" data-service="tesseract-ocr-pipeline"></i>
</a>
</div>
</div>
<div class="col s12 m9 pull-m3">
<div class="card service-color-border border-darken" data-service="tesseract-ocr-pipeline" style="border-top: 10px solid;">
<div class="card-content">
<div class="row">
<div class="col s12">
<div class="card-panel z-depth-0">
<span class="card-title"><i class="left material-icons">layers</i>Tesseract OCR Pipeline Model</span>
<p>You can create a new Tesseract OCR Pipeline Model using the form below. They will automatically appear in the list of usable models on the <a href="{{ url_for('services.tesseract_ocr_pipeline') }}">Tesseract OCR Pipeline</a> service page.</p>
</div>
</div>
</div>
</div>
</div>
</div>
<div class="col s12">
<div class="card">
<form class="create-contribution-form" enctype="multipart/form-data" method="POST">
<div class="card-content">
{{ form.hidden_tag() }}
<div class="row">
<div class="col s12 l5">
{{ wtf.render_field(form.tesseract_model_file, accept='.traineddata', placeholder='Choose a .traineddata file') }}
</div>
<div class="col s12 l7">
{{ wtf.render_field(form.title, material_icon='title') }}
</div>
<div class="col s12">
{{ wtf.render_field(form.description, material_icon='description') }}
</div>
<div class="col s12 l6">
{{ wtf.render_field(form.publisher, material_icon='account_balance') }}
</div>
<div class="col s12 l6">
{{ wtf.render_field(form.publishing_year, material_icon='calendar_month') }}
</div>
<div class="col s12">
{{ wtf.render_field(form.publisher_url, material_icon='link') }}
</div>
<div class="col s12">
{{ wtf.render_field(form.publishing_url, material_icon='link') }}
</div>
<div class="col s12 l10">
{{ wtf.render_field(form.version, material_icon='apps') }}
</div>
<div class="col s12 l6">
{{ wtf.render_field(form.compatible_service_versions) }}
</div>
</div>
</div>
<div class="card-action right-align">
{{ wtf.render_field(form.submit, material_icon='send') }}
</div>
</form>
</div>
</div>
</div>
</div>
{% endblock page_content %}
{% block modals %}
{{ super() }}
<div id="models-modal" class="modal">
<div class="modal-content">
<h4>Tesseract OCR Pipeline models</h4>
<table>
<thead>
<tr>
<th>Title</th>
<th>Description</th>
<th>Biblio</th>
</tr>
</thead>
<tbody>
{% for m in tesseract_ocr_pipeline_models %}
<tr id="tesseract-ocr-pipeline-model-{{ m.hashid }}">
<td>{{ m.title }}</td>
{% if m.description == '' %}
<td>Description is not available.</td>
{% else %}
<td>{{ m.description }}</td>
{% endif %}
<td><a href="{{ m.publisher_url }}">{{ m.publisher }}</a> ({{ m.publishing_year }}), {{ m.title }} {{ m.version}}, <a href="{{ m.publishing_url }}">{{ m.publishing_url }}</a></td>
</tr>
{% endfor %}
</tbody>
</table>
</div>
<div class="modal-footer">
<a href="#!" class="modal-close waves-effect waves-light btn">Close</a>
</div>
</div>
{% endblock modals %}

View File

@ -0,0 +1,57 @@
{% extends "base.html.j2" %}
{% import "materialize/wtf.html.j2" as wtf %}
{% from "contributions/_breadcrumbs.html.j2" import breadcrumbs with context %}
{% block main_attribs %} class="service-scheme" data-service="spacy-nlp-pipeline"{% endblock main_attribs %}
{% block page_content %}
<div class="container">
<div class="row">
<div class="col s12">
<h1 id="title">{{ title }}</h1>
</div>
<div class="col s12">
<div class="card">
<form method="POST">
<div class="card-content">
{{ form.hidden_tag() }}
<div class="row">
<div class="col s12 l7">
{{ wtf.render_field(form.title, material_icon='title') }}
</div>
<div class="col s12">
{{ wtf.render_field(form.description, material_icon='description') }}
</div>
<div class="col s12">
{{ wtf.render_field(form.pipeline_name, material_icon='emoji_objects') }}
</div>
<div class="col s12 l6">
{{ wtf.render_field(form.publisher, material_icon='account_balance') }}
</div>
<div class="col s12 l6">
{{ wtf.render_field(form.publishing_year, material_icon='calendar_month') }}
</div>
<div class="col s12">
{{ wtf.render_field(form.publisher_url, material_icon='link') }}
</div>
<div class="col s12">
{{ wtf.render_field(form.publishing_url, material_icon='link') }}
</div>
<div class="col s12 l10">
{{ wtf.render_field(form.version, material_icon='apps') }}
</div>
<div class="col s12 l6">
{{ wtf.render_field(form.compatible_service_versions) }}
</div>
</div>
</div>
<div class="card-action right-align">
{{ wtf.render_field(form.submit, material_icon='send') }}
</div>
</form>
</div>
</div>
</div>
</div>
{% endblock page_content %}

View File

@ -0,0 +1,25 @@
{% extends "base.html.j2" %}
{% import "materialize/wtf.html.j2" as wtf %}
{% from "contributions/_breadcrumbs.html.j2" import breadcrumbs with context %}
{% block page_content %}
<div class="container">
<div class="row">
<div class="col s12">
<h1 id="title">{{ title }}</h1>
<p>Here you can see and edit the models that you have created. You can also create new models.</p>
</div>
<div class="col s12">
<div class="card">
<div class="card-content">
<div class="spacy-nlp-pipeline-model-list" data-user-id="{{ current_user.hashid }}"></div>
</div>
<div class="card-action right-align">
<a href="{{ url_for('.create_spacy_nlp_pipeline_model') }}" class="btn waves-effect waves-light"><i class="material-icons left">add</i>Create</a>
</div>
</div>
</div>
</div>
</div>
{% endblock page_content %}

View File

@ -0,0 +1,54 @@
{% extends "base.html.j2" %}
{% import "materialize/wtf.html.j2" as wtf %}
{% from "contributions/_breadcrumbs.html.j2" import breadcrumbs with context %}
{% block main_attribs %} class="service-scheme" data-service="tesseract-ocr-pipeline"{% endblock main_attribs %}
{% block page_content %}
<div class="container">
<div class="row">
<div class="col s12">
<h1 id="title">{{ title }}</h1>
</div>
<div class="col s12">
<div class="card">
<form method="POST">
<div class="card-content">
{{ form.hidden_tag() }}
<div class="row">
<div class="col s12 l7">
{{ wtf.render_field(form.title, material_icon='title') }}
</div>
<div class="col s12">
{{ wtf.render_field(form.description, material_icon='description') }}
</div>
<div class="col s12 l6">
{{ wtf.render_field(form.publisher, material_icon='account_balance') }}
</div>
<div class="col s12 l6">
{{ wtf.render_field(form.publishing_year, material_icon='calendar_month') }}
</div>
<div class="col s12">
{{ wtf.render_field(form.publisher_url, material_icon='link') }}
</div>
<div class="col s12">
{{ wtf.render_field(form.publishing_url, material_icon='link') }}
</div>
<div class="col s12 l10">
{{ wtf.render_field(form.version, material_icon='apps') }}
</div>
<div class="col s12 l6">
{{ wtf.render_field(form.compatible_service_versions) }}
</div>
</div>
</div>
<div class="card-action right-align">
{{ wtf.render_field(form.submit, material_icon='send') }}
</div>
</form>
</div>
</div>
</div>
</div>
{% endblock page_content %}

View File

@ -0,0 +1,25 @@
{% extends "base.html.j2" %}
{% import "materialize/wtf.html.j2" as wtf %}
{% from "contributions/_breadcrumbs.html.j2" import breadcrumbs with context %}
{% block page_content %}
<div class="container">
<div class="row">
<div class="col s12">
<h1 id="title">{{ title }}</h1>
<p>Here you can see and edit the models that you have created. You can also create new models.</p>
</div>
<div class="col s12">
<div class="card">
<div class="card-content">
<div class="tesseract-ocr-pipeline-model-list" data-user-id="{{ current_user.hashid }}"></div>
</div>
<div class="card-action right-align">
<a href="{{ url_for('.create_tesseract_ocr_pipeline_model') }}" class="btn waves-effect waves-light"><i class="material-icons left">add</i>Create</a>
</div>
</div>
</div>
</div>
</div>
{% endblock page_content %}

View File

@ -1,9 +1,5 @@
{% extends "base.html.j2" %}
{% import "materialize/wtf.html.j2" as wtf %}
<style>
a {color: #FFFFFF;}
</style>
{% block main_attribs %} class="service-scheme" data-service="corpus-analysis" id="corpus-analysis-app-container"{% endblock main_attribs %}
{% block page_content %}

View File

@ -6,27 +6,18 @@
<div class="row">
<div class="col s12">
<h1 id="title">{{ title }}</h1>
</div>
<h3>My Corpora and Query results</h3>
<p>Create a corpus to interactively perform linguistic analysis or import query results to save interesting passages.</p>
<div class="row">
<div class="col s12">
<ul class="tabs">
<li class="tab col s6"><a class="active" href="#corpora">Corpora</a></li>
<li class="tab col s6 disabled"><a href="#query-results">Query results</a></li>
</ul>
<div class="col s12" id="corpora">
<h3>My Corpora</h3>
<p>Create a corpus to interactively perform linguistic analysis.</p>
<div class="card">
<div class="card-content">
<div class="corpus-list" data-user-id="{{ current_user.hashid }}"></div>
</div>
<div class="col s12" id="corpora">
<div class="card">
<div class="card-content">
<div class="corpus-list" data-user-id="{{ current_user.hashid }}"></div>
</div>
<div class="card-action right-align">
<a class="btn disabled waves-effect waves-light" href="{{ url_for('corpora.import_corpus') }}">Import Corpus<i class="material-icons right">import_export</i></a>
<a class="btn waves-effect waves-light" href="{{ url_for('corpora.create_corpus') }}">Create corpus<i class="material-icons right">add</i></a>
</div>
</div>
<div class="card-action right-align">
<a class="btn disabled waves-effect waves-light" href="{{ url_for('corpora.import_corpus') }}">Import Corpus<i class="material-icons right">import_export</i></a>
<a class="btn waves-effect waves-light" href="{{ url_for('corpora.create_corpus') }}">Create corpus<i class="material-icons right">add</i></a>
</div>
</div>
</div>

View File

@ -70,8 +70,16 @@
<div class="col s12 l5">
{{ wtf.render_field(form.txt, accept='text/plain', placeholder='Choose a plain text file') }}
</div>
<div class="col s12 l4">
{{ wtf.render_field(form.model, material_icon='language') }}
<div class="col s12 l4">
<div class="input-field">
<i class="material-icons prefix">language</i>
{{ form.model() }}
{{ form.model.label }}
<span class="helper-text">
<a class="modal-trigger tooltipped" href="#models-modal" data-position="bottom" data-tooltip="See more information about models"><i class="material-icons" style="color:#0064A3;">help_outline</i></a>
<a class="tooltipped" href="{{ url_for('contributions.create_spacy_nlp_pipeline_model') }}" data-position="bottom" data-tooltip="Add your own spaCy NLP models"><i class="material-icons" style="color:#0064A3">new_label</i></a>
</span>
</div>
</div>
<div class="col s12 l3">
{{ wtf.render_field(form.version, material_icon='apps') }}
@ -79,6 +87,7 @@
<div class="col s12">
<span class="card-title">Preprocessing</span>
</div>
{% if 'disabled' not in form.encoding_detection.render_kw or not form.encoding_detection.render_kw['disabled'] %}
<div class="col s9">
<p>{{ form.encoding_detection.label.text }}</p>
<p class="light">If the input files are not created with the nopaque OCR service or you do not know if your text files are UTF-8 encoded, check this switch. We will try to automatically determine the right encoding for your texts to process them.</p>
@ -91,6 +100,7 @@
</label>
</div>
</div>
{% endif %}
<!--
Seperate each setting with the following
<div class="col s12"><p>&nbsp;</p></div>
@ -122,4 +132,35 @@
<a href="#!" class="modal-close waves-effect waves-light btn red abort-request">Cancel</a>
</div>
</div>
<div id="models-modal" class="modal">
<div class="modal-content">
<h4>spaCy NLP Pipeline models</h4>
<table>
<thead>
<tr>
<th>Title</th>
<th>Description</th>
<th>Biblio</th>
</tr>
</thead>
<tbody>
{% for m in spacy_nlp_pipeline_models %}
<tr id="spacy-nlp-pipeline-model-{{ m.hashid }}">
<td>{{ m.title }}</td>
{% if m.description == '' %}
<td>Description is not available.</td>
{% else %}
<td>{{ m.description }}</td>
{% endif %}
<td><a href="{{ m.publisher_url }}">{{ m.publisher }}</a> ({{ m.publishing_year }}), {{ m.title }} {{ m.version}}, <a href="{{ m.publishing_url }}">{{ m.publishing_url }}</a></td>
</tr>
{% endfor %}
</tbody>
</table>
</div>
<div class="modal-footer">
<a href="#!" class="modal-close waves-effect waves-light btn">Close</a>
</div>
</div>
{% endblock modals %}

View File

@ -58,7 +58,8 @@
{{ form.model() }}
{{ form.model.label }}
<span class="helper-text">
<a class="modal-trigger" href="#models-modal">More details about models</a>
<a class="modal-trigger tooltipped" href="#models-modal" data-position="bottom" data-tooltip="See more information about models"><i class="material-icons" style="color:#00A58B;">help_outline</i></a>
<a class="tooltipped" href="{{ url_for('contributions.create_tesseract_ocr_pipeline_model') }}" data-position="bottom" data-tooltip="Add your own Tesseract OCR models"><i class="material-icons" style="color:#00A58B">new_label</i></a>
</span>
{% for error in form.model.errors %}
<span class="helper-text error-color-text">{{ error }}</span>
@ -71,6 +72,7 @@
<div class="col s12">
<span class="card-title">Preprocessing</span>
</div>
{% if 'disabled' not in form.binarization.render_kw or not form.binarization.render_kw['disabled'] %}
<div class="col s9">
<p>{{ form.binarization.label.text }}</p>
<p class="light">Based on a brightness threshold pixels are converted into either black or white. It is useful to reduce noise in images. (<b>longer duration</b>)</p>
@ -83,51 +85,14 @@
</label>
</div>
</div>
<div class="col s12"><p>&nbsp;</p></div>
<div class="col s12 divider"></div>
<div class="col s12"><p>&nbsp;</p></div>
<div class="col s9">
<p>Page range</p>
<p class="light"></p>
</div>
<div class="col s3 right-align">
<div class="switch">
<label>
<input disabled type="checkbox">
<span class="lever"></span>
</label>
</div>
</div>
<div class="col s12"><p>&nbsp;</p></div>
<div class="col s12 divider"></div>
<div class="col s12"><p>&nbsp;</p></div>
<div class="col s9">
<p>Page rotation</p>
<p class="light"></p>
</div>
<div class="col s3 right-align">
<div class="switch">
<label>
<input disabled type="checkbox">
<span class="lever"></span>
</label>
</div>
</div>
<div class="col s12"><p>&nbsp;</p></div>
<div class="col s12 divider"></div>
<div class="col s12"><p>&nbsp;</p></div>
<div class="col s9">
<p>Page split</p>
<p class="light"></p>
</div>
<div class="col s3 right-align">
<div class="switch">
<label>
<input disabled type="checkbox">
<span class="lever"></span>
</label>
</div>
{% if 'disabled' not in form.ocropus_nlbin_threshold.render_kw or not form.ocropus_nlbin_threshold.render_kw['disabled'] %}
<div class="col s9 hide" id="create-job-form-ocropus_nlbin_threshold-container">
<br>
<p>Intensity (between 0 and 1)</p>
<p class="range-field">{{ form.ocropus_nlbin_threshold() }}</p>
</div>
{% endif %}
{% endif %}
<!--
Seperate each setting with the following
<div class="col s12"><p>&nbsp;</p></div>
@ -179,3 +144,12 @@
</div>
</div>
{% endblock modals %}
{% block scripts %}
{{ super() }}
<script>
document.querySelector('#create-job-form-binarization').addEventListener('change', (event) => {
document.querySelector('#create-job-form-ocropus_nlbin_threshold-container').classList.toggle('hide');
});
</script>
{% endblock scripts %}

View File

@ -76,6 +76,7 @@
<div class="col s12">
<span class="card-title">Preprocessing</span>
</div>
{% if 'disabled' not in form.binarization.render_kw or not form.binarization.render_kw['disabled'] %}
<div class="col s9">
<p>{{ form.binarization.label.text }}</p>
<p class="light">Based on a brightness threshold pixels are converted into either black or white. It is useful to reduce noise in images. (<b>longer duration</b>)</p>
@ -88,51 +89,7 @@
</label>
</div>
</div>
<div class="col s12"><p>&nbsp;</p></div>
<div class="col s12 divider"></div>
<div class="col s12"><p>&nbsp;</p></div>
<div class="col s9">
<p>Page range</p>
<p class="light"></p>
</div>
<div class="col s3 right-align">
<div class="switch">
<label>
<input disabled type="checkbox">
<span class="lever"></span>
</label>
</div>
</div>
<div class="col s12"><p>&nbsp;</p></div>
<div class="col s12 divider"></div>
<div class="col s12"><p>&nbsp;</p></div>
<div class="col s9">
<p>Page rotation</p>
<p class="light"></p>
</div>
<div class="col s3 right-align">
<div class="switch">
<label>
<input disabled type="checkbox">
<span class="lever"></span>
</label>
</div>
</div>
<div class="col s12"><p>&nbsp;</p></div>
<div class="col s12 divider"></div>
<div class="col s12"><p>&nbsp;</p></div>
<div class="col s9">
<p>Page split</p>
<p class="light"></p>
</div>
<div class="col s3 right-align">
<div class="switch">
<label>
<input disabled type="checkbox">
<span class="lever"></span>
</label>
</div>
</div>
{% endif %}
<!--
Seperate each setting with the following
<div class="col s12"><p>&nbsp;</p></div>

View File

@ -27,38 +27,6 @@
</div>
</form>
<form method="POST">
{{ edit_interface_settings_form.hidden_tag() }}
<div class="card">
<div class="card-content">
<span class="card-title">Interface settings</span>
<div class="row">
<div class="col s12"><p>&nbsp;</p></div>
<div class="col s1">
<p><i class="material-icons">brightness_3</i></p>
</div>
<div class="col s8">
<p>{{ edit_interface_settings_form.dark_mode.label.text }}</p>
<p class="light">Enable dark mode to ease your eyes.</p>
</div>
<div class="col s3 right-align">
<div class="switch">
<label>
{{ edit_interface_settings_form.dark_mode() }}
<span class="lever"></span>
</label>
</div>
</div>
</div>
</div>
<div class="card-action">
<div class="right-align">
{{ wtf.render_field(edit_interface_settings_form.submit, material_icon='send') }}
</div>
</div>
</div>
</form>
<form method="POST">
{{ edit_notification_settings_form.hidden_tag() }}
<div class="card">

View File

@ -1,7 +1,5 @@
#!/bin/bash
source venv/bin/activate
display_help() {
local script_name=$(basename "${0}")
echo ""
@ -26,7 +24,7 @@ if [[ "${#}" -eq 0 ]]; then
sleep 5
done
fi
python nopaque.py
python3 nopaque.py
elif [[ "${1}" == "flask" ]]; then
flask "${@:2}"
elif [[ "${1}" == "--help" || "${1}" == "-h" ]]; then

View File

@ -0,0 +1,27 @@
"""Add pipeline_name column to spacy_nlp_pipeline_models table
Revision ID: 721829b5dd25
Revises: 31dd42e5ea6f
Create Date: 2022-11-04 13:58:13.008301
"""
from alembic import op
import sqlalchemy as sa
# revision identifiers, used by Alembic.
revision = '721829b5dd25'
down_revision = '31dd42e5ea6f'
branch_labels = None
depends_on = None
def upgrade():
op.add_column(
'spacy_nlp_pipeline_models',
sa.Column('pipeline_name', sa.String(length=64), nullable=True)
)
def downgrade():
op.drop_column('spacy_nlp_pipeline_models', 'pipeline_name')

View File

@ -0,0 +1,26 @@
"""Remove setting_dark_mode column from users table
Revision ID: 89e9526089bf
Revises: 721829b5dd25
Create Date: 2022-11-17 09:47:27.724692
"""
from alembic import op
import sqlalchemy as sa
# revision identifiers, used by Alembic.
revision = '89e9526089bf'
down_revision = '721829b5dd25'
branch_labels = None
depends_on = None
def upgrade():
op.drop_column('users', 'setting_dark_mode')
def downgrade():
op.add_column('users',
sa.Column('setting_dark_mode', sa.Boolean(), nullable=True)
)