diff --git a/.dockerignore b/.dockerignore index 07d50b4b..9960fd26 100644 --- a/.dockerignore +++ b/.dockerignore @@ -1,29 +1,12 @@ -**/__pycache__ -**/.venv -**/.classpath -**/.dockerignore -**/.env -**/.git -**/.gitignore -**/.project -**/.settings -**/.toolstarget -**/.vs -**/.vscode -**/*.*proj.user -**/*.dbmdl -**/*.jfm -**/bin -**/charts -**/docker-compose* -**/compose* -**/Dockerfile* -**/node_modules -**/npm-debug.log -**/obj -**/secrets.dev.yaml -**/values.dev.yaml -README.md - - -data +# Exclude everything +* + +# Include what we need +!app +!migrations +!tests +!.flaskenv +!boot.sh +!config.py +!nopaque.py +!requirements.txt diff --git a/.gitignore b/.gitignore index 14a22fe1..59ada396 100644 --- a/.gitignore +++ b/.gitignore @@ -1,42 +1,168 @@ +# nopaque specifics +app/static/gen/ +data/ +docker-compose.override.yml +logs/ +!logs/dummy +*.env + +# Byte-compiled / optimized / DLL files +__pycache__/ *.py[cod] +*$py.class # C extensions *.so -# Flask-Assets files -.webassets-cache -app/static/gen +# Distribution / packaging +.Python +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +wheels/ +share/python-wheels/ +*.egg-info/ +.installed.cfg +*.egg +MANIFEST -# Docker related files -docker-compose.override.yml -data/** - -# Environment files -*.env +# PyInstaller +# Usually these files are written by a python script from a template +# before PyInstaller builds the exe, so as to inject date/other infos into it. +*.manifest +*.spec # Installer logs pip-log.txt +pip-delete-this-directory.txt -# Logs in log folder -logs/* -!logs/dummy +# Unit test / coverage reports +htmlcov/ +.tox/ +.nox/ +.coverage +.coverage.* +.cache +nosetests.xml +coverage.xml +*.cover +*.py,cover +.hypothesis/ +.pytest_cache/ +cover/ -# Packages -*.egg -*.egg-info -dist -build -eggs -parts -bin -var -sdist -develop-eggs -.installed.cfg -lib -lib64 -__pycache__ +# Translations +*.mo +*.pot -# Virtual environment -venv -.idea +# Django stuff: +*.log +local_settings.py +db.sqlite3 +db.sqlite3-journal + +# Flask stuff: +instance/ +.webassets-cache + +# Scrapy stuff: +.scrapy + +# Sphinx documentation +docs/_build/ + +# PyBuilder +.pybuilder/ +target/ + +# Jupyter Notebook +.ipynb_checkpoints + +# IPython +profile_default/ +ipython_config.py + +# pyenv +# For a library or package, you might want to ignore these files since the code is +# intended to run in multiple environments; otherwise, check them in: +# .python-version + +# pipenv +# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. +# However, in case of collaboration, if having platform-specific dependencies or dependencies +# having no cross-platform support, pipenv may install dependencies that don't work, or not +# install all needed dependencies. +#Pipfile.lock + +# poetry +# Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control. +# This is especially recommended for binary packages to ensure reproducibility, and is more +# commonly ignored for libraries. +# https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control +#poetry.lock + +# pdm +# Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control. +#pdm.lock +# pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it +# in version control. +# https://pdm.fming.dev/#use-with-ide +.pdm.toml + +# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm +__pypackages__/ + +# Celery stuff +celerybeat-schedule +celerybeat.pid + +# SageMath parsed files +*.sage.py + +# Environments +.env +.venv +env/ +venv/ +ENV/ +env.bak/ +venv.bak/ + +# Spyder project settings +.spyderproject +.spyproject + +# Rope project settings +.ropeproject + +# mkdocs documentation +/site + +# mypy +.mypy_cache/ +.dmypy.json +dmypy.json + +# Pyre type checker +.pyre/ + +# pytype static type analyzer +.pytype/ + +# Cython debug symbols +cython_debug/ + +# PyCharm +# JetBrains specific template is maintained in a separate JetBrains.gitignore that can +# be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore +# and can be added to the global gitignore or merged into this file. For a more nuclear +# option (not recommended) you can uncomment the following to ignore the entire idea folder. +#.idea/ diff --git a/Dockerfile b/Dockerfile index 0a2309f5..8ccdf7f6 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,44 +1,50 @@ -FROM python:3.8.10-slim-buster - - -LABEL authors="Patrick Jentsch " - - -ARG DOCKER_GID -ARG UID -ARG GID - - -ENV FLASK_APP nopaque.py -ENV LANG=C.UTF-8 -ENV PYTHONDONTWRITEBYTECODE=1 -ENV PYTHONUNBUFFERED=1 - - -RUN apt-get update \ - && apt-get install --no-install-recommends --yes \ - build-essential \ - libpq-dev \ - && rm -r /var/lib/apt/lists/* - - -RUN groupadd --gid ${DOCKER_GID} --system docker \ - && groupadd --gid ${GID} --system nopaque \ - && useradd --create-home --gid ${GID} --groups ${DOCKER_GID} --no-log-init --system --uid ${UID} nopaque -USER nopaque -WORKDIR /home/nopaque - -COPY --chown=nopaque:nopaque requirements.txt ./ -RUN python -m venv venv \ - && venv/bin/pip install --requirement requirements.txt - - -COPY --chown=nopaque:nopaque app app -COPY --chown=nopaque:nopaque migrations migrations -COPY --chown=nopaque:nopaque tests tests -COPY --chown=nopaque:nopaque boot.sh config.py nopaque.py ./ - - -# run-time configuration -EXPOSE 5000 -ENTRYPOINT ["./boot.sh"] +FROM python:3.9.15-slim-bullseye + + +LABEL authors="Patrick Jentsch " + + +ARG DOCKER_GID +ARG UID +ARG GID + + +ENV LANG="C.UTF-8" +ENV PYTHONDONTWRITEBYTECODE="1" +ENV PYTHONUNBUFFERED="1" + + +RUN apt-get update \ + && apt-get install --no-install-recommends --yes \ + build-essential \ + libpq-dev \ + && rm --recursive /var/lib/apt/lists/* + + +RUN groupadd --gid "${DOCKER_GID}" docker \ + && groupadd --gid "${GID}" nopaque \ + && useradd --create-home --gid nopaque --groups "${DOCKER_GID}" --no-log-init --uid "${UID}" nopaque +USER nopaque +WORKDIR /home/nopaque + + +ENV PYTHON3_VENV_PATH="/home/nopaque/venv" +RUN python3 -m venv "${PYTHON3_VENV_PATH}" +ENV PATH="${PYTHON3_VENV_PATH}/bin:${PATH}" + + +COPY --chown=nopaque:nopaque requirements.txt . +RUN python3 -m pip install --requirement requirements.txt \ + && rm requirements.txt + + +COPY --chown=nopaque:nopaque app app +COPY --chown=nopaque:nopaque migrations migrations +COPY --chown=nopaque:nopaque tests tests +COPY --chown=nopaque:nopaque .flaskenv boot.sh config.py nopaque.py ./ + + +EXPOSE 5000 + + +ENTRYPOINT ["./boot.sh"] diff --git a/app/SpaCyNLPPipelineModel.defaults.yml b/app/SpaCyNLPPipelineModel.defaults.yml index 576f85e4..f8f3116b 100644 --- a/app/SpaCyNLPPipelineModel.defaults.yml +++ b/app/SpaCyNLPPipelineModel.defaults.yml @@ -1,10 +1,221 @@ -- title: 'de_core_news_md-3.4.0' +- title: 'Catalan' + description: 'Catalan pipeline optimized for CPU. Components: tok2vec, morphologizer, parser, senter, ner, attribute_ruler, lemmatizer.' + url: 'https://github.com/explosion/spacy-models/releases/download/ca_core_news_md-3.2.0/ca_core_news_md-3.2.0.tar.gz' + publisher: 'Explosion' + publisher_url: 'https://github.com/explosion' + publishing_url: 'https://github.com/explosion/spacy-models/releases/tag/ca_core_news_md-3.2.0' + publishing_year: 2021 + pipeline_name: 'ca_core_news_md' + version: '3.2.0' + compatible_service_versions: + - '0.1.0' +- title: 'German' + description: 'German pipeline optimized for CPU. Components: tok2vec, tagger, morphologizer, parser, senter, ner, attribute_ruler, lemmatizer.' + url: 'https://github.com/explosion/spacy-models/releases/download/de_core_news_md-3.2.0/de_core_news_md-3.2.0.tar.gz' + publisher: 'Explosion' + publisher_url: 'https://github.com/explosion' + publishing_url: 'https://github.com/explosion/spacy-models/releases/tag/de_core_news_md-3.2.0' + publishing_year: 2021 + pipeline_name: 'de_core_news_md' + version: '3.2.0' + compatible_service_versions: + - '0.1.0' +- title: 'Greek' + description: 'Greek pipeline optimized for CPU. Components: tok2vec, morphologizer, parser, senter, ner, attribute_ruler, lemmatizer.' + url: 'https://github.com/explosion/spacy-models/releases/download/el_core_news_md-3.2.0/el_core_news_md-3.2.0.tar.gz' + publisher: 'Explosion' + publisher_url: 'https://github.com/explosion' + publishing_url: 'https://github.com/explosion/spacy-models/releases/tag/el_core_news_md-3.2.0' + publishing_year: 2021 + pipeline_name: 'el_core_news_md' + version: '3.2.0' + compatible_service_versions: + - '0.1.0' +- title: 'English' + description: 'English pipeline optimized for CPU. Components: tok2vec, tagger, parser, senter, ner, attribute_ruler, lemmatizer.' + url: 'https://github.com/explosion/spacy-models/releases/download/en_core_web_md-3.2.0/en_core_web_md-3.2.0.tar.gz' + publisher: 'Explosion' + publisher_url: 'https://github.com/explosion' + publishing_url: 'https://github.com/explosion/spacy-models/releases/tag/en_core_web_md-3.2.0' + publishing_year: 2021 + pipeline_name: 'en_core_web_md' + version: '3.2.0' + compatible_service_versions: + - '0.1.0' +- title: 'Spanish' + description: 'Spanish pipeline optimized for CPU. Components: tok2vec, morphologizer, parser, senter, ner, attribute_ruler, lemmatizer.' + url: 'https://github.com/explosion/spacy-models/releases/download/es_core_news_md-3.2.0/es_core_news_md-3.2.0.tar.gz' + publisher: 'Explosion' + publisher_url: 'https://github.com/explosion' + publishing_url: 'https://github.com/explosion/spacy-models/releases/tag/es_core_news_md-3.2.0' + publishing_year: 2021 + pipeline_name: 'es_core_news_md' + version: '3.2.0' + compatible_service_versions: + - '0.1.0' +- title: 'French' + description: 'French pipeline optimized for CPU. Components: tok2vec, morphologizer, parser, senter, ner, attribute_ruler, lemmatizer.' + url: 'https://github.com/explosion/spacy-models/releases/download/fr_core_news_md-3.2.0/fr_core_news_md-3.2.0.tar.gz' + publisher: 'Explosion' + publisher_url: 'https://github.com/explosion' + publishing_url: 'https://github.com/explosion/spacy-models/releases/tag/fr_core_news_md-3.2.0' + publishing_year: 2021 + pipeline_name: 'fr_core_news_md' + version: '3.2.0' + compatible_service_versions: + - '0.1.0' +- title: 'Italian' + description: 'Italian pipeline optimized for CPU. Components: tok2vec, morphologizer, parser, senter, ner, attribute_ruler, lemmatizer.' + url: 'https://github.com/explosion/spacy-models/releases/download/it_core_news_md-3.2.0/it_core_news_md-3.2.0.tar.gz' + publisher: 'Explosion' + publisher_url: 'https://github.com/explosion' + publishing_url: 'https://github.com/explosion/spacy-models/releases/tag/it_core_news_md-3.2.0' + publishing_year: 2021 + pipeline_name: 'it_core_news_md' + version: '3.2.0' + compatible_service_versions: + - '0.1.0' +- title: 'Polish' + description: 'Polish pipeline optimized for CPU. Components: tok2vec, morphologizer, parser, senter, ner, attribute_ruler, lemmatizer.' + url: 'https://github.com/explosion/spacy-models/releases/download/pl_core_news_md-3.2.0/pl_core_news_md-3.2.0.tar.gz' + publisher: 'Explosion' + publisher_url: 'https://github.com/explosion' + publishing_url: 'https://github.com/explosion/spacy-models/releases/tag/pl_core_news_md-3.2.0' + publishing_year: 2021 + pipeline_name: 'pl_core_news_md' + version: '3.2.0' + compatible_service_versions: + - '0.1.0' +- title: 'Russian' + description: 'Russian pipeline optimized for CPU. Components: tok2vec, morphologizer, parser, senter, ner, attribute_ruler, lemmatizer.' + url: 'https://github.com/explosion/spacy-models/releases/download/ru_core_news_md-3.2.0/ru_core_news_md-3.2.0.tar.gz' + publisher: 'Explosion' + publisher_url: 'https://github.com/explosion' + publishing_url: 'https://github.com/explosion/spacy-models/releases/tag/ru_core_news_md-3.2.0' + publishing_year: 2021 + pipeline_name: 'ru_core_news_md' + version: '3.2.0' + compatible_service_versions: + - '0.1.0' +- title: 'Chinese' + description: 'Chinese pipeline optimized for CPU. Components: tok2vec, tagger, parser, senter, ner, attribute_ruler.' + url: 'https://github.com/explosion/spacy-models/releases/download/zh_core_web_md-3.2.0/zh_core_web_md-3.2.0.tar.gz' + publisher: 'Explosion' + publisher_url: 'https://github.com/explosion' + publishing_url: 'https://github.com/explosion/spacy-models/releases/tag/zh_core_web_md-3.2.0' + publishing_year: 2021 + pipeline_name: 'zh_core_web_md' + version: '3.2.0' + compatible_service_versions: + - '0.1.0' + +- title: 'Catalan' + description: 'Catalan pipeline optimized for CPU. Components: tok2vec, morphologizer, parser, senter, ner, attribute_ruler, lemmatizer.' + url: 'https://github.com/explosion/spacy-models/releases/download/ca_core_news_md-3.4.0/ca_core_news_md-3.4.0.tar.gz' + publisher: 'Explosion' + publisher_url: 'https://github.com/explosion' + publishing_url: 'https://github.com/explosion/spacy-models/releases/tag/ca_core_news_md-3.4.0' + publishing_year: 2022 + pipeline_name: 'ca_core_news_md' + version: '3.4.0' + compatible_service_versions: + - '0.1.1' +- title: 'German' description: 'German pipeline optimized for CPU. Components: tok2vec, tagger, morphologizer, parser, lemmatizer (trainable_lemmatizer), senter, ner.' url: 'https://github.com/explosion/spacy-models/releases/download/de_core_news_md-3.4.0/de_core_news_md-3.4.0.tar.gz' publisher: 'Explosion' publisher_url: 'https://github.com/explosion' publishing_url: 'https://github.com/explosion/spacy-models/releases/tag/de_core_news_md-3.4.0' publishing_year: 2022 + pipeline_name: 'de_core_news_md' version: '3.4.0' compatible_service_versions: - - '0.1.0' + - '0.1.1' +- title: 'Greek' + description: 'Greek pipeline optimized for CPU. Components: tok2vec, morphologizer, parser, lemmatizer (trainable_lemmatizer), senter, ner, attribute_ruler.' + url: 'https://github.com/explosion/spacy-models/releases/download/el_core_news_md-3.4.0/el_core_news_md-3.4.0.tar.gz' + publisher: 'Explosion' + publisher_url: 'https://github.com/explosion' + publishing_url: 'https://github.com/explosion/spacy-models/releases/tag/el_core_news_md-3.4.0' + publishing_year: 2022 + pipeline_name: 'el_core_news_md' + version: '3.4.0' + compatible_service_versions: + - '0.1.1' +- title: 'English' + description: 'English pipeline optimized for CPU. Components: tok2vec, tagger, parser, senter, ner, attribute_ruler, lemmatizer.' + url: 'https://github.com/explosion/spacy-models/releases/download/en_core_web_md-3.4.1/en_core_web_md-3.4.1.tar.gz' + publisher: 'Explosion' + publisher_url: 'https://github.com/explosion' + publishing_url: 'https://github.com/explosion/spacy-models/releases/tag/en_core_web_md-3.4.1' + publishing_year: 2022 + pipeline_name: 'en_core_web_md' + version: '3.4.1' + compatible_service_versions: + - '0.1.1' +- title: 'Spanish' + description: 'Spanish pipeline optimized for CPU. Components: tok2vec, morphologizer, parser, senter, ner, attribute_ruler, lemmatizer.' + url: 'https://github.com/explosion/spacy-models/releases/download/es_core_news_md-3.4.0/es_core_news_md-3.4.0.tar.gz' + publisher: 'Explosion' + publisher_url: 'https://github.com/explosion' + publishing_url: 'https://github.com/explosion/spacy-models/releases/tag/es_core_news_md-3.4.0' + publishing_year: 2022 + pipeline_name: 'es_core_news_md' + version: '3.4.0' + compatible_service_versions: + - '0.1.1' +- title: 'French' + description: 'French pipeline optimized for CPU. Components: tok2vec, morphologizer, parser, senter, ner, attribute_ruler, lemmatizer.' + url: 'https://github.com/explosion/spacy-models/releases/download/fr_core_news_md-3.4.0/fr_core_news_md-3.4.0.tar.gz' + publisher: 'Explosion' + publisher_url: 'https://github.com/explosion' + publishing_url: 'https://github.com/explosion/spacy-models/releases/tag/fr_core_news_md-3.4.0' + publishing_year: 2022 + pipeline_name: 'fr_core_news_md' + version: '3.4.0' + compatible_service_versions: + - '0.1.1' +- title: 'Italian' + description: 'Italian pipeline optimized for CPU. Components: tok2vec, morphologizer, tagger, parser, lemmatizer (trainable_lemmatizer), senter, ner' + url: 'https://github.com/explosion/spacy-models/releases/download/it_core_news_md-3.4.0/it_core_news_md-3.4.0.tar.gz' + publisher: 'Explosion' + publisher_url: 'https://github.com/explosion' + publishing_url: 'https://github.com/explosion/spacy-models/releases/tag/it_core_news_md-3.4.0' + publishing_year: 2022 + pipeline_name: 'it_core_news_md' + version: '3.4.0' + compatible_service_versions: + - '0.1.1' +- title: 'Polish' + description: 'Polish pipeline optimized for CPU. Components: tok2vec, morphologizer, parser, lemmatizer (trainable_lemmatizer), tagger, senter, ner.' + url: 'https://github.com/explosion/spacy-models/releases/download/pl_core_news_md-3.4.0/pl_core_news_md-3.4.0.tar.gz' + publisher: 'Explosion' + publisher_url: 'https://github.com/explosion' + publishing_url: 'https://github.com/explosion/spacy-models/releases/tag/pl_core_news_md-3.4.0' + publishing_year: 2022 + pipeline_name: 'pl_core_news_md' + version: '3.4.0' + compatible_service_versions: + - '0.1.1' +- title: 'Russian' + description: 'Russian pipeline optimized for CPU. Components: tok2vec, morphologizer, parser, senter, ner, attribute_ruler, lemmatizer.' + url: 'https://github.com/explosion/spacy-models/releases/download/ru_core_news_md-3.4.0/ru_core_news_md-3.4.0.tar.gz' + publisher: 'Explosion' + publisher_url: 'https://github.com/explosion' + publishing_url: 'https://github.com/explosion/spacy-models/releases/tag/ru_core_news_md-3.4.0' + publishing_year: 2022 + pipeline_name: 'ru_core_news_md' + version: '3.4.0' + compatible_service_versions: + - '0.1.1' +- title: 'Chinese' + description: 'Chinese pipeline optimized for CPU. Components: tok2vec, tagger, parser, senter, ner, attribute_ruler.' + url: 'https://github.com/explosion/spacy-models/releases/download/zh_core_web_md-3.4.0/zh_core_web_md-3.4.0.tar.gz' + publisher: 'Explosion' + publisher_url: 'https://github.com/explosion' + publishing_url: 'https://github.com/explosion/spacy-models/releases/tag/zh_core_web_md-3.4.0' + publishing_year: 2022 + pipeline_name: 'zh_core_web_md' + version: '3.4.0' + compatible_service_versions: + - '0.1.1' diff --git a/app/admin/routes.py b/app/admin/routes.py index 011de1bb..8f4370fd 100644 --- a/app/admin/routes.py +++ b/app/admin/routes.py @@ -6,7 +6,6 @@ from app.decorators import admin_required from app.models import Role, User, UserSettingJobStatusMailNotificationLevel from app.settings.forms import ( EditGeneralSettingsForm, - EditInterfaceSettingsForm, EditNotificationSettingsForm ) from . import bp @@ -49,15 +48,14 @@ def user(user_id): def edit_user(user_id): user = User.query.get_or_404(user_id) admin_edit_user_form = AdminEditUserForm( + obj=user, prefix='admin-edit-user-form' ) edit_general_settings_form = EditGeneralSettingsForm( user, + obj=user, prefix='edit-general-settings-form' ) - edit_interface_settings_form = EditInterfaceSettingsForm( - prefix='edit-interface-settings-form' - ) edit_notification_settings_form = EditNotificationSettingsForm( prefix='edit-notification-settings-form' ) @@ -76,12 +74,6 @@ def edit_user(user_id): db.session.commit() flash('Your changes have been saved') return redirect(url_for('.edit_user', user_id=user.id)) - if (edit_interface_settings_form.submit.data - and edit_interface_settings_form.validate()): - user.setting_dark_mode = edit_interface_settings_form.dark_mode.data - db.session.commit() - flash('Your changes have been saved') - return redirect(url_for('.edit_user', user_id=user.id)) if (edit_notification_settings_form.submit.data and edit_notification_settings_form.validate()): user.setting_job_status_mail_notification_level = \ @@ -91,15 +83,11 @@ def edit_user(user_id): db.session.commit() flash('Your changes have been saved') return redirect(url_for('.edit_user', user_id=user.id)) - admin_edit_user_form.prefill(user) - edit_general_settings_form.prefill(user) - edit_interface_settings_form.prefill(user) edit_notification_settings_form.prefill(user) return render_template( 'admin/edit_user.html.j2', admin_edit_user_form=admin_edit_user_form, edit_general_settings_form=edit_general_settings_form, - edit_interface_settings_form=edit_interface_settings_form, edit_notification_settings_form=edit_notification_settings_form, title='Edit user', user=user diff --git a/app/api/schemas.py b/app/api/schemas.py index 9474bd1a..7abb56de 100644 --- a/app/api/schemas.py +++ b/app/api/schemas.py @@ -150,7 +150,6 @@ class UserSchema(ma.SQLAlchemySchema): last_seen = ma.auto_field(dump_only=True) password = ma.String(load_only=True) last_seen = ma.auto_field(dump_only=True) - setting_dark_mode = ma.auto_field() setting_job_status_mail_notification_level = ma.String( validate=validate.OneOf(list(UserSettingJobStatusMailNotificationLevel.__members__.keys())) ) diff --git a/app/contributions/forms.py b/app/contributions/forms.py index 44279a1d..0ba8f5d5 100644 --- a/app/contributions/forms.py +++ b/app/contributions/forms.py @@ -1,3 +1,4 @@ +from flask import current_app from flask_wtf import FlaskForm from flask_wtf.file import FileField, FileRequired from wtforms import ( @@ -5,13 +6,14 @@ from wtforms import ( StringField, SubmitField, SelectMultipleField, - IntegerField + IntegerField, + ValidationError ) from wtforms.validators import InputRequired, Length from app.services import SERVICES -class TesseractOCRModelContributionForm(FlaskForm): +class ContributionBaseForm(FlaskForm): title = StringField( 'Title', validators=[InputRequired(), Length(max=64)] @@ -24,9 +26,6 @@ class TesseractOCRModelContributionForm(FlaskForm): 'Version', validators=[InputRequired(), Length(max=16)] ) - compatible_service_versions = SelectMultipleField( - 'Compatible service versions' - ) publisher = StringField( 'Publisher', validators=[InputRequired(), Length(max=128)] @@ -43,11 +42,23 @@ class TesseractOCRModelContributionForm(FlaskForm): 'Publishing year', validators=[InputRequired()] ) - shared = BooleanField('Shared', validators=[InputRequired()]) - model_file = FileField('File',validators=[FileRequired()]) + compatible_service_versions = SelectMultipleField( + 'Compatible service versions' + ) submit = SubmitField() +class CreateTesseractOCRPipelineModelForm(ContributionBaseForm): + tesseract_model_file = FileField( + 'File', + validators=[FileRequired()] + ) + + def validate_tesseract_model_file(self, field): + current_app.logger.warning(field.data.filename) + if not field.data.filename.lower().endswith('.traineddata'): + raise ValidationError('traineddata files only!') + def __init__(self, *args, **kwargs): service_manifest = SERVICES['tesseract-ocr-pipeline'] super().__init__(*args, **kwargs) @@ -56,3 +67,57 @@ class TesseractOCRModelContributionForm(FlaskForm): (x, x) for x in service_manifest['versions'].keys() ] self.compatible_service_versions.default = '' + + +class CreateSpaCyNLPPipelineModelForm(ContributionBaseForm): + spacy_model_file = FileField( + 'File', + validators=[FileRequired()] + ) + pipeline_name = StringField( + 'Pipeline name', + validators=[InputRequired(), Length(max=64)] + ) + + def validate_spacy_model_file(self, field): + current_app.logger.warning(field.data.filename) + if not field.data.filename.lower().endswith('.tar.gz'): + raise ValidationError('.tar.gz files only!') + + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + service_manifest = SERVICES['spacy-nlp-pipeline'] + self.compatible_service_versions.choices = [('', 'Choose your option')] + self.compatible_service_versions.choices += [ + (x, x) for x in service_manifest['versions'].keys() + ] + self.compatible_service_versions.default = '' + + +class EditContributionBaseForm(ContributionBaseForm): + pass + +class EditTesseractOCRPipelineModelForm(EditContributionBaseForm): + def __init__(self, *args, **kwargs): + service_manifest = SERVICES['tesseract-ocr-pipeline'] + super().__init__(*args, **kwargs) + self.compatible_service_versions.choices = [('', 'Choose your option')] + self.compatible_service_versions.choices += [ + (x, x) for x in service_manifest['versions'].keys() + ] + self.compatible_service_versions.default = '' + + +class EditSpaCyNLPPipelineModelForm(EditContributionBaseForm): + pipeline_name = StringField( + 'Pipeline name', + validators=[InputRequired(), Length(max=64)] + ) + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + service_manifest = SERVICES['spacy-nlp-pipeline'] + self.compatible_service_versions.choices = [('', 'Choose your option')] + self.compatible_service_versions.choices += [ + (x, x) for x in service_manifest['versions'].keys() + ] + self.compatible_service_versions.default = '' diff --git a/app/contributions/routes.py b/app/contributions/routes.py index 287eda18..6852979f 100644 --- a/app/contributions/routes.py +++ b/app/contributions/routes.py @@ -1,54 +1,233 @@ -from flask import abort, flash, Markup, render_template, url_for -from flask_login import login_required +from flask import ( + abort, + current_app, + flash, + Markup, + redirect, + render_template, + url_for +) +from flask_login import login_required, current_user +from threading import Thread from app import db -from app.decorators import permission_required -from app.models import TesseractOCRPipelineModel, Permission +from app.decorators import permission_required +from app.models import ( + Permission, + SpaCyNLPPipelineModel, + TesseractOCRPipelineModel +) from . import bp -from .forms import TesseractOCRModelContributionForm +from .forms import ( + CreateSpaCyNLPPipelineModelForm, + CreateTesseractOCRPipelineModelForm, + EditSpaCyNLPPipelineModelForm, + EditTesseractOCRPipelineModelForm +) @bp.before_request @login_required -@permission_required(Permission.CONTRIBUTE) def before_request(): pass -@bp.route('') +@bp.route('/') def contributions(): - pass - - -@bp.route('/tesseract-ocr-pipeline-models', methods=['GET', 'POST']) -def tesseract_ocr_pipeline_models(): - form = TesseractOCRModelContributionForm( - prefix='contribute-tesseract-ocr-pipeline-model-form' + return render_template( + 'contributions/contributions.html.j2', + title='Contributions' ) + + +@bp.route('/tesseract-ocr-pipeline-models') +def tesseract_ocr_pipeline_models(): + return render_template( + 'contributions/tesseract_ocr_pipeline_models.html.j2', + title='Tesseract OCR Pipeline Models' + ) + + +@bp.route('/tesseract-ocr-pipeline-models/', methods=['GET', 'POST']) +def tesseract_ocr_pipeline_model(tesseract_ocr_pipeline_model_id): + tesseract_ocr_pipeline_model = TesseractOCRPipelineModel.query.get_or_404(tesseract_ocr_pipeline_model_id) + form = EditTesseractOCRPipelineModelForm( + obj=tesseract_ocr_pipeline_model, + prefix='edit-tesseract-ocr-pipeline-model-form' + ) + if form.validate_on_submit(): + form.populate_obj(tesseract_ocr_pipeline_model) + if db.session.is_modified(tesseract_ocr_pipeline_model): + message = Markup(f'Tesseract OCR Pipeline model "{tesseract_ocr_pipeline_model.title}" updated') + flash(message) + db.session.commit() + return redirect(url_for('.tesseract_ocr_pipeline_models')) + return render_template( + 'contributions/tesseract_ocr_pipeline_model.html.j2', + form=form, + tesseract_ocr_pipeline_model=tesseract_ocr_pipeline_model, + title=f'{tesseract_ocr_pipeline_model.title} {tesseract_ocr_pipeline_model.version}' + ) + + +@bp.route('/tesseract-ocr-pipeline-models/', methods=['DELETE']) +def delete_tesseract_model(tesseract_ocr_pipeline_model_id): + def _delete_tesseract_ocr_pipeline_model(app, tesseract_ocr_pipeline_model_id): + with app.app_context(): + tesseract_ocr_pipeline_model = TesseractOCRPipelineModel.query.get(tesseract_ocr_pipeline_model_id) + tesseract_ocr_pipeline_model.delete() + db.session.commit() + + tesseract_ocr_pipeline_model = TesseractOCRPipelineModel.query.get_or_404(tesseract_ocr_pipeline_model_id) + if not (tesseract_ocr_pipeline_model.user == current_user or current_user.is_administrator()): + abort(403) + thread = Thread( + target=_delete_tesseract_ocr_pipeline_model, + args=(current_app._get_current_object(), tesseract_ocr_pipeline_model_id) + ) + thread.start() + return {}, 202 + + +@bp.route('/tesseract-ocr-pipeline-models/create', methods=['GET', 'POST']) +def create_tesseract_ocr_pipeline_model(): + form = CreateTesseractOCRPipelineModelForm(prefix='create-tesseract-ocr-pipeline-model-form') if form.is_submitted(): if not form.validate(): response = {'errors': form.errors} return response, 400 try: - tesseract_ocr_model = TesseractOCRPipelineModel.create( - form.file.data, + tesseract_ocr_pipeline_model = TesseractOCRPipelineModel.create( + form.tesseract_model_file.data, compatible_service_versions=form.compatible_service_versions.data, description=form.description.data, publisher=form.publisher.data, publisher_url=form.publisher_url.data, publishing_url=form.publishing_url.data, publishing_year=form.publishing_year.data, - shared=form.shared.data, + shared=False, title=form.title.data, - version=form.version.data + version=form.version.data, + user=current_user ) except OSError: abort(500) db.session.commit() - message = Markup(f'Model "{tesseract_ocr_model.title}" created') + tesseract_ocr_pipeline_model_url = url_for( + '.tesseract_ocr_pipeline_model', + tesseract_ocr_pipeline_model_id=tesseract_ocr_pipeline_model.id + ) + message = Markup(f'Tesseract OCR Pipeline model "{tesseract_ocr_pipeline_model.title}" created') flash(message) - return {}, 201, {'Location': url_for('contributions.contributions')} + return {}, 201, {'Location': tesseract_ocr_pipeline_model_url} return render_template( - 'contributions/contribute.html.j2', + 'contributions/create_tesseract_ocr_pipeline_model.html.j2', form=form, - title='Contribution' + title='Create Tesseract OCR Pipeline Model' ) + +@bp.route('/tesseract-ocr-pipeline-models//toggle-public-status', methods=['POST']) +@permission_required(Permission.CONTRIBUTE) +def toggle_tesseract_ocr_pipeline_model_public_status(tesseract_ocr_pipeline_model_id): + tesseract_ocr_pipeline_model = TesseractOCRPipelineModel.query.get_or_404(tesseract_ocr_pipeline_model_id) + if not (tesseract_ocr_pipeline_model.user == current_user or current_user.is_administrator()): + abort(403) + tesseract_ocr_pipeline_model.shared = not tesseract_ocr_pipeline_model.shared + db.session.commit() + return {}, 201 + + +@bp.route('/spacy-nlp-pipeline-models') +def spacy_nlp_pipeline_models(): + return render_template( + 'contributions/spacy_nlp_pipeline_models.html.j2', + title='SpaCy NLP Pipeline Models' + ) + + +@bp.route('/spacy-nlp-pipeline-models/', methods=['GET', 'POST']) +def spacy_nlp_pipeline_model(spacy_nlp_pipeline_model_id): + spacy_nlp_pipeline_model = SpaCyNLPPipelineModel.query.get_or_404(spacy_nlp_pipeline_model_id) + form = EditSpaCyNLPPipelineModelForm( + obj=spacy_nlp_pipeline_model, + prefix='edit-spacy-nlp-pipeline-model-form' + ) + if form.validate_on_submit(): + form.populate_obj(spacy_nlp_pipeline_model) + if db.session.is_modified(spacy_nlp_pipeline_model): + message = Markup(f'SpaCy NLP Pipeline model "{spacy_nlp_pipeline_model.title}" updated') + flash(message) + db.session.commit() + return redirect(url_for('.spacy_nlp_pipeline_models')) + return render_template( + 'contributions/spacy_nlp_pipeline_model.html.j2', + form=form, + spacy_nlp_pipeline_model=spacy_nlp_pipeline_model, + title=f'{spacy_nlp_pipeline_model.title} {spacy_nlp_pipeline_model.version}' + ) + +@bp.route('/spacy-nlp-pipeline-models/', methods=['DELETE']) +def delete_spacy_model(spacy_nlp_pipeline_model_id): + def _delete_spacy_model(app, spacy_nlp_pipeline_model_id): + with app.app_context(): + spacy_nlp_pipeline_model = SpaCyNLPPipelineModel.query.get(spacy_nlp_pipeline_model_id) + spacy_nlp_pipeline_model.delete() + db.session.commit() + + spacy_nlp_pipeline_model = SpaCyNLPPipelineModel.query.get_or_404(spacy_nlp_pipeline_model_id) + if not (spacy_nlp_pipeline_model.user == current_user or current_user.is_administrator()): + abort(403) + thread = Thread( + target=_delete_spacy_model, + args=(current_app._get_current_object(), spacy_nlp_pipeline_model_id) + ) + thread.start() + return {}, 202 + + +@bp.route('/spacy-nlp-pipeline-models/create', methods=['GET', 'POST']) +def create_spacy_nlp_pipeline_model(): + form = CreateSpaCyNLPPipelineModelForm(prefix='create-spacy-nlp-pipeline-model-form') + if form.is_submitted(): + if not form.validate(): + response = {'errors': form.errors} + return response, 400 + try: + spacy_nlp_pipeline_model = SpaCyNLPPipelineModel.create( + form.spacy_model_file.data, + compatible_service_versions=form.compatible_service_versions.data, + description=form.description.data, + pipeline_name=form.pipeline_name.data, + publisher=form.publisher.data, + publisher_url=form.publisher_url.data, + publishing_url=form.publishing_url.data, + publishing_year=form.publishing_year.data, + shared=False, + title=form.title.data, + version=form.version.data, + user=current_user + ) + except OSError: + abort(500) + db.session.commit() + spacy_nlp_pipeline_model_url = url_for( + '.spacy_nlp_pipeline_model', + spacy_nlp_pipeline_model_id=spacy_nlp_pipeline_model.id + ) + message = Markup(f'SpaCy NLP Pipeline model "{spacy_nlp_pipeline_model.title}" created') + flash(message) + return {}, 201, {'Location': spacy_nlp_pipeline_model_url} + return render_template( + 'contributions/create_spacy_nlp_pipeline_model.html.j2', + form=form, + title='Create SpaCy NLP Pipeline Model' + ) + +@bp.route('/spacy-nlp-pipeline-models//toggle-public-status', methods=['POST']) +@permission_required(Permission.CONTRIBUTE) +def toggle_spacy_nlp_pipeline_model_public_status(spacy_nlp_pipeline_model_id): + spacy_nlp_pipeline_model = SpaCyNLPPipelineModel.query.get_or_404(spacy_nlp_pipeline_model_id) + if not (spacy_nlp_pipeline_model.user == current_user or current_user.is_administrator()): + abort(403) + spacy_nlp_pipeline_model.shared = not spacy_nlp_pipeline_model.shared + db.session.commit() + return {}, 201 diff --git a/app/corpora/forms.py b/app/corpora/forms.py index 73002edc..db46b0ad 100644 --- a/app/corpora/forms.py +++ b/app/corpora/forms.py @@ -47,20 +47,7 @@ class CreateCorpusFileForm(CorpusFileBaseForm): class EditCorpusFileForm(CorpusFileBaseForm): - def prefill(self, corpus_file): - ''' Pre-fill the form with data of an exististing corpus file ''' - self.address.data = corpus_file.address - self.author.data = corpus_file.author - self.booktitle.data = corpus_file.booktitle - self.chapter.data = corpus_file.chapter - self.editor.data = corpus_file.editor - self.institution.data = corpus_file.institution - self.journal.data = corpus_file.journal - self.pages.data = corpus_file.pages - self.publisher.data = corpus_file.publisher - self.publishing_year.data = corpus_file.publishing_year - self.school.data = corpus_file.school - self.title.data = corpus_file.title + pass class ImportCorpusForm(FlaskForm): diff --git a/app/corpora/query_results_forms.py b/app/corpora/query_results_forms.py deleted file mode 100644 index bb55e513..00000000 --- a/app/corpora/query_results_forms.py +++ /dev/null @@ -1,21 +0,0 @@ -from flask_wtf import FlaskForm -from werkzeug.utils import secure_filename -from wtforms import FileField, StringField, SubmitField, ValidationError -from wtforms.validators import DataRequired, Length - - -class AddQueryResultForm(FlaskForm): - ''' - Form used to import one result json file. - ''' - description = StringField('Description', - validators=[DataRequired(), Length(1, 255)]) - file = FileField('File', validators=[DataRequired()]) - title = StringField('Title', validators=[DataRequired(), Length(1, 32)]) - submit = SubmitField() - - def validate_file(self, field): - if not field.data.filename.lower().endswith('.json'): - raise ValidationError('File does not have an approved extension: ' - '.json') - field.data.filename = secure_filename(field.data.filename) diff --git a/app/corpora/query_results_routes.py b/app/corpora/query_results_routes.py deleted file mode 100644 index 6c22e11e..00000000 --- a/app/corpora/query_results_routes.py +++ /dev/null @@ -1,135 +0,0 @@ -from flask import (abort, current_app, flash, make_response, redirect, request, - render_template, url_for, send_from_directory) -from flask_login import current_user, login_required -from . import bp -from . import tasks -from .forms import (AddQueryResultForm, DisplayOptionsForm, - InspectDisplayOptionsForm) -from .. import db -from ..models import QueryResult -import json -import os - - -@bp.route('/result/add', methods=['GET', 'POST']) -@login_required -def add_query_result(): - ''' - View to import a result as a json file. - ''' - abort(503) - form = AddQueryResultForm(prefix='add-query-result-form') - if form.is_submitted(): - if not form.validate(): - return make_response(form.errors, 400) - query_result = QueryResult(user=current_user, - description=form.description.data, - filename=form.file.data.filename, - title=form.title.data) - db.session.add(query_result) - db.session.flush() - db.session.refresh(query_result) - try: - os.makedirs(os.path.dirname(query_result.path)) - except OSError: - current_app.logger.error( - f'Make dir {query_result.path} led to an OSError!') - db.session.rollback() - flash('Internal Server Error', 'error') - return make_response( - {'redirect_url': url_for('.add_query_result')}, 500) - # save the uploaded file - form.file.data.save(query_result.path) - # parse json from file - with open(query_result.path, 'r') as file: - query_result_file_content = json.load(file) - # parse json schema - # with open('app/static/json_schema/nopaque_cqi_py_results_schema.json', 'r') as file: # noqa - # schema = json.load(file) - # try: - # # validate imported json file - # validate(instance=query_result_file_content, schema=schema) - # except Exception: - # tasks.delete_query_result(query_result.id) - # flash('Uploaded file is invalid', 'result') - # return make_response( - # {'redirect_url': url_for('.add_query_result')}, 201) - query_result_file_content.pop('matches') - query_result_file_content.pop('cpos_lookup') - query_result.query_metadata = query_result_file_content - db.session.commit() - flash('Query result added', 'result') - return make_response({'redirect_url': url_for('.query_result', query_result_id=query_result.id)}, 201) # noqa - return render_template('corpora/query_results/add_query_result.html.j2', - form=form, title='Add query result') - - -@bp.route('/result/') -@login_required -def query_result(query_result_id): - abort(503) - query_result = QueryResult.query.get_or_404(query_result_id) - if not (query_result.user == current_user - or current_user.is_administrator()): - abort(403) - return render_template('corpora/query_results/query_result.html.j2', - query_result=query_result, title='Query result') - - -@bp.route('/result//inspect') -@login_required -def inspect_query_result(query_result_id): - ''' - View to inspect imported result file in a corpus analysis like interface - ''' - abort(503) - query_result = QueryResult.query.get_or_404(query_result_id) - query_metadata = query_result.query_metadata - if not (query_result.user == current_user - or current_user.is_administrator()): - abort(403) - display_options_form = DisplayOptionsForm( - prefix='display-options-form', - results_per_page=request.args.get('results_per_page', 30), - result_context=request.args.get('context', 20) - ) - inspect_display_options_form = InspectDisplayOptionsForm( - prefix='inspect-display-options-form' - ) - with open(query_result.path, 'r') as query_result_file: - query_result_file_content = json.load(query_result_file) - return render_template( - 'corpora/query_results/inspect.html.j2', - query_result=query_result, - display_options_form=display_options_form, - inspect_display_options_form=inspect_display_options_form, # noqa - query_result_file_content=query_result_file_content, - query_metadata=query_metadata, - title='Inspect query result' - ) - - -@bp.route('/result//delete') -@login_required -def delete_query_result(query_result_id): - abort(503) - query_result = QueryResult.query.get_or_404(query_result_id) - if not (query_result.user == current_user - or current_user.is_administrator()): - abort(403) - flash(f'Query result "{query_result}" marked for deletion', 'result') - tasks.delete_query_result(query_result_id) - return redirect(url_for('services.service', service="corpus_analysis")) - - -@bp.route('/result//download') -@login_required -def download_query_result(query_result_id): - abort(503) - query_result = QueryResult.query.get_or_404(query_result_id) - if not (query_result.user == current_user - or current_user.is_administrator()): - abort(403) - return send_from_directory(as_attachment=True, - directory=os.path.dirname(query_result.path), - filename=query_result.filename) diff --git a/app/corpora/query_results_tasks.py b/app/corpora/query_results_tasks.py deleted file mode 100644 index 653096b3..00000000 --- a/app/corpora/query_results_tasks.py +++ /dev/null @@ -1,13 +0,0 @@ -from .. import db -from ..decorators import background -from ..models import QueryResult - - -@background -def delete_query_result(query_result_id, *args, **kwargs): - with kwargs['app'].app_context(): - query_result = QueryResult.query.get(query_result_id) - if query_result is None: - raise Exception(f'QueryResult {query_result_id} not found') - query_result.delete() - db.session.commit() diff --git a/app/corpora/routes.py b/app/corpora/routes.py index 57c14e65..8bf7b1fb 100644 --- a/app/corpora/routes.py +++ b/app/corpora/routes.py @@ -176,52 +176,15 @@ def corpus_file(corpus_id, corpus_file_id): abort(404) if not (corpus_file.corpus.user == current_user or current_user.is_administrator()): abort(403) - form = EditCorpusFileForm(prefix='edit-corpus-file-form') + form = EditCorpusFileForm(obj=corpus_file, prefix='edit-corpus-file-form') if form.validate_on_submit(): - has_changes = False - if corpus_file.address != form.address.data: - corpus_file.address = form.address.data - has_changes = True - if corpus_file.author != form.author.data: - corpus_file.author = form.author.data - has_changes = True - if corpus_file.booktitle != form.booktitle.data: - corpus_file.booktitle = form.booktitle.data - has_changes = True - if corpus_file.chapter != form.chapter.data: - corpus_file.chapter = form.chapter.data - has_changes = True - if corpus_file.editor != form.editor.data: - corpus_file.editor = form.editor.data - has_changes = True - if corpus_file.institution != form.institution.data: - corpus_file.institution = form.institution.data - has_changes = True - if corpus_file.journal != form.journal.data: - corpus_file.journal = form.journal.data - has_changes = True - if corpus_file.pages != form.pages.data: - corpus_file.pages = form.pages.data - has_changes = True - if corpus_file.publisher != form.publisher.data: - corpus_file.publisher = form.publisher.data - has_changes = True - if corpus_file.publishing_year != form.publishing_year.data: - corpus_file.publishing_year = form.publishing_year.data - has_changes = True - if corpus_file.school != form.school.data: - corpus_file.school = form.school.data - has_changes = True - if corpus_file.title != form.title.data: - corpus_file.title = form.title.data - has_changes = True - if has_changes: + form.populate_obj(corpus_file) + if db.session.is_modified(corpus_file): corpus_file.corpus.status = CorpusStatus.UNPREPARED - db.session.commit() - message = Markup(f'Corpus file "{corpus_file.filename}" updated') - flash(message, category='corpus') + db.session.commit() + message = Markup(f'Corpus file "{corpus_file.filename}" updated') + flash(message, category='corpus') return redirect(corpus_file.corpus.url) - form.prefill(corpus_file) return render_template( 'corpora/corpus_file.html.j2', corpus=corpus_file.corpus, diff --git a/app/daemon/job_utils.py b/app/daemon/job_utils.py index 32def73d..cfb362db 100644 --- a/app/daemon/job_utils.py +++ b/app/daemon/job_utils.py @@ -3,7 +3,8 @@ from app.models import ( Job, JobResult, JobStatus, - TesseractOCRPipelineModel + TesseractOCRPipelineModel, + SpaCyNLPPipelineModel ) from datetime import datetime from flask import current_app @@ -52,13 +53,21 @@ def _create_job_service(job): command += f' --mem-mb {mem_mb}' command += f' --n-cores {n_cores}' if job.service == 'spacy-nlp-pipeline': - command += f' -m {job.service_args["model"]}' + model_id = hashids.decode(job.service_args['model']) + model = SpaCyNLPPipelineModel.query.get(model_id) + if model is None: + job.status = JobStatus.FAILED + return + command += f' -m {model.pipeline_name}' if 'encoding_detection' in job.service_args and job.service_args['encoding_detection']: command += ' --check-encoding' elif job.service == 'tesseract-ocr-pipeline': command += f' -m {job.service_args["model"]}' if 'binarization' in job.service_args and job.service_args['binarization']: command += ' --binarize' + if 'ocropus_nlbin_threshold' in job.service_args and job.service_args['ocropus_nlbin_threshold']: + value = job.service_args['ocropus_nlbin_threshold'] + command += f' --ocropus-nlbin-threshold {value}' elif job.service == 'transkribus-htr-pipeline': transkribus_htr_pipeline_model_id = job.service_args['model'] command += f' -m {transkribus_htr_pipeline_model_id}' @@ -103,6 +112,16 @@ def _create_job_service(job): models_mount_target = f'/usr/local/share/tessdata/{model.filename}' models_mount = f'{models_mount_source}:{models_mount_target}:ro' mounts.append(models_mount) + elif job.service == 'spacy-nlp-pipeline': + model_id = hashids.decode(job.service_args['model']) + model = SpaCyNLPPipelineModel.query.get(model_id) + if model is None: + job.status = JobStatus.FAILED + return + models_mount_source = model.path + models_mount_target = f'/usr/local/share/spacy/models/{model.filename}' + models_mount = f'{models_mount_source}:{models_mount_target}:ro' + mounts.append(models_mount) ''' ### Output mount ### ''' output_mount_source = os.path.join(job.path, 'results') output_mount_target = '/output' diff --git a/app/models.py b/app/models.py index cc5d60ce..cbe25379 100644 --- a/app/models.py +++ b/app/models.py @@ -1,6 +1,5 @@ from datetime import datetime, timedelta from enum import Enum, IntEnum -import re from flask import current_app, url_for from flask_hashids import HashidMixin from flask_login import UserMixin @@ -263,7 +262,6 @@ class User(HashidMixin, UserMixin, db.Model): password_hash = db.Column(db.String(128)) confirmed = db.Column(db.Boolean, default=False) member_since = db.Column(db.DateTime(), default=datetime.utcnow) - setting_dark_mode = db.Column(db.Boolean, default=False) setting_job_status_mail_notification_level = db.Column( IntEnumColumn(UserSettingJobStatusMailNotificationLevel), default=UserSettingJobStatusMailNotificationLevel.END @@ -500,7 +498,6 @@ class User(HashidMixin, UserMixin, db.Model): 'member_since': f'{self.member_since.isoformat()}Z', 'username': self.username, 'settings': { - 'dark_mode': self.setting_dark_mode, 'job_status_mail_notification_level': \ self.setting_job_status_mail_notification_level.name } @@ -520,6 +517,10 @@ class User(HashidMixin, UserMixin, db.Model): x.hashid: x.to_json(relationships=True) for x in self.tesseract_ocr_pipeline_models } + _json['spacy_nlp_pipeline_models'] = { + x.hashid: x.to_json(relationships=True) + for x in self.spacy_nlp_pipeline_models + } return _json class TesseractOCRPipelineModel(FileMixin, HashidMixin, db.Model): @@ -548,6 +549,21 @@ class TesseractOCRPipelineModel(FileMixin, HashidMixin, db.Model): str(self.id) ) + @property + def jsonpatch_path(self): + return f'{self.user.jsonpatch_path}/tesseract_ocr_pipeline_models/{self.hashid}' + + @property + def url(self): + return url_for( + 'contributions.tesseract_ocr_pipeline_model', + tesseract_ocr_pipeline_model_id=self.id + ) + + @property + def user_hashid(self): + return self.user.hashid + @staticmethod def insert_defaults(): nopaque_user = User.query.filter_by(username='nopaque').first() @@ -603,6 +619,13 @@ class TesseractOCRPipelineModel(FileMixin, HashidMixin, db.Model): pbar.close() db.session.commit() + def delete(self): + try: + os.remove(self.path) + except OSError as e: + current_app.logger.error(e) + db.session.delete(self) + def to_json(self, backrefs=False, relationships=False): _json = { 'id': self.hashid, @@ -614,6 +637,7 @@ class TesseractOCRPipelineModel(FileMixin, HashidMixin, db.Model): 'publishing_year': self.publishing_year, 'shared': self.shared, 'title': self.title, + 'version': self.version, **self.file_mixin_to_json() } if backrefs: @@ -636,6 +660,7 @@ class SpaCyNLPPipelineModel(FileMixin, HashidMixin, db.Model): publisher_url = db.Column(db.String(512)) publishing_url = db.Column(db.String(512)) publishing_year = db.Column(db.Integer) + pipeline_name = db.Column(db.String(64)) shared = db.Column(db.Boolean, default=False) # Backrefs: user: User @@ -647,6 +672,21 @@ class SpaCyNLPPipelineModel(FileMixin, HashidMixin, db.Model): str(self.id) ) + @property + def jsonpatch_path(self): + return f'{self.user.jsonpatch_path}/spacy_nlp_pipeline_models/{self.hashid}' + + @property + def url(self): + return url_for( + 'contributions.spacy_nlp_pipeline_model', + spacy_nlp_pipeline_model_id=self.id + ) + + @property + def user_hashid(self): + return self.user.hashid + @staticmethod def insert_defaults(): nopaque_user = User.query.filter_by(username='nopaque').first() @@ -668,6 +708,7 @@ class SpaCyNLPPipelineModel(FileMixin, HashidMixin, db.Model): model.shared = True model.title = m['title'] model.version = m['version'] + model.pipeline_name = m['pipeline_name'] continue model = SpaCyNLPPipelineModel( compatible_service_versions=m['compatible_service_versions'], @@ -679,12 +720,13 @@ class SpaCyNLPPipelineModel(FileMixin, HashidMixin, db.Model): shared=True, title=m['title'], user=nopaque_user, - version=m['version'] + version=m['version'], + pipeline_name=m['pipeline_name'] ) db.session.add(model) db.session.flush(objects=[model]) db.session.refresh(model) - model.filename = f'{model.id}.traineddata' + model.filename = m['url'].split('/')[-1] r = requests.get(m['url'], stream=True) pbar = tqdm( desc=f'{model.title} ({model.filename})', @@ -701,6 +743,13 @@ class SpaCyNLPPipelineModel(FileMixin, HashidMixin, db.Model): f.write(chunk) pbar.close() db.session.commit() + + def delete(self): + try: + os.remove(self.path) + except OSError as e: + current_app.logger.error(e) + db.session.delete(self) def to_json(self, backrefs=False, relationships=False): _json = { @@ -711,8 +760,10 @@ class SpaCyNLPPipelineModel(FileMixin, HashidMixin, db.Model): 'publisher_url': self.publisher_url, 'publishing_url': self.publishing_url, 'publishing_year': self.publishing_year, + 'pipeline_name': self.pipeline_name, 'shared': self.shared, 'title': self.title, + 'version': self.version, **self.file_mixin_to_json() } if backrefs: @@ -1023,11 +1074,8 @@ class CorpusFile(FileMixin, HashidMixin, db.Model): def delete(self): try: os.remove(self.path) - except OSError: - current_app.logger.error( - f'Removing {self.path} led to an OSError!' - ) - pass + except OSError as e: + current_app.logger.error(e) db.session.delete(self) self.corpus.status = CorpusStatus.UNPREPARED diff --git a/app/query_results_models.py b/app/query_results_models.py deleted file mode 100644 index 132a4cc3..00000000 --- a/app/query_results_models.py +++ /dev/null @@ -1,58 +0,0 @@ -class QueryResult(FileMixin, HashidMixin, db.Model): - __tablename__ = 'query_results' - # Primary key - id = db.Column(db.Integer, primary_key=True) - # Foreign keys - user_id = db.Column(db.Integer, db.ForeignKey('users.id')) - # Fields - description = db.Column(db.String(255)) - query_metadata = db.Column(db.JSON()) - title = db.Column(db.String(32)) - # Backrefs: user: User - - def __repr__(self): - ''' - String representation of the QueryResult. For human readability. - ''' - return f'' - - @property - def download_url(self): - return url_for( - 'corpora.download_query_result', query_result_id=self.id) - - @property - def jsonpatch_path(self): - return f'{self.user.jsonpatch_path}/query_results/{self.hashid}' - - @property - def path(self): - return os.path.join( - self.user.path, 'query_results', str(self.id), self.filename) - - @property - def url(self): - return url_for('corpora.query_result', query_result_id=self.id) - - @property - def user_hashid(self): - return self.user.hashid - - def delete(self): - shutil.rmtree(self.path, ignore_errors=True) - db.session.delete(self) - - def to_json(self, backrefs=False, relationships=False): - _json = { - 'id': self.hashid, - 'corpus_title': self.query_metadata['corpus_name'], - 'description': self.description, - 'filename': self.filename, - 'query': self.query_metadata['query'], - 'query_metadata': self.query_metadata, - 'title': self.title, - **self.file_mixin_to_json( - backrefs=backrefs, relationships=relationships) - } - if backrefs: - _json['user'] = self.user.to_json(backrefs=True, relationships=False) diff --git a/app/services/forms.py b/app/services/forms.py index 5c0af906..562696eb 100644 --- a/app/services/forms.py +++ b/app/services/forms.py @@ -1,16 +1,12 @@ from flask_login import current_user from flask_wtf import FlaskForm from flask_wtf.file import FileField, FileRequired -from wtforms import ( - BooleanField, - MultipleFileField, - SelectField, - StringField, - SubmitField, - ValidationError -) +from wtforms import (BooleanField, DecimalRangeField, MultipleFileField, + SelectField, StringField, SubmitField, ValidationError) from wtforms.validators import InputRequired, Length -from app.models import TesseractOCRPipelineModel + +from app.models import SpaCyNLPPipelineModel, TesseractOCRPipelineModel + from . import SERVICES @@ -49,13 +45,16 @@ class CreateTesseractOCRPipelineJobForm(CreateJobBaseForm): binarization = BooleanField('Binarization') pdf = FileField('File', validators=[FileRequired()]) model = SelectField('Model', validators=[InputRequired()]) + ocropus_nlbin_threshold = DecimalRangeField( + render_kw={'min': 0, 'max': 1, 'step': 0.1, 'start': [0.5], 'disabled': True} + ) def validate_binarization(self, field): service_info = SERVICES['tesseract-ocr-pipeline']['versions'][self.version.data] if field.data: if not('methods' in service_info and 'binarization' in service_info['methods']): raise ValidationError('Binarization is not available') - + def validate_pdf(self, field): if field.data.mimetype != 'application/pdf': raise ValidationError('PDF files only!') @@ -68,16 +67,20 @@ class CreateTesseractOCRPipelineJobForm(CreateJobBaseForm): if self.binarization.render_kw is None: self.binarization.render_kw = {} self.binarization.render_kw['disabled'] = True + if self.ocropus_nlbin_threshold.render_kw is None: + self.ocropus_nlbin_threshold.render_kw = {} + self.ocropus_nlbin_threshold.render_kw['disabled'] = True if 'methods' in service_info: if 'binarization' in service_info['methods']: - if 'disabled' in self.binarization.render_kw: - del self.binarization.render_kw['disabled'] + del self.binarization.render_kw['disabled'] + if 'ocropus_nlbin_threshold' in service_info['methods']: + del self.ocropus_nlbin_threshold.render_kw['disabled'] models = [ - x for x in TesseractOCRPipelineModel.query.filter().all() + x for x in TesseractOCRPipelineModel.query.order_by(TesseractOCRPipelineModel.title).all() if version in x.compatible_service_versions and (x.shared == True or x.user == current_user) ] self.model.choices = [('', 'Choose your option')] - self.model.choices += [(x.hashid, x.title) for x in models] + self.model.choices += [(x.hashid, f'{x.title} [{x.version}]') for x in models] self.model.default = '' self.version.choices = [(x, x) for x in service_manifest['versions']] self.version.data = version @@ -113,8 +116,7 @@ class CreateTranskribusHTRPipelineJobForm(CreateJobBaseForm): self.binarization.render_kw['disabled'] = True if 'methods' in service_info: if 'binarization' in service_info['methods']: - if 'disabled' in self.binarization.render_kw: - del self.binarization.render_kw['disabled'] + del self.binarization.render_kw['disabled'] self.model.choices = [('', 'Choose your option')] self.model.choices += [(x['modelId'], x['name']) for x in transkribus_htr_pipeline_models] self.model.default = '' @@ -127,7 +129,7 @@ class CreateSpacyNLPPipelineJobForm(CreateJobBaseForm): encoding_detection = BooleanField('Encoding detection', render_kw={'disabled': True}) txt = FileField('File', validators=[FileRequired()]) model = SelectField('Model', validators=[InputRequired()]) - + def validate_encoding_detection(self, field): service_info = SERVICES['spacy-nlp-pipeline']['versions'][self.version.data] if field.data: @@ -146,15 +148,19 @@ class CreateSpacyNLPPipelineJobForm(CreateJobBaseForm): version = kwargs.pop('version', service_manifest['latest_version']) super().__init__(*args, **kwargs) service_info = service_manifest['versions'][version] + print(service_info) if self.encoding_detection.render_kw is None: self.encoding_detection.render_kw = {} self.encoding_detection.render_kw['disabled'] = True if 'methods' in service_info: if 'encoding_detection' in service_info['methods']: - if 'disabled' in self.encoding_detection.render_kw: - del self.encoding_detection.render_kw['disabled'] + del self.encoding_detection.render_kw['disabled'] + models = [ + x for x in SpaCyNLPPipelineModel.query.order_by(SpaCyNLPPipelineModel.title).all() + if version in x.compatible_service_versions and (x.shared == True or x.user == current_user) + ] self.model.choices = [('', 'Choose your option')] - self.model.choices += [(x, y) for x, y in service_info['models'].items()] # noqa + self.model.choices += [(x.hashid, f'{x.title} [{x.version}]') for x in models] self.model.default = '' self.version.choices = [(x, x) for x in service_manifest['versions']] self.version.data = version diff --git a/app/services/routes.py b/app/services/routes.py index b34d0619..7748240c 100644 --- a/app/services/routes.py +++ b/app/services/routes.py @@ -6,7 +6,8 @@ from app.models import ( Job, JobInput, JobStatus, - TesseractOCRPipelineModel + TesseractOCRPipelineModel, + SpaCyNLPPipelineModel ) from . import bp, SERVICES from .forms import ( @@ -78,7 +79,8 @@ def tesseract_ocr_pipeline(): service=service_name, service_args={ 'binarization': form.binarization.data, - 'model': hashids.decode(form.model.data) + 'model': hashids.decode(form.model.data), + 'ocropus_nlbin_threshold': float(form.ocropus_nlbin_threshold.data) }, service_version=form.version.data, user=current_user @@ -172,6 +174,7 @@ def spacy_nlp_pipeline(): if version not in service_manifest['versions']: abort(404) form = CreateSpacyNLPPipelineJobForm(prefix='create-job-form', version=version) + spacy_nlp_pipeline_models = SpaCyNLPPipelineModel.query.all() if form.is_submitted(): if not form.validate(): response = {'errors': form.errors} @@ -202,6 +205,7 @@ def spacy_nlp_pipeline(): return render_template( 'services/spacy_nlp_pipeline.html.j2', form=form, + spacy_nlp_pipeline_models=spacy_nlp_pipeline_models, title=service_manifest['name'] ) diff --git a/app/services/services.yml b/app/services/services.yml index e8db1b33..4d00163d 100644 --- a/app/services/services.yml +++ b/app/services/services.yml @@ -1,6 +1,6 @@ # TODO: This could also be done via GitLab/GitHub APIs file-setup-pipeline: - name: 'File setup pipeline' + name: 'File Setup Pipeline' publisher: 'Bielefeld University - CRC 1288 - INF' latest_version: '0.1.0' versions: @@ -20,6 +20,7 @@ tesseract-ocr-pipeline: 0.1.1: methods: - 'binarization' + - 'ocropus_nlbin_threshold' publishing_year: 2022 url: 'https://gitlab.ub.uni-bielefeld.de/sfb1288inf/tesseract-ocr-pipeline/-/releases/v0.1.1' transkribus-htr-pipeline: @@ -38,23 +39,17 @@ transkribus-htr-pipeline: publishing_year: 2022 url: 'https://gitlab.ub.uni-bielefeld.de/sfb1288inf/transkribus-htr-pipeline/-/releases/v0.1.1' spacy-nlp-pipeline: - name: 'spaCy NLP Pipeline' + name: 'SpaCy NLP Pipeline' publisher: 'Bielefeld University - CRC 1288 - INF' - latest_version: '0.1.0' + latest_version: '0.1.1' versions: 0.1.0: methods: - 'encoding_detection' - models: - ca: 'Catalan' - de: 'German' - el: 'Greek' - en: 'English' - es: 'Spanish' - fr: 'French' - it: 'Italian' - pl: 'Polish' - ru: 'Russian' - zh: 'Chinese' publishing_year: 2022 url: 'https://gitlab.ub.uni-bielefeld.de/sfb1288inf/spacy-nlp-pipeline/-/releases/v0.1.0' + 0.1.1: + methods: + - 'encoding_detection' + publishing_year: 2022 + url: 'https://gitlab.ub.uni-bielefeld.de/sfb1288inf/spacy-nlp-pipeline/-/releases/v0.1.1' diff --git a/app/settings/forms.py b/app/settings/forms.py index 3bd3b5ab..00904e00 100644 --- a/app/settings/forms.py +++ b/app/settings/forms.py @@ -71,10 +71,6 @@ class EditGeneralSettingsForm(FlaskForm): super().__init__(*args, **kwargs) self.user = user - def prefill(self, user): - self.email.data = user.email - self.username.data = user.username - def validate_email(self, field): if (field.data != self.user.email and User.query.filter_by(email=field.data).first()): @@ -86,13 +82,6 @@ class EditGeneralSettingsForm(FlaskForm): raise ValidationError('Username already in use') -class EditInterfaceSettingsForm(FlaskForm): - dark_mode = BooleanField('Dark mode') - submit = SubmitField() - - def prefill(self, user): - self.dark_mode.data = user.setting_dark_mode - class EditNotificationSettingsForm(FlaskForm): job_status_mail_notification_level = SelectField( 'Job status mail notification level', diff --git a/app/settings/routes.py b/app/settings/routes.py index eb2636e8..0604eb06 100644 --- a/app/settings/routes.py +++ b/app/settings/routes.py @@ -6,7 +6,6 @@ from . import bp from .forms import ( ChangePasswordForm, EditGeneralSettingsForm, - EditInterfaceSettingsForm, EditNotificationSettingsForm ) @@ -20,11 +19,9 @@ def settings(): ) edit_general_settings_form = EditGeneralSettingsForm( current_user, + obj=current_user, prefix='edit-general-settings-form' ) - edit_interface_settings_form = EditInterfaceSettingsForm( - prefix='edit-interface-settings-form' - ) edit_notification_settings_form = EditNotificationSettingsForm( prefix='edit-notification-settings-form' ) @@ -41,13 +38,6 @@ def settings(): db.session.commit() flash('Your changes have been saved') return redirect(url_for('.settings')) - if (edit_interface_settings_form.submit.data - and edit_interface_settings_form.validate()): - current_user.setting_dark_mode = ( - edit_interface_settings_form.dark_mode.data) - db.session.commit() - flash('Your changes have been saved') - return redirect(url_for('.settings')) if (edit_notification_settings_form.submit.data and edit_notification_settings_form.validate()): current_user.setting_job_status_mail_notification_level = ( @@ -58,14 +48,11 @@ def settings(): db.session.commit() flash('Your changes have been saved') return redirect(url_for('.settings')) - edit_general_settings_form.prefill(current_user) - edit_interface_settings_form.prefill(current_user) edit_notification_settings_form.prefill(current_user) return render_template( 'settings/settings.html.j2', change_password_form=change_password_form, edit_general_settings_form=edit_general_settings_form, - edit_interface_settings_form=edit_interface_settings_form, edit_notification_settings_form=edit_notification_settings_form, title='Settings' ) diff --git a/app/static/js/Forms/CreateContributionForm.js b/app/static/js/Forms/CreateContributionForm.js new file mode 100644 index 00000000..e7651ab0 --- /dev/null +++ b/app/static/js/Forms/CreateContributionForm.js @@ -0,0 +1,18 @@ +class CreateContributionForm extends Form { + static autoInit() { + let createContributionFormElements = document.querySelectorAll('.create-contribution-form'); + for (let createContributionFormElement of createContributionFormElements) { + new CreateContributionForm(createContributionFormElement); + } + } + + constructor(formElement) { + super(formElement); + + this.addEventListener('requestLoad', (event) => { + if (event.target.status === 201) { + window.location.href = event.target.getResponseHeader('Location'); + } + }); + } +} diff --git a/app/static/js/Forms/Form.js b/app/static/js/Forms/Form.js index 9a21e986..d93f3e2c 100644 --- a/app/static/js/Forms/Form.js +++ b/app/static/js/Forms/Form.js @@ -1,5 +1,6 @@ class Form { static autoInit() { + CreateContributionForm.autoInit(); CreateCorpusFileForm.autoInit(); CreateJobForm.autoInit(); } diff --git a/app/static/js/RessourceLists/QueryResultList.js b/app/static/js/RessourceLists/QueryResultList.js deleted file mode 100644 index cffc4318..00000000 --- a/app/static/js/RessourceLists/QueryResultList.js +++ /dev/null @@ -1,134 +0,0 @@ -class QueryResultList extends RessourceList { - static autoInit() { - for (let queryResultListElement of document.querySelectorAll('.query-result-list:not(.no-autoinit)')) { - new QueryResultList(queryResultListElement); - } - } - - static options = { - item: ` - -

-
- - delete - send - - - `.trim(), - ressourceMapper: queryResult => { - return { - 'id': queryResult.id, - 'corpus-title': queryResult.corpus_title, - 'creation-date': queryResult.creation_date, - 'description': queryResult.description, - 'query': queryResult.query, - 'title': queryResult.title - }; - }, - sortArgs: ['creation-date', {order: 'desc'}], - valueNames: [ - {data: ['id']}, - {data: ['creation-date']}, - 'corpus-title', - 'description', - 'query', - 'title' - ] - }; - - constructor(listElement, options = {}) { - super(listElement, {...QueryResultList.options, ...options}); - } - - init(user) { - super._init(user.query_results); - } - - onclick(event) { - let action; - let actionButtonElement; - let deleteModal; - let deleteModalElement; - let queryResultElement; - let queryResultId; - let tmp; - - queryResultElement = event.target.closest('tr[data-id]'); - if (queryResultElement === null) {return;} - queryResultId = queryResultElement.dataset.id; - actionButtonElement = event.target.closest('.action-button[data-action]'); - action = actionButtonElement === null ? 'view' : actionButtonElement.dataset.action; - switch (action) { - case 'delete': - tmp = document.createElement('div'); - tmp.innerHTML = ` -
-
-

Confirm query result deletion

-

Do you really want to delete the query result ${app.users[this.userId].query_results[queryResultId].title}? It will be permanently deleted!

-
-
- Cancel - deleteDelete -
-
- `.trim(); - deleteModalElement = document.querySelector('#modals').appendChild(tmp.firstChild); - deleteModal = M.Modal.init( - deleteModalElement, - { - onCloseEnd: () => { - deleteModal.destroy(); - deleteModalElement.remove(); - } - } - ); - deleteModal.open(); - break; - case 'view': - window.location.href = `/query_results/${queryResultId}`; - break; - default: - break; - } - } - - onPATCH(patch) { - let filteredPatch; - let match; - let operation; - let queryResultId; - let re; - let valueName; - - re = new RegExp(`^/users/${this.userId}/query_results/([A-Za-z0-9]*)`); - filteredPatch = patch.filter(operation => re.test(operation.path)); - for (operation of filteredPatch) { - switch(operation.op) { - case 'add': - re = new RegExp(`^/users/${this.userId}/query_results/([A-Za-z0-9]*)$`); - if (re.test(operation.path)) { - this.add(operation.value); - } - break; - case 'remove': - re = new RegExp(`^/users/${this.userId}/query_results/([A-Za-z0-9]*)$`); - if (re.test(operation.path)) { - [match, queryResultId] = operation.path.match(re); - this.remove(queryResultId); - } - break; - case 'replace': - re = new RegExp(`^/users/${this.userId}/query_results/([A-Za-z0-9]*)/(corpus_title|description|query|title)$`); - if (re.test(operation.path)) { - [match, queryResultId, valueName] = operation.path.match(re); - this.replace(queryResultId, valueName.replace('_', '-'), operation.value); - } - break; - default: - break; - } - } - } -} diff --git a/app/static/js/RessourceLists/RessourceList.js b/app/static/js/RessourceLists/RessourceList.js index 824db3d1..871a1e2f 100644 --- a/app/static/js/RessourceLists/RessourceList.js +++ b/app/static/js/RessourceLists/RessourceList.js @@ -10,7 +10,8 @@ class RessourceList { JobList.autoInit(); JobInputList.autoInit(); JobResultList.autoInit(); - QueryResultList.autoInit(); + SpaCyNLPPipelineModelList.autoInit(); + TesseractOCRPipelineModelList.autoInit(); UserList.autoInit(); } diff --git a/app/static/js/RessourceLists/SpacyNLPPipelineModelList.js b/app/static/js/RessourceLists/SpacyNLPPipelineModelList.js new file mode 100644 index 00000000..057e7c4c --- /dev/null +++ b/app/static/js/RessourceLists/SpacyNLPPipelineModelList.js @@ -0,0 +1,135 @@ +class SpaCyNLPPipelineModelList extends RessourceList { + static autoInit() { + for (let spaCyNLPPipelineModelListElement of document.querySelectorAll('.spacy-nlp-pipeline-model-list:not(.no-autoinit)')) { + new SpaCyNLPPipelineModelList(spaCyNLPPipelineModelListElement); + } + } + + static options = { + initialHtmlGenerator: (id) => { + return ` +
+ search + + +
+ + + + + + + + + +
Title and DescriptionPublisher
+
    + `.trim(); + }, + item: ` + +
    + ()
    + +
    + + +
    + + + delete + send + + + `.trim(), + ressourceMapper: (spaCyNLPPipelineModel) => { + return { + 'id': spaCyNLPPipelineModel.id, + 'creation-date': spaCyNLPPipelineModel.creation_date, + 'description': spaCyNLPPipelineModel.description, + 'publisher': spaCyNLPPipelineModel.publisher, + 'publisher-url': spaCyNLPPipelineModel.publisher_url, + 'publishing-url': spaCyNLPPipelineModel.publishing_url, + 'publishing-url-2': spaCyNLPPipelineModel.publishing_url, + 'publishing-year': spaCyNLPPipelineModel.publishing_year, + 'title': spaCyNLPPipelineModel.title, + 'title-2': spaCyNLPPipelineModel.title, + 'version': spaCyNLPPipelineModel.version, + 'shared': spaCyNLPPipelineModel.shared ? 'True' : 'False' + }; + }, + sortArgs: ['creation-date', {order: 'desc'}], + valueNames: [ + {data: ['id']}, + {data: ['creation-date']}, + {name: 'publisher-url', attr: 'href'}, + {name: 'publishing-url', attr: 'href'}, + 'description', + 'publisher', + 'publishing-url-2', + 'publishing-year', + 'title', + 'title-2', + 'version', + {name: 'shared', attr: 'data-checked'} + ] + }; + + constructor(listElement, options = {}) { + super(listElement, {...SpaCyNLPPipelineModelList.options, ...options}); + this.listjs.list.addEventListener('change', (event) => {this.onChange(event)}); + } + + init (user) { + this._init(user.spacy_nlp_pipeline_models); + } + + _init(ressources) { + super._init(ressources); + for (let uncheckedCheckbox of this.listjs.list.querySelectorAll('input[data-checked="True"]')) { + uncheckedCheckbox.setAttribute('checked', ''); + } + } + + onClick(event) { + if (event.target.closest('.action-switch')) {return;} + let actionButtonElement = event.target.closest('.action-button'); + let action = actionButtonElement === null ? 'view' : actionButtonElement.dataset.action; + let spaCyNLPPipelineModelElement = event.target.closest('tr'); + let spaCyNLPPipelineModelId = spaCyNLPPipelineModelElement.dataset.id; + switch (action) { + case 'delete-request': { + Utils.deleteSpaCyNLPPipelineModelRequest(this.userId, spaCyNLPPipelineModelId); + break; + } + case 'view': { + window.location.href = `/contributions/spacy-nlp-pipeline-models/${spaCyNLPPipelineModelId}`; + break; + } + default: { + break; + } + } + } + + onChange(event) { + let actionSwitchElement = event.target.closest('.action-switch'); + let action = actionSwitchElement.dataset.action; + let spaCyNLPPipelineModelElement = event.target.closest('tr'); + let spaCyNLPPipelineModelId = spaCyNLPPipelineModelElement.dataset.id; + switch (action) { + case 'share-request': { + let shared = actionSwitchElement.querySelector('input').checked; + Utils.shareSpaCyNLPPipelineModelRequest(this.userId, spaCyNLPPipelineModelId, shared); + break; + } + default: { + break; + } + } + } +} diff --git a/app/static/js/RessourceLists/TesseractOCRPipelineModelList.js b/app/static/js/RessourceLists/TesseractOCRPipelineModelList.js new file mode 100644 index 00000000..33e7f432 --- /dev/null +++ b/app/static/js/RessourceLists/TesseractOCRPipelineModelList.js @@ -0,0 +1,135 @@ +class TesseractOCRPipelineModelList extends RessourceList { + static autoInit() { + for (let tesseractOCRPipelineModelListElement of document.querySelectorAll('.tesseract-ocr-pipeline-model-list:not(.no-autoinit)')) { + new TesseractOCRPipelineModelList(tesseractOCRPipelineModelListElement); + } + } + + static options = { + initialHtmlGenerator: (id) => { + return ` +
    + search + + +
    + + + + + + + + + +
    Title and DescriptionPublisher
    +
      + `.trim(); + }, + item: ` + +
      + ()
      + +
      + + +
      + + + delete + send + + + `.trim(), + ressourceMapper: (tesseractOCRPipelineModel) => { + return { + 'id': tesseractOCRPipelineModel.id, + 'creation-date': tesseractOCRPipelineModel.creation_date, + 'description': tesseractOCRPipelineModel.description, + 'publisher': tesseractOCRPipelineModel.publisher, + 'publisher-url': tesseractOCRPipelineModel.publisher_url, + 'publishing-url': tesseractOCRPipelineModel.publishing_url, + 'publishing-url-2': tesseractOCRPipelineModel.publishing_url, + 'publishing-year': tesseractOCRPipelineModel.publishing_year, + 'title': tesseractOCRPipelineModel.title, + 'title-2': tesseractOCRPipelineModel.title, + 'version': tesseractOCRPipelineModel.version, + 'shared': tesseractOCRPipelineModel.shared ? 'True' : 'False' + }; + }, + sortArgs: ['creation-date', {order: 'desc'}], + valueNames: [ + {data: ['id']}, + {data: ['creation-date']}, + {name: 'publisher-url', attr: 'href'}, + {name: 'publishing-url', attr: 'href'}, + 'description', + 'publisher', + 'publishing-url-2', + 'publishing-year', + 'title', + 'title-2', + 'version', + {name: 'shared', attr: 'data-checked'} + ] + }; + + constructor(listElement, options = {}) { + super(listElement, {...TesseractOCRPipelineModelList.options, ...options}); + this.listjs.list.addEventListener('change', (event) => {this.onChange(event)}); + } + + init (user) { + this._init(user.tesseract_ocr_pipeline_models); + } + + _init(ressources) { + super._init(ressources); + for (let uncheckedCheckbox of this.listjs.list.querySelectorAll('input[data-checked="True"]')) { + uncheckedCheckbox.setAttribute('checked', ''); + } + } + + onClick(event) { + if (event.target.closest('.action-switch')) {return;} + let actionButtonElement = event.target.closest('.action-button'); + let action = actionButtonElement === null ? 'view' : actionButtonElement.dataset.action; + let tesseractOCRPipelineModelElement = event.target.closest('tr'); + let tesseractOCRPipelineModelId = tesseractOCRPipelineModelElement.dataset.id; + switch (action) { + case 'delete-request': { + Utils.deleteTesseractOCRPipelineModelRequest(this.userId, tesseractOCRPipelineModelId); + break; + } + case 'view': { + window.location.href = `/contributions/tesseract-ocr-pipeline-models/${tesseractOCRPipelineModelId}`; + break; + } + default: { + break; + } + } + } + + onChange(event) { + let actionSwitchElement = event.target.closest('.action-switch'); + let action = actionSwitchElement.dataset.action; + let tesseractOCRPipelineModelElement = event.target.closest('tr'); + let tesseractOCRPipelineModelId = tesseractOCRPipelineModelElement.dataset.id; + switch (action) { + case 'share-request': { + let shared = actionSwitchElement.querySelector('input').checked; + Utils.shareTesseractOCRPipelineModelRequest(this.userId, tesseractOCRPipelineModelId, shared); + break; + } + default: { + break; + } + } + } +} diff --git a/app/static/js/Utils.js b/app/static/js/Utils.js index 2e7cbd4c..01de8207 100644 --- a/app/static/js/Utils.js +++ b/app/static/js/Utils.js @@ -33,8 +33,8 @@ class Utils { `
      -

      Confirm job deletion

      -

      Do you really want to delete the job ${corpus.title}? All files will be permanently deleted!

      +

      Confirm Corpus deletion

      +

      Do you really want to delete the Corpus ${corpus.title}? All files will be permanently deleted!

      Cancel @@ -84,8 +84,8 @@ class Utils { `
      -

      Confirm job deletion

      -

      Do you really want to delete the job ${corpusFile.title}? All files will be permanently deleted!

      +

      Confirm Corpus File deletion

      +

      Do you really want to delete the Corpus File ${corpusFile.title}? All files will be permanently deleted!

      Cancel @@ -126,6 +126,102 @@ class Utils { }); } + static deleteSpaCyNLPPipelineModelRequest(userId, spaCyNLPPipelineModelId) { + return new Promise((resolve, reject) => { + let spaCyNLPPipelineModel = app.data.users[userId].spacy_nlp_pipeline_models[spaCyNLPPipelineModelId]; + let modalElement = Utils.elementFromString( + ` +
      +
      +

      Confirm SpaCy NLP Pipeline Model deletion

      +

      Do you really want to delete the SpaCy NLP Pipeline Model ${spaCyNLPPipelineModel.title}? All files will be permanently deleted!

      +
      +
      + Cancel + Delete +
      +
      + ` + ); + document.querySelector('#modals').appendChild(modalElement); + let modal = M.Modal.init( + modalElement, + { + dismissible: false, + onCloseEnd: () => { + modal.destroy(); + modalElement.remove(); + } + } + ); + let confirmElement = modalElement.querySelector('.action-button[data-action="confirm"]'); + confirmElement.addEventListener('click', (event) => { + let spaCyNLPPipelineModelTitle = spaCyNLPPipelineModel.title; + fetch(`/contributions/spacy-nlp-pipeline-models/${spaCyNLPPipelineModelId}`, {method: 'DELETE'}) + .then( + (response) => { + app.flash(`SpaCy NLP Pipeline Model "${spaCyNLPPipelineModelTitle}" marked for deletion`); + resolve(response); + }, + (response) => { + if (response.status === 403) {app.flash('Forbidden', 'error');} + if (response.status === 404) {app.flash('Not Found', 'error');} + reject(response); + } + ); + }); + modal.open(); + }); + } + + static deleteTesseractOCRPipelineModelRequest(userId, tesseractOCRPipelineModelId) { + return new Promise((resolve, reject) => { + let tesseractOCRPipelineModel = app.data.users[userId].tesseract_ocr_pipeline_models[tesseractOCRPipelineModelId]; + let modalElement = Utils.elementFromString( + ` +
      +
      +

      Confirm Tesseract OCR Pipeline Model deletion

      +

      Do you really want to delete the Tesseract OCR Pipeline Model ${tesseractOCRPipelineModel.title}? All files will be permanently deleted!

      +
      +
      + Cancel + Delete +
      +
      + ` + ); + document.querySelector('#modals').appendChild(modalElement); + let modal = M.Modal.init( + modalElement, + { + dismissible: false, + onCloseEnd: () => { + modal.destroy(); + modalElement.remove(); + } + } + ); + let confirmElement = modalElement.querySelector('.action-button[data-action="confirm"]'); + confirmElement.addEventListener('click', (event) => { + let tesseractOCRPipelineModelTitle = tesseractOCRPipelineModel.title; + fetch(`/contributions/tesseract-ocr-pipeline-models/${tesseractOCRPipelineModelId}`, {method: 'DELETE'}) + .then( + (response) => { + app.flash(`Tesseract OCR Pipeline Model "${tesseractOCRPipelineModelTitle}" marked for deletion`); + resolve(response); + }, + (response) => { + if (response.status === 403) {app.flash('Forbidden', 'error');} + if (response.status === 404) {app.flash('Not Found', 'error');} + reject(response); + } + ); + }); + modal.open(); + }); + } + static deleteJobRequest(userId, jobId) { return new Promise((resolve, reject) => { let job = app.data.users[userId].jobs[jobId]; @@ -134,8 +230,8 @@ class Utils { `
      -

      Confirm job deletion

      -

      Do you really want to delete the job ${job.title}? All files will be permanently deleted!

      +

      Confirm Job deletion

      +

      Do you really want to delete the Job ${job.title}? All files will be permanently deleted!

      Cancel @@ -231,8 +327,8 @@ class Utils { `
      -

      Confirm job restart

      -

      Do you really want to restart the job ${job.title}? All log and result files will be permanently deleted.

      +

      Confirm Job restart

      +

      Do you really want to restart the Job ${job.title}? All Job Results will be permanently deleted.

      Cancel @@ -282,8 +378,8 @@ class Utils { `
      -

      Confirm job deletion

      -

      Do you really want to delete the user ${user.username}? All files will be permanently deleted!

      +

      Confirm User deletion

      +

      Do you really want to delete the User ${user.username}? All files will be permanently deleted!

      Cancel @@ -323,4 +419,50 @@ class Utils { modal.open(); }); } + + static shareTesseractOCRPipelineModelRequest(userId, tesseractOCRPipelineModelId, shared) { + return new Promise((resolve, reject) => { + let tesseractOCRPipelineModel = app.data.users[userId].tesseract_ocr_pipeline_models[tesseractOCRPipelineModelId]; + let msg = ''; + if (shared) { + msg = `Model "${tesseractOCRPipelineModel.title}" is now public`; + } else { + msg = `Model "${tesseractOCRPipelineModel.title}" is now private`; + } + fetch(`/contributions/tesseract-ocr-pipeline-models/${tesseractOCRPipelineModel.id}/toggle-public-status`, {method: 'POST', headers: {Accept: 'application/json'}}) + .then( + (response) => { + app.flash(msg); + resolve(response); + }, + (response) => { + if (response.status === 403) {app.flash('Forbidden', 'error');} + reject(response); + } + ); + }); + } + + static shareSpaCyNLPPipelineModelRequest(userId, spaCyNLPPipelineModelId, shared) { + return new Promise((resolve, reject) => { + let spaCyNLPPipelineModel = app.data.users[userId].spacy_nlp_pipeline_models[spaCyNLPPipelineModelId]; + let msg = ''; + if (shared) { + msg = `Model "${spaCyNLPPipelineModel.title}" is now public`; + } else { + msg = `Model "${spaCyNLPPipelineModel.title}" is now private`; + } + fetch(`/contributions/spacy-nlp-pipeline-models/${spaCyNLPPipelineModel.id}/toggle-public-status`, {method: 'POST', headers: {Accept: 'application/json'}}) + .then( + (response) => { + app.flash(msg); + resolve(response); + }, + (response) => { + if (response.status === 403) {app.flash('Forbidden', 'error');} + reject(response); + } + ); + }); + } } diff --git a/app/templates/_scripts.html.j2 b/app/templates/_scripts.html.j2 index ccc32a05..323c576c 100644 --- a/app/templates/_scripts.html.j2 +++ b/app/templates/_scripts.html.j2 @@ -9,6 +9,7 @@ 'js/Forms/Form.js', 'js/Forms/CreateCorpusFileForm.js', 'js/Forms/CreateJobForm.js', + 'js/Forms/CreateContributionForm.js', 'js/CorpusAnalysis/CQiClient.js', 'js/CorpusAnalysis/CorpusAnalysisApp.js', 'js/CorpusAnalysis/CorpusAnalysisConcordance.js', @@ -23,7 +24,8 @@ 'js/RessourceLists/JobList.js', 'js/RessourceLists/JobInputList.js', 'js/RessourceLists/JobResultList.js', - 'js/RessourceLists/QueryResultList.js', + 'js/RessourceLists/SpacyNLPPipelineModelList.js', + 'js/RessourceLists/TesseractOCRPipelineModelList.js', 'js/RessourceLists/UserList.js' %} diff --git a/app/templates/_sidenav.html.j2 b/app/templates/_sidenav.html.j2 index 553eb9db..b0ea6de8 100644 --- a/app/templates/_sidenav.html.j2 +++ b/app/templates/_sidenav.html.j2 @@ -12,6 +12,7 @@
    • dashboardDashboard
    • IMy Corpora
    • JMy Jobs
      • +
    • new_labelContribute
    • Processes & Services
    • File setup
      • @@ -34,7 +35,4 @@ {% if current_user.can(Permission.USE_API) %}
    • apiAPI
      • {% endif %} - {% if current_user.can(Permission.CONTRIBUTE) %} -
    • new_labelContribute
      • - {% endif %} diff --git a/app/templates/admin/edit_user.html.j2 b/app/templates/admin/edit_user.html.j2 index 45c27c6a..13e8a473 100644 --- a/app/templates/admin/edit_user.html.j2 +++ b/app/templates/admin/edit_user.html.j2 @@ -27,38 +27,6 @@
      -
      - {{ edit_interface_settings_form.hidden_tag() }} -
      -
      - Interface settings -
      -

       

      -
      -

      brightness_3

      -
      -
      -

      {{ edit_interface_settings_form.dark_mode.label.text }}

      -

      Enable dark mode to ease your eyes.

      -
      -
      -
      - -
      -
      -
      -
      -
      -
      - {{ wtf.render_field(edit_interface_settings_form.submit, material_icon='send') }} -
      -
      -
      -
      -
      {{ edit_notification_settings_form.hidden_tag() }}
      diff --git a/app/templates/base.html.j2 b/app/templates/base.html.j2 index af8bc2d3..dbecd35e 100644 --- a/app/templates/base.html.j2 +++ b/app/templates/base.html.j2 @@ -5,12 +5,6 @@ {% block head %} {{ super() }} -{% if current_user.setting_dark_mode %} - - -{% endif %} {% endblock head %} {% block metas %} diff --git a/app/templates/contributions/_breadcrumbs.html.j2 b/app/templates/contributions/_breadcrumbs.html.j2 new file mode 100644 index 00000000..ab64dec4 --- /dev/null +++ b/app/templates/contributions/_breadcrumbs.html.j2 @@ -0,0 +1,46 @@ +{% set breadcrumbs %} +
    • navigate_next
      • +{% if request.path == url_for('.contributions') %} +
    • Contributions Overview
      • +{% elif request.path == url_for('.tesseract_ocr_pipeline_models')%} +
    • Contributions Overview
      • +
    • navigate_next
      • +
    • Tesseract OCR Pipeline Models
      • +{% elif request.path == url_for('.spacy_nlp_pipeline_models')%} +
    • Contributions Overview
      • +
    • navigate_next
      • +
    • SpaCy NLP Pipeline Models
      • +{% elif request.path == url_for('.create_tesseract_ocr_pipeline_model') %} +
    • Contributions Overview
      • +
    • navigate_next
      • +
    • Tesseract OCR Pipeline Models
      • +
    • navigate_next
      • +
    • {{ title }}
      • +{% elif request.path == url_for('.create_spacy_nlp_pipeline_model') %} +
    • Contributions Overview
      • +
    • navigate_next
      • +
    • SpaCy NLP Pipeline Models
      • +
    • navigate_next
      • +
    • {{ title }}
      • +{% elif tesseract_ocr_pipeline_model and request.path == url_for('.tesseract_ocr_pipeline_model', tesseract_ocr_pipeline_model_id=tesseract_ocr_pipeline_model.id) %} +
    • Contributions Overview
      • +
    • navigate_next
      • +
    • Tesseract OCR Pipeline Models
      • +
    • navigate_next
      • +
    • + + {{ tesseract_ocr_pipeline_model.title }} {{ tesseract_ocr_pipeline_model.version }} + +
      • +{% elif spacy_nlp_pipeline_model and request.path == url_for('.spacy_nlp_pipeline_model', spacy_nlp_pipeline_model_id=spacy_nlp_pipeline_model.id) %} +
    • Contributions Overview
      • +
    • navigate_next
      • +
    • SpaCy NLP Pipeline Models
      • +
    • navigate_next
      • +
    • + + {{ spacy_nlp_pipeline_model.title }} {{ spacy_nlp_pipeline_model.version }} + +
      • +{% endif %} +{% endset %} diff --git a/app/templates/contributions/contribute.html.j2 b/app/templates/contributions/contribute.html.j2 deleted file mode 100644 index 6789e1f8..00000000 --- a/app/templates/contributions/contribute.html.j2 +++ /dev/null @@ -1,32 +0,0 @@ -{% extends "base.html.j2" %} -{% import "materialize/wtf.html.j2" as wtf %} - - -{% block page_content %} -
      -
      -
      -

      {{ title }}

      -

      - In order to add a new model, please fill in the form below. -

      - - -
      - {{ form.hidden_tag() }} - {{ wtf.render_field(form.title) }} - {{ wtf.render_field(form.description) }} - {{ wtf.render_field(form.publisher) }} - {{ wtf.render_field(form.publisher_url) }} - {{ wtf.render_field(form.publishing_url) }} - {{ wtf.render_field(form.publishing_year) }} - {{ wtf.render_field(form.shared) }} - {{ wtf.render_field(form.version) }} - {{ wtf.render_field(form.compatible_service_versions) }} - {{ wtf.render_field(form.submit, class_='width-100', material_icon='send') }} - -
      - -
      -
      -{% endblock page_content %} \ No newline at end of file diff --git a/app/templates/contributions/contributions.html.j2 b/app/templates/contributions/contributions.html.j2 new file mode 100644 index 00000000..4e513b9e --- /dev/null +++ b/app/templates/contributions/contributions.html.j2 @@ -0,0 +1,45 @@ +{% extends "base.html.j2" %} +{% import "materialize/wtf.html.j2" as wtf %} +{% from "contributions/_breadcrumbs.html.j2" import breadcrumbs with context %} + +{% block page_content %} +
      +
      +
      +

      {{ title }}

      +
      + +
      +
      + +
      + Tesseract OCR Pipeline Models +

      Here you can see and edit the models that you have created. You can also create new models.

      +
      +
      +
      + +
      +
      + +
      + SpaCy NLP Pipeline Models +

      Here you can see and edit the models that you have created. You can also create new models.

      +
      +
      +
      + + +
      +
      +{% endblock page_content %} diff --git a/app/templates/contributions/create_spacy_nlp_pipeline_model.html.j2 b/app/templates/contributions/create_spacy_nlp_pipeline_model.html.j2 new file mode 100644 index 00000000..c1f452d3 --- /dev/null +++ b/app/templates/contributions/create_spacy_nlp_pipeline_model.html.j2 @@ -0,0 +1,85 @@ +{% extends "base.html.j2" %} +{% import "materialize/wtf.html.j2" as wtf %} +{% from "contributions/_breadcrumbs.html.j2" import breadcrumbs with context %} + +{% block main_attribs %} class="service-scheme" data-service="spacy-nlp-pipeline"{% endblock main_attribs %} + +{% block page_content %} +
      +
      +
      +

      {{ title }}

      +
      + +
      +
      +

       

      +

       

      + + + +
      +
      + +
      +
      +
      +
      +
      +
      + layersSpaCy NLP Pipeline Model +

      You can create a new SpaCy NLP Pipeline Model using the form below. They will automatically appear in the list of usable models on the SpaCy NLP Pipeline service page.

      +
      +
      +
      +
      +
      +
      + +
      +
      +
      +
      + {{ form.hidden_tag() }} +
      +
      + {{ wtf.render_field(form.spacy_model_file, accept='.tar.gz', placeholder='Choose a .tar.gz file') }} +
      +
      + {{ wtf.render_field(form.title, material_icon='title') }} +
      +
      + {{ wtf.render_field(form.description, material_icon='description') }} +
      +
      + {{ wtf.render_field(form.pipeline_name, material_icon='emoji_objects') }} +
      +
      + {{ wtf.render_field(form.publisher, material_icon='account_balance') }} +
      +
      + {{ wtf.render_field(form.publishing_year, material_icon='calendar_month') }} +
      +
      + {{ wtf.render_field(form.publisher_url, material_icon='link') }} +
      +
      + {{ wtf.render_field(form.publishing_url, material_icon='link') }} +
      +
      + {{ wtf.render_field(form.version, material_icon='apps') }} +
      +
      + {{ wtf.render_field(form.compatible_service_versions) }} +
      +
      +
      +
      + {{ wtf.render_field(form.submit, material_icon='send') }} +
      +
      +
      +
      +
      +
      +{% endblock page_content %} diff --git a/app/templates/contributions/create_tesseract_ocr_pipeline_model.html.j2 b/app/templates/contributions/create_tesseract_ocr_pipeline_model.html.j2 new file mode 100644 index 00000000..e4c5a04b --- /dev/null +++ b/app/templates/contributions/create_tesseract_ocr_pipeline_model.html.j2 @@ -0,0 +1,116 @@ +{% extends "base.html.j2" %} +{% import "materialize/wtf.html.j2" as wtf %} +{% from "contributions/_breadcrumbs.html.j2" import breadcrumbs with context %} + +{% block main_attribs %} class="service-scheme" data-service="tesseract-ocr-pipeline"{% endblock main_attribs %} + +{% block page_content %} +
      +
      +
      +

      {{ title }}

      +
      + +
      +
      +

       

      +

       

      + + + +
      +
      + +
      +
      +
      +
      +
      +
      + layersTesseract OCR Pipeline Model +

      You can create a new Tesseract OCR Pipeline Model using the form below. They will automatically appear in the list of usable models on the Tesseract OCR Pipeline service page.

      +
      +
      +
      +
      +
      +
      + +
      +
      +
      +
      + {{ form.hidden_tag() }} +
      +
      + {{ wtf.render_field(form.tesseract_model_file, accept='.traineddata', placeholder='Choose a .traineddata file') }} +
      +
      + {{ wtf.render_field(form.title, material_icon='title') }} +
      +
      + {{ wtf.render_field(form.description, material_icon='description') }} +
      +
      + {{ wtf.render_field(form.publisher, material_icon='account_balance') }} +
      +
      + {{ wtf.render_field(form.publishing_year, material_icon='calendar_month') }} +
      +
      + {{ wtf.render_field(form.publisher_url, material_icon='link') }} +
      +
      + {{ wtf.render_field(form.publishing_url, material_icon='link') }} +
      +
      + {{ wtf.render_field(form.version, material_icon='apps') }} +
      +
      + {{ wtf.render_field(form.compatible_service_versions) }} +
      +
      +
      +
      + {{ wtf.render_field(form.submit, material_icon='send') }} +
      +
      +
      +
      +
      +
      +{% endblock page_content %} + +{% block modals %} +{{ super() }} + +{% endblock modals %} diff --git a/app/templates/contributions/spacy_nlp_pipeline_model.html.j2 b/app/templates/contributions/spacy_nlp_pipeline_model.html.j2 new file mode 100644 index 00000000..32f27303 --- /dev/null +++ b/app/templates/contributions/spacy_nlp_pipeline_model.html.j2 @@ -0,0 +1,57 @@ +{% extends "base.html.j2" %} +{% import "materialize/wtf.html.j2" as wtf %} +{% from "contributions/_breadcrumbs.html.j2" import breadcrumbs with context %} + +{% block main_attribs %} class="service-scheme" data-service="spacy-nlp-pipeline"{% endblock main_attribs %} + +{% block page_content %} +
      +
      +
      +

      {{ title }}

      +
      + +
      +
      +
      +
      + {{ form.hidden_tag() }} +
      +
      + {{ wtf.render_field(form.title, material_icon='title') }} +
      +
      + {{ wtf.render_field(form.description, material_icon='description') }} +
      +
      + {{ wtf.render_field(form.pipeline_name, material_icon='emoji_objects') }} +
      +
      + {{ wtf.render_field(form.publisher, material_icon='account_balance') }} +
      +
      + {{ wtf.render_field(form.publishing_year, material_icon='calendar_month') }} +
      +
      + {{ wtf.render_field(form.publisher_url, material_icon='link') }} +
      +
      + {{ wtf.render_field(form.publishing_url, material_icon='link') }} +
      +
      + {{ wtf.render_field(form.version, material_icon='apps') }} +
      +
      + {{ wtf.render_field(form.compatible_service_versions) }} +
      +
      +
      +
      + {{ wtf.render_field(form.submit, material_icon='send') }} +
      +
      +
      +
      +
      +
      +{% endblock page_content %} diff --git a/app/templates/contributions/spacy_nlp_pipeline_models.html.j2 b/app/templates/contributions/spacy_nlp_pipeline_models.html.j2 new file mode 100644 index 00000000..f3c7a40e --- /dev/null +++ b/app/templates/contributions/spacy_nlp_pipeline_models.html.j2 @@ -0,0 +1,25 @@ +{% extends "base.html.j2" %} +{% import "materialize/wtf.html.j2" as wtf %} +{% from "contributions/_breadcrumbs.html.j2" import breadcrumbs with context %} + +{% block page_content %} +
      +
      +
      +

      {{ title }}

      +

      Here you can see and edit the models that you have created. You can also create new models.

      +
      + +
      +
      +
      +
      +
      +
      + addCreate +
      +
      +
      +
      +
      +{% endblock page_content %} diff --git a/app/templates/contributions/tesseract_ocr_pipeline_model.html.j2 b/app/templates/contributions/tesseract_ocr_pipeline_model.html.j2 new file mode 100644 index 00000000..02322d8a --- /dev/null +++ b/app/templates/contributions/tesseract_ocr_pipeline_model.html.j2 @@ -0,0 +1,54 @@ +{% extends "base.html.j2" %} +{% import "materialize/wtf.html.j2" as wtf %} +{% from "contributions/_breadcrumbs.html.j2" import breadcrumbs with context %} + +{% block main_attribs %} class="service-scheme" data-service="tesseract-ocr-pipeline"{% endblock main_attribs %} + +{% block page_content %} +
      +
      +
      +

      {{ title }}

      +
      + +
      +
      +
      +
      + {{ form.hidden_tag() }} +
      +
      + {{ wtf.render_field(form.title, material_icon='title') }} +
      +
      + {{ wtf.render_field(form.description, material_icon='description') }} +
      +
      + {{ wtf.render_field(form.publisher, material_icon='account_balance') }} +
      +
      + {{ wtf.render_field(form.publishing_year, material_icon='calendar_month') }} +
      +
      + {{ wtf.render_field(form.publisher_url, material_icon='link') }} +
      +
      + {{ wtf.render_field(form.publishing_url, material_icon='link') }} +
      +
      + {{ wtf.render_field(form.version, material_icon='apps') }} +
      +
      + {{ wtf.render_field(form.compatible_service_versions) }} +
      +
      +
      +
      + {{ wtf.render_field(form.submit, material_icon='send') }} +
      +
      +
      +
      +
      +
      +{% endblock page_content %} diff --git a/app/templates/contributions/tesseract_ocr_pipeline_models.html.j2 b/app/templates/contributions/tesseract_ocr_pipeline_models.html.j2 new file mode 100644 index 00000000..3d43d727 --- /dev/null +++ b/app/templates/contributions/tesseract_ocr_pipeline_models.html.j2 @@ -0,0 +1,25 @@ +{% extends "base.html.j2" %} +{% import "materialize/wtf.html.j2" as wtf %} +{% from "contributions/_breadcrumbs.html.j2" import breadcrumbs with context %} + +{% block page_content %} +
      +
      +
      +

      {{ title }}

      +

      Here you can see and edit the models that you have created. You can also create new models.

      +
      + +
      +
      +
      +
      +
      +
      + addCreate +
      +
      +
      +
      +
      +{% endblock page_content %} diff --git a/app/templates/corpora/analyse_corpus.html.j2 b/app/templates/corpora/analyse_corpus.html.j2 index 779eafd4..73ebfd77 100644 --- a/app/templates/corpora/analyse_corpus.html.j2 +++ b/app/templates/corpora/analyse_corpus.html.j2 @@ -1,9 +1,5 @@ {% extends "base.html.j2" %} {% import "materialize/wtf.html.j2" as wtf %} - - {% block main_attribs %} class="service-scheme" data-service="corpus-analysis" id="corpus-analysis-app-container"{% endblock main_attribs %} {% block page_content %} diff --git a/app/templates/main/dashboard.html.j2 b/app/templates/main/dashboard.html.j2 index 0d156517..ec03609d 100644 --- a/app/templates/main/dashboard.html.j2 +++ b/app/templates/main/dashboard.html.j2 @@ -6,27 +6,18 @@

      {{ title }}

      +
      -

      My Corpora and Query results

      -

      Create a corpus to interactively perform linguistic analysis or import query results to save interesting passages.

      - -
      -
      - +
      +

      My Corpora

      +

      Create a corpus to interactively perform linguistic analysis.

      +
      +
      +
      -
      diff --git a/app/templates/services/spacy_nlp_pipeline.html.j2 b/app/templates/services/spacy_nlp_pipeline.html.j2 index 37f70210..6443c506 100644 --- a/app/templates/services/spacy_nlp_pipeline.html.j2 +++ b/app/templates/services/spacy_nlp_pipeline.html.j2 @@ -70,8 +70,16 @@
      {{ wtf.render_field(form.txt, accept='text/plain', placeholder='Choose a plain text file') }}
      -
      - {{ wtf.render_field(form.model, material_icon='language') }} +
      +
      + language + {{ form.model() }} + {{ form.model.label }} + + help_outline + new_label + +
      {{ wtf.render_field(form.version, material_icon='apps') }} @@ -79,6 +87,7 @@
      Preprocessing
      + {% if 'disabled' not in form.encoding_detection.render_kw or not form.encoding_detection.render_kw['disabled'] %}

      {{ form.encoding_detection.label.text }}

      If the input files are not created with the nopaque OCR service or you do not know if your text files are UTF-8 encoded, check this switch. We will try to automatically determine the right encoding for your texts to process them.

      @@ -91,6 +100,7 @@
      + {% endif %}