diff --git a/.dockerignore b/.dockerignore index 9960fd26..518de1be 100644 --- a/.dockerignore +++ b/.dockerignore @@ -8,5 +8,6 @@ !.flaskenv !boot.sh !config.py +!docker-nopaque-entrypoint.sh !nopaque.py !requirements.txt diff --git a/.env.tpl b/.env.tpl index b63e2920..1f6731cd 100644 --- a/.env.tpl +++ b/.env.tpl @@ -1,204 +1,32 @@ -################################################################################ -# Docker # -################################################################################ -# DEFAULT: ./data -# NOTE: Use `.` as -# HOST_DATA_DIR= - -# Example: 1000 +############################################################################## +# Variables for use in Docker Compose YAML files # +############################################################################## # HINT: Use this bash command `id -u` +# NOTE: 0 (= root user) is not allowed HOST_UID= -# Example: 1000 # HINT: Use this bash command `id -g` HOST_GID= -# Example: 999 # HINT: Use this bash command `getent group docker | cut -d: -f3` HOST_DOCKER_GID= -# DEFAULT: ./logs -# NOTES: Use `.` as -# HOST_LOG_DIR= +# DEFAULT: nopaque +# DOCKER_DEFAULT_NETWORK_NAME= -# DEFAULT: nopaque_default -# DOCKER_NETWORK_NAME= +# DEFAULT: ./volumes/db/data +# NOTE: Use `.` as +# DOCKER_DB_SERVICE_DATA_VOLUME_SOURCE_PATH= -################################################################################ -# Flask # -# https://flask.palletsprojects.com/en/1.1.x/config/ # -################################################################################ -# CHOOSE ONE: http, https -# DEFAULT: http -# PREFERRED_URL_SCHEME= +# DEFAULT: ./volumes/mq/data +# NOTE: Use `.` as +# DOCKER_MQ_SERVICE_DATA_VOLUME_SOURCE_PATH= -# DEFAULT: hard to guess string -# HINT: Use this bash command `python -c "import uuid; print(uuid.uuid4().hex)"` -# SECRET_KEY= +# NOTE: This must be a network share and it must be available on all +# Docker Swarm nodes, mounted to the same path with the same +# user and group ownership. +DOCKER_NOPAQUE_SERVICE_DATA_VOLUME_SOURCE_PATH= -# DEFAULT: localhost:5000 -# Example: nopaque.example.com/nopaque.example.com:5000 -# HINT: If your instance is publicly available on a different Port then 80/443, -# you will have to add this to the server name -# SERVER_NAME= - -# CHOOSE ONE: False, True -# DEFAULT: False -# HINT: Set to true if you redirect http to https -# SESSION_COOKIE_SECURE= - - -################################################################################ -# Flask-Assets # -# https://webassets.readthedocs.io/en/latest/ # -################################################################################ -# CHOOSE ONE: False, True -# DEFAULT: False -# ASSETS_DEBUG= - - -################################################################################ -# Flask-Hashids # -# https://github.com/Pevtrick/Flask-Hashids # -################################################################################ -# DEFAULT: 16 -# HASHIDS_MIN_LENGTH= - -# NOTE: Use this bash command `python -c "import uuid; print(uuid.uuid4().hex)"` -# It is strongly recommended that this is NEVER the same as the SECRET_KEY -HASHIDS_SALT= - - -################################################################################ -# Flask-Login # -# https://flask-login.readthedocs.io/en/latest/ # -################################################################################ -# CHOOSE ONE: False, True -# DEFAULT: False -# HINT: Set to true if you redirect http to https -# REMEMBER_COOKIE_SECURE= - - -################################################################################ -# Flask-Mail # -# https://pythonhosted.org/Flask-Mail/ # -################################################################################ -# EXAMPLE: nopaque Admin -MAIL_DEFAULT_SENDER= - -MAIL_PASSWORD= - -# EXAMPLE: smtp.example.com -MAIL_SERVER= - -# EXAMPLE: 587 -MAIL_PORT= - -# CHOOSE ONE: False, True -# DEFAULT: False -# MAIL_USE_SSL= - -# CHOOSE ONE: False, True -# DEFAULT: False -# MAIL_USE_TLS= - -# EXAMPLE: nopaque@example.com -MAIL_USERNAME= - - -################################################################################ -# Flask-SQLAlchemy # -# https://flask-sqlalchemy.palletsprojects.com/en/2.x/config/ # -################################################################################ -# DEFAULT: 'sqlite:////data.sqlite' -# NOTE: Use `.` as , -# Don't use a SQLite database when using Docker -# SQLALCHEMY_DATABASE_URI= - - -################################################################################ -# nopaque # -################################################################################ -# An account is registered with this email adress gets automatically assigned -# the administrator role. -# EXAMPLE: admin.nopaque@example.com -NOPAQUE_ADMIN= - -# DEFAULT: /mnt/nopaque -# NOTE: This must be a network share and it must be available on all Docker -# Swarm nodes -# NOPAQUE_DATA_DIR= - -# CHOOSE ONE: False, True -# DEFAULT: True -# NOPAQUE_IS_PRIMARY_INSTANCE= - -# transport://[userid:password]@hostname[:port]/[virtual_host] -NOPAQUE_SOCKETIO_MESSAGE_QUEUE_URI= - -# NOTE: Get these from the nopaque development team -NOPAQUE_DOCKER_REGISTRY_USERNAME= -NOPAQUE_DOCKER_REGISTRY_PASSWORD= - -# DEFAULT: %Y-%m-%d %H:%M:%S -# NOPAQUE_LOG_DATE_FORMAT= - -# DEFAULT: [%(asctime)s] %(levelname)s in %(pathname)s (function: %(funcName)s, line: %(lineno)d): %(message)s -# NOPAQUE_LOG_FORMAT= - -# DEFAULT: INFO -# CHOOSE ONE: CRITICAL, ERROR, WARNING, INFO, DEBUG -# NOPAQUE_LOG_LEVEL= - -# CHOOSE ONE: False, True -# DEFAULT: True -# NOPAQUE_LOG_FILE_ENABLED= - -# DEFAULT: /logs -# NOTE: Use `.` as -# NOPAQUE_LOG_FILE_DIR= - -# DEFAULT: NOPAQUE_LOG_LEVEL -# CHOOSE ONE: CRITICAL, ERROR, WARNING, INFO, DEBUG -# NOPAQUE_LOG_FILE_LEVEL= - -# CHOOSE ONE: False, True -# DEFAULT: False -# NOPAQUE_LOG_STDERR_ENABLED= - -# CHOOSE ONE: CRITICAL, ERROR, WARNING, INFO, DEBUG -# DEFAULT: NOPAQUE_LOG_LEVEL -# NOPAQUE_LOG_STDERR_LEVEL= - -# CHOOSE ONE: False, True -# DEFAULT: False -# HINT: Set this to True only if you are using a proxy in front of nopaque -# NOPAQUE_PROXY_FIX_ENABLED= - -# DEFAULT: 0 -# Number of values to trust for X-Forwarded-For -# NOPAQUE_PROXY_FIX_X_FOR= - -# DEFAULT: 0 -# Number of values to trust for X-Forwarded-Host -# NOPAQUE_PROXY_FIX_X_HOST= - -# DEFAULT: 0 -# Number of values to trust for X-Forwarded-Port -# NOPAQUE_PROXY_FIX_X_PORT= - -# DEFAULT: 0 -# Number of values to trust for X-Forwarded-Prefix -# NOPAQUE_PROXY_FIX_X_PREFIX= - -# DEFAULT: 0 -# Number of values to trust for X-Forwarded-Proto -# NOPAQUE_PROXY_FIX_X_PROTO= - -# CHOOSE ONE: False, True -# DEFAULT: False -# NOPAQUE_TRANSKRIBUS_ENABLED= - -# READ-COOP account data: https://readcoop.eu/ -# NOPAQUE_READCOOP_USERNAME= -# NOPAQUE_READCOOP_PASSWORD= +# DEFAULT: ./volumes/nopaque/logs +# NOTE: Use `.` as +# DOCKER_NOPAQUE_SERVICE_LOGS_VOLUME_SOURCE_PATH=. diff --git a/.gitignore b/.gitignore index b7a84431..d9a03f48 100644 --- a/.gitignore +++ b/.gitignore @@ -1,6 +1,6 @@ # nopaque specifics app/static/gen/ -data/ +volumes/ docker-compose.override.yml logs/ !logs/dummy diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml new file mode 100644 index 00000000..f5759d43 --- /dev/null +++ b/.gitlab-ci.yml @@ -0,0 +1,84 @@ +include: + - template: Security/Container-Scanning.gitlab-ci.yml + +############################################################################## +# Pipeline stages in order of execution # +############################################################################## +stages: + - build + - publish + - sca + +############################################################################## +# Pipeline behavior # +############################################################################## +workflow: + rules: + # Run the pipeline on commits to the default branch + - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH + variables: + # Set the Docker image tag to `latest` + DOCKER_IMAGE: $CI_REGISTRY_IMAGE:latest + when: always + # Run the pipeline on tag creation + - if: $CI_COMMIT_TAG + variables: + # Set the Docker image tag to the Git tag name + DOCKER_IMAGE: $CI_REGISTRY_IMAGE:$CI_COMMIT_REF_NAME + when: always + # Don't run the pipeline on all other occasions + - when: never + +############################################################################## +# Default values for pipeline jobs # +############################################################################## +default: + image: docker:24.0.6 + services: + - docker:24.0.6-dind + tags: + - docker + +############################################################################## +# CI/CD variables for all jobs in the pipeline # +############################################################################## +variables: + DOCKER_TLS_CERTDIR: /certs + DOCKER_BUILD_PATH: . + DOCKERFILE: Dockerfile + +############################################################################## +# Pipeline jobs # +############################################################################## +build: + stage: build + script: + - docker build --tag $DOCKER_IMAGE --file $DOCKERFILE $DOCKER_BUILD_PATH + - docker save $DOCKER_IMAGE > docker_image.tar + artifacts: + paths: + - docker_image.tar + +publish: + stage: publish + before_script: + - docker login --username gitlab-ci-token --password $CI_JOB_TOKEN $CI_REGISTRY + script: + - docker load --input docker_image.tar + - docker push $DOCKER_IMAGE + after_script: + - docker logout $CI_REGISTRY + +container_scanning: + stage: sca + rules: + # Run the job on commits to the default branch + - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH + when: always + # Run the job on tag creation + - if: $CI_COMMIT_TAG + when: always + # Don't run the job on all other occasions + - when: never + variables: + CS_IMAGE: $DOCKER_IMAGE diff --git a/.vscode/extensions.json b/.vscode/extensions.json index ecf868e0..6dab7cd6 100644 --- a/.vscode/extensions.json +++ b/.vscode/extensions.json @@ -1,7 +1,8 @@ { "recommendations": [ - "samuelcolvin.jinjahtml", + "irongeek.vscode-env", "ms-azuretools.vscode-docker", - "ms-python.python" + "ms-python.python", + "samuelcolvin.jinjahtml" ] } diff --git a/.vscode/settings.json b/.vscode/settings.json index abafcbe3..a8cd0c3d 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -1,13 +1,9 @@ { "editor.rulers": [79], "files.insertFinalNewline": true, - "python.terminal.activateEnvironment": false, "[css]": { "editor.tabSize": 2 }, - "[scss]": { - "editor.tabSize": 2 - }, "[html]": { "editor.tabSize": 2 }, @@ -17,7 +13,7 @@ "[jinja-html]": { "editor.tabSize": 2 }, - "[jinja-js]": { + "[scss]": { "editor.tabSize": 2 } } diff --git a/Dockerfile b/Dockerfile index fe63463e..1342134f 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,14 +1,9 @@ -FROM python:3.8.10-slim-buster +FROM python:3.10.13-slim-bookworm LABEL authors="Patrick Jentsch " -ARG DOCKER_GID -ARG UID -ARG GID - - ENV LANG="C.UTF-8" ENV PYTHONDONTWRITEBYTECODE="1" ENV PYTHONUNBUFFERED="1" @@ -17,34 +12,42 @@ ENV PYTHONUNBUFFERED="1" RUN apt-get update \ && apt-get install --no-install-recommends --yes \ build-essential \ + gosu \ libpq-dev \ && rm --recursive /var/lib/apt/lists/* -RUN groupadd --gid "${DOCKER_GID}" docker \ - && groupadd --gid "${GID}" nopaque \ - && useradd --create-home --gid nopaque --groups "${DOCKER_GID}" --no-log-init --uid "${UID}" nopaque +RUN useradd --create-home --no-log-init nopaque \ + && groupadd docker \ + && usermod --append --groups docker nopaque + + USER nopaque WORKDIR /home/nopaque -ENV PYTHON3_VENV_PATH="/home/nopaque/venv" -RUN python3 -m venv "${PYTHON3_VENV_PATH}" -ENV PATH="${PYTHON3_VENV_PATH}/bin:${PATH}" - - -COPY --chown=nopaque:nopaque requirements.txt . -RUN python3 -m pip install --requirement requirements.txt \ - && rm requirements.txt +ENV NOPAQUE_PYTHON3_VENV_PATH="/home/nopaque/.venv" +RUN python3 -m venv "${NOPAQUE_PYTHON3_VENV_PATH}" +ENV PATH="${NOPAQUE_PYTHON3_VENV_PATH}/bin:${PATH}" COPY --chown=nopaque:nopaque app app COPY --chown=nopaque:nopaque migrations migrations COPY --chown=nopaque:nopaque tests tests -COPY --chown=nopaque:nopaque .flaskenv boot.sh config.py nopaque.py ./ +COPY --chown=nopaque:nopaque .flaskenv boot.sh config.py nopaque.py requirements.txt ./ + + +RUN python3 -m pip install --requirement requirements.txt \ + && mkdir logs + + +USER root + + +COPY docker-nopaque-entrypoint.sh /usr/local/bin/ EXPOSE 5000 -ENTRYPOINT ["./boot.sh"] +ENTRYPOINT ["docker-nopaque-entrypoint.sh"] diff --git a/README.md b/README.md index 30c84ae2..cabdf34b 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,8 @@ # nopaque +![release badge](https://gitlab.ub.uni-bielefeld.de/sfb1288inf/nopaque/-/badges/release.svg) +![pipeline badge](https://gitlab.ub.uni-bielefeld.de/sfb1288inf/nopaque/badges/master/pipeline.svg?ignore_skipped=true) + nopaque bundles various tools and services that provide humanities scholars with DH methods and thus can support their various individual research processes. Using nopaque, researchers can subject digitized sources to Optical Character Recognition (OCR). The resulting text files can then be used as a data basis for Natural Language Processing (NLP). The texts are automatically subjected to various linguistic annotations. The data processed via NLP can then be summarized in the web application as corpora and analyzed by means of an information retrieval system through complex search queries. The range of functions of the web application will be successively extended according to the needs of the researchers. ## Prerequisites and requirements diff --git a/app/SpaCyNLPPipelineModel.defaults.yml b/app/SpaCyNLPPipelineModel.defaults.yml index 62dc5e65..cabbb7a3 100644 --- a/app/SpaCyNLPPipelineModel.defaults.yml +++ b/app/SpaCyNLPPipelineModel.defaults.yml @@ -8,7 +8,7 @@ pipeline_name: 'ca_core_news_md' version: '3.2.0' compatible_service_versions: - - '0.1.0' + - '0.1.0' - title: 'German' description: 'German pipeline optimized for CPU. Components: tok2vec, tagger, morphologizer, parser, senter, ner, attribute_ruler, lemmatizer.' url: 'https://github.com/explosion/spacy-models/releases/download/de_core_news_md-3.2.0/de_core_news_md-3.2.0.tar.gz' @@ -19,7 +19,7 @@ pipeline_name: 'de_core_news_md' version: '3.2.0' compatible_service_versions: - - '0.1.0' + - '0.1.0' - title: 'Greek' description: 'Greek pipeline optimized for CPU. Components: tok2vec, morphologizer, parser, senter, ner, attribute_ruler, lemmatizer.' url: 'https://github.com/explosion/spacy-models/releases/download/el_core_news_md-3.2.0/el_core_news_md-3.2.0.tar.gz' @@ -120,7 +120,6 @@ version: '3.4.0' compatible_service_versions: - '0.1.1' - - '0.1.2' - title: 'German' description: 'German pipeline optimized for CPU. Components: tok2vec, tagger, morphologizer, parser, lemmatizer (trainable_lemmatizer), senter, ner.' url: 'https://github.com/explosion/spacy-models/releases/download/de_core_news_md-3.4.0/de_core_news_md-3.4.0.tar.gz' @@ -132,7 +131,6 @@ version: '3.4.0' compatible_service_versions: - '0.1.1' - - '0.1.2' - title: 'Greek' description: 'Greek pipeline optimized for CPU. Components: tok2vec, morphologizer, parser, lemmatizer (trainable_lemmatizer), senter, ner, attribute_ruler.' url: 'https://github.com/explosion/spacy-models/releases/download/el_core_news_md-3.4.0/el_core_news_md-3.4.0.tar.gz' @@ -144,7 +142,6 @@ version: '3.4.0' compatible_service_versions: - '0.1.1' - - '0.1.2' - title: 'English' description: 'English pipeline optimized for CPU. Components: tok2vec, tagger, parser, senter, ner, attribute_ruler, lemmatizer.' url: 'https://github.com/explosion/spacy-models/releases/download/en_core_web_md-3.4.1/en_core_web_md-3.4.1.tar.gz' @@ -156,7 +153,6 @@ version: '3.4.1' compatible_service_versions: - '0.1.1' - - '0.1.2' - title: 'Spanish' description: 'Spanish pipeline optimized for CPU. Components: tok2vec, morphologizer, parser, senter, ner, attribute_ruler, lemmatizer.' url: 'https://github.com/explosion/spacy-models/releases/download/es_core_news_md-3.4.0/es_core_news_md-3.4.0.tar.gz' @@ -168,7 +164,6 @@ version: '3.4.0' compatible_service_versions: - '0.1.1' - - '0.1.2' - title: 'French' description: 'French pipeline optimized for CPU. Components: tok2vec, morphologizer, parser, senter, ner, attribute_ruler, lemmatizer.' url: 'https://github.com/explosion/spacy-models/releases/download/fr_core_news_md-3.4.0/fr_core_news_md-3.4.0.tar.gz' @@ -180,7 +175,6 @@ version: '3.4.0' compatible_service_versions: - '0.1.1' - - '0.1.2' - title: 'Italian' description: 'Italian pipeline optimized for CPU. Components: tok2vec, morphologizer, tagger, parser, lemmatizer (trainable_lemmatizer), senter, ner' url: 'https://github.com/explosion/spacy-models/releases/download/it_core_news_md-3.4.0/it_core_news_md-3.4.0.tar.gz' @@ -192,7 +186,6 @@ version: '3.4.0' compatible_service_versions: - '0.1.1' - - '0.1.2' - title: 'Polish' description: 'Polish pipeline optimized for CPU. Components: tok2vec, morphologizer, parser, lemmatizer (trainable_lemmatizer), tagger, senter, ner.' url: 'https://github.com/explosion/spacy-models/releases/download/pl_core_news_md-3.4.0/pl_core_news_md-3.4.0.tar.gz' @@ -204,7 +197,6 @@ version: '3.4.0' compatible_service_versions: - '0.1.1' - - '0.1.2' - title: 'Russian' description: 'Russian pipeline optimized for CPU. Components: tok2vec, morphologizer, parser, senter, ner, attribute_ruler, lemmatizer.' url: 'https://github.com/explosion/spacy-models/releases/download/ru_core_news_md-3.4.0/ru_core_news_md-3.4.0.tar.gz' @@ -216,7 +208,6 @@ version: '3.4.0' compatible_service_versions: - '0.1.1' - - '0.1.2' - title: 'Chinese' description: 'Chinese pipeline optimized for CPU. Components: tok2vec, tagger, parser, senter, ner, attribute_ruler.' url: 'https://github.com/explosion/spacy-models/releases/download/zh_core_web_md-3.4.0/zh_core_web_md-3.4.0.tar.gz' @@ -228,4 +219,3 @@ version: '3.4.0' compatible_service_versions: - '0.1.1' - - '0.1.2' diff --git a/app/TesseractOCRPipelineModel.defaults.yml b/app/TesseractOCRPipelineModel.defaults.yml index 834b0ea5..e83bb503 100644 --- a/app/TesseractOCRPipelineModel.defaults.yml +++ b/app/TesseractOCRPipelineModel.defaults.yml @@ -9,6 +9,7 @@ # compatible_service_versions: # - '0.1.0' # - '0.1.1' +# - '0.1.2' # - title: 'Amharic' # description: '' # url: 'https://github.com/tesseract-ocr/tessdata/raw/4.1.0/amh.traineddata' @@ -20,6 +21,7 @@ # compatible_service_versions: # - '0.1.0' # - '0.1.1' +# - '0.1.2' - title: 'Arabic' description: '' url: 'https://github.com/tesseract-ocr/tessdata/raw/4.1.0/ara.traineddata' @@ -31,6 +33,7 @@ compatible_service_versions: - '0.1.0' - '0.1.1' + - '0.1.2' # - title: 'Assamese' # description: '' # url: 'https://github.com/tesseract-ocr/tessdata/raw/4.1.0/asm.traineddata' @@ -42,6 +45,7 @@ # compatible_service_versions: # - '0.1.0' # - '0.1.1' +# - '0.1.2' # - title: 'Azerbaijani' # description: '' # url: 'https://github.com/tesseract-ocr/tessdata/raw/4.1.0/aze.traineddata' @@ -53,6 +57,7 @@ # compatible_service_versions: # - '0.1.0' # - '0.1.1' +# - '0.1.2' # - title: 'Azerbaijani - Cyrillic' # description: '' # url: 'https://github.com/tesseract-ocr/tessdata/raw/4.1.0/aze_cyrl.traineddata' @@ -64,6 +69,7 @@ # compatible_service_versions: # - '0.1.0' # - '0.1.1' +# - '0.1.2' # - title: 'Belarusian' # description: '' # url: 'https://github.com/tesseract-ocr/tessdata/raw/4.1.0/bel.traineddata' @@ -75,6 +81,7 @@ # compatible_service_versions: # - '0.1.0' # - '0.1.1' +# - '0.1.2' # - title: 'Bengali' # description: '' # url: 'https://github.com/tesseract-ocr/tessdata/raw/4.1.0/ben.traineddata' @@ -86,6 +93,7 @@ # compatible_service_versions: # - '0.1.0' # - '0.1.1' +# - '0.1.2' # - title: 'Tibetan' # description: '' # url: 'https://github.com/tesseract-ocr/tessdata/raw/4.1.0/bod.traineddata' @@ -97,6 +105,7 @@ # compatible_service_versions: # - '0.1.0' # - '0.1.1' +# - '0.1.2' # - title: 'Bosnian' # description: '' # url: 'https://github.com/tesseract-ocr/tessdata/raw/4.1.0/bos.traineddata' @@ -108,6 +117,7 @@ # compatible_service_versions: # - '0.1.0' # - '0.1.1' +# - '0.1.2' # - title: 'Bulgarian' # description: '' # url: 'https://github.com/tesseract-ocr/tessdata/raw/4.1.0/bul.traineddata' @@ -119,6 +129,7 @@ # compatible_service_versions: # - '0.1.0' # - '0.1.1' +# - '0.1.2' # - title: 'Catalan; Valencian' # description: '' # url: 'https://github.com/tesseract-ocr/tessdata/raw/4.1.0/cat.traineddata' @@ -130,6 +141,7 @@ # compatible_service_versions: # - '0.1.0' # - '0.1.1' +# - '0.1.2' # - title: 'Cebuano' # description: '' # url: 'https://github.com/tesseract-ocr/tessdata/raw/4.1.0/ceb.traineddata' @@ -141,6 +153,7 @@ # compatible_service_versions: # - '0.1.0' # - '0.1.1' +# - '0.1.2' # - title: 'Czech' # description: '' # url: 'https://github.com/tesseract-ocr/tessdata/raw/4.1.0/ces.traineddata' @@ -152,6 +165,7 @@ # compatible_service_versions: # - '0.1.0' # - '0.1.1' +# - '0.1.2' # - title: 'Chinese - Simplified' # description: '' # url: 'https://github.com/tesseract-ocr/tessdata/raw/4.1.0/chi_sim.traineddata' @@ -163,6 +177,7 @@ # compatible_service_versions: # - '0.1.0' # - '0.1.1' +# - '0.1.2' - title: 'Chinese - Traditional' description: '' url: 'https://github.com/tesseract-ocr/tessdata/raw/4.1.0/chi_tra.traineddata' @@ -174,6 +189,7 @@ compatible_service_versions: - '0.1.0' - '0.1.1' + - '0.1.2' # - title: 'Cherokee' # description: '' # url: 'https://github.com/tesseract-ocr/tessdata/raw/4.1.0/chr.traineddata' @@ -185,6 +201,7 @@ # compatible_service_versions: # - '0.1.0' # - '0.1.1' +# - '0.1.2' # - title: 'Welsh' # description: '' # url: 'https://github.com/tesseract-ocr/tessdata/raw/4.1.0/cym.traineddata' @@ -196,6 +213,7 @@ # compatible_service_versions: # - '0.1.0' # - '0.1.1' +# - '0.1.2' - title: 'Danish' description: '' url: 'https://github.com/tesseract-ocr/tessdata/raw/4.1.0/dan.traineddata' @@ -207,6 +225,7 @@ compatible_service_versions: - '0.1.0' - '0.1.1' + - '0.1.2' - title: 'German' description: '' url: 'https://github.com/tesseract-ocr/tessdata/raw/4.1.0/deu.traineddata' @@ -218,6 +237,7 @@ compatible_service_versions: - '0.1.0' - '0.1.1' + - '0.1.2' # - title: 'Dzongkha' # description: '' # url: 'https://github.com/tesseract-ocr/tessdata/raw/4.1.0/dzo.traineddata' @@ -229,6 +249,7 @@ # compatible_service_versions: # - '0.1.0' # - '0.1.1' +# - '0.1.2' - title: 'Greek, Modern (1453-)' description: '' url: 'https://github.com/tesseract-ocr/tessdata/raw/4.1.0/ell.traineddata' @@ -240,6 +261,7 @@ compatible_service_versions: - '0.1.0' - '0.1.1' + - '0.1.2' - title: 'English' description: '' url: 'https://github.com/tesseract-ocr/tessdata/raw/4.1.0/eng.traineddata' @@ -251,6 +273,7 @@ compatible_service_versions: - '0.1.0' - '0.1.1' + - '0.1.2' - title: 'English, Middle (1100-1500)' description: '' url: 'https://github.com/tesseract-ocr/tessdata/raw/4.1.0/enm.traineddata' @@ -262,6 +285,7 @@ compatible_service_versions: - '0.1.0' - '0.1.1' + - '0.1.2' # - title: 'Esperanto' # description: '' # url: 'https://github.com/tesseract-ocr/tessdata/raw/4.1.0/epo.traineddata' @@ -273,6 +297,7 @@ # compatible_service_versions: # - '0.1.0' # - '0.1.1' +# - '0.1.2' # - title: 'Estonian' # description: '' # url: 'https://github.com/tesseract-ocr/tessdata/raw/4.1.0/est.traineddata' @@ -284,6 +309,7 @@ # compatible_service_versions: # - '0.1.0' # - '0.1.1' +# - '0.1.2' # - title: 'Basque' # description: '' # url: 'https://github.com/tesseract-ocr/tessdata/raw/4.1.0/eus.traineddata' @@ -295,6 +321,7 @@ # compatible_service_versions: # - '0.1.0' # - '0.1.1' +# - '0.1.2' # - title: 'Persian' # description: '' # url: 'https://github.com/tesseract-ocr/tessdata/raw/4.1.0/fas.traineddata' @@ -306,6 +333,7 @@ # compatible_service_versions: # - '0.1.0' # - '0.1.1' +# - '0.1.2' # - title: 'Finnish' # description: '' # url: 'https://github.com/tesseract-ocr/tessdata/raw/4.1.0/fin.traineddata' @@ -317,6 +345,7 @@ # compatible_service_versions: # - '0.1.0' # - '0.1.1' +# - '0.1.2' - title: 'French' description: '' url: 'https://github.com/tesseract-ocr/tessdata/raw/4.1.0/fra.traineddata' @@ -328,6 +357,7 @@ compatible_service_versions: - '0.1.0' - '0.1.1' + - '0.1.2' - title: 'German Fraktur' description: '' url: 'https://github.com/tesseract-ocr/tessdata/raw/4.1.0/frk.traineddata' @@ -339,6 +369,7 @@ compatible_service_versions: - '0.1.0' - '0.1.1' + - '0.1.2' - title: 'French, Middle (ca. 1400-1600)' description: '' url: 'https://github.com/tesseract-ocr/tessdata/raw/4.1.0/frm.traineddata' @@ -350,6 +381,7 @@ compatible_service_versions: - '0.1.0' - '0.1.1' + - '0.1.2' # - title: 'Irish' # description: '' # url: 'https://github.com/tesseract-ocr/tessdata/raw/4.1.0/gle.traineddata' @@ -361,6 +393,7 @@ # compatible_service_versions: # - '0.1.0' # - '0.1.1' +# - '0.1.2' # - title: 'Galician' # description: '' # url: 'https://github.com/tesseract-ocr/tessdata/raw/4.1.0/glg.traineddata' @@ -372,6 +405,7 @@ # compatible_service_versions: # - '0.1.0' # - '0.1.1' +# - '0.1.2' - title: 'Greek, Ancient (-1453)' description: '' url: 'https://github.com/tesseract-ocr/tessdata/raw/4.1.0/grc.traineddata' @@ -383,6 +417,7 @@ compatible_service_versions: - '0.1.0' - '0.1.1' + - '0.1.2' # - title: 'Gujarati' # description: '' # url: 'https://github.com/tesseract-ocr/tessdata/raw/4.1.0/guj.traineddata' @@ -394,6 +429,7 @@ # compatible_service_versions: # - '0.1.0' # - '0.1.1' +# - '0.1.2' # - title: 'Haitian; Haitian Creole' # description: '' # url: 'https://github.com/tesseract-ocr/tessdata/raw/4.1.0/hat.traineddata' @@ -405,6 +441,7 @@ # compatible_service_versions: # - '0.1.0' # - '0.1.1' +# - '0.1.2' # - title: 'Hebrew' # description: '' # url: 'https://github.com/tesseract-ocr/tessdata/raw/4.1.0/heb.traineddata' @@ -416,6 +453,7 @@ # compatible_service_versions: # - '0.1.0' # - '0.1.1' +# - '0.1.2' # - title: 'Hindi' # description: '' # url: 'https://github.com/tesseract-ocr/tessdata/raw/4.1.0/hin.traineddata' @@ -427,6 +465,7 @@ # compatible_service_versions: # - '0.1.0' # - '0.1.1' +# - '0.1.2' # - title: 'Croatian' # description: '' # url: 'https://github.com/tesseract-ocr/tessdata/raw/4.1.0/hrv.traineddata' @@ -438,6 +477,7 @@ # compatible_service_versions: # - '0.1.0' # - '0.1.1' +# - '0.1.2' # - title: 'Hungarian' # description: '' # url: 'https://github.com/tesseract-ocr/tessdata/raw/4.1.0/hun.traineddata' @@ -449,6 +489,7 @@ # compatible_service_versions: # - '0.1.0' # - '0.1.1' +# - '0.1.2' # - title: 'Inuktitut' # description: '' # url: 'https://github.com/tesseract-ocr/tessdata/raw/4.1.0/iku.traineddata' @@ -460,6 +501,7 @@ # compatible_service_versions: # - '0.1.0' # - '0.1.1' +# - '0.1.2' # - title: 'Indonesian' # description: '' # url: 'https://github.com/tesseract-ocr/tessdata/raw/4.1.0/ind.traineddata' @@ -471,6 +513,7 @@ # compatible_service_versions: # - '0.1.0' # - '0.1.1' +# - '0.1.2' # - title: 'Icelandic' # description: '' # url: 'https://github.com/tesseract-ocr/tessdata/raw/4.1.0/isl.traineddata' @@ -482,6 +525,7 @@ # compatible_service_versions: # - '0.1.0' # - '0.1.1' +# - '0.1.2' - title: 'Italian' description: '' url: 'https://github.com/tesseract-ocr/tessdata/raw/4.1.0/ita.traineddata' @@ -493,6 +537,7 @@ compatible_service_versions: - '0.1.0' - '0.1.1' + - '0.1.2' - title: 'Italian - Old' description: '' url: 'https://github.com/tesseract-ocr/tessdata/raw/4.1.0/ita_old.traineddata' @@ -504,6 +549,7 @@ compatible_service_versions: - '0.1.0' - '0.1.1' + - '0.1.2' # - title: 'Javanese' # description: '' # url: 'https://github.com/tesseract-ocr/tessdata/raw/4.1.0/jav.traineddata' @@ -515,6 +561,7 @@ # compatible_service_versions: # - '0.1.0' # - '0.1.1' +# - '0.1.2' # - title: 'Japanese' # description: '' # url: 'https://github.com/tesseract-ocr/tessdata/raw/4.1.0/jpn.traineddata' @@ -526,6 +573,7 @@ # compatible_service_versions: # - '0.1.0' # - '0.1.1' +# - '0.1.2' # - title: 'Kannada' # description: '' # url: 'https://github.com/tesseract-ocr/tessdata/raw/4.1.0/kan.traineddata' @@ -537,6 +585,7 @@ # compatible_service_versions: # - '0.1.0' # - '0.1.1' +# - '0.1.2' # - title: 'Georgian' # description: '' # url: 'https://github.com/tesseract-ocr/tessdata/raw/4.1.0/kat.traineddata' @@ -548,6 +597,7 @@ # compatible_service_versions: # - '0.1.0' # - '0.1.1' +# - '0.1.2' # - title: 'Georgian - Old' # description: '' # url: 'https://github.com/tesseract-ocr/tessdata/raw/4.1.0/kat_old.traineddata' @@ -559,6 +609,7 @@ # compatible_service_versions: # - '0.1.0' # - '0.1.1' +# - '0.1.2' # - title: 'Kazakh' # description: '' # url: 'https://github.com/tesseract-ocr/tessdata/raw/4.1.0/kaz.traineddata' @@ -570,6 +621,7 @@ # compatible_service_versions: # - '0.1.0' # - '0.1.1' +# - '0.1.2' # - title: 'Central Khmer' # description: '' # url: 'https://github.com/tesseract-ocr/tessdata/raw/4.1.0/khm.traineddata' @@ -581,6 +633,7 @@ # compatible_service_versions: # - '0.1.0' # - '0.1.1' +# - '0.1.2' # - title: 'Kirghiz; Kyrgyz' # description: '' # url: 'https://github.com/tesseract-ocr/tessdata/raw/4.1.0/kir.traineddata' @@ -592,6 +645,7 @@ # compatible_service_versions: # - '0.1.0' # - '0.1.1' +# - '0.1.2' # - title: 'Korean' # description: '' # url: 'https://github.com/tesseract-ocr/tessdata/raw/4.1.0/kor.traineddata' @@ -603,6 +657,7 @@ # compatible_service_versions: # - '0.1.0' # - '0.1.1' +# - '0.1.2' # - title: 'Kurdish' # description: '' # url: 'https://github.com/tesseract-ocr/tessdata/raw/4.1.0/kur.traineddata' @@ -614,6 +669,7 @@ # compatible_service_versions: # - '0.1.0' # - '0.1.1' +# - '0.1.2' # - title: 'Lao' # description: '' # url: 'https://github.com/tesseract-ocr/tessdata/raw/4.1.0/lao.traineddata' @@ -625,6 +681,7 @@ # compatible_service_versions: # - '0.1.0' # - '0.1.1' +# - '0.1.2' # - title: 'Latin' # description: '' # url: 'https://github.com/tesseract-ocr/tessdata/raw/4.1.0/lat.traineddata' @@ -636,6 +693,7 @@ # compatible_service_versions: # - '0.1.0' # - '0.1.1' +# - '0.1.2' # - title: 'Latvian' # description: '' # url: 'https://github.com/tesseract-ocr/tessdata/raw/4.1.0/lav.traineddata' @@ -647,6 +705,7 @@ # compatible_service_versions: # - '0.1.0' # - '0.1.1' +# - '0.1.2' # - title: 'Lithuanian' # description: '' # url: 'https://github.com/tesseract-ocr/tessdata/raw/4.1.0/lit.traineddata' @@ -658,6 +717,7 @@ # compatible_service_versions: # - '0.1.0' # - '0.1.1' +# - '0.1.2' # - title: 'Malayalam' # description: '' # url: 'https://github.com/tesseract-ocr/tessdata/raw/4.1.0/mal.traineddata' @@ -669,6 +729,7 @@ # compatible_service_versions: # - '0.1.0' # - '0.1.1' +# - '0.1.2' # - title: 'Marathi' # description: '' # url: 'https://github.com/tesseract-ocr/tessdata/raw/4.1.0/mar.traineddata' @@ -680,6 +741,7 @@ # compatible_service_versions: # - '0.1.0' # - '0.1.1' +# - '0.1.2' # - title: 'Macedonian' # description: '' # url: 'https://github.com/tesseract-ocr/tessdata/raw/4.1.0/mkd.traineddata' @@ -691,6 +753,7 @@ # compatible_service_versions: # - '0.1.0' # - '0.1.1' +# - '0.1.2' # - title: 'Maltese' # description: '' # url: 'https://github.com/tesseract-ocr/tessdata/raw/4.1.0/mlt.traineddata' @@ -702,6 +765,7 @@ # compatible_service_versions: # - '0.1.0' # - '0.1.1' +# - '0.1.2' # - title: 'Malay' # description: '' # url: 'https://github.com/tesseract-ocr/tessdata/raw/4.1.0/msa.traineddata' @@ -713,6 +777,7 @@ # compatible_service_versions: # - '0.1.0' # - '0.1.1' +# - '0.1.2' # - title: 'Burmese' # description: '' # url: 'https://github.com/tesseract-ocr/tessdata/raw/4.1.0/mya.traineddata' @@ -724,6 +789,7 @@ # compatible_service_versions: # - '0.1.0' # - '0.1.1' +# - '0.1.2' # - title: 'Nepali' # description: '' # url: 'https://github.com/tesseract-ocr/tessdata/raw/4.1.0/nep.traineddata' @@ -735,6 +801,7 @@ # compatible_service_versions: # - '0.1.0' # - '0.1.1' +# - '0.1.2' # - title: 'Dutch; Flemish' # description: '' # url: 'https://github.com/tesseract-ocr/tessdata/raw/4.1.0/nld.traineddata' @@ -746,6 +813,7 @@ # compatible_service_versions: # - '0.1.0' # - '0.1.1' +# - '0.1.2' # - title: 'Norwegian' # description: '' # url: 'https://github.com/tesseract-ocr/tessdata/raw/4.1.0/nor.traineddata' @@ -757,6 +825,7 @@ # compatible_service_versions: # - '0.1.0' # - '0.1.1' +# - '0.1.2' # - title: 'Oriya' # description: '' # url: 'https://github.com/tesseract-ocr/tessdata/raw/4.1.0/ori.traineddata' @@ -768,6 +837,7 @@ # compatible_service_versions: # - '0.1.0' # - '0.1.1' +# - '0.1.2' # - title: 'Panjabi; Punjabi' # description: '' # url: 'https://github.com/tesseract-ocr/tessdata/raw/4.1.0/pan.traineddata' @@ -779,6 +849,7 @@ # compatible_service_versions: # - '0.1.0' # - '0.1.1' +# - '0.1.2' # - title: 'Polish' # description: '' # url: 'https://github.com/tesseract-ocr/tessdata/raw/4.1.0/pol.traineddata' @@ -790,6 +861,7 @@ # compatible_service_versions: # - '0.1.0' # - '0.1.1' +# - '0.1.2' - title: 'Portuguese' description: '' url: 'https://github.com/tesseract-ocr/tessdata/raw/4.1.0/por.traineddata' @@ -801,6 +873,7 @@ compatible_service_versions: - '0.1.0' - '0.1.1' + - '0.1.2' # - title: 'Pushto; Pashto' # description: '' # url: 'https://github.com/tesseract-ocr/tessdata/raw/4.1.0/pus.traineddata' @@ -812,6 +885,7 @@ # compatible_service_versions: # - '0.1.0' # - '0.1.1' +# - '0.1.2' # - title: 'Romanian; Moldavian; Moldovan' # description: '' # url: 'https://github.com/tesseract-ocr/tessdata/raw/4.1.0/ron.traineddata' @@ -823,6 +897,7 @@ # compatible_service_versions: # - '0.1.0' # - '0.1.1' +# - '0.1.2' - title: 'Russian' description: '' url: 'https://github.com/tesseract-ocr/tessdata/raw/4.1.0/rus.traineddata' @@ -834,6 +909,7 @@ compatible_service_versions: - '0.1.0' - '0.1.1' + - '0.1.2' # - title: 'Sanskrit' # description: '' # url: 'https://github.com/tesseract-ocr/tessdata/raw/4.1.0/san.traineddata' @@ -845,6 +921,7 @@ # compatible_service_versions: # - '0.1.0' # - '0.1.1' +# - '0.1.2' # - title: 'Sinhala; Sinhalese' # description: '' # url: 'https://github.com/tesseract-ocr/tessdata/raw/4.1.0/sin.traineddata' @@ -856,6 +933,7 @@ # compatible_service_versions: # - '0.1.0' # - '0.1.1' +# - '0.1.2' # - title: 'Slovak' # description: '' # url: 'https://github.com/tesseract-ocr/tessdata/raw/4.1.0/slk.traineddata' @@ -867,6 +945,7 @@ # compatible_service_versions: # - '0.1.0' # - '0.1.1' +# - '0.1.2' # - title: 'Slovenian' # description: '' # url: 'https://github.com/tesseract-ocr/tessdata/raw/4.1.0/slv.traineddata' @@ -878,6 +957,7 @@ # compatible_service_versions: # - '0.1.0' # - '0.1.1' +# - '0.1.2' - title: 'Spanish; Castilian' description: '' url: 'https://github.com/tesseract-ocr/tessdata/raw/4.1.0/spa.traineddata' @@ -889,6 +969,7 @@ compatible_service_versions: - '0.1.0' - '0.1.1' + - '0.1.2' - title: 'Spanish; Castilian - Old' description: '' url: 'https://github.com/tesseract-ocr/tessdata/raw/4.1.0/spa_old.traineddata' @@ -900,6 +981,7 @@ compatible_service_versions: - '0.1.0' - '0.1.1' + - '0.1.2' # - title: 'Albanian' # description: '' # url: 'https://github.com/tesseract-ocr/tessdata/raw/4.1.0/sqi.traineddata' @@ -911,6 +993,7 @@ # compatible_service_versions: # - '0.1.0' # - '0.1.1' +# - '0.1.2' # - title: 'Serbian' # description: '' # url: 'https://github.com/tesseract-ocr/tessdata/raw/4.1.0/srp.traineddata' @@ -922,6 +1005,7 @@ # compatible_service_versions: # - '0.1.0' # - '0.1.1' +# - '0.1.2' # - title: 'Serbian - Latin' # description: '' # url: 'https://github.com/tesseract-ocr/tessdata/raw/4.1.0/srp_latn.traineddata' @@ -933,6 +1017,7 @@ # compatible_service_versions: # - '0.1.0' # - '0.1.1' +# - '0.1.2' # - title: 'Swahili' # description: '' # url: 'https://github.com/tesseract-ocr/tessdata/raw/4.1.0/swa.traineddata' @@ -944,6 +1029,7 @@ # compatible_service_versions: # - '0.1.0' # - '0.1.1' +# - '0.1.2' # - title: 'Swedish' # description: '' # url: 'https://github.com/tesseract-ocr/tessdata/raw/4.1.0/swe.traineddata' @@ -955,6 +1041,7 @@ # compatible_service_versions: # - '0.1.0' # - '0.1.1' +# - '0.1.2' # - title: 'Syriac' # description: '' # url: 'https://github.com/tesseract-ocr/tessdata/raw/4.1.0/syr.traineddata' @@ -966,6 +1053,7 @@ # compatible_service_versions: # - '0.1.0' # - '0.1.1' +# - '0.1.2' # - title: 'Tamil' # description: '' # url: 'https://github.com/tesseract-ocr/tessdata/raw/4.1.0/tam.traineddata' @@ -977,6 +1065,7 @@ # compatible_service_versions: # - '0.1.0' # - '0.1.1' +# - '0.1.2' # - title: 'Telugu' # description: '' # url: 'https://github.com/tesseract-ocr/tessdata/raw/4.1.0/tel.traineddata' @@ -988,6 +1077,7 @@ # compatible_service_versions: # - '0.1.0' # - '0.1.1' +# - '0.1.2' # - title: 'Tajik' # description: '' # url: 'https://github.com/tesseract-ocr/tessdata/raw/4.1.0/tgk.traineddata' @@ -999,6 +1089,7 @@ # compatible_service_versions: # - '0.1.0' # - '0.1.1' +# - '0.1.2' # - title: 'Tagalog' # description: '' # url: 'https://github.com/tesseract-ocr/tessdata/raw/4.1.0/tgl.traineddata' @@ -1010,6 +1101,7 @@ # compatible_service_versions: # - '0.1.0' # - '0.1.1' +# - '0.1.2' # - title: 'Thai' # description: '' # url: 'https://github.com/tesseract-ocr/tessdata/raw/4.1.0/tha.traineddata' @@ -1021,6 +1113,7 @@ # compatible_service_versions: # - '0.1.0' # - '0.1.1' +# - '0.1.2' # - title: 'Tigrinya' # description: '' # url: 'https://github.com/tesseract-ocr/tessdata/raw/4.1.0/tir.traineddata' @@ -1032,6 +1125,7 @@ # compatible_service_versions: # - '0.1.0' # - '0.1.1' +# - '0.1.2' # - title: 'Turkish' # description: '' # url: 'https://github.com/tesseract-ocr/tessdata/raw/4.1.0/tur.traineddata' @@ -1043,6 +1137,7 @@ # compatible_service_versions: # - '0.1.0' # - '0.1.1' +# - '0.1.2' # - title: 'Uighur; Uyghur' # description: '' # url: 'https://github.com/tesseract-ocr/tessdata/raw/4.1.0/uig.traineddata' @@ -1054,6 +1149,7 @@ # compatible_service_versions: # - '0.1.0' # - '0.1.1' +# - '0.1.2' # - title: 'Ukrainian' # description: '' # url: 'https://github.com/tesseract-ocr/tessdata/raw/4.1.0/ukr.traineddata' @@ -1065,6 +1161,7 @@ # compatible_service_versions: # - '0.1.0' # - '0.1.1' +# - '0.1.2' # - title: 'Urdu' # description: '' # url: 'https://github.com/tesseract-ocr/tessdata/raw/4.1.0/urd.traineddata' @@ -1076,6 +1173,7 @@ # compatible_service_versions: # - '0.1.0' # - '0.1.1' +# - '0.1.2' # - title: 'Uzbek' # description: '' # url: 'https://github.com/tesseract-ocr/tessdata/raw/4.1.0/uzb.traineddata' @@ -1087,6 +1185,7 @@ # compatible_service_versions: # - '0.1.0' # - '0.1.1' +# - '0.1.2' # - title: 'Uzbek - Cyrillic' # description: '' # url: 'https://github.com/tesseract-ocr/tessdata/raw/4.1.0/uzb_cyrl.traineddata' @@ -1098,6 +1197,7 @@ # compatible_service_versions: # - '0.1.0' # - '0.1.1' +# - '0.1.2' # - title: 'Vietnamese' # description: '' # url: 'https://github.com/tesseract-ocr/tessdata/raw/4.1.0/vie.traineddata' @@ -1109,6 +1209,7 @@ # compatible_service_versions: # - '0.1.0' # - '0.1.1' +# - '0.1.2' # - title: 'Yiddish' # description: '' # url: 'https://github.com/tesseract-ocr/tessdata/raw/4.1.0/yid.traineddata' @@ -1120,3 +1221,4 @@ # compatible_service_versions: # - '0.1.0' # - '0.1.1' +# - '0.1.2' diff --git a/app/__init__.py b/app/__init__.py index 41b3eeb1..41a8074d 100644 --- a/app/__init__.py +++ b/app/__init__.py @@ -13,7 +13,6 @@ from flask_paranoid import Paranoid from flask_socketio import SocketIO from flask_sqlalchemy import SQLAlchemy from flask_hashids import Hashids -from werkzeug.exceptions import HTTPException apifairy = APIFairy() diff --git a/app/corpora/cqi_over_sio/__init__.py b/app/corpora/cqi_over_sio/__init__.py index 6c093a9e..83ccf695 100644 --- a/app/corpora/cqi_over_sio/__init__.py +++ b/app/corpora/cqi_over_sio/__init__.py @@ -1,13 +1,14 @@ from cqi import CQiClient from cqi.errors import CQiException from cqi.status import CQiStatus -from flask import session +from docker.models.containers import Container +from flask import current_app, session from flask_login import current_user from flask_socketio import Namespace from inspect import signature from threading import Lock -from typing import Callable, Dict, List -from app import db, hashids, socketio +from typing import Callable, Dict, List, Optional +from app import db, docker_client, hashids, socketio from app.decorators import socketio_login_required from app.models import Corpus, CorpusStatus from . import extensions @@ -92,8 +93,8 @@ class CQiNamespace(Namespace): @socketio_login_required def on_init(self, db_corpus_hashid: str): - db_corpus_id = hashids.decode(db_corpus_hashid) - db_corpus = Corpus.query.get(db_corpus_id) + db_corpus_id: int = hashids.decode(db_corpus_hashid) + db_corpus: Optional[Corpus] = Corpus.query.get(db_corpus_id) if db_corpus is None: return {'code': 404, 'msg': 'Not Found'} if not (db_corpus.user == current_user @@ -112,7 +113,7 @@ class CQiNamespace(Namespace): db.session.commit() db_corpus.num_analysis_sessions = Corpus.num_analysis_sessions + 1 db.session.commit() - retry_counter = 20 + retry_counter: int = 20 while db_corpus.status != CorpusStatus.RUNNING_ANALYSIS_SESSION: if retry_counter == 0: db_corpus.num_analysis_sessions = Corpus.num_analysis_sessions - 1 @@ -121,11 +122,16 @@ class CQiNamespace(Namespace): socketio.sleep(3) retry_counter -= 1 db.session.refresh(db_corpus) - cqi_client = CQiClient(f'cqpserver_{db_corpus_id}', timeout=float('inf')) - session['cqi_over_sio'] = {} - session['cqi_over_sio']['cqi_client'] = cqi_client - session['cqi_over_sio']['cqi_client_lock'] = Lock() - session['cqi_over_sio']['db_corpus_id'] = db_corpus_id + # cqi_client: CQiClient = CQiClient(f'cqpserver_{db_corpus_id}') + cqpserver_container_name: str = f'cqpserver_{db_corpus_id}' + cqpserver_container: Container = docker_client.containers.get(cqpserver_container_name) + cqpserver_host: str = cqpserver_container.attrs['NetworkSettings']['Networks'][current_app.config['NOPAQUE_DOCKER_NETWORK_NAME']]['IPAddress'] + cqi_client: CQiClient = CQiClient(cqpserver_host) + session['cqi_over_sio'] = { + 'cqi_client': cqi_client, + 'cqi_client_lock': Lock(), + 'db_corpus_id': db_corpus_id + } return {'code': 200, 'msg': 'OK'} @socketio_login_required @@ -193,7 +199,8 @@ class CQiNamespace(Namespace): except (BrokenPipeError, CQiException): pass cqi_client_lock.release() - db_corpus = Corpus.query.get(db_corpus_id) - if db_corpus is not None: - db_corpus.num_analysis_sessions = Corpus.num_analysis_sessions - 1 - db.session.commit() + db_corpus: Optional[Corpus] = Corpus.query.get(db_corpus_id) + if db_corpus is None: + return + db_corpus.num_analysis_sessions = Corpus.num_analysis_sessions - 1 + db.session.commit() diff --git a/app/corpora/cqi_over_sio/extensions.py b/app/corpora/cqi_over_sio/extensions.py index 70ee5d97..6748b963 100644 --- a/app/corpora/cqi_over_sio/extensions.py +++ b/app/corpora/cqi_over_sio/extensions.py @@ -1,6 +1,7 @@ from collections import Counter from cqi import CQiClient from cqi.models.corpora import Corpus as CQiCorpus +from cqi.models.subcorpora import Subcorpus as CQiSubcorpus from cqi.models.attributes import ( PositionalAttribute as CQiPositionalAttribute, StructuralAttribute as CQiStructuralAttribute @@ -40,161 +41,132 @@ def ext_corpus_update_db(corpus: str) -> CQiStatusOk: def ext_corpus_static_data(corpus: str) -> Dict: db_corpus_id: int = session['cqi_over_sio']['db_corpus_id'] db_corpus: Corpus = Corpus.query.get(db_corpus_id) - cache_file_path: str = os.path.join(db_corpus.path, 'cwb', 'static.json.gz') - if os.path.exists(cache_file_path): - with open(cache_file_path, 'rb') as f: + + static_data_file_path: str = os.path.join(db_corpus.path, 'cwb', 'static.json.gz') + if os.path.exists(static_data_file_path): + with open(static_data_file_path, 'rb') as f: return f.read() + cqi_client: CQiClient = session['cqi_over_sio']['cqi_client'] cqi_corpus: CQiCorpus = cqi_client.corpora.get(corpus) - cqi_p_attrs: Dict[str, CQiPositionalAttribute] = { - p_attr.name: p_attr - for p_attr in cqi_corpus.positional_attributes.list() - } - cqi_s_attrs: Dict[str, CQiStructuralAttribute] = { - s_attr.name: s_attr - for s_attr in cqi_corpus.structural_attributes.list() - } - static_corpus_data = { + cqi_p_attrs: List[CQiPositionalAttribute] = cqi_corpus.positional_attributes.list() + cqi_s_attrs: List[CQiStructuralAttribute] = cqi_corpus.structural_attributes.list() + + static_data = { 'corpus': { 'bounds': [0, cqi_corpus.size - 1], - 'counts': { - 'token': cqi_corpus.size - }, 'freqs': {} }, 'p_attrs': {}, 's_attrs': {}, 'values': {'p_attrs': {}, 's_attrs': {}} } - for p_attr in cqi_p_attrs.values(): - static_corpus_data['corpus']['freqs'][p_attr.name] = {} - chunk_size = 10000 - p_attr_id_list = list(range(p_attr.lexicon_size)) - chunks = [p_attr_id_list[i:i+chunk_size] for i in range(0, len(p_attr_id_list), chunk_size)] + + for p_attr in cqi_p_attrs: + print(f'corpus.freqs.{p_attr.name}') + static_data['corpus']['freqs'][p_attr.name] = [] + p_attr_id_list: List[int] = list(range(p_attr.lexicon_size)) + static_data['corpus']['freqs'][p_attr.name].extend(p_attr.freqs_by_ids(p_attr_id_list)) del p_attr_id_list - for chunk in chunks: - # print(f'corpus.freqs.{p_attr.name}: {chunk[0]} - {chunk[-1]}') - static_corpus_data['corpus']['freqs'][p_attr.name].update( - dict(zip(chunk, p_attr.freqs_by_ids(chunk))) - ) - del chunks - static_corpus_data['p_attrs'][p_attr.name] = {} - cpos_list = list(range(cqi_corpus.size)) - chunks = [cpos_list[i:i+chunk_size] for i in range(0, len(cpos_list), chunk_size)] + + print(f'p_attrs.{p_attr.name}') + static_data['p_attrs'][p_attr.name] = [] + cpos_list: List[int] = list(range(cqi_corpus.size)) + static_data['p_attrs'][p_attr.name].extend(p_attr.ids_by_cpos(cpos_list)) del cpos_list - for chunk in chunks: - # print(f'p_attrs.{p_attr.name}: {chunk[0]} - {chunk[-1]}') - static_corpus_data['p_attrs'][p_attr.name].update( - dict(zip(chunk, p_attr.ids_by_cpos(chunk))) - ) - del chunks - static_corpus_data['values']['p_attrs'][p_attr.name] = {} - p_attr_id_list = list(range(p_attr.lexicon_size)) - chunks = [p_attr_id_list[i:i+chunk_size] for i in range(0, len(p_attr_id_list), chunk_size)] + + print(f'values.p_attrs.{p_attr.name}') + static_data['values']['p_attrs'][p_attr.name] = [] + p_attr_id_list: List[int] = list(range(p_attr.lexicon_size)) + static_data['values']['p_attrs'][p_attr.name].extend(p_attr.values_by_ids(p_attr_id_list)) del p_attr_id_list - for chunk in chunks: - # print(f'values.p_attrs.{p_attr.name}: {chunk[0]} - {chunk[-1]}') - static_corpus_data['values']['p_attrs'][p_attr.name].update( - dict(zip(chunk, p_attr.values_by_ids(chunk))) - ) - del chunks - for s_attr in cqi_s_attrs.values(): + + for s_attr in cqi_s_attrs: if s_attr.has_values: continue - static_corpus_data['corpus']['counts'][s_attr.name] = s_attr.size - static_corpus_data['s_attrs'][s_attr.name] = {'lexicon': {}, 'values': None} - static_corpus_data['values']['s_attrs'][s_attr.name] = {} - ########################################################################## - # A faster way to get cpos boundaries for smaller s_attrs # - ########################################################################## - # if s_attr.name in ['s', 'ent']: - # cqi_corpus.query('Last', f'<{s_attr.name}> []* ;') - # cqi_subcorpus = cqi_corpus.subcorpora.get('Last') - # first_match = 0 - # last_match = cqi_subcorpus.size - 1 - # match_boundaries = zip( - # range(first_match, last_match + 1), - # cqi_subcorpus.dump(cqi_subcorpus.fields['match'], first_match, last_match), - # cqi_subcorpus.dump(cqi_subcorpus.fields['matchend'], first_match, last_match) - # ) - # for id, lbound, rbound in match_boundaries: - # static_corpus_data['s_attrs'][s_attr.name]['lexicon'][id] = {} - # static_corpus_data['s_attrs'][s_attr.name]['lexicon'][id]['bounds'] = [lbound, rbound] - # static_corpus_data['s_attrs'][s_attr.name]['lexicon'][id]['counts'] = {} - # static_corpus_data['s_attrs'][s_attr.name]['lexicon'][id]['counts']['token'] = rbound - lbound + 1 - # cqi_subcorpus.drop() + + static_data['s_attrs'][s_attr.name] = {'lexicon': [], 'values': None} + + if s_attr.name in ['s', 'ent']: + ############################################################## + # A faster way to get cpos boundaries for smaller s_attrs # + # Note: Needs more testing, don't use it in production # + ############################################################## + cqi_corpus.query('Last', f'<{s_attr.name}> []* ;') + cqi_subcorpus: CQiSubcorpus = cqi_corpus.subcorpora.get('Last') + first_match: int = 0 + last_match: int = cqi_subcorpus.size - 1 + match_boundaries = zip( + range(first_match, last_match + 1), + cqi_subcorpus.dump( + cqi_subcorpus.fields['match'], + first_match, + last_match + ), + cqi_subcorpus.dump( + cqi_subcorpus.fields['matchend'], + first_match, + last_match + ) + ) + cqi_subcorpus.drop() + del cqi_subcorpus, first_match, last_match + for id, lbound, rbound in match_boundaries: + static_data['s_attrs'][s_attr.name]['lexicon'].append({}) + print(f's_attrs.{s_attr.name}.lexicon.{id}.bounds') + static_data['s_attrs'][s_attr.name]['lexicon'][id]['bounds'] = [lbound, rbound] + del match_boundaries + + if s_attr.name != 'text': + continue + for id in range(0, s_attr.size): - # print(f's_attrs.{s_attr.name}.lexicon.{id}') - static_corpus_data['s_attrs'][s_attr.name]['lexicon'][id] = { - 'bounds': None, - 'counts': None, - 'freqs': None - } - if s_attr.name != 'text': - continue + static_data['s_attrs'][s_attr.name]['lexicon'].append({}) + # This is a very slow operation, thats why we only use it for + # the text attribute lbound, rbound = s_attr.cpos_by_id(id) - # print(f's_attrs.{s_attr.name}.lexicon.{id}.bounds') - static_corpus_data['s_attrs'][s_attr.name]['lexicon'][id]['bounds'] = [lbound, rbound] - # print(f's_attrs.{s_attr.name}.lexicon.{id}.counts') - static_corpus_data['s_attrs'][s_attr.name]['lexicon'][id]['counts'] = {} - static_corpus_data['s_attrs'][s_attr.name]['lexicon'][id]['counts']['token'] = rbound - lbound + 1 - cpos_list = list(range(lbound, rbound + 1)) - chunks = [cpos_list[i:i+chunk_size] for i in range(0, len(cpos_list), chunk_size)] - del cpos_list - ent_ids = set() - for chunk in chunks: - # print(f'Gather ent_ids from cpos: {chunk[0]} - {chunk[-1]}') - ent_ids.update({x for x in cqi_s_attrs['ent'].ids_by_cpos(chunk) if x != -1}) - static_corpus_data['s_attrs'][s_attr.name]['lexicon'][id]['counts']['ent'] = len(ent_ids) - del ent_ids - s_ids = set() - for chunk in chunks: - # print(f'Gather s_ids from cpos: {chunk[0]} - {chunk[-1]}') - s_ids.update({x for x in cqi_s_attrs['s'].ids_by_cpos(chunk) if x != -1}) - static_corpus_data['s_attrs'][s_attr.name]['lexicon'][id]['counts']['s'] = len(s_ids) - del s_ids - # print(f's_attrs.{s_attr.name}.lexicon.{id}.freqs') - static_corpus_data['s_attrs'][s_attr.name]['lexicon'][id]['freqs'] = {} - for p_attr in cqi_p_attrs.values(): - p_attr_ids = [] - for chunk in chunks: - # print(f'Gather p_attr_ids from cpos: {chunk[0]} - {chunk[-1]}') - p_attr_ids.extend(p_attr.ids_by_cpos(chunk)) - static_corpus_data['s_attrs'][s_attr.name]['lexicon'][id]['freqs'][p_attr.name] = dict(Counter(p_attr_ids)) + print(f's_attrs.{s_attr.name}.lexicon.{id}.bounds') + static_data['s_attrs'][s_attr.name]['lexicon'][id]['bounds'] = [lbound, rbound] + static_data['s_attrs'][s_attr.name]['lexicon'][id]['freqs'] = {} + cpos_list: List[int] = list(range(lbound, rbound + 1)) + for p_attr in cqi_p_attrs: + p_attr_ids: List[int] = [] + p_attr_ids.extend(p_attr.ids_by_cpos(cpos_list)) + print(f's_attrs.{s_attr.name}.lexicon.{id}.freqs.{p_attr.name}') + static_data['s_attrs'][s_attr.name]['lexicon'][id]['freqs'][p_attr.name] = dict(Counter(p_attr_ids)) del p_attr_ids - del chunks - sub_s_attrs = cqi_corpus.structural_attributes.list(filters={'part_of': s_attr}) - s_attr_value_names: List[str] = [ + del cpos_list + + sub_s_attrs: List[CQiStructuralAttribute] = cqi_corpus.structural_attributes.list(filters={'part_of': s_attr}) + print(f's_attrs.{s_attr.name}.values') + static_data['s_attrs'][s_attr.name]['values'] = [ sub_s_attr.name[(len(s_attr.name) + 1):] for sub_s_attr in sub_s_attrs ] - s_attr_id_list = list(range(s_attr.size)) - chunks = [s_attr_id_list[i:i+chunk_size] for i in range(0, len(s_attr_id_list), chunk_size)] - del s_attr_id_list - sub_s_attr_values = [] + s_attr_id_list: List[int] = list(range(s_attr.size)) + sub_s_attr_values: List[str] = [] for sub_s_attr in sub_s_attrs: tmp = [] - for chunk in chunks: - tmp.extend(sub_s_attr.values_by_ids(chunk)) + tmp.extend(sub_s_attr.values_by_ids(s_attr_id_list)) sub_s_attr_values.append(tmp) del tmp - del chunks - # print(f's_attrs.{s_attr.name}.values') - static_corpus_data['s_attrs'][s_attr.name]['values'] = s_attr_value_names - # print(f'values.s_attrs.{s_attr.name}') - static_corpus_data['values']['s_attrs'][s_attr.name] = { - s_attr_id: { - s_attr_value_name: sub_s_attr_values[s_attr_value_name_idx][s_attr_id_idx] + del s_attr_id_list + print(f'values.s_attrs.{s_attr.name}') + static_data['values']['s_attrs'][s_attr.name] = [ + { + s_attr_value_name: sub_s_attr_values[s_attr_value_name_idx][s_attr_id] for s_attr_value_name_idx, s_attr_value_name in enumerate( - static_corpus_data['s_attrs'][s_attr.name]['values'] + static_data['s_attrs'][s_attr.name]['values'] ) - } for s_attr_id_idx, s_attr_id in enumerate(range(0, s_attr.size)) - } + } for s_attr_id in range(0, s_attr.size) + ] del sub_s_attr_values - with gzip.open(cache_file_path, 'wt') as f: - json.dump(static_corpus_data, f) - del static_corpus_data - with open(cache_file_path, 'rb') as f: + print('Saving static data to file') + with gzip.open(static_data_file_path, 'wt') as f: + json.dump(static_data, f) + del static_data + print('Sending static data to client') + with open(static_data_file_path, 'rb') as f: return f.read() diff --git a/app/corpora/cqi_over_sio/utils.py b/app/corpora/cqi_over_sio/utils.py index 121c3233..a69d4809 100644 --- a/app/corpora/cqi_over_sio/utils.py +++ b/app/corpora/cqi_over_sio/utils.py @@ -1,46 +1,44 @@ -from cqi.models.corpora import Corpus -from cqi.models.subcorpora import Subcorpus +from cqi.models.corpora import Corpus as CQiCorpus +from cqi.models.subcorpora import Subcorpus as CQiSubcorpus from typing import Dict, List -from app.models import Corpus -def lookups_by_cpos(corpus: Corpus, cpos_list: List[int]) -> Dict: +def lookups_by_cpos(corpus: CQiCorpus, cpos_list: List[int]) -> Dict: lookups = {} lookups['cpos_lookup'] = {cpos: {} for cpos in cpos_list} for attr in corpus.positional_attributes.list(): - cpos_attr_values = attr.values_by_cpos(cpos_list) + cpos_attr_values: List[str] = attr.values_by_cpos(cpos_list) for i, cpos in enumerate(cpos_list): - lookups['cpos_lookup'][cpos][attr.attrs['name']] = \ - cpos_attr_values[i] + lookups['cpos_lookup'][cpos][attr.name] = cpos_attr_values[i] for attr in corpus.structural_attributes.list(): # We only want to iterate over non subattributes, identifiable by - # attr.attrs['has_values'] == False - if attr.attrs['has_values']: + # attr.has_values == False + if attr.has_values: continue - cpos_attr_ids = attr.ids_by_cpos(cpos_list) + cpos_attr_ids: List[int] = attr.ids_by_cpos(cpos_list) for i, cpos in enumerate(cpos_list): if cpos_attr_ids[i] == -1: continue - lookups['cpos_lookup'][cpos][attr.attrs['name']] = cpos_attr_ids[i] + lookups['cpos_lookup'][cpos][attr.name] = cpos_attr_ids[i] occured_attr_ids = [x for x in set(cpos_attr_ids) if x != -1] - if not occured_attr_ids: + if len(occured_attr_ids) == 0: continue subattrs = corpus.structural_attributes.list(filters={'part_of': attr}) - if not subattrs: + if len(subattrs) == 0: continue - lookup_name = f'{attr.attrs["name"]}_lookup' + lookup_name: str = f'{attr.name}_lookup' lookups[lookup_name] = {} for attr_id in occured_attr_ids: lookups[lookup_name][attr_id] = {} for subattr in subattrs: - subattr_name = subattr.attrs['name'][(len(attr.attrs['name']) + 1):] # noqa + subattr_name = subattr.name[(len(attr.name) + 1):] # noqa for i, subattr_value in enumerate(subattr.values_by_ids(occured_attr_ids)): # noqa lookups[lookup_name][occured_attr_ids[i]][subattr_name] = subattr_value # noqa return lookups def partial_export_subcorpus( - subcorpus: Subcorpus, + subcorpus: CQiSubcorpus, match_id_list: List[int], context: int = 25 ) -> Dict: @@ -89,7 +87,7 @@ def partial_export_subcorpus( def export_subcorpus( - subcorpus: Subcorpus, + subcorpus: CQiSubcorpus, context: int = 25, cutoff: float = float('inf'), offset: int = 0 diff --git a/app/corpora/followers/json_routes.py b/app/corpora/followers/json_routes.py index db6bb635..87299862 100644 --- a/app/corpora/followers/json_routes.py +++ b/app/corpora/followers/json_routes.py @@ -12,65 +12,65 @@ from ..decorators import corpus_follower_permission_required from . import bp -# @bp.route('//followers', methods=['POST']) -# @corpus_follower_permission_required('MANAGE_FOLLOWERS') -# @content_negotiation(consumes='application/json', produces='application/json') -# def create_corpus_followers(corpus_id): -# usernames = request.json -# if not (isinstance(usernames, list) or all(isinstance(u, str) for u in usernames)): -# abort(400) -# corpus = Corpus.query.get_or_404(corpus_id) -# for username in usernames: -# user = User.query.filter_by(username=username, is_public=True).first_or_404() -# user.follow_corpus(corpus) -# db.session.commit() -# response_data = { -# 'message': f'Users are now following "{corpus.title}"', -# 'category': 'corpus' -# } -# return response_data, 200 +@bp.route('//followers', methods=['POST']) +@corpus_follower_permission_required('MANAGE_FOLLOWERS') +@content_negotiation(consumes='application/json', produces='application/json') +def create_corpus_followers(corpus_id): + usernames = request.json + if not (isinstance(usernames, list) or all(isinstance(u, str) for u in usernames)): + abort(400) + corpus = Corpus.query.get_or_404(corpus_id) + for username in usernames: + user = User.query.filter_by(username=username, is_public=True).first_or_404() + user.follow_corpus(corpus) + db.session.commit() + response_data = { + 'message': f'Users are now following "{corpus.title}"', + 'category': 'corpus' + } + return response_data, 200 -# @bp.route('//followers//role', methods=['PUT']) -# @corpus_follower_permission_required('MANAGE_FOLLOWERS') -# @content_negotiation(consumes='application/json', produces='application/json') -# def update_corpus_follower_role(corpus_id, follower_id): -# role_name = request.json -# if not isinstance(role_name, str): -# abort(400) -# cfr = CorpusFollowerRole.query.filter_by(name=role_name).first() -# if cfr is None: -# abort(400) -# cfa = CorpusFollowerAssociation.query.filter_by(corpus_id=corpus_id, follower_id=follower_id).first_or_404() -# cfa.role = cfr -# db.session.commit() -# response_data = { -# 'message': f'User "{cfa.follower.username}" is now {cfa.role.name}', -# 'category': 'corpus' -# } -# return response_data, 200 +@bp.route('//followers//role', methods=['PUT']) +@corpus_follower_permission_required('MANAGE_FOLLOWERS') +@content_negotiation(consumes='application/json', produces='application/json') +def update_corpus_follower_role(corpus_id, follower_id): + role_name = request.json + if not isinstance(role_name, str): + abort(400) + cfr = CorpusFollowerRole.query.filter_by(name=role_name).first() + if cfr is None: + abort(400) + cfa = CorpusFollowerAssociation.query.filter_by(corpus_id=corpus_id, follower_id=follower_id).first_or_404() + cfa.role = cfr + db.session.commit() + response_data = { + 'message': f'User "{cfa.follower.username}" is now {cfa.role.name}', + 'category': 'corpus' + } + return response_data, 200 -# @bp.route('//followers/', methods=['DELETE']) -# def delete_corpus_follower(corpus_id, follower_id): -# cfa = CorpusFollowerAssociation.query.filter_by(corpus_id=corpus_id, follower_id=follower_id).first_or_404() -# if not ( -# current_user.id == follower_id -# or current_user == cfa.corpus.user -# or CorpusFollowerAssociation.query.filter_by(corpus_id=corpus_id, follower_id=current_user.id).first().role.has_permission('MANAGE_FOLLOWERS') -# or current_user.is_administrator()): -# abort(403) -# if current_user.id == follower_id: -# flash(f'You are no longer following "{cfa.corpus.title}"', 'corpus') -# response = make_response() -# response.status_code = 204 -# else: -# response_data = { -# 'message': f'"{cfa.follower.username}" is not following "{cfa.corpus.title}" anymore', -# 'category': 'corpus' -# } -# response = jsonify(response_data) -# response.status_code = 200 -# cfa.follower.unfollow_corpus(cfa.corpus) -# db.session.commit() -# return response +@bp.route('//followers/', methods=['DELETE']) +def delete_corpus_follower(corpus_id, follower_id): + cfa = CorpusFollowerAssociation.query.filter_by(corpus_id=corpus_id, follower_id=follower_id).first_or_404() + if not ( + current_user.id == follower_id + or current_user == cfa.corpus.user + or CorpusFollowerAssociation.query.filter_by(corpus_id=corpus_id, follower_id=current_user.id).first().role.has_permission('MANAGE_FOLLOWERS') + or current_user.is_administrator()): + abort(403) + if current_user.id == follower_id: + flash(f'You are no longer following "{cfa.corpus.title}"', 'corpus') + response = make_response() + response.status_code = 204 + else: + response_data = { + 'message': f'"{cfa.follower.username}" is not following "{cfa.corpus.title}" anymore', + 'category': 'corpus' + } + response = jsonify(response_data) + response.status_code = 200 + cfa.follower.unfollow_corpus(cfa.corpus) + db.session.commit() + return response diff --git a/app/corpora/json_routes.py b/app/corpora/json_routes.py index 6a3b5f29..79283aaf 100644 --- a/app/corpora/json_routes.py +++ b/app/corpora/json_routes.py @@ -61,7 +61,7 @@ def build_corpus(corpus_id): @bp.route('/stopwords') @content_negotiation(produces='application/json') def get_stopwords(): - nltk.download('stopwords') + nltk.download('stopwords', quiet=True) languages = ["german", "english", "catalan", "greek", "spanish", "french", "italian", "russian", "chinese"] stopwords = {} for language in languages: @@ -71,55 +71,55 @@ def get_stopwords(): response_data = stopwords return response_data, 202 -# @bp.route('//generate-share-link', methods=['POST']) -# @corpus_follower_permission_required('MANAGE_FOLLOWERS') -# @content_negotiation(consumes='application/json', produces='application/json') -# def generate_corpus_share_link(corpus_id): -# data = request.json -# if not isinstance(data, dict): -# abort(400) -# expiration = data.get('expiration') -# if not isinstance(expiration, str): -# abort(400) -# role_name = data.get('role') -# if not isinstance(role_name, str): -# abort(400) -# expiration_date = datetime.strptime(expiration, '%b %d, %Y') -# cfr = CorpusFollowerRole.query.filter_by(name=role_name).first() -# if cfr is None: -# abort(400) -# corpus = Corpus.query.get_or_404(corpus_id) -# token = current_user.generate_follow_corpus_token(corpus.hashid, role_name, expiration_date) -# corpus_share_link = url_for( -# 'corpora.follow_corpus', -# corpus_id=corpus_id, -# token=token, -# _external=True -# ) -# response_data = { -# 'message': 'Corpus share link generated', -# 'category': 'corpus', -# 'corpusShareLink': corpus_share_link -# } -# return response_data, 200 +@bp.route('//generate-share-link', methods=['POST']) +@corpus_follower_permission_required('MANAGE_FOLLOWERS') +@content_negotiation(consumes='application/json', produces='application/json') +def generate_corpus_share_link(corpus_id): + data = request.json + if not isinstance(data, dict): + abort(400) + expiration = data.get('expiration') + if not isinstance(expiration, str): + abort(400) + role_name = data.get('role') + if not isinstance(role_name, str): + abort(400) + expiration_date = datetime.strptime(expiration, '%b %d, %Y') + cfr = CorpusFollowerRole.query.filter_by(name=role_name).first() + if cfr is None: + abort(400) + corpus = Corpus.query.get_or_404(corpus_id) + token = current_user.generate_follow_corpus_token(corpus.hashid, role_name, expiration_date) + corpus_share_link = url_for( + 'corpora.follow_corpus', + corpus_id=corpus_id, + token=token, + _external=True + ) + response_data = { + 'message': 'Corpus share link generated', + 'category': 'corpus', + 'corpusShareLink': corpus_share_link + } + return response_data, 200 -# @bp.route('//is_public', methods=['PUT']) -# @corpus_owner_or_admin_required -# @content_negotiation(consumes='application/json', produces='application/json') -# def update_corpus_is_public(corpus_id): -# is_public = request.json -# if not isinstance(is_public, bool): -# abort(400) -# corpus = Corpus.query.get_or_404(corpus_id) -# corpus.is_public = is_public -# db.session.commit() -# response_data = { -# 'message': ( -# f'Corpus "{corpus.title}" is now' -# f' {"public" if is_public else "private"}' -# ), -# 'category': 'corpus' -# } -# return response_data, 200 +@bp.route('//is_public', methods=['PUT']) +@corpus_owner_or_admin_required +@content_negotiation(consumes='application/json', produces='application/json') +def update_corpus_is_public(corpus_id): + is_public = request.json + if not isinstance(is_public, bool): + abort(400) + corpus = Corpus.query.get_or_404(corpus_id) + corpus.is_public = is_public + db.session.commit() + response_data = { + 'message': ( + f'Corpus "{corpus.title}" is now' + f' {"public" if is_public else "private"}' + ), + 'category': 'corpus' + } + return response_data, 200 diff --git a/app/corpora/routes.py b/app/corpora/routes.py index b1b9142d..66e6c2df 100644 --- a/app/corpora/routes.py +++ b/app/corpora/routes.py @@ -68,20 +68,19 @@ def corpus(corpus_id): corpus=corpus, cfr=cfr, cfrs=cfrs, - users = users + users=users ) if (current_user.is_following_corpus(corpus) or corpus.is_public): - abort(404) - # cfas = CorpusFollowerAssociation.query.filter(Corpus.id == corpus_id, CorpusFollowerAssociation.follower_id != corpus.user.id).all() - # return render_template( - # 'corpora/public_corpus.html.j2', - # title=corpus.title, - # corpus=corpus, - # cfrs=cfrs, - # cfr=cfr, - # cfas=cfas, - # users = users - # ) + cfas = CorpusFollowerAssociation.query.filter(Corpus.id == corpus_id, CorpusFollowerAssociation.follower_id != corpus.user.id).all() + return render_template( + 'corpora/public_corpus.html.j2', + title=corpus.title, + corpus=corpus, + cfrs=cfrs, + cfr=cfr, + cfas=cfas, + users=users + ) abort(403) @@ -98,14 +97,14 @@ def analysis(corpus_id): ) -# @bp.route('//follow/') -# def follow_corpus(corpus_id, token): -# corpus = Corpus.query.get_or_404(corpus_id) -# if current_user.follow_corpus_by_token(token): -# db.session.commit() -# flash(f'You are following "{corpus.title}" now', category='corpus') -# return redirect(url_for('corpora.corpus', corpus_id=corpus_id)) -# abort(403) +@bp.route('//follow/') +def follow_corpus(corpus_id, token): + corpus = Corpus.query.get_or_404(corpus_id) + if current_user.follow_corpus_by_token(token): + db.session.commit() + flash(f'You are following "{corpus.title}" now', category='corpus') + return redirect(url_for('corpora.corpus', corpus_id=corpus_id)) + abort(403) @bp.route('/import', methods=['GET', 'POST']) diff --git a/app/daemon/corpus_utils.py b/app/daemon/corpus_utils.py index 5b885db7..88a272f8 100644 --- a/app/daemon/corpus_utils.py +++ b/app/daemon/corpus_utils.py @@ -45,7 +45,7 @@ def _create_build_corpus_service(corpus): ''' ## Constraints ## ''' constraints = ['node.role==worker'] ''' ## Image ## ''' - image = f'{current_app.config["NOPAQUE_DOCKER_IMAGE_PREFIX"]}cwb:r1702' + image = f'{current_app.config["NOPAQUE_DOCKER_IMAGE_PREFIX"]}cwb:r1879' ''' ## Labels ## ''' labels = { 'origin': current_app.config['SERVER_NAME'], @@ -139,11 +139,11 @@ def _create_cqpserver_container(corpus): ''' ## Entrypoint ## ''' entrypoint = ['bash', '-c'] ''' ## Image ## ''' - image = f'{current_app.config["NOPAQUE_DOCKER_IMAGE_PREFIX"]}cwb:r1702' + image = f'{current_app.config["NOPAQUE_DOCKER_IMAGE_PREFIX"]}cwb:r1879' ''' ## Name ## ''' name = f'cqpserver_{corpus.id}' ''' ## Network ## ''' - network = f'{current_app.config["DOCKER_NETWORK_NAME"]}' + network = f'{current_app.config["NOPAQUE_DOCKER_NETWORK_NAME"]}' ''' ## Volumes ## ''' volumes = [] ''' ### Corpus data volume ### ''' diff --git a/app/main/cli.py b/app/main/cli.py index 0284bb88..45fabf38 100644 --- a/app/main/cli.py +++ b/app/main/cli.py @@ -43,3 +43,5 @@ def deploy(): SpaCyNLPPipelineModel.insert_defaults() print('Insert/Update default TesseractOCRPipelineModels') TesseractOCRPipelineModel.insert_defaults() + + # TODO: Implement checks for if the nopaque network exists diff --git a/app/main/routes.py b/app/main/routes.py index 3be92196..255edb2d 100644 --- a/app/main/routes.py +++ b/app/main/routes.py @@ -45,12 +45,6 @@ def dashboard(): ) -# @bp.route('/user_manual') -# @register_breadcrumb(bp, '.user_manual', 'helpUser manual') -# def user_manual(): -# return render_template('main/user_manual.html.j2', title='User manual') - - @bp.route('/news') @register_breadcrumb(bp, '.news', 'emailNews') def news(): @@ -78,15 +72,17 @@ def terms_of_use(): ) -# @bp.route('/social-area') -# @register_breadcrumb(bp, '.social_area', 'groupSocial Area') -# @login_required -# def social_area(): -# corpora = Corpus.query.filter(Corpus.is_public == True, Corpus.user != current_user).all() -# users = User.query.filter(User.is_public == True, User.id != current_user.id).all() -# return render_template( -# 'main/social_area.html.j2', -# title='Social Area', -# corpora=corpora, -# users=users -# ) +@bp.route('/social-area') +@register_breadcrumb(bp, '.social_area', 'groupSocial Area') +@login_required +def social_area(): + print('test') + corpora = Corpus.query.filter(Corpus.is_public == True, Corpus.user != current_user).all() + print(corpora) + users = User.query.filter(User.is_public == True, User.id != current_user.id).all() + return render_template( + 'main/social_area.html.j2', + title='Social Area', + corpora=corpora, + users=users + ) diff --git a/app/models.py b/app/models.py index 8121f7a9..ba90ca08 100644 --- a/app/models.py +++ b/app/models.py @@ -853,7 +853,7 @@ class User(HashidMixin, UserMixin, db.Model): json_serializeable = { 'id': self.hashid, 'confirmed': self.confirmed, - # 'avatar': url_for('users.user_avatar', user_id=self.id), + 'avatar': url_for('users.user_avatar', user_id=self.id), 'email': self.email, 'last_seen': ( None if self.last_seen is None @@ -953,7 +953,7 @@ class TesseractOCRPipelineModel(FileMixin, HashidMixin, db.Model): return self.user.hashid @staticmethod - def insert_defaults(): + def insert_defaults(force_download=False): nopaque_user = User.query.filter_by(username='nopaque').first() defaults_file = os.path.join( os.path.dirname(os.path.abspath(__file__)), @@ -966,6 +966,7 @@ class TesseractOCRPipelineModel(FileMixin, HashidMixin, db.Model): if model is not None: model.compatible_service_versions = m['compatible_service_versions'] model.description = m['description'] + model.filename = f'{model.id}.traineddata' model.publisher = m['publisher'] model.publisher_url = m['publisher_url'] model.publishing_url = m['publishing_url'] @@ -973,38 +974,39 @@ class TesseractOCRPipelineModel(FileMixin, HashidMixin, db.Model): model.is_public = True model.title = m['title'] model.version = m['version'] - continue - model = TesseractOCRPipelineModel( - compatible_service_versions=m['compatible_service_versions'], - description=m['description'], - publisher=m['publisher'], - publisher_url=m['publisher_url'], - publishing_url=m['publishing_url'], - publishing_year=m['publishing_year'], - is_public=True, - title=m['title'], - user=nopaque_user, - version=m['version'] - ) - db.session.add(model) - db.session.flush(objects=[model]) - db.session.refresh(model) - model.filename = f'{model.id}.traineddata' - r = requests.get(m['url'], stream=True) - pbar = tqdm( - desc=f'{model.title} ({model.filename})', - unit="B", - unit_scale=True, - unit_divisor=1024, - total=int(r.headers['Content-Length']) - ) - pbar.clear() - with open(model.path, 'wb') as f: - for chunk in r.iter_content(chunk_size=1024): - if chunk: # filter out keep-alive new chunks - pbar.update(len(chunk)) - f.write(chunk) - pbar.close() + else: + model = TesseractOCRPipelineModel( + compatible_service_versions=m['compatible_service_versions'], + description=m['description'], + publisher=m['publisher'], + publisher_url=m['publisher_url'], + publishing_url=m['publishing_url'], + publishing_year=m['publishing_year'], + is_public=True, + title=m['title'], + user=nopaque_user, + version=m['version'] + ) + db.session.add(model) + db.session.flush(objects=[model]) + db.session.refresh(model) + model.filename = f'{model.id}.traineddata' + if not os.path.exists(model.path) or force_download: + r = requests.get(m['url'], stream=True) + pbar = tqdm( + desc=f'{model.title} ({model.filename})', + unit="B", + unit_scale=True, + unit_divisor=1024, + total=int(r.headers['Content-Length']) + ) + pbar.clear() + with open(model.path, 'wb') as f: + for chunk in r.iter_content(chunk_size=1024): + if chunk: # filter out keep-alive new chunks + pbar.update(len(chunk)) + f.write(chunk) + pbar.close() db.session.commit() def delete(self): @@ -1080,7 +1082,7 @@ class SpaCyNLPPipelineModel(FileMixin, HashidMixin, db.Model): return self.user.hashid @staticmethod - def insert_defaults(): + def insert_defaults(force_download=False): nopaque_user = User.query.filter_by(username='nopaque').first() defaults_file = os.path.join( os.path.dirname(os.path.abspath(__file__)), @@ -1093,6 +1095,7 @@ class SpaCyNLPPipelineModel(FileMixin, HashidMixin, db.Model): if model is not None: model.compatible_service_versions = m['compatible_service_versions'] model.description = m['description'] + model.filename = m['url'].split('/')[-1] model.publisher = m['publisher'] model.publisher_url = m['publisher_url'] model.publishing_url = m['publishing_url'] @@ -1101,39 +1104,40 @@ class SpaCyNLPPipelineModel(FileMixin, HashidMixin, db.Model): model.title = m['title'] model.version = m['version'] model.pipeline_name = m['pipeline_name'] - continue - model = SpaCyNLPPipelineModel( - compatible_service_versions=m['compatible_service_versions'], - description=m['description'], - publisher=m['publisher'], - publisher_url=m['publisher_url'], - publishing_url=m['publishing_url'], - publishing_year=m['publishing_year'], - is_public=True, - title=m['title'], - user=nopaque_user, - version=m['version'], - pipeline_name=m['pipeline_name'] - ) - db.session.add(model) - db.session.flush(objects=[model]) - db.session.refresh(model) - model.filename = m['url'].split('/')[-1] - r = requests.get(m['url'], stream=True) - pbar = tqdm( - desc=f'{model.title} ({model.filename})', - unit="B", - unit_scale=True, - unit_divisor=1024, - total=int(r.headers['Content-Length']) - ) - pbar.clear() - with open(model.path, 'wb') as f: - for chunk in r.iter_content(chunk_size=1024): - if chunk: # filter out keep-alive new chunks - pbar.update(len(chunk)) - f.write(chunk) - pbar.close() + else: + model = SpaCyNLPPipelineModel( + compatible_service_versions=m['compatible_service_versions'], + description=m['description'], + filename=m['url'].split('/')[-1], + publisher=m['publisher'], + publisher_url=m['publisher_url'], + publishing_url=m['publishing_url'], + publishing_year=m['publishing_year'], + is_public=True, + title=m['title'], + user=nopaque_user, + version=m['version'], + pipeline_name=m['pipeline_name'] + ) + db.session.add(model) + db.session.flush(objects=[model]) + db.session.refresh(model) + if not os.path.exists(model.path) or force_download: + r = requests.get(m['url'], stream=True) + pbar = tqdm( + desc=f'{model.title} ({model.filename})', + unit="B", + unit_scale=True, + unit_divisor=1024, + total=int(r.headers['Content-Length']) + ) + pbar.clear() + with open(model.path, 'wb') as f: + for chunk in r.iter_content(chunk_size=1024): + if chunk: # filter out keep-alive new chunks + pbar.update(len(chunk)) + f.write(chunk) + pbar.close() db.session.commit() def delete(self): diff --git a/app/services/services.yml b/app/services/services.yml index a686f683..0c6ba958 100644 --- a/app/services/services.yml +++ b/app/services/services.yml @@ -10,7 +10,7 @@ file-setup-pipeline: tesseract-ocr-pipeline: name: 'Tesseract OCR Pipeline' publisher: 'Bielefeld University - CRC 1288 - INF' - latest_version: '0.1.1' + latest_version: '0.1.2' versions: 0.1.0: methods: @@ -23,6 +23,12 @@ tesseract-ocr-pipeline: - 'ocropus_nlbin_threshold' publishing_year: 2022 url: 'https://gitlab.ub.uni-bielefeld.de/sfb1288inf/tesseract-ocr-pipeline/-/releases/v0.1.1' + 0.1.2: + methods: + - 'binarization' + - 'ocropus_nlbin_threshold' + publishing_year: 2023 + url: 'https://gitlab.ub.uni-bielefeld.de/sfb1288inf/tesseract-ocr-pipeline/-/releases/v0.1.2' transkribus-htr-pipeline: name: 'Transkribus HTR Pipeline' publisher: 'Bielefeld University - CRC 1288 - INF' @@ -41,7 +47,7 @@ transkribus-htr-pipeline: spacy-nlp-pipeline: name: 'SpaCy NLP Pipeline' publisher: 'Bielefeld University - CRC 1288 - INF' - latest_version: '0.1.2' + latest_version: '0.1.1' versions: 0.1.0: methods: @@ -53,8 +59,3 @@ spacy-nlp-pipeline: - 'encoding_detection' publishing_year: 2022 url: 'https://gitlab.ub.uni-bielefeld.de/sfb1288inf/spacy-nlp-pipeline/-/releases/v0.1.1' - 0.1.2: - methods: - - 'encoding_detection' - publishing_year: 2022 - url: 'https://gitlab.ub.uni-bielefeld.de/sfb1288inf/spacy-nlp-pipeline/-/releases/v0.1.2' diff --git a/app/static/css/queryBuilder.css b/app/static/css/queryBuilder.css index 4ff7eb9d..31c13113 100644 --- a/app/static/css/queryBuilder.css +++ b/app/static/css/queryBuilder.css @@ -1,132 +1,108 @@ -.modal-conent { +#corpus-analysis-concordance-query-builder-input-field { + border-bottom: #9E9E9E 1px solid; + min-height: 38px; + margin-top: 23px; +} + +#corpus-analysis-concordance-query-builder-input-field-placeholder { + color: #9E9E9E; +} + +.modal-content { overflow-x: hidden; } -#concordance-query-builder { +#corpus-analysis-concordance-positional-attr-modal, #corpus-analysis-concordance-corpus-analysis-concordance-structural-attr-modal { width: 70%; } -#concordance-query-builder nav { - background-color: #6B3F89; - margin-top: -25px; - margin-left: -25px; - width: 105%; -} - -#query-builder-nav{ - padding-left: 15px; -} - -#close-query-builder { - margin-right: 50px; - cursor: pointer; -} - -#general-options-query-builder-tutorial-info-icon { +#corpus-analysis-concordance-general-options-query-builder-tutorial-info-icon { color: black; } -#your-query { - border-bottom-style: solid; - border-bottom-width: 1px; -} - -#insert-query-button { +#corpus-analysis-concordance-insert-query-button { background-color: #00426f; text-align: center; } -#structural-attr h6 { - margin-left: 15px; -} - -#add-structural-attribute-tutorial-info-icon { - color: black; -} - -#sentence { - background-color:#FD9720; -} - -#entity { - background-color: #A6E22D; -} - -#text-annotation { - background-color: #2FBBAB; -} - -#no-value-metadata-message { - padding-top: 25px; - margin-left: -20px; -} - -#token-kind-selector { +.attr-modal-header { background-color: #f2eff7; padding: 15px; - border-top-style: solid; - border-color: #6B3F89; + padding-left: 25px; + border-top: 10px solid #6B3F89; + margin-left: -24px; + margin-top: -24px; + margin-right: -24px; } -#token-kind-selector.s5 { - margin-top: 15px; -} - -#token-kind-selector h6 { +.attr-modal-header h6 { margin-left: 15px; } -#token-tutorial-info-icon { +#corpus-analysis-concordance-add-structural-attribute-tutorial-info-icon { color: black; } -#no-value-message { +[data-structural-attr-modal-action-button="sentence"]{ + background-color:#FD9720 !important; +} + +[data-structural-attr-modal-action-button="entity"]{ + background-color: #A6E22D !important; +} + +[data-structural-attr-modal-action-button="meta-data"]{ + background-color: #2FBBAB !important; +} + +#corpus-analysis-concordance-no-value-metadata-message { padding-top: 25px; margin-left: -20px; } -#token-edit-options h6 { - margin-left: 15px; +.attr-modal-header.input-field { + margin-left: 41px; } -#edit-options-tutorial-info-icon { +#corpus-analysis-concordance-token-attr { + margin-left: 41px; +} + +#corpus-analysis-concordance-token-tutorial-info-icon { color: black; } -#incidence-modifiers-button a{ - background-color: #2FBBAB; +#corpus-analysis-concordance-no-value-message { + padding-top: 25px; + margin-left: -20px; } -#incidence-modifiers a{ - background-color: white; +#corpus-analysis-concordance-token-edit-options h6 { + margin-left: 15px; } -#ignore-case { - margin-left: 5px; +#corpus-analysis-concordance-edit-options-tutorial-info-icon { + color: black; } -#or, #and { - background-color: #fc0; +[data-toggle-area="input-field-options"] a { + margin-right: 10px; } -#betweenNM { - width: 60%; +[data-target="corpus-analysis-concordance-character-incidence-modifiers-dropdown"], [data-target="corpus-analysis-concordance-token-incidence-modifiers-dropdown"] { + background-color: #2FBBAB !important; } -#query-builder-tutorial-modal { - width: 60%; +#corpus-analysis-concordance-exactly-n-token-modal, #corpus-analysis-concordance-between-nm-token-modal { + width: 30%; } -#query-builder-tutorial-modal ul { - margin-top: 10px; +[data-modal-id="corpus-analysis-concordance-exactly-n-token-modal"], [data-modal-id="corpus-analysis-concordance-between-nm-token-modal"] { + margin-top: 15px !important; } -#query-builder-tutorial { - padding:15px; -} - -#scroll-up-button-query-builder-tutorial { - background-color: #28B3D1; +[data-options-action="and"], [data-options-action="or"] { + background-color: #fc0 !important; } [data-type="start-sentence"], [data-type="end-sentence"] { @@ -134,13 +110,18 @@ } [data-type="start-empty-entity"], [data-type="start-entity"], [data-type="end-entity"] { - background-color: #A6E22D; + background-color: #a6e22d; } -[data-type="start-text-annotation"]{ +[data-type="text-annotation"]{ background-color: #2FBBAB; } [data-type="token"] { background-color: #28B3D1; } + +[data-type="token-incidence-modifier"] { + background-color: #4db6ac; + color: white; +} diff --git a/app/static/images/manual/query_builder/delete.gif b/app/static/images/manual/query_builder/delete.gif index a5dc39b3..af4f50ee 100644 Binary files a/app/static/images/manual/query_builder/delete.gif and b/app/static/images/manual/query_builder/delete.gif differ diff --git a/app/static/images/manual/query_builder/drag_and_drop.gif b/app/static/images/manual/query_builder/drag_and_drop.gif index 1c80fa8a..df87671a 100644 Binary files a/app/static/images/manual/query_builder/drag_and_drop.gif and b/app/static/images/manual/query_builder/drag_and_drop.gif differ diff --git a/app/static/images/manual/query_builder/editing_chips.gif b/app/static/images/manual/query_builder/editing_chips.gif new file mode 100644 index 00000000..28823f45 Binary files /dev/null and b/app/static/images/manual/query_builder/editing_chips.gif differ diff --git a/app/static/images/manual/query_builder/entity.gif b/app/static/images/manual/query_builder/entity.gif index c496c01d..80d52937 100644 Binary files a/app/static/images/manual/query_builder/entity.gif and b/app/static/images/manual/query_builder/entity.gif differ diff --git a/app/static/images/manual/query_builder/expert_mode.gif b/app/static/images/manual/query_builder/expert_mode.gif new file mode 100644 index 00000000..19e9fef6 Binary files /dev/null and b/app/static/images/manual/query_builder/expert_mode.gif differ diff --git a/app/static/images/manual/query_builder/incidence_modifier.gif b/app/static/images/manual/query_builder/incidence_modifier.gif new file mode 100644 index 00000000..8b817b0d Binary files /dev/null and b/app/static/images/manual/query_builder/incidence_modifier.gif differ diff --git a/app/static/images/manual/query_builder/option_group.gif b/app/static/images/manual/query_builder/option_group.gif index e27d5116..45616d5e 100644 Binary files a/app/static/images/manual/query_builder/option_group.gif and b/app/static/images/manual/query_builder/option_group.gif differ diff --git a/app/static/images/manual/query_builder/or_and.gif b/app/static/images/manual/query_builder/or_and.gif index c24499a2..66e093c1 100644 Binary files a/app/static/images/manual/query_builder/or_and.gif and b/app/static/images/manual/query_builder/or_and.gif differ diff --git a/app/static/images/manual/query_builder/pos.gif b/app/static/images/manual/query_builder/pos.gif index 79367d55..863eecfa 100644 Binary files a/app/static/images/manual/query_builder/pos.gif and b/app/static/images/manual/query_builder/pos.gif differ diff --git a/app/static/images/manual/query_builder/word_lemma.gif b/app/static/images/manual/query_builder/word_lemma.gif index 63ab9b0b..d406eda3 100644 Binary files a/app/static/images/manual/query_builder/word_lemma.gif and b/app/static/images/manual/query_builder/word_lemma.gif differ diff --git a/app/static/images/nopaque_slogan_transparent.png b/app/static/images/nopaque_slogan_transparent.png new file mode 100644 index 00000000..3b7b4aaf Binary files /dev/null and b/app/static/images/nopaque_slogan_transparent.png differ diff --git a/app/static/js/App.js b/app/static/js/App.js deleted file mode 100644 index cfcb3a05..00000000 --- a/app/static/js/App.js +++ /dev/null @@ -1,104 +0,0 @@ -class App { - constructor() { - this.data = { - promises: {getUser: {}, subscribeUser: {}}, - users: {}, - }; - this.socket = io({transports: ['websocket'], upgrade: false}); - this.socket.on('PATCH', (patch) => {this.onPatch(patch);}); - } - - getUser(userId) { - if (userId in this.data.promises.getUser) { - return this.data.promises.getUser[userId]; - } - - this.data.promises.getUser[userId] = new Promise((resolve, reject) => { - this.socket.emit('GET /users/', userId, (response) => { - if (response.status === 200) { - this.data.users[userId] = response.body; - resolve(this.data.users[userId]); - } else { - reject(`[${response.status}] ${response.statusText}`); - } - }); - }); - - return this.data.promises.getUser[userId]; - } - - subscribeUser(userId) { - if (userId in this.data.promises.subscribeUser) { - return this.data.promises.subscribeUser[userId]; - } - - this.data.promises.subscribeUser[userId] = new Promise((resolve, reject) => { - this.socket.emit('SUBSCRIBE /users/', userId, (response) => { - if (response.status !== 200) { - reject(response); - return; - } - resolve(response); - }); - }); - - return this.data.promises.subscribeUser[userId]; - } - - flash(message, category) { - let iconPrefix = ''; - switch (category) { - case 'corpus': { - iconPrefix = 'book'; - break; - } - case 'error': { - iconPrefix = 'error'; - break; - } - case 'job': { - iconPrefix = 'J'; - break; - } - case 'settings': { - iconPrefix = 'settings'; - break; - } - default: { - iconPrefix = 'notifications'; - break; - } - } - let toast = M.toast( - { - html: ` - ${iconPrefix}${message} - - `.trim() - } - ); - let toastCloseActionElement = toast.el.querySelector('.action-button[data-action="close"]'); - toastCloseActionElement.addEventListener('click', () => {toast.dismiss();}); - } - - onPatch(patch) { - // Filter Patch to only include operations on users that are initialized - let regExp = new RegExp(`^/users/(${Object.keys(this.data.users).join('|')})`); - let filteredPatch = patch.filter(operation => regExp.test(operation.path)); - - // Handle job status updates - let subRegExp = new RegExp(`^/users/([A-Za-z0-9]*)/jobs/([A-Za-z0-9]*)/status$`); - let subFilteredPatch = filteredPatch - .filter((operation) => {return operation.op === 'replace';}) - .filter((operation) => {return subRegExp.test(operation.path);}); - for (let operation of subFilteredPatch) { - let [match, userId, jobId] = operation.path.match(subRegExp); - this.flash(`[${this.data.users[userId].jobs[jobId].title}] New status: `, 'job'); - } - - // Apply Patch - jsonpatch.applyPatch(this.data, filteredPatch); - } -} diff --git a/app/static/js/CorpusAnalysis/QueryBuilder.js b/app/static/js/CorpusAnalysis/QueryBuilder.js deleted file mode 100644 index 9163b4bc..00000000 --- a/app/static/js/CorpusAnalysis/QueryBuilder.js +++ /dev/null @@ -1,1007 +0,0 @@ -class ConcordanceQueryBuilder { - - constructor() { - - - this.elements = { - - counter: 0, - yourQueryContent: [], - queryContent:[], - concordanceQueryBuilder: document.querySelector('#concordance-query-builder'), - concordanceQueryBuilderButton: document.querySelector('#concordance-query-builder-button'), - closeQueryBuilder: document.querySelector('#close-query-builder'), - queryBuilderTutorialModal: document.querySelector('#query-builder-tutorial-modal'), - valueValidator: true, - - - //#region QueryBuilder Elements - - positionalAttrButton: document.querySelector('#positional-attr-button'), - positionalAttrArea: document.querySelector('#positional-attr'), - positionalAttr: document.querySelector('#token-attr'), - structuralAttrButton: document.querySelector('#structural-attr-button'), - structuralAttrArea: document.querySelector('#structural-attr'), - queryContainer: document.querySelector('#query-container'), - buttonPreparer: document.querySelector('#button-preparer'), - yourQuery: document.querySelector('#your-query'), - insertQueryButton: document.querySelector('#insert-query-button'), - queryPreview: document.querySelector('#query-preview'), - tokenQuery: document.querySelector('#token-query'), - tokenBuilderContent: document.querySelector('#token-builder-content'), - tokenSubmitButton: document.querySelector('#token-submit'), - extFormQuery: document.querySelector('#concordance-extension-form-query'), - dropButton: '', - - queryBuilderTutorialInfoIcon: document.querySelector('#query-builder-tutorial-info-icon'), - tokenTutorialInfoIcon: document.querySelector('#token-tutorial-info-icon'), - editTokenTutorialInfoIcon: document.querySelector('#edit-options-tutorial-info-icon'), - structuralAttributeTutorialInfoIcon: document.querySelector('#add-structural-attribute-tutorial-info-icon'), - generalOptionsQueryBuilderTutorialInfoIcon: document.querySelector('#general-options-query-builder-tutorial-info-icon'), - - - //#endregion QueryBuilder Elements - - //#region Strucutral Attributes - - sentence:document.querySelector('#sentence'), - entity: document.querySelector('#entity'), - textAnnotation: document.querySelector('#text-annotation'), - - entityBuilder: document.querySelector('#entity-builder'), - englishEntType: document.querySelector('#english-ent-type'), - germanEntType: document.querySelector('#german-ent-type'), - emptyEntity: document.querySelector('#empty-entity'), - entityAnyType: false, - - textAnnotationBuilder: document.querySelector('#text-annotation-builder'), - textAnnotationOptions: document.querySelector('#text-annotation-options'), - textAnnotationInput: document.querySelector('#text-annotation-input'), - textAnnotationSubmit: document.querySelector('#text-annotation-submit'), - noValueMetadataMessage: document.querySelector('#no-value-metadata-message'), - //#endregion Structural Attributes - - //#region Token Attributes - tokenQueryFilled: false, - - lemma: document.querySelector('#lemma'), - emptyToken: document.querySelector('#empty-token'), - word: document.querySelector('#word'), - lemma: document.querySelector('#lemma'), - pos: document.querySelector('#pos'), - simplePosButton: document.querySelector('#simple-pos-button'), - incidenceModifiers: document.querySelector('[data-target="incidence-modifiers"]'), - or: document.querySelector('#or'), - and: document.querySelector('#and'), - - //#region Word and Lemma Elements - wordBuilder: document.querySelector('#word-builder'), - lemmaBuilder: document.querySelector('#lemma-builder'), - inputOptions: document.querySelector('#input-options'), - incidenceModifiersButton: document.querySelector('#incidence-modifiers-button'), - conditionContainer: document.querySelector('#condition-container'), - wordInput: document.querySelector('#word-input'), - lemmaInput: document.querySelector('#lemma-input'), - ignoreCaseCheckbox : document.querySelector('#ignore-case-checkbox'), - ignoreCase: document.querySelector('input[type="checkbox"]'), - wildcardChar: document.querySelector('#wildcard-char'), - optionGroup: document.querySelector('#option-group'), - //#endregion Word and Lemma Elements - - //#region posBuilder Elements - englishPosBuilder: document.querySelector('#english-pos-builder'), - englishPos: document.querySelector('#english-pos'), - germanPosBuilder: document.querySelector('#german-pos-builder'), - germanPos: document.querySelector('#german-pos'), - //#endregion posBuilder Elements - - //#region simple_posBuilder Elements - simplePosBuilder: document.querySelector('#simplepos-builder'), - simplePos: document.querySelector('#simple-pos'), - //#endregion simple_posBuilder Elements - - //#region incidence modifiers - oneOrMore: document.querySelector('#one-or-more'), - zeroOrMore: document.querySelector('#zero-or-more'), - zeroOrOne: document.querySelector('#zero-or-one'), - exactlyN: document.querySelector('#exactlyN'), - betweenNM: document.querySelector('#betweenNM'), - nInput: document.querySelector('#n-input'), - nSubmit: document.querySelector('#n-submit'), - nmInput: document.querySelector('#n-m-input'), - mInput: document.querySelector('#m-input'), - nmSubmit: document.querySelector('#n-m-submit'), - //#endregion incidence modifiers - - cancelBool: false, - noValueMessage: document.querySelector('#no-value-message'), - //#endregion Token Attributes - } - - this.elements.closeQueryBuilder.addEventListener('click', () => {this.closeQueryBuilderModal(this.elements.concordanceQueryBuilder);}); - this.elements.concordanceQueryBuilderButton.addEventListener('click', () => {this.clearAll();}); - this.elements.insertQueryButton.addEventListener('click', () => {this.insertQuery();}); - this.elements.positionalAttrButton.addEventListener('click', () => {this.showPositionalAttrArea();}); - this.elements.structuralAttrButton.addEventListener('click', () => {this.showStructuralAttrArea();}); - - //#region Structural Attribute Event Listeners - this.elements.sentence.addEventListener('click', () => {this.addSentence();}); - this.elements.entity.addEventListener('click', () => {this.addEntity();}); - this.elements.textAnnotation.addEventListener('click', () => {this.addTextAnnotation();}); - - this.elements.englishEntType.addEventListener('change', () => {this.englishEntTypeHandler();}); - this.elements.germanEntType.addEventListener('change', () => {this.germanEntTypeHandler();}); - this.elements.emptyEntity.addEventListener('click', () => {this.emptyEntityButton();}); - - this.elements.textAnnotationSubmit.addEventListener('click', () => {this.textAnnotationSubmitHandler();}); - - //#endregion - - //#region Token Attribute Event Listeners - this.elements.queryBuilderTutorialInfoIcon.addEventListener('click', () => {this.tutorialIconHandler('#query-builder-tutorial-start');}); - this.elements.tokenTutorialInfoIcon.addEventListener('click', () => {this.tutorialIconHandler('#add-new-token-tutorial');}); - this.elements.editTokenTutorialInfoIcon.addEventListener('click', () => {this.tutorialIconHandler('#edit-options-tutorial');}); - this.elements.structuralAttributeTutorialInfoIcon.addEventListener('click', () => {this.tutorialIconHandler('#add-structural-attribute-tutorial');}); - this.elements.generalOptionsQueryBuilderTutorialInfoIcon.addEventListener('click', () => {this.tutorialIconHandler('#general-options-query-builder');}); - - this.elements.positionalAttr.addEventListener('change', () => {this.tokenTypeSelector();}); - this.elements.tokenSubmitButton.addEventListener('click', () => {this.addTokenToQuery();}); - - this.elements.wordInput.addEventListener('input', () => {this.inputFieldHandler();}); - this.elements.lemmaInput.addEventListener('input', () => {this.inputFieldHandler();}); - this.elements.ignoreCase.addEventListener('change', () => {this.inputOptionHandler(this.elements.ignoreCase);}); - this.elements.wildcardChar.addEventListener('click', () => {this.inputOptionHandler(this.elements.wildcardChar);}); - this.elements.optionGroup.addEventListener('click', () => {this.inputOptionHandler(this.elements.optionGroup);}); - - this.elements.oneOrMore.addEventListener('click', () => {this.incidenceModifiersHandler(this.elements.oneOrMore);}); - this.elements.zeroOrMore.addEventListener('click', () => {this.incidenceModifiersHandler(this.elements.zeroOrMore);}); - this.elements.zeroOrOne.addEventListener('click', () => {this.incidenceModifiersHandler(this.elements.zeroOrOne);}); - this.elements.nSubmit.addEventListener('click', () => {this.nSubmitHandler();}); - this.elements.nmSubmit.addEventListener('click', () => {this.nmSubmitHandler();}); - - this.elements.or.addEventListener('click', () => {this.orHandler();}); - this.elements.and.addEventListener('click', () => {this.andHandler();}); - - //#endregion Token Attribute Event Listeners - let selectInstances = this.elements.concordanceQueryBuilder.querySelectorAll('select'); - M.FormSelect.init( - selectInstances, - { - dropdownOptions: { - alignment: 'bottom', - coverTrigger: false - } - } - ) - let dropdownContents = this.elements.concordanceQueryBuilder.querySelectorAll('.dropdown-content'); - dropdownContents.forEach((dropdownContent) => { - dropdownContent.style.paddingBottom = '15px'; - }); - } - - - // ########################################################################## - // #################### General Functions ################################### - // ########################################################################## - - //#region General Functions - - closeQueryBuilderModal(closeInstance) { - let instance = M.Modal.getInstance(closeInstance); - instance.close(); - - } - - showPositionalAttrArea() { - this.elements.positionalAttrArea.classList.remove('hide'); - this.elements.structuralAttrArea.classList.add('hide'); - this.wordBuilder(); - - this.elements.tokenQueryFilled = false; - - window.location.href = '#token-builder-content'; - } - - showStructuralAttrArea() { - this.elements.positionalAttrArea.classList.add('hide'); - this.elements.structuralAttrArea.classList.remove('hide'); - } - - queryChipFactory(dataType, prettyQueryText, queryText) { - this.elements.counter++; - window.location.href = '#query-container'; - queryText = Utils.escape(queryText); - prettyQueryText = Utils.escape(prettyQueryText); - let queryChipElement = Utils.HTMLToElement( - ` - - ${prettyQueryText} - close - - ` - ); - queryChipElement.addEventListener('click', () => {this.deleteAttr(queryChipElement);}); - queryChipElement.addEventListener('dragstart', (event) => { - // selects all nodes without target class - let queryChips = this.elements.yourQuery.querySelectorAll('.query-component'); - - // Adds a target chip in front of all draggable childnodes - setTimeout(() => { - let targetChipElement = Utils.HTMLToElement('Drop here'); - for (let element of queryChips) { - if (element === queryChipElement.nextSibling) {continue;} - let targetChipClone = targetChipElement.cloneNode(true); - if (element === queryChipElement) { - // If the dragged element is not at the very end, a target chip is also inserted at the end - if (queryChips[queryChips.length - 1] !== element) { - queryChips[queryChips.length - 1].insertAdjacentElement('afterend', targetChipClone); - } - } else { - element.insertAdjacentElement('beforebegin', targetChipClone); - } - targetChipClone.addEventListener('dragover', (event) => { - event.preventDefault(); - }); - targetChipClone.addEventListener('dragenter', (event) => { - event.preventDefault(); - event.target.style.borderStyle = 'solid dotted'; - }); - targetChipClone.addEventListener('dragleave', (event) => { - event.preventDefault(); - event.target.style.borderStyle = 'hidden'; - }); - targetChipClone.addEventListener('drop', (event) => { - let dropzone = event.target; - dropzone.parentElement.replaceChild(queryChipElement, dropzone); - this.queryPreviewBuilder(); - }); - } - }, 0); - }); - - queryChipElement.addEventListener('dragend', (event) => { - let targets = document.querySelectorAll('.drop-target'); - for (let target of targets) { - target.remove(); - } - }); - - // Ensures that metadata is always at the end of the query: - const lastChild = this.elements.yourQuery.lastChild; - const isLastChildTextAnnotation = lastChild && lastChild.dataset.type === 'text-annotation'; - - if (!isLastChildTextAnnotation) { - this.elements.yourQuery.appendChild(queryChipElement); - } else { - this.elements.yourQuery.insertBefore(queryChipElement, lastChild); - } - - this.elements.queryContainer.classList.remove('hide'); - this.queryPreviewBuilder(); - - // Shows a hint about possible functions for editing the query at the first added element in the query - if (this.elements.yourQuery.childNodes.length === 1) { - app.flash('You can edit your query by deleting individual elements or moving them via drag and drop.'); - } - } - - queryPreviewBuilder() { - this.elements.yourQueryContent = []; - for (let element of this.elements.yourQuery.childNodes) { - let queryElement = element.dataset.query; - if (queryElement !== undefined) { - queryElement = Utils.escape(queryElement); - this.elements.yourQueryContent.push(queryElement); - } - } - - let queryString = this.elements.yourQueryContent.join(' '); - queryString += ';'; - this.elements.queryPreview.innerHTML = queryString; - } - - - deleteAttr(attr) { - this.elements.yourQuery.removeChild(attr); - if (attr.dataset.type === "start-sentence") { - this.elements.sentence.innerHTML = 'Sentence'; - } else if (attr.dataset.type === "start-entity" || attr.dataset.type === "start-empty-entity") { - this.elements.entity.innerHTML = 'Entity'; - } - this.elements.counter -= 1; - if (this.elements.counter === 0) { - this.elements.queryContainer.classList.add('hide'); - } - this.queryPreviewBuilder(); - } - - insertQuery() { - this.elements.yourQueryContent = []; - this.validateValue(); - if (this.elements.valueValidator) { - for (let element of this.elements.yourQuery.childNodes) { - let queryElement = element.dataset.query; - if (queryElement !== 'undefined') { - this.elements.yourQueryContent.push(queryElement); - } - } - - let queryString = this.elements.yourQueryContent.join(' '); - queryString += ';'; - - this.elements.concordanceQueryBuilder.classList.add('modal-close'); - this.elements.extFormQuery.value = queryString; - } - } - - validateValue() { - this.elements.valueValidator = true; - let sentenceCounter = 0; - let sentenceEndCounter = 0; - let entityCounter = 0; - let entityEndCounter = 0; - for (let element of this.elements.yourQuery.childNodes) { - if (element.dataset.type === 'start-sentence') { - sentenceCounter += 1; - }else if (element.dataset.type === 'end-sentence') { - sentenceEndCounter += 1; - }else if (element.dataset.type === 'start-entity' || element.dataset.type === 'start-empty-entity') { - entityCounter += 1; - }else if (element.dataset.type === 'end-entity') { - entityEndCounter += 1; - } - } - // Checks if the same number of opening and closing tags (entity and sentence) are present. Depending on what is missing, the corresponding error message is ejected - if (sentenceCounter > sentenceEndCounter) { - app.flash('Please add the closing sentence tag', 'error'); - this.elements.valueValidator = false; - } else if (sentenceCounter < sentenceEndCounter) { - app.flash('Please remove the closing sentence tag', 'error'); - this.elements.valueValidator = false; - } - if (entityCounter > entityEndCounter) { - app.flash('Please add the closing entity tag', 'error'); - this.elements.valueValidator = false; - } else if (entityCounter < entityEndCounter) { - app.flash('Please remove the closing entity tag', 'error'); - this.elements.valueValidator = false; - } - } - - clearAll() { - // Everything is reset. - let instance = M.Tooltip.getInstance(this.elements.queryBuilderTutorialInfoIcon); - - this.hideEverything(); - this.elements.counter = 0; - this.elements.concordanceQueryBuilder.classList.remove('modal-close'); - this.elements.positionalAttrArea.classList.add('hide'); - this.elements.structuralAttrArea.classList.add('hide'); - this.elements.yourQuery.innerHTML = ''; - this.elements.queryContainer.classList.add('hide'); - this.elements.entity.innerHTML = 'Entity'; - this.elements.sentence.innerHTML = 'Sentence'; - - // If the Modal is open after 5 seconds for 5 seconds (with 'instance'), a message is displayed indicating that further information can be obtained via the question mark icon - instance.tooltipEl.style.background = '#98ACD2'; - instance.tooltipEl.style.borderTop = 'solid 4px #0064A3'; - instance.tooltipEl.style.padding = '10px'; - instance.tooltipEl.style.color = 'black'; - - setTimeout(() => { - let modalInstance = M.Modal.getInstance(this.elements.concordanceQueryBuilder); - if (modalInstance.isOpen) { - instance.open(); - setTimeout(() => { - instance.close(); - }, 5000); - } - }, 5000); - - } - - tutorialIconHandler(id) { - setTimeout(() => { - window.location.href= id; - }, 0); - - } - - //#endregion General Functions - - - // ########################################################################## - // ############## Token Attribute Builder Functions ######################### - // ########################################################################## - - //#region Token Attribute Builder Functions - - //#region General functions of the Token Builder - tokenTypeSelector() { - this.hideEverything(); - switch (this.elements.positionalAttr.value) { - case 'word': - this.wordBuilder(); - break; - case 'lemma': - this.lemmaBuilder(); - break; - case 'english-pos': - this.englishPosHandler(); - break; - case 'german-pos': - this.germanPosHandler(); - break; - case 'simple-pos-button': - this.simplePosBuilder(); - break; - case 'empty-token': - this.emptyTokenHandler(); - break; - default: - this.wordBuilder(); - break; - } - } - - hideEverything() { - - this.elements.wordBuilder.classList.add('hide'); - this.elements.lemmaBuilder.classList.add('hide'); - this.elements.ignoreCaseCheckbox.classList.add('hide'); - this.elements.inputOptions.classList.add('hide'); - this.elements.incidenceModifiersButton.classList.add('hide'); - this.elements.conditionContainer.classList.add('hide'); - this.elements.englishPosBuilder.classList.add('hide'); - this.elements.germanPosBuilder.classList.add('hide'); - this.elements.simplePosBuilder.classList.add('hide'); - this.elements.entityBuilder.classList.add('hide'); - this.elements.textAnnotationBuilder.classList.add('hide'); - - } - - tokenChipFactory(prettyQueryText, tokenText) { - tokenText = encodeURI(tokenText); - let builderElement; - let queryChipElement; - builderElement = document.createElement('div'); - builderElement.innerHTML = ` -
- ${prettyQueryText} - close -
`; - queryChipElement = builderElement.firstElementChild; - queryChipElement.addEventListener('click', () => {this.deleteTokenAttr(queryChipElement);}); - this.elements.tokenQuery.appendChild(queryChipElement); - } - - deleteTokenAttr(attr) { - if (this.elements.tokenQuery.childNodes.length < 2) { - this.elements.tokenQuery.removeChild(attr); - this.wordBuilder(); - } else { - this.elements.tokenQuery.removeChild(attr); - } - - } - - addTokenToQuery() { - let c; - let tokenQueryContent = ''; //for ButtonFactory(prettyQueryText) - let tokenQueryText = ''; //for ButtonFactory(queryText) - this.elements.cancelBool = false; - let tokenIsEmpty = false; - - if (this.elements.ignoreCase.checked) { - c = ' %c'; - } else { - c = ''; - } - - for (let element of this.elements.tokenQuery.childNodes) { - tokenQueryContent += ' ' + element.firstChild.data + ' '; - tokenQueryText += decodeURI(element.dataset.tokentext); - if (element.innerText.indexOf('empty token') !== -1) { - tokenIsEmpty = true; - } - } - - if (this.elements.tokenQueryFilled === false) { - switch (this.elements.positionalAttr.value) { - case 'word': - if (this.elements.wordInput.value === '') { - this.disableTokenSubmit(); - } else { - tokenQueryContent += `word=${this.elements.wordInput.value}${c}`; - tokenQueryText += `word="${this.elements.wordInput.value}"${c}`; - this.elements.wordInput.value = ''; - } - break; - case 'lemma': - if (this.elements.lemmaInput.value === '') { - this.disableTokenSubmit(); - } else { - tokenQueryContent += `lemma=${this.elements.lemmaInput.value}${c}`; - tokenQueryText += `lemma="${this.elements.lemmaInput.value}"${c}`; - this.elements.lemmaInput.value = ''; - } - break; - case 'english-pos': - if (this.elements.englishPos.value === 'default') { - this.disableTokenSubmit(); - } else { - tokenQueryContent += `pos=${this.elements.englishPos.value}`; - tokenQueryText += `pos="${this.elements.englishPos.value}"`; - this.elements.englishPos.value = ''; - } - break; - case 'german-pos': - if (this.elements.germanPos.value === 'default') { - this.disableTokenSubmit(); - } else { - tokenQueryContent += `pos=${this.elements.germanPos.value}`; - tokenQueryText += `pos="${this.elements.germanPos.value}"`; - this.elements.germanPos.value = ''; - } - break; - case 'simple-pos-button': - if (this.elements.simplePos.value === 'default') { - this.disableTokenSubmit(); - } else { - tokenQueryContent += `simple_pos=${this.elements.simplePos.value}`; - tokenQueryText += `simple_pos="${this.elements.simplePos.value}"`; - this.elements.simplePos.value = ''; - } - break; - default: - this.wordBuilder(); - break; - } - } - - // cancelBool looks in disableTokenSubmit() whether a value is passed. If the input fields/dropdowns are empty (cancelBool === true), no token is added. - if (this.elements.cancelBool === false) { - // Square brackets are added only if it is not an empty token (where they are already present). - if (tokenIsEmpty === false) { - tokenQueryText = '[' + tokenQueryText + ']'; - } - this.queryChipFactory('token', tokenQueryContent, tokenQueryText); - this.hideEverything(); - this.elements.positionalAttrArea.classList.add('hide'); - this.elements.tokenQuery.innerHTML = ''; - } - - } - - disableTokenSubmit() { - this.elements.cancelBool = true; - this.elements.tokenSubmitButton.classList.add('red'); - this.elements.noValueMessage.classList.remove('hide'); - setTimeout(() => { - this.elements.tokenSubmitButton.classList.remove('red'); - }, 500); - setTimeout(() => { - this.elements.noValueMessage.classList.add('hide'); - }, 3000); - } - - inputFieldHandler() { - let input; - - if (this.elements.wordBuilder.classList.contains('hide') === false) { - input = this.elements.wordInput; - } else { - input = this.elements.lemmaInput; - } - - if (input.value === '') { - this.elements.incidenceModifiersButton.firstElementChild.classList.add('disabled'); - this.elements.or.classList.add('disabled'); - this.elements.and.classList.add('disabled'); - } else { - this.elements.incidenceModifiersButton.firstElementChild.classList.remove('disabled'); - this.elements.or.classList.remove('disabled'); - this.elements.and.classList.remove('disabled'); - } - } - - //#endregion General functions of the Token Builder - - //#region Dropdown Select Handler - wordBuilder() { - this.hideEverything(); - this.elements.wordInput.value = ''; - this.elements.wordBuilder.classList.remove('hide'); - this.elements.inputOptions.classList.remove('hide'); - this.elements.incidenceModifiersButton.classList.remove('hide'); - this.elements.conditionContainer.classList.remove('hide'); - this.elements.ignoreCaseCheckbox.classList.remove('hide'); - - this.elements.incidenceModifiersButton.firstElementChild.classList.add('disabled'); - this.elements.or.classList.add('disabled'); - this.elements.and.classList.add('disabled'); - - // Resets materialize select field to default value - let SelectInstance = M.FormSelect.getInstance(this.elements.positionalAttr); - SelectInstance.input.value = 'word'; - this.elements.positionalAttr.value = 'word'; - - } - - lemmaBuilder() { - this.hideEverything(); - this.elements.lemmaInput.value = ''; - this.elements.lemmaBuilder.classList.remove('hide'); - this.elements.inputOptions.classList.remove('hide'); - this.elements.incidenceModifiersButton.classList.remove('hide'); - this.elements.incidenceModifiersButton.firstElementChild.classList.add('disabled'); - this.elements.conditionContainer.classList.remove('hide'); - this.elements.ignoreCaseCheckbox.classList.remove('hide'); - - this.elements.incidenceModifiersButton.firstElementChild.classList.add('disabled'); - this.elements.or.classList.add('disabled'); - this.elements.and.classList.add('disabled'); - } - - englishPosHandler() { - this.hideEverything(); - this.elements.englishPosBuilder.classList.remove('hide'); - this.elements.incidenceModifiersButton.classList.remove('hide'); - this.elements.conditionContainer.classList.remove('hide'); - this.elements.incidenceModifiersButton.firstElementChild.classList.remove('disabled'); - this.elements.or.classList.remove('disabled'); - this.elements.and.classList.remove('disabled'); - - // Resets materialize select dropdown - let selectInstance = M.FormSelect.getInstance(this.elements.englishPos); - selectInstance.input.value = 'English pos tagset'; - this.elements.englishPos.value = 'default'; - } - - germanPosHandler() { - this.hideEverything(); - this.elements.germanPosBuilder.classList.remove('hide'); - this.elements.incidenceModifiersButton.classList.remove('hide'); - this.elements.conditionContainer.classList.remove('hide'); - this.elements.incidenceModifiersButton.firstElementChild.classList.remove('disabled'); - this.elements.or.classList.remove('disabled'); - this.elements.and.classList.remove('disabled'); - - // Resets materialize select dropdown - let selectInstance = M.FormSelect.getInstance(this.elements.germanPos); - selectInstance.input.value = 'German pos tagset'; - this.elements.germanPos.value = 'default'; - } - - simplePosBuilder() { - this.hideEverything(); - this.elements.simplePosBuilder.classList.remove('hide'); - this.elements.incidenceModifiersButton.classList.remove('hide'); - this.elements.conditionContainer.classList.remove('hide'); - this.elements.simplePos.selectedIndex = 0; - this.elements.incidenceModifiersButton.firstElementChild.classList.remove('disabled'); - this.elements.or.classList.remove('disabled'); - this.elements.and.classList.remove('disabled'); - - // Resets materialize select dropdown - let selectInstance = M.FormSelect.getInstance(this.elements.simplePos); - selectInstance.input.value = 'simple_pos tagset'; - this.elements.simplePos.value = 'default'; - M.FormSelect.init( - selectInstance, - { - dropdownOptions: { - direction: 'bottom', - coverTrigger: false - } - } - ) - - } - - emptyTokenHandler() { - this.tokenChipFactory('empty token', '[]'); - this.elements.tokenQueryFilled = true; - this.hideEverything(); - this.elements.incidenceModifiersButton.classList.remove('hide'); - this.elements.incidenceModifiersButton.firstElementChild.classList.remove('disabled'); - - } - //#endregion Dropdown Select Handler - - //#region Options to edit your token - Wildcard Charakter, Option Group, Incidence Modifiers, Ignore Case, 'and', 'or' - - inputOptionHandler(elem) { - let input; - - if (this.elements.wordBuilder.classList.contains('hide') === false) { - input = this.elements.wordInput; - } else { - input = this.elements.lemmaInput; - } - - if (elem === this.elements.optionGroup) { - input.value += '(option1|option2)'; - let firstIndex = input.value.indexOf('option1'); - let lastIndex = firstIndex + 'option1'.length; - input.focus(); - input.setSelectionRange(firstIndex, lastIndex); - } else if (elem === this.elements.wildcardChar) { - input.value += '.'; - } - this.inputFieldHandler(); - } - - nSubmitHandler() { - let instance = M.Modal.getInstance(this.elements.exactlyN); - instance.close(); - - switch (this.elements.positionalAttr.value) { - case 'word': - this.elements.wordInput.value += ' {' + this.elements.nInput.value + '}'; - break; - case 'lemma': - this.elements.lemmaInput.value += ' {' + this.elements.nInput.value + '}'; - break; - case 'english-pos': - this.elements.tokenQueryFilled = true; - this.tokenChipFactory(`pos=${this.elements.englishPos.value}`, `pos="${this.elements.englishPos.value}"`); - this.tokenChipFactory('{' + this.elements.nInput.value + '}', '{' + this.elements.nInput.value + '}'); - this.elements.englishPosBuilder.classList.add('hide'); - this.elements.incidenceModifiersButton.classList.add('hide'); - break; - case 'german-pos': - this.elements.tokenQueryFilled = true; - this.tokenChipFactory(`pos=${this.elements.germanPos.value}`, `pos="${this.elements.germanPos.value}"`); - this.tokenChipFactory('{' + this.elements.nInput.value + '}', '{' + this.elements.nInput.value + '}'); - this.elements.germanPosBuilder.classList.add('hide'); - this.elements.incidenceModifiersButton.classList.add('hide'); - break; - case 'simple-pos-button': - this.elements.tokenQueryFilled = true; - this.tokenChipFactory(`simple_pos=${this.elements.simplePos.value}`, `simple_pos="${this.elements.simplePos.value}"`); - this.tokenChipFactory('{' + this.elements.nInput.value + '}', '{' + this.elements.nInput.value + '}'); - this.elements.simplePosBuilder.classList.add('hide'); - this.elements.incidenceModifiersButton.classList.add('hide'); - break; - case 'empty-token': - this.tokenChipFactory('{' + this.elements.nInput.value + '}', '{' + this.elements.nInput.value + '}'); - break; - default: - break; - } - - } - - nmSubmitHandler() { - let instance = M.Modal.getInstance(this.elements.betweenNM); - instance.close(); - - switch (this.elements.positionalAttr.value) { - case 'word': - this.elements.wordInput.value += `{${this.elements.nmInput.value}, ${this.elements.mInput.value}}`; - break; - case 'lemma': - this.elements.lemmaInput.value += `{${this.elements.nmInput.value}, ${this.elements.mInput.value}}`; - break; - case 'english-pos': - this.elements.tokenQueryFilled = true; - this.tokenChipFactory(`pos=${this.elements.englishPos.value}`, `pos="${this.elements.englishPos.value}"`); - this.tokenChipFactory(`{${this.elements.nmInput.value}, ${this.elements.mInput.value}}`, `{${this.elements.nmInput.value}, ${this.elements.mInput.value}}`); - this.elements.englishPosBuilder.classList.add('hide'); - this.elements.incidenceModifiersButton.classList.add('hide'); - break; - case 'german-pos': - this.elements.tokenQueryFilled = true; - this.tokenChipFactory(`pos=${this.elements.germanPos.value}`, `pos="${this.elements.germanPos.value}"`); - this.tokenChipFactory(`{${this.elements.nmInput.value}, ${this.elements.mInput.value}}`, `{${this.elements.nmInput.value}, ${this.elements.mInput.value}}`); - this.elements.germanPosBuilder.classList.add('hide'); - this.elements.incidenceModifiersButton.classList.add('hide'); - break; - case 'simple-pos-button': - this.elements.tokenQueryFilled = true; - this.tokenChipFactory(`simple_pos=${this.elements.simplePos.value}`, `simple_pos="${this.elements.simplePos.value}"`); - this.tokenChipFactory(`{${this.elements.nmInput.value}, ${this.elements.mInput.value}}`, `{${this.elements.nmInput.value}, ${this.elements.mInput.value}}`); - this.elements.simplePosBuilder.classList.add('hide'); - this.elements.incidenceModifiersButton.classList.add('hide'); - break; - case 'empty-token': - this.tokenChipFactory(`{${this.elements.nmInput.value}, ${this.elements.mInput.value}}`, `{${this.elements.nmInput.value}, ${this.elements.mInput.value}}`); - break; - default: - break; - } - } - - incidenceModifiersHandler(elem) { - // For word and lemma, the incidence modifiers are inserted in the input field. For the others, one or two chips are created which contain the respective value of the token and the incidence modifier. - if (this.elements.positionalAttr.value === 'empty-token') { - this.tokenChipFactory(elem.innerText, elem.dataset.token); - } else if (this.elements.positionalAttr.value === 'english-pos') { - this.tokenChipFactory(`pos=${this.elements.englishPos.value}`, `pos="${this.elements.englishPos.value}"`); - this.tokenChipFactory(elem.innerText, elem.dataset.token); - this.elements.englishPosBuilder.classList.add('hide'); - this.elements.incidenceModifiersButton.classList.add('hide'); - this.elements.tokenQueryFilled = true; - } else if (this.elements.positionalAttr.value === 'german-pos') { - this.tokenChipFactory(`pos=${this.elements.germanPos.value}`, `pos="${this.elements.germanPos.value}"`); - this.tokenChipFactory(elem.innerText, elem.dataset.token); - this.elements.germanPosBuilder.classList.add('hide'); - this.elements.incidenceModifiersButton.classList.add('hide'); - this.elements.tokenQueryFilled = true; - } else if (this.elements.positionalAttr.value === 'simple-pos-button') { - this.tokenChipFactory(`simple_pos=${this.elements.simplePos.value}`, `simple_pos="${this.elements.simplePos.value}"`); - this.tokenChipFactory(elem.innerText, elem.dataset.token); - this.elements.simplePosBuilder.classList.add('hide'); - this.elements.incidenceModifiersButton.classList.add('hide'); - this.elements.tokenQueryFilled = true; - } else { - let input; - - if (this.elements.wordBuilder.classList.contains('hide') === false) { - input = this.elements.wordInput; - } else { - input = this.elements.lemmaInput; - } - input.value += elem.dataset.token; - } - - } - - orHandler() { - this.conditionHandler('or', ' | '); - } - - andHandler() { - this.conditionHandler('and', ' & '); - } - - conditionHandler(conditionText, conditionQueryContent) { - this.hideEverything(); - let tokenQueryContent; - let tokenQueryText; - let c; - - if (this.elements.ignoreCase.checked) { - c = ' %c'; - } else { - c = ''; - } - - switch (this.elements.positionalAttr.value) { - case 'word': - tokenQueryContent = `word=${this.elements.wordInput.value}${c}`; - tokenQueryText = `word="${this.elements.wordInput.value}"${c}`; - this.elements.wordInput.value = ''; - break; - case 'lemma': - tokenQueryContent = `lemma=${this.elements.lemmaInput.value}${c}`; - tokenQueryText = `lemma="${this.elements.lemmaInput.value}"${c}`; - this.elements.lemmaInput.value = ''; - break; - case 'english-pos': - tokenQueryContent = `pos=${this.elements.englishPos.value}`; - tokenQueryText = `pos="${this.elements.englishPos.value}"`; - this.elements.englishPos.value = ''; - break; - case 'german-pos': - tokenQueryContent = `pos=${this.elements.germanPos.value}`; - tokenQueryText = `pos="${this.elements.germanPos.value}"`; - this.elements.germanPos.value = ''; - break; - case 'simple-pos-button': - tokenQueryContent = `simple_pos=${this.elements.simplePos.value}`; - tokenQueryText = `simple_pos="${this.elements.simplePos.value}"`; - this.elements.simplePos.value = ''; - break; - default: - this.wordBuilder(); - break; - } - - this.tokenChipFactory(tokenQueryContent, tokenQueryText); - this.tokenChipFactory(conditionText, conditionQueryContent); - this.wordBuilder(); - } - - //#endregion Options to edit your token - Wildcard Charakter, Option Group, Incidence Modifiers, Ignore Case, 'and', 'or' - - //#endregion Token Attribute Builder Functions - - - // ########################################################################## - // ############ Structural Attribute Builder Functions ###################### - // ########################################################################## - - //#region Structural Attribute Builder Functions - addSentence() { - this.hideEverything(); - if (this.elements.sentence.text === 'End Sentence') { - this.queryChipFactory('end-sentence', 'Sentence End', ''); - this.elements.sentence.innerHTML = 'Sentence'; - } else { - this.queryChipFactory('start-sentence', 'Sentence Start', ''); - this.elements.queryContent.push('sentence'); - this.elements.sentence.innerHTML = 'End Sentence'; - } - } - - addEntity() { - if (this.elements.entity.text === 'End Entity') { - let queryText; - if (this.elements.entityAnyType === false) { - queryText = ''; - } else { - queryText = ''; - } - this.queryChipFactory('end-entity', 'Entity End', queryText); - this.elements.entity.innerHTML = 'Entity'; - } else { - this.hideEverything(); - this.elements.entityBuilder.classList.remove('hide'); - window.location.href = '#entity-builder'; - } - } - - englishEntTypeHandler() { - this.queryChipFactory('start-entity', 'Entity Type=' + this.elements.englishEntType.value, ''); - this.elements.entity.innerHTML = 'End Entity'; - this.hideEverything(); - this.elements.entityAnyType = false; - - // Resets materialize select dropdown - let SelectInstance = M.FormSelect.getInstance(this.elements.englishEntType); - SelectInstance.input.value = 'English ent_type'; - this.elements.englishEntType.value = 'default'; - } - - germanEntTypeHandler() { - this.queryChipFactory('start-entity', 'Entity Type=' + this.elements.germanEntType.value, ''); - this.elements.entity.innerHTML = 'End Entity'; - this.hideEverything(); - this.elements.entityAnyType = false; - - // Resets materialize select dropdown - let SelectInstance = M.FormSelect.getInstance(this.elements.germanEntType); - SelectInstance.input.value = 'German ent_type'; - this.elements.germanEntType.value = 'default'; - } - - emptyEntityButton() { - this.queryChipFactory('start-empty-entity', 'Entity Start', ''); - this.elements.entity.innerHTML = 'End Entity'; - this.hideEverything(); - this.elements.entityAnyType = true; - } - - addTextAnnotation() { - this.hideEverything(); - this.elements.textAnnotationBuilder.classList.remove('hide'); - window.location.href = '#text-annotation-builder'; - - // Resets materialize select dropdown - let SelectInstance = M.FormSelect.getInstance(this.elements.textAnnotationOptions); - SelectInstance.input.value = 'address'; - this.elements.textAnnotationOptions.value = 'address'; - this.elements.textAnnotationInput.value= ''; - } - - textAnnotationSubmitHandler() { - if (this.elements.textAnnotationInput.value === '') { - this.elements.textAnnotationSubmit.classList.add('red'); - this.elements.noValueMetadataMessage.classList.remove('hide'); - setTimeout(() => { - this.elements.textAnnotationSubmit.classList.remove('red'); - }, 500); - setTimeout(() => { - this.elements.noValueMetadataMessage.classList.add('hide'); - }, 3000); - } else { - let queryText = `:: match.text_${this.elements.textAnnotationOptions.value}="${this.elements.textAnnotationInput.value}"`; - this.queryChipFactory('text-annotation', `${this.elements.textAnnotationOptions.value}=${this.elements.textAnnotationInput.value}`, queryText); - this.hideEverything(); - } - } - //#endregion Structural Attribute Builder Functions - -} diff --git a/app/static/js/Forms/CreateContributionForm.js b/app/static/js/Forms/CreateContributionForm.js deleted file mode 100644 index e7651ab0..00000000 --- a/app/static/js/Forms/CreateContributionForm.js +++ /dev/null @@ -1,18 +0,0 @@ -class CreateContributionForm extends Form { - static autoInit() { - let createContributionFormElements = document.querySelectorAll('.create-contribution-form'); - for (let createContributionFormElement of createContributionFormElements) { - new CreateContributionForm(createContributionFormElement); - } - } - - constructor(formElement) { - super(formElement); - - this.addEventListener('requestLoad', (event) => { - if (event.target.status === 201) { - window.location.href = event.target.getResponseHeader('Location'); - } - }); - } -} diff --git a/app/static/js/Forms/CreateCorpusFileForm.js b/app/static/js/Forms/CreateCorpusFileForm.js deleted file mode 100644 index ae8dba3b..00000000 --- a/app/static/js/Forms/CreateCorpusFileForm.js +++ /dev/null @@ -1,18 +0,0 @@ -class CreateCorpusFileForm extends Form { - static autoInit() { - let createCorpusFileFormElements = document.querySelectorAll('.create-corpus-file-form'); - for (let createCorpusFileFormElement of createCorpusFileFormElements) { - new CreateCorpusFileForm(createCorpusFileFormElement); - } - } - - constructor(formElement) { - super(formElement); - - this.addEventListener('requestLoad', (event) => { - if (event.target.status === 201) { - window.location.href = event.target.getResponseHeader('Location'); - } - }); - } -} diff --git a/app/static/js/Requests/admin/admin.js b/app/static/js/Requests/admin/admin.js deleted file mode 100644 index 77fdb6b1..00000000 --- a/app/static/js/Requests/admin/admin.js +++ /dev/null @@ -1,20 +0,0 @@ -/***************************************************************************** -* Admin * -* Fetch requests for /admin routes * -*****************************************************************************/ -Requests.admin = {}; - -Requests.admin.users = {}; - -Requests.admin.users.entity = {}; - -Requests.admin.users.entity.confirmed = {}; - -Requests.admin.users.entity.confirmed.update = (userId, value) => { - let input = `/admin/users/${userId}/confirmed`; - let init = { - method: 'PUT', - body: JSON.stringify(value) - }; - return Requests.JSONfetch(input, init); -}; diff --git a/app/static/js/Requests/contributions/contributions.js b/app/static/js/Requests/contributions/contributions.js deleted file mode 100644 index 2d9cf26a..00000000 --- a/app/static/js/Requests/contributions/contributions.js +++ /dev/null @@ -1,5 +0,0 @@ -/***************************************************************************** -* Contributions * -* Fetch requests for /contributions routes * -*****************************************************************************/ -Requests.contributions = {}; diff --git a/app/static/js/Requests/contributions/spacy_nlp_pipeline_models.js b/app/static/js/Requests/contributions/spacy_nlp_pipeline_models.js deleted file mode 100644 index e1422c1e..00000000 --- a/app/static/js/Requests/contributions/spacy_nlp_pipeline_models.js +++ /dev/null @@ -1,26 +0,0 @@ -/***************************************************************************** -* SpaCy NLP Pipeline Models * -* Fetch requests for /contributions/spacy-nlp-pipeline-models routes * -*****************************************************************************/ -Requests.contributions.spacy_nlp_pipeline_models = {}; - -Requests.contributions.spacy_nlp_pipeline_models.entity = {}; - -Requests.contributions.spacy_nlp_pipeline_models.entity.delete = (spacyNlpPipelineModelId) => { - let input = `/contributions/spacy-nlp-pipeline-models/${spacyNlpPipelineModelId}`; - let init = { - method: 'DELETE' - }; - return Requests.JSONfetch(input, init); -}; - -Requests.contributions.spacy_nlp_pipeline_models.entity.isPublic = {}; - -Requests.contributions.spacy_nlp_pipeline_models.entity.isPublic.update = (spacyNlpPipelineModelId, value) => { - let input = `/contributions/spacy-nlp-pipeline-models/${spacyNlpPipelineModelId}/is_public`; - let init = { - method: 'PUT', - body: JSON.stringify(value) - }; - return Requests.JSONfetch(input, init); -}; diff --git a/app/static/js/Requests/contributions/tesseract_ocr_pipeline_models.js b/app/static/js/Requests/contributions/tesseract_ocr_pipeline_models.js deleted file mode 100644 index 13feb42a..00000000 --- a/app/static/js/Requests/contributions/tesseract_ocr_pipeline_models.js +++ /dev/null @@ -1,26 +0,0 @@ -/***************************************************************************** -* Tesseract OCR Pipeline Models * -* Fetch requests for /contributions/tesseract-ocr-pipeline-models routes * -*****************************************************************************/ -Requests.contributions.tesseract_ocr_pipeline_models = {}; - -Requests.contributions.tesseract_ocr_pipeline_models.entity = {}; - -Requests.contributions.tesseract_ocr_pipeline_models.entity.delete = (tesseractOcrPipelineModelId) => { - let input = `/contributions/tesseract-ocr-pipeline-models/${tesseractOcrPipelineModelId}`; - let init = { - method: 'DELETE' - }; - return Requests.JSONfetch(input, init); -}; - -Requests.contributions.tesseract_ocr_pipeline_models.entity.isPublic = {}; - -Requests.contributions.tesseract_ocr_pipeline_models.entity.isPublic.update = (tesseractOcrPipelineModelId, value) => { - let input = `/contributions/tesseract-ocr-pipeline-models/${tesseractOcrPipelineModelId}/is_public`; - let init = { - method: 'PUT', - body: JSON.stringify(value) - }; - return Requests.JSONfetch(input, init); -}; diff --git a/app/static/js/Requests/corpora/corpora.js b/app/static/js/Requests/corpora/corpora.js deleted file mode 100644 index 3118a153..00000000 --- a/app/static/js/Requests/corpora/corpora.js +++ /dev/null @@ -1,53 +0,0 @@ -/***************************************************************************** -* Corpora * -* Fetch requests for /corpora routes * -*****************************************************************************/ -Requests.corpora = {}; - -Requests.corpora.entity = {}; - -Requests.corpora.entity.delete = (corpusId) => { - let input = `/corpora/${corpusId}`; - let init = { - method: 'DELETE' - }; - return Requests.JSONfetch(input, init); -}; - -Requests.corpora.entity.build = (corpusId) => { - let input = `/corpora/${corpusId}/build`; - let init = { - method: 'POST', - }; - return Requests.JSONfetch(input, init); -}; - -Requests.corpora.entity.generateShareLink = (corpusId, role, expiration) => { - let input = `/corpora/${corpusId}/generate-share-link`; - let init = { - method: 'POST', - body: JSON.stringify({role: role, expiration: expiration}) - }; - return Requests.JSONfetch(input, init); -}; - -Requests.corpora.entity.getStopwords = () => { - let input = `/corpora/stopwords`; - let init = { - method: 'GET' - }; - return Requests.JSONfetch(input, init); -}; - -Requests.corpora.entity.isPublic = {}; - -Requests.corpora.entity.isPublic.update = (corpusId, isPublic) => { - let input = `/corpora/${corpusId}/is_public`; - let init = { - method: 'PUT', - body: JSON.stringify(isPublic) - }; - return Requests.JSONfetch(input, init); -}; - - diff --git a/app/static/js/Requests/corpora/files.js b/app/static/js/Requests/corpora/files.js deleted file mode 100644 index 9ff9ba87..00000000 --- a/app/static/js/Requests/corpora/files.js +++ /dev/null @@ -1,15 +0,0 @@ -/***************************************************************************** -* Corpora * -* Fetch requests for /corpora//files routes * -*****************************************************************************/ -Requests.corpora.entity.files = {}; - -Requests.corpora.entity.files.ent = {}; - -Requests.corpora.entity.files.ent.delete = (corpusId, corpusFileId) => { - let input = `/corpora/${corpusId}/files/${corpusFileId}`; - let init = { - method: 'DELETE', - }; - return Requests.JSONfetch(input, init); -}; diff --git a/app/static/js/Requests/corpora/followers.js b/app/static/js/Requests/corpora/followers.js deleted file mode 100644 index f7f7877f..00000000 --- a/app/static/js/Requests/corpora/followers.js +++ /dev/null @@ -1,35 +0,0 @@ -/***************************************************************************** -* Corpora * -* Fetch requests for /corpora//followers routes * -*****************************************************************************/ -Requests.corpora.entity.followers = {}; - -Requests.corpora.entity.followers.add = (corpusId, usernames) => { - let input = `/corpora/${corpusId}/followers`; - let init = { - method: 'POST', - body: JSON.stringify(usernames) - }; - return Requests.JSONfetch(input, init); -}; - -Requests.corpora.entity.followers.entity = {}; - -Requests.corpora.entity.followers.entity.delete = (corpusId, followerId) => { - let input = `/corpora/${corpusId}/followers/${followerId}`; - let init = { - method: 'DELETE', - }; - return Requests.JSONfetch(input, init); -}; - -Requests.corpora.entity.followers.entity.role = {}; - -Requests.corpora.entity.followers.entity.role.update = (corpusId, followerId, value) => { - let input = `/corpora/${corpusId}/followers/${followerId}/role`; - let init = { - method: 'PUT', - body: JSON.stringify(value) - }; - return Requests.JSONfetch(input, init); -}; diff --git a/app/static/js/Requests/jobs/jobs.js b/app/static/js/Requests/jobs/jobs.js deleted file mode 100644 index 64e523db..00000000 --- a/app/static/js/Requests/jobs/jobs.js +++ /dev/null @@ -1,31 +0,0 @@ -/***************************************************************************** -* Jobs * -* Fetch requests for /jobs routes * -*****************************************************************************/ -Requests.jobs = {}; - -Requests.jobs.entity = {}; - -Requests.jobs.entity.delete = (jobId) => { - let input = `/jobs/${jobId}`; - let init = { - method: 'DELETE' - }; - return Requests.JSONfetch(input, init); -} - -Requests.jobs.entity.log = (jobId) => { - let input = `/jobs/${jobId}/log`; - let init = { - method: 'GET' - }; - return Requests.JSONfetch(input, init); -} - -Requests.jobs.entity.restart = (jobId) => { - let input = `/jobs/${jobId}/restart`; - let init = { - method: 'POST' - }; - return Requests.JSONfetch(input, init); -} diff --git a/app/static/js/Requests/users/settings.js b/app/static/js/Requests/users/settings.js deleted file mode 100644 index 609ecb35..00000000 --- a/app/static/js/Requests/users/settings.js +++ /dev/null @@ -1,17 +0,0 @@ -/***************************************************************************** -* Settings * -* Fetch requests for /users//settings routes * -*****************************************************************************/ -Requests.users.entity.settings = {}; - -Requests.users.entity.settings.profilePrivacy = {}; - -Requests.users.entity.settings.profilePrivacy.update = (userId, profilePrivacySetting, enabled) => { - let input = `/users/${userId}/settings/profile-privacy/${profilePrivacySetting}`; - let init = { - method: 'PUT', - body: JSON.stringify(enabled) - }; - return Requests.JSONfetch(input, init); -}; - diff --git a/app/static/js/Requests/users/users.js b/app/static/js/Requests/users/users.js deleted file mode 100644 index 4baf4717..00000000 --- a/app/static/js/Requests/users/users.js +++ /dev/null @@ -1,35 +0,0 @@ -/***************************************************************************** -* Users * -* Fetch requests for /users routes * -*****************************************************************************/ -Requests.users = {}; - -Requests.users.entity = {}; - -Requests.users.entity.delete = (userId) => { - let input = `/users/${userId}`; - let init = { - method: 'DELETE' - }; - return Requests.JSONfetch(input, init); -}; - -Requests.users.entity.acceptTermsOfUse = () => { - let input = `/users/accept-terms-of-use`; - let init = { - method: 'POST' - }; - return Requests.JSONfetch(input, init); -}; - - -Requests.users.entity.avatar = {}; - -Requests.users.entity.avatar.delete = (userId) => { - let input = `/users/${userId}/avatar`; - let init = { - method: 'DELETE' - }; - return Requests.JSONfetch(input, init); -} - diff --git a/app/static/js/app.js b/app/static/js/app.js new file mode 100644 index 00000000..57dd7ca1 --- /dev/null +++ b/app/static/js/app.js @@ -0,0 +1,204 @@ +nopaque.App = class App { + constructor() { + this.data = { + promises: {getUser: {}, subscribeUser: {}}, + users: {}, + }; + this.socket = io({transports: ['websocket'], upgrade: false}); + this.socket.on('PATCH', (patch) => {this.onPatch(patch);}); + } + + getUser(userId) { + if (userId in this.data.promises.getUser) { + return this.data.promises.getUser[userId]; + } + + this.data.promises.getUser[userId] = new Promise((resolve, reject) => { + this.socket.emit('GET /users/', userId, (response) => { + if (response.status === 200) { + this.data.users[userId] = response.body; + resolve(this.data.users[userId]); + } else { + reject(`[${response.status}] ${response.statusText}`); + } + }); + }); + + return this.data.promises.getUser[userId]; + } + + subscribeUser(userId) { + if (userId in this.data.promises.subscribeUser) { + return this.data.promises.subscribeUser[userId]; + } + + this.data.promises.subscribeUser[userId] = new Promise((resolve, reject) => { + this.socket.emit('SUBSCRIBE /users/', userId, (response) => { + if (response.status !== 200) { + reject(response); + return; + } + resolve(response); + }); + }); + + return this.data.promises.subscribeUser[userId]; + } + + flash(message, category) { + let iconPrefix = ''; + switch (category) { + case 'corpus': { + iconPrefix = 'book'; + break; + } + case 'error': { + iconPrefix = 'error'; + break; + } + case 'job': { + iconPrefix = 'J'; + break; + } + case 'settings': { + iconPrefix = 'settings'; + break; + } + default: { + iconPrefix = 'notifications'; + break; + } + } + let toast = M.toast( + { + html: ` + ${iconPrefix}${message} + + `.trim() + } + ); + let toastCloseActionElement = toast.el.querySelector('.action-button[data-action="close"]'); + toastCloseActionElement.addEventListener('click', () => {toast.dismiss();}); + } + + onPatch(patch) { + // Filter Patch to only include operations on users that are initialized + let regExp = new RegExp(`^/users/(${Object.keys(this.data.users).join('|')})`); + let filteredPatch = patch.filter(operation => regExp.test(operation.path)); + + // Handle job status updates + let subRegExp = new RegExp(`^/users/([A-Za-z0-9]*)/jobs/([A-Za-z0-9]*)/status$`); + let subFilteredPatch = filteredPatch + .filter((operation) => {return operation.op === 'replace';}) + .filter((operation) => {return subRegExp.test(operation.path);}); + for (let operation of subFilteredPatch) { + let [match, userId, jobId] = operation.path.match(subRegExp); + this.flash(`[${this.data.users[userId].jobs[jobId].title}] New status: `, 'job'); + } + + // Apply Patch + jsonpatch.applyPatch(this.data, filteredPatch); + } + + init() { + this.initUi(); + } + + initUi() { + /* Pre-Initialization fixes */ + // #region + + // Flask-WTF sets the standard HTML maxlength Attribute on input/textarea + // elements to specify their maximum length (in characters). Unfortunatly + // Materialize won't recognize the maxlength Attribute, instead it uses + // the data-length Attribute. It's conversion time :) + for (let elem of document.querySelectorAll('input[maxlength], textarea[maxlength]')) { + elem.dataset.length = elem.getAttribute('maxlength'); + elem.removeAttribute('maxlength'); + } + + // To work around some limitations with the Form setup of Flask-WTF. + // HTML option elements with an empty value are considered as placeholder + // elements. The user should not be able to actively select these options. + // So they get the disabled attribute. + for (let optionElement of document.querySelectorAll('option[value=""]')) { + optionElement.disabled = true; + } + + // TODO: Check why we are doing this. + for (let optgroupElement of document.querySelectorAll('optgroup[label=""]')) { + for (let c of optgroupElement.children) { + optgroupElement.parentElement.insertAdjacentElement('afterbegin', c); + } + optgroupElement.remove(); + } + // #endregion + + + /* Initialize Materialize Components */ + // #region + + // Automatically initialize Materialize Components that do not require + // additional configuration. + M.AutoInit(); + + // CharacterCounters + // Materialize didn't include the CharacterCounter plugin within the + // AutoInit method (maybe they forgot it?). Anyway... We do it here. :) + M.CharacterCounter.init(document.querySelectorAll('input[data-length]:not(.no-autoinit), textarea[data-length]:not(.no-autoinit)')); + + // Header navigation "more" Dropdown. + M.Dropdown.init( + document.querySelector('#nav-more-dropdown-trigger'), + { + alignment: 'right', + constrainWidth: false, + coverTrigger: false + } + ); + + // Manual modal + M.Modal.init( + document.querySelector('#manual-modal'), + { + onOpenStart: (modalElement, modalTriggerElement) => { + if ('manualModalChapter' in modalTriggerElement.dataset) { + let manualModalTocElement = document.querySelector('#manual-modal-toc'); + let manualModalToc = M.Tabs.getInstance(manualModalTocElement); + manualModalToc.select(modalTriggerElement.dataset.manualModalChapter); + // TODO: Make this work. + // if ('manualModalChapterAnchor' in modalTriggerElement.dataset) { + // let manualModalChapterAnchor = document.querySelector(`#${modalTriggerElement.dataset.manualModalChapterAnchor}`); + // let xCoord = manualModalChapterAnchor.getBoundingClientRect().left; + // let yCoord = manualModalChapterAnchor.getBoundingClientRect().top; + // let modalContentElement = modalElement.querySelector('.modal-content'); + // modalContentElement.scroll(xCoord, yCoord); + // } + } + } + } + ); + + // Terms of use modal + M.Modal.init( + document.querySelector('#terms-of-use-modal'), + { + dismissible: false, + onCloseEnd: (modalElement) => { + nopaque.requests.users.entity.acceptTermsOfUse(); + } + } + ); + // #endregion + + + /* Initialize nopaque Components */ + // #region + nopaque.resource_displays.AutoInit(); + nopaque.resource_lists.AutoInit(); + nopaque.forms.AutoInit(); + // #endregion + } +}; diff --git a/app/static/js/CorpusAnalysis/CorpusAnalysisApp.js b/app/static/js/corpus-analysis/app.js similarity index 90% rename from app/static/js/CorpusAnalysis/CorpusAnalysisApp.js rename to app/static/js/corpus-analysis/app.js index d6274f32..522dc7f4 100644 --- a/app/static/js/CorpusAnalysis/CorpusAnalysisApp.js +++ b/app/static/js/corpus-analysis/app.js @@ -1,4 +1,4 @@ -class CorpusAnalysisApp { +nopaque.corpus_analysis.App = class App { constructor(corpusId) { this.corpusId = corpusId; @@ -6,10 +6,10 @@ class CorpusAnalysisApp { // HTML elements this.elements = { - container: document.querySelector('#corpus-analysis-app-container'), - extensionCards: document.querySelector('#corpus-analysis-app-extension-cards'), - extensionTabs: document.querySelector('#corpus-analysis-app-extension-tabs'), - initModal: document.querySelector('#corpus-analysis-app-init-modal') + container: document.querySelector('#corpus-analysis-container'), + extensionCards: document.querySelector('#corpus-analysis-extension-cards'), + extensionTabs: document.querySelector('#corpus-analysis-extension-tabs'), + initModal: document.querySelector('#corpus-analysis-init-modal') }; // Materialize elements this.elements.m = { @@ -25,12 +25,12 @@ class CorpusAnalysisApp { async init() { this.disableActionElements(); this.elements.m.initModal.open(); - + try { // Setup CQi over SocketIO connection and gather data from the CQPServer const statusTextElement = this.elements.initModal.querySelector('.status-text'); statusTextElement.innerText = 'Creating CQi over SocketIO client...'; - const cqiClient = new cqi.CQiClient('/cqi_over_sio'); + const cqiClient = new nopaque.corpus_analysis.cqi.Client('/cqi_over_sio'); statusTextElement.innerText += ' Done'; statusTextElement.innerHTML = 'Waiting for the CQP server...'; const response = await cqiClient.api.socket.emitWithAck('init', this.corpusId); diff --git a/app/static/js/CorpusAnalysis/CorpusAnalysisConcordance.js b/app/static/js/corpus-analysis/concordance-extension.js similarity index 86% rename from app/static/js/CorpusAnalysis/CorpusAnalysisConcordance.js rename to app/static/js/corpus-analysis/concordance-extension.js index 6af78603..46a5c32a 100644 --- a/app/static/js/CorpusAnalysis/CorpusAnalysisConcordance.js +++ b/app/static/js/corpus-analysis/concordance-extension.js @@ -1,4 +1,4 @@ -class CorpusAnalysisConcordance { +nopaque.corpus_analysis.ConcordanceExtension = class ConcordanceExtension { name = 'Concordance'; constructor(app) { @@ -7,33 +7,38 @@ class CorpusAnalysisConcordance { this.data = {}; this.elements = { - // TODO: Prefix elements with "corpus-analysis-app-" - container: document.querySelector('#concordance-extension-container'), - error: document.querySelector('#concordance-extension-error'), - form: document.querySelector('#concordance-extension-form'), - progress: document.querySelector('#concordance-extension-progress'), - subcorpusInfo: document.querySelector('#concordance-extension-subcorpus-info'), - subcorpusActions: document.querySelector('#concordance-extension-subcorpus-actions'), - subcorpusItems: document.querySelector('#concordance-extension-subcorpus-items'), - subcorpusList: document.querySelector('#concordance-extension-subcorpus-list'), - subcorpusPagination: document.querySelector('#concordance-extension-subcorpus-pagination') + container: document.querySelector(`#corpus-analysis-concordance-container`), + error: document.querySelector(`#corpus-analysis-concordance-error`), + userInterfaceForm: document.querySelector(`#corpus-analysis-concordance-user-interface-form`), + expertModeForm: document.querySelector(`#corpus-analysis-concordance-expert-mode-form`), + queryBuilderForm: document.querySelector(`#corpus-analysis-concordance-query-builder-form`), + progress: document.querySelector(`#corpus-analysis-concordance-progress`), + subcorpusInfo: document.querySelector(`#corpus-analysis-concordance-subcorpus-info`), + subcorpusActions: document.querySelector(`#corpus-analysis-concordance-subcorpus-actions`), + subcorpusItems: document.querySelector(`#corpus-analysis-concordance-subcorpus-items`), + subcorpusList: document.querySelector(`#corpus-analysis-concordance-subcorpus-list`), + subcorpusPagination: document.querySelector(`#corpus-analysis-concordance-subcorpus-pagination`) }; this.settings = { - context: parseInt(this.elements.form['context'].value), - perPage: parseInt(this.elements.form['per-page'].value), + context: parseInt(this.elements.userInterfaceForm['context'].value), + perPage: parseInt(this.elements.userInterfaceForm['per-page'].value), selectedSubcorpus: undefined, - textStyle: parseInt(this.elements.form['text-style'].value), - tokenRepresentation: this.elements.form['token-representation'].value + textStyle: parseInt(this.elements.userInterfaceForm['text-style'].value), + tokenRepresentation: this.elements.userInterfaceForm['token-representation'].value }; this.app.registerExtension(this); } - async submitForm() { + async submitForm(queryModeId) { this.app.disableActionElements(); - let query = this.elements.form.query.value.trim(); - let subcorpusName = this.elements.form['subcorpus-name'].value; + let queryBuilderQuery = nopaque.Utils.unescape(document.querySelector('#corpus-analysis-concordance-query-preview').innerHTML.trim()); + let expertModeQuery = this.elements.expertModeForm.query.value.trim(); + let query = queryModeId === 'corpus-analysis-concordance-expert-mode-form' ? expertModeQuery : queryBuilderQuery; + let form = queryModeId === 'corpus-analysis-concordance-expert-mode-form' ? this.elements.expertModeForm : this.elements.queryBuilderForm; + + let subcorpusName = form['subcorpus-name'].value; this.elements.error.innerText = ''; this.elements.error.classList.add('hide'); this.elements.progress.classList.remove('hide'); @@ -72,25 +77,29 @@ class CorpusAnalysisConcordance { this.data.corpus = this.app.data.corpus; this.data.subcorpora = {}; // Add event listeners - this.elements.form.addEventListener('submit', (event) => { + this.elements.expertModeForm.addEventListener('submit', (event) => { event.preventDefault(); - this.submitForm(); + this.submitForm(this.elements.expertModeForm.id); }); - this.elements.form.addEventListener('change', (event) => { - if (event.target === this.elements.form['context']) { - this.settings.context = parseInt(this.elements.form['context'].value); + this.elements.queryBuilderForm.addEventListener('submit', (event) => { + event.preventDefault(); + this.submitForm(this.elements.queryBuilderForm.id); + }); + this.elements.userInterfaceForm.addEventListener('change', (event) => { + if (event.target === this.elements.userInterfaceForm['context']) { + this.settings.context = parseInt(this.elements.userInterfaceForm['context'].value); this.submitForm(); } - if (event.target === this.elements.form['per-page']) { - this.settings.perPage = parseInt(this.elements.form['per-page'].value); + if (event.target === this.elements.userInterfaceForm['per-page']) { + this.settings.perPage = parseInt(this.elements.userInterfaceForm['per-page'].value); this.submitForm(); } - if (event.target === this.elements.form['text-style']) { - this.settings.textStyle = parseInt(this.elements.form['text-style'].value); + if (event.target === this.elements.userInterfaceForm['text-style']) { + this.settings.textStyle = parseInt(this.elements.userInterfaceForm['text-style'].value); this.setTextStyle(); } - if (event.target === this.elements.form['token-representation']) { - this.settings.tokenRepresentation = this.elements.form['token-representation'].value; + if (event.target === this.elements.userInterfaceForm['token-representation']) { + this.settings.tokenRepresentation = this.elements.userInterfaceForm['token-representation'].value; this.setTokenRepresentation(); } }); @@ -162,11 +171,11 @@ class CorpusAnalysisConcordance { this.elements.subcorpusActions.querySelector('.subcorpus-export-trigger').addEventListener('click', (event) => { event.preventDefault(); let subcorpus = this.data.subcorpora[this.settings.selectedSubcorpus]; - let modalElementId = Utils.generateElementId('export-subcorpus-modal-'); - let exportFormatSelectElementId = Utils.generateElementId('export-format-select-'); - let exportSelectedMatchesOnlyCheckboxElementId = Utils.generateElementId('export-selected-matches-only-checkbox-'); - let exportFileNameInputElementId = Utils.generateElementId('export-file-name-input-'); - let modalElement = Utils.HTMLToElement( + let modalElementId = nopaque.Utils.generateElementId('export-subcorpus-modal-'); + let exportFormatSelectElementId = nopaque.Utils.generateElementId('export-format-select-'); + let exportSelectedMatchesOnlyCheckboxElementId = nopaque.Utils.generateElementId('export-selected-matches-only-checkbox-'); + let exportFileNameInputElementId = nopaque.Utils.generateElementId('export-file-name-input-'); + let modalElement = nopaque.Utils.HTMLToElement( `