From cb2b64fa9dc4b8b38711737a5ed623601e73eed3 Mon Sep 17 00:00:00 2001 From: Patrick Jentsch Date: Fri, 5 Jun 2020 14:42:04 +0200 Subject: [PATCH] integrate nopaque repo --- .env.tpl | 6 + .gitignore | 5 - .gitlab-ci.yml | 55 --- README.md | 109 ++--- daemon/Dockerfile | 40 ++ daemon/decorators.py | 14 + daemon/docker-entrypoint.sh | 8 + {tests => daemon/logger}/__init__.py | 0 daemon/logger/logger.py | 26 + daemon/nopaqued.bak.py | 455 ++++++++++++++++++ daemon/nopaqued.py | 18 + daemon/notify/__init__.py | 0 daemon/notify/notification.py | 27 ++ daemon/notify/service.py | 41 ++ .../notification_messages/notification.html | 15 + .../notification_messages/notification.txt | 10 + daemon/requirements.txt | 3 + daemon/tasks/Models.py | 52 ++ daemon/tasks/__init__.py | 22 + daemon/tasks/check_corpora.py | 134 ++++++ daemon/tasks/check_jobs.py | 152 ++++++ daemon/tasks/notify.py | 87 ++++ docker-compose.yml | 48 +- nopaque.env.tpl | 27 +- Dockerfile => web/Dockerfile | 6 +- {app => web/app}/__init__.py | 0 {app => web/app}/admin/__init__.py | 0 {app => web/app}/admin/forms.py | 0 {app => web/app}/admin/tables.py | 0 {app => web/app}/admin/views.py | 0 {app => web/app}/auth/__init__.py | 0 {app => web/app}/auth/forms.py | 0 {app => web/app}/auth/views.py | 0 {app => web/app}/corpora/__init__.py | 0 {app => web/app}/corpora/events.py | 0 {app => web/app}/corpora/forms.py | 0 {app => web/app}/corpora/tasks.py | 0 {app => web/app}/corpora/views.py | 0 {app => web/app}/decorators.py | 0 {app => web/app}/email.py | 0 {app => web/app}/events.py | 0 {app => web/app}/jobs/__init__.py | 0 {app => web/app}/jobs/forms.py | 0 {app => web/app}/jobs/tasks.py | 0 {app => web/app}/jobs/views.py | 0 {app => web/app}/main/__init__.py | 0 {app => web/app}/main/errors.py | 0 {app => web/app}/main/forms.py | 0 {app => web/app}/main/views.py | 0 {app => web/app}/models.py | 0 {app => web/app}/profile/__init__.py | 0 {app => web/app}/profile/forms.py | 0 {app => web/app}/profile/tasks.py | 0 {app => web/app}/profile/views.py | 0 {app => web/app}/services/__init__.py | 0 {app => web/app}/services/views.py | 0 .../app}/static/css/Materialize/LICENSE | 0 .../css/Materialize/materialize.min.css | 0 {app => web/app}/static/css/nopaque.css | 0 .../fonts/Material_design_icons/LICENSE | 0 .../MaterialIcons-Regular.eot | Bin .../MaterialIcons-Regular.ttf | Bin .../MaterialIcons-Regular.woff | Bin .../MaterialIcons-Regular.woff2 | Bin .../Material_design_icons/material-icons.css | 0 {app => web/app}/static/images/logo_-_dfg.gif | Bin .../app}/static/images/logo_-_sfb_1288.png | Bin .../app}/static/images/nopaque_logo.png | Bin .../parallax_hq/book_text_read_paper.jpg | Bin .../parallax_hq/books_antique_book_old.jpg | Bin .../concept_document_focus_letter.jpg | Bin .../images/parallax_hq/text_data_wide.png | Bin .../parallax_lq/01_books_antique_book_old.jpg | Bin .../02_concept_document_focus_letter.jpg | Bin .../images/parallax_lq/03_text_data_wide.png | Bin .../parallax_lq/04_german_text_book_paper.jpg | Bin .../parallax_lq/05_chapter_book_text_tale.jpg | Bin .../static/images/parallax_lq/bible_text.jpg | Bin .../static/images/parallax_lq/text_data.png | Bin {app => web/app}/static/images/qr_-_inf.svg | 0 .../app}/static/images/server_activity.png | Bin .../app}/static/images/sfb_background.jpeg | Bin .../app}/static/js/Dark_Reader/LICENSE | 0 .../app}/static/js/Dark_Reader/darkreader.js | 0 .../app}/static/js/JSONPatch.js/LICENSE | 0 .../static/js/JSONPatch.js/jsonpatch.min.js | 0 {app => web/app}/static/js/List.js/LICENSE | 0 .../app}/static/js/List.js/list.min.js | 0 .../app}/static/js/Materialize/LICENSE | 0 .../static/js/Materialize/materialize.min.js | 0 {app => web/app}/static/js/Socket.IO/LICENSE | 0 .../static/js/Socket.IO/socket.io.slim.js | 0 .../static/js/Socket.IO/socket.io.slim.js.map | 0 .../static/js/nopaque.CorpusAnalysisClient.js | 0 {app => web/app}/static/js/nopaque.Results.js | 0 .../app}/static/js/nopaque.callbacks.js | 0 {app => web/app}/static/js/nopaque.js | 0 {app => web/app}/static/js/nopaque.lists.js | 0 {app => web/app}/templates/403.html.j2 | 0 {app => web/app}/templates/404.html.j2 | 0 {app => web/app}/templates/500.html.j2 | 0 .../app}/templates/admin/edit_user.html.j2 | 0 .../app}/templates/admin/index.html.j2 | 0 {app => web/app}/templates/admin/user.html.j2 | 0 .../app}/templates/auth/email/confirm.html.j2 | 0 .../app}/templates/auth/email/confirm.txt.j2 | 0 .../auth/email/reset_password.html.j2 | 0 .../auth/email/reset_password.txt.j2 | 0 {app => web/app}/templates/auth/login.html.j2 | 0 .../app}/templates/auth/register.html.j2 | 0 .../templates/auth/reset_password.html.j2 | 0 .../auth/reset_password_request.html.j2 | 0 .../app}/templates/auth/unconfirmed.html.j2 | 0 .../app}/templates/corpora/add_corpus.html.j2 | 0 .../templates/corpora/add_corpus_file.html.j2 | 0 .../templates/corpora/analyse_corpus.html.j2 | 0 .../app}/templates/corpora/corpus.html.j2 | 0 .../corpora/edit_corpus_file.html.j2 | 0 {app => web/app}/templates/jobs/job.html.j2 | 0 .../app}/templates/macros/materialize.html.j2 | 0 .../app}/templates/main/dashboard.html.j2 | 0 .../app}/templates/main/feedback.html.j2 | 0 {app => web/app}/templates/main/index.html.j2 | 0 .../app}/templates/main/poster.html.j2 | 0 .../templates/main/privacy_policy.html.j2 | 0 {app => web/app}/templates/nopaque.html.j2 | 0 .../app}/templates/profile/settings.html.j2 | 0 .../services/corpus_analysis.html.j2 | 0 .../templates/services/file-setup.html.j2 | 0 .../app}/templates/services/nlp.html.j2 | 0 .../app}/templates/services/ocr.html.j2 | 0 .../app}/templates/services/roadmap.html.j2 | 0 config.py => web/config.py | 17 +- .../docker-entrypoint.sh | 0 {migrations => web/migrations}/README | 0 {migrations => web/migrations}/alembic.ini | 0 {migrations => web/migrations}/env.py | 0 {migrations => web/migrations}/script.py.mako | 0 .../migrations}/versions/099037c4aa06_.py | 0 .../migrations}/versions/0aa38a7973c5_.py | 0 .../migrations}/versions/10a92d8f4616_.py | 0 .../migrations}/versions/1210adfe1e34_.py | 0 .../migrations}/versions/3d9a20b8b26c_.py | 0 .../migrations}/versions/421ba4373e50_.py | 0 .../migrations}/versions/4638e6509e13_.py | 0 .../migrations}/versions/471aa04c1a92_.py | 0 .../migrations}/versions/4886241e0f5d_.py | 0 .../migrations}/versions/49a42c69e523_.py | 0 .../migrations}/versions/5ba6786a847e_.py | 0 .../migrations}/versions/62233e0cb2c7_.py | 0 .../migrations}/versions/6227310c2112_.py | 0 .../migrations}/versions/66253783654f_.py | 0 .../migrations}/versions/68772b6560c3_.py | 0 .../migrations}/versions/7378391345fa_.py | 0 .../migrations}/versions/776761fb7466_.py | 0 .../migrations}/versions/abf60427ff84_.py | 0 .../migrations}/versions/da9fd175af8c_.py | 0 .../migrations}/versions/ded5a37f8a7b_.py | 0 nopaque.py => web/nopaque.py | 3 +- requirements.txt => web/requirements.txt | 0 web/tests/__init__.py | 0 {tests => web/tests}/test_basics.py | 0 {tests => web/tests}/test_client.py | 0 {tests => web/tests}/test_user_model.py | 0 164 files changed, 1212 insertions(+), 168 deletions(-) create mode 100644 .env.tpl delete mode 100644 .gitlab-ci.yml create mode 100644 daemon/Dockerfile create mode 100644 daemon/decorators.py create mode 100644 daemon/docker-entrypoint.sh rename {tests => daemon/logger}/__init__.py (100%) create mode 100644 daemon/logger/logger.py create mode 100644 daemon/nopaqued.bak.py create mode 100644 daemon/nopaqued.py create mode 100644 daemon/notify/__init__.py create mode 100644 daemon/notify/notification.py create mode 100644 daemon/notify/service.py create mode 100644 daemon/notify/templates/notification_messages/notification.html create mode 100644 daemon/notify/templates/notification_messages/notification.txt create mode 100644 daemon/requirements.txt create mode 100644 daemon/tasks/Models.py create mode 100644 daemon/tasks/__init__.py create mode 100644 daemon/tasks/check_corpora.py create mode 100644 daemon/tasks/check_jobs.py create mode 100644 daemon/tasks/notify.py rename Dockerfile => web/Dockerfile (80%) rename {app => web/app}/__init__.py (100%) rename {app => web/app}/admin/__init__.py (100%) rename {app => web/app}/admin/forms.py (100%) rename {app => web/app}/admin/tables.py (100%) rename {app => web/app}/admin/views.py (100%) rename {app => web/app}/auth/__init__.py (100%) rename {app => web/app}/auth/forms.py (100%) rename {app => web/app}/auth/views.py (100%) rename {app => web/app}/corpora/__init__.py (100%) rename {app => web/app}/corpora/events.py (100%) rename {app => web/app}/corpora/forms.py (100%) rename {app => web/app}/corpora/tasks.py (100%) rename {app => web/app}/corpora/views.py (100%) rename {app => web/app}/decorators.py (100%) rename {app => web/app}/email.py (100%) rename {app => web/app}/events.py (100%) rename {app => web/app}/jobs/__init__.py (100%) rename {app => web/app}/jobs/forms.py (100%) rename {app => web/app}/jobs/tasks.py (100%) rename {app => web/app}/jobs/views.py (100%) rename {app => web/app}/main/__init__.py (100%) rename {app => web/app}/main/errors.py (100%) rename {app => web/app}/main/forms.py (100%) rename {app => web/app}/main/views.py (100%) rename {app => web/app}/models.py (100%) rename {app => web/app}/profile/__init__.py (100%) rename {app => web/app}/profile/forms.py (100%) rename {app => web/app}/profile/tasks.py (100%) rename {app => web/app}/profile/views.py (100%) rename {app => web/app}/services/__init__.py (100%) rename {app => web/app}/services/views.py (100%) rename {app => web/app}/static/css/Materialize/LICENSE (100%) rename {app => web/app}/static/css/Materialize/materialize.min.css (100%) rename {app => web/app}/static/css/nopaque.css (100%) rename {app => web/app}/static/fonts/Material_design_icons/LICENSE (100%) rename {app => web/app}/static/fonts/Material_design_icons/MaterialIcons-Regular.eot (100%) rename {app => web/app}/static/fonts/Material_design_icons/MaterialIcons-Regular.ttf (100%) rename {app => web/app}/static/fonts/Material_design_icons/MaterialIcons-Regular.woff (100%) rename {app => web/app}/static/fonts/Material_design_icons/MaterialIcons-Regular.woff2 (100%) rename {app => web/app}/static/fonts/Material_design_icons/material-icons.css (100%) rename {app => web/app}/static/images/logo_-_dfg.gif (100%) rename {app => web/app}/static/images/logo_-_sfb_1288.png (100%) rename {app => web/app}/static/images/nopaque_logo.png (100%) rename {app => web/app}/static/images/parallax_hq/book_text_read_paper.jpg (100%) rename {app => web/app}/static/images/parallax_hq/books_antique_book_old.jpg (100%) rename {app => web/app}/static/images/parallax_hq/concept_document_focus_letter.jpg (100%) rename {app => web/app}/static/images/parallax_hq/text_data_wide.png (100%) rename {app => web/app}/static/images/parallax_lq/01_books_antique_book_old.jpg (100%) rename {app => web/app}/static/images/parallax_lq/02_concept_document_focus_letter.jpg (100%) rename {app => web/app}/static/images/parallax_lq/03_text_data_wide.png (100%) rename {app => web/app}/static/images/parallax_lq/04_german_text_book_paper.jpg (100%) rename {app => web/app}/static/images/parallax_lq/05_chapter_book_text_tale.jpg (100%) rename {app => web/app}/static/images/parallax_lq/bible_text.jpg (100%) rename {app => web/app}/static/images/parallax_lq/text_data.png (100%) rename {app => web/app}/static/images/qr_-_inf.svg (100%) rename {app => web/app}/static/images/server_activity.png (100%) rename {app => web/app}/static/images/sfb_background.jpeg (100%) rename {app => web/app}/static/js/Dark_Reader/LICENSE (100%) rename {app => web/app}/static/js/Dark_Reader/darkreader.js (100%) rename {app => web/app}/static/js/JSONPatch.js/LICENSE (100%) rename {app => web/app}/static/js/JSONPatch.js/jsonpatch.min.js (100%) rename {app => web/app}/static/js/List.js/LICENSE (100%) rename {app => web/app}/static/js/List.js/list.min.js (100%) rename {app => web/app}/static/js/Materialize/LICENSE (100%) rename {app => web/app}/static/js/Materialize/materialize.min.js (100%) rename {app => web/app}/static/js/Socket.IO/LICENSE (100%) rename {app => web/app}/static/js/Socket.IO/socket.io.slim.js (100%) rename {app => web/app}/static/js/Socket.IO/socket.io.slim.js.map (100%) rename {app => web/app}/static/js/nopaque.CorpusAnalysisClient.js (100%) rename {app => web/app}/static/js/nopaque.Results.js (100%) rename {app => web/app}/static/js/nopaque.callbacks.js (100%) rename {app => web/app}/static/js/nopaque.js (100%) rename {app => web/app}/static/js/nopaque.lists.js (100%) rename {app => web/app}/templates/403.html.j2 (100%) rename {app => web/app}/templates/404.html.j2 (100%) rename {app => web/app}/templates/500.html.j2 (100%) rename {app => web/app}/templates/admin/edit_user.html.j2 (100%) rename {app => web/app}/templates/admin/index.html.j2 (100%) rename {app => web/app}/templates/admin/user.html.j2 (100%) rename {app => web/app}/templates/auth/email/confirm.html.j2 (100%) rename {app => web/app}/templates/auth/email/confirm.txt.j2 (100%) rename {app => web/app}/templates/auth/email/reset_password.html.j2 (100%) rename {app => web/app}/templates/auth/email/reset_password.txt.j2 (100%) rename {app => web/app}/templates/auth/login.html.j2 (100%) rename {app => web/app}/templates/auth/register.html.j2 (100%) rename {app => web/app}/templates/auth/reset_password.html.j2 (100%) rename {app => web/app}/templates/auth/reset_password_request.html.j2 (100%) rename {app => web/app}/templates/auth/unconfirmed.html.j2 (100%) rename {app => web/app}/templates/corpora/add_corpus.html.j2 (100%) rename {app => web/app}/templates/corpora/add_corpus_file.html.j2 (100%) rename {app => web/app}/templates/corpora/analyse_corpus.html.j2 (100%) rename {app => web/app}/templates/corpora/corpus.html.j2 (100%) rename {app => web/app}/templates/corpora/edit_corpus_file.html.j2 (100%) rename {app => web/app}/templates/jobs/job.html.j2 (100%) rename {app => web/app}/templates/macros/materialize.html.j2 (100%) rename {app => web/app}/templates/main/dashboard.html.j2 (100%) rename {app => web/app}/templates/main/feedback.html.j2 (100%) rename {app => web/app}/templates/main/index.html.j2 (100%) rename {app => web/app}/templates/main/poster.html.j2 (100%) rename {app => web/app}/templates/main/privacy_policy.html.j2 (100%) rename {app => web/app}/templates/nopaque.html.j2 (100%) rename {app => web/app}/templates/profile/settings.html.j2 (100%) rename {app => web/app}/templates/services/corpus_analysis.html.j2 (100%) rename {app => web/app}/templates/services/file-setup.html.j2 (100%) rename {app => web/app}/templates/services/nlp.html.j2 (100%) rename {app => web/app}/templates/services/ocr.html.j2 (100%) rename {app => web/app}/templates/services/roadmap.html.j2 (100%) rename config.py => web/config.py (90%) rename docker-entrypoint.sh => web/docker-entrypoint.sh (100%) mode change 100755 => 100644 rename {migrations => web/migrations}/README (100%) rename {migrations => web/migrations}/alembic.ini (100%) rename {migrations => web/migrations}/env.py (100%) rename {migrations => web/migrations}/script.py.mako (100%) rename {migrations => web/migrations}/versions/099037c4aa06_.py (100%) rename {migrations => web/migrations}/versions/0aa38a7973c5_.py (100%) rename {migrations => web/migrations}/versions/10a92d8f4616_.py (100%) rename {migrations => web/migrations}/versions/1210adfe1e34_.py (100%) rename {migrations => web/migrations}/versions/3d9a20b8b26c_.py (100%) rename {migrations => web/migrations}/versions/421ba4373e50_.py (100%) rename {migrations => web/migrations}/versions/4638e6509e13_.py (100%) rename {migrations => web/migrations}/versions/471aa04c1a92_.py (100%) rename {migrations => web/migrations}/versions/4886241e0f5d_.py (100%) rename {migrations => web/migrations}/versions/49a42c69e523_.py (100%) rename {migrations => web/migrations}/versions/5ba6786a847e_.py (100%) rename {migrations => web/migrations}/versions/62233e0cb2c7_.py (100%) rename {migrations => web/migrations}/versions/6227310c2112_.py (100%) rename {migrations => web/migrations}/versions/66253783654f_.py (100%) rename {migrations => web/migrations}/versions/68772b6560c3_.py (100%) rename {migrations => web/migrations}/versions/7378391345fa_.py (100%) rename {migrations => web/migrations}/versions/776761fb7466_.py (100%) rename {migrations => web/migrations}/versions/abf60427ff84_.py (100%) rename {migrations => web/migrations}/versions/da9fd175af8c_.py (100%) rename {migrations => web/migrations}/versions/ded5a37f8a7b_.py (100%) rename nopaque.py => web/nopaque.py (96%) rename requirements.txt => web/requirements.txt (100%) create mode 100644 web/tests/__init__.py rename {tests => web/tests}/test_basics.py (100%) rename {tests => web/tests}/test_client.py (100%) rename {tests => web/tests}/test_user_model.py (100%) diff --git a/.env.tpl b/.env.tpl new file mode 100644 index 00000000..26e41eb5 --- /dev/null +++ b/.env.tpl @@ -0,0 +1,6 @@ +# docker GID (getent group docker | cut -d: -f3) +docker_gid= +# GID (id -g) +gid= +# UID (id -u) +uid= diff --git a/.gitignore b/.gitignore index 439b4c22..89f44724 100644 --- a/.gitignore +++ b/.gitignore @@ -1,9 +1,4 @@ -# Files .DS_Store *.env - - -# Directories logs __pycache__ -certs diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml deleted file mode 100644 index b3bf5524..00000000 --- a/.gitlab-ci.yml +++ /dev/null @@ -1,55 +0,0 @@ -image: docker:stable - -services: - - docker:stable-dind - -variables: - DOCKER_DRIVER: overlay2 - -stages: - - build - - push - -before_script: - - docker login -u gitlab-ci-token -p $CI_JOB_TOKEN $CI_REGISTRY - -Build: - script: - - docker build --pull -t $CI_REGISTRY_IMAGE:tmp . - - docker push $CI_REGISTRY_IMAGE:tmp - stage: build - tags: - - docker - -Push development: - only: - - development - script: - - docker pull $CI_REGISTRY_IMAGE:tmp - - docker tag $CI_REGISTRY_IMAGE:tmp $CI_REGISTRY_IMAGE:development - - docker push $CI_REGISTRY_IMAGE:development - stage: push - tags: - - docker - -Push latest: - only: - - master - script: - - docker pull $CI_REGISTRY_IMAGE:tmp - - docker tag $CI_REGISTRY_IMAGE:tmp $CI_REGISTRY_IMAGE:latest - - docker push $CI_REGISTRY_IMAGE:latest - stage: push - tags: - - docker - -Push tag: - only: - - tags - script: - - docker pull $CI_REGISTRY_IMAGE:tmp - - docker tag $CI_REGISTRY_IMAGE:tmp $CI_REGISTRY_IMAGE:$CI_COMMIT_REF_NAME - - docker push $CI_REGISTRY_IMAGE:$CI_COMMIT_REF_NAME - stage: push - tags: - - docker diff --git a/README.md b/README.md index ad52d00d..38096673 100644 --- a/README.md +++ b/README.md @@ -1,46 +1,22 @@ -# Opaque +# nopaque -Opaque is a virtual research environment (VRE) bundling OCR, NLP and additional computer linguistic methods for research purposes in the field of Digital Humanities. +_nopaque_ bundles various tools and services that provide humanities scholars with DH methods and thus can support their various individual research processes. Using _nopaque_, researchers can subject digitized sources to Optical Character Recognition (OCR). The resulting text files can then be used as a data basis for Natural Language Processing (NLP). The texts are automatically subjected to various linguistic annotations. The data processed via NLP can then be summarized in the web application as corpora and analyzed by means of an information retrieval system through complex search queries. The range of functions of the web application will be successively extended according to the needs of the researchers. -Opaque is designed as a web application which can be easily used by researchers to aid them during their research process. +## Prerequisites and requirements -In particular researchers can use Opaque to start OCR jobs for digitized sources. The text output of these OCR jobs can then be used as an input for tagging processes (POS, NER etc.). - -As a last step texts can be loaded into an information retrieval system to query for specific words, phrases in connection with linguistic features. +1. Install docker for your system. Following the official instructions. (LINK) +2. Install docker-compose. Following the official instructions. (LINK) -## Dependencies - -- cifs-utils -- Docker -- Docker Compose - - -## Configuration and startup +## Configuration and startup: Run a docker swarm and setup a Samba share 1. **Create Docker swarm:** -The following part is for **users** and not the development team. The development team uses a script which sets up a local development swarm. - -The generated computational workload is handled by a [Docker](https://docs.docker.com/) swarm. A swarm is a group of machines that are running Docker and joined into a cluster. It consists out of two different kinds of members, managers and workers. Currently it is not possible to specify a dedicated Docker host, instead Opaque expects the executing system to be a swarm manager of a cluster with at least one dedicated worker machine. The swarm setup process is described best in the [Docker documentation](https://docs.docker.com/engine/swarm/swarm-tutorial/). - -The dev team can use dind_swarm_setup.sh. If the workers cannot join the manager node. Try opening the following ports using the ubuntu firewall ufw: -```bash -sudo ufw allow 2376/tcp \ -&& sudo ufw allow 7946/udp \ -&& sudo ufw allow 7946/tcp \ -&& sudo ufw allow 80/tcp \ -&& sudo ufw allow 2377/tcp \ -&& sudo ufw allow 4789/udp - -sudo ufw reload && sudo ufw enable -sudo systemctl restart docker -``` +The generated computational workload is handled by a [Docker](https://docs.docker.com/) swarm. A swarm is a group of machines that are running Docker and joined into a cluster. It consists out of two different kinds of members, manager and worker nodes. The swarm setup process is described best in the [Docker documentation](https://docs.docker.com/engine/swarm/swarm-tutorial/). 2. **Create a network storage:** -The dind_swarm_setup.sh script handles this step for the dev team aswell. -A shared network space is necessary so that all swarm members have access to all the data. To achieve this a [Samba](https://www.samba.org/) can be used. +A shared network space is necessary so that all swarm members have access to all the data. To achieve this a [samba](https://www.samba.org/) share is used. ``` bash # Example: Create a Samba share via Docker # More details can be found under https://hub.docker.com/r/dperson/samba/ @@ -54,42 +30,57 @@ docker run \ -s storage.nopaque;/srv/nopaque/storage;no;no;no;nopaque \ -u nopaque;nopaque -# Mount the Samba share on all swarm member nodes with the following code +# Mount the Samba share on all swarm nodes (managers and workers) sudo mkdir /mnt/nopaque -sudo mount --types cifs --options gid=${USER},password=nopaque,uid=${USER},user=nopaque,vers=3.0 ///storage.nopaque /mnt/nopaque +sudo mount --types cifs --options gid=${USER},password=nopaque,uid=${USER},user=nopaque,vers=3.0 ///storage.nopaque /mnt/nopaque ``` -3. **Download Opaque** -``` bash -git clone https://gitlab.ub.uni-bielefeld.de/sfb1288inf/opaque.git -cd opaque -docker-compose pull -``` -4. **Configure your instance:** -For production environments it is recommended to activate and secure the Docker HTTP API. You can read more [here](https://gitlab.ub.uni-bielefeld.de/sfb1288inf/opaque_daemon). +## Download, configure and build _nopaque_** + +3. **Download, configure and build _nopaque_** + ``` bash +git clone https://gitlab.ub.uni-bielefeld.de/sfb1288inf/nopaque.git mkdir logs cp nopaque.env.tpl nopaque.env - nopaque.env # Fill out the empty variables within this file. For the gitlab login either use your credentials (not recommended) Or create a gitlab token + nopaque.env # Fill out the variables within this file. For the GitLab variables either use your credentials (not recommended) or create an access token with the read_registry scope. If this repository is public no credentials are needed. ``` +### Variables and their values are explained here: +The first three DOCKER variables should only be used if you want to use the Docker HTTP API. Check the daemon readme to see how to create certificates and activate the API. + +FLASK_CONFIG=development|testing|production \ +SECRET_KEY=92b461ba136e4ca48e430003acd56977 \ +Uses this for example to create a secret key: `python -c "import uuid; print(uuid.uuid4().hex)"` + +The **GitLab Registry** Variables are not needed if this repository is public. If needed use your GitLab username and a token as a password + +**Flask Mail** variables are needed for sending password reset mails etc. Use your own mail server configs here.\ +MAIL_SERVER=smtp.example.com \ +MAIL_PORT=587 \ +MAIL_USE_TLS=true \ +MAIL_USERNAME=user@example.com \ +MAIL_PASSWORD=password \ +NOPAQUE_MAIL_SENDER=Nopaque Admin _Name shown as sender._ + +**Nopaque** variables are needed for the web app.\ +NOPAQUE_ADMIN=yourmail@example.com _If a user is registered using this mail the user will automatically be granted admin rights._ \ +NOPAQUE_CONTACT=contactmailadress@example.com _Contact mail address shown in the footer of the web application._\ +NOPAQUE_DOMAIN=yourdomain.com _The domain your nopaque installation is hosted on. use https://nopaque.localhost for a local running instance._ \ +NOPAQUE_LOG_LEVEL=WARNING|INFO|ERROR|DEBUG \ +NOPAQUE_STORAGE=path/to/your/samba/share + +**PostgreSQL Database** credentials: \ +POSTGRES_DB_NAME=dbanme \ +POSTGRES_USER=username \ +POSTGRES_PASSWORD=password -5. Further development instructions -Use the following command to allow docker to pull images from your gitlab registry. TODO: Check if this could also work wit a token? ```bash -docker login gitlab.ub.uni-bielefeld.de:4567 +cp .env.tpl .env + .env # Fill out the variables within this file. +docker-compose build ``` -6. **Start your instance** -``` bash -# Execute the following 3 steps only on first startup -docker-compose run web flask db upgrade -docker-compose run web flask insert-initial-database-entries -docker-compose down - +**Start your instance** +```bash +# for background execution add the -d flag and to scale the app, add --scale web= docker-compose up ``` - -6. **Alter Database Models** -``` bash -docker-compose run web flask db migrate -docker-compose run web flask db upgrade -``` diff --git a/daemon/Dockerfile b/daemon/Dockerfile new file mode 100644 index 00000000..db20eaee --- /dev/null +++ b/daemon/Dockerfile @@ -0,0 +1,40 @@ +FROM python:3.6-slim-stretch + + +LABEL maintainer="inf_sfb1288@lists.uni-bielefeld.de" + + +ARG docker_gid=998 +ARG gid=1000 +ARG uid=1000 +ENV LANG=C.UTF-8 + + +RUN apt-get update \ + && apt-get install --no-install-recommends --yes \ + build-essential \ + libpq-dev \ + wait-for-it \ + && rm -rf /var/lib/apt/lists/* + + +RUN groupadd --gid ${docker_gid} --system docker \ + && groupadd --gid ${gid} --system nopaqued \ + && useradd --create-home --gid ${gid} --groups ${docker_gid} --no-log-init --system --uid ${uid} nopaqued +USER nopaqued +WORKDIR /home/nopaqued + + +COPY ["notify", "notify"] +COPY ["tasks", "tasks"] +COPY ["logger", "logger"] +COPY ["decorators.py", "nopaqued.py", "requirements.txt", "./"] +RUN python -m venv venv \ + && venv/bin/pip install --requirement requirements.txt \ + && mkdir logs + + +COPY docker-entrypoint.sh /usr/local/bin/ + + +ENTRYPOINT ["docker-entrypoint.sh"] diff --git a/daemon/decorators.py b/daemon/decorators.py new file mode 100644 index 00000000..19d80d8e --- /dev/null +++ b/daemon/decorators.py @@ -0,0 +1,14 @@ +from functools import wraps +from threading import Thread + + +def background(f): + ''' + ' This decorator executes a function in a Thread. + ''' + @wraps(f) + def wrapped(*args, **kwargs): + thread = Thread(target=f, args=args, kwargs=kwargs) + thread.start() + return thread + return wrapped \ No newline at end of file diff --git a/daemon/docker-entrypoint.sh b/daemon/docker-entrypoint.sh new file mode 100644 index 00000000..e2f4d358 --- /dev/null +++ b/daemon/docker-entrypoint.sh @@ -0,0 +1,8 @@ +#!/bin/bash + +echo "Waiting for db..." +wait-for-it db:5432 --strict --timeout=0 +echo "Waiting for web..." +wait-for-it web:5000 --strict --timeout=0 + +venv/bin/python nopaqued.py diff --git a/tests/__init__.py b/daemon/logger/__init__.py similarity index 100% rename from tests/__init__.py rename to daemon/logger/__init__.py diff --git a/daemon/logger/logger.py b/daemon/logger/logger.py new file mode 100644 index 00000000..de602eb6 --- /dev/null +++ b/daemon/logger/logger.py @@ -0,0 +1,26 @@ +import os +import logging + + +def init_logger(): + ''' + Functions initiates a logger instance. + ''' + if not os.path.isfile('logs/nopaqued.log'): + file_path = os.path.join(os.getcwd(), 'logs/nopaqued.log') + log = open(file_path, 'w+') + log.close() + logging.basicConfig(datefmt='%Y-%m-%d %H:%M:%S', + filemode='w', filename='logs/nopaqued.log', + format='%(asctime)s - %(levelname)s - %(name)s - ' + '%(filename)s - %(lineno)d - %(message)s') + logger = logging.getLogger(__name__) + if os.environ.get('FLASK_CONFIG') == 'development': + logger.setLevel(logging.DEBUG) + if os.environ.get('FLASK_CONFIG') == 'production': + logger.setLevel(logging.WARNING) + return logger + + +if __name__ == '__main__': + init_logger() \ No newline at end of file diff --git a/daemon/nopaqued.bak.py b/daemon/nopaqued.bak.py new file mode 100644 index 00000000..b1c71739 --- /dev/null +++ b/daemon/nopaqued.bak.py @@ -0,0 +1,455 @@ +from notify.notification import Notification +from notify.service import NotificationService +from sqlalchemy import create_engine, asc +from sqlalchemy.orm import Session, relationship +from sqlalchemy.ext.automap import automap_base +from datetime import datetime +from time import sleep +import docker +import json +import logging +import os +import shutil + + +''' Global constants ''' +NOPAQUE_STORAGE = os.environ.get('NOPAQUE_STORAGE') + +''' Global variables ''' +docker_client = None +session = None + + +# Classes for database models +Base = automap_base() + + +class Corpus(Base): + __tablename__ = 'corpora' + files = relationship('CorpusFile', collection_class=set) + + +class CorpusFile(Base): + __tablename__ = 'corpus_files' + + +class Job(Base): + __tablename__ = 'jobs' + inputs = relationship('JobInput', collection_class=set) + results = relationship('JobResult', collection_class=set) + notification_data = relationship('NotificationData', collection_class=list) + notification_email_data = relationship('NotificationEmailData', collection_class=list) + + +class NotificationData(Base): + __tablename__ = 'notification_data' + job = relationship('Job', collection_class=set) + + +class NotificationEmailData(Base): + __tablename__ = 'notification_email_data' + job = relationship('Job', collection_class=set) + + +class JobInput(Base): + __tablename__ = 'job_results' + + +class JobResult(Base): + __tablename__ = 'job_results' + + +class User(Base): + __tablename__ = 'users' + jobs = relationship('Job', collection_class=set) + corpora = relationship('Corpus', collection_class=set) + + +def check_corpora(): + corpora = session.query(Corpus).all() + for corpus in filter(lambda corpus: corpus.status == 'submitted', corpora): + __create_build_corpus_service(corpus) + for corpus in filter(lambda corpus: (corpus.status == 'queued' + or corpus.status == 'running'), + corpora): + __checkout_build_corpus_service(corpus) + for corpus in filter(lambda corpus: corpus.status == 'start analysis', + corpora): + __create_cqpserver_container(corpus) + for corpus in filter(lambda corpus: corpus.status == 'stop analysis', + corpora): + __remove_cqpserver_container(corpus) + + +def __create_build_corpus_service(corpus): + corpus_dir = os.path.join(NOPAQUE_STORAGE, str(corpus.user_id), + 'corpora', str(corpus.id)) + corpus_data_dir = os.path.join(corpus_dir, 'data') + corpus_file = os.path.join(corpus_dir, 'merged', 'corpus.vrt') + corpus_registry_dir = os.path.join(corpus_dir, 'registry') + if os.path.exists(corpus_data_dir): + shutil.rmtree(corpus_data_dir) + if os.path.exists(corpus_registry_dir): + shutil.rmtree(corpus_registry_dir) + os.mkdir(corpus_data_dir) + os.mkdir(corpus_registry_dir) + service_args = {'command': 'docker-entrypoint.sh build-corpus', + 'constraints': ['node.role==worker'], + 'labels': {'origin': 'nopaque', + 'type': 'corpus.prepare', + 'corpus_id': str(corpus.id)}, + 'mounts': [corpus_file + ':/root/files/corpus.vrt:ro', + corpus_data_dir + ':/corpora/data:rw', + corpus_registry_dir + ':/usr/local/share/cwb/registry:rw'], + 'name': 'build-corpus_{}'.format(corpus.id), + 'restart_policy': docker.types.RestartPolicy()} + service_image = ('gitlab.ub.uni-bielefeld.de:4567/sfb1288inf/cqpserver:latest') + try: + service = docker_client.services.get(service_args['name']) + except docker.errors.NotFound: + pass + except docker.errors.DockerException: + return + else: + service.remove() + try: + docker_client.services.create(service_image, **service_args) + except docker.errors.DockerException: + corpus.status = 'failed' + else: + corpus.status = 'queued' + + +def __checkout_build_corpus_service(corpus): + service_name = 'build-corpus_{}'.format(corpus.id) + try: + service = docker_client.services.get(service_name) + except docker.errors.NotFound: + logger.error('__checkout_build_corpus_service({}):'.format(corpus.id) + + ' The service does not exist.' + + ' (stauts: {} -> failed)'.format(corpus.status)) + corpus.status = 'failed' + return + except docker.errors.DockerException: + return + service_tasks = service.tasks() + if not service_tasks: + return + task_state = service_tasks[0].get('Status').get('State') + if corpus.status == 'queued' and task_state != 'pending': + corpus.status = 'running' + elif corpus.status == 'running' and task_state == 'complete': + service.remove() + corpus.status = 'prepared' + elif corpus.status == 'running' and task_state == 'failed': + service.remove() + corpus.status = task_state + + +def __create_cqpserver_container(corpus): + corpus_dir = os.path.join(NOPAQUE_STORAGE, str(corpus.user_id), + 'corpora', str(corpus.id)) + corpus_data_dir = os.path.join(corpus_dir, 'data') + corpus_registry_dir = os.path.join(corpus_dir, 'registry') + container_args = {'command': 'cqpserver', + 'detach': True, + 'volumes': [corpus_data_dir + ':/corpora/data:rw', + corpus_registry_dir + ':/usr/local/share/cwb/registry:rw'], + 'name': 'cqpserver_{}'.format(corpus.id), + 'network': 'opaque_default'} + container_image = ('gitlab.ub.uni-bielefeld.de:4567/sfb1288inf/cqpserver:latest') + try: + container = docker_client.containers.get(container_args['name']) + except docker.errors.NotFound: + pass + except docker.errors.DockerException: + return + else: + container.remove(force=True) + try: + docker_client.containers.run(container_image, **container_args) + except docker.errors.DockerException: + return + else: + corpus.status = 'analysing' + + +def __remove_cqpserver_container(corpus): + container_name = 'cqpserver_{}'.format(corpus.id) + try: + container = docker_client.containers.get(container_name) + except docker.errors.NotFound: + pass + except docker.errors.DockerException: + return + else: + container.remove(force=True) + corpus.status = 'prepared' + + +def check_jobs(): + jobs = session.query(Job).all() + for job in filter(lambda job: job.status == 'submitted', jobs): + __create_job_service(job) + for job in filter(lambda job: (job.status == 'queued'), jobs): + __checkout_job_service(job) + # __add_notification_data(job, 'queued') + for job in filter(lambda job: (job.status == 'running'), jobs): + __checkout_job_service(job) + # __add_notification_data(job, 'running') + # for job in filter(lambda job: job.status == 'complete', jobs): + # __add_notification_data(job, 'complete') + # for job in filter(lambda job: job.status == 'failed', jobs): + #__add_notification_data(job, 'failed') + for job in filter(lambda job: job.status == 'canceling', jobs): + __remove_job_service(job) + + +def __add_notification_data(job, notified_on_status): + # checks if user wants any notifications at all + if (job.user.setting_job_status_mail_notifications == 'none'): + # logger.warning('User does not want any notifications!') + return + # checks if user wants only notification on completed jobs + elif (job.user.setting_job_status_mail_notifications == 'end' + and notified_on_status != 'complete'): + # logger.warning('User only wants notifications on job completed!') + return + else: + # check if a job already has associated NotificationData + notification_exists = len(job.notification_data) + # create notification_data for current job if there is none + if (notification_exists == 0): + notification_data = NotificationData(job_id=job.id) + session.add(notification_data) + session.commit() # If no commit job will have no NotificationData + # logger.warning('Created NotificationData for current Job.')) + else: + pass + # logger.warning('Job already had notification: {}'.format(notification_exists)) + if (job.notification_data[0].notified_on != notified_on_status): + notification_email_data = NotificationEmailData(job_id=job.id) + notification_email_data.notify_status = notified_on_status + notification_email_data.creation_date = datetime.utcnow() + job.notification_data[0].notified_on = notified_on_status + session.add(notification_email_data) + # logger.warning('Created NotificationEmailData for current Job.') + else: + # logger.warning('NotificationEmailData has already been created for current Job!') + pass + + +def __create_job_service(job): + job_dir = os.path.join(NOPAQUE_STORAGE, str(job.user_id), 'jobs', + str(job.id)) + service_args = {'command': ('{} /files /files/output'.format(job.service) + + ' {}'.format(job.secure_filename if job.service == 'file-setup' else '') + + ' --log-dir /files' + + ' --zip [{}]_{}'.format(job.service, job.secure_filename) + + ' ' + ' '.join(json.loads(job.service_args))), + 'constraints': ['node.role==worker'], + 'labels': {'origin': 'nopaque', + 'type': 'service.{}'.format(job.service), + 'job_id': str(job.id)}, + 'mounts': [job_dir + ':/files:rw'], + 'name': 'job_{}'.format(job.id), + 'resources': docker.types.Resources( + cpu_reservation=job.n_cores * (10 ** 9), + mem_reservation=job.mem_mb * (10 ** 6)), + 'restart_policy': docker.types.RestartPolicy()} + service_image = ('gitlab.ub.uni-bielefeld.de:4567/sfb1288inf/' + + job.service + ':' + job.service_version) + try: + service = docker_client.services.get(service_args['name']) + except docker.errors.NotFound: + pass + except docker.errors.DockerException: + return + else: + service.remove() + try: + docker_client.services.create(service_image, **service_args) + except docker.errors.DockerException: + job.status = 'failed' + else: + job.status = 'queued' + + +def __checkout_job_service(job): + service_name = 'job_{}'.format(job.id) + try: + service = docker_client.services.get(service_name) + except docker.errors.NotFound: + logger.error('__checkout_job_service({}):'.format(job.id) + + ' The service does not exist.' + + ' (stauts: {} -> failed)'.format(job.status)) + job.status = 'failed' + return + except docker.errors.DockerException: + return + service_tasks = service.tasks() + if not service_tasks: + return + task_state = service_tasks[0].get('Status').get('State') + if job.status == 'queued' and task_state != 'pending': + job.status = 'running' + elif (job.status == 'running' + and (task_state == 'complete' or task_state == 'failed')): + service.remove() + job.end_date = datetime.utcnow() + job.status = task_state + if task_state == 'complete': + results_dir = os.path.join(NOPAQUE_STORAGE, str(job.user_id), + 'jobs', str(job.id), 'output') + results = filter(lambda x: x.endswith('.zip'), + os.listdir(results_dir)) + for result in results: + job_result = JobResult(dir=results_dir, filename=result, + job_id=job.id) + session.add(job_result) + + +def __remove_job_service(job): + service_name = 'job_{}'.format(job.id) + try: + service = docker_client.services.get(service_name) + except docker.errors.NotFound: + job.status = 'canceled' + except docker.errors.DockerException: + return + else: + service.update(mounts=None) + service.remove() + + +def handle_jobs(): + check_jobs() + + +def handle_corpora(): + check_corpora() + + +# Email notification functions +def create_mail_notifications(notification_service): + notification_email_data = session.query(NotificationEmailData).order_by(asc(NotificationEmailData.creation_date)).all() + notifications = {} + for data in notification_email_data: + notification = Notification() + notification.set_addresses(notification_service.email_address, + data.job.user.email) + subject_template = '[nopaque] Status update for your Job/Corpora: {title}!' + subject_template_values_dict = {'title': data.job.title} + domain = os.environ.get('NOPAQUE_DOMAIN') + url = '{domain}/{jobs}/{id}'.format(domain=domain, + jobs='jobs', + id=data.job.id) + body_template_values_dict = {'username': data.job.user.username, + 'id': data.job.id, + 'title': data.job.title, + 'status': data.notify_status, + 'time': data.creation_date, + 'url': url} + notification.set_notification_content(subject_template, + subject_template_values_dict, + 'templates/notification_messages/notification.txt', + 'templates/notification_messages/notification.html', + body_template_values_dict) + notifications[data.job.id] = notification + # Using a dictionary for notifications avoids sending multiple mails + # if the status of a job changes in a few seconds. The user will not get + # swamped with mails for queued, running and complete if those happen in + # in a few seconds. Only the last update will be sent. + session.delete(data) + return notifications + + +def send_mail_notifications(notifications, notification_service): + for key, notification in notifications.items(): + try: + notification_service.send(notification) + notification_service.mail_limit_exceeded = False + except Exception as e: + # Adds notifications to unsent if mail server exceded limit for + # consecutive mail sending + notification_service.not_sent[key] = notification + notification_service.mail_limit_exceeded = True + + +def notify(): + # Initialize notification service + notification_service = NotificationService() + notification_service.get_smtp_configs() + notification_service.set_server() + # create notifications (content, recipient etc.) + notifications = create_mail_notifications(notification_service) + # only login and send mails if there are any notifications + if (len(notifications) > 0): + try: + notification_service.login() + # combine new and unsent notifications + notifications.update(notification_service.not_sent) + # send all notifications + send_mail_notifications(notifications, notification_service) + # remove unsent notifications because they have been sent now + # but only if mail limit has not been exceeded + if (notification_service.mail_limit_exceeded is not True): + notification_service.not_sent = {} + notification_service.quit() + except Exception as e: + notification_service.not_sent.update(notifications) + + +# Logger functions # +def init_logger(): + ''' + Functions initiates a logger instance. + ''' + global logger + + if not os.path.isfile('logs/nopaqued.log'): + file_path = os.path.join(os.getcwd(), 'logs/nopaqued.log') + log = open(file_path, 'w+') + log.close() + logging.basicConfig(datefmt='%Y-%m-%d %H:%M:%S', + filemode='w', filename='logs/nopaqued.log', + format='%(asctime)s - %(levelname)s - %(name)s - ' + '%(filename)s - %(lineno)d - %(message)s') + logger = logging.getLogger(__name__) + if os.environ.get('FLASK_CONFIG') == 'development': + logger.setLevel(logging.DEBUG) + if os.environ.get('FLASK_CONFIG') == 'production': + logger.setLevel(logging.WARNING) + + +def nopaqued(): + global Base + global docker_client + global session + + engine = create_engine( + 'postgresql://{}:{}@db/{}'.format( + os.environ.get('POSTGRES_USER'), + os.environ.get('POSTGRES_PASSWORD'), + os.environ.get('POSTGRES_DB_NAME'))) + Base.prepare(engine, reflect=True) + session = Session(engine) + session.commit() + + docker_client = docker.from_env() + docker_client.login(password=os.environ.get('GITLAB_PASSWORD'), + registry="gitlab.ub.uni-bielefeld.de:4567", + username=os.environ.get('GITLAB_USERNAME')) + + # executing background functions + while True: + handle_jobs() + handle_corpora() + # notify() + session.commit() + sleep(3) + + +if __name__ == '__main__': + init_logger() + nopaqued() diff --git a/daemon/nopaqued.py b/daemon/nopaqued.py new file mode 100644 index 00000000..537b3e02 --- /dev/null +++ b/daemon/nopaqued.py @@ -0,0 +1,18 @@ +from tasks.check_jobs import check_jobs +from tasks.check_corpora import check_corpora +from tasks.notify import notify +from time import sleep + + +def nopaqued(): + # executing background functions + while True: + check_jobs() + check_corpora() + notify(True) # If True mails are sent. If False no mails are sent. + # But notification status will be set nonetheless. + sleep(3) + + +if __name__ == '__main__': + nopaqued() diff --git a/daemon/notify/__init__.py b/daemon/notify/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/daemon/notify/notification.py b/daemon/notify/notification.py new file mode 100644 index 00000000..ed88b6be --- /dev/null +++ b/daemon/notify/notification.py @@ -0,0 +1,27 @@ +from email.message import EmailMessage + + +class Notification(EmailMessage): + """docstring for Email.""" + + def set_notification_content(self, + subject_template, + subject_template_values_dict, + body_txt_template_path, + body_html_template_path, + body_template_values_dict): + # Create subject with subject_template_values_dict + self['subject'] = subject_template.format(**subject_template_values_dict) + # Open template files and insert values from body_template_values_dict + with open(body_txt_template_path) as nfile: + self.body_txt = nfile.read().format(**body_template_values_dict) + with open(body_html_template_path) as nfile: + self.body_html = nfile.read().format(**body_template_values_dict) + # Set txt of email + self.set_content(self.body_txt) + # Set html alternative + self.add_alternative(self.body_html, subtype='html') + + def set_addresses(self, sender, recipient): + self['From'] = sender + self['to'] = recipient \ No newline at end of file diff --git a/daemon/notify/service.py b/daemon/notify/service.py new file mode 100644 index 00000000..5687ac42 --- /dev/null +++ b/daemon/notify/service.py @@ -0,0 +1,41 @@ +import os +import smtplib + + +class NotificationService(object): + """This is a nopaque notifcation service object.""" + + def __init__(self, execute_flag): + super(NotificationService, self).__init__() + self.execute_flag = execute_flag # If True mails are sent normaly + # If False mails are not sent. Used to avoid sending mails for jobs that + # have been completed a long time ago. Use this if you implement notify + # into an already existing nopaque instance. Change it to True after the + # daemon has run one time with the flag set to False + self.not_sent = {} # Holds due to an error unsent email notifications + self.mail_limit_exceeded = False # Bool to show if the mail server + # stoped sending mails due to exceeding its sending limit + + def get_smtp_configs(self): + self.password = os.environ.get('MAIL_PASSWORD') + self.port = os.environ.get('MAIL_PORT') + self.server_str = os.environ.get('MAIL_SERVER') + self.tls = os.environ.get('MAIL_USE_TLS') + self.username = os.environ.get('MAIL_USERNAME').split("@")[0] + self.email_address = os.environ.get('MAIL_USERNAME') + + def set_server(self): + self.smtp_server = smtplib.SMTP(host=self.server_str, port=self.port) + + def login(self): + self.smtp_server.starttls() + self.smtp_server.login(self.username, self.password) + + def send(self, email): + if self.execute_flag: + self.smtp_server.send_message(email) + else: + return + + def quit(self): + self.smtp_server.quit() \ No newline at end of file diff --git a/daemon/notify/templates/notification_messages/notification.html b/daemon/notify/templates/notification_messages/notification.html new file mode 100644 index 00000000..e2edfe75 --- /dev/null +++ b/daemon/notify/templates/notification_messages/notification.html @@ -0,0 +1,15 @@ + + +

Dear {username},

+ +

The status of your Job/Corpus({id}) with the title "{title}" has changed!

+

It is now {status}!

+

Time of this status update was: {time} UTC

+ +

You can access your Job/Corpus here: {url} +

+ +

Kind regards!
+ Your nopaque team

+ + diff --git a/daemon/notify/templates/notification_messages/notification.txt b/daemon/notify/templates/notification_messages/notification.txt new file mode 100644 index 00000000..0e221c54 --- /dev/null +++ b/daemon/notify/templates/notification_messages/notification.txt @@ -0,0 +1,10 @@ +Dear {username}, + +The status of your Job/Corpus({id}) with the title "{title}" has changed! +It is now {status}! +Time of this status update was: {time} UTC + +You can access your Job/Corpus here: {url} + +Kind regards! +Your nopaque team \ No newline at end of file diff --git a/daemon/requirements.txt b/daemon/requirements.txt new file mode 100644 index 00000000..bafd77ed --- /dev/null +++ b/daemon/requirements.txt @@ -0,0 +1,3 @@ +docker +psycopg2 +SQLAlchemy diff --git a/daemon/tasks/Models.py b/daemon/tasks/Models.py new file mode 100644 index 00000000..42cc4021 --- /dev/null +++ b/daemon/tasks/Models.py @@ -0,0 +1,52 @@ +from sqlalchemy.ext.automap import automap_base +from sqlalchemy.orm import relationship +from tasks import engine + + +Base = automap_base() + + +# Classes for database models +class Corpus(Base): + __tablename__ = 'corpora' + files = relationship('CorpusFile', collection_class=set) + + +class CorpusFile(Base): + __tablename__ = 'corpus_files' + + +class Job(Base): + __tablename__ = 'jobs' + inputs = relationship('JobInput', collection_class=set) + results = relationship('JobResult', collection_class=set) + notification_data = relationship('NotificationData', collection_class=list) + notification_email_data = relationship('NotificationEmailData', + collection_class=list) + + +class JobInput(Base): + __tablename__ = 'job_results' + + +class JobResult(Base): + __tablename__ = 'job_results' + + +class NotificationData(Base): + __tablename__ = 'notification_data' + job = relationship('Job', collection_class=set) + + +class NotificationEmailData(Base): + __tablename__ = 'notification_email_data' + job = relationship('Job', collection_class=set) + + +class User(Base): + __tablename__ = 'users' + jobs = relationship('Job', collection_class=set) + corpora = relationship('Corpus', collection_class=set) + + +Base.prepare(engine, reflect=True) diff --git a/daemon/tasks/__init__.py b/daemon/tasks/__init__.py new file mode 100644 index 00000000..e3e6eb51 --- /dev/null +++ b/daemon/tasks/__init__.py @@ -0,0 +1,22 @@ +from sqlalchemy import create_engine +from sqlalchemy.orm import scoped_session, sessionmaker +import os +import docker + +''' Global constants ''' +NOPAQUE_STORAGE = os.environ.get('NOPAQUE_STORAGE') + +''' Docker client ''' +docker_client = docker.from_env() +docker_client.login(password=os.environ.get('GITLAB_PASSWORD'), + registry="gitlab.ub.uni-bielefeld.de:4567", + username=os.environ.get('GITLAB_USERNAME')) + +''' Scoped session ''' +engine = create_engine( + 'postgresql://{}:{}@db/{}'.format( + os.environ.get('POSTGRES_USER'), + os.environ.get('POSTGRES_PASSWORD'), + os.environ.get('POSTGRES_DB_NAME'))) +session_factory = sessionmaker(bind=engine) +Session = scoped_session(session_factory) diff --git a/daemon/tasks/check_corpora.py b/daemon/tasks/check_corpora.py new file mode 100644 index 00000000..9ade3db5 --- /dev/null +++ b/daemon/tasks/check_corpora.py @@ -0,0 +1,134 @@ +from decorators import background +from logger.logger import init_logger +from tasks import Session, docker_client, NOPAQUE_STORAGE +from tasks.Models import Corpus +import docker +import os +import shutil + + +@background +def check_corpora(): + c_session = Session() + corpora = c_session.query(Corpus).all() + for corpus in filter(lambda corpus: corpus.status == 'submitted', corpora): + __create_build_corpus_service(corpus) + for corpus in filter(lambda corpus: (corpus.status == 'queued' + or corpus.status == 'running'), + corpora): + __checkout_build_corpus_service(corpus) + for corpus in filter(lambda corpus: corpus.status == 'start analysis', + corpora): + __create_cqpserver_container(corpus) + for corpus in filter(lambda corpus: corpus.status == 'stop analysis', + corpora): + __remove_cqpserver_container(corpus) + c_session.commit() + Session.remove() + + +def __create_build_corpus_service(corpus): + corpus_dir = os.path.join(NOPAQUE_STORAGE, str(corpus.user_id), + 'corpora', str(corpus.id)) + corpus_data_dir = os.path.join(corpus_dir, 'data') + corpus_file = os.path.join(corpus_dir, 'merged', 'corpus.vrt') + corpus_registry_dir = os.path.join(corpus_dir, 'registry') + if os.path.exists(corpus_data_dir): + shutil.rmtree(corpus_data_dir) + if os.path.exists(corpus_registry_dir): + shutil.rmtree(corpus_registry_dir) + os.mkdir(corpus_data_dir) + os.mkdir(corpus_registry_dir) + service_args = {'command': 'docker-entrypoint.sh build-corpus', + 'constraints': ['node.role==worker'], + 'labels': {'origin': 'nopaque', + 'type': 'corpus.prepare', + 'corpus_id': str(corpus.id)}, + 'mounts': [corpus_file + ':/root/files/corpus.vrt:ro', + corpus_data_dir + ':/corpora/data:rw', + corpus_registry_dir + ':/usr/local/share/cwb/registry:rw'], + 'name': 'build-corpus_{}'.format(corpus.id), + 'restart_policy': docker.types.RestartPolicy()} + service_image = ('gitlab.ub.uni-bielefeld.de:4567/sfb1288inf/cqpserver:latest') + try: + service = docker_client.services.get(service_args['name']) + except docker.errors.NotFound: + pass + except docker.errors.DockerException: + return + else: + service.remove() + try: + docker_client.services.create(service_image, **service_args) + except docker.errors.DockerException: + corpus.status = 'failed' + else: + corpus.status = 'queued' + + +def __checkout_build_corpus_service(corpus): + logger = init_logger() + service_name = 'build-corpus_{}'.format(corpus.id) + try: + service = docker_client.services.get(service_name) + except docker.errors.NotFound: + logger.error('__checkout_build_corpus_service({}):'.format(corpus.id) + + ' The service does not exist.' + + ' (stauts: {} -> failed)'.format(corpus.status)) + corpus.status = 'failed' + return + except docker.errors.DockerException: + return + service_tasks = service.tasks() + if not service_tasks: + return + task_state = service_tasks[0].get('Status').get('State') + if corpus.status == 'queued' and task_state != 'pending': + corpus.status = 'running' + elif corpus.status == 'running' and task_state == 'complete': + service.remove() + corpus.status = 'prepared' + elif corpus.status == 'running' and task_state == 'failed': + service.remove() + corpus.status = task_state + + +def __create_cqpserver_container(corpus): + corpus_dir = os.path.join(NOPAQUE_STORAGE, str(corpus.user_id), + 'corpora', str(corpus.id)) + corpus_data_dir = os.path.join(corpus_dir, 'data') + corpus_registry_dir = os.path.join(corpus_dir, 'registry') + container_args = {'command': 'cqpserver', + 'detach': True, + 'volumes': [corpus_data_dir + ':/corpora/data:rw', + corpus_registry_dir + ':/usr/local/share/cwb/registry:rw'], + 'name': 'cqpserver_{}'.format(corpus.id), + 'network': 'opaque_default'} + container_image = ('gitlab.ub.uni-bielefeld.de:4567/sfb1288inf/cqpserver:latest') + try: + container = docker_client.containers.get(container_args['name']) + except docker.errors.NotFound: + pass + except docker.errors.DockerException: + return + else: + container.remove(force=True) + try: + docker_client.containers.run(container_image, **container_args) + except docker.errors.DockerException: + return + else: + corpus.status = 'analysing' + + +def __remove_cqpserver_container(corpus): + container_name = 'cqpserver_{}'.format(corpus.id) + try: + container = docker_client.containers.get(container_name) + except docker.errors.NotFound: + pass + except docker.errors.DockerException: + return + else: + container.remove(force=True) + corpus.status = 'prepared' \ No newline at end of file diff --git a/daemon/tasks/check_jobs.py b/daemon/tasks/check_jobs.py new file mode 100644 index 00000000..e699e12e --- /dev/null +++ b/daemon/tasks/check_jobs.py @@ -0,0 +1,152 @@ +from datetime import datetime +from decorators import background +from logger.logger import init_logger +from tasks import Session, docker_client, NOPAQUE_STORAGE +from tasks.Models import Job, NotificationData, NotificationEmailData, JobResult +import docker +import json +import os + + +@background +def check_jobs(): + # logger = init_logger() + cj_session = Session() + jobs = cj_session.query(Job).all() + for job in filter(lambda job: job.status == 'submitted', jobs): + __create_job_service(job) + for job in filter(lambda job: (job.status == 'queued'), jobs): + __checkout_job_service(job, cj_session) + __add_notification_data(job, 'queued', cj_session) + for job in filter(lambda job: (job.status == 'running'), jobs): + __checkout_job_service(job, cj_session) + __add_notification_data(job, 'running', cj_session) + for job in filter(lambda job: job.status == 'complete', jobs): + __add_notification_data(job, 'complete', cj_session) + for job in filter(lambda job: job.status == 'failed', jobs): + __add_notification_data(job, 'failed', cj_session) + for job in filter(lambda job: job.status == 'canceling', jobs): + __remove_job_service(job) + cj_session.commit() + Session.remove() + + +def __add_notification_data(job, notified_on_status, scoped_session): + logger = init_logger() + # checks if user wants any notifications at all + if (job.user.setting_job_status_mail_notifications == 'none'): + # logger.warning('User does not want any notifications!') + return + # checks if user wants only notification on completed jobs + elif (job.user.setting_job_status_mail_notifications == 'end' + and notified_on_status != 'complete'): + # logger.warning('User only wants notifications on job completed!') + return + else: + # check if a job already has associated NotificationData + notification_exists = len(job.notification_data) + # create notification_data for current job if there is none + if (notification_exists == 0): + notification_data = NotificationData(job_id=job.id) + scoped_session.add(notification_data) + scoped_session.commit() # If no commit job will have no NotificationData + # logger.warning('Created NotificationData for current Job.')) + else: + pass + # logger.warning('Job already had notification: {}'.format(notification_exists)) + if (job.notification_data[0].notified_on != notified_on_status): + notification_email_data = NotificationEmailData(job_id=job.id) + notification_email_data.notify_status = notified_on_status + notification_email_data.creation_date = datetime.utcnow() + job.notification_data[0].notified_on = notified_on_status + scoped_session.add(notification_email_data) + logger.warning('Created NotificationEmailData for current Job.') + else: + # logger.warning('NotificationEmailData has already been created for current Job!') + pass + + +def __create_job_service(job): + job_dir = os.path.join(NOPAQUE_STORAGE, str(job.user_id), 'jobs', + str(job.id)) + service_args = {'command': ('{} /files /files/output'.format(job.service) + + ' {}'.format(job.secure_filename if job.service == 'file-setup' else '') + + ' --log-dir /files' + + ' --zip [{}]_{}'.format(job.service, job.secure_filename) + + ' ' + ' '.join(json.loads(job.service_args))), + 'constraints': ['node.role==worker'], + 'labels': {'origin': 'nopaque', + 'type': 'service.{}'.format(job.service), + 'job_id': str(job.id)}, + 'mounts': [job_dir + ':/files:rw'], + 'name': 'job_{}'.format(job.id), + 'resources': docker.types.Resources( + cpu_reservation=job.n_cores * (10 ** 9), + mem_reservation=job.mem_mb * (10 ** 6)), + 'restart_policy': docker.types.RestartPolicy()} + service_image = ('gitlab.ub.uni-bielefeld.de:4567/sfb1288inf/' + + job.service + ':' + job.service_version) + try: + service = docker_client.services.get(service_args['name']) + except docker.errors.NotFound: + pass + except docker.errors.DockerException: + return + else: + service.remove() + try: + docker_client.services.create(service_image, **service_args) + except docker.errors.DockerException: + job.status = 'failed' + else: + job.status = 'queued' + + +def __checkout_job_service(job, scoped_session): + logger = init_logger() + scoped_session = Session() + service_name = 'job_{}'.format(job.id) + try: + service = docker_client.services.get(service_name) + except docker.errors.NotFound: + logger.error('__checkout_job_service({}):'.format(job.id) + + ' The service does not exist.' + + ' (stauts: {} -> failed)'.format(job.status)) + job.status = 'failed' + return + except docker.errors.DockerException: + return + service_tasks = service.tasks() + if not service_tasks: + return + task_state = service_tasks[0].get('Status').get('State') + if job.status == 'queued' and task_state != 'pending': + job.status = 'running' + elif (job.status == 'running' + and (task_state == 'complete' or task_state == 'failed')): + service.remove() + job.end_date = datetime.utcnow() + job.status = task_state + if task_state == 'complete': + results_dir = os.path.join(NOPAQUE_STORAGE, str(job.user_id), + 'jobs', str(job.id), 'output') + results = filter(lambda x: x.endswith('.zip'), + os.listdir(results_dir)) + for result in results: + job_result = JobResult(dir=results_dir, filename=result, + job_id=job.id) + scoped_session.add(job_result) + scoped_session.commit() + + +def __remove_job_service(job): + service_name = 'job_{}'.format(job.id) + try: + service = docker_client.services.get(service_name) + except docker.errors.NotFound: + job.status = 'canceled' + except docker.errors.DockerException: + return + else: + service.update(mounts=None) + service.remove() \ No newline at end of file diff --git a/daemon/tasks/notify.py b/daemon/tasks/notify.py new file mode 100644 index 00000000..e28b66a3 --- /dev/null +++ b/daemon/tasks/notify.py @@ -0,0 +1,87 @@ +from decorators import background +from notify.notification import Notification +from notify.service import NotificationService +from sqlalchemy import asc +from tasks import Session +from tasks.Models import NotificationEmailData +import os + + +# Email notification functions +def __create_mail_notifications(notification_service): + mn_session = Session() + notification_email_data = mn_session.query(NotificationEmailData).order_by(asc(NotificationEmailData.creation_date)).all() + notifications = {} + for data in notification_email_data: + notification = Notification() + notification.set_addresses(notification_service.email_address, + data.job.user.email) + subject_template = '[nopaque] Status update for your Job/Corpora: {title}!' + subject_template_values_dict = {'title': data.job.title} + domain = os.environ.get('NOPAQUE_DOMAIN') + url = '{domain}/{jobs}/{id}'.format(domain=domain, + jobs='jobs', + id=data.job.id) + body_template_values_dict = {'username': data.job.user.username, + 'id': data.job.id, + 'title': data.job.title, + 'status': data.notify_status, + 'time': data.creation_date, + 'url': url} + notification.set_notification_content(subject_template, + subject_template_values_dict, + 'notify/templates/notification_messages/notification.txt', + 'notify/templates/notification_messages/notification.html', + body_template_values_dict) + notifications[data.job.id] = notification + # Using a dictionary for notifications avoids sending multiple mails + # if the status of a job changes in a few seconds. The user will not get + # swamped with mails for queued, running and complete if those happen in + # in a few seconds. Only the last update will be sent. This depends on + # the sleep time interval though. + mn_session.delete(data) + mn_session.commit() + Session.remove() + return notifications + + +def __send_mail_notifications(notifications, notification_service): + for key, notification in notifications.items(): + try: + notification_service.send(notification) + notification_service.mail_limit_exceeded = False + except Exception as e: + # Adds notifications to unsent if mail server exceded limit for + # consecutive mail sending + notification_service.not_sent[key] = notification + notification_service.mail_limit_exceeded = True + + +@background +def notify(execute_flag): + # If True mails are sent normaly + # If False mails are not sent. Used to avoid sending mails for jobs that + # have been completed a long time ago. Use this if you implement notify + # into an already existing nopaque instance. Change it to True after the + # daemon has run one time with the flag set to False. + # Initialize notification service + notification_service = NotificationService(execute_flag) + notification_service.get_smtp_configs() + notification_service.set_server() + # create notifications (content, recipient etc.) + notifications = __create_mail_notifications(notification_service) + # only login and send mails if there are any notifications + if (len(notifications) > 0): + try: + notification_service.login() + # combine new and unsent notifications + notifications.update(notification_service.not_sent) + # send all notifications + __send_mail_notifications(notifications, notification_service) + # remove unsent notifications because they have been sent now + # but only if mail limit has not been exceeded + if (notification_service.mail_limit_exceeded is not True): + notification_service.not_sent = {} + notification_service.quit() + except Exception as e: + notification_service.not_sent.update(notifications) \ No newline at end of file diff --git a/docker-compose.yml b/docker-compose.yml index 1ffdeab8..98ae4143 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -10,11 +10,16 @@ volumes: services: web: + build: + args: + gid: ${gid} + uid: ${uid} + context: ./web depends_on: - db - redis env_file: nopaque.env - image: gitlab.ub.uni-bielefeld.de:4567/sfb1288inf/opaque:development + image: nopaque/web labels: - "traefik.docker.network=reverse-proxy" - "traefik.enable=true" @@ -22,13 +27,13 @@ services: - "traefik.http.middlewares.nopaque-header.headers.customrequestheaders.X-Forwarded-Proto=http" - "traefik.http.routers.nopaque.entrypoints=web" - "traefik.http.routers.nopaque.middlewares=nopaque-header, redirect-to-https@file" - - "traefik.http.routers.nopaque.rule=Host(`nopaque.localhost`)" # Change this to match your nopaque domain + - "traefik.http.routers.nopaque.rule=Host(`nopaque.localhost`)" ### ### ### ### - "traefik.http.middlewares.nopaque-secure-header.headers.customrequestheaders.X-Forwarded-Proto=https" - "traefik.http.routers.nopaque-secure.entrypoints=web-secure" - "traefik.http.routers.nopaque-secure.middlewares=hsts-header@file, nopaque-secure-header" - - "traefik.http.routers.nopaque-secure.rule=Host(`nopaque.localhost`)" # Change this to match your nopaque domain + - "traefik.http.routers.nopaque-secure.rule=Host(`nopaque.localhost`)" - "traefik.http.routers.nopaque-secure.tls.options=intermediate@file" ### ### ### ### @@ -41,32 +46,35 @@ services: - reverse-proxy volumes: - "/mnt/dind-swarm/nopaque:/mnt/dind-swarm/nopaque" - - "./app:/home/nopaque/app" - "./logs:/home/nopaque/logs" - - "./migrations:/home/nopaque/migrations" - - "./tests:/home/nopaque/tests" - - "./config.py:/home/nopaque/config.py" - - "./docker-entrypoint.sh:/usr/local/bin/docker-entrypoint.sh" - - "./nopaque.py:/home/nopaque/nopaque.py" - - "./requirements.txt:/home/nopaque/requirements.txt" + - "./web/app:/home/nopaque/app" + - "./web/migrations:/home/nopaque/migrations" + - "./web/tests:/home/nopaque/tests" + - "./web/config.py:/home/nopaque/config.py" + - "./web/docker-entrypoint.sh:/usr/local/bin/docker-entrypoint.sh" + - "./web/nopaque.py:/home/nopaque/nopaque.py" + - "./web/requirements.txt:/home/nopaque/requirements.txt" daemon: + build: + args: + docker_gid: ${docker_gid} + gid: ${gid} + uid: ${uid} + context: ./daemon depends_on: - db - web env_file: nopaque.env - extra_hosts: - - "host.docker.internal:172.17.0.1" - image: gitlab.ub.uni-bielefeld.de:4567/sfb1288inf/opaque_daemon:latest + image: nopaque/daemon volumes: - "/mnt/dind-swarm/nopaque:/mnt/dind-swarm/nopaque" + - "/var/run/docker.sock:/var/run/docker.sock" - "./logs:/home/nopaqued/logs" - - "../opaque_daemon/docker-entrypoint.sh:/usr/local/bin/docker-entrypoint.sh" - - "../opaque_daemon/nopaqued.py:/home/nopaqued/nopaqued.py" - - "../opaque_daemon/decorators.py:/home/nopaqued/decorators.py" - - "../opaque_daemon/notify:/home/nopaqued/notify" - - "../opaque_daemon/templates:/home/nopaqued/templates" - - "../opaque_daemon/requirements.txt:/home/nopaqued/requirements.txt" - - "$HOME/.docker:/home/nopaqued/.docker" + - "./daemon/notify:/home/nopaqued/notify" + - "./daemon/decorators.py:/home/nopaqued/decorators.py" + - "./daemon/docker-entrypoint.sh:/usr/local/bin/docker-entrypoint.sh" + - "./daemon/nopaqued.py:/home/nopaqued/nopaqued.py" + - "./daemon/requirements.txt:/home/nopaqued/requirements.txt" db: env_file: nopaque.env image: postgres:11 diff --git a/nopaque.env.tpl b/nopaque.env.tpl index 7f8dbcf8..ffea48ba 100644 --- a/nopaque.env.tpl +++ b/nopaque.env.tpl @@ -1,21 +1,18 @@ -### PostgreSQL ### -POSTGRES_DB_NAME= -POSTGRES_USER= -POSTGRES_PASSWORD= - ### Docker ### -DOCKER_CERT_PATH= -DOCKER_HOST= -DOCKER_TLS_VERIFY= - -### GitLab Registry ### -GITLAB_USERNAME= -GITLAB_PASSWORD= +# Fill out these variables to use the Docker HTTP socket. When doing this, you +# can remove the Docker UNIX socket mount from the docker-compose file. +# DOCKER_CERT_PATH= +# DOCKER_HOST= +# DOCKER_TLS_VERIFY= ### Flask ### FLASK_CONFIG= SECRET_KEY= +### GitLab Registry ### +GITLAB_USERNAME= +GITLAB_PASSWORD= + ### Flask-Mail ### MAIL_SERVER= MAIL_PORT= @@ -27,5 +24,11 @@ MAIL_PASSWORD= NOPAQUE_ADMIN= NOPAQUE_CONTACT= NOPAQUE_DOMAIN= +NOPAQUE_LOG_LEVEL= NOPAQUE_MAIL_SENDER= NOPAQUE_STORAGE= + +### PostgreSQL ### +POSTGRES_DB_NAME= +POSTGRES_USER= +POSTGRES_PASSWORD= diff --git a/Dockerfile b/web/Dockerfile similarity index 80% rename from Dockerfile rename to web/Dockerfile index 499c8f65..80ab3920 100644 --- a/Dockerfile +++ b/web/Dockerfile @@ -4,6 +4,8 @@ FROM python:3.6-slim-stretch LABEL maintainer="inf_sfb1288@lists.uni-bielefeld.de" +ARG uid=1000 +ARG gid=1000 ENV FLASK_APP=nopaque.py ENV LANG=C.UTF-8 @@ -19,8 +21,8 @@ RUN apt-get update \ && rm -rf /var/lib/apt/lists/* -RUN groupadd --gid 1000 --system nopaque \ - && useradd --create-home --gid nopaque --no-log-init --system --uid 1000 nopaque +RUN groupadd --gid "${gid}" --system nopaque \ + && useradd --create-home --gid "${gid}" --no-log-init --system --uid "${uid}" nopaque USER nopaque WORKDIR /home/nopaque diff --git a/app/__init__.py b/web/app/__init__.py similarity index 100% rename from app/__init__.py rename to web/app/__init__.py diff --git a/app/admin/__init__.py b/web/app/admin/__init__.py similarity index 100% rename from app/admin/__init__.py rename to web/app/admin/__init__.py diff --git a/app/admin/forms.py b/web/app/admin/forms.py similarity index 100% rename from app/admin/forms.py rename to web/app/admin/forms.py diff --git a/app/admin/tables.py b/web/app/admin/tables.py similarity index 100% rename from app/admin/tables.py rename to web/app/admin/tables.py diff --git a/app/admin/views.py b/web/app/admin/views.py similarity index 100% rename from app/admin/views.py rename to web/app/admin/views.py diff --git a/app/auth/__init__.py b/web/app/auth/__init__.py similarity index 100% rename from app/auth/__init__.py rename to web/app/auth/__init__.py diff --git a/app/auth/forms.py b/web/app/auth/forms.py similarity index 100% rename from app/auth/forms.py rename to web/app/auth/forms.py diff --git a/app/auth/views.py b/web/app/auth/views.py similarity index 100% rename from app/auth/views.py rename to web/app/auth/views.py diff --git a/app/corpora/__init__.py b/web/app/corpora/__init__.py similarity index 100% rename from app/corpora/__init__.py rename to web/app/corpora/__init__.py diff --git a/app/corpora/events.py b/web/app/corpora/events.py similarity index 100% rename from app/corpora/events.py rename to web/app/corpora/events.py diff --git a/app/corpora/forms.py b/web/app/corpora/forms.py similarity index 100% rename from app/corpora/forms.py rename to web/app/corpora/forms.py diff --git a/app/corpora/tasks.py b/web/app/corpora/tasks.py similarity index 100% rename from app/corpora/tasks.py rename to web/app/corpora/tasks.py diff --git a/app/corpora/views.py b/web/app/corpora/views.py similarity index 100% rename from app/corpora/views.py rename to web/app/corpora/views.py diff --git a/app/decorators.py b/web/app/decorators.py similarity index 100% rename from app/decorators.py rename to web/app/decorators.py diff --git a/app/email.py b/web/app/email.py similarity index 100% rename from app/email.py rename to web/app/email.py diff --git a/app/events.py b/web/app/events.py similarity index 100% rename from app/events.py rename to web/app/events.py diff --git a/app/jobs/__init__.py b/web/app/jobs/__init__.py similarity index 100% rename from app/jobs/__init__.py rename to web/app/jobs/__init__.py diff --git a/app/jobs/forms.py b/web/app/jobs/forms.py similarity index 100% rename from app/jobs/forms.py rename to web/app/jobs/forms.py diff --git a/app/jobs/tasks.py b/web/app/jobs/tasks.py similarity index 100% rename from app/jobs/tasks.py rename to web/app/jobs/tasks.py diff --git a/app/jobs/views.py b/web/app/jobs/views.py similarity index 100% rename from app/jobs/views.py rename to web/app/jobs/views.py diff --git a/app/main/__init__.py b/web/app/main/__init__.py similarity index 100% rename from app/main/__init__.py rename to web/app/main/__init__.py diff --git a/app/main/errors.py b/web/app/main/errors.py similarity index 100% rename from app/main/errors.py rename to web/app/main/errors.py diff --git a/app/main/forms.py b/web/app/main/forms.py similarity index 100% rename from app/main/forms.py rename to web/app/main/forms.py diff --git a/app/main/views.py b/web/app/main/views.py similarity index 100% rename from app/main/views.py rename to web/app/main/views.py diff --git a/app/models.py b/web/app/models.py similarity index 100% rename from app/models.py rename to web/app/models.py diff --git a/app/profile/__init__.py b/web/app/profile/__init__.py similarity index 100% rename from app/profile/__init__.py rename to web/app/profile/__init__.py diff --git a/app/profile/forms.py b/web/app/profile/forms.py similarity index 100% rename from app/profile/forms.py rename to web/app/profile/forms.py diff --git a/app/profile/tasks.py b/web/app/profile/tasks.py similarity index 100% rename from app/profile/tasks.py rename to web/app/profile/tasks.py diff --git a/app/profile/views.py b/web/app/profile/views.py similarity index 100% rename from app/profile/views.py rename to web/app/profile/views.py diff --git a/app/services/__init__.py b/web/app/services/__init__.py similarity index 100% rename from app/services/__init__.py rename to web/app/services/__init__.py diff --git a/app/services/views.py b/web/app/services/views.py similarity index 100% rename from app/services/views.py rename to web/app/services/views.py diff --git a/app/static/css/Materialize/LICENSE b/web/app/static/css/Materialize/LICENSE similarity index 100% rename from app/static/css/Materialize/LICENSE rename to web/app/static/css/Materialize/LICENSE diff --git a/app/static/css/Materialize/materialize.min.css b/web/app/static/css/Materialize/materialize.min.css similarity index 100% rename from app/static/css/Materialize/materialize.min.css rename to web/app/static/css/Materialize/materialize.min.css diff --git a/app/static/css/nopaque.css b/web/app/static/css/nopaque.css similarity index 100% rename from app/static/css/nopaque.css rename to web/app/static/css/nopaque.css diff --git a/app/static/fonts/Material_design_icons/LICENSE b/web/app/static/fonts/Material_design_icons/LICENSE similarity index 100% rename from app/static/fonts/Material_design_icons/LICENSE rename to web/app/static/fonts/Material_design_icons/LICENSE diff --git a/app/static/fonts/Material_design_icons/MaterialIcons-Regular.eot b/web/app/static/fonts/Material_design_icons/MaterialIcons-Regular.eot similarity index 100% rename from app/static/fonts/Material_design_icons/MaterialIcons-Regular.eot rename to web/app/static/fonts/Material_design_icons/MaterialIcons-Regular.eot diff --git a/app/static/fonts/Material_design_icons/MaterialIcons-Regular.ttf b/web/app/static/fonts/Material_design_icons/MaterialIcons-Regular.ttf similarity index 100% rename from app/static/fonts/Material_design_icons/MaterialIcons-Regular.ttf rename to web/app/static/fonts/Material_design_icons/MaterialIcons-Regular.ttf diff --git a/app/static/fonts/Material_design_icons/MaterialIcons-Regular.woff b/web/app/static/fonts/Material_design_icons/MaterialIcons-Regular.woff similarity index 100% rename from app/static/fonts/Material_design_icons/MaterialIcons-Regular.woff rename to web/app/static/fonts/Material_design_icons/MaterialIcons-Regular.woff diff --git a/app/static/fonts/Material_design_icons/MaterialIcons-Regular.woff2 b/web/app/static/fonts/Material_design_icons/MaterialIcons-Regular.woff2 similarity index 100% rename from app/static/fonts/Material_design_icons/MaterialIcons-Regular.woff2 rename to web/app/static/fonts/Material_design_icons/MaterialIcons-Regular.woff2 diff --git a/app/static/fonts/Material_design_icons/material-icons.css b/web/app/static/fonts/Material_design_icons/material-icons.css similarity index 100% rename from app/static/fonts/Material_design_icons/material-icons.css rename to web/app/static/fonts/Material_design_icons/material-icons.css diff --git a/app/static/images/logo_-_dfg.gif b/web/app/static/images/logo_-_dfg.gif similarity index 100% rename from app/static/images/logo_-_dfg.gif rename to web/app/static/images/logo_-_dfg.gif diff --git a/app/static/images/logo_-_sfb_1288.png b/web/app/static/images/logo_-_sfb_1288.png similarity index 100% rename from app/static/images/logo_-_sfb_1288.png rename to web/app/static/images/logo_-_sfb_1288.png diff --git a/app/static/images/nopaque_logo.png b/web/app/static/images/nopaque_logo.png similarity index 100% rename from app/static/images/nopaque_logo.png rename to web/app/static/images/nopaque_logo.png diff --git a/app/static/images/parallax_hq/book_text_read_paper.jpg b/web/app/static/images/parallax_hq/book_text_read_paper.jpg similarity index 100% rename from app/static/images/parallax_hq/book_text_read_paper.jpg rename to web/app/static/images/parallax_hq/book_text_read_paper.jpg diff --git a/app/static/images/parallax_hq/books_antique_book_old.jpg b/web/app/static/images/parallax_hq/books_antique_book_old.jpg similarity index 100% rename from app/static/images/parallax_hq/books_antique_book_old.jpg rename to web/app/static/images/parallax_hq/books_antique_book_old.jpg diff --git a/app/static/images/parallax_hq/concept_document_focus_letter.jpg b/web/app/static/images/parallax_hq/concept_document_focus_letter.jpg similarity index 100% rename from app/static/images/parallax_hq/concept_document_focus_letter.jpg rename to web/app/static/images/parallax_hq/concept_document_focus_letter.jpg diff --git a/app/static/images/parallax_hq/text_data_wide.png b/web/app/static/images/parallax_hq/text_data_wide.png similarity index 100% rename from app/static/images/parallax_hq/text_data_wide.png rename to web/app/static/images/parallax_hq/text_data_wide.png diff --git a/app/static/images/parallax_lq/01_books_antique_book_old.jpg b/web/app/static/images/parallax_lq/01_books_antique_book_old.jpg similarity index 100% rename from app/static/images/parallax_lq/01_books_antique_book_old.jpg rename to web/app/static/images/parallax_lq/01_books_antique_book_old.jpg diff --git a/app/static/images/parallax_lq/02_concept_document_focus_letter.jpg b/web/app/static/images/parallax_lq/02_concept_document_focus_letter.jpg similarity index 100% rename from app/static/images/parallax_lq/02_concept_document_focus_letter.jpg rename to web/app/static/images/parallax_lq/02_concept_document_focus_letter.jpg diff --git a/app/static/images/parallax_lq/03_text_data_wide.png b/web/app/static/images/parallax_lq/03_text_data_wide.png similarity index 100% rename from app/static/images/parallax_lq/03_text_data_wide.png rename to web/app/static/images/parallax_lq/03_text_data_wide.png diff --git a/app/static/images/parallax_lq/04_german_text_book_paper.jpg b/web/app/static/images/parallax_lq/04_german_text_book_paper.jpg similarity index 100% rename from app/static/images/parallax_lq/04_german_text_book_paper.jpg rename to web/app/static/images/parallax_lq/04_german_text_book_paper.jpg diff --git a/app/static/images/parallax_lq/05_chapter_book_text_tale.jpg b/web/app/static/images/parallax_lq/05_chapter_book_text_tale.jpg similarity index 100% rename from app/static/images/parallax_lq/05_chapter_book_text_tale.jpg rename to web/app/static/images/parallax_lq/05_chapter_book_text_tale.jpg diff --git a/app/static/images/parallax_lq/bible_text.jpg b/web/app/static/images/parallax_lq/bible_text.jpg similarity index 100% rename from app/static/images/parallax_lq/bible_text.jpg rename to web/app/static/images/parallax_lq/bible_text.jpg diff --git a/app/static/images/parallax_lq/text_data.png b/web/app/static/images/parallax_lq/text_data.png similarity index 100% rename from app/static/images/parallax_lq/text_data.png rename to web/app/static/images/parallax_lq/text_data.png diff --git a/app/static/images/qr_-_inf.svg b/web/app/static/images/qr_-_inf.svg similarity index 100% rename from app/static/images/qr_-_inf.svg rename to web/app/static/images/qr_-_inf.svg diff --git a/app/static/images/server_activity.png b/web/app/static/images/server_activity.png similarity index 100% rename from app/static/images/server_activity.png rename to web/app/static/images/server_activity.png diff --git a/app/static/images/sfb_background.jpeg b/web/app/static/images/sfb_background.jpeg similarity index 100% rename from app/static/images/sfb_background.jpeg rename to web/app/static/images/sfb_background.jpeg diff --git a/app/static/js/Dark_Reader/LICENSE b/web/app/static/js/Dark_Reader/LICENSE similarity index 100% rename from app/static/js/Dark_Reader/LICENSE rename to web/app/static/js/Dark_Reader/LICENSE diff --git a/app/static/js/Dark_Reader/darkreader.js b/web/app/static/js/Dark_Reader/darkreader.js similarity index 100% rename from app/static/js/Dark_Reader/darkreader.js rename to web/app/static/js/Dark_Reader/darkreader.js diff --git a/app/static/js/JSONPatch.js/LICENSE b/web/app/static/js/JSONPatch.js/LICENSE similarity index 100% rename from app/static/js/JSONPatch.js/LICENSE rename to web/app/static/js/JSONPatch.js/LICENSE diff --git a/app/static/js/JSONPatch.js/jsonpatch.min.js b/web/app/static/js/JSONPatch.js/jsonpatch.min.js similarity index 100% rename from app/static/js/JSONPatch.js/jsonpatch.min.js rename to web/app/static/js/JSONPatch.js/jsonpatch.min.js diff --git a/app/static/js/List.js/LICENSE b/web/app/static/js/List.js/LICENSE similarity index 100% rename from app/static/js/List.js/LICENSE rename to web/app/static/js/List.js/LICENSE diff --git a/app/static/js/List.js/list.min.js b/web/app/static/js/List.js/list.min.js similarity index 100% rename from app/static/js/List.js/list.min.js rename to web/app/static/js/List.js/list.min.js diff --git a/app/static/js/Materialize/LICENSE b/web/app/static/js/Materialize/LICENSE similarity index 100% rename from app/static/js/Materialize/LICENSE rename to web/app/static/js/Materialize/LICENSE diff --git a/app/static/js/Materialize/materialize.min.js b/web/app/static/js/Materialize/materialize.min.js similarity index 100% rename from app/static/js/Materialize/materialize.min.js rename to web/app/static/js/Materialize/materialize.min.js diff --git a/app/static/js/Socket.IO/LICENSE b/web/app/static/js/Socket.IO/LICENSE similarity index 100% rename from app/static/js/Socket.IO/LICENSE rename to web/app/static/js/Socket.IO/LICENSE diff --git a/app/static/js/Socket.IO/socket.io.slim.js b/web/app/static/js/Socket.IO/socket.io.slim.js similarity index 100% rename from app/static/js/Socket.IO/socket.io.slim.js rename to web/app/static/js/Socket.IO/socket.io.slim.js diff --git a/app/static/js/Socket.IO/socket.io.slim.js.map b/web/app/static/js/Socket.IO/socket.io.slim.js.map similarity index 100% rename from app/static/js/Socket.IO/socket.io.slim.js.map rename to web/app/static/js/Socket.IO/socket.io.slim.js.map diff --git a/app/static/js/nopaque.CorpusAnalysisClient.js b/web/app/static/js/nopaque.CorpusAnalysisClient.js similarity index 100% rename from app/static/js/nopaque.CorpusAnalysisClient.js rename to web/app/static/js/nopaque.CorpusAnalysisClient.js diff --git a/app/static/js/nopaque.Results.js b/web/app/static/js/nopaque.Results.js similarity index 100% rename from app/static/js/nopaque.Results.js rename to web/app/static/js/nopaque.Results.js diff --git a/app/static/js/nopaque.callbacks.js b/web/app/static/js/nopaque.callbacks.js similarity index 100% rename from app/static/js/nopaque.callbacks.js rename to web/app/static/js/nopaque.callbacks.js diff --git a/app/static/js/nopaque.js b/web/app/static/js/nopaque.js similarity index 100% rename from app/static/js/nopaque.js rename to web/app/static/js/nopaque.js diff --git a/app/static/js/nopaque.lists.js b/web/app/static/js/nopaque.lists.js similarity index 100% rename from app/static/js/nopaque.lists.js rename to web/app/static/js/nopaque.lists.js diff --git a/app/templates/403.html.j2 b/web/app/templates/403.html.j2 similarity index 100% rename from app/templates/403.html.j2 rename to web/app/templates/403.html.j2 diff --git a/app/templates/404.html.j2 b/web/app/templates/404.html.j2 similarity index 100% rename from app/templates/404.html.j2 rename to web/app/templates/404.html.j2 diff --git a/app/templates/500.html.j2 b/web/app/templates/500.html.j2 similarity index 100% rename from app/templates/500.html.j2 rename to web/app/templates/500.html.j2 diff --git a/app/templates/admin/edit_user.html.j2 b/web/app/templates/admin/edit_user.html.j2 similarity index 100% rename from app/templates/admin/edit_user.html.j2 rename to web/app/templates/admin/edit_user.html.j2 diff --git a/app/templates/admin/index.html.j2 b/web/app/templates/admin/index.html.j2 similarity index 100% rename from app/templates/admin/index.html.j2 rename to web/app/templates/admin/index.html.j2 diff --git a/app/templates/admin/user.html.j2 b/web/app/templates/admin/user.html.j2 similarity index 100% rename from app/templates/admin/user.html.j2 rename to web/app/templates/admin/user.html.j2 diff --git a/app/templates/auth/email/confirm.html.j2 b/web/app/templates/auth/email/confirm.html.j2 similarity index 100% rename from app/templates/auth/email/confirm.html.j2 rename to web/app/templates/auth/email/confirm.html.j2 diff --git a/app/templates/auth/email/confirm.txt.j2 b/web/app/templates/auth/email/confirm.txt.j2 similarity index 100% rename from app/templates/auth/email/confirm.txt.j2 rename to web/app/templates/auth/email/confirm.txt.j2 diff --git a/app/templates/auth/email/reset_password.html.j2 b/web/app/templates/auth/email/reset_password.html.j2 similarity index 100% rename from app/templates/auth/email/reset_password.html.j2 rename to web/app/templates/auth/email/reset_password.html.j2 diff --git a/app/templates/auth/email/reset_password.txt.j2 b/web/app/templates/auth/email/reset_password.txt.j2 similarity index 100% rename from app/templates/auth/email/reset_password.txt.j2 rename to web/app/templates/auth/email/reset_password.txt.j2 diff --git a/app/templates/auth/login.html.j2 b/web/app/templates/auth/login.html.j2 similarity index 100% rename from app/templates/auth/login.html.j2 rename to web/app/templates/auth/login.html.j2 diff --git a/app/templates/auth/register.html.j2 b/web/app/templates/auth/register.html.j2 similarity index 100% rename from app/templates/auth/register.html.j2 rename to web/app/templates/auth/register.html.j2 diff --git a/app/templates/auth/reset_password.html.j2 b/web/app/templates/auth/reset_password.html.j2 similarity index 100% rename from app/templates/auth/reset_password.html.j2 rename to web/app/templates/auth/reset_password.html.j2 diff --git a/app/templates/auth/reset_password_request.html.j2 b/web/app/templates/auth/reset_password_request.html.j2 similarity index 100% rename from app/templates/auth/reset_password_request.html.j2 rename to web/app/templates/auth/reset_password_request.html.j2 diff --git a/app/templates/auth/unconfirmed.html.j2 b/web/app/templates/auth/unconfirmed.html.j2 similarity index 100% rename from app/templates/auth/unconfirmed.html.j2 rename to web/app/templates/auth/unconfirmed.html.j2 diff --git a/app/templates/corpora/add_corpus.html.j2 b/web/app/templates/corpora/add_corpus.html.j2 similarity index 100% rename from app/templates/corpora/add_corpus.html.j2 rename to web/app/templates/corpora/add_corpus.html.j2 diff --git a/app/templates/corpora/add_corpus_file.html.j2 b/web/app/templates/corpora/add_corpus_file.html.j2 similarity index 100% rename from app/templates/corpora/add_corpus_file.html.j2 rename to web/app/templates/corpora/add_corpus_file.html.j2 diff --git a/app/templates/corpora/analyse_corpus.html.j2 b/web/app/templates/corpora/analyse_corpus.html.j2 similarity index 100% rename from app/templates/corpora/analyse_corpus.html.j2 rename to web/app/templates/corpora/analyse_corpus.html.j2 diff --git a/app/templates/corpora/corpus.html.j2 b/web/app/templates/corpora/corpus.html.j2 similarity index 100% rename from app/templates/corpora/corpus.html.j2 rename to web/app/templates/corpora/corpus.html.j2 diff --git a/app/templates/corpora/edit_corpus_file.html.j2 b/web/app/templates/corpora/edit_corpus_file.html.j2 similarity index 100% rename from app/templates/corpora/edit_corpus_file.html.j2 rename to web/app/templates/corpora/edit_corpus_file.html.j2 diff --git a/app/templates/jobs/job.html.j2 b/web/app/templates/jobs/job.html.j2 similarity index 100% rename from app/templates/jobs/job.html.j2 rename to web/app/templates/jobs/job.html.j2 diff --git a/app/templates/macros/materialize.html.j2 b/web/app/templates/macros/materialize.html.j2 similarity index 100% rename from app/templates/macros/materialize.html.j2 rename to web/app/templates/macros/materialize.html.j2 diff --git a/app/templates/main/dashboard.html.j2 b/web/app/templates/main/dashboard.html.j2 similarity index 100% rename from app/templates/main/dashboard.html.j2 rename to web/app/templates/main/dashboard.html.j2 diff --git a/app/templates/main/feedback.html.j2 b/web/app/templates/main/feedback.html.j2 similarity index 100% rename from app/templates/main/feedback.html.j2 rename to web/app/templates/main/feedback.html.j2 diff --git a/app/templates/main/index.html.j2 b/web/app/templates/main/index.html.j2 similarity index 100% rename from app/templates/main/index.html.j2 rename to web/app/templates/main/index.html.j2 diff --git a/app/templates/main/poster.html.j2 b/web/app/templates/main/poster.html.j2 similarity index 100% rename from app/templates/main/poster.html.j2 rename to web/app/templates/main/poster.html.j2 diff --git a/app/templates/main/privacy_policy.html.j2 b/web/app/templates/main/privacy_policy.html.j2 similarity index 100% rename from app/templates/main/privacy_policy.html.j2 rename to web/app/templates/main/privacy_policy.html.j2 diff --git a/app/templates/nopaque.html.j2 b/web/app/templates/nopaque.html.j2 similarity index 100% rename from app/templates/nopaque.html.j2 rename to web/app/templates/nopaque.html.j2 diff --git a/app/templates/profile/settings.html.j2 b/web/app/templates/profile/settings.html.j2 similarity index 100% rename from app/templates/profile/settings.html.j2 rename to web/app/templates/profile/settings.html.j2 diff --git a/app/templates/services/corpus_analysis.html.j2 b/web/app/templates/services/corpus_analysis.html.j2 similarity index 100% rename from app/templates/services/corpus_analysis.html.j2 rename to web/app/templates/services/corpus_analysis.html.j2 diff --git a/app/templates/services/file-setup.html.j2 b/web/app/templates/services/file-setup.html.j2 similarity index 100% rename from app/templates/services/file-setup.html.j2 rename to web/app/templates/services/file-setup.html.j2 diff --git a/app/templates/services/nlp.html.j2 b/web/app/templates/services/nlp.html.j2 similarity index 100% rename from app/templates/services/nlp.html.j2 rename to web/app/templates/services/nlp.html.j2 diff --git a/app/templates/services/ocr.html.j2 b/web/app/templates/services/ocr.html.j2 similarity index 100% rename from app/templates/services/ocr.html.j2 rename to web/app/templates/services/ocr.html.j2 diff --git a/app/templates/services/roadmap.html.j2 b/web/app/templates/services/roadmap.html.j2 similarity index 100% rename from app/templates/services/roadmap.html.j2 rename to web/app/templates/services/roadmap.html.j2 diff --git a/config.py b/web/config.py similarity index 90% rename from config.py rename to web/config.py index 07b748c6..5ccd09f1 100644 --- a/config.py +++ b/web/config.py @@ -20,6 +20,10 @@ class Config: MAIL_PASSWORD = os.environ.get('MAIL_PASSWORD') ''' ### Flask-SQLAlchemy ### ''' + SQLALCHEMY_DATABASE_URI = 'postgresql://{}:{}@db/{}'.format( + os.environ.get('POSTGRES_USER'), + os.environ.get('POSTGRES_PASSWORD'), + os.environ.get('POSTGRES_DB_NAME')) SQLALCHEMY_RECORD_QUERIES = True SQLALCHEMY_TRACK_MODIFICATIONS = False @@ -52,12 +56,6 @@ class DevelopmentConfig(Config): ''' ### Flask ### ''' DEBUG = True - ''' ### Flask-SQLAlchemy ### ''' - SQLALCHEMY_DATABASE_URI = 'postgresql://{}:{}@db/{}'.format( - os.environ.get('POSTGRES_USER'), - os.environ.get('POSTGRES_PASSWORD'), - os.environ.get('POSTGRES_DB_NAME')) - ''' ### nopaque ### ''' NOPAQUE_LOG_LEVEL = os.environ.get('NOPAQUE_LOG_LEVEL') or 'DEBUG' logging.basicConfig(level=NOPAQUE_LOG_LEVEL) @@ -75,13 +73,6 @@ class TestingConfig(Config): class ProductionConfig(Config): - ''' ### Flask-SQLAlchemy ### ''' - SQLALCHEMY_DATABASE_URI = 'postgresql://{}:{}@db/{}'.format( - os.environ.get('POSTGRES_USER'), - os.environ.get('POSTGRES_PASSWORD'), - os.environ.get('POSTGRES_DB_NAME') - ) - ''' ### nopaque ### ''' NOPAQUE_LOG_LEVEL = os.environ.get('NOPAQUE_LOG_LEVEL') or 'ERROR' logging.basicConfig(level=NOPAQUE_LOG_LEVEL) diff --git a/docker-entrypoint.sh b/web/docker-entrypoint.sh old mode 100755 new mode 100644 similarity index 100% rename from docker-entrypoint.sh rename to web/docker-entrypoint.sh diff --git a/migrations/README b/web/migrations/README similarity index 100% rename from migrations/README rename to web/migrations/README diff --git a/migrations/alembic.ini b/web/migrations/alembic.ini similarity index 100% rename from migrations/alembic.ini rename to web/migrations/alembic.ini diff --git a/migrations/env.py b/web/migrations/env.py similarity index 100% rename from migrations/env.py rename to web/migrations/env.py diff --git a/migrations/script.py.mako b/web/migrations/script.py.mako similarity index 100% rename from migrations/script.py.mako rename to web/migrations/script.py.mako diff --git a/migrations/versions/099037c4aa06_.py b/web/migrations/versions/099037c4aa06_.py similarity index 100% rename from migrations/versions/099037c4aa06_.py rename to web/migrations/versions/099037c4aa06_.py diff --git a/migrations/versions/0aa38a7973c5_.py b/web/migrations/versions/0aa38a7973c5_.py similarity index 100% rename from migrations/versions/0aa38a7973c5_.py rename to web/migrations/versions/0aa38a7973c5_.py diff --git a/migrations/versions/10a92d8f4616_.py b/web/migrations/versions/10a92d8f4616_.py similarity index 100% rename from migrations/versions/10a92d8f4616_.py rename to web/migrations/versions/10a92d8f4616_.py diff --git a/migrations/versions/1210adfe1e34_.py b/web/migrations/versions/1210adfe1e34_.py similarity index 100% rename from migrations/versions/1210adfe1e34_.py rename to web/migrations/versions/1210adfe1e34_.py diff --git a/migrations/versions/3d9a20b8b26c_.py b/web/migrations/versions/3d9a20b8b26c_.py similarity index 100% rename from migrations/versions/3d9a20b8b26c_.py rename to web/migrations/versions/3d9a20b8b26c_.py diff --git a/migrations/versions/421ba4373e50_.py b/web/migrations/versions/421ba4373e50_.py similarity index 100% rename from migrations/versions/421ba4373e50_.py rename to web/migrations/versions/421ba4373e50_.py diff --git a/migrations/versions/4638e6509e13_.py b/web/migrations/versions/4638e6509e13_.py similarity index 100% rename from migrations/versions/4638e6509e13_.py rename to web/migrations/versions/4638e6509e13_.py diff --git a/migrations/versions/471aa04c1a92_.py b/web/migrations/versions/471aa04c1a92_.py similarity index 100% rename from migrations/versions/471aa04c1a92_.py rename to web/migrations/versions/471aa04c1a92_.py diff --git a/migrations/versions/4886241e0f5d_.py b/web/migrations/versions/4886241e0f5d_.py similarity index 100% rename from migrations/versions/4886241e0f5d_.py rename to web/migrations/versions/4886241e0f5d_.py diff --git a/migrations/versions/49a42c69e523_.py b/web/migrations/versions/49a42c69e523_.py similarity index 100% rename from migrations/versions/49a42c69e523_.py rename to web/migrations/versions/49a42c69e523_.py diff --git a/migrations/versions/5ba6786a847e_.py b/web/migrations/versions/5ba6786a847e_.py similarity index 100% rename from migrations/versions/5ba6786a847e_.py rename to web/migrations/versions/5ba6786a847e_.py diff --git a/migrations/versions/62233e0cb2c7_.py b/web/migrations/versions/62233e0cb2c7_.py similarity index 100% rename from migrations/versions/62233e0cb2c7_.py rename to web/migrations/versions/62233e0cb2c7_.py diff --git a/migrations/versions/6227310c2112_.py b/web/migrations/versions/6227310c2112_.py similarity index 100% rename from migrations/versions/6227310c2112_.py rename to web/migrations/versions/6227310c2112_.py diff --git a/migrations/versions/66253783654f_.py b/web/migrations/versions/66253783654f_.py similarity index 100% rename from migrations/versions/66253783654f_.py rename to web/migrations/versions/66253783654f_.py diff --git a/migrations/versions/68772b6560c3_.py b/web/migrations/versions/68772b6560c3_.py similarity index 100% rename from migrations/versions/68772b6560c3_.py rename to web/migrations/versions/68772b6560c3_.py diff --git a/migrations/versions/7378391345fa_.py b/web/migrations/versions/7378391345fa_.py similarity index 100% rename from migrations/versions/7378391345fa_.py rename to web/migrations/versions/7378391345fa_.py diff --git a/migrations/versions/776761fb7466_.py b/web/migrations/versions/776761fb7466_.py similarity index 100% rename from migrations/versions/776761fb7466_.py rename to web/migrations/versions/776761fb7466_.py diff --git a/migrations/versions/abf60427ff84_.py b/web/migrations/versions/abf60427ff84_.py similarity index 100% rename from migrations/versions/abf60427ff84_.py rename to web/migrations/versions/abf60427ff84_.py diff --git a/migrations/versions/da9fd175af8c_.py b/web/migrations/versions/da9fd175af8c_.py similarity index 100% rename from migrations/versions/da9fd175af8c_.py rename to web/migrations/versions/da9fd175af8c_.py diff --git a/migrations/versions/ded5a37f8a7b_.py b/web/migrations/versions/ded5a37f8a7b_.py similarity index 100% rename from migrations/versions/ded5a37f8a7b_.py rename to web/migrations/versions/ded5a37f8a7b_.py diff --git a/nopaque.py b/web/nopaque.py similarity index 96% rename from nopaque.py rename to web/nopaque.py index b027530c..ee20b8f7 100644 --- a/nopaque.py +++ b/web/nopaque.py @@ -22,8 +22,7 @@ def make_shell_context(): 'NotificationData': NotificationData, 'NotificationEmailData': NotificationEmailData, 'Role': Role, - 'User': User - } + 'User': User} @app.cli.command() diff --git a/requirements.txt b/web/requirements.txt similarity index 100% rename from requirements.txt rename to web/requirements.txt diff --git a/web/tests/__init__.py b/web/tests/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tests/test_basics.py b/web/tests/test_basics.py similarity index 100% rename from tests/test_basics.py rename to web/tests/test_basics.py diff --git a/tests/test_client.py b/web/tests/test_client.py similarity index 100% rename from tests/test_client.py rename to web/tests/test_client.py diff --git a/tests/test_user_model.py b/web/tests/test_user_model.py similarity index 100% rename from tests/test_user_model.py rename to web/tests/test_user_model.py