From da2f7f6c7d6f32971949cc62d5f57bb472dcf1c3 Mon Sep 17 00:00:00 2001 From: Patrick Jentsch Date: Mon, 2 Dec 2019 11:06:46 +0100 Subject: [PATCH 1/4] Updates for stack deployment --- app/__init__.py | 2 +- docker_stack_deploy.yml | 20 ++++++++------------ 2 files changed, 9 insertions(+), 13 deletions(-) diff --git a/app/__init__.py b/app/__init__.py index 4b32e927..3a1f54fc 100644 --- a/app/__init__.py +++ b/app/__init__.py @@ -23,7 +23,7 @@ def create_app(config_name): db.init_app(app) login_manager.init_app(app) mail.init_app(app) - socketio.init_app(app, message_queue='redis://redis:6379') + socketio.init_app(app, message_queue='redis://redis:6379/') from . import events diff --git a/docker_stack_deploy.yml b/docker_stack_deploy.yml index 62dc689d..dc232c8a 100644 --- a/docker_stack_deploy.yml +++ b/docker_stack_deploy.yml @@ -6,6 +6,10 @@ services: placement: constraints: - node.role == manager + labels: + - com.docker.lb.hosts=web.opaque + - com.docker.lb.port=8080 + - com.docker.lb.sticky_session_cookie=session env_file: - db.env - web.env @@ -13,7 +17,7 @@ services: ports: - 5000:5000 volumes: - - storage:/mnt/opaque + - /mnt/opaque:/mnt/opaque - ./logs:/home/opaque/logs daemon: deploy: @@ -25,7 +29,7 @@ services: - web.env image: gitlab.ub.uni-bielefeld.de:4567/sfb1288inf/opaque_daemon:latest volumes: - - storage:/mnt/opaque + - /mnt/opaque:/mnt/opaque - ./logs:/home/opaque_daemon/logs - $HOME/.docker/config.json:/home/opaque_daemon/.docker/config.json db: @@ -34,7 +38,7 @@ services: constraints: - node.role == manager env_file: db.env - image: postgres:11-alpine + image: postgres:11 volumes: - /srv/opaque/database:/var/lib/postgresql/data redis: @@ -42,12 +46,4 @@ services: placement: constraints: - node.role == manager - image: redis:alpine - -volumes: - storage: - driver: local - driver_opts: - device: "//127.0.0.1/opaque_storage" - o: "gid=1000,password=opaque,uid=1000,username=opaque" - type: cifs + image: redis:5 From f63a85017590107231e89209f83245abe3eda050 Mon Sep 17 00:00:00 2001 From: Patrick Jentsch Date: Mon, 2 Dec 2019 11:34:28 +0100 Subject: [PATCH 2/4] Renaming: opaque->nopaque --- Dockerfile | 14 ++++++------- README.md | 2 +- app/auth/views.py | 2 +- app/corpora/views.py | 6 +++--- app/jobs/views.py | 4 ++-- app/main/views.py | 2 +- app/models.py | 12 +++++------ app/services/views.py | 2 +- app/static/css/{opaque.css => nopaque.css} | 0 app/templates/base.html.j2 | 10 ++++----- app/templates/main/index.html.j2 | 2 +- config.py | 4 ++-- docker-compose.yml | 24 +++++++++++----------- docker-entrypoint.sh | 2 +- opaque.py => nopaque.py | 0 web.env.tpl | 4 ++-- 16 files changed, 45 insertions(+), 45 deletions(-) rename app/static/css/{opaque.css => nopaque.css} (100%) rename opaque.py => nopaque.py (100%) diff --git a/Dockerfile b/Dockerfile index 59511c03..e9042052 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,7 +1,7 @@ FROM python:3.6-slim-stretch -ENV FLASK_APP=opaque.py +ENV FLASK_APP=nopaque.py RUN apt-get update \ @@ -12,23 +12,23 @@ RUN apt-get update \ && rm -rf /var/lib/apt/lists/* -RUN groupadd --gid 1000 --system opaque \ - && useradd --create-home --gid opaque --no-log-init --system --uid 1000 opaque -USER opaque -WORKDIR /home/opaque +RUN groupadd --gid 1000 --system nopaque \ + && useradd --create-home --gid nopaque --no-log-init --system --uid 1000 nopaque +USER nopaque +WORKDIR /home/nopaque COPY ["app", "app"] COPY ["migrations", "migrations"] COPY ["tests", "tests"] -COPY ["config.py", "opaque.py", "requirements.txt", "./"] +COPY ["config.py", "nopaque.py", "requirements.txt", "./"] RUN python -m venv venv \ && venv/bin/pip install --requirement requirements.txt \ && mkdir logs EXPOSE 5000 -VOLUME ["/home/opaque/logs", "/home/opaque/migrations"] +VOLUME ["/home/nopaque/logs", "/home/nopaque/migrations"] COPY ["docker-entrypoint.sh", "/usr/local/bin/"] diff --git a/README.md b/README.md index dd872578..6c25284c 100644 --- a/README.md +++ b/README.md @@ -72,7 +72,7 @@ echo "MAIL_SERVER=smtp.example.com" >> .env echo "MAIL_PORT=587" >> .env echo "MAIL_USE_TLS=true" >> .env # A user registering with this email address will automatically promoted as an admin. -echo "OPAQUE_ADMIN=admin.opaque@example.com" >> .env +echo "NOPAQUE_ADMIN=admin.opaque@example.com" >> .env # Absolut path to an existing directory to save all opaque files. echo "OPAQUE_STORAGE=/home/opaque/mnt/opaque" >> .env ``` diff --git a/app/auth/views.py b/app/auth/views.py index 74afc490..22a11954 100644 --- a/app/auth/views.py +++ b/app/auth/views.py @@ -48,7 +48,7 @@ def register(): username=registration_form.username.data) db.session.add(user) db.session.commit() - dir = os.path.join(current_app.config['OPAQUE_STORAGE_DIRECTORY'], + dir = os.path.join(current_app.config['NOPAQUE_STORAGE'], str(user.id)) try: os.makedirs(dir) diff --git a/app/corpora/views.py b/app/corpora/views.py index 5f07173b..f2d6f83a 100644 --- a/app/corpora/views.py +++ b/app/corpora/views.py @@ -23,7 +23,7 @@ def add_corpus(): status='unprepared', title=add_corpus_form.title.data) db.session.add(corpus) db.session.commit() - dir = os.path.join(current_app.config['OPAQUE_STORAGE_DIRECTORY'], + dir = os.path.join(current_app.config['NOPAQUE_STORAGE'], str(corpus.user_id), 'corpora', str(corpus.id)) try: os.makedirs(dir) @@ -95,7 +95,7 @@ def add_corpus_file(corpus_id): corpus_id=corpus_id)) # Save the file dir = os.path.join(str(corpus.user_id), 'corpora', str(corpus.id)) - file.save(os.path.join(current_app.config['OPAQUE_STORAGE_DIRECTORY'], + file.save(os.path.join(current_app.config['NOPAQUE_STORAGE'], dir, filename)) corpus_file = CorpusFile(author=add_corpus_file_form.author.data, corpus=corpus, dir=dir, filename=filename, @@ -139,7 +139,7 @@ def download_corpus_file(corpus_id, corpus_file_id): if not (corpus_file.corpus.creator == current_user or current_user.is_administrator()): abort(403) - dir = os.path.join(current_app.config['OPAQUE_STORAGE_DIRECTORY'], + dir = os.path.join(current_app.config['NOPAQUE_STORAGE'], corpus_file.dir) return send_from_directory(as_attachment=True, directory=dir, filename=corpus_file.filename) diff --git a/app/jobs/views.py b/app/jobs/views.py index 08fd825c..a5bbe723 100644 --- a/app/jobs/views.py +++ b/app/jobs/views.py @@ -39,7 +39,7 @@ def download_job_input(job_id, job_input_id): if not (job_input.job.creator == current_user or current_user.is_administrator()): abort(403) - dir = os.path.join(current_app.config['OPAQUE_STORAGE_DIRECTORY'], + dir = os.path.join(current_app.config['NOPAQUE_STORAGE'], job_input.dir) return send_from_directory(as_attachment=True, directory=dir, filename=job_input.filename) @@ -54,7 +54,7 @@ def download_job_result(job_id, job_result_id): if not (job_result.job.creator == current_user or current_user.is_administrator()): abort(403) - dir = os.path.join(current_app.config['OPAQUE_STORAGE_DIRECTORY'], + dir = os.path.join(current_app.config['NOPAQUE_STORAGE'], job_result.dir) return send_from_directory(as_attachment=True, directory=dir, filename=job_result.filename) diff --git a/app/main/views.py b/app/main/views.py index fe8ec583..5f20d853 100644 --- a/app/main/views.py +++ b/app/main/views.py @@ -5,7 +5,7 @@ from . import main @main.route('/') def index(): - return render_template('main/index.html.j2', title='Opaque') + return render_template('main/index.html.j2', title='nopaque') @main.route('/dashboard') diff --git a/app/models.py b/app/models.py index 87cd38df..32f0d109 100644 --- a/app/models.py +++ b/app/models.py @@ -128,7 +128,7 @@ class User(UserMixin, db.Model): def __init__(self, **kwargs): super(User, self).__init__(**kwargs) if self.role is None: - if self.email == current_app.config['OPAQUE_ADMIN']: + if self.email == current_app.config['NOPAQUE_ADMIN']: self.role = Role.query.filter_by(name='Administrator').first() if self.role is None: self.role = Role.query.filter_by(default=True).first() @@ -213,7 +213,7 @@ class User(UserMixin, db.Model): job.delete() for corpus in self.corpora: corpus.delete() - path = os.path.join(current_app.config['OPAQUE_STORAGE_DIRECTORY'], + path = os.path.join(current_app.config['NOPAQUE_STORAGE'], str(self.id)) try: shutil.rmtree(path) @@ -333,7 +333,7 @@ class Job(db.Model): while self.status != 'deleted': sleep(1) db.session.refresh(self) - path = os.path.join(current_app.config['OPAQUE_STORAGE_DIRECTORY'], + path = os.path.join(current_app.config['NOPAQUE_STORAGE'], str(self.user_id), 'jobs', str(self.id)) try: shutil.rmtree(path) @@ -377,7 +377,7 @@ class CorpusFile(db.Model): corpus_id = db.Column(db.Integer, db.ForeignKey('corpora.id')) def delete(self): - path = os.path.join(current_app.config['OPAQUE_STORAGE_DIRECTORY'], + path = os.path.join(current_app.config['NOPAQUE_STORAGE'], self.dir, self.filename) try: os.remove(path) @@ -390,7 +390,7 @@ class CorpusFile(db.Model): db.session.commit() def insert_metadata(self): - file = os.path.join(current_app.config['OPAQUE_STORAGE_DIRECTORY'], + file = os.path.join(current_app.config['NOPAQUE_STORAGE'], self.dir, self.filename) element_tree = ET.parse(file) text_node = element_tree.find('text') @@ -437,7 +437,7 @@ class Corpus(db.Model): def delete(self): for corpus_file in self.files: corpus_file.delete() - path = os.path.join(current_app.config['OPAQUE_STORAGE_DIRECTORY'], + path = os.path.join(current_app.config['NOPAQUE_STORAGE'], str(self.user_id), 'corpora', str(self.id)) try: shutil.rmtree(path) diff --git a/app/services/views.py b/app/services/views.py index 6cbf9d39..787ebac0 100644 --- a/app/services/views.py +++ b/app/services/views.py @@ -45,7 +45,7 @@ def service(service): db.session.commit() relative_dir = os.path.join(str(job.user_id), 'jobs', str(job.id)) absolut_dir = os.path.join( - current_app.config['OPAQUE_STORAGE_DIRECTORY'], relative_dir) + current_app.config['NOPAQUE_STORAGE'], relative_dir) try: os.makedirs(absolut_dir) except OSError: diff --git a/app/static/css/opaque.css b/app/static/css/nopaque.css similarity index 100% rename from app/static/css/opaque.css rename to app/static/css/nopaque.css diff --git a/app/templates/base.html.j2 b/app/templates/base.html.j2 index a6d4c559..04d92c74 100644 --- a/app/templates/base.html.j2 +++ b/app/templates/base.html.j2 @@ -3,14 +3,14 @@ {% if title %} - Opaque – {{ title }} + nopaque – {{ title }} {% else %} - Opaque + nopaque {% endif %} - + @@ -111,7 +111,7 @@