sporada/nopaque
1
0
Fork 0
mirror of https://gitlab.ub.uni-bielefeld.de/sfb1288inf/nopaque.git synced 2024-12-24 18:34:18 +00:00
Code Issues Projects Releases Wiki Activity

Merge branch 'development' of gitlab.ub.uni-bielefeld.de:sfb1288inf/opaque into development

Browse Source
This commit is contained in:
Stephan Porada 2019-12-02 14:19:51 +01:00
commit d2334e6a1e
25 changed files with 72 additions and 76 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

14
Dockerfile
View File

@ -1,7 +1,7 @@
FROM python:3.6-slim-stretch
ENV FLASK_APP=opaque.py
ENV FLASK_APP=nopaque.py
RUN apt-get update \
@ -12,23 +12,23 @@ RUN apt-get update \
&& rm -rf /var/lib/apt/lists/*
RUN groupadd --gid 1000 --system opaque \
&& useradd --create-home --gid opaque --no-log-init --system --uid 1000 opaque
USER opaque
WORKDIR /home/opaque
RUN groupadd --gid 1000 --system nopaque \
&& useradd --create-home --gid nopaque --no-log-init --system --uid 1000 nopaque
USER nopaque
WORKDIR /home/nopaque
COPY ["app", "app"]
COPY ["migrations", "migrations"]
COPY ["tests", "tests"]
COPY ["config.py", "opaque.py", "requirements.txt", "./"]
COPY ["config.py", "nopaque.py", "requirements.txt", "./"]
RUN python -m venv venv \
&& venv/bin/pip install --requirement requirements.txt \
&& mkdir logs
EXPOSE 5000
VOLUME ["/home/opaque/logs", "/home/opaque/migrations"]
VOLUME ["/home/nopaque/logs", "/home/nopaque/migrations"]
COPY ["docker-entrypoint.sh", "/usr/local/bin/"]

2
README.md
View File

@ -72,7 +72,7 @@ echo "MAIL_SERVER=smtp.example.com" >> .env
echo "MAIL_PORT=587" >> .env
echo "MAIL_USE_TLS=true" >> .env
# A user registering with this email address will automatically promoted as an admin.
echo "OPAQUE_ADMIN=admin.opaque@example.com" >> .env
echo "NOPAQUE_ADMIN=admin.opaque@example.com" >> .env
# Absolut path to an existing directory to save all opaque files.
echo "OPAQUE_STORAGE=/home/opaque/mnt/opaque" >> .env
```

2
app/__init__.py
View File

@ -23,7 +23,7 @@ def create_app(config_name):
db.init_app(app)
login_manager.init_app(app)
mail.init_app(app)
socketio.init_app(app, message_queue='redis://redis:6379')
socketio.init_app(app, message_queue='redis://redis:6379/')
from . import events

2
app/auth/views.py
View File

@ -48,7 +48,7 @@ def register():
username=registration_form.username.data)
db.session.add(user)
db.session.commit()
dir = os.path.join(current_app.config['OPAQUE_STORAGE_DIRECTORY'],
dir = os.path.join(current_app.config['NOPAQUE_STORAGE'],
str(user.id))
try:
os.makedirs(dir)

6
app/corpora/views.py
View File

@ -23,7 +23,7 @@ def add_corpus():
status='unprepared', title=add_corpus_form.title.data)
db.session.add(corpus)
db.session.commit()
dir = os.path.join(current_app.config['OPAQUE_STORAGE_DIRECTORY'],
dir = os.path.join(current_app.config['NOPAQUE_STORAGE'],
str(corpus.user_id), 'corpora', str(corpus.id))
try:
os.makedirs(dir)
@ -95,7 +95,7 @@ def add_corpus_file(corpus_id):
corpus_id=corpus_id))
# Save the file
dir = os.path.join(str(corpus.user_id), 'corpora', str(corpus.id))
file.save(os.path.join(current_app.config['OPAQUE_STORAGE_DIRECTORY'],
file.save(os.path.join(current_app.config['NOPAQUE_STORAGE'],
dir, filename))
corpus_file = CorpusFile(author=add_corpus_file_form.author.data,
corpus=corpus, dir=dir, filename=filename,
@ -139,7 +139,7 @@ def download_corpus_file(corpus_id, corpus_file_id):
if not (corpus_file.corpus.creator == current_user
or current_user.is_administrator()):
abort(403)
dir = os.path.join(current_app.config['OPAQUE_STORAGE_DIRECTORY'],
dir = os.path.join(current_app.config['NOPAQUE_STORAGE'],
corpus_file.dir)
return send_from_directory(as_attachment=True, directory=dir,
filename=corpus_file.filename)

2
app/email.py
View File

@ -10,7 +10,7 @@ def send_async_email(app, msg):
def send_email(to, subject, template, **kwargs):
msg = Message('[Opaque] {}'.format(subject), recipients=[to])
msg = Message('[nopaque] {}'.format(subject), recipients=[to])
msg.body = render_template(template + '.txt.j2', **kwargs)
msg.html = render_template(template + '.html.j2', **kwargs)
thread = Thread(target=send_async_email,

4
app/jobs/views.py
View File

@ -39,7 +39,7 @@ def download_job_input(job_id, job_input_id):
if not (job_input.job.creator == current_user
or current_user.is_administrator()):
abort(403)
dir = os.path.join(current_app.config['OPAQUE_STORAGE_DIRECTORY'],
dir = os.path.join(current_app.config['NOPAQUE_STORAGE'],
job_input.dir)
return send_from_directory(as_attachment=True, directory=dir,
filename=job_input.filename)
@ -54,7 +54,7 @@ def download_job_result(job_id, job_result_id):
if not (job_result.job.creator == current_user
or current_user.is_administrator()):
abort(403)
dir = os.path.join(current_app.config['OPAQUE_STORAGE_DIRECTORY'],
dir = os.path.join(current_app.config['NOPAQUE_STORAGE'],
job_result.dir)
return send_from_directory(as_attachment=True, directory=dir,
filename=job_result.filename)

2
app/main/views.py
View File

@ -5,7 +5,7 @@ from . import main
@main.route('/')
def index():
return render_template('main/index.html.j2', title='Opaque')
return render_template('main/index.html.j2', title='nopaque')
@main.route('/dashboard')

12
app/models.py
View File

@ -128,7 +128,7 @@ class User(UserMixin, db.Model):
def __init__(self, **kwargs):
super(User, self).__init__(**kwargs)
if self.role is None:
if self.email == current_app.config['OPAQUE_ADMIN']:
if self.email == current_app.config['NOPAQUE_ADMIN']:
self.role = Role.query.filter_by(name='Administrator').first()
if self.role is None:
self.role = Role.query.filter_by(default=True).first()
@ -213,7 +213,7 @@ class User(UserMixin, db.Model):
job.delete()
for corpus in self.corpora:
corpus.delete()
path = os.path.join(current_app.config['OPAQUE_STORAGE_DIRECTORY'],
path = os.path.join(current_app.config['NOPAQUE_STORAGE'],
str(self.id))
try:
shutil.rmtree(path)
@ -333,7 +333,7 @@ class Job(db.Model):
while self.status != 'deleted':
sleep(1)
db.session.refresh(self)
path = os.path.join(current_app.config['OPAQUE_STORAGE_DIRECTORY'],
path = os.path.join(current_app.config['NOPAQUE_STORAGE'],
str(self.user_id), 'jobs', str(self.id))
try:
shutil.rmtree(path)
@ -377,7 +377,7 @@ class CorpusFile(db.Model):
corpus_id = db.Column(db.Integer, db.ForeignKey('corpora.id'))
def delete(self):
path = os.path.join(current_app.config['OPAQUE_STORAGE_DIRECTORY'],
path = os.path.join(current_app.config['NOPAQUE_STORAGE'],
self.dir, self.filename)
try:
os.remove(path)
@ -390,7 +390,7 @@ class CorpusFile(db.Model):
db.session.commit()
def insert_metadata(self):
file = os.path.join(current_app.config['OPAQUE_STORAGE_DIRECTORY'],
file = os.path.join(current_app.config['NOPAQUE_STORAGE'],
self.dir, self.filename)
element_tree = ET.parse(file)
text_node = element_tree.find('text')
@ -437,7 +437,7 @@ class Corpus(db.Model):
def delete(self):
for corpus_file in self.files:
corpus_file.delete()
path = os.path.join(current_app.config['OPAQUE_STORAGE_DIRECTORY'],
path = os.path.join(current_app.config['NOPAQUE_STORAGE'],
str(self.user_id), 'corpora', str(self.id))
try:
shutil.rmtree(path)

2
app/services/views.py
View File

@ -45,7 +45,7 @@ def service(service):
db.session.commit()
relative_dir = os.path.join(str(job.user_id), 'jobs', str(job.id))
absolut_dir = os.path.join(
current_app.config['OPAQUE_STORAGE_DIRECTORY'], relative_dir)
current_app.config['NOPAQUE_STORAGE'], relative_dir)
try:
os.makedirs(absolut_dir)
except OSError:

0
app/static/css/opaque.css → app/static/css/nopaque.css
View File

2
app/templates/auth/email/confirm.html.j2
View File

@ -3,5 +3,5 @@
<p>Alternatively, you can paste the following link in your browser's address bar:</p>
<p>{{ url_for('auth.confirm', token=token, _external=True) }}</p>
<p>Sincerely,</p>
<p>The Opaque Team</p>
<p>The nopaque Team</p>
<p><small>Note: replies to this email address are not monitored.</small></p>

2
app/templates/auth/email/confirm.txt.j2
View File

@ -4,6 +4,6 @@ To confirm your account please click on the following link:
{{ url_for('auth.confirm', token=token, _external=True) }}
Sincerely,
The Opaque Team
The nopaque Team
Note: replies to this email address are not monitored.

2
app/templates/auth/email/reset_password.html.j2
View File

@ -4,5 +4,5 @@
<p>{{ url_for('auth.password_reset', token=token, _external=True) }}</p>
<p>If you have not requested a password reset simply ignore this message.</p>
<p>Sincerely,</p>
<p>The Opaque Team</p>
<p>The nopaque Team</p>
<p><small>Note: replies to this email address are not monitored.</small></p>

2
app/templates/auth/email/reset_password.txt.j2
View File

@ -8,6 +8,6 @@ If you have not requested a password reset simply ignore this message.
Sincerely,
The Opaque Team
The nopaque Team
Note: replies to this email address are not monitored.

10
app/templates/base.html.j2
View File

@ -3,14 +3,14 @@
<head>
<meta charset="UTF-8">
{% if title %}
<title>Opaque – {{ title }}</title>
<title>nopaque – {{ title }}</title>
{% else %}
<title>Opaque</title>
<title>nopaque</title>
{% endif %}
<link href="{{ url_for('static', filename='images/favicon.png') }}" rel="icon" type="image/png">
<link type="text/css" rel="stylesheet" href="{{ url_for('static', filename='fonts/material-icons/material-icons.css') }}">
<link type="text/css" rel="stylesheet" href="{{ url_for('static', filename='css/materialize.min.css') }}" media="screen,projection"/>
<link type="text/css" rel="stylesheet" href="{{ url_for('static', filename='css/opaque.css') }}" media="screen,projection"/>
<link type="text/css" rel="stylesheet" href="{{ url_for('static', filename='css/nopaque.css') }}" media="screen,projection"/>
<script src="{{ url_for('static', filename='js/add_job.js') }}"></script>
<script src="{{ url_for('static', filename='js/jsonpatch.min.js') }}"></script>
<script src="{{ url_for('static', filename='js/socket.io.js') }}"></script>
@ -111,7 +111,7 @@
<div class="navbar-fixed">
<nav>
<div class="nav-wrapper">
<a href="{{ url_for('main.index') }}" class="brand-logo center"><i class="material-icons">opacity</i>Opaque</a>
<a href="{{ url_for('main.index') }}" class="brand-logo center"><i class="material-icons">opacity</i>nopaque</a>
<a href="#" data-target="slide-out" class="sidenav-trigger"><i class="material-icons">menu</i></a>
<ul class="right hide-on-med-and-down">
<li><a id="nav-notifications" class="dropdown-trigger no-autoinit" href="#!" data-target="nav-notifications-dropdown"><i class="material-icons">notifications</i></a></li>
@ -130,7 +130,7 @@
</div>
<ul id="slide-out" class="sidenav sidenav-fixed">
<li><a href="{{ url_for('main.index') }}"><i class="material-icons">opacity</i>Opaque</a></li>
<li><a href="{{ url_for('main.index') }}"><i class="material-icons">opacity</i>nopaque</a></li>
<li><a href="#"><i class="material-icons">linear_scale</i>Workflow</a></li>
<li><a href="{{ url_for('main.dashboard') }}"><i class="material-icons">dashboard</i>Dashboard</a></li>
<li><div class="divider"></div></li>

2
app/templates/main/index.html.j2
View File

@ -2,7 +2,7 @@
{% block page_content %}
<div class="col s12 m4">
<h3>Opaque?</h3>
<h3>nopaque?</h3>
<p>Lorem ipsum dolor sit amet, consetetur sadipscing elitr, sed diam nonumy eirmod tempor invidunt ut labore et dolore magna aliquyam erat, sed diam voluptua. At vero eos et accusam et justo duo dolores et ea rebum. Stet clita kasd gubergren,</p>
</div>

4
config.py
View File

@ -18,8 +18,8 @@ class Config:
SQLALCHEMY_TRACK_MODIFICATIONS = False
''' ### Opaque ### '''
OPAQUE_ADMIN = os.environ.get('OPAQUE_ADMIN')
OPAQUE_STORAGE_DIRECTORY = '/mnt/opaque'
NOPAQUE_ADMIN = os.environ.get('NOPAQUE_ADMIN')
NOPAQUE_STORAGE = '/mnt/nopaque'
@staticmethod
def init_app(app):

6
dind_swarm.yml
View File

@ -2,13 +2,13 @@ version: '3'
services:
storage:
command: ["-p", "-s", "opaque_storage;/srv/opaque/storage;no;no;no;opaque", "-u", "opaque;opaque"]
command: ["-p", "-s", "storage.nopaque;/srv/nopaque/storage;no;no;no;nopaque", "-u", "nopaque;nopaque"]
image: dperson/samba:latest
ports:
- 445:445
restart: on-failure
volumes:
- /srv/opaque/storage:/srv/opaque/storage
- /srv/nopaque/storage:/srv/nopaque/storage
worker:
image: docker:dind
ports:
@ -16,7 +16,7 @@ services:
privileged: true
restart: on-failure
volumes:
- /mnt/opaque:/mnt/opaque
- /mnt/nopaque:/mnt/nopaque
viz:
image: dockersamples/visualizer:latest
ports:

6
dind_swarm_setup.sh
View File

@ -17,9 +17,9 @@ docker-compose --file dind_swarm.yml up --detach storage
sleep 3
echo "Mount network storage to host system..."
sudo mkdir -p /mnt/opaque
sudo umount /mnt/opaque
sudo mount -t cifs -o gid=${USER},password=opaque,uid=${USER},user=opaque,vers=3.0 //localhost/opaque_storage /mnt/opaque
sudo mkdir -p /mnt/nopaque
sudo umount /mnt/nopaque
sudo mount -t cifs -o gid=${USER},password=nopaque,uid=${USER},user=nopaque,vers=3.0 //localhost/storage.nopaque /mnt/nopaque
echo "Start worker service(s)"
docker-compose --file dind_swarm.yml up --detach --scale worker=${SWARM_WORKER_NUMBER} worker viz

30
docker-compose.yml
View File

@ -12,15 +12,15 @@ services:
ports:
- 5000:5000
volumes:
- /mnt/opaque:/mnt/opaque
- ./app:/home/opaque/app
- ./logs:/home/opaque/logs
- ./migrations:/home/opaque/migrations
- ./tests:/home/opaque/tests
- ./config.py:/home/opaque/config.py
- /mnt/nopaque:/mnt/nopaque
- ./app:/home/nopaque/app
- ./logs:/home/nopaque/logs
- ./migrations:/home/nopaque/migrations
- ./tests:/home/nopaque/tests
- ./config.py:/home/nopaque/config.py
- ./docker-entrypoint.sh:/usr/local/bin/docker-entrypoint.sh
- ./opaque.py:/home/opaque/opaque.py
- ./requirements.txt:/home/opaque/requirements.txt
- ./nopaque.py:/home/nopaque/nopaque.py
- ./requirements.txt:/home/nopaque/requirements.txt
daemon:
depends_on:
- db
@ -29,17 +29,17 @@ services:
- web.env
image: gitlab.ub.uni-bielefeld.de:4567/sfb1288inf/opaque_daemon:latest
volumes:
- /mnt/opaque:/mnt/opaque
- ./logs:/home/opaque_daemon/logs
- /mnt/nopaque:/mnt/nopaque
- ./logs:/home/nopaqued/logs
- ../opaque_daemon/docker-entrypoint.sh:/usr/local/bin/docker-entrypoint.sh
- ../opaque_daemon/opaque_daemon.py:/home/opaque_daemon/opaque_daemon.py
- ../opaque_daemon/merge_corpus_files.py:/home/opaque_daemon/merge_corpus_files.py
- ../opaque_daemon/requirements.txt:/home/opaque_daemon/requirements.txt
- $HOME/.docker/config.json:/home/opaque_daemon/.docker/config.json
- ../opaque_daemon/nopaqued.py:/home/nopaqued/opaque_daemon.py
- ../opaque_daemon/merge_corpus_files.py:/home/nopaqued/merge_corpus_files.py
- ../opaque_daemon/requirements.txt:/home/nopaqued/requirements.txt
- $HOME/.docker/config.json:/home/nopaqued/.docker/config.json
db:
env_file: db.env
image: postgres:11
volumes:
- /srv/opaque/database:/var/lib/postgresql/data
- /srv/nopaque/database:/var/lib/postgresql/data
redis:
image: redis:5

2
docker-entrypoint.sh
View File

@ -7,7 +7,7 @@ wait-for-it redis:6379 --strict --timeout=0
if [ $# -eq 0 ]
then
venv/bin/python -u opaque.py
venv/bin/python -u nopaque.py
elif [ $1 == "flask" ]
then
echo "$@"

28
docker_stack_deploy.yml
View File

@ -6,6 +6,10 @@ services:
placement:
constraints:
- node.role == manager
labels:
- com.docker.lb.hosts=nopaque
- com.docker.lb.port=8080
- com.docker.lb.sticky_session_cookie=session
env_file:
- db.env
- web.env
@ -13,8 +17,8 @@ services:
ports:
- 5000:5000
volumes:
- storage:/mnt/opaque
- ./logs:/home/opaque/logs
- /mnt/nopaque:/mnt/nopaque
- ./logs:/home/nopaque/logs
daemon:
deploy:
placement:
@ -25,29 +29,21 @@ services:
- web.env
image: gitlab.ub.uni-bielefeld.de:4567/sfb1288inf/opaque_daemon:latest
volumes:
- storage:/mnt/opaque
- ./logs:/home/opaque_daemon/logs
- $HOME/.docker/config.json:/home/opaque_daemon/.docker/config.json
- /mnt/nopaque:/mnt/nopaque
- ./logs:/home/nopaqued/logs
- $HOME/.docker/config.json:/home/nopaqued/.docker/config.json
db:
deploy:
placement:
constraints:
- node.role == manager
env_file: db.env
image: postgres:11-alpine
image: postgres:11
volumes:
- /srv/opaque/database:/var/lib/postgresql/data
- /srv/nopaque/database:/var/lib/postgresql/data
redis:
deploy:
placement:
constraints:
- node.role == manager
image: redis:alpine
volumes:
storage:
driver: local
driver_opts:
device: "//127.0.0.1/opaque_storage"
o: "gid=1000,password=opaque,uid=1000,username=opaque"
type: cifs
image: redis:5

0
opaque.py → nopaque.py
View File

4
web.env.tpl
View File

@ -10,5 +10,5 @@ MAIL_USERNAME=username@example.com
MAIL_PASSWORD=password
MAIL_DEFAULT_SENDER=username@example.com
### Opaque ###
OPAQUE_ADMIN=admin.opaque@example.com
### nopaque ###
NOPAQUE_ADMIN=admin.opaque@example.com