sporada/nopaque
1
0
Fork 0
mirror of https://gitlab.ub.uni-bielefeld.de/sfb1288inf/nopaque.git synced 2024-12-26 03:14:19 +00:00
Code Issues Projects Releases Wiki Activity

Merge branch 'development'

Browse Source
This commit is contained in:
Patrick Jentsch 2023-02-15 11:37:09 +01:00
commit 525723818e
27 changed files with 792 additions and 67 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
app/corpora/cqi_over_socketio/__init__.py
View File

@ -62,7 +62,9 @@ def connect(auth):
if corpus is None:
# return {'code': 404, 'msg': 'Not Found'}
raise ConnectionRefusedError('Not Found')
if not (corpus.user == current_user or current_user.is_administrator()):
if not (corpus.user == current_user
or current_user.is_following_corpus(corpus)
or current_user.is_administrator()):
# return {'code': 403, 'msg': 'Forbidden'}
raise ConnectionRefusedError('Forbidden')
if corpus.status not in [

1
app/corpora/forms.py
View File

@ -1,6 +1,7 @@
from flask_wtf import FlaskForm
from flask_wtf.file import FileField, FileRequired
from wtforms import (
BooleanField,
StringField,
SubmitField,
TextAreaField,

194
app/corpora/routes.py
View File

@ -1,3 +1,4 @@
from datetime import datetime
from flask import (
abort,
current_app,
@ -5,37 +6,130 @@ from flask import (
Markup,
redirect,
render_template,
send_from_directory
request,
send_from_directory,
url_for
)
from flask_login import current_user, login_required
from threading import Thread
import jwt
import os
from app import db
from app.models import Corpus, CorpusFile, CorpusStatus
from app import db, hashids
from app.models import Corpus, CorpusFile, CorpusStatus, User
from . import bp
from .forms import CreateCorpusFileForm, CreateCorpusForm, UpdateCorpusFileForm
from .forms import (
CreateCorpusFileForm,
CreateCorpusForm,
UpdateCorpusFileForm
)
def user_can_read_corpus(user, corpus):
return corpus.user == user or user.is_administrator() or corpus.is_public
# @bp.route('/share/<token>', methods=['GET', 'POST'])
# def share_corpus(token):
# try:
# payload = jwt.decode(
# token,
# current_app.config['SECRET_KEY'],
# algorithms=['HS256'],
# issuer=current_app.config['SERVER_NAME'],
# options={'require': ['iat', 'iss', 'sub']}
# )
# except jwt.PyJWTError:
# return False
# corpus_hashid = payload.get('sub')
# corpus_id = hashids.decode(corpus_hashid)
# return redirect(url_for('.corpus', corpus_id=corpus_id))
def user_can_update_corpus(user, corpus):
return corpus.user == user or user.is_administrator()
def user_can_delete_corpus(user, corpus):
return user_can_update_corpus(user, corpus)
@bp.route('')
@bp.route('/<hashid:corpus_id>/enable_is_public', methods=['POST'])
@login_required
def corpora():
query = Corpus.query.filter(
(Corpus.user_id == current_user.id) | (Corpus.is_public == True)
def enable_corpus_is_public(corpus_id):
corpus = Corpus.query.get_or_404(corpus_id)
if not (corpus.user == current_user or current_user.is_administrator()):
abort(403)
corpus.is_public = True
db.session.commit()
return '', 204
@bp.route('/<hashid:corpus_id>/disable_is_public', methods=['POST'])
@login_required
def disable_corpus_is_public(corpus_id):
corpus = Corpus.query.get_or_404(corpus_id)
if not (corpus.user == current_user or current_user.is_administrator()):
abort(403)
corpus.is_public = False
db.session.commit()
return '', 204
# @bp.route('/<hashid:corpus_id>/follow', methods=['GET', 'POST'])
# @login_required
# def follow_corpus(corpus_id):
# corpus = Corpus.query.get_or_404(corpus_id)
# user_hashid = request.args.get('user_id')
# if user_hashid is None:
# user = current_user
# else:
# if not current_user.is_administrator():
# abort(403)
# else:
# user_id = hashids.decode(user_hashid)
# user = User.query.get_or_404(user_id)
# if not user.is_following_corpus(corpus):
# user.follow_corpus(corpus)
# db.session.commit()
# flash(f'You are following {corpus.title} now', category='corpus')
# return {}, 202
@bp.route('/<hashid:corpus_id>/unfollow', methods=['GET', 'POST'])
@login_required
def unfollow_corpus(corpus_id):
corpus = Corpus.query.get_or_404(corpus_id)
user_hashid = request.args.get('user_id')
if user_hashid is None:
user = current_user
elif current_user.is_administrator():
user_id = hashids.decode(user_hashid)
user = User.query.get_or_404(user_id)
else:
abort(403)
if user.is_following_corpus(corpus):
user.unfollow_corpus(corpus)
db.session.commit()
flash(f'You are not following {corpus.title} anymore', category='corpus')
return {}, 202
# @bp.route('/add_permission/<hashid:corpus_id>/<hashid:user_id>/<int:permission>')
# def add_permission(corpus_id, user_id, permission):
# a = CorpusFollowerAssociation.query.filter_by(followed_corpus_id=corpus_id, following_user_id=user_id).first_or_404()
# a.add_permission(permission)
# db.session.commit()
# return 'ok'
# @bp.route('/remove_permission/<hashid:corpus_id>/<hashid:user_id>/<int:permission>')
# def remove_permission(corpus_id, user_id, permission):
# a = CorpusFollowerAssociation.query.filter_by(followed_corpus_id=corpus_id, following_user_id=user_id).first_or_404()
# a.remove_permission(permission)
# db.session.commit()
# return 'ok'
@bp.route('/public')
@login_required
def public_corpora():
corpora = [
c.to_json_serializeable()
for c in Corpus.query.filter(Corpus.is_public == True).all()
]
return render_template(
'corpora/public_corpora.html.j2',
corpora=corpora,
title='Corpora'
)
corpora = [c.to_json_serializeable() for c in query.all()]
return render_template('corpora/corpora.html.j2', corpora=corpora, title='Corpora')
@bp.route('/create', methods=['GET', 'POST'])
@ -64,30 +158,38 @@ def create_corpus():
)
@bp.route('/<hashid:corpus_id>')
@bp.route('/<hashid:corpus_id>', methods=['GET', 'POST'])
@login_required
def corpus(corpus_id):
corpus = Corpus.query.get_or_404(corpus_id)
if not user_can_read_corpus(current_user, corpus):
abort(403)
return render_template(
'corpora/corpus.html.j2',
corpus=corpus,
title='Corpus'
)
# @bp.route('/<hashid:corpus_id>/update')
# @login_required
# def update_corpus(corpus_id):
# corpus = Corpus.query.get_or_404(corpus_id)
# if not user_can_update_corpus(current_user, corpus):
# abort(403)
# return render_template(
# 'corpora/update_corpus.html.j2',
# corpus=corpus,
# title='Corpus'
# )
if corpus.user == current_user or current_user.is_administrator():
# now = datetime.utcnow()
# payload = {
# 'exp': now + timedelta(weeks=1),
# 'iat': now,
# 'iss': current_app.config['SERVER_NAME'],
# 'sub': corpus.hashid
# }
# token = jwt.encode(
# payload,
# current_app.config['SECRET_KEY'],
# algorithm='HS256'
# )
return render_template(
'corpora/corpus.html.j2',
corpus=corpus,
# token=token,
title='Corpus'
)
if current_user.is_following_corpus(corpus) or corpus.is_public:
corpus_files = [x.to_json_serializeable() for x in corpus.files]
return render_template(
'corpora/public_corpus.html.j2',
corpus=corpus,
corpus_files=corpus_files,
title='Corpus'
)
abort(403)
@bp.route('/<hashid:corpus_id>', methods=['DELETE'])
@ -100,7 +202,7 @@ def delete_corpus(corpus_id):
db.session.commit()
corpus = Corpus.query.get_or_404(corpus_id)
if not user_can_delete_corpus(current_user, corpus):
if not (corpus.user == current_user or current_user.is_administrator()):
abort(403)
thread = Thread(
target=_delete_corpus,
@ -114,7 +216,9 @@ def delete_corpus(corpus_id):
@login_required
def analyse_corpus(corpus_id):
corpus = Corpus.query.get_or_404(corpus_id)
if not user_can_read_corpus(current_user, corpus):
if not (corpus.user == current_user
or current_user.is_administrator()
or current_user.is_following_corpus(corpus)):
abort(403)
return render_template(
'corpora/analyse_corpus.html.j2',
@ -133,7 +237,7 @@ def build_corpus(corpus_id):
db.session.commit()
corpus = Corpus.query.get_or_404(corpus_id)
if not user_can_update_corpus(current_user, corpus):
if not (corpus.user == current_user or current_user.is_administrator()):
abort(403)
# Check if the corpus has corpus files
if not corpus.files.all():
@ -151,7 +255,7 @@ def build_corpus(corpus_id):
@login_required
def create_corpus_file(corpus_id):
corpus = Corpus.query.get_or_404(corpus_id)
if not user_can_update_corpus(current_user, corpus):
if not (corpus.user == current_user or current_user.is_administrator()):
abort(403)
form = CreateCorpusFileForm()
if form.is_submitted():

21
app/main/routes.py
View File

@ -1,7 +1,7 @@
from flask import flash, redirect, render_template, url_for
from flask_login import current_user, login_required, login_user
from app.auth.forms import LoginForm
from app.models import User
from app.models import Corpus, User
from . import bp
@ -27,11 +27,20 @@ def faq():
@bp.route('/dashboard')
@login_required
def dashboard():
users = [
u.to_json_serializeable(filter_by_privacy_settings=True) for u
in User.query.filter(User.is_public == True, User.id != current_user.id).all()
]
return render_template('main/dashboard.html.j2', title='Dashboard', users=users)
# users = [
# u.to_json_serializeable(filter_by_privacy_settings=True) for u
# in User.query.filter(User.is_public == True, User.id != current_user.id).all()
# ]
# corpora = [
# c.to_json_serializeable() for c
# in Corpus.query.filter(Corpus.is_public == True, Corpus.user != current_user).all()
# ]
return render_template(
'main/dashboard.html.j2',
title='Dashboard',
# users=users,
# corpora=corpora
)
@bp.route('/dashboard2')

31
app/models.py
View File

@ -68,6 +68,11 @@ class ProfilePrivacySettings(IntEnum):
SHOW_EMAIL = 1
SHOW_LAST_SEEN = 2
SHOW_MEMBER_SINCE = 4
class CorpusFollowPermission(IntEnum):
VIEW = 1
CONTRIBUTE = 2
ADMINISTRATE = 4
# endregion enums
@ -298,6 +303,16 @@ class CorpusFollowerAssociation(db.Model):
def __repr__(self):
return f'<CorpusFollowerAssociation {self.following_user.__repr__()} ~ {self.followed_corpus.__repr__()}>'
def has_permission(self, permission):
return self.permissions & permission == permission
def add_permission(self, permission):
if not self.has_permission(permission):
self.permissions += permission
def remove_permission(self, permission):
if self.has_permission(permission):
self.permissions -= permission
class User(HashidMixin, UserMixin, db.Model):
__tablename__ = 'users'
@ -576,6 +591,18 @@ class User(HashidMixin, UserMixin, db.Model):
self.profile_privacy_settings = 0
#endregion Profile Privacy settings
def follow_corpus(self, corpus):
if not self.is_following_corpus(corpus):
self.followed_corpora.append(corpus)
def unfollow_corpus(self, corpus):
if self.is_following_corpus(corpus):
self.followed_corpora.remove(corpus)
def is_following_corpus(self, corpus):
return corpus in self.followed_corpora
def to_json_serializeable(self, backrefs=False, relationships=False, filter_by_privacy_settings=False):
json_serializeable = {
'id': self.hashid,
@ -623,6 +650,10 @@ class User(HashidMixin, UserMixin, db.Model):
x.hashid: x.to_json_serializeable(relationships=True)
for x in self.spacy_nlp_pipeline_models
}
json_serializeable['followed_corpora'] = {
x.hashid: x.to_json_serializeable(relationships=True)
for x in self.followed_corpora
}
if filter_by_privacy_settings:
if not self.has_profile_privacy_setting(ProfilePrivacySettings.SHOW_EMAIL):

2
app/static/js/App.js
View File

@ -8,7 +8,7 @@ class App {
this.socket.on('PATCH', (patch) => {this.onPatch(patch);});
}
getUser(userId, backrefs=false, relationships=false) {
getUser(userId, backrefs=true, relationships=true) {
if (userId in this.data.promises.getUser) {
return this.data.promises.getUser[userId];
}

3
app/static/js/ResourceLists/CorpusFileList.js
View File

@ -11,13 +11,14 @@ class CorpusFileList extends ResourceList {
this.isInitialized = false;
this.userId = listContainerElement.dataset.userId;
this.corpusId = listContainerElement.dataset.corpusId;
if (this.userId === undefined || this.corpusId === undefined) {return;}
app.subscribeUser(this.userId).then((response) => {
app.socket.on('PATCH', (patch) => {
if (this.isInitialized) {this.onPatch(patch);}
});
});
app.getUser(this.userId).then((user) => {
this.add(Object.values(user.corpora[this.corpusId].files));
this.add(Object.values(user.corpora[this.corpusId].files || user.followed_corpora[this.corpusId].files));
this.isInitialized = true;
});
}

3
app/static/js/ResourceLists/CorpusList.js
View File

@ -8,8 +8,9 @@ class CorpusList extends ResourceList {
constructor(listContainerElement, options = {}) {
super(listContainerElement, options);
this.listjs.list.addEventListener('click', (event) => {this.onClick(event)});
this.isInitialized = false;
this.isInitialized = false
this.userId = listContainerElement.dataset.userId;
if (this.userId === undefined) {return;}
app.subscribeUser(this.userId).then((response) => {
app.socket.on('PATCH', (patch) => {
if (this.isInitialized) {this.onPatch(patch);}

1
app/static/js/ResourceLists/JobInputList.js
View File

@ -11,6 +11,7 @@ class JobInputList extends ResourceList {
this.isInitialized = false;
this.userId = listContainerElement.dataset.userId;
this.jobId = listContainerElement.dataset.jobId;
if (this.userId === undefined || this.jobId === undefined) {return;}
app.subscribeUser(this.userId).then((response) => {
app.socket.on('PATCH', (patch) => {
if (this.isInitialized) {this.onPatch(patch);}

1
app/static/js/ResourceLists/JobList.js
View File

@ -10,6 +10,7 @@ class JobList extends ResourceList {
this.listjs.list.addEventListener('click', (event) => {this.onClick(event)});
this.isInitialized = false;
this.userId = listContainerElement.dataset.userId;
if (this.userId === undefined) {return;}
app.subscribeUser(this.userId).then((response) => {
app.socket.on('PATCH', (patch) => {
if (this.isInitialized) {this.onPatch(patch);}

1
app/static/js/ResourceLists/JobResultList.js
View File

@ -11,6 +11,7 @@ class JobResultList extends ResourceList {
this.isInitialized = false;
this.userId = listContainerElement.dataset.userId;
this.jobId = listContainerElement.dataset.jobId;
if (this.userId === undefined || this.jobId === undefined) {return;}
app.subscribeUser(this.userId).then((response) => {
app.socket.on('PATCH', (patch) => {
if (this.isInitialized) {this.onPatch(patch);}

15
app/static/js/ResourceLists/PublicCorpusFileList.js Normal file
View File

@ -0,0 +1,15 @@
class PublicCorpusFileList extends CorpusFileList {
get item() {
return `
<tr class="list-item clickable hoverable">
<td><span class="filename"></span></td>
<td><span class="author"></span></td>
<td><span class="title"></span></td>
<td><span class="publishing-year"></span></td>
<td class="right-align">
<a class="list-action-trigger btn-floating service-color darken waves-effect waves-light" data-list-action="view" data-service="corpus-analysis"><i class="material-icons">send</i></a>
</td>
</tr>
`.trim();
}
}

14
app/static/js/ResourceLists/PublicCorpusList.js Normal file
View File

@ -0,0 +1,14 @@
class PublicCorpusList extends CorpusList {
get item() {
return `
<tr class="list-item clickable hoverable">
<td><a class="btn-floating disabled"><i class="material-icons service-color darken" data-service="corpus-analysis">book</i></a></td>
<td><b class="title"></b><br><i class="description"></i></td>
<td><span class="status badge new corpus-status-color corpus-status-text" data-badge-caption=""></span></td>
<td class="right-align">
<a class="list-action-trigger btn-floating service-color darken waves-effect waves-light" data-list-action="view" data-service="corpus-analysis"><i class="material-icons">send</i></a>
</td>
</tr>
`.trim();
}
}

1
app/static/js/ResourceLists/SpacyNLPPipelineModelList.js
View File

@ -11,6 +11,7 @@ class SpaCyNLPPipelineModelList extends ResourceList {
this.listjs.list.addEventListener('click', (event) => {this.onClick(event)});
this.isInitialized = false;
this.userId = listContainerElement.dataset.userId;
if (this.userId === undefined) {return;}
app.subscribeUser(this.userId).then((response) => {
app.socket.on('PATCH', (patch) => {
if (this.isInitialized) {this.onPatch(patch);}

1
app/static/js/ResourceLists/TesseractOCRPipelineModelList.js
View File

@ -11,6 +11,7 @@ class TesseractOCRPipelineModelList extends ResourceList {
this.listjs.list.addEventListener('click', (event) => {this.onClick(event)});
this.isInitialized = false;
this.userId = listContainerElement.dataset.userId;
if (this.userId === undefined) {return;}
app.subscribeUser(this.userId).then((response) => {
app.socket.on('PATCH', (patch) => {
if (this.isInitialized) {this.onPatch(patch);}

10
app/static/js/ResourceLists/UserList.js
View File

@ -1,7 +1,7 @@
class UserList extends ResourceList {
static autoInit() {
for (let publicUserListElement of document.querySelectorAll('.user-list:not(.no-autoinit)')) {
new UserList(publicUserListElement);
for (let userListElement of document.querySelectorAll('.user-list:not(.no-autoinit)')) {
new UserList(userListElement);
}
}
@ -41,14 +41,14 @@ class UserList extends ResourceList {
initListContainerElement() {
if (!this.listContainerElement.hasAttribute('id')) {
this.listContainerElement.id = Utils.generateElementId('public-user-list-');
this.listContainerElement.id = Utils.generateElementId('user-list-');
}
let listSearchElementId = Utils.generateElementId(`${this.listContainerElement.id}-search-`);
this.listContainerElement.innerHTML = `
<div class="input-field">
<i class="material-icons prefix">search</i>
<input id="${listSearchElementId}" class="search" type="text"></input>
<label for="${listSearchElementId}">Search public user</label>
<label for="${listSearchElementId}">Search user</label>
</div>
<table>
<thead>
@ -77,7 +77,7 @@ class UserList extends ResourceList {
'full-name': user.full_name ? user.full_name : '',
'location': user.location ? user.location : '',
'organization': user.organization ? user.organization : '',
'corpora-online': '0'
'corpora-online': '-'
};
};

10
app/static/js/RessourceDisplays/CorpusDisplay.js
View File

@ -12,6 +12,16 @@ class CorpusDisplay extends RessourceDisplay {
.addEventListener('click', (event) => {
Utils.deleteCorpusRequest(this.userId, this.corpusId);
});
this.displayElement
.querySelector('.action-switch[data-action="toggle-is-public"]')
.addEventListener('click', (event) => {
if (event.target.tagName !== 'INPUT') {return;}
if (event.target.checked) {
Utils.enableCorpusIsPublicRequest(this.userId, this.corpusId);
} else {
Utils.disableCorpusIsPublicRequest(this.userId, this.corpusId);
}
});
}
init(user) {

82
app/static/js/Utils.js
View File

@ -69,6 +69,88 @@ class Utils {
return Utils.mergeObjectsDeep(mergedObject, ...objects.slice(2));
}
static enableCorpusIsPublicRequest(userId, corpusId) {
return new Promise((resolve, reject) => {
let corpus;
try {
corpus = app.data.users[userId].corpora[corpusId];
} catch (error) {
corpus = {};
}
let modalElement = Utils.HTMLToElement(
`
<div class="modal">
<div class="modal-content">
<h4>Hier könnte eine Warnung stehen</h4>
<p></p>
</div>
<div class="modal-footer">
<a class="action-button btn modal-close waves-effect waves-light" data-action="cancel">Cancel</a>
<a class="action-button btn modal-close red waves-effect waves-light" data-action="confirm">Confirm</a>
</div>
</div>
`
);
document.querySelector('#modals').appendChild(modalElement);
let modal = M.Modal.init(
modalElement,
{
dismissible: false,
onCloseEnd: () => {
modal.destroy();
modalElement.remove();
}
}
);
let confirmElement = modalElement.querySelector('.action-button[data-action="confirm"]');
confirmElement.addEventListener('click', (event) => {
let corpusTitle = corpus?.title;
fetch(`/corpora/${corpusId}/enable_is_public`, {method: 'POST', headers: {Accept: 'application/json'}})
.then(
(response) => {
if (response.status === 403) {app.flash('Forbidden', 'error'); reject(response);}
if (response.status === 404) {app.flash('Not Found', 'error'); reject(response);}
app.flash(`Corpus "${corpusTitle}" is public now`, 'corpus');
resolve(response);
},
(response) => {
app.flash('Something went wrong', 'error');
reject(response);
}
);
});
modal.open();
});
}
static disableCorpusIsPublicRequest(userId, corpusId) {
return new Promise((resolve, reject) => {
let corpus;
try {
corpus = app.data.users[userId].corpora[corpusId];
} catch (error) {
corpus = {};
}
let corpusTitle = corpus?.title;
fetch(`/corpora/${corpusId}/disable_is_public`, {method: 'POST', headers: {Accept: 'application/json'}})
.then(
(response) => {
if (response.status === 403) {app.flash('Forbidden', 'error'); reject(response);}
if (response.status === 404) {app.flash('Not Found', 'error'); reject(response);}
app.flash(`Corpus "${corpusTitle}" is private now`, 'corpus');
resolve(response);
},
(response) => {
app.flash('Something went wrong', 'error');
reject(response);
}
);
});
}
static buildCorpusRequest(userId, corpusId) {
return new Promise((resolve, reject) => {
let corpus;

2
app/templates/_scripts.html.j2
View File

@ -20,7 +20,9 @@
'js/RessourceDisplays/JobDisplay.js',
'js/ResourceLists/ResourceList.js',
'js/ResourceLists/CorpusFileList.js',
'js/ResourceLists/PublicCorpusFileList.js',
'js/ResourceLists/CorpusList.js',
'js/ResourceLists/PublicCorpusList.js',
'js/ResourceLists/JobList.js',
'js/ResourceLists/JobInputList.js',
'js/ResourceLists/JobResultList.js',

1
app/templates/_sidenav.html.j2
View File

@ -20,6 +20,7 @@
<li><a href="{{ url_for('main.dashboard') }}"><i class="material-icons">dashboard</i>Dashboard</a></li>
<li><a href="{{ url_for('main.dashboard', _anchor='corpora') }}" style="padding-left: 47px;"><i class="nopaque-icons">I</i>My Corpora</a></li>
<li><a href="{{ url_for('main.dashboard', _anchor='jobs') }}" style="padding-left: 47px;"><i class="nopaque-icons">J</i>My Jobs</a></li>
<li><a href="{{ url_for('main.dashboard', _anchor='social') }}" style="padding-left: 47px;"><i class="material-icons">groups</i>Social</a></li>
<li><a href="{{ url_for('contributions.contributions') }}"><i class="material-icons">new_label</i>Contribute</a></li>
<li><div class="divider"></div></li>
<li><a class="subheader">Processes & Services</a></li>

62
app/templates/corpora/corpus.html.j2
View File

@ -1,4 +1,5 @@
{% extends "base.html.j2" %}
{% import "materialize/wtf.html.j2" as wtf %}
{% from "corpora/_breadcrumbs.html.j2" import breadcrumbs with context %}
{% block main_attribs %} class="service-scheme" data-service="corpus-analysis"{% endblock main_attribs %}
@ -56,11 +57,13 @@
</div>
</div>
</div>
<div class="card-action right-align">
<a class="btn corpus-analyse-trigger disabled waves-effect waves-light" href="{{ url_for('corpora.analyse_corpus', corpus_id=corpus.id) }}"><i class="material-icons left">search</i>Analyze</a>
<a class="action-button btn disabled waves-effect waves-light" data-action="build-request"><i class="nopaque-icons left">K</i>Build</a>
<a class="btn disabled export-corpus-trigger waves-effect waves-light" href="{{ url_for('corpora.export_corpus', corpus_id=corpus.id) }}"><i class="material-icons left">import_export</i>Export</a>
<a class="action-button btn red waves-effect waves-light" data-action="delete-request"><i class="material-icons left">delete</i>Delete</a>
<div class="card-action">
<div class="right-align">
<a class="btn corpus-analyse-trigger disabled waves-effect waves-light" href="{{ url_for('corpora.analyse_corpus', corpus_id=corpus.id) }}"><i class="material-icons left">search</i>Analyze</a>
<a class="action-button btn disabled waves-effect waves-light" data-action="build-request"><i class="nopaque-icons left">K</i>Build</a>
<a class="btn disabled export-corpus-trigger waves-effect waves-light" href="{{ url_for('corpora.export_corpus', corpus_id=corpus.id) }}"><i class="material-icons left">import_export</i>Export</a>
<a class="action-button btn red waves-effect waves-light" data-action="delete-request"><i class="material-icons left">delete</i>Delete</a>
</div>
</div>
</div>
</div>
@ -76,6 +79,34 @@
</div>
</div>
</div>
{# <div class="col s12">
<div class="card">
<div class="card-content">
<div class="action-switch switch center-align" data-action="toggle-is-public">
<span class="share"></span>
<label>
<input class="corpus-is-public" {% if corpus.is_public %}checked{% endif %} type="checkbox">
<span class="lever"></span>
public
</label>
</div>
<a class="action-button btn waves-effect waves-light" id="generate-share-link-button">Generate Share Link</a>
<div id="share-link"></div>
<a class="action-button btn-small waves-effect waves-light hide" id="copy-share-link-button">Copy</a>
</div>
</div>
</div>
<div class="col s12">
<div class="card">
<div class="card-content">
<span class="card-title" id="files">Corpus followers</span>
<div class="user-list no-autoinit"></div>
</div>
</div>
</div> #}
</div>
</div>
{% endblock page_content %}
@ -85,4 +116,25 @@
<script>
let corpusDisplay = new CorpusDisplay(document.querySelector('#corpus-display'));
</script>
{# <script>
let generateShareLinkButton = document.querySelector('#generate-share-link-button');
let copyShareLinkButton = document.querySelector('#copy-share-link-button');
let shareLink = document.querySelector('#share-link');
let linkValue = '{{ url_for('corpora.share_corpus', token=token, _external=True) }}';
generateShareLinkButton.addEventListener('click', () => {
let shareLinkElement = document.createElement('input');
shareLinkElement.value = linkValue;
shareLinkElement.setAttribute('readonly', '');
shareLink.appendChild(shareLinkElement);
copyShareLinkButton.classList.remove('hide');
});
copyShareLinkButton.addEventListener('click', () => {
let shareLinkElement = document.querySelector('#share-link input');
shareLinkElement.select();
document.execCommand('copy');
app.flash(`Copied!`, 'success');
});
</script> #}
{% endblock scripts %}

0
app/templates/corpora/corpora.html.j2 → app/templates/corpora/public_corpora.html.j2
View File

80
app/templates/corpora/public_corpus.html.j2 Normal file
View File

@ -0,0 +1,80 @@
{% extends "base.html.j2" %}
{% import "materialize/wtf.html.j2" as wtf %}
{% block main_attribs %} class="service-scheme" data-service="corpus-analysis"{% endblock main_attribs %}
{% block page_content %}
<div class="container">
<div class="row">
<div class="col s12">
<h1>{{ corpus.title }} </h1>
<div class="row">
<div class="col s8 m9 l10">
{% if current_user.is_following_corpus(corpus) %}
<a class="action-button btn waves-effect waves-light" data-action="unfollow-request"><i class="material-icons left">add</i>Unfollow Corpus</a>
{% endif %}
{% if corpus.status.name in ['BUILT', 'STARTING_ANALYSIS_SESSION', 'RUNNING_ANALYSIS_SESSION', 'CANCELING_ANALYSIS_SESSION'] and current_user.is_following_corpus(corpus) %}
<a class="btn waves-effect waves-light" href="{{ url_for('corpora.analyse_corpus', corpus_id=corpus.id) }}">Analyze</a>
{% endif %}
</div>
</div>
<div class="card service-color-border border-darken" data-service="corpus-analysis" style="border-top: 10px solid">
<div class="card-content">
<div class="row">
<div class="col s12">
<p><b>Status:</b> <span class="chip corpus-status-text corpus-status-color white-text" data-status="{{ corpus.status.name }}"></span></p>
<p></p>
<br>
</div>
<div class="col s12">
<p><b>Description:</b> {{ corpus.description }}</p>
<br>
<p></p>
</div>
<div class="col s6">
<p><b>Creation date:</b> {{ corpus.creation_date }}</p>
</div>
<div class="col s6">
<p><b>Number of tokens used:</b> {{ corpus.num_tokens }}</p>
</div>
</div>
</div>
</div>
<div class="card">
<div class="card-content">
<span class="card-title" id="files">Corpus files</span>
<div class="corpus-file-list no-autoinit" data-user-id="{{ corpus.user.hashid }}" data-corpus-id="{{ corpus.hashid }}"></div>
</div>
</div>
</div>
</div>
</div>
</div>
{% endblock page_content %}
{% block scripts %}
{{ super() }}
<script>
let corpusFileList = new PublicCorpusFileList(document.querySelector('.corpus-file-list'));
corpusFileList.add({{ corpus_files|tojson }});
let unfollowRequestElement = document.querySelector('.action-button[data-action="unfollow-request"]');
unfollowRequestElement.addEventListener('click', () => {
return new Promise((resolve, reject) => {
fetch('{{ url_for("corpora.unfollow_corpus", corpus_id=corpus.id) }}', {method: 'POST', headers: {Accept: 'application/json'}})
.then(
(response) => {
if (response.status === 403) {app.flash('Forbidden', 'error'); reject(response);}
if (response.status === 404) {app.flash('Not Found', 'error'); reject(response);}
resolve(response);
window.location.href = '{{ url_for("corpora.corpus", corpus_id=corpus.id) }}';
},
(response) => {
app.flash('Something went wrong', 'error');
reject(response);
}
);
});
});
</script>
{% endblock scripts %}

28
app/templates/main/dashboard.html.j2
View File

@ -42,6 +42,23 @@
</div>
</div>
</div>
{# <div class="col s12" id="social">
<h3>Social</h3>
<div class="card">
<div class="card-content">
<span class="card-title">Other users</span>
<p>Find other users and follow them to see their corpora.</p>
<div class="user-list no-autoinit"></div>
</div>
</div>
<div class="card">
<div class="card-content">
<span class="card-title">Public corpora</span>
<p>Find public corpora</p>
<div class="public-corpus-list no-autoinit"></div>
</div>
</div>
</div> #}
</div>
</div>
{% endblock page_content %}
@ -96,3 +113,14 @@
</div>
</div>
{% endblock modals %}
{% block scripts %}
{{ super() }}
{# <script>
let userList = new UserList(document.querySelector('.user-list'));
userList.add({{ users|tojson }});
let publicCorpusList = new PublicCorpusList(document.querySelector('.public-corpus-list'));
publicCorpusList.add({{ corpora|tojson }});
</script> #}
{% endblock scripts %}

269
app/templates/main/news_new.html.j2 Normal file
View File

@ -0,0 +1,269 @@
{% extends "base.html.j2" %}
{% from "main/_breadcrumbs.html.j2" import breadcrumbs with context %}
{% block page_content %}
<div class="container">
<div class="row">
<div class="col s12">
<h1 id="title">{{ title }}</h1>
</div>
<div class="col s12">
<div id="aggregated-news"></div>
<div class="card" id="april-2022-update">
<div class="card-content">
<span class="card-title">April 2022 update</span>
<p>Dear users</p>
<br>
<p>
with the April 2022 update we have improved nopaque in all places.
We have significantly reworked our backend code to utilize our servers more efficiently,
integrated a new service, updated all previously existing ones, rewrote a lot of code and made a few minor design improvements.
</p>
<br>
<span class="card-title">Where is my Job data?</span>
<p>
At the beginning of the year, we realized that our storage limit had been reached.
This was the time when some users may have noticed system instabilities.
We were fortunately able to temporarily solve this problem without data loss
by deleting some non-nopaque related data on our system (yes we also do <a href="https://digital-history.uni-bielefeld.de">other things then nopaque</a>).
In order to not face the same problem again, we had to dedicate ourselves to a long-term solution.
This consists of deleting all previous job data with this update and henceforth storing new job data
only for three months after job creation (important note: <b>corpora are not affected</b>).
All job data prior to this update has been backed up for you,
feel free to contact us at nopaque@uni-bielefeld.de if you would like to get this data back.
</p>
<br>
<span class="card-title">What's new?</span>
<p>
By partnering up with <a href="https://readcoop.eu/transkribus/?sc=Transkribus">Transkribus</a> we reached one of our long term goals: integrate a HTR service into nopaque.
The <a href="{{ url_for('services.transkribus_htr_pipeline') }}">Transkribus HTR Pipeline</a> service is implemented as a kind of proxied service where the work is split between Transkribus and us.
That means we do the preprocessing, storage and postprocessing, while Transkribus handles the HTR itself.
</p>
<br>
<p>
One of the changes in the background was to fix our performance issues. While implementing the <a href="{{ url_for('services.transkribus_htr_pipeline') }}">Transkribus HTR Pipeline</a> service we
found some optimization potential within different steps of our processing routine. These optimizations are now also
available in our <a href="{{ url_for('services.transkribus_htr_pipeline') }}">Tesseract OCR Pipeline</a> service, resulting in a speed up of about 4x.
For now we are done with the most obvious optimizations but we may include more in the near future, so stay tuned!
</p>
<br>
<p>
The next step was to reorganize our <a href="{{ url_for('services.corpus_analysis') }}">Corpus Analysis</a> code. Unfortunatly it was a bit messy, after a complete rewrite we are
now able to query a corpus without long loading times and with better error handling, resulting in way more stable user experience.
The Corpus Analysis service is now modularized and comes with 2 modules that recreate and extend the functionality of the old service.<br>
For now we had to disable the Query Result viewer, the code was based on the old Corpus Analysis service and will be reintegrated as a module to the Corpus Analysis.
</p>
<br>
<p>
The <a href="{{ url_for('services.spacy_nlp_pipeline') }}">spaCy NLP Pipeline</a> service got some love in the form of smaller updates too.
This is important preliminary work to support more models/languages that does not provide the full set of linguistic features (lemma, ner, pos, simple_pos). It still needs some testing and tweaking but will be ready soon!
</p>
<br>
<p>
Last but not least we made some design changes. Now you can find colors in places where we had just black and white before.
Nothing big but the new colors will help you identify ressources more efficient!
</p>
<br>
<span class="card-title">Database cleanup</span>
<p>
We may be a bit late with our spring cleaning but with this update we tidied up within our database system.
This means we deleted old corpora with no corpus files, unconfirmed user accounts and in general unnecessary data fields.
</p>
<br>
<p>
That's it, thank you for using nopaque! We hope you like the update and appreciate all your past and future feedback.
</p>
</div>
</div>
</div>
<div class="col s12">
<div class="card" id="maintenance">
<div class="card-content">
<span class="card-title">Maintenance</span>
<p>Dear users</p>
<br>
<p>Currently we are rewriting big parts of our project infrastructure. Due to this the following features are not available:</p>
<ul>
<li>Corpus export and import</li>
<li>Query result export, import and view</li>
</ul>
<p>We hope to add these features back in the near future, until then check out our updated corpus analysis.</p>
</div>
</div>
</div>
<div class="col s12">
<div class="card" id="nlp-removed-language-support">
<div class="card-content">
<span class="card-title">Natural Language Processing removed language support</span>
<p>Dear users</p>
<br>
<p>Not all language models support all features we utizlize in our NLP service. Thats why we had to drop them, as soon as they meet our requirements we will add them back!</p>
</div>
</div>
</div>
<div class="col s12">
<div class="card" id="beta-launch">
<div class="card-content">
<span class="card-title">nopaque's beta launch</span>
<p>Dear users</p>
<br>
<p>A few days ago we went live with nopaque. Right now nopaque is still in its Beta phase. So some bugs are to be expected. If you encounter any bugs or some feature is not working as expected please send as an email using the feedback button at the botton of the page in the footer!</p>
<p>We are happy to help you with any issues and will use the feedback to fix all mentioned bugs!</p>
</div>
</div>
</div>
</div>
</div>
{% endblock page_content %}
{% block scripts %}
{{ super() }}
<script>
function getMastodonStatuses() {
return new Promise((resolve, reject) => {
fetch(`https://fedihum.org/api/v1/accounts/109386364241901080/statuses`, {method: 'GET', headers: {Accept: 'application/json'}})
.then((response) => {
if (!response.ok) {reject(response);}
return response.json();
})
.then((statuses) => {resolve(statuses);})
});
}
function getBisBlogsEntries() {
return new Promise((resolve, reject) => {
fetch(`https://blogs.uni-bielefeld.de/blog/uniintern/feed/entries/atom?cat=%2FAllgemein`, {method: 'GET', headers: {Accept: 'application/xml'}})
.then((response) => {
if (!response.ok) {reject(response);}
return response.text();
})
.then((responseText) => {return new DOMParser().parseFromString(responseText, 'application/xml');})
.then((xmlDocument) => {return xmlDocument.toObject();})
.then((feed) => {resolve(feed);});
});
}
function sortAggregatedNews(a, b) {
let aDate;
let bDate;
switch (a.source) {
case 'mastodon':
aDate = new Date(a.created_at);
break;
case 'big-blogs':
aDate = new Date(a.published);
break;
default:
throw new Error('Unknown source');
}
switch (b.source) {
case 'mastodon':
bDate = new Date(b.created_at);
break;
case 'big-blogs':
bDate = new Date(b.published);
break;
default:
throw new Error('Unknown source');
}
return bDate - aDate;
}
function aggregateNews() {
return new Promise((resolve, reject) => {
Promise.all([getMastodonStatuses(), getBisBlogsEntries()])
.then(
(responses) => {
console.log(responses[1]);
let mastodonStatuses = responses[0].map((obj) => {return { ...obj, source: 'mastodon'}});
let bisBlogsEntries = responses[1].feed.entry.map((obj) => {return { ...obj, source: 'big-blogs'};});
let aggregatedNews = [...mastodonStatuses, ...bisBlogsEntries];
aggregatedNews.sort(sortAggregatedNews);
resolve(aggregatedNews);
},
(error) => {reject(error);}
);
});
}
function mastodonStatusToElement(status) {
let date = new Date(status.created_at).toLocaleString('en-US');
let newsElement = Utils.HTMLToElement(
`
<div class="row">
<div class="col s11">
<div class="card white-text" style="background-color:#5D50E7; border-radius:10px;">
<div class="card-content">
<span class="card-title">New Actitvity on Mastodon</span>
<p><i>Published on ${date}</i></p>
<br>
<p>${status.content}</p>
</div>
</div>
</div>
<div class="col s1">
<img src="https://joinmastodon.org/logos/logo-purple.svg" alt="Mastodon" class="responsive-img hide-on-small-only" style="width:70%; margin-top:30px;">
</div>
</div>
`
);
return newsElement;
}
function bisBlogsEntryToElement(entry) {
let date = new Date(entry.published).toLocaleString('en-US');
let newsElement = Utils.HTMLToElement(
`
<div class="row">
<div class="col s1">
<img src="https://blogs.uni-bielefeld.de/blog/uniintern/resource/themabilder/unilogo-square.svg" alt="Bielefeld University Blogs" class="responsive-img hide-on-small-only" style="width:70%; margin-top:40px;">
</div>
<div class="col s11">
<div class="card" style="background-color: #A5BDCC; border-radius:10px;">
<div class="card-content">
<span class="card-title">${entry.title['#text']}</span>
<p><i>Published on ${date}</i></p>
<br>
<p>${entry.content['#text']}</p>
</div>
</div>
</div>
</div>
`
);
let newsImageElements = newsElement.querySelectorAll('img');
for (let newsImageElement of newsImageElements) {
newsImageElement.classList.add('responsive-img');
}
return newsElement;
}
let aggregatedNewsElement = document.querySelector('#aggregated-news');
aggregateNews().then((aggregatedNews) => {
for (let item of aggregatedNews) {
let newsElement;
switch (item.source) {
case 'mastodon':
newsElement = mastodonStatusToElement(item);
break;
case 'big-blogs':
newsElement = bisBlogsEntryToElement(item);
break;
default:
throw new Error('Unknown source');
}
aggregatedNewsElement.appendChild(newsElement);
}
});
</script>
{% endblock scripts %}

18
app/templates/users/profile.html.j2
View File

@ -89,6 +89,24 @@
</div>
</div>
</div>
<div class="row">
<div class="col s6">
<div class="card">
<div class="card-content">
<h4>Groups</h4>
</div>
</div>
</div>
<div class="col s6">
<div class="card">
<div class="card-content">
<h4>Public corpora</h4>
<div class="public-corpora-list" data-user-id="{{ user.hashid }}"></div>
</div>
</div>
</div>
</div>
</div>
{% endblock page_content %}

4
requirements.txt
View File

@ -6,7 +6,7 @@ eventlet
Flask==2.1.3
Flask-APScheduler
Flask-Assets
Flask-Hashids==1.0.0
Flask-Hashids==1.0.1
Flask-HTTPAuth
Flask-Login
Flask-Mail
@ -24,6 +24,6 @@ pyScss
python-dotenv
pyyaml
redis
SQLAlchemy==1.4.46
SQLAlchemy==1.4.45
tqdm
wtforms[email]