Implement corpora endpoint/socket.io namespace

This commit is contained in:
Patrick Jentsch 2024-12-09 16:12:49 +01:00
parent 93344c9573
commit 328f85ba52
16 changed files with 339 additions and 182 deletions

View File

@ -133,6 +133,9 @@ def create_app(config: Config = Config) -> Flask:
from .namespaces.cqi_over_sio import CQiOverSocketIONamespace
socketio.on_namespace(CQiOverSocketIONamespace('/cqi_over_sio'))
from .namespaces.corpora import CorporaNamespace
socketio.on_namespace(CorporaNamespace('/corpora'))
from .namespaces.jobs import JobsNamespace
socketio.on_namespace(JobsNamespace('/jobs'))

View File

@ -1,45 +0,0 @@
from flask_login import current_user
from flask_socketio import join_room
from app import hashids, socketio
from app.decorators import socketio_login_required
from app.models import Corpus
@socketio.on('GET /corpora/<corpus_id>')
@socketio_login_required
def get_corpus(corpus_hashid):
corpus_id = hashids.decode(corpus_hashid)
corpus = Corpus.query.get(corpus_id)
if corpus is None:
return {'options': {'status': 404, 'statusText': 'Not found'}}
if not (
corpus.is_public
or corpus.user == current_user
or current_user.is_administrator
):
return {'options': {'status': 403, 'statusText': 'Forbidden'}}
return {
'body': corpus.to_json_serializable(),
'options': {
'status': 200,
'statusText': 'OK',
'headers': {'Content-Type: application/json'}
}
}
@socketio.on('SUBSCRIBE /corpora/<corpus_id>')
@socketio_login_required
def subscribe_corpus(corpus_hashid):
corpus_id = hashids.decode(corpus_hashid)
corpus = Corpus.query.get(corpus_id)
if corpus is None:
return {'options': {'status': 404, 'statusText': 'Not found'}}
if not (
corpus.is_public
or corpus.user == current_user
or current_user.is_administrator
):
return {'options': {'status': 403, 'statusText': 'Forbidden'}}
join_room(f'/corpora/{corpus.hashid}')
return {'options': {'status': 200, 'statusText': 'OK'}}

215
app/namespaces/corpora.py Normal file
View File

@ -0,0 +1,215 @@
from datetime import datetime
from flask import current_app, Flask, url_for
from flask_login import current_user
from flask_socketio import Namespace
from string import punctuation
import nltk
from app import db, hashids, socketio
from app.decorators import socketio_login_required
from app.models import Corpus, CorpusFollowerAssociation, CorpusFollowerRole
def _delete_corpus(app: Flask, corpus_id: int):
with app.app_context():
corpus = Corpus.query.get(corpus_id)
corpus.delete()
db.session.commit()
def _build_corpus(app: Flask, corpus_id: int):
with app.app_context():
corpus = Corpus.query.get(corpus_id)
corpus.build()
db.session.commit()
class CorporaNamespace(Namespace):
@socketio_login_required
def on_delete(self, corpus_hashid: str) -> dict:
if not isinstance(corpus_hashid, str):
return {'status': 400, 'statusText': 'Bad Request'}
corpus_id = hashids.decode(corpus_hashid)
if not isinstance(corpus_id, int):
return {'status': 400, 'statusText': 'Bad Request'}
corpus = Corpus.query.get(corpus_id)
if corpus is None:
return {'status': 404, 'statusText': 'Not Found'}
if not (
corpus.user == current_user
or current_user.is_administrator
):
return {'status': 403, 'statusText': 'Forbidden'}
socketio.start_background_task(
_delete_corpus,
current_app._get_current_object(),
corpus_id
)
return {
'body': f'Corpus "{corpus.title}" marked for deletion',
'status': 202,
'statusText': 'Accepted'
}
@socketio_login_required
def on_build(self, corpus_hashid: str) -> dict:
if not isinstance(corpus_hashid, str):
return {'status': 400, 'statusText': 'Bad Request'}
corpus_id = hashids.decode(corpus_hashid)
if not isinstance(corpus_id, int):
return {'status': 400, 'statusText': 'Bad Request'}
corpus = Corpus.query.get(corpus_id)
if corpus is None:
return {'status': 404, 'statusText': 'Not Found'}
cfa = CorpusFollowerAssociation.query.filter_by(corpus_id=corpus_id, follower_id=current_user.id).first()
if not (
cfa is not None and cfa.role.has_permission('MANAGE_FILES')
or corpus.user == current_user
or current_user.is_administrator
):
return {'status': 403, 'statusText': 'Forbidden'}
if len(corpus.files.all()) == 0:
return {'status': 409, 'statusText': 'Conflict'}
socketio.start_background_task(
_build_corpus,
current_app._get_current_object(),
corpus_id
)
return {
'body': f'Corpus "{corpus.title}" marked for building',
'status': 202,
'statusText': 'Accepted'
}
# TODO: Think about where to place this, as this does not belong here...
@socketio_login_required
def on_get_stopwords(self):
languages = [
'german',
'english',
'catalan',
'greek',
'spanish',
'french',
'italian',
'russian',
'chinese'
]
nltk.download('stopwords', quiet=True)
stopwords = {
language: nltk.corpus.stopwords.words(language)
for language in languages
}
stopwords['punctuation'] = list(punctuation)
stopwords['punctuation'] += ['', '|', '', '', '', '--']
stopwords['user_stopwords'] = []
return {
'body': stopwords,
'status': 200,
'statusText': 'OK'
}
@socketio_login_required
def on_create_share_link(self, corpus_hashid: str, expiration_date: str, role_name: str) -> dict:
if not isinstance(corpus_hashid, str):
return {'status': 400, 'statusText': 'Bad Request'}
if not isinstance(expiration_date, str):
return {'status': 400, 'statusText': 'Bad Request'}
if not isinstance(role_name, str):
return {'status': 400, 'statusText': 'Bad Request'}
print(corpus_hashid, expiration_date, role_name)
corpus_id = hashids.decode(corpus_hashid)
if not isinstance(corpus_id, int):
return {'status': 400, 'statusText': 'Bad Request'}
corpus = Corpus.query.get(corpus_id)
if corpus is None:
return {'status': 404, 'statusText': 'Not Found'}
cfa = CorpusFollowerAssociation.query.filter_by(corpus_id=corpus_id, follower_id=current_user.id).first()
if not (
cfa is not None and cfa.role.has_permission('MANAGE_FOLLOWERS')
or corpus.user == current_user
or current_user.is_administrator
):
return {'status': 403, 'statusText': 'Forbidden'}
_expiration_date = datetime.strptime(expiration_date, '%b %d, %Y')
cfr = CorpusFollowerRole.query.filter_by(name=role_name).first()
if cfr is None:
return {'status': 400, 'statusText': 'Bad Request'}
token = current_user.generate_follow_corpus_token(
corpus.hashid,
role_name,
_expiration_date
)
corpus_share_link = url_for(
'corpora.follow_corpus',
corpus_id=corpus_id,
token=token,
_external=True
)
return {
'body': corpus_share_link,
'status': 200,
'statusText': 'OK'
}
@socketio_login_required
def on_set_is_public(corpus_hashid: str, new_value: bool) -> dict:
if not isinstance(corpus_id, str):
return {'status': 400, 'statusText': 'Bad Request'}
if not isinstance(new_value, bool):
return {'status': 400, 'statusText': 'Bad Request'}
corpus_id = hashids.decode(corpus_hashid)
if not isinstance(corpus_id, int):
return {'status': 400, 'statusText': 'Bad Request'}
corpus = Corpus.query.get(corpus_id)
if corpus is None:
return {'status': 404, 'statusText': 'Not Found'}
if not (
corpus.user == current_user
or current_user.is_administrator
):
return {'status': 403, 'statusText': 'Forbidden'}
corpus.is_public = new_value
db.session.commit()
return {
'body': f'Corpus "{corpus.title}" is now {"public" if new_value else "private"}',
'status': 200,
'statusText': 'OK'
}

View File

@ -169,6 +169,7 @@ class CQiOverSocketIONamespace(Namespace):
for param in signature(fn).parameters.values():
# Check if the parameter is optional or required
# The following is true for required parameters
if param.default is param.empty:
if param.name not in fn_args:
return {'code': 400, 'msg': 'Bad Request'}

View File

@ -23,6 +23,9 @@ def _restart_job(app: Flask, job_id: int):
class JobsNamespace(Namespace):
@socketio_login_required
def on_delete(self, job_hashid: str) -> dict:
if not isinstance(job_hashid, str):
return {'status': 400, 'statusText': 'Bad Request'}
job_id = hashids.decode(job_hashid)
if not isinstance(job_id, int):
@ -31,7 +34,7 @@ class JobsNamespace(Namespace):
job = Job.query.get(job_id)
if job is None:
return {'status': 404, 'statusText': 'Not found'}
return {'status': 404, 'statusText': 'Not Found'}
if not (
job.user == current_user
@ -53,6 +56,9 @@ class JobsNamespace(Namespace):
@socketio_admin_required
def on_log(self, job_hashid: str) -> dict:
if not isinstance(job_hashid, str):
return {'status': 400, 'statusText': 'Bad Request'}
job_id = hashids.decode(job_hashid)
if not isinstance(job_id, int):
@ -61,7 +67,7 @@ class JobsNamespace(Namespace):
job = Job.query.get(job_id)
if job is None:
return {'status': 404, 'statusText': 'Not found'}
return {'status': 404, 'statusText': 'Not Found'}
if job.status not in [JobStatus.COMPLETED, JobStatus.FAILED]:
return {'status': 409, 'statusText': 'Conflict'}
@ -72,11 +78,14 @@ class JobsNamespace(Namespace):
return {
'body': log,
'status': 200,
'statusText': 'Forbidden'
'statusText': 'OK'
}
socketio_login_required
def on_restart(self, job_hashid: str) -> dict:
if not isinstance(job_hashid, str):
return {'status': 400, 'statusText': 'Bad Request'}
job_id = hashids.decode(job_hashid)
if not isinstance(job_id, int):
@ -85,7 +94,7 @@ class JobsNamespace(Namespace):
job = Job.query.get(job_id)
if job is None:
return {'status': 404, 'statusText': 'Not found'}
return {'status': 404, 'statusText': 'Not Found'}
if not (
job.user == current_user

View File

@ -16,6 +16,9 @@ def _delete_user(app: Flask, user_id: int):
class UsersNamespace(Namespace):
@socketio_login_required
def on_get(self, user_hashid: str) -> dict:
if not isinstance(user_hashid, str):
return {'status': 400, 'statusText': 'Bad Request'}
user_id = hashids.decode(user_hashid)
if not isinstance(user_id, int):
@ -24,7 +27,7 @@ class UsersNamespace(Namespace):
user = User.query.get(user_id)
if user is None:
return {'status': 404, 'statusText': 'Not found'}
return {'status': 404, 'statusText': 'Not Found'}
if not (
user == current_user
@ -43,6 +46,9 @@ class UsersNamespace(Namespace):
@socketio_login_required
def on_subscribe(self, user_hashid: str) -> dict:
if not isinstance(user_hashid, str):
return {'status': 400, 'statusText': 'Bad Request'}
user_id = hashids.decode(user_hashid)
if not isinstance(user_id, int):
@ -51,7 +57,7 @@ class UsersNamespace(Namespace):
user = User.query.get(user_id)
if user is None:
return {'status': 404, 'statusText': 'Not found'}
return {'status': 404, 'statusText': 'Not Found'}
if not (
user == current_user
@ -65,6 +71,9 @@ class UsersNamespace(Namespace):
@socketio_login_required
def on_unsubscribe(self, user_hashid: str) -> dict:
if not isinstance(user_hashid, str):
return {'status': 400, 'statusText': 'Bad Request'}
user_id = hashids.decode(user_hashid)
if not isinstance(user_id, int):
@ -73,7 +82,7 @@ class UsersNamespace(Namespace):
user = User.query.get(user_id)
if user is None:
return {'status': 404, 'statusText': 'Not found'}
return {'status': 404, 'statusText': 'Not Found'}
if not (
user == current_user
@ -87,6 +96,9 @@ class UsersNamespace(Namespace):
@socketio_login_required
def on_delete(self, user_hashid: str) -> dict:
if not isinstance(user_hashid, str):
return {'status': 400, 'statusText': 'Bad Request'}
user_id = hashids.decode(user_hashid)
if not isinstance(user_id, int):
@ -95,7 +107,7 @@ class UsersNamespace(Namespace):
user = User.query.get(user_id)
if user is None:
return {'status': 404, 'statusText': 'Not found'}
return {'status': 404, 'statusText': 'Not Found'}
if not (
user == current_user

View File

@ -3,6 +3,7 @@ nopaque.App = class App {
this.socket = io({transports: ['websocket'], upgrade: false});
// Endpoints
this.corpora = new nopaque.app.endpoints.Corpora(this);
this.jobs = new nopaque.app.endpoints.Jobs(this);
this.users = new nopaque.app.endpoints.Users(this);

View File

@ -0,0 +1,57 @@
nopaque.app.endpoints.Corpora = class Corpora {
constructor(app) {
this.app = app;
this.socket = io('/corpora', {transports: ['websocket'], upgrade: false});
}
async delete(id) {
const response = await this.socket.emitWithAck('delete', id);
if (response.status != 202) {
throw new Error(`[${response.status}] ${response.statusText}`);
}
return response.body;
}
async build(id) {
const response = await this.socket.emitWithAck('build', id);
if (response.status != 202) {
throw new Error(`[${response.status}] ${response.statusText}`);
}
return response.body;
}
async getStopwords() {
const response = await this.socket.emitWithAck('get_stopwords');
if (response.status != 200) {
throw new Error(`[${response.status}] ${response.statusText}`);
}
return response.body;
}
async createShareLink(id, expirationDate, roleName) {
const response = await this.socket.emitWithAck('create_share_link', id, expirationDate, roleName);
if (response.status != 200) {
throw new Error(`[${response.status}] ${response.statusText}`);
}
return response.body;
}
async setIsPublic(id, newValue) {
const response = await this.socket.emitWithAck('set_is_public', id, newValue);
if (response.status != 200) {
throw new Error(`[${response.status}] ${response.statusText}`);
}
return response.body;
}
}

View File

@ -7,7 +7,6 @@ nopaque.corpus_analysis.StaticVisualizationExtension = class StaticVisualization
stopwords: undefined,
originalStopwords: {},
stopwordCache: {},
promises: {getStopwords: undefined},
tokenSet: new Set()
};
@ -73,22 +72,11 @@ nopaque.corpus_analysis.StaticVisualizationExtension = class StaticVisualization
}
}
getStopwords() {
this.data.promises.getStopwords = new Promise((resolve, reject) => {
nopaque.requests.corpora.entity.getStopwords()
.then((response) => {
response.json()
.then((json) => {
this.data.originalStopwords = structuredClone(json);
this.data.stopwords = structuredClone(json);
resolve(this.data.stopwords);
})
.catch((error) => {
reject(error);
});
});
});
return this.data.promises.getStopwords;
async getStopwords() {
const stopwords = await app.corpora.getStopwords();
this.data.originalStopwords = structuredClone(stopwords);
this.data.stopwords = structuredClone(stopwords);
return stopwords;
}
renderGeneralCorpusInfo() {

View File

@ -5,50 +5,6 @@ nopaque.requests.corpora = {};
nopaque.requests.corpora.entity = {};
nopaque.requests.corpora.entity.delete = (corpusId) => {
let input = `/corpora/${corpusId}`;
let init = {
method: 'DELETE'
};
return nopaque.requests.JSONfetch(input, init);
};
nopaque.requests.corpora.entity.build = (corpusId) => {
let input = `/corpora/${corpusId}/build`;
let init = {
method: 'POST',
};
return nopaque.requests.JSONfetch(input, init);
};
nopaque.requests.corpora.entity.generateShareLink = (corpusId, role, expiration) => {
let input = `/corpora/${corpusId}/generate-share-link`;
let init = {
method: 'POST',
body: JSON.stringify({role: role, expiration: expiration})
};
return nopaque.requests.JSONfetch(input, init);
};
nopaque.requests.corpora.entity.getStopwords = () => {
let input = `/corpora/stopwords`;
let init = {
method: 'GET'
};
return nopaque.requests.JSONfetch(input, init);
};
nopaque.requests.corpora.entity.isPublic = {};
nopaque.requests.corpora.entity.isPublic.update = (corpusId, isPublic) => {
let input = `/corpora/${corpusId}/is_public`;
let init = {
method: 'PUT',
body: JSON.stringify(isPublic)
};
return nopaque.requests.JSONfetch(input, init);
};
/*****************************************************************************
* Requests for /corpora/<entity>/files routes *

View File

@ -1,30 +0,0 @@
/*****************************************************************************
* Requests for /jobs routes *
*****************************************************************************/
nopaque.requests.jobs = {};
nopaque.requests.jobs.entity = {};
nopaque.requests.jobs.entity.delete = (jobId) => {
let input = `/jobs/${jobId}`;
let init = {
method: 'DELETE'
};
return nopaque.requests.JSONfetch(input, init);
};
nopaque.requests.jobs.entity.log = (jobId) => {
let input = `/jobs/${jobId}/log`;
let init = {
method: 'GET'
};
return nopaque.requests.JSONfetch(input, init);
};
nopaque.requests.jobs.entity.restart = (jobId) => {
let input = `/jobs/${jobId}/restart`;
let init = {
method: 'POST'
};
return nopaque.requests.JSONfetch(input, init);
};

View File

@ -7,7 +7,7 @@ nopaque.resource_displays.CorpusDisplay = class CorpusDisplay extends nopaque.re
this.displayElement
.querySelector('.action-button[data-action="build-request"]')
.addEventListener('click', (event) => {
nopaque.requests.corpora.entity.build(this.corpusId);
app.corpora.build(this.corpusId);
});
}

View File

@ -180,7 +180,7 @@ nopaque.resource_lists.CorpusList = class CorpusList extends nopaque.resource_li
window.location.reload();
});
} else {
nopaque.requests.corpora.entity.delete(itemId);
app.corpora.delete(itemId);
}
});
modal.open();
@ -276,7 +276,7 @@ nopaque.resource_lists.CorpusList = class CorpusList extends nopaque.resource_li
let listItem = this.listjs.get('id', selectedItemId)[0].elm;
let values = this.listjs.get('id', listItem.dataset.id)[0].values();
if (values['is-owner']) {
nopaque.requests.corpora.entity.delete(selectedItemId);
app.corpora.delete(selectedItemId);
} else {
nopaque.requests.corpora.entity.followers.entity.delete(selectedItemId, currentUserId);
setTimeout(() => {

View File

@ -11,6 +11,7 @@
'js/app.js',
'js/app/index.js',
'js/app/endpoints/index.js',
'js/app/endpoints/corpora.js',
'js/app/endpoints/jobs.js',
'js/app/endpoints/users.js',
'js/app/extensions/index.js',
@ -51,7 +52,6 @@
'js/requests/admin.js',
'js/requests/contributions.js',
'js/requests/corpora.js',
'js/requests/jobs.js',
'js/requests/users.js',
'js/corpus-analysis/index.js',

View File

@ -246,7 +246,7 @@
let publishingModalIsPublicSwitchElement = document.querySelector('#publishing-modal-is-public-switch');
publishingModalIsPublicSwitchElement.addEventListener('change', (event) => {
let newIsPublic = publishingModalIsPublicSwitchElement.checked;
nopaque.requests.corpora.entity.isPublic.update({{ corpus.hashid|tojson }}, newIsPublic)
app.corpora.setIsPublic.update({{ corpus.hashid|tojson }}, newIsPublic)
.catch((response) => {
publishingModalIsPublicSwitchElement.checked = !newIsPublic;
});
@ -256,7 +256,7 @@ publishingModalIsPublicSwitchElement.addEventListener('change', (event) => {
// #region Delete
let deleteModalDeleteButtonElement = document.querySelector('#delete-modal-delete-button');
deleteModalDeleteButtonElement.addEventListener('click', (event) => {
nopaque.requests.corpora.entity.delete({{ corpus.hashid|tojson }})
app.corpora.delete({{ corpus.hashid|tojson }})
.then((response) => {
window.location.href = {{ url_for('main.dashboard')|tojson }};
});
@ -346,19 +346,14 @@ M.Modal.init(
shareLinkModalOutputContainerElement.classList.add('hide');
}
}
)
);
shareLinkModalCreateButtonElement.addEventListener('click', (event) => {
let role = shareLinkModalCorpusFollowerRoleSelectElement.value;
let expiration = shareLinkModalExpirationDateDatepickerElement.value
nopaque.requests.corpora.entity.generateShareLink({{ corpus.hashid|tojson }}, role, expiration)
.then((response) => {
response.json()
.then((json) => {
shareLinkModalCreateButtonElement.addEventListener('click', async (event) => {
const role = shareLinkModalCorpusFollowerRoleSelectElement.value;
const expiration = shareLinkModalExpirationDateDatepickerElement.value;
const shareLink = await app.corpora.createShareLink({{ corpus.hashid|tojson }}, expiration, role);
shareLinkModalOutputContainerElement.classList.remove('hide');
shareLinkModalOutputFieldElement.value = json.corpusShareLink;
});
});
shareLinkModalOutputFieldElement.value = shareLink;
});
shareLinkModalOutputCopyButtonElement.addEventListener('click', (event) => {

View File

@ -273,7 +273,7 @@ publicCorpusFollowerList.add(
{% if cfr.has_permission('MANAGE_FILES') %}
let followerBuildRequest = document.querySelector('#follower-build-request');
followerBuildRequest.addEventListener('click', () => {
nopaque.requests.corpora.entity.build({{ corpus.hashid|tojson }})
app.corpora.build({{ corpus.hashid|tojson }})
.then((response) => {
window.location.reload();
});
@ -380,17 +380,12 @@ M.Modal.init(
}
)
shareLinkModalCreateButtonElement.addEventListener('click', (event) => {
let role = shareLinkModalCorpusFollowerRoleSelectElement.value;
let expiration = shareLinkModalExpirationDateDatepickerElement.value
nopaque.requests.corpora.entity.generateShareLink({{ corpus.hashid|tojson }}, role, expiration)
.then((response) => {
response.json()
.then((json) => {
shareLinkModalCreateButtonElement.addEventListener('click', async (event) => {
const roleName = shareLinkModalCorpusFollowerRoleSelectElement.value;
const expirationDate = shareLinkModalExpirationDateDatepickerElement.value;
const shareLink = await app.corpora.createShareLink({{ corpus.hashid|tojson }}, expiration, role);
shareLinkModalOutputContainerElement.classList.remove('hide');
shareLinkModalOutputFieldElement.value = json.corpusShareLink;
});
});
shareLinkModalOutputFieldElement.value = shareLink;
});
shareLinkModalOutputCopyButtonElement.addEventListener('click', (event) => {