sporada/nopaque
1
0
Fork 0
mirror of https://gitlab.ub.uni-bielefeld.de/sfb1288inf/nopaque.git synced 2024-11-15 01:05:42 +00:00
Code Issues Projects Releases Wiki Activity

Merge branch 'development' of gitlab.ub.uni-bielefeld.de:sfb1288inf/opaque into development

Browse Source
This commit is contained in:
Stephan Porada 2020-02-13 14:41:07 +01:00
commit 2edb6f9fdf
10 changed files with 251 additions and 203 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

21
README.md
View File

@ -19,8 +19,27 @@ As a last step texts can be loaded into an information retrieval system to query
## Configuration and startup ## Configuration and startup
1. **Create Docker swarm:** 1. **Create Docker swarm:**
The following part is for **users** and not the development team. The development team uses a script which sets up a local development swarm.
The generated computational workload is handled by a [Docker](https://docs.docker.com/) swarm. A swarm is a group of machines that are running Docker and joined into a cluster. It consists out of two different kinds of members, managers and workers. Currently it is not possible to specify a dedicated Docker host, instead Opaque expects the executing system to be a swarm manager of a cluster with at least one dedicated worker machine. The swarm setup process is described best in the [Docker documentation](https://docs.docker.com/engine/swarm/swarm-tutorial/). The generated computational workload is handled by a [Docker](https://docs.docker.com/) swarm. A swarm is a group of machines that are running Docker and joined into a cluster. It consists out of two different kinds of members, managers and workers. Currently it is not possible to specify a dedicated Docker host, instead Opaque expects the executing system to be a swarm manager of a cluster with at least one dedicated worker machine. The swarm setup process is described best in the [Docker documentation](https://docs.docker.com/engine/swarm/swarm-tutorial/).
The dev team can use dind_swarm_setup.sh. If the workers cannot join the manager node. Try opening the following ports using the ubuntu firewall ufw:
```bash
sudo ufw allow 2376/tcp \
&& sudo ufw allow 7946/udp \
&& sudo ufw allow 7946/tcp \
&& sudo ufw allow 80/tcp \
&& sudo ufw allow 2377/tcp \
&& sudo ufw allow 4789/udp
sudo ufw reload && sudo ufw enable
sudo systemctl restart docker
```
2. **Create a network storage:** 2. **Create a network storage:**
The dind_swarm_setup.sh script handles this step for the dev team aswell.
A shared network space is necessary so that all swarm members have access to all the data. To achieve this a [Samba](https://www.samba.org/) can be used. A shared network space is necessary so that all swarm members have access to all the data. To achieve this a [Samba](https://www.samba.org/) can be used.
``` bash ``` bash
# Example: Create a Samba share via Docker # Example: Create a Samba share via Docker
@ -55,7 +74,7 @@ $ <YOUR EDITOR> nopaque.env # Fill out the empty variables within this file.
``` bash ``` bash
# Execute the following 3 steps only on first startup # Execute the following 3 steps only on first startup
$ docker-compose run web flask db upgrade $ docker-compose run web flask db upgrade
$ docker-compose run web flask db insert-initial-database-entries $ docker-compose run web flask insert-initial-database-entries
$ docker-compose down $ docker-compose down
$ docker-compose up $ docker-compose up

3
app/static/css/nopaque.css
View File

@ -101,9 +101,6 @@ indicator will show up how the column is sorted right now.; */
.service[data-service]:before { .service[data-service]:before {
content: "help"; content: "help";
} }
.service[data-service="corpus"]:before {
content: "book";
}
.service[data-service="setup_files"]:before { .service[data-service="setup_files"]:before {
content: "burst_mode"; content: "burst_mode";
} }

59
app/static/js/nopaque.lists.js
View File

@ -1,7 +1,13 @@
class RessourceList extends List { class RessourceList extends List {
constructor(idOrElement, subscriberList, dataMapper=null, options={}) { constructor(idOrElement, subscriberList, type, options={}) {
super(idOrElement, {...RessourceList.options, ...options}); if (!['corpus', 'job'].includes(type)) {
this.dataMapper = dataMapper; console.error("Unknown Type!");
return;
}
super(idOrElement, {...RessourceList.options['common'],
...RessourceList.options[type],
...options});
this.type = type;
subscriberList.push(this); subscriberList.push(this);
} }
@ -43,19 +49,15 @@ class RessourceList extends List {
addRessources(ressources) { addRessources(ressources) {
if (this.dataMapper) { this.add(ressources.map(x => RessourceList.dataMapper[this.type](x)));
this.add(ressources.map(x => this.dataMapper(x)));
} else {
this.add(ressources);
}
} }
} }
RessourceList.dataMapper = { RessourceList.dataMapper = {
corpus: corpus => ({creation_date: corpus.creation_date, corpus: corpus => ({creation_date: corpus.creation_date,
description: corpus.description, description: corpus.description,
id: corpus.id, id: corpus.id,
link: `/corpora/${corpus.id}`, "analyse-link": `/corpora/${corpus.id}/analyse`,
service: "corpus", "edit-link": `/corpora/${corpus.id}`,
status: corpus.status, status: corpus.status,
title: corpus.title}), title: corpus.title}),
job: job => ({creation_date: job.creation_date, job: job => ({creation_date: job.creation_date,
@ -67,7 +69,31 @@ RessourceList.dataMapper = {
title: job.title}) title: job.title})
}; };
RessourceList.options = { RessourceList.options = {
item: `<tr> common: {page: 4, pagination: {innerWindow: 8, outerWindow: 1}},
corpus: {item: `<tr>
<td>
<a class="btn-floating disabled">
<i class="material-icons service">book</i>
</a>
</td>
<td>
<b class="title"></b><br>
<i class="description"></i>
</td>
<td>
<span class="badge new status" data-badge-caption=""></span>
</td>
<td class="right-align">
<a class="btn-small edit-link waves-effect waves-light"><i class="material-icons">edit</i></a>
<a class="btn-small analyse-link waves-effect waves-light">Analyse<i class="material-icons right">search</i></a>
</td>
</tr>`,
valueNames: ["creation_date", "description", "title",
{data: ["id"]},
{name: "analyse-link", attr: "href"},
{name: "edit-link", attr: "href"},
{name: "status", attr: "data-status"}]},
job: {item: `<tr>
<td> <td>
<a class="btn-floating disabled"> <a class="btn-floating disabled">
<i class="material-icons service"></i> <i class="material-icons service"></i>
@ -81,18 +107,15 @@ RessourceList.options = {
<span class="badge new status" data-badge-caption=""></span> <span class="badge new status" data-badge-caption=""></span>
</td> </td>
<td class="right-align"> <td class="right-align">
<a class="btn-small link waves-effect waves-light">View<i class="material-icons right">send</i> <a class="btn-small link waves-effect waves-light">View<i class="material-icons right">send</i></a>
</td> </td>
</tr>`, </tr>`,
page: 4, valueNames: ["creation_date", "description", "title",
pagination: {innerWindow: 8, outerWindow: 1},
valueNames: ["creation_date",
"description",
"title",
{data: ["id"]}, {data: ["id"]},
{name: "link", attr: "href"}, {name: "link", attr: "href"},
{name: "service", attr: "data-service"}, {name: "service", attr: "data-service"},
{name: "status", attr: "data-status"}]}; {name: "status", attr: "data-status"}]}
};
class ResultList extends List { class ResultList extends List {

5
app/templates/admin/user.html.j2
View File

@ -103,8 +103,9 @@
<script> <script>
var corpusList = new RessourceList("corpora", nopaque.foreignCorporaSubscribers, RessourceList.dataMapper.corpus); var corpusList = new RessourceList("corpora", nopaque.foreignCorporaSubscribers, "corpus");
var jobList = new RessourceList("jobs", nopaque.foreignJobsSubscribers, RessourceList.dataMapper.job); var jobList = new RessourceList("jobs", nopaque.foreignJobsSubscribers,
"job");
nopaque.socket.emit("foreign_user_ressources_init", {{ user.id }}); nopaque.socket.emit("foreign_user_ressources_init", {{ user.id }});
</script> </script>
{% endblock %} {% endblock %}

21
app/templates/corpora/corpus.html.j2
View File

@ -7,7 +7,20 @@
<div class="col s12 m4"> <div class="col s12 m4">
<h3 id="title">{{ corpus.title }}</h3> <h3 id="title">{{ corpus.title }}</h3>
<p id="description">{{ corpus.description }}</p> <p id="description">{{ corpus.description }}</p>
<a class="btn status waves-effect waves-light" id="status"></a> <div class="active preloader-wrapper small" id="progress-indicator">
<div class="spinner-layer spinner-blue-only">
<div class="circle-clipper left">
<div class="circle"></div>
</div>
<div class="gap-patch">
<div class="circle"></div>
</div>
<div class="circle-clipper right">
<div class="circle"></div>
</div>
</div>
</div>
<span class="chip status white-text" id="status"></span>
</div> </div>
<div class="col s12 m8"> <div class="col s12 m8">
@ -150,8 +163,12 @@
} }
setStatus(status) { setStatus(status) {
let analyseBtn, statusElement; let analyseBtn, progressIndicator, statusElement;
if (status != "preparable" && status != "preparing") {
progressIndicator = document.getElementById("progress-indicator");
progressIndicator.classList.add("hide");
}
statusElement = document.getElementById("status"); statusElement = document.getElementById("status");
statusElement.dataset.status = status; statusElement.dataset.status = status;
analyseBtn = document.getElementById('analyse'); analyseBtn = document.getElementById('analyse');

197
app/templates/jobs/job.html.j2
View File

@ -2,87 +2,36 @@
{% set full_width = False %} {% set full_width = False %}
{% set roadmap = False %} {% set roadmap = False %}
{% set title_override = '<i class="material-icons left service" data-service="{service}" style="font-size: inherit;"></i>Job'.format(service=job.service) %}
{% block page_content %} {% block page_content %}
<div class="col s12 m4"> <div class="col s12 m4">
<h3 id="title">{{ job.title }}</h3> <h3 id="title">{{ job.title }}</h3>
<p id="description">{{ job.description }}</p> <p id="description">{{ job.description }}</p>
<a class="btn status waves-effect waves-light" id="status"></a> <div class="active preloader-wrapper small" id="progress-indicator">
<div class="spinner-layer spinner-blue-only">
<div class="circle-clipper left">
<div class="circle"></div>
</div>
<div class="gap-patch">
<div class="circle"></div>
</div>
<div class="circle-clipper right">
<div class="circle"></div>
</div>
</div>
</div>
<span class="chip status white-text" id="status"></span>
</div> </div>
<div class="col s12 m8"> <div class="col s12 m8">
<div class="card"> <ul class="collapsible expandable" id="job-list">
<div class="card-content"> <li class="active">
<span class="card-title">Chronometrics</span> <div class="collapsible-header">
<div class="row"> <i class="material-icons">input</i>Input files
<div class="col s12 m6">
<div class="input-field">
<input disabled value="{{ job.creation_date.strftime('%m/%d/%Y, %H:%M:%S %p') }}" id="creation-date" type="text" class="validate">
<label for="creation-date">Creation date</label>
</div> </div>
</div> <div class="collapsible-body">
<div class="col s12 m6">
<div class="input-field">
<input disabled value="" id="end-date" type="text" class="validate">
<label for="end-date">End date</label>
</div>
</div>
</div>
<span class="card-title">Ressource allocations</span>
<div class="row">
<div class="col s12 m6">
<div class="input-field">
<input disabled value="{{ job.mem_mb }}" id="mem-mb" type="text" class="validate">
<label for="mem-mb">Memory</label>
</div>
</div>
<div class="col s12 m6">
<div class="input-field">
<input disabled value="{{ job.n_cores }}" id="n-cores" type="text" class="validate">
<label for="n-cores">CPU cores</label>
</div>
</div>
</div>
<span class="card-title">Service informations</span>
<div class="row">
<div class="col s12 m4">
<div class="input-field">
<input disabled value="{{ job.service }}" id="service" type="text" class="validate">
<label for="service">Service</label>
</div>
</div>
<div class="col s12 m4">
<div class="input-field">
<input disabled value="{{ job.service_args|e }}" id="service-args" type="text" class="validate">
<label for="service-args">Service arguments</label>
</div>
</div>
<div class="col s12 m4">
<div class="input-field">
<input disabled value="{{ job.service_version }}" id="service-version" type="text" class="validate">
<label for="service-version">Service version</label>
</div>
</div>
</div>
</div>
<div class="card-action right-align">
<a href="#" class="waves-effect waves-light btn"><i class="material-icons left">settings</i>Export Parameters</a>
<a data-target="delete-job-modal" class="waves-effect waves-light btn red modal-trigger"><i class="material-icons left">delete</i>Delete Job</a>
</div>
</div>
</div>
<div class="col s12">
<h4>Input and result files</h4>
</div>
<div class="col s12 m7">
<div class="card">
<div class="card-content">
<span class="card-title">Filewise</span>
<table class="highlight responsive-table"> <table class="highlight responsive-table">
<thead> <thead>
<tr> <tr>
@ -104,13 +53,10 @@
</tbody> </tbody>
</table> </table>
</div> </div>
</div> </li>
</div> <li>
<div class="collapsible-header"><i class="material-icons">done</i>Result files</div>
<div class="col s12 m5"> <div class="collapsible-body">
<div class="card">
<div class="card-content">
<span class="card-title">Bundled</span>
<table class="highlight responsive-table"> <table class="highlight responsive-table">
<thead> <thead>
<tr> <tr>
@ -121,37 +67,66 @@
<tbody id="results"></tbody> <tbody id="results"></tbody>
</table> </table>
</div> </div>
</li>
<li>
<div class="collapsible-header"><i class="material-icons">settings</i>Settings</div>
<div class="collapsible-body">
<div class="row">
<div class="col s12">Chronometrics</div>
<div class="col s12 m6">
<div class="input-field">
<input disabled value="{{ job.creation_date.strftime('%m/%d/%Y, %H:%M:%S %p') }}" id="creation-date" type="text" class="validate">
<label for="creation-date">Creation date</label>
</div>
</div>
<div class="col s12 m6">
<div class="input-field">
<input disabled value="" id="end-date" type="text" class="validate">
<label for="end-date">End date</label>
</div> </div>
</div> </div>
<div class="col s12">Ressource allocations</div>
<div class="col s12 m6">
<div class="input-field">
<input disabled value="{{ job.mem_mb }}" id="mem-mb" type="text" class="validate">
<label for="mem-mb">Memory</label>
</div>
</div>
<div class="col s12 m6">
<div class="input-field">
<input disabled value="{{ job.n_cores }}" id="n-cores" type="text" class="validate">
<label for="n-cores">CPU cores</label>
</div>
</div>
<div class="col s12 hide"> <div class="col s12">Service informations</div>
<div class="card"> <div class="col s12 m4">
<div class="card-content"> <div class="input-field">
<span class="card-title">Files</span> <input disabled value="{{ job.service }}" id="service" type="text" class="validate">
<table class="highlight responsive-table"> <label for="service">Service</label>
<thead>
<tr>
<th>Filename</th>
<th>Download</th>
</tr>
</thead>
<tbody>
{% for input in job.inputs %}
<tr>
<td id="input-{{ input.id }}-filename">{{ input.filename }}</td>
<td id="input-{{ input.id }}-download">
<a class="waves-effect waves-light btn-small" download href="{{ url_for('jobs.download_job_input', job_id=job.id, job_input_id=input.id) }}">
<i class="material-icons">file_download</i>
</a>
</td>
<td id="input-{{ input.id }}-results"></td>
</tr>
{% endfor %}
</tbody>
</table>
</div> </div>
</div> </div>
<div class="col s12 m4">
<div class="input-field">
<input disabled value="{{ job.service_args|e }}" id="service-args" type="text" class="validate">
<label for="service-args">Service arguments</label>
</div>
</div>
<div class="col s12 m4">
<div class="input-field">
<input disabled value="{{ job.service_version }}" id="service-version" type="text" class="validate">
<label for="service-version">Service version</label>
</div>
</div>
<div class="col s12 right-align">
<a href="#" class="waves-effect waves-light btn"><i class="material-icons left">settings</i>Export Parameters</a>
<a data-target="delete-job-modal" class="waves-effect waves-light btn red modal-trigger"><i class="material-icons left">delete</i>Delete Job</a>
</div>
</div>
</div>
</li>
</ul>
</div> </div>
@ -246,21 +221,27 @@
resultsElement = document.getElementById("results"); resultsElement = document.getElementById("results");
for (let result of results) { for (let result of results) {
resultsElement.insertAdjacentHTML("beforeend", ` resultsElement.insertAdjacentHTML(
<tr> "beforeend",
`<tr>
<td>${result.filename}</td> <td>${result.filename}</td>
<td> <td>
<a class="btn-small waves-effect waves-light" download href="/jobs/${result.job_id}/results/${result.id}/download"> <a class="btn-small waves-effect waves-light" download href="/jobs/${result.job_id}/results/${result.id}/download">
<i class="material-icons">file_download</i> <i class="material-icons">file_download</i>
</a> </a>
</td> </td>
</tr>`); </tr>`
);
} }
} }
setStatus(status) { setStatus(status) {
let statusElement; let progressIndicator, statusElement;
if (status === "complete") {
progressIndicator = document.getElementById("progress-indicator");
progressIndicator.classList.add("hide");
M.Collapsible.getInstance(document.getElementById("job-list")).open(1);
}
statusElement = document.getElementById("status"); statusElement = document.getElementById("status");
statusElement.dataset.status = status; statusElement.dataset.status = status;
} }

13
app/templates/main/dashboard.html.j2
View File

@ -4,11 +4,11 @@
{% set roadmap = False %} {% set roadmap = False %}
{% block page_content %} {% block page_content %}
<div class="col s12"> <div class="col s12" id="corpora">
<h3>My Corpora</h3> <h3>My Corpora</h3>
<p>Lorem ipsum dolor sit amet, consetetur sadipscing elitr, sed diam nonumy eirmod tempor invidunt ut labore et dolore magna aliquyam erat, sed diam voluptua. At vero eos et accusam et justo duo dolores et ea rebum. Stet clita kasd gubergren,</p> <p>Lorem ipsum dolor sit amet, consetetur sadipscing elitr, sed diam nonumy eirmod tempor invidunt ut labore et dolore magna aliquyam erat, sed diam voluptua. At vero eos et accusam et justo duo dolores et ea rebum. Stet clita kasd gubergren,</p>
<div class="card"> <div class="card">
<div class="card-content" id="corpora"> <div class="card-content">
<div class="input-field"> <div class="input-field">
<i class="material-icons prefix">search</i> <i class="material-icons prefix">search</i>
<input id="search-corpus" class="search" type="search"></input> <input id="search-corpus" class="search" type="search"></input>
@ -36,11 +36,11 @@
</div> </div>
</div> </div>
<div class="col s12"> <div class="col s12" id="jobs">
<h3>My Jobs</h3> <h3>My Jobs</h3>
<p>Lorem ipsum dolor sit amet, consetetur sadipscing elitr, sed diam nonumy eirmod tempor invidunt ut labore et dolore magna aliquyam erat, sed diam voluptua. At vero eos et accusam et justo duo dolores et ea rebum. Stet clita kasd gubergren,</p> <p>Lorem ipsum dolor sit amet, consetetur sadipscing elitr, sed diam nonumy eirmod tempor invidunt ut labore et dolore magna aliquyam erat, sed diam voluptua. At vero eos et accusam et justo duo dolores et ea rebum. Stet clita kasd gubergren,</p>
<div class="card"> <div class="card">
<div class="card-content" id="jobs"> <div class="card-content">
<div class="input-field"> <div class="input-field">
<i class="material-icons prefix">search</i> <i class="material-icons prefix">search</i>
<input id="search-job" class="search" type="search"></input> <input id="search-job" class="search" type="search"></input>
@ -89,7 +89,8 @@
</div> </div>
<script> <script>
var corpusList = new RessourceList("corpora", nopaque.corporaSubscribers, RessourceList.dataMapper.corpus); var corpusList = new RessourceList("corpora", nopaque.corporaSubscribers,
var jobList = new RessourceList("jobs", nopaque.jobsSubscribers, RessourceList.dataMapper.job); "corpus");
var jobList = new RessourceList("jobs", nopaque.jobsSubscribers, "job");
</script> </script>
{% endblock %} {% endblock %}

12
app/templates/main/poster.html.j2
View File

@ -157,10 +157,9 @@
<div class="card-content"> <div class="card-content">
<span class="card-title"><i class="material-icons left">search</i>Corpus Analysis</span> <span class="card-title"><i class="material-icons left">search</i>Corpus Analysis</span>
<p> <p>
Mittels CQP Query Language als Abfragesprache können komplexe Mittels CQP Query Language können komplexe Suchanfragen unter
Suchanfragen unter Zu­hil­fe­nah­me von Metadaten und NLP-Auszeichnungen Zuhilfenahme von Metadaten und NLP-Auszeichnungen an eigens erstellte
ausgeführt werden. Ergebnisse können als Text oder in abstrakter Korpora gestellt werden. Ergebnisse können als Text oder in abstrakter Darstellung ausgewertet werden.
Darstellung ausgewertet werden.
</p> </p>
<div class="input-field"> <div class="input-field">
<i class="material-icons prefix">search</i> <i class="material-icons prefix">search</i>
@ -182,7 +181,10 @@
<div class="card"> <div class="card">
<div class="card-content"> <div class="card-content">
<span class="card-title">Ausgabe</span> <span class="card-title">Ausgabe</span>
<p>Export der Ergebnisse in CSV, Excel, JSON und HTML.</p> <p>
Export der Ergebnisse in JSON. (Zunkünftig angedacht: CSV, Excel und
HTML)
</p>
</div> </div>
</div> </div>
<p> <p>

14
app/templates/nopaque.html.j2
View File

@ -60,8 +60,8 @@
<li><a href="{{ url_for('main.index') }}"><i class="material-icons">opacity</i>nopaque</a></li> <li><a href="{{ url_for('main.index') }}"><i class="material-icons">opacity</i>nopaque</a></li>
<li><a href="#"><i class="material-icons">linear_scale</i>Workflow</a></li> <li><a href="#"><i class="material-icons">linear_scale</i>Workflow</a></li>
<li><a href="{{ url_for('main.dashboard') }}"><i class="material-icons">dashboard</i>Dashboard</a></li> <li><a href="{{ url_for('main.dashboard') }}"><i class="material-icons">dashboard</i>Dashboard</a></li>
<li><a href="{{ url_for('main.dashboard', _anchor='my_anchor') }}" style="padding-left: 47px;"><i class="material-icons">book</i>Corpora</a></li> <li><a href="{{ url_for('main.dashboard', _anchor='corpora') }}" style="padding-left: 47px;"><i class="material-icons">book</i>Corpora</a></li>
<li><a href="{{ url_for('main.dashboard', _anchor='my_anchor') }}" style="padding-left: 47px;"><i class="material-icons">work</i>Jobs</a></li> <li><a href="{{ url_for('main.dashboard', _anchor='jobs') }}" style="padding-left: 47px;"><i class="material-icons">work</i>Jobs</a></li>
<li><div class="divider"></div></li> <li><div class="divider"></div></li>
<li><a class="subheader">Processes and Services</a></li> <li><a class="subheader">Processes and Services</a></li>
<li><a href="{{ url_for('services.service', service='corpus_analysis') }}"><i class="material-icons">search</i>Corpus Analysis</a></li> <li><a href="{{ url_for('services.service', service='corpus_analysis') }}"><i class="material-icons">search</i>Corpus Analysis</a></li>
@ -94,7 +94,15 @@
{% endif %} {% endif %}
<div class="row"> <div class="row">
<div class="col s12"> <div class="col s12">
<h2>{% if title %}{{ title }}{% else %}Unnamed page{% endif %}</h2> <h2>
{% if title_override %}
{{ title_override }}
{% elif title %}
{{ title }}
{% else %}
Unnamed page
{% endif %}
</h2>
</div> </div>
{% if roadmap %} {% if roadmap %}
<div class="col s12 m9 l10"> <div class="col s12 m9 l10">

3
app/templates/services/corpus_analysis.html.j2
View File

@ -38,7 +38,6 @@
<script> <script>
var corpusList = new RessourceList("corpora", nopaque.corporaSubscribers, var corpusList = new RessourceList("corpora", nopaque.corporaSubscribers,
RessourceList.dataMapper.corpus, "corpus", {page: 10});
{page: 10});
</script> </script>
{% endblock %} {% endblock %}