Merge branch 'development' of gitlab.ub.uni-bielefeld.de:sfb1288inf/opaque into development

app/main/views.py

@@ -5,7 +5,7 @@ from flask_login import current_user, login_required
 from . import main
 from .forms import CreateCorpusForm, QueryForm
 from .. import db
-from ..models import Corpus, CorpusFile, Job
+from ..models import Corpus, CorpusFile, Job, JobInput, JobResult
 from werkzeug.utils import secure_filename
 import os
 import threading
@@ -20,47 +20,30 @@ def index():
 @main.route('/corpora/<int:corpus_id>')
 @login_required
 def corpus(corpus_id):
-    if (current_user.is_administrator()):
     corpus = Corpus.query.get_or_404(corpus_id)
-    else:
+    if not (corpus.creator == current_user
-        corpus = current_user.corpora.filter_by(id=corpus_id).first()
+            or current_user.is_administrator()):
-    if not corpus:
+        abort(403)
-        print('Corpus not found.')
-        abort(404)
-
-    dir = os.path.join(current_app.config['OPAQUE_STORAGE_DIRECTORY'],
-                       str(corpus.user_id),
-                       'corpora',
-                       str(corpus.id))
-    files = {}
-    for file in sorted(os.listdir(dir)):
-        files[file] = {}
-        files[file]['path'] = os.path.join(file)
-
     return render_template('main/corpora/corpus.html.j2',
                            corpus=corpus,
-                           files=files,
+                           title='Corpus')
-                           title='Corpus: ' + corpus.title)
 
 
 @main.route('/corpora/<int:corpus_id>/download')
 @login_required
 def corpus_download(corpus_id):
-    file = request.args.get('file')
+    corpus_file_id = request.args.get('corpus_file_id')
-    if (current_user.is_administrator()):
+    corpus_file = CorpusFile.query.get_or_404(corpus_file_id)
-        corpus = Corpus.query.get_or_404(corpus_id)
+    if not corpus_file.corpus_id == corpus_id:
-    else:
-        corpus = current_user.corpora.filter_by(id=corpus_id).first()
-    if not file or not corpus:
-        print('File not found.')
         abort(404)
+    if not (corpus_file.corpus.creator == current_user
+            or current_user.is_administrator()):
+        abort(403)
     dir = os.path.join(current_app.config['OPAQUE_STORAGE_DIRECTORY'],
-                       str(corpus.user_id),
+                       corpus_file.dir)
-                       'corpora',
-                       str(corpus.id))
     return send_from_directory(as_attachment=True,
                                directory=dir,
-                               filename=file)
+                               filename=corpus_file.filename)
 
 
 @main.route('/corpora/<int:corpus_id>/analysis', methods=['GET', 'POST'])
@@ -83,20 +66,16 @@ def corpus_analysis(corpus_id):
 @login_required
 def dashboard():
     create_corpus_form = CreateCorpusForm()
-
     if create_corpus_form.validate_on_submit():
-        app = current_app._get_current_object()
         corpus = Corpus(creator=current_user._get_current_object(),
                         description=create_corpus_form.description.data,
                         title=create_corpus_form.title.data)
         db.session.add(corpus)
         db.session.commit()
-
+        dir = os.path.join(current_app.config['OPAQUE_STORAGE_DIRECTORY'],
-        dir = os.path.join(app.config['OPAQUE_STORAGE_DIRECTORY'],
                            str(corpus.user_id),
                            'corpora',
                            str(corpus.id))
-
         try:
             os.makedirs(dir)
         except OSError:
@@ -115,7 +94,6 @@ def dashboard():
             db.session.commit()
             flash('Corpus created!')
         return redirect(url_for('main.dashboard'))
-
     return render_template('main/dashboard.html.j2',
                            create_corpus_form=create_corpus_form,
                            title='Dashboard')
@@ -124,58 +102,33 @@ def dashboard():
 @main.route('/jobs/<int:job_id>')
 @login_required
 def job(job_id):
-    if (current_user.is_administrator()):
     job = Job.query.get_or_404(job_id)
-    else:
+    if not (job.creator == current_user or current_user.is_administrator()):
-        job = current_user.jobs.filter_by(id=job_id).first()
+        abort(403)
-    if not job:
+    return render_template('main/jobs/job.html.j2', job=job, title='Job')
-        print('Job not found.')
-        abort(404)
-
-    dir = os.path.join(current_app.config['OPAQUE_STORAGE_DIRECTORY'],
-                       str(job.user_id),
-                       'jobs',
-                       str(job.id))
-    files = {}
-    for file in sorted(os.listdir(dir)):
-        if file == 'output':
-            continue
-        files[file] = {}
-        files[file]['path'] = os.path.join(file)
-        if job.status == 'complete':
-            files[file]['results'] = {}
-            results_dir = os.path.join(dir, 'output', file)
-            for result in sorted(os.listdir(results_dir)):
-                result_type = result.rsplit(".", 1)[1]
-                files[file]['results'][result_type] = {}
-                files[file]['results'][result_type]['path'] = os.path.join(
-                    'output', files[file]['path'], result
-                )
-
-    return render_template('main/jobs/job.html.j2',
-                           files=files,
-                           job=job,
-                           title='Job')
 
 
 @main.route('/jobs/<int:job_id>/download')
 @login_required
 def job_download(job_id):
-    file = request.args.get('file')
+    ressource_id = request.args.get('ressource_id')
-    if (current_user.is_administrator()):
+    ressource_type = request.args.get('ressource_type')
-        job = Job.query.get_or_404(job_id)
+    if ressource_type == 'input':
+        ressource = JobInput.query.get_or_404(ressource_id)
+    elif ressource_type == 'result':
+        ressource = JobResult.query.get_or_404(ressource_id)
     else:
-        job = current_user.jobs.filter_by(id=job_id).first()
+        abort(400)
-    if not file or not job:
+    if not ressource.job_id == job_id:
-        print('File not found.')
         abort(404)
+    if not (ressource.job.creator == current_user
+            or current_user.is_administrator()):
+        abort(403)
     dir = os.path.join(current_app.config['OPAQUE_STORAGE_DIRECTORY'],
-                       str(job.user_id),
+                       ressource.dir)
-                       'jobs',
-                       str(job.id))
     return send_from_directory(as_attachment=True,
                                directory=dir,
-                               filename=file)
+                               filename=ressource.filename)
 
 
 @main.route('/jobs/<int:job_id>/delete')

app/templates/main/corpora/corpus.html.j2

@@ -1,85 +1,9 @@
 {% extends "limited_width.html.j2" %}
 
 {% block page_content %}
-<script>
-  var corpus_user_id = {{ corpus.user_id|tojson|safe }}
-  socket.emit('inspect_user', {{ corpus_user_id }});
-</script>
-<script>
-  var CORPUS_ID = {{ corpus.id|tojson|safe }}
-  var foreignCorpusFlag;
-  {% if current_user.id == corpus.user_id %}
-    foreignCorpusFlag = false;
-  {% else %}
-    foreignCorpusFlag = true;
-  {% endif %}
-
-  class InformationUpdater {
-    constructor(corpusId) {
-      this.corpusId = corpusId;
-      if (foreignCorpusFlag) {
-        foreignCorpusSubscribers.push(this);
-      } else {
-        corporaSubscribers.push(this);
-      }
-    }
-
-    _init() {
-      var creationDateElement, descriptionElement, titleElement;
-
-      if (foreignCorpusFlag) {
-        this.corpus = foreignCorpora[this.corpusId];
-      } else {
-        this.corpus = corpora[this.corpusId];
-      }
-      creationDateElement = document.getElementById("creation-date");
-      creationDateElement.value = (new Date(this.corpus.creation_date * 1000)).toLocaleString();
-      descriptionElement = document.getElementById("description");
-      descriptionElement.innerHTML = this.corpus.description;
-      titleElement = document.getElementById("title");
-      titleElement.innerHTML = this.corpus.title;
-
-      M.updateTextFields();
-    }
-
-    _update(patch) {
-      var newStatusColor, operation, pathArray, status, statusColor,
-          updatedElement;
-
-      for (operation of patch) {
-        /* "/corpusId/valueName" -> ["corpusId", "valueName"] */
-        pathArray = operation.path.split("/").slice(1);
-        if (pathArray[0] != this.jobId) {continue;}
-        switch(operation.op) {
-          case "delete":
-            location.reload();
-            break;
-          case "replace":
-            switch(pathArray[1]) {
-              case "description":
-                updatedElement = document.getElementById("description");
-                updatedElement.innerHTML = operation.value;
-                break;
-              case "title":
-                updatedElement = document.getElementById("title");
-                updatedElement.innerHTML = operation.value;
-                break;
-              default:
-                break;
-            }
-            break;
-          default:
-            break;
-        }
-      }
-    }
-  }
-  var informationUpdater = new InformationUpdater(CORPUS_ID);
-</script>
-
 <div class="col s12 m4">
-  <h3 id="title"></h3>
+  <h3 id="title">{{ corpus.title }}</h3>
-  <p id="description"></p>
+  <p id="description">{{ corpus.description }}</p>
   <h2>Actions:</h2>
   <!-- Confirm deletion of job with modal dialogue
   Modal Trigger-->
@@ -106,29 +30,33 @@
       <div class="row">
         <div class="col s12 m6">
           <div class="input-field">
-            <input disabled value="" id="creation-date" type="text" class="validate">
+            <input disabled value="{{ corpus.creation_date.strftime('%m/%d/%Y, %H:%M:%S %p') }}" id="creation-date" type="text" class="validate">
             <label for="creation-date">Creation date</label>
           </div>
         </div>
       </div>
       <span class="card-title">Files</span>
-      <table>
+      <table class="highlight responsive-table">
         <thead>
           <tr>
-            <th style="width: 50%;">Inputs</th>
+            <th>Filename</th>
+            <th>Download</th>
           </tr>
         </thead>
         <tbody>
-          {% for file in files %}
+          {% for file in corpus.files %}
           <tr>
-            <td>
+            <td id="file-{{ file.id }}-filename">{{ file.filename }}</td>
-              <a href="{{ url_for('main.corpus_download', corpus_id=corpus.id, file=files[file]['path']) }}" class="waves-effect waves-light btn-small"><i class="material-icons left">file_download</i>{{ file }}</a>
+            <td id="file-{{ file.id }}-download">
+              <a class="waves-effect waves-light btn-small" download href="{{ url_for('main.corpus_download', corpus_id=corpus.id, corpus_file_id=file.id) }}">
+                <i class="material-icons">file_download</i>
+              </a>
             </td>
           </tr>
           {% endfor %}
         </tbody>
       </table>
     </div>
+  </div>
 </div>
-
 {% endblock %}

app/templates/main/jobs/job.html.j2

@@ -2,11 +2,11 @@
 
 {% block page_content %}
 <script>
-  var job_user_id = {{ job.user_id|tojson|safe }}
+  var job_user_id = {{ job.user_id }}
   socket.emit('inspect_user', job_user_id);
 </script>
 <script>
-  var JOB_ID = {{ job.id|tojson|safe }}
+  var JOB_ID = {{ job.id }}
   var foreignJobFlag;
   {% if current_user.id == job.user_id %}
   foreignJobFlag = false;
@@ -25,111 +25,45 @@
     }
 
     _init() {
-      var creationDateElement, descriptionElement, endDateElement,
-          memMbElement, nCoresElement, serviceElement, serviceArgsElement,
-          serviceVersionElement, statusColor, statusElement, titleElement;
-
       if (foreignJobFlag) {
         this.job = foreignJobs[this.jobId];
       } else {
         this.job = jobs[this.jobId];
       }
 
-      // Title
-      this.setTitle(this.job.title);
-      // Description
-      this.setDescription(this.job.description);
       // Status
       this.setStatus(this.job.status);
-      // Creation date
-      this.setCreationDate(this.job.creation_date);
       // End date
       if (this.job.end_date) {this.setEndDate(this.job.end_date);}
-      // Memory
+      // Input results
-      this.setMemMb(this.job.mem_mb);
+      for (let input of this.job.inputs) {
-      // CPU cores
+        for (let result of input.results) {
-      this.setNCores(this.job.n_cores);
+          this.setResult(result);
-      // Service
-      this.setService(this.job.service);
-      // Service arguments
-      this.setServiceArgs(this.job.service_args);
-      // Service version
-      this.setServiceVersion(this.job.service_version);
-
-      var filesElement, input, inputDownloadElement, inputElement,
-          inputFilenameElement, inputResultsElement;
-
-      filesElement = document.getElementById("files");
-      for (input of this.job.inputs) {
-        // Data row
-        inputElement = document.createElement("tr");
-        filesElement.append(inputElement);
-        // Input filename
-        inputFilenameElement = document.createElement("td");
-        inputFilenameElement.id = `input-${input.id}-filename`;
-        inputElement.append(inputFilenameElement);
-        // Input download
-        inputDownloadElement = document.createElement("td");
-        inputDownloadElement.id = `input-${input.id}-download`;
-        inputElement.append(inputDownloadElement);
-        // Third column for input result file download buttons
-        inputResultsElement = document.createElement("td");
-        inputResultsElement.id = `input-${input.id}-results`;
-        inputElement.append(inputResultsElement);
-        this.setInputFilename(input);
-        this.setInputDownload(input);
-        this.setInputResults(input);
         }
       }
-
-    setInputDownload(input) {
-      var inputDownloadButtonElement, inputDownloadButtonIconElement,
-          inputDownloadElement;
-      inputDownloadElement = document.getElementById(`input-${input.id}-download`);
-      inputDownloadButtonElement = document.createElement("a");
-      inputDownloadButtonElement.classList.add("waves-effect", "waves-light", "btn-small");
-      inputDownloadButtonElement.href = `${this.jobId}/download?file=${input.filename}`;
-      inputDownloadButtonElement.setAttribute("download", "");
-      inputDownloadButtonIconElement = document.createElement("i");
-      inputDownloadButtonIconElement.classList.add("material-icons");
-      inputDownloadButtonIconElement.innerText = "file_download";
-      inputDownloadButtonElement.append(inputDownloadButtonIconElement);
-      inputDownloadElement.append(inputDownloadButtonElement);
-    }
-
-    setInputFilename(input) {
-      var inputFilenameElement;
-      inputFilenameElement = document.getElementById(`input-${input.id}-filename`);
-      inputFilenameElement.innerText = input.filename;
     }
 
     _update(patch) {
-      var input, operation, pathArray;
+      var pathArray;
 
-      for (operation of patch) {
+      for (let operation of patch) {
         /* "/jobId/valueName" -> ["jobId", "valueName"] */
         pathArray = operation.path.split("/").slice(1);
         if (pathArray[0] != this.jobId) {continue;}
         switch(operation.op) {
           case "add":
             if (pathArray[1] === "inputs" && pathArray[3] === "results") {
-              console.log(operation.value);
+              this.setResult(operation.value);
-              this.setInputResult(operation.value);
             }
             break;
           case "delete":
             location.reload();
             break;
           case "replace":
-            switch(pathArray[1]) {
+            if (pathArray[1] === "end_date") {
-              case "end_date":
               this.setEndDate(operation.value);
-                break;
+            } else if (pathArray[1] === "status") {
-              case "status":
               this.setStatus(operation.value);
-                break;
-              default:
-                break;
             }
             break;
           default:
@@ -138,109 +72,43 @@
       }
     }
 
-    setTitle(title) {
-      var titleElement;
-      titleElement = document.getElementById("title");
-      titleElement.innerText = title;
-    }
-
-    setDescription(description) {
-      var descriptionElement;
-      descriptionElement = document.getElementById("description");
-      descriptionElement.innerText = description;
-    }
-
-    setStatus(status) {
-      var statusColor, statusElement;
-      statusElement = document.getElementById("status");
-      for (statusColor of Object.values(JobList.STATUS_COLORS)) {
-        statusElement.classList.remove(statusColor);
-      }
-      statusElement.classList.add(JobList.STATUS_COLORS[status] || JobList.STATUS_COLORS['default']);
-      statusElement.innerText = status;
-    }
-
-    setCreationDate(timestamp) {
-      var creationDateElement;
-      creationDateElement = document.getElementById("creation-date");
-      creationDateElement.value = new Date(timestamp * 1000).toLocaleString();
-      M.updateTextFields();
-    }
-
     setEndDate(timestamp) {
-      var endDateElement;
+      document.getElementById("end-date").value = new Date(timestamp * 1000).toLocaleString();
-      endDateElement = document.getElementById("end-date");
-      endDateElement.value = new Date(timestamp * 1000).toLocaleString();
       M.updateTextFields();
     }
 
-    setMemMb(memMb) {
+    setResult(result) {
-      var memMbElement;
+      var resultsElement, resultDownloadButtonElement,
-      memMbElement = document.getElementById("mem-mb");
-      memMbElement.value = memMb;
-      M.updateTextFields();
-    }
-
-    setNCores(nCores) {
-      var nCoresElement;
-      nCoresElement = document.getElementById("n-cores");
-      nCoresElement.value = nCores;
-      M.updateTextFields();
-    }
-
-    setService(service) {
-      var serviceElement;
-      serviceElement = document.getElementById("service");
-      serviceElement.value = service;
-      M.updateTextFields();
-    }
-
-    setServiceArgs(serviceArgs) {
-      var serviceArgsElement;
-      serviceArgsElement = document.getElementById("service-args");
-      serviceArgsElement.value = serviceArgs;
-      M.updateTextFields();
-    }
-
-    setServiceVersion(serviceVersion) {
-      var serviceVersionElement;
-      serviceVersionElement = document.getElementById("service-version");
-      serviceVersionElement.value = serviceVersion;
-      M.updateTextFields();
-    }
-
-    setInputResults(input) {
-      var result;
-      for (result of input.results) {
-        this.setInputResult(result);
-      }
-    }
-
-    setInputResult(result) {
-      var inputResultsElement, resultDownloadButtonElement,
           resultDownloadButtonIconElement;
-      inputResultsElement = document.getElementById(`input-${result.job_input_id}-results`);
+      resultsElement = document.getElementById(`input-${result.job_input_id}-results`);
       resultDownloadButtonElement = document.createElement("a");
       resultDownloadButtonElement.classList.add("waves-effect", "waves-light", "btn-small");
-      var resultFile = `${result.dir}/${result.filename}`;
+      resultDownloadButtonElement.href = `/jobs/${this.jobId}/download?ressource_id=${result.id}&ressource_type=result`;
-      resultFile = resultFile.substring(resultFile.indexOf("output/"));
-      resultDownloadButtonElement.href = `${this.jobId}/download?file=${resultFile}`;
       resultDownloadButtonElement.innerText = result.filename.split(".").reverse()[0];
       resultDownloadButtonElement.setAttribute("download", "");
       resultDownloadButtonIconElement = document.createElement("i");
       resultDownloadButtonIconElement.classList.add("material-icons", "left");
       resultDownloadButtonIconElement.innerText = "file_download";
       resultDownloadButtonElement.prepend(resultDownloadButtonIconElement);
-      inputResultsElement.append(resultDownloadButtonElement);
+      resultsElement.append(resultDownloadButtonElement);
-      inputResultsElement.append(" ");
+      resultsElement.append(" ");
+    }
+
+    setStatus(status) {
+      var statusElement;
+      statusElement = document.getElementById("status");
+      statusElement.classList.remove(...Object.values(JobList.STATUS_COLORS));
+      statusElement.classList.add(JobList.STATUS_COLORS[status] || JobList.STATUS_COLORS['default']);
+      statusElement.innerText = status;
     }
   }
+
   var informationUpdater = new InformationUpdater(JOB_ID);
 </script>
 
 <div class="col s12 m4">
-  <h3 id="title"></h3>
+  <h3 id="title">{{ job.title }}</h3>
-  <p id="description"></p>
+  <p id="description">{{ job.description }}</p>
   <a class="waves-effect waves-light btn" id="status"></a>
   <h2>Actions:</h2>
   <!-- Confirm deletion of job with modal dialogue
@@ -269,7 +137,7 @@
       <div class="row">
         <div class="col s12 m6">
           <div class="input-field">
-            <input disabled value="" id="creation-date" type="text" class="validate">
+            <input disabled value="{{ job.creation_date.strftime('%m/%d/%Y, %H:%M:%S %p') }}" id="creation-date" type="text" class="validate">
             <label for="creation-date">Creation date</label>
           </div>
         </div>
@@ -285,13 +153,13 @@
       <div class="row">
         <div class="col s12 m6">
           <div class="input-field">
-            <input disabled value="" id="mem-mb" type="text" class="validate">
+            <input disabled value="{{ job.mem_mb }}" id="mem-mb" type="text" class="validate">
             <label for="mem-mb">Memory</label>
           </div>
         </div>
         <div class="col s12 m6">
           <div class="input-field">
-            <input disabled value="" id="n-cores" type="text" class="validate">
+            <input disabled value="{{ job.n_cores }}" id="n-cores" type="text" class="validate">
             <label for="n-cores">CPU cores</label>
           </div>
         </div>
@@ -301,19 +169,19 @@
       <div class="row">
         <div class="col s12 m4">
           <div class="input-field">
-            <input disabled value="" id="service" type="text" class="validate">
+            <input disabled value="{{ job.service }}" id="service" type="text" class="validate">
             <label for="service">Service</label>
           </div>
         </div>
         <div class="col s12 m4">
           <div class="input-field">
-            <input disabled value="" id="service-args" type="text" class="validate">
+            <input disabled value="{{ job.service_args|e }}" id="service-args" type="text" class="validate">
             <label for="service-args">Service arguments</label>
           </div>
         </div>
         <div class="col s12 m4">
           <div class="input-field">
-            <input disabled value="" id="service-version" type="text" class="validate">
+            <input disabled value="{{ job.service_version }}" id="service-version" type="text" class="validate">
             <label for="service-version">Service version</label>
           </div>
         </div>
@@ -329,12 +197,24 @@
       <table class="highlight responsive-table">
         <thead>
           <tr>
-            <th>File</th>
+            <th>Filename</th>
-            <th>Input</th>
+            <th>Download</th>
             <th>Results</th>
           </tr>
         </thead>
-        <tbody id="files"></tbody>
+        <tbody>
+          {% for input in job.inputs %}
+          <tr>
+            <td id="input-{{ input.id }}-filename">{{ input.filename }}</td>
+            <td id="input-{{ input.id }}-download">
+              <a class="waves-effect waves-light btn-small" download href="{{ url_for('main.job_download', job_id=job.id, ressource_id=input.id, ressource_type='input') }}">
+                <i class="material-icons">file_download</i>
+              </a>
+            </td>
+            <td id="input-{{ input.id }}-results"></td>
+          </tr>
+          {% endfor %}
+        </tbody>
       </table>
     </div>
   </div>

docker-entrypoint.sh

@@ -1,6 +1,10 @@
 #!/bin/sh
 
-./wait-for-it/wait-for-it.sh db:5432 --strict --timeout=0
+echo "Waiting for db..."
+wait-for-it/wait-for-it.sh db:5432 --strict --timeout=0
+echo "Waiting for redis..."
+wait-for-it/wait-for-it.sh redis:6379 --strict --timeout=0
+
 if [ $# -eq 0 ]
 then
     venv/bin/python -u opaque.py