Results import fixes and additions. Table creation rework.

This commit is contained in:
Stephan Porada 2020-07-07 15:08:15 +02:00
parent 5867871de2
commit 7648149584
13 changed files with 296 additions and 167 deletions

View File

@ -547,6 +547,16 @@ class Result (db.Model):
file = db.relationship('ResultFile', backref='result', lazy='dynamic',
cascade='save-update, merge, delete')
def delete(self):
db.session.delete(self)
db.session.commit()
def __repr__(self):
'''
String representation of the Result. For human readability.
'''
return '<Result ID: {result_id}>'.format(result_id=self.id)
class ResultFile(db.Model):
'''
@ -561,6 +571,16 @@ class ResultFile(db.Model):
filename = db.Column(db.String(255))
dir = db.Column(db.String(255))
def delete(self):
db.session.delete(self)
db.session.commit()
def __repr__(self):
'''
String representation of the ResultFile. For human readability.
'''
return '<ResultFile {result_file_name}>'.format(result_file_name=self.filename) # noqa
'''
' Flask-Login is told to use the applications custom anonymous user by setting

View File

@ -0,0 +1,18 @@
from flask_wtf import FlaskForm
from werkzeug.utils import secure_filename
from wtforms import FileField, SubmitField, ValidationError
from wtforms.validators import DataRequired
class ImportResultsForm(FlaskForm):
'''
Form used to import one result json file.
'''
file = FileField('File', validators=[DataRequired()])
submit = SubmitField()
def validate_file(self, field):
if not field.data.filename.lower().endswith('.json'):
raise ValidationError('File does not have an approved extension: '
'.json')
field.data.filename = secure_filename(field.data.filename)

17
web/app/results/tasks.py Normal file
View File

@ -0,0 +1,17 @@
from ..decorators import background
from ..models import Result
import os
import shutil
@background
def delete_result(result_id, *args, **kwargs):
app = kwargs['app']
with app.app_context():
result = Result.query.get(result_id)
if result is None:
return
result_file_path = os.path.join(app.config['NOPAQUE_STORAGE'],
result.file[0].dir)
shutil.rmtree(result_file_path)
result.delete() # cascades down and also deletes ResultFile

View File

@ -1,12 +1,94 @@
from . import results
from ..models import Result
from flask import abort, render_template, current_app, request
from flask_login import current_user, login_required
from . import tasks
from .. import db
from ..corpora.forms import DisplayOptionsForm
from ..models import Result, ResultFile, User
from .forms import ImportResultsForm
from datetime import datetime
from flask import (abort, render_template, current_app, request, redirect,
flash, url_for, make_response)
from flask_login import current_user, login_required
import json
import os
@results.route('/import_results', methods=['GET', 'POST'])
@login_required
def import_results():
'''
View to import one json result file. Uses the ImportReultFileForm.
'''
import_results_form = ImportResultsForm(prefix='add-result-file-form')
if import_results_form.is_submitted():
if not import_results_form.validate():
return make_response(import_results_form.errors, 400)
# Save the file
# result creation only happens on file save to avoid creating a result
# object in the db everytime by just visiting the import_results page
result = Result(user_id=current_user.id)
db.session.add(result)
db.session.commit()
if not (result.creator == current_user
or current_user.is_administrator()):
abort(403)
dir = os.path.join(str(result.user_id),
'results',
'corpus_analysis_results',
str(result.id))
abs_dir = os.path.join(current_app.config['NOPAQUE_STORAGE'], dir)
abs_file_path = os.path.join(abs_dir,
import_results_form.file.data.filename)
os.makedirs(abs_dir)
import_results_form.file.data.save(abs_file_path)
# Saves all needed metadata entries in one json field
with open(abs_file_path, 'r') as f:
corpus_metadata = json.load(f)
del corpus_metadata['matches']
del corpus_metadata['cpos_lookup']
result_file = ResultFile(
result_id=result.id,
dir=dir,
filename=import_results_form.file.data.filename)
result.corpus_metadata = corpus_metadata
db.session.add(result_file)
db.session.commit()
flash('Result file added!', 'result')
return make_response(
{'redirect_url': url_for('results.results_overview')},
201)
return render_template('results/import_results.html.j2',
import_results_form=import_results_form,
title='Add corpus file')
@results.route('/')
@login_required
def results_overview():
'''
Shows an overview of imported results.
'''
# get all results of current user
results = User.query.get(current_user.id).results
def __p_time(time_str):
# helper to convert the datetime into a nice readable string
return datetime.strptime(time_str, '%Y-%m-%dT%H:%M:%S.%f')
# convert results into a list of dicts to add the measier to list.js in
# the template
results = [dict(query=r.corpus_metadata['query'],
match_count=r.corpus_metadata['match_count'],
corpus_name=r.corpus_metadata['corpus_name'],
corpus_creation_date=__p_time(r.corpus_metadata['corpus_creation_date']), # noqa
corpus_analysis_date=__p_time(r.corpus_metadata['corpus_analysis_date']), # noqa
corpus_type=r.corpus_metadata['corpus_type'],
id=r.id)
for r in results]
return render_template('results/results.html.j2',
title='Imported Results',
# table=table,
results=results)
@results.route('/<int:result_id>/details')
@login_required
def result_details(result_id):
@ -44,3 +126,17 @@ def result_inspect(result_id):
result=result,
result_json=result_json,
title='Result Insepct')
@results.route('/<int:result_id>/delete')
@login_required
def result_delete(result_id):
result = Result.query.get_or_404(result_id)
if not result.id == result_id:
abort(404)
if not (result.creator == current_user
or current_user.is_administrator()):
abort(403)
tasks.delete_result(result_id)
flash('Result deleted!')
return redirect(url_for('results.results_overview'))

View File

@ -1,18 +0,0 @@
from flask_wtf import FlaskForm
from werkzeug.utils import secure_filename
from wtforms import FileField, SubmitField, ValidationError
from wtforms.validators import DataRequired
class ImportResultsForm(FlaskForm):
'''
Form used to import one result json file.
'''
file = FileField('File', validators=[DataRequired()])
submit = SubmitField()
def validate_file(self, field):
if not field.data.filename.lower().endswith('.json'):
raise ValidationError('File does not have an approved extension: '
'.json')
field.data.filename = secure_filename(field.data.filename)

View File

@ -1,17 +1,13 @@
from flask import (abort, current_app, flash, make_response, render_template,
url_for)
from flask_login import current_user, login_required
from .forms import ImportResultsForm
from werkzeug.utils import secure_filename
from . import services
from .. import db
from ..jobs.forms import AddFileSetupJobForm, AddNLPJobForm, AddOCRJobForm
from ..models import Job, JobInput, Result, ResultFile, User
from .tables import ResultTable, ResultItem
from ..models import Job, JobInput
import json
import os
import html
from datetime import datetime
SERVICES = {'corpus_analysis': {'name': 'Corpus analysis'},
@ -85,84 +81,3 @@ def service(service):
return render_template('services/{}.html.j2'.format(service),
title=SERVICES[service]['name'],
add_job_form=add_job_form)
@services.route('/import_results', methods=['GET', 'POST'])
@login_required
def import_results():
'''
View to import one json result file. Uses the ImportReultFileForm.
'''
# TODO: Build in a check if uploaded json is actually a result file and
# not something different
# Add the possibility to add several result files at once.
import_results_form = ImportResultsForm(prefix='add-result-file-form')
if import_results_form.is_submitted():
if not import_results_form.validate():
return make_response(import_results_form.errors, 400)
# Save the file
# result creation only happens on file save to avoid creating a result
# object in the db everytime by just visiting the import_results page
result = Result(user_id=current_user.id)
db.session.add(result)
db.session.commit()
if not (result.creator == current_user
or current_user.is_administrator()):
abort(403)
dir = os.path.join(str(result.user_id),
'results',
'corpus_analysis_results',
str(result.id))
abs_dir = os.path.join(current_app.config['NOPAQUE_STORAGE'], dir)
abs_file_path = os.path.join(abs_dir,
import_results_form.file.data.filename)
os.makedirs(abs_dir)
import_results_form.file.data.save(abs_file_path)
# Saves all needed metadata entries in one json field
with open(abs_file_path, 'r') as f:
corpus_metadata = json.load(f)
del corpus_metadata['matches']
del corpus_metadata['cpos_lookup']
result_file = ResultFile(
result_id=result.id,
dir=dir,
filename=import_results_form.file.data.filename)
result.corpus_metadata = corpus_metadata
db.session.add(result_file)
db.session.commit()
flash('Result file added!', 'result')
return make_response(
{'redirect_url': url_for('services.results')},
201)
return render_template('services/import_results.html.j2',
import_results_form=import_results_form,
title='Add corpus file')
@services.route('/results')
@login_required
def results():
'''
Shows an overview of imported results.
'''
# get all results of current user
results = User.query.get(current_user.id).results
# create table row for every result#
def __p_time(time_str):
return datetime.strptime(time_str, '%Y-%m-%dT%H:%M:%S.%f')
items = [ResultItem(r.corpus_metadata['query'],
r.corpus_metadata['match_count'],
r.corpus_metadata['corpus_name'],
__p_time(r.corpus_metadata['corpus_creation_date']),
__p_time(r.corpus_metadata['corpus_analysis_date']),
r.corpus_metadata['corpus_type'],
r.id) for r in results]
# create table with items and save it as html
table = html.unescape(ResultTable(items).__html__())
# add class=list to table body with string replacement
table = table.replace('tbody', 'tbody class=list', 1)
return render_template('services/results.html.j2',
title='Imported Results',
table=table)

View File

@ -1,14 +1,21 @@
class RessourceList extends List {
constructor(idOrElement, subscriberList, type, options={}) {
if (!['corpus', 'job'].includes(type)) {
if (!["corpus", "job", "result"].includes(type)) {
console.error("Unknown Type!");
return;
}
if (subscriberList) {
super(idOrElement, {...RessourceList.options['common'],
...RessourceList.options[type],
...options});
this.type = type;
subscriberList.push(this);
} else {
super(idOrElement, {...RessourceList.options['extended'],
...RessourceList.options[type],
...options});
this.type = type;
}
}
@ -53,6 +60,8 @@ class RessourceList extends List {
this.add(ressources.map(x => RessourceList.dataMapper[this.type](x)));
}
}
RessourceList.dataMapper = {
corpus: corpus => ({creation_date: corpus.creation_date,
description: corpus.description,
@ -67,10 +76,35 @@ RessourceList.dataMapper = {
link: `/jobs/${job.id}`,
service: job.service,
status: job.status,
title: job.title})
title: job.title}),
result : result => ({ query: result.query,
match_count: result.match_count,
corpus_name: result.corpus_name,
corpus_creation_date: result.corpus_creation_date,
corpus_analysis_date: result.corpus_analysis_date,
corpus_type : result.corpus_type,
"details-link": `${result.id}/details`,
"inspect-link": `${result.id}/inspect`,
"delete-modal": `delete-result-${result.id}-modal`})
};
RessourceList.options = {
common: {page: 4, pagination: {innerWindow: 8, outerWindow: 1}},
extended: {page: 10,
pagination: [
{
name: "paginationTop",
paginationClass: "paginationTop",
innerWindow: 8,
outerWindow: 1
},
{
paginationClass: "paginationBottom",
innerWindow: 8,
outerWindow: 1
}
]},
corpus: {item: `<tr>
<td>
<a class="btn-floating disabled">
@ -122,7 +156,33 @@ RessourceList.options = {
{data: ["id"]},
{name: "link", attr: "href"},
{name: "service", attr: "data-service"},
{name: "status", attr: "data-status"}]}
{name: "status", attr: "data-status"}]},
result : {item: `<tr>
<td class="query"></td>
<td class="match_count"></td>
<td class="corpus_name"></td>
<td class="corpus_creation_date"></td>
<td class="corpus_analysis_date"></td>
<td class="corpus_type"></td>
<td class="actions right-align">
<a class="btn-floating details-link waves-effect waves-light"><i class="material-icons">info_outline</i>
</a>
<a class="btn-floating inspect-link waves-effect waves-light"><i class="material-icons">search</i>
</a>
<a class="btn-floating red delete-modal waves-effect waves-light modal-trigger"><i class="material-icons">delete</i>
</a>
</td>
</tr>`,
valueNames: ["query",
"match_count",
"corpus_name",
"corpus_creation_date",
"corpus_analysis_date",
"corpus_type",
{name: "details-link", attr: "href"},
{name: "inspect-link", attr: "href"},
{name: "delete-modal", attr: "data-target"}]
}
};

View File

@ -11,7 +11,7 @@
<input id="search-corpus" class="search" type="search"></input>
<label for="search-corpus">Search corpus</label>
</div>
<table>
<table class="highlight">
<thead>
<tr>
<th></th>
@ -28,7 +28,8 @@
<ul class="pagination"></ul>
</div>
<div class="card-action right-align">
<a class="waves-effect waves-light btn" href="{{ url_for('services.results') }}">Show Imported Results<i class="material-icons right">folder</i></a>
<a class="waves-effect waves-light btn" href="{{ url_for('results.import_results') }}">Import Results<i class="material-icons right">file_upload</i></a>
<a class="waves-effect waves-light btn" href="{{ url_for('results.results_overview') }}">Show Imported Results<i class="material-icons right">folder</i></a>
<a class="waves-effect waves-light btn" href="{{ url_for('corpora.add_corpus') }}">New corpus<i class="material-icons right">add</i></a>
</div>
</div>
@ -44,7 +45,7 @@
<input id="search-job" class="search" type="search"></input>
<label for="search-job">Search job</label>
</div>
<table>
<table class="highlight">
<thead>
<tr>
<th><span class="sort" data-sort="service">Service</span></th>

View File

@ -63,7 +63,8 @@
</table>
</div>
<div class="card-action right-align">
<a class="waves-effect waves-light btn" href="{{ url_for('services.import_results') }}">Inspect Results<i class="material-icons right">search</i></a>
<a class="waves-effect waves-light btn left-align" href="{{ url_for('results.results_overview') }}">Back To Overview<i class="material-icons right">arrow_back</i></a>
<a class="waves-effect waves-light btn" href="{{ url_for('results.result_inspect', result_id=result.id) }}">Inspect Results<i class="material-icons right">search</i></a>
</div>
</div>
</div>

View File

@ -0,0 +1,70 @@
{% extends "nopaque.html.j2" %}
{% set full_width = True %}
{% block page_content %}
<div class="col s12">
<p>This is an overview of all your imported results.</p>
</div>
<div class="col s12">
<div class="card">
<div class="card-content" id="results">
<div class="input-field">
<i class="material-icons prefix">search</i>
<input id="search-results" class="search" type="search"></input>
<label for="search-results">Search results</label>
</div>
<ul class="pagination paginationTop"></ul>
<table class="highlight responsive-table">
<thead>
<tr>
<th class="sort" data-sort="query">Query</th>
<th class="sort" data-sort="match_count">Match count</th>
<th class="sort" data-sort="corpus_name">Corpus name</th>
<th class="sort" data-sort="corpus_creation_date">Corpus creation date</th>
<th class="sort" data-sort="corpus_analysis_date">Corpus analysis date</th>
<th class="sort" data-sort="corpus_type">Corpus type</th>
<th>{# Actions #}</th>
</tr>
</thead>
<tbody class="list">
<tr class="show-if-only-child">
<td colspan="5">
<span class="card-title"><i class="material-icons left">folder</i>Nothing here...</span>
<p>No results yet improted.</p>
</td>
</tr>
</tbody>
</table>
<ul class="pagination paginationBottom"></ul>
</div>
<div class="card-action right-align">
<a class="waves-effect waves-light btn" href="{{ url_for('results.import_results') }}">Import Results<i class="material-icons right">file_upload</i></a>
</div>
</div>
</div>
{# Delete modals #}
{% for result in results %}
<div id="delete-result-{{ result.id }}-modal" class="modal">
<div class="modal-content">
<h4>Confirm result file deletion</h4>
<p>Do you really want to delete the result file created on <i>{{ result.corpus_analysis_date }}</i>?
<p>The file holds results for the query <i>{{ result.query }}</i>.</p>
<p>The file will be permanently deleted!</p>
</div>
<div class="modal-footer">
<a href="#!" class="btn modal-close waves-effect waves-light">Cancel</a>
<a class="btn modal-close red waves-effect waves-light" href="{{ url_for('results.result_delete', result_id=result.id) }}"><i class="material-icons left">delete</i>Delete</a>
</div>
</div>
{% endfor %}
<script>
var ressources = {{ results|tojson|safe }};
var importedResultsList = new RessourceList("results", null, "result");
importedResultsList.addRessources(ressources);
</script>
{% endblock %}

View File

@ -36,7 +36,8 @@
<ul class="pagination"></ul>
</div>
<div class="card-action right-align">
<a class="waves-effect waves-light btn" href="{{ url_for('services.results') }}">Show Imported Results<i class="material-icons right">folder</i></a>
<a class="waves-effect waves-light btn" href="{{ url_for('results.import_results') }}">Import Results<i class="material-icons right">file_upload</i></a>
<a class="waves-effect waves-light btn" href="{{ url_for('results.results_overview') }}">Show Imported Results<i class="material-icons right">folder</i></a>
<a class="waves-effect waves-light btn" href="{{ url_for('corpora.add_corpus') }}">New corpus<i class="material-icons right">add</i></a>
</div>
</div>

View File

@ -1,52 +0,0 @@
{% extends "nopaque.html.j2" %}
{% block page_content %}
<div class="col s12">
<p>This is an overview of all your imported results.</p>
</div>
<div class="col s12">
<div class="card">
<div class="card-content" id="results">
<div class="input-field">
<i class="material-icons prefix">search</i>
<input id="search-results" class="search" type="search"></input>
<label for="search-results">Search results</label>
</div>
<ul class="pagination paginationTop"></ul>
{{ table }}
<ul class="pagination paginationBottom"></ul>
<ul class="pagination"></ul>
</div>
<div class="card-action right-align">
<a class="waves-effect waves-light btn" href="{{ url_for('services.import_results') }}">Import Results<i class="material-icons right">file_upload</i></a>
</div>
</div>
</div>
<script>
var options = {page: 10,
pagination: [
{
name: "paginationTop",
paginationClass: "paginationTop",
innerWindow: 8,
outerWindow: 1
},
{
paginationClass: "paginationBottom",
innerWindow: 8,
outerWindow: 1
}
],
valueNames: ['query',
'match-count',
'corpus-name',
'corpus-creation-date',
'corpus-analysis-date',
'corpus-type']
};
var resultsList = new List('results', options);
</script>
{% endblock %}