From 37d5802b42accb32bc65f40b7681fce81208c0c5 Mon Sep 17 00:00:00 2001 From: Patrick Jentsch Date: Mon, 4 Nov 2019 15:06:54 +0100 Subject: [PATCH 1/6] Update --- .gitlab-ci.yml | 16 ++++++++-------- app/corpora/views.py | 4 +++- app/models.py | 10 ++++++---- app/utils.py | 7 ++++++- 4 files changed, 23 insertions(+), 14 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 205d949e..b3bf5524 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -15,8 +15,8 @@ before_script: Build: script: - - docker build --pull -t $CI_REGISTRY_IMAGE:$CI_COMMIT_SHA . - - docker push $CI_REGISTRY_IMAGE:$CI_COMMIT_SHA + - docker build --pull -t $CI_REGISTRY_IMAGE:tmp . + - docker push $CI_REGISTRY_IMAGE:tmp stage: build tags: - docker @@ -25,8 +25,8 @@ Push development: only: - development script: - - docker pull $CI_REGISTRY_IMAGE:$CI_COMMIT_SHA - - docker tag $CI_REGISTRY_IMAGE:$CI_COMMIT_SHA $CI_REGISTRY_IMAGE:development + - docker pull $CI_REGISTRY_IMAGE:tmp + - docker tag $CI_REGISTRY_IMAGE:tmp $CI_REGISTRY_IMAGE:development - docker push $CI_REGISTRY_IMAGE:development stage: push tags: @@ -36,8 +36,8 @@ Push latest: only: - master script: - - docker pull $CI_REGISTRY_IMAGE:$CI_COMMIT_SHA - - docker tag $CI_REGISTRY_IMAGE:$CI_COMMIT_SHA $CI_REGISTRY_IMAGE:latest + - docker pull $CI_REGISTRY_IMAGE:tmp + - docker tag $CI_REGISTRY_IMAGE:tmp $CI_REGISTRY_IMAGE:latest - docker push $CI_REGISTRY_IMAGE:latest stage: push tags: @@ -47,8 +47,8 @@ Push tag: only: - tags script: - - docker pull $CI_REGISTRY_IMAGE:$CI_COMMIT_SHA - - docker tag $CI_REGISTRY_IMAGE:$CI_COMMIT_SHA $CI_REGISTRY_IMAGE:$CI_COMMIT_REF_NAME + - docker pull $CI_REGISTRY_IMAGE:tmp + - docker tag $CI_REGISTRY_IMAGE:tmp $CI_REGISTRY_IMAGE:$CI_COMMIT_REF_NAME - docker push $CI_REGISTRY_IMAGE:$CI_COMMIT_REF_NAME stage: push tags: diff --git a/app/corpora/views.py b/app/corpora/views.py index aef61908..3a06b204 100644 --- a/app/corpora/views.py +++ b/app/corpora/views.py @@ -19,7 +19,7 @@ def add_corpus(): if add_corpus_form.validate_on_submit(): corpus = Corpus(creator=current_user, description=add_corpus_form.description.data, - title=add_corpus_form.title.data) + status='unprepared', title=add_corpus_form.title.data) db.session.add(corpus) db.session.commit() dir = os.path.join(current_app.config['OPAQUE_STORAGE_DIRECTORY'], @@ -113,6 +113,7 @@ def add_corpus_file(corpus_id): corpus=corpus, dir=dir, filename=filename, publishing_year=add_corpus_file_form.publishing_year.data, title=add_corpus_file_form.title.data) + corpus_file.insert_metadata() db.session.add(corpus_file) db.session.commit() flash('Corpus file added!') @@ -166,6 +167,7 @@ def edit_corpus_file(corpus_id, corpus_file_id): corpus_file.author = edit_corpus_file_form.author.data corpus_file.publishing_year = edit_corpus_file_form.publishing_year.data corpus_file.title = edit_corpus_file_form.title.data + corpus_file.insert_metadata() db.session.commit() flash('Corpus file edited!') return redirect(url_for('corpora.corpus', corpus_id=corpus_id)) diff --git a/app/models.py b/app/models.py index 77892c02..ac5578d1 100644 --- a/app/models.py +++ b/app/models.py @@ -424,6 +424,9 @@ class CorpusFile(db.Model): db.session.delete(self) db.session.commit() + def insert_metadata(self): + pass + class Corpus(db.Model): """ @@ -436,16 +439,12 @@ class Corpus(db.Model): description = db.Column(db.String(255)) title = db.Column(db.String(32)) user_id = db.Column(db.Integer, db.ForeignKey('users.id')) - status = db.Column(db.String(16)) # Relationships files = db.relationship('CorpusFile', backref='corpus', lazy='dynamic', cascade='save-update, merge, delete') - def __init__(self, **kwargs): - super(Corpus, self).__init__(**kwargs) - def __repr__(self): """ String representation of the corpus. For human readability. @@ -482,6 +481,9 @@ class Corpus(db.Model): db.session.delete(self) db.session.commit() + def prepare(self): + pass + ''' ' Flask-Login is told to use the application’s custom anonymous user by setting diff --git a/app/utils.py b/app/utils.py index 7c1d9638..6eae9d54 100644 --- a/app/utils.py +++ b/app/utils.py @@ -1,4 +1,4 @@ -from .models import Job, User, Corpus +from .models import Job, User, Corpus, CorpusFile from . import db import logging @@ -64,3 +64,8 @@ def background_delete_corpus(app, corpus_id): logger.warning('Corpus id is: {}.'.format(corpus_id)) corpus = Corpus.query.filter_by(id=corpus_id).first() corpus.delete() + + +def background_prepare_corpus_file(app, corpus_file_id): + with app.app_context(): + corpus_file = CorpusFile.query.filter_by(id=corpus_file_id).first() From ffcc13ef5b1d1ca8186906cc475ead783f7dadac Mon Sep 17 00:00:00 2001 From: Patrick Jentsch Date: Mon, 4 Nov 2019 15:35:09 +0100 Subject: [PATCH 2/6] Add status to corpus model again... --- app/models.py | 1 + 1 file changed, 1 insertion(+) diff --git a/app/models.py b/app/models.py index ac5578d1..5c08da59 100644 --- a/app/models.py +++ b/app/models.py @@ -437,6 +437,7 @@ class Corpus(db.Model): id = db.Column(db.Integer, primary_key=True) creation_date = db.Column(db.DateTime(), default=datetime.utcnow) description = db.Column(db.String(255)) + status = db.Column(db.String(16)) title = db.Column(db.String(32)) user_id = db.Column(db.Integer, db.ForeignKey('users.id')) # Relationships From 7176ea26ade207ccfbe33ea71c725942ccfd3528 Mon Sep 17 00:00:00 2001 From: Patrick Jentsch Date: Mon, 4 Nov 2019 15:40:55 +0100 Subject: [PATCH 3/6] Show Corpus status on corpus page. --- app/templates/corpora/corpus.html.j2 | 1 + 1 file changed, 1 insertion(+) diff --git a/app/templates/corpora/corpus.html.j2 b/app/templates/corpora/corpus.html.j2 index fbcf321b..59eabf75 100644 --- a/app/templates/corpora/corpus.html.j2 +++ b/app/templates/corpora/corpus.html.j2 @@ -4,6 +4,7 @@

{{ corpus.title }}

{{ corpus.description }}

+ {{ corpus.status }}
From 84578c0d768c50204a4246bf593ef0f2f7625888 Mon Sep 17 00:00:00 2001 From: Patrick Jentsch Date: Mon, 4 Nov 2019 16:58:15 +0100 Subject: [PATCH 4/6] UI enhancements --- app/templates/corpora/corpus.html.j2 | 11 ++++------ app/templates/jobs/job.html.j2 | 32 +++++++++++++--------------- 2 files changed, 19 insertions(+), 24 deletions(-) diff --git a/app/templates/corpora/corpus.html.j2 b/app/templates/corpora/corpus.html.j2 index 59eabf75..bf04d74d 100644 --- a/app/templates/corpora/corpus.html.j2 +++ b/app/templates/corpora/corpus.html.j2 @@ -19,13 +19,10 @@
- Actions - - helpAnalyse - - - deleteDelete Corpus - + +
+ helpAnalyse + deleteDelete Corpus
diff --git a/app/templates/jobs/job.html.j2 b/app/templates/jobs/job.html.j2 index 512dc734..896d2d25 100644 --- a/app/templates/jobs/job.html.j2 +++ b/app/templates/jobs/job.html.j2 @@ -118,23 +118,6 @@

{{ job.title }}

{{ job.description }}

-

Actions:

- - deleteDelete Job - settingsExport Parameters - - @@ -195,6 +178,10 @@ +
+ settingsExport Parameters + deleteDelete Job +
@@ -228,4 +215,15 @@ + + {% endblock %} From 8ef6d3a04bacd98c474cfd498231693a1ef18e70 Mon Sep 17 00:00:00 2001 From: Patrick Jentsch Date: Tue, 5 Nov 2019 09:11:37 +0100 Subject: [PATCH 5/6] Fix date formatting for end-date display --- app/templates/jobs/job.html.j2 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/templates/jobs/job.html.j2 b/app/templates/jobs/job.html.j2 index 896d2d25..30a722e8 100644 --- a/app/templates/jobs/job.html.j2 +++ b/app/templates/jobs/job.html.j2 @@ -79,7 +79,7 @@ if (timestamp === null) { end_date = "N.a."; } else { - end_date = new Date(timestamp * 1000).toLocaleString(); + end_date = new Date(timestamp * 1000).toLocaleString("en-US"); } document.getElementById("end-date").value = end_date; M.updateTextFields(); From 69387ee183e95287b12d0a19f3c5ca38ea30f1cb Mon Sep 17 00:00:00 2001 From: Patrick Jentsch Date: Tue, 5 Nov 2019 09:12:06 +0100 Subject: [PATCH 6/6] Insert metadata to corpus file in constructor --- app/corpora/views.py | 1 - app/models.py | 16 +++++++++++++--- 2 files changed, 13 insertions(+), 4 deletions(-) diff --git a/app/corpora/views.py b/app/corpora/views.py index 3a06b204..e7862b17 100644 --- a/app/corpora/views.py +++ b/app/corpora/views.py @@ -113,7 +113,6 @@ def add_corpus_file(corpus_id): corpus=corpus, dir=dir, filename=filename, publishing_year=add_corpus_file_form.publishing_year.data, title=add_corpus_file_form.title.data) - corpus_file.insert_metadata() db.session.add(corpus_file) db.session.commit() flash('Corpus file added!') diff --git a/app/models.py b/app/models.py index 5c08da59..4987fb4c 100644 --- a/app/models.py +++ b/app/models.py @@ -2,6 +2,7 @@ from flask import current_app from flask_login import UserMixin, AnonymousUserMixin from itsdangerous import BadSignature, TimedJSONWebSignatureSerializer from werkzeug.security import generate_password_hash, check_password_hash +import xml.etree.ElementTree as ET from . import db from . import login_manager from datetime import datetime @@ -410,6 +411,18 @@ class CorpusFile(db.Model): title = db.Column(db.String(64)) corpus_id = db.Column(db.Integer, db.ForeignKey('corpora.id')) + def __init__(self, **kwargs): + super(CorpusFile, self).__init__(**kwargs) + file = os.path.join(current_app.config['OPAQUE_STORAGE_DIRECTORY'], + self.dir, + self.filename) + element_tree = ET.parse(file) + text_node = element_tree.find('text') + text_node.set('author', self.author) + text_node.set('publishing_year', str(self.publishing_year)) + text_node.set('title', self.title) + element_tree.write(file) + def delete(self): logger = logging.getLogger(__name__) logger.warning('Called CorpusFile.delete') @@ -424,9 +437,6 @@ class CorpusFile(db.Model): db.session.delete(self) db.session.commit() - def insert_metadata(self): - pass - class Corpus(db.Model): """