diff --git a/.env.tpl b/.env.tpl index 588e2f45..30a89416 100644 --- a/.env.tpl +++ b/.env.tpl @@ -21,6 +21,9 @@ HOST_DOCKER_GID= # NOTES: Use `.` as # HOST_LOG_DIR= +# DEFAULT: nopaque_default +# DOCKER_NETWORK_NAME= + ################################################################################ # Flask # # https://flask.palletsprojects.com/en/1.1.x/config/ # @@ -186,4 +189,4 @@ NOPAQUE_DOCKER_REGISTRY_PASSWORD= # READ-COOP account data: https://readcoop.eu/ # NOPAQUE_READCOOP_USERNAME= -# NOPAQUE_READCOOP_PASSWORD= \ No newline at end of file +# NOPAQUE_READCOOP_PASSWORD= diff --git a/.gitignore b/.gitignore index 76c4e06b..14a22fe1 100644 --- a/.gitignore +++ b/.gitignore @@ -39,3 +39,4 @@ __pycache__ # Virtual environment venv +.idea diff --git a/app/SpaCyNLPPipelineModel.defaults.yml b/app/SpaCyNLPPipelineModel.defaults.yml new file mode 100644 index 00000000..576f85e4 --- /dev/null +++ b/app/SpaCyNLPPipelineModel.defaults.yml @@ -0,0 +1,10 @@ +- title: 'de_core_news_md-3.4.0' + description: 'German pipeline optimized for CPU. Components: tok2vec, tagger, morphologizer, parser, lemmatizer (trainable_lemmatizer), senter, ner.' + url: 'https://github.com/explosion/spacy-models/releases/download/de_core_news_md-3.4.0/de_core_news_md-3.4.0.tar.gz' + publisher: 'Explosion' + publisher_url: 'https://github.com/explosion' + publishing_url: 'https://github.com/explosion/spacy-models/releases/tag/de_core_news_md-3.4.0' + publishing_year: 2022 + version: '3.4.0' + compatible_service_versions: + - '0.1.0' diff --git a/app/TesseractOCRModel.defaults.yml b/app/TesseractOCRPipelineModel.defaults.yml similarity index 100% rename from app/TesseractOCRModel.defaults.yml rename to app/TesseractOCRPipelineModel.defaults.yml diff --git a/app/api/auth.py b/app/api/auth.py index afda3a30..398052f5 100644 --- a/app/api/auth.py +++ b/app/api/auth.py @@ -1,4 +1,3 @@ -from flask import current_app from flask_httpauth import HTTPBasicAuth, HTTPTokenAuth from werkzeug.exceptions import Forbidden, Unauthorized from app.models import User diff --git a/app/api/jobs.py b/app/api/jobs.py index e730f2e6..2eaecd3f 100644 --- a/app/api/jobs.py +++ b/app/api/jobs.py @@ -4,8 +4,8 @@ from apifairy.decorators import body, other_responses from flask import abort, Blueprint from werkzeug.exceptions import InternalServerError from app import db, hashids -from app.models import Job, JobInput, JobStatus, TesseractOCRModel -from .schemas import EmptySchema, JobSchema, SpaCyNLPPipelineJobSchema, TesseractOCRPipelineJobSchema, TesseractOCRModelSchema +from app.models import Job, JobInput, JobStatus, TesseractOCRPipelineModel +from .schemas import EmptySchema, JobSchema, SpaCyNLPPipelineJobSchema, TesseractOCRPipelineJobSchema, TesseractOCRPipelineModelSchema from .auth import auth_error_responses, token_auth @@ -14,8 +14,8 @@ job_schema = JobSchema() jobs_schema = JobSchema(many=True) spacy_nlp_pipeline_job_schema = SpaCyNLPPipelineJobSchema() tesseract_ocr_pipeline_job_schema = TesseractOCRPipelineJobSchema() -tesseract_ocr_model_schema = TesseractOCRModelSchema() -tesseract_ocr_models_schema = TesseractOCRModelSchema(many=True) +tesseract_ocr_pipeline_model_schema = TesseractOCRPipelineModelSchema() +tesseract_ocr_pipeline_models_schema = TesseractOCRPipelineModelSchema(many=True) @bp.route('', methods=['GET']) @@ -60,11 +60,11 @@ def create_tesseract_ocr_pipeline_job(args): @bp.route('/tesseract-ocr-pipeline/models', methods=['GET']) @authenticate(token_auth) -@response(tesseract_ocr_models_schema) +@response(tesseract_ocr_pipeline_models_schema) @other_responses(auth_error_responses) def get_tesseract_ocr_models(): """Get all Tesseract OCR Models""" - return TesseractOCRModel.query.all() + return TesseractOCRPipelineModel.query.all() @bp.route('/', methods=['DELETE']) diff --git a/app/api/schemas.py b/app/api/schemas.py index 394b1ebb..9474bd1a 100644 --- a/app/api/schemas.py +++ b/app/api/schemas.py @@ -3,7 +3,14 @@ from marshmallow import validate, validates, ValidationError from marshmallow.decorators import post_dump from app import ma from app.auth import USERNAME_REGEX -from app.models import Job, JobStatus, TesseractOCRModel, Token, User, UserSettingJobStatusMailNotificationLevel +from app.models import ( + Job, + JobStatus, + TesseractOCRPipelineModel, + Token, + User, + UserSettingJobStatusMailNotificationLevel +) from app.services import SERVICES @@ -21,9 +28,9 @@ class TokenSchema(ma.SQLAlchemySchema): refresh_token = ma.String() -class TesseractOCRModelSchema(ma.SQLAlchemySchema): +class TesseractOCRPipelineModelSchema(ma.SQLAlchemySchema): class Meta: - model = TesseractOCRModel + model = TesseractOCRPipelineModel ordered = True hashid = ma.String(data_key='id', dump_only=True) diff --git a/app/api/users.py b/app/api/users.py index fc180df0..c9ea5d39 100644 --- a/app/api/users.py +++ b/app/api/users.py @@ -1,7 +1,7 @@ from apifairy import authenticate, body, response from apifairy.decorators import other_responses -from flask import abort, Blueprint, current_app +from flask import abort, Blueprint from werkzeug.exceptions import InternalServerError from app import db from app.email import create_message, send diff --git a/app/cli.py b/app/cli.py index d9b4fdf0..826aa790 100644 --- a/app/cli.py +++ b/app/cli.py @@ -2,7 +2,12 @@ from flask import current_app from flask_migrate import upgrade import click import os -from app.models import Role, User, TesseractOCRModel, TranskribusHTRModel +from app.models import ( + Role, + User, + TesseractOCRPipelineModel, + SpaCyNLPPipelineModel +) def _make_default_dirs(): @@ -35,10 +40,10 @@ def register(app): Role.insert_defaults() current_app.logger.info('Insert/Update default users') User.insert_defaults() - current_app.logger.info('Insert/Update default TesseractOCRModels') - TesseractOCRModel.insert_defaults() - current_app.logger.info('Insert/Update default TranskribusHTRModels') - TranskribusHTRModel.insert_defaults() + current_app.logger.info('Insert/Update default SpaCyNLPPipelineModels') + SpaCyNLPPipelineModel.insert_defaults() + current_app.logger.info('Insert/Update default TesseractOCRPipelineModels') + TesseractOCRPipelineModel.insert_defaults() @app.cli.group() def converter(): diff --git a/app/contributions/forms.py b/app/contributions/forms.py new file mode 100644 index 00000000..44279a1d --- /dev/null +++ b/app/contributions/forms.py @@ -0,0 +1,58 @@ +from flask_wtf import FlaskForm +from flask_wtf.file import FileField, FileRequired +from wtforms import ( + BooleanField, + StringField, + SubmitField, + SelectMultipleField, + IntegerField +) +from wtforms.validators import InputRequired, Length +from app.services import SERVICES + + +class TesseractOCRModelContributionForm(FlaskForm): + title = StringField( + 'Title', + validators=[InputRequired(), Length(max=64)] + ) + description = StringField( + 'Description', + validators=[InputRequired(), Length(max=255)] + ) + version = StringField( + 'Version', + validators=[InputRequired(), Length(max=16)] + ) + compatible_service_versions = SelectMultipleField( + 'Compatible service versions' + ) + publisher = StringField( + 'Publisher', + validators=[InputRequired(), Length(max=128)] + ) + publisher_url = StringField( + 'Publisher URL', + validators=[InputRequired(), Length(max=512)] + ) + publishing_url = StringField( + 'Publishing URL', + validators=[InputRequired(), Length(max=512)] + ) + publishing_year = IntegerField( + 'Publishing year', + validators=[InputRequired()] + ) + shared = BooleanField('Shared', validators=[InputRequired()]) + model_file = FileField('File',validators=[FileRequired()]) + submit = SubmitField() + + + def __init__(self, *args, **kwargs): + service_manifest = SERVICES['tesseract-ocr-pipeline'] + super().__init__(*args, **kwargs) + self.compatible_service_versions.choices = [('', 'Choose your option')] + self.compatible_service_versions.choices += [ + (x, x) for x in service_manifest['versions'].keys() + ] + self.compatible_service_versions.default = '' diff --git a/app/contributions/routes.py b/app/contributions/routes.py index 80c6a82d..287eda18 100644 --- a/app/contributions/routes.py +++ b/app/contributions/routes.py @@ -1,7 +1,10 @@ +from flask import abort, flash, Markup, render_template, url_for from flask_login import login_required +from app import db from app.decorators import permission_required -from app.models import Permission +from app.models import TesseractOCRPipelineModel, Permission from . import bp +from .forms import TesseractOCRModelContributionForm @bp.before_request @@ -14,3 +17,38 @@ def before_request(): @bp.route('') def contributions(): pass + + +@bp.route('/tesseract-ocr-pipeline-models', methods=['GET', 'POST']) +def tesseract_ocr_pipeline_models(): + form = TesseractOCRModelContributionForm( + prefix='contribute-tesseract-ocr-pipeline-model-form' + ) + if form.is_submitted(): + if not form.validate(): + response = {'errors': form.errors} + return response, 400 + try: + tesseract_ocr_model = TesseractOCRPipelineModel.create( + form.file.data, + compatible_service_versions=form.compatible_service_versions.data, + description=form.description.data, + publisher=form.publisher.data, + publisher_url=form.publisher_url.data, + publishing_url=form.publishing_url.data, + publishing_year=form.publishing_year.data, + shared=form.shared.data, + title=form.title.data, + version=form.version.data + ) + except OSError: + abort(500) + db.session.commit() + message = Markup(f'Model "{tesseract_ocr_model.title}" created') + flash(message) + return {}, 201, {'Location': url_for('contributions.contributions')} + return render_template( + 'contributions/contribute.html.j2', + form=form, + title='Contribution' + ) diff --git a/app/corpora/routes.py b/app/corpora/routes.py index bbe98090..57c14e65 100644 --- a/app/corpora/routes.py +++ b/app/corpora/routes.py @@ -149,7 +149,7 @@ def create_corpus_file(corpus_id): mimetype='application/vrt+xml', corpus=corpus ) - except OSError: + except (AttributeError, OSError): abort(500) corpus.status = CorpusStatus.UNPREPARED db.session.commit() diff --git a/app/daemon/corpus_utils.py b/app/daemon/corpus_utils.py index 1703521a..4d807c14 100644 --- a/app/daemon/corpus_utils.py +++ b/app/daemon/corpus_utils.py @@ -143,7 +143,7 @@ def _create_cqpserver_container(corpus): ''' ## Name ## ''' name = f'cqpserver_{corpus.id}' ''' ## Network ## ''' - network = 'nopaque_default' + network = f'{current_app.config["DOCKER_NETWORK_NAME"]}' ''' ## Volumes ## ''' volumes = [] ''' ### Corpus data volume ### ''' diff --git a/app/daemon/job_utils.py b/app/daemon/job_utils.py index 38d6c48b..32def73d 100644 --- a/app/daemon/job_utils.py +++ b/app/daemon/job_utils.py @@ -3,8 +3,7 @@ from app.models import ( Job, JobResult, JobStatus, - TesseractOCRModel, - TranskribusHTRModel + TesseractOCRPipelineModel ) from datetime import datetime from flask import current_app @@ -61,8 +60,8 @@ def _create_job_service(job): if 'binarization' in job.service_args and job.service_args['binarization']: command += ' --binarize' elif job.service == 'transkribus-htr-pipeline': - transkribus_htr_model = TranskribusHTRModel.query.get(job.service_args['model']) - command += f' -m {transkribus_htr_model.transkribus_model_id}' + transkribus_htr_pipeline_model_id = job.service_args['model'] + command += f' -m {transkribus_htr_pipeline_model_id}' readcoop_username = current_app.config.get('NOPAQUE_READCOOP_USERNAME') command += f' --readcoop-username "{readcoop_username}"' readcoop_password = current_app.config.get('NOPAQUE_READCOOP_PASSWORD') @@ -96,7 +95,7 @@ def _create_job_service(job): else: job.status = JobStatus.FAILED return - model = TesseractOCRModel.query.get(model_id) + model = TesseractOCRPipelineModel.query.get(model_id) if model is None: job.status = JobStatus.FAILED return diff --git a/app/models.py b/app/models.py index 6719ca72..cc5d60ce 100644 --- a/app/models.py +++ b/app/models.py @@ -1,5 +1,6 @@ from datetime import datetime, timedelta from enum import Enum, IntEnum +import re from flask import current_app, url_for from flask_hashids import HashidMixin from flask_login import UserMixin @@ -20,10 +21,6 @@ from app.converters.vrt import normalize_vrt_file from app.email import create_message -TRANSKRIBUS_HTR_MODELS = \ - json.loads(requests.get('https://transkribus.eu/TrpServer/rest/models/text', params={'docType': 'handwritten'}).content)['trpModelMetadata'] # noqa - - ############################################################################## # enums # ############################################################################## @@ -91,6 +88,26 @@ class FileMixin: ), 'mimetype': self.mimetype } + + @classmethod + def create(cls, file_storage, **kwargs): + filename = kwargs.pop('filename', file_storage.filename) + mimetype = kwargs.pop('mimetype', file_storage.mimetype) + obj = cls( + filename=secure_filename(filename), + mimetype=mimetype, + **kwargs + ) + db.session.add(obj) + db.session.flush(objects=[obj]) + db.session.refresh(obj) + try: + file_storage.save(obj.path) + except (AttributeError, OSError) as e: + current_app.logger.error(e) + db.session.rollback() + raise e + return obj # endregion mixins @@ -254,14 +271,14 @@ class User(HashidMixin, UserMixin, db.Model): last_seen = db.Column(db.DateTime()) # Backrefs: role: Role # Relationships - tesseract_ocr_models = db.relationship( - 'TesseractOCRModel', + tesseract_ocr_pipeline_models = db.relationship( + 'TesseractOCRPipelineModel', backref='user', cascade='all, delete-orphan', lazy='dynamic' ) - transkribus_htr_models = db.relationship( - 'TranskribusHTRModel', + spacy_nlp_pipeline_models = db.relationship( + 'SpaCyNLPPipelineModel', backref='user', cascade='all, delete-orphan', lazy='dynamic' @@ -322,7 +339,8 @@ class User(HashidMixin, UserMixin, db.Model): db.session.refresh(user) try: os.mkdir(user.path) - os.mkdir(os.path.join(user.path, 'tesseract_ocr_models')) + os.mkdir(os.path.join(user.path, 'spacy_nlp_pipeline_models')) + os.mkdir(os.path.join(user.path, 'tesseract_ocr_pipeline_models')) os.mkdir(os.path.join(user.path, 'corpora')) os.mkdir(os.path.join(user.path, 'jobs')) except OSError as e: @@ -498,14 +516,14 @@ class User(HashidMixin, UserMixin, db.Model): x.hashid: x.to_json(relationships=True) for x in self.jobs } - _json['tesseract_ocr_models'] = { + _json['tesseract_ocr_pipeline_models'] = { x.hashid: x.to_json(relationships=True) - for x in self.tesseract_ocr_models + for x in self.tesseract_ocr_pipeline_models } return _json -class TesseractOCRModel(FileMixin, HashidMixin, db.Model): - __tablename__ = 'tesseract_ocr_models' +class TesseractOCRPipelineModel(FileMixin, HashidMixin, db.Model): + __tablename__ = 'tesseract_ocr_pipeline_models' # Primary key id = db.Column(db.Integer, primary_key=True) # Foreign keys @@ -526,7 +544,7 @@ class TesseractOCRModel(FileMixin, HashidMixin, db.Model): def path(self): return os.path.join( self.user.path, - 'tesseract_ocr_models', + 'tesseract_ocr_pipeline_models', str(self.id) ) @@ -535,12 +553,12 @@ class TesseractOCRModel(FileMixin, HashidMixin, db.Model): nopaque_user = User.query.filter_by(username='nopaque').first() defaults_file = os.path.join( os.path.dirname(os.path.abspath(__file__)), - 'TesseractOCRModel.defaults.yml' + 'TesseractOCRPipelineModel.defaults.yml' ) with open(defaults_file, 'r') as f: defaults = yaml.safe_load(f) for m in defaults: - model = TesseractOCRModel.query.filter_by(title=m['title'], version=m['version']).first() # noqa + model = TesseractOCRPipelineModel.query.filter_by(title=m['title'], version=m['version']).first() # noqa if model is not None: model.compatible_service_versions = m['compatible_service_versions'] model.description = m['description'] @@ -552,7 +570,7 @@ class TesseractOCRModel(FileMixin, HashidMixin, db.Model): model.title = m['title'] model.version = m['version'] continue - model = TesseractOCRModel( + model = TesseractOCRPipelineModel( compatible_service_versions=m['compatible_service_versions'], description=m['description'], publisher=m['publisher'], @@ -603,45 +621,99 @@ class TesseractOCRModel(FileMixin, HashidMixin, db.Model): return _json -class TranskribusHTRModel(HashidMixin, db.Model): - __tablename__ = 'transkribus_htr_models' +class SpaCyNLPPipelineModel(FileMixin, HashidMixin, db.Model): + __tablename__ = 'spacy_nlp_pipeline_models' # Primary key id = db.Column(db.Integer, primary_key=True) # Foreign keys user_id = db.Column(db.Integer, db.ForeignKey('users.id')) # Fields + title = db.Column(db.String(64)) + description = db.Column(db.String(255)) + version = db.Column(db.String(16)) + compatible_service_versions = db.Column(ContainerColumn(list, 255)) + publisher = db.Column(db.String(128)) + publisher_url = db.Column(db.String(512)) + publishing_url = db.Column(db.String(512)) + publishing_year = db.Column(db.Integer) shared = db.Column(db.Boolean, default=False) - transkribus_model_id = db.Column(db.Integer) # Backrefs: user: User + @property + def path(self): + return os.path.join( + self.user.path, + 'spacy_nlp_pipeline_models', + str(self.id) + ) + @staticmethod def insert_defaults(): nopaque_user = User.query.filter_by(username='nopaque').first() - # models = [ - # m for m in TRANSKRIBUS_HTR_MODELS if True - # and 'creator' in m and m['creator'] == 'Transkribus Team' - # and 'docType' in m and m['docType'] == 'handwritten' - # ] - for m in TRANSKRIBUS_HTR_MODELS: - model = TranskribusHTRModel.query.filter_by(transkribus_model_id=m['modelId']).first() # noqa + defaults_file = os.path.join( + os.path.dirname(os.path.abspath(__file__)), + 'SpaCyNLPPipelineModel.defaults.yml' + ) + with open(defaults_file, 'r') as f: + defaults = yaml.safe_load(f) + for m in defaults: + model = SpaCyNLPPipelineModel.query.filter_by(title=m['title'], version=m['version']).first() # noqa if model is not None: + model.compatible_service_versions = m['compatible_service_versions'] + model.description = m['description'] + model.publisher = m['publisher'] + model.publisher_url = m['publisher_url'] + model.publishing_url = m['publishing_url'] + model.publishing_year = m['publishing_year'] model.shared = True - model.transkribus_model_id = m['modelId'] + model.title = m['title'] + model.version = m['version'] continue - model = TranskribusHTRModel( - transkribus_model_id=m['modelId'], + model = SpaCyNLPPipelineModel( + compatible_service_versions=m['compatible_service_versions'], + description=m['description'], + publisher=m['publisher'], + publisher_url=m['publisher_url'], + publishing_url=m['publishing_url'], + publishing_year=m['publishing_year'], shared=True, + title=m['title'], user=nopaque_user, + version=m['version'] ) db.session.add(model) + db.session.flush(objects=[model]) + db.session.refresh(model) + model.filename = f'{model.id}.traineddata' + r = requests.get(m['url'], stream=True) + pbar = tqdm( + desc=f'{model.title} ({model.filename})', + unit="B", + unit_scale=True, + unit_divisor=1024, + total=int(r.headers['Content-Length']) + ) + pbar.clear() + with open(model.path, 'wb') as f: + for chunk in r.iter_content(chunk_size=1024): + if chunk: # filter out keep-alive new chunks + pbar.update(len(chunk)) + f.write(chunk) + pbar.close() db.session.commit() def to_json(self, backrefs=False, relationships=False): _json = { 'id': self.hashid, - 'user_id': self.user.hashid, + 'compatible_service_versions': self.compatible_service_versions, + 'description': self.description, + 'publisher': self.publisher, + 'publisher_url': self.publisher_url, + 'publishing_url': self.publishing_url, + 'publishing_year': self.publishing_year, 'shared': self.shared, - 'transkribus_model_id': self.transkribus_model_id, + 'title': self.title, + **self.file_mixin_to_json() } if backrefs: _json['user'] = self.user.to_json(backrefs=True) @@ -691,26 +763,6 @@ class JobInput(FileMixin, HashidMixin, db.Model): def user_id(self): return self.job.user_id - @staticmethod - def create(input_file, **kwargs): - filename = kwargs.get('filename', input_file.filename) - mimetype = kwargs.get('mimetype', input_file.mimetype) - job_input = JobInput( - filename=secure_filename(filename), - mimetype=mimetype, - **kwargs - ) - db.session.add(job_input) - db.session.flush(objects=[job_input]) - db.session.refresh(job_input) - try: - input_file.save(job_input.path) - except OSError as e: - current_app.logger.error(e) - db.session.rollback() - raise e - return job_input - def to_json(self, backrefs=False, relationships=False): _json = { 'id': self.hashid, @@ -766,26 +818,6 @@ class JobResult(FileMixin, HashidMixin, db.Model): def user_id(self): return self.job.user_id - @staticmethod - def create(input_file, **kwargs): - filename = kwargs.get('filename', input_file.filename) - mimetype = kwargs.get('mimetype', input_file.mimetype) - job_result = JobResult( - filename=secure_filename(filename), - mimetype=mimetype, - **kwargs - ) - db.session.add(job_result) - db.session.flush(objects=[job_result]) - db.session.refresh(job_result) - try: - input_file.save(job_result.path) - except OSError as e: - current_app.logger.error(e) - db.session.rollback() - raise e - return job_result - def to_json(self, backrefs=False, relationships=False): _json = { 'id': self.hashid, @@ -1024,26 +1056,6 @@ class CorpusFile(FileMixin, HashidMixin, db.Model): _json['corpus'] = self.corpus.to_json(backrefs=True) return _json - @staticmethod - def create(input_file, **kwargs): - filename = kwargs.pop('filename', input_file.filename) - mimetype = kwargs.pop('mimetype', input_file.mimetype) - corpus_file = CorpusFile( - filename=secure_filename(filename), - mimetype=mimetype, - **kwargs, - ) - db.session.add(corpus_file) - db.session.flush(objects=[corpus_file]) - db.session.refresh(corpus_file) - try: - input_file.save(corpus_file.path) - except OSError as e: - current_app.logger.error(e) - db.session.rollback() - raise e - return corpus_file - class Corpus(HashidMixin, db.Model): ''' Class to define a corpus. diff --git a/app/services/forms.py b/app/services/forms.py index 008e0d0a..5c0af906 100644 --- a/app/services/forms.py +++ b/app/services/forms.py @@ -10,11 +10,7 @@ from wtforms import ( ValidationError ) from wtforms.validators import InputRequired, Length -from app.models import ( - TRANSKRIBUS_HTR_MODELS, - TesseractOCRModel, - TranskribusHTRModel -) +from app.models import TesseractOCRPipelineModel from . import SERVICES @@ -77,7 +73,7 @@ class CreateTesseractOCRPipelineJobForm(CreateJobBaseForm): if 'disabled' in self.binarization.render_kw: del self.binarization.render_kw['disabled'] models = [ - x for x in TesseractOCRModel.query.filter().all() + x for x in TesseractOCRPipelineModel.query.filter().all() if version in x.compatible_service_versions and (x.shared == True or x.user == current_user) ] self.model.choices = [('', 'Choose your option')] @@ -107,6 +103,7 @@ class CreateTranskribusHTRPipelineJobForm(CreateJobBaseForm): raise ValidationError('PDF files only!') def __init__(self, *args, **kwargs): + transkribus_htr_pipeline_models = kwargs.pop('transkribus_htr_pipeline_models', []) service_manifest = SERVICES['transkribus-htr-pipeline'] version = kwargs.pop('version', service_manifest['latest_version']) super().__init__(*args, **kwargs) @@ -118,12 +115,8 @@ class CreateTranskribusHTRPipelineJobForm(CreateJobBaseForm): if 'binarization' in service_info['methods']: if 'disabled' in self.binarization.render_kw: del self.binarization.render_kw['disabled'] - models = [ - x for x in TranskribusHTRModel.query.filter().all() - if x.shared == True or x.user == current_user - ] self.model.choices = [('', 'Choose your option')] - self.model.choices += [(x.hashid, [y['name'] for y in TRANSKRIBUS_HTR_MODELS if y['modelId'] == x.transkribus_model_id ][0]) for x in models] + self.model.choices += [(x['modelId'], x['name']) for x in transkribus_htr_pipeline_models] self.model.default = '' self.version.choices = [(x, x) for x in service_manifest['versions']] self.version.data = version diff --git a/app/services/routes.py b/app/services/routes.py index 9f5c81ef..b34d0619 100644 --- a/app/services/routes.py +++ b/app/services/routes.py @@ -1,13 +1,12 @@ -from flask import abort, current_app, flash, Markup, render_template, request +from flask import abort, current_app, flash, make_response, Markup, render_template, request from flask_login import current_user, login_required +import requests from app import db, hashids from app.models import ( Job, JobInput, JobStatus, - TesseractOCRModel, - TRANSKRIBUS_HTR_MODELS, - TranskribusHTRModel + TesseractOCRPipelineModel ) from . import bp, SERVICES from .forms import ( @@ -45,7 +44,7 @@ def file_setup_pipeline(): for input_file in form.images.data: try: JobInput.create(input_file, job=job) - except OSError: + except (AttributeError, OSError): abort(500) job.status = JobStatus.SUBMITTED db.session.commit() @@ -88,21 +87,21 @@ def tesseract_ocr_pipeline(): abort(500) try: JobInput.create(form.pdf.data, job=job) - except OSError: + except (AttributeError, OSError): abort(500) job.status = JobStatus.SUBMITTED db.session.commit() message = Markup(f'Job "{job.title}" created') flash(message, 'job') return {}, 201, {'Location': job.url} - tesseract_ocr_models = [ - x for x in TesseractOCRModel.query.all() + tesseract_ocr_pipeline_models = [ + x for x in TesseractOCRPipelineModel.query.all() if version in x.compatible_service_versions and (x.shared == True or x.user == current_user) ] return render_template( 'services/tesseract_ocr_pipeline.html.j2', form=form, - tesseract_ocr_models=tesseract_ocr_models, + tesseract_ocr_pipeline_models=tesseract_ocr_pipeline_models, title=service_manifest['name'] ) @@ -117,7 +116,18 @@ def transkribus_htr_pipeline(): version = request.args.get('version', service_manifest['latest_version']) if version not in service_manifest['versions']: abort(404) - form = CreateTranskribusHTRPipelineJobForm(prefix='create-job-form', version=version) + r = requests.get( + 'https://transkribus.eu/TrpServer/rest/models/text', + headers={'Accept': 'application/json'} + ) + if r.status_code != 200: + abort(500) + transkribus_htr_pipeline_models = r.json()['trpModelMetadata'] + form = CreateTranskribusHTRPipelineJobForm( + transkribus_htr_pipeline_models=transkribus_htr_pipeline_models, + prefix='create-job-form', + version=version + ) if form.is_submitted(): if not form.validate(): response = {'errors': form.errors} @@ -129,7 +139,7 @@ def transkribus_htr_pipeline(): service=service, service_args={ 'binarization': form.binarization.data, - 'model': hashids.decode(form.model.data) + 'model': form.model.data }, service_version=form.version.data, user=current_user @@ -138,23 +148,18 @@ def transkribus_htr_pipeline(): abort(500) try: JobInput.create(form.pdf.data, job=job) - except OSError: + except (AttributeError, OSError): abort(500) job.status = JobStatus.SUBMITTED db.session.commit() message = Markup(f'Job "{job.title}" created') flash(message, 'job') return {}, 201, {'Location': job.url} - transkribus_htr_models = [ - x for x in TranskribusHTRModel.query.all() - if x.shared == True or x.user == current_user - ] return render_template( 'services/transkribus_htr_pipeline.html.j2', form=form, title=service_manifest['name'], - TRANSKRIBUS_HTR_MODELS=TRANSKRIBUS_HTR_MODELS, - transkribus_htr_models=transkribus_htr_models + transkribus_htr_pipeline_models=transkribus_htr_pipeline_models ) @@ -187,7 +192,7 @@ def spacy_nlp_pipeline(): abort(500) try: JobInput.create(form.txt.data, job=job) - except OSError: + except (AttributeError, OSError): abort(500) job.status = JobStatus.SUBMITTED db.session.commit() diff --git a/app/static/css/queryBuilder.css b/app/static/css/queryBuilder.css new file mode 100644 index 00000000..4ff7eb9d --- /dev/null +++ b/app/static/css/queryBuilder.css @@ -0,0 +1,146 @@ +.modal-conent { + overflow-x: hidden; +} + +#concordance-query-builder { + width: 70%; +} + +#concordance-query-builder nav { + background-color: #6B3F89; + margin-top: -25px; + margin-left: -25px; + width: 105%; +} + +#query-builder-nav{ + padding-left: 15px; +} + +#close-query-builder { + margin-right: 50px; + cursor: pointer; +} + +#general-options-query-builder-tutorial-info-icon { + color: black; +} + +#your-query { + border-bottom-style: solid; + border-bottom-width: 1px; +} + +#insert-query-button { + background-color: #00426f; + text-align: center; +} + +#structural-attr h6 { + margin-left: 15px; +} + +#add-structural-attribute-tutorial-info-icon { + color: black; +} + +#sentence { + background-color:#FD9720; +} + +#entity { + background-color: #A6E22D; +} + +#text-annotation { + background-color: #2FBBAB; +} + +#no-value-metadata-message { + padding-top: 25px; + margin-left: -20px; +} + +#token-kind-selector { + background-color: #f2eff7; + padding: 15px; + border-top-style: solid; + border-color: #6B3F89; +} + +#token-kind-selector.s5 { + margin-top: 15px; +} + +#token-kind-selector h6 { + margin-left: 15px; +} + +#token-tutorial-info-icon { + color: black; +} + +#no-value-message { + padding-top: 25px; + margin-left: -20px; +} + +#token-edit-options h6 { + margin-left: 15px; +} + +#edit-options-tutorial-info-icon { + color: black; +} + +#incidence-modifiers-button a{ + background-color: #2FBBAB; +} + +#incidence-modifiers a{ + background-color: white; +} + +#ignore-case { + margin-left: 5px; +} + +#or, #and { + background-color: #fc0; +} + +#betweenNM { + width: 60%; +} + +#query-builder-tutorial-modal { + width: 60%; +} + +#query-builder-tutorial-modal ul { + margin-top: 10px; +} + +#query-builder-tutorial { + padding:15px; +} + +#scroll-up-button-query-builder-tutorial { + background-color: #28B3D1; +} + +[data-type="start-sentence"], [data-type="end-sentence"] { + background-color: #FD9720; +} + +[data-type="start-empty-entity"], [data-type="start-entity"], [data-type="end-entity"] { + background-color: #A6E22D; +} + +[data-type="start-text-annotation"]{ + background-color: #2FBBAB; +} + +[data-type="token"] { + background-color: #28B3D1; +} diff --git a/app/static/js/CorpusAnalysis/QueryBuilder.js b/app/static/js/CorpusAnalysis/QueryBuilder.js index 256dadd2..522054dd 100644 --- a/app/static/js/CorpusAnalysis/QueryBuilder.js +++ b/app/static/js/CorpusAnalysis/QueryBuilder.js @@ -8,31 +8,32 @@ class ConcordanceQueryBuilder { counter: 0, yourQueryContent: [], queryContent:[], - concordanceQueryBuilder: document.querySelector("#concordance-query-builder"), - concordanceQueryBuilderButton: document.querySelector("#concordance-query-builder-button"), - closeQueryBuilder: document.querySelector("#close-query-builder"), + concordanceQueryBuilder: document.querySelector('#concordance-query-builder'), + concordanceQueryBuilderButton: document.querySelector('#concordance-query-builder-button'), + closeQueryBuilder: document.querySelector('#close-query-builder'), queryBuilderTutorialModal: document.querySelector('#query-builder-tutorial-modal'), + valueValidator: true, //#region QueryBuilder Elements positionalAttrButton: document.querySelector('#positional-attr-button'), positionalAttrArea: document.querySelector('#positional-attr'), - positionalAttr: document.querySelector("#token-attr"), + positionalAttr: document.querySelector('#token-attr'), structuralAttrButton: document.querySelector('#structural-attr-button'), - structuralAttrArea: document.querySelector("#structural-attr"), + structuralAttrArea: document.querySelector('#structural-attr'), queryContainer: document.querySelector('#query-container'), - buttonPreparer: document.querySelector("#button-preparer"), - yourQuery: document.querySelector("#your-query"), - insertQueryButton: document.querySelector("#insert-query-button"), + buttonPreparer: document.querySelector('#button-preparer'), + yourQuery: document.querySelector('#your-query'), + insertQueryButton: document.querySelector('#insert-query-button'), queryPreview: document.querySelector('#query-preview'), - tokenQuery: document.querySelector("#token-query"), - tokenBuilderContent: document.querySelector("#token-builder-content"), - tokenSubmitButton: document.querySelector("#token-submit"), + tokenQuery: document.querySelector('#token-query'), + tokenBuilderContent: document.querySelector('#token-builder-content'), + tokenSubmitButton: document.querySelector('#token-submit'), extFormQuery: document.querySelector('#concordance-extension-form-query'), - dropButton: "", + dropButton: '', queryBuilderTutorialInfoIcon: document.querySelector('#query-builder-tutorial-info-icon'), - tokenTutorialInfoIcon: document.querySelector("#token-tutorial-info-icon"), + tokenTutorialInfoIcon: document.querySelector('#token-tutorial-info-icon'), editTokenTutorialInfoIcon: document.querySelector('#edit-options-tutorial-info-icon'), structuralAttributeTutorialInfoIcon: document.querySelector('#add-structural-attribute-tutorial-info-icon'), generalOptionsQueryBuilderTutorialInfoIcon: document.querySelector('#general-options-query-builder-tutorial-info-icon'), @@ -42,73 +43,73 @@ class ConcordanceQueryBuilder { //#region Strucutral Attributes - sentence:document.querySelector("#sentence"), - entity: document.querySelector("#entity"), - textAnnotation: document.querySelector("#text-annotation"), + sentence:document.querySelector('#sentence'), + entity: document.querySelector('#entity'), + textAnnotation: document.querySelector('#text-annotation'), - entityBuilder: document.querySelector("#entity-builder"), - englishEntType: document.querySelector("#english-ent-type"), - germanEntType: document.querySelector("#german-ent-type"), - emptyEntity: document.querySelector("#empty-entity"), + entityBuilder: document.querySelector('#entity-builder'), + englishEntType: document.querySelector('#english-ent-type'), + germanEntType: document.querySelector('#german-ent-type'), + emptyEntity: document.querySelector('#empty-entity'), entityAnyType: false, - textAnnotationBuilder: document.querySelector("#text-annotation-builder"), - textAnnotationOptions: document.querySelector("#text-annotation-options"), - textAnnotationInput: document.querySelector("#text-annotation-input"), - textAnnotationSubmit: document.querySelector("#text-annotation-submit"), + textAnnotationBuilder: document.querySelector('#text-annotation-builder'), + textAnnotationOptions: document.querySelector('#text-annotation-options'), + textAnnotationInput: document.querySelector('#text-annotation-input'), + textAnnotationSubmit: document.querySelector('#text-annotation-submit'), noValueMetadataMessage: document.querySelector('#no-value-metadata-message'), //#endregion Structural Attributes //#region Token Attributes tokenQueryFilled: false, - lemma: document.querySelector("#lemma"), - emptyToken: document.querySelector("#empty-token"), - word: document.querySelector("#word"), - lemma: document.querySelector("#lemma"), - pos: document.querySelector("#pos"), - simplePosButton: document.querySelector("#simple-pos-button"), - incidenceModifiers: document.querySelector("[data-target='incidence-modifiers']"), - or: document.querySelector("#or"), - and: document.querySelector("#and"), + lemma: document.querySelector('#lemma'), + emptyToken: document.querySelector('#empty-token'), + word: document.querySelector('#word'), + lemma: document.querySelector('#lemma'), + pos: document.querySelector('#pos'), + simplePosButton: document.querySelector('#simple-pos-button'), + incidenceModifiers: document.querySelector('[data-target="incidence-modifiers"]'), + or: document.querySelector('#or'), + and: document.querySelector('#and'), //#region Word and Lemma Elements - wordBuilder: document.querySelector("#word-builder"), - lemmaBuilder: document.querySelector("#lemma-builder"), - inputOptions: document.querySelector("#input-options"), - incidenceModifiersButton: document.querySelector("#incidence-modifiers-button"), + wordBuilder: document.querySelector('#word-builder'), + lemmaBuilder: document.querySelector('#lemma-builder'), + inputOptions: document.querySelector('#input-options'), + incidenceModifiersButton: document.querySelector('#incidence-modifiers-button'), conditionContainer: document.querySelector('#condition-container'), - wordInput: document.querySelector("#word-input"), - lemmaInput: document.querySelector("#lemma-input"), - ignoreCaseCheckbox : document.querySelector("#ignore-case-checkbox"), - ignoreCase: document.querySelector("input[type='checkbox']"), - wildcardChar: document.querySelector("#wildcard-char"), - optionGroup: document.querySelector("#option-group"), + wordInput: document.querySelector('#word-input'), + lemmaInput: document.querySelector('#lemma-input'), + ignoreCaseCheckbox : document.querySelector('#ignore-case-checkbox'), + ignoreCase: document.querySelector('input[type="checkbox"]'), + wildcardChar: document.querySelector('#wildcard-char'), + optionGroup: document.querySelector('#option-group'), //#endregion Word and Lemma Elements //#region posBuilder Elements - englishPosBuilder: document.querySelector("#english-pos-builder"), - englishPos: document.querySelector("#english-pos"), - germanPosBuilder: document.querySelector("#german-pos-builder"), - germanPos: document.querySelector("#german-pos"), + englishPosBuilder: document.querySelector('#english-pos-builder'), + englishPos: document.querySelector('#english-pos'), + germanPosBuilder: document.querySelector('#german-pos-builder'), + germanPos: document.querySelector('#german-pos'), //#endregion posBuilder Elements //#region simple_posBuilder Elements - simplePosBuilder: document.querySelector("#simplepos-builder"), - simplePos: document.querySelector("#simple-pos"), + simplePosBuilder: document.querySelector('#simplepos-builder'), + simplePos: document.querySelector('#simple-pos'), //#endregion simple_posBuilder Elements //#region incidence modifiers - oneOrMore: document.querySelector("#one-or-more"), - zeroOrMore: document.querySelector("#zero-or-more"), - zeroOrOne: document.querySelector("#zero-or-one"), - exactlyN: document.querySelector("#exactlyN"), - betweenNM: document.querySelector("#betweenNM"), - nInput: document.querySelector("#n-input"), - nSubmit: document.querySelector("#n-submit"), - nmInput: document.querySelector("#n-m-input"), - mInput: document.querySelector("#m-input"), - nmSubmit: document.querySelector("#n-m-submit"), + oneOrMore: document.querySelector('#one-or-more'), + zeroOrMore: document.querySelector('#zero-or-more'), + zeroOrOne: document.querySelector('#zero-or-one'), + exactlyN: document.querySelector('#exactlyN'), + betweenNM: document.querySelector('#betweenNM'), + nInput: document.querySelector('#n-input'), + nSubmit: document.querySelector('#n-submit'), + nmInput: document.querySelector('#n-m-input'), + mInput: document.querySelector('#m-input'), + nmSubmit: document.querySelector('#n-m-submit'), //#endregion incidence modifiers cancelBool: false, @@ -116,68 +117,73 @@ class ConcordanceQueryBuilder { //#endregion Token Attributes } - this.elements.closeQueryBuilder.addEventListener("click", () => {this.closeQueryBuilderModal(this.elements.concordanceQueryBuilder);}); - this.elements.concordanceQueryBuilderButton.addEventListener("click", () => {this.clearAll();}); - this.elements.insertQueryButton.addEventListener("click", () => {this.insertQuery();}); - this.elements.positionalAttrButton.addEventListener("click", () => {this.showPositionalAttrArea();}); - this.elements.structuralAttrButton.addEventListener("click", () => {this.showStructuralAttrArea();}); + this.elements.closeQueryBuilder.addEventListener('click', () => {this.closeQueryBuilderModal(this.elements.concordanceQueryBuilder);}); + this.elements.concordanceQueryBuilderButton.addEventListener('click', () => {this.clearAll();}); + this.elements.insertQueryButton.addEventListener('click', () => {this.insertQuery();}); + this.elements.positionalAttrButton.addEventListener('click', () => {this.showPositionalAttrArea();}); + this.elements.structuralAttrButton.addEventListener('click', () => {this.showStructuralAttrArea();}); //#region Structural Attribute Event Listeners - this.elements.sentence.addEventListener("click", () => {this.addSentence();}); - this.elements.entity.addEventListener("click", () => {this.addEntity();}); - this.elements.textAnnotation.addEventListener("click", () => {this.addTextAnnotation();}); + this.elements.sentence.addEventListener('click', () => {this.addSentence();}); + this.elements.entity.addEventListener('click', () => {this.addEntity();}); + this.elements.textAnnotation.addEventListener('click', () => {this.addTextAnnotation();}); - this.elements.englishEntType.addEventListener("change", () => {this.englishEntTypeHandler();}); - this.elements.germanEntType.addEventListener("change", () => {this.germanEntTypeHandler();}); - this.elements.emptyEntity.addEventListener("click", () => {this.emptyEntityButton();}); + this.elements.englishEntType.addEventListener('change', () => {this.englishEntTypeHandler();}); + this.elements.germanEntType.addEventListener('change', () => {this.germanEntTypeHandler();}); + this.elements.emptyEntity.addEventListener('click', () => {this.emptyEntityButton();}); - this.elements.textAnnotationSubmit.addEventListener("click", () => {this.textAnnotationSubmitHandler();}); + this.elements.textAnnotationSubmit.addEventListener('click', () => {this.textAnnotationSubmitHandler();}); //#endregion //#region Token Attribute Event Listeners - this.elements.queryBuilderTutorialInfoIcon.addEventListener("click", () => {this.tutorialIconHandler('#query-builder-tutorial-start');}); - this.elements.tokenTutorialInfoIcon.addEventListener("click", () => {this.tutorialIconHandler('#add-new-token-tutorial');}); - this.elements.editTokenTutorialInfoIcon.addEventListener("click", () => {this.tutorialIconHandler('#edit-options-tutorial');}); - this.elements.structuralAttributeTutorialInfoIcon.addEventListener("click", () => {this.tutorialIconHandler('#add-structural-attribute-tutorial');}); - this.elements.generalOptionsQueryBuilderTutorialInfoIcon.addEventListener("click", () => {this.tutorialIconHandler('#general-options-query-builder');}); + this.elements.queryBuilderTutorialInfoIcon.addEventListener('click', () => {this.tutorialIconHandler('#query-builder-tutorial-start');}); + this.elements.tokenTutorialInfoIcon.addEventListener('click', () => {this.tutorialIconHandler('#add-new-token-tutorial');}); + this.elements.editTokenTutorialInfoIcon.addEventListener('click', () => {this.tutorialIconHandler('#edit-options-tutorial');}); + this.elements.structuralAttributeTutorialInfoIcon.addEventListener('click', () => {this.tutorialIconHandler('#add-structural-attribute-tutorial');}); + this.elements.generalOptionsQueryBuilderTutorialInfoIcon.addEventListener('click', () => {this.tutorialIconHandler('#general-options-query-builder');}); - this.elements.positionalAttr.addEventListener("change", () => {this.tokenTypeSelector();}); - this.elements.tokenSubmitButton.addEventListener("click", () => {this.addToken();}); + this.elements.positionalAttr.addEventListener('change', () => {this.tokenTypeSelector();}); + this.elements.tokenSubmitButton.addEventListener('click', () => {this.addToken();}); - this.elements.ignoreCase.addEventListener("change", () => {this.inputOptionHandler(this.elements.ignoreCase);}); - this.elements.wildcardChar.addEventListener("click", () => {this.inputOptionHandler(this.elements.wildcardChar);}); - this.elements.optionGroup.addEventListener("click", () => {this.inputOptionHandler(this.elements.optionGroup);}); + this.elements.ignoreCase.addEventListener('change', () => {this.inputOptionHandler(this.elements.ignoreCase);}); + this.elements.wildcardChar.addEventListener('click', () => {this.inputOptionHandler(this.elements.wildcardChar);}); + this.elements.optionGroup.addEventListener('click', () => {this.inputOptionHandler(this.elements.optionGroup);}); - this.elements.oneOrMore.addEventListener("click", () => {this.incidenceModifiersHandler(this.elements.oneOrMore);}); - this.elements.zeroOrMore.addEventListener("click", () => {this.incidenceModifiersHandler(this.elements.zeroOrMore);}); - this.elements.zeroOrOne.addEventListener("click", () => {this.incidenceModifiersHandler(this.elements.zeroOrOne);}); - this.elements.nSubmit.addEventListener("click", () => {this.nSubmitHandler();}); - this.elements.nmSubmit.addEventListener("click", () => {this.nmSubmitHandler();}); + this.elements.oneOrMore.addEventListener('click', () => {this.incidenceModifiersHandler(this.elements.oneOrMore);}); + this.elements.zeroOrMore.addEventListener('click', () => {this.incidenceModifiersHandler(this.elements.zeroOrMore);}); + this.elements.zeroOrOne.addEventListener('click', () => {this.incidenceModifiersHandler(this.elements.zeroOrOne);}); + this.elements.nSubmit.addEventListener('click', () => {this.nSubmitHandler();}); + this.elements.nmSubmit.addEventListener('click', () => {this.nmSubmitHandler();}); - this.elements.or.addEventListener("click", () => {this.orHandler();}); - this.elements.and.addEventListener("click", () => {this.andHandler();}); + this.elements.or.addEventListener('click', () => {this.orHandler();}); + this.elements.and.addEventListener('click', () => {this.andHandler();}); //#endregion Token Attribute Event Listeners } + + // ########################################################################## + // #################### General Functions ################################### + // ########################################################################## + //#region General Functions - closeQueryBuilderModal(closeInstance){ + closeQueryBuilderModal(closeInstance) { let instance = M.Modal.getInstance(closeInstance); instance.close(); } - showPositionalAttrArea(){ + showPositionalAttrArea() { this.elements.positionalAttrArea.classList.remove('hide'); - this.elements.wordBuilder.classList.remove("hide"); - this.elements.inputOptions.classList.remove("hide"); + this.elements.wordBuilder.classList.remove('hide'); + this.elements.inputOptions.classList.remove('hide'); this.elements.incidenceModifiersButton.classList.remove('hide'); - this.elements.conditionContainer.classList.remove("hide"); - this.elements.ignoreCaseCheckbox.classList.remove("hide"); + this.elements.conditionContainer.classList.remove('hide'); + this.elements.ignoreCaseCheckbox.classList.remove('hide'); this.elements.structuralAttrArea.classList.add('hide'); this.elements.lemmaBuilder.classList.add('hide'); this.elements.englishPosBuilder.classList.add('hide'); @@ -186,198 +192,195 @@ class ConcordanceQueryBuilder { this.elements.tokenQueryFilled = false; - window.location.href = "#token-builder-content"; + window.location.href = '#token-builder-content'; // Resets materialize select field to default value let SelectInstance = M.FormSelect.getInstance(this.elements.positionalAttr); - SelectInstance.input.value = "word"; - this.elements.positionalAttr.value = "word"; + SelectInstance.input.value = 'word'; + this.elements.positionalAttr.value = 'word'; } - showStructuralAttrArea(){ + showStructuralAttrArea() { this.elements.positionalAttrArea.classList.add('hide'); this.elements.structuralAttrArea.classList.remove('hide'); } buttonfactory(dataType, prettyText, queryText) { - - window.location.href = "#query-container"; + window.location.href = '#query-container'; this.elements.counter += 1; queryText = encodeURI(queryText); - let chipColor = 'style="background-color:#'; - - // Sets chip color, depending on the type of element - if (dataType === 'start-sentence' || dataType === 'end-sentence'){ - chipColor += 'FD9720'; - }else if (dataType === "start-empty-entity" || dataType === "start-entity" || dataType === "end-entity"){ - chipColor += 'A6E22D'; - }else if (dataType === "text-annotation"){ - chipColor += '2FBBAB'; - }else if (dataType === "token"){ - chipColor += '28B3D1'; - }else { - chipColor = ''; - } - - // Creates a chip with the previously selected element. Is first created in the "BuilderElement" and populated with an EventListener, then moved to "yourQuery". - let builderElement = document.createElement('div'); - builderElement.innerHTML +=` -
- ${prettyText} - close -
- `.trim(); - - let buttonElement = builderElement.firstElementChild; - buttonElement.addEventListener("click", () => {this.deleteAttr(buttonElement);}); + let buttonElement = Utils.elementFromString( + ` +
+ ${prettyText} + close +
+ ` + ); + buttonElement.addEventListener('click', () => {this.deleteAttr(buttonElement);}); // Ensures that metadata is always at the end of the query: - if(this.elements.yourQuery.lastChild === null || this.elements.yourQuery.lastChild.dataset.type !== "text-annotation"){ + if (this.elements.yourQuery.lastChild === null || this.elements.yourQuery.lastChild.dataset.type !== 'text-annotation') { this.elements.yourQuery.appendChild(buttonElement); - }else if (this.elements.yourQuery.lastChild.dataset.type === "text-annotation"){ + } else if (this.elements.yourQuery.lastChild.dataset.type === 'text-annotation') { this.elements.yourQuery.insertBefore(buttonElement, this.elements.yourQuery.lastChild); } - - this.elements.queryContainer.classList.remove("hide"); + this.elements.queryContainer.classList.remove('hide'); this.queryPreviewBuilder(); - // Opens a hint about the possible functions for editing the query when the first chip is added. It is displayed for 5 seconds and then deleted. - if (this.elements.yourQuery.classList.contains("tooltipped")){ - let tooltipInstance = M.Tooltip.getInstance(this.elements.yourQuery); - tooltipInstance.tooltipEl.style.background = "#98ACD2"; - tooltipInstance.tooltipEl.style.borderTop = "solid 4px #0064A3" - tooltipInstance.tooltipEl.style.padding = "10px"; - tooltipInstance.tooltipEl.style.color = "black"; - - if (tooltipInstance !== undefined){ - setTimeout(() => { - tooltipInstance.open(); - setTimeout(() => { - tooltipInstance.destroy(); - }, 5000); - }, 500); - } - this.elements.yourQuery.classList.remove("tooltipped"); + // Shows a hint about possible functions for editing the query at the first added element in the query + if (this.elements.yourQuery.childNodes.length === 1) { + app.flash('You can edit your query by deleting individual elements or moving them via drag and drop.'); } } - //#region Drag&Drop Events - dragStartHandler(event){ + //#region Drag&Drop Events + dragStartHandler(event) { + // Creates element with the class 'target' and all necessary drop functions, in which drop content can be released this.elements.dropButton = event.target; let targetChip = ` -
- Drop here -
+
+ Drop here +
`.trim(); - let childNodes = this.elements.yourQuery.querySelectorAll("div:not(.target)"); + // selects all nodes without target class + let childNodes = this.elements.yourQuery.querySelectorAll('div:not(.target)'); + // Adds a target chip in front of all draggable childnodes setTimeout(() => { - for (let element of childNodes){ - if(element === event.target){ + for (let element of childNodes) { + if (element === this.elements.dropButton) { + // If the dragged element is not at the very end, a target chip is also inserted at the end + if (childNodes[childNodes.length - 1] !== element) { + childNodes[childNodes.length - 1].insertAdjacentHTML('afterend', targetChip); + } + } else if (element === this.elements.dropButton.nextSibling) { continue; - }else if (element === event.target.nextSibling){ - continue; - }else { - element.insertAdjacentHTML("beforebegin", targetChip) + } else { + element.insertAdjacentHTML('beforebegin', targetChip) } } - childNodes[childNodes.length-1].insertAdjacentHTML("afterend", targetChip); },0); } - dragOverHandler(event){ + dragOverHandler(event) { event.preventDefault(); } - dragEnterHandler(event){ + dragEnterHandler(event) { event.preventDefault(); - event.target.style.borderStyle = "solid dotted"; + event.target.style.borderStyle = 'solid dotted'; } - dragLeaveHandler(event){ + dragLeaveHandler(event) { event.preventDefault(); - event.target.style.borderStyle = "hidden"; + event.target.style.borderStyle = 'hidden'; } - dragEndHandler(event){ + dragEndHandler(event) { let targets = document.querySelectorAll('.target'); - for (let target of targets){ + for (let target of targets) { target.remove(); } } - dropHandler(event){ + dropHandler(event) { let dropzone = event.target; - - for (let i = 0; i < dropzone.parentElement.childNodes.length; i++){ - if (dropzone === dropzone.parentElement.childNodes[i]){ - nodeIndex = i; - } - } - for (let i = 0; i < dropzone.parentElement.childNodes.length; i++){ - if (this.elements.dropButton === dropzone.parentElement.childNodes[i]){ - draggedElementIndex = i; - } - } - dropzone.parentElement.replaceChild(this.elements.dropButton, dropzone); this.queryPreviewBuilder(); } //#endregion Drag&Drop Events - queryPreviewBuilder(){ + queryPreviewBuilder() { this.elements.yourQueryContent = []; - for (let element of this.elements.yourQuery.childNodes) { let queryElement = decodeURI(element.dataset.query); - if (queryElement.includes("<")){ - queryElement = queryElement.replace("<", "<"); + if (queryElement.includes('<')) { + queryElement = queryElement.replace('<', '<'); } - if (queryElement.includes(">")){ - queryElement = queryElement.replace(">", ">"); + if (queryElement.includes('>')) { + queryElement = queryElement.replace('>', '>'); } - if (queryElement !== "undefined") { + if (queryElement !== 'undefined') { this.elements.yourQueryContent.push(queryElement); } } let queryString = this.elements.yourQueryContent.join(' '); - queryString += ";"; + queryString += ';'; this.elements.queryPreview.innerHTML = queryString; } deleteAttr(attr) { this.elements.yourQuery.removeChild(attr); - - this.elements.counter -= 1; - if(this.elements.counter === 0){ - this.elements.queryContainer.classList.add("hide"); + if (attr.dataset.type === "start-sentence") { + this.elements.sentence.innerHTML = 'Sentence'; + } else if (attr.dataset.type === "start-entity" || attr.dataset.type === "start-empty-entity") { + this.elements.entity.innerHTML = 'Entity'; + } + this.elements.counter -= 1; + if (this.elements.counter === 0) { + this.elements.queryContainer.classList.add('hide'); } - this.queryPreviewBuilder(); } insertQuery() { this.elements.yourQueryContent = []; + this.validateValue(); + if (this.elements.valueValidator === true) { + for (let element of this.elements.yourQuery.childNodes) { + let queryElement = decodeURI(element.dataset.query); + if (queryElement !== 'undefined') { + this.elements.yourQueryContent.push(queryElement); + } + } + let queryString = this.elements.yourQueryContent.join(' '); + queryString += ';'; + + this.elements.concordanceQueryBuilder.classList.add('modal-close'); + this.elements.extFormQuery.value = queryString; + } + } + + validateValue() { + this.elements.valueValidator = true; + let sentenceCounter = 0; + let sentenceEndCounter = 0; + let entityCounter = 0; + let entityEndCounter = 0; for (let element of this.elements.yourQuery.childNodes) { - let queryElement = decodeURI(element.dataset.query); - if (queryElement !== "undefined"){ - this.elements.yourQueryContent.push(queryElement); + if (element.dataset.type === 'start-sentence') { + sentenceCounter += 1; + }else if (element.dataset.type === 'end-sentence') { + sentenceEndCounter += 1; + }else if (element.dataset.type === 'start-entity' || element.dataset.type === 'start-empty-entity') { + entityCounter += 1; + }else if (element.dataset.type === 'end-entity') { + entityEndCounter += 1; + } } + // Checks if the same number of opening and closing tags (entity and sentence) are present. Depending on what is missing, the corresponding error message is ejected + if (sentenceCounter > sentenceEndCounter) { + app.flash('Please add the closing sentence tag', 'error'); + this.elements.valueValidator = false; + } else if (sentenceCounter < sentenceEndCounter) { + app.flash('Please remove the closing sentence tag', 'error'); + this.elements.valueValidator = false; + } + if (entityCounter > entityEndCounter) { + app.flash('Please add the closing entity tag', 'error'); + this.elements.valueValidator = false; + } else if (entityCounter < entityEndCounter) { + app.flash('Please remove the closing entity tag', 'error'); + this.elements.valueValidator = false; } - - let queryString = this.elements.yourQueryContent.join(' '); - queryString += ";"; - - this.elements.concordanceQueryBuilder.classList.add('modal-close'); - this.elements.extFormQuery.value = queryString; - } clearAll() { - // Everything is reset. After 5 seconds for 5 seconds (with "instance"), a message is displayed indicating that further information can be obtained via the question mark icon + // Everything is reset. After 5 seconds for 5 seconds (with 'instance'), a message is displayed indicating that further information can be obtained via the question mark icon let instance = M.Tooltip.getInstance(this.elements.queryBuilderTutorialInfoIcon); this.hideEverything(); @@ -387,6 +390,8 @@ class ConcordanceQueryBuilder { this.elements.structuralAttrArea.classList.add('hide'); this.elements.yourQuery.innerHTML = ''; this.elements.queryContainer.classList.add('hide'); + this.elements.entity.innerHTML = 'Entity'; + this.elements.sentence.innerHTML = 'Sentence'; instance.tooltipEl.style.background = '#98ACD2'; instance.tooltipEl.style.borderTop = 'solid 4px #0064A3'; @@ -411,28 +416,33 @@ class ConcordanceQueryBuilder { //#endregion General Functions + + // ########################################################################## + // ############## Token Attribute Builder Functions ######################### + // ########################################################################## + //#region Token Attribute Builder Functions - //#region General functions of the Token Builder + //#region General functions of the Token Builder tokenTypeSelector() { this.hideEverything(); switch (this.elements.positionalAttr.value) { - case "word": + case 'word': this.wordBuilder(); break; - case "lemma": + case 'lemma': this.lemmaBuilder(); break; - case "english-pos": + case 'english-pos': this.englishPosHandler(); break; - case "german-pos": + case 'german-pos': this.germanPosHandler(); break; - case "simple-pos-button": + case 'simple-pos-button': this.simplePosBuilder(); break; - case "empty-token": + case 'empty-token': this.emptyTokenHandler(); break; default: @@ -441,19 +451,19 @@ class ConcordanceQueryBuilder { } } - hideEverything(){ + hideEverything() { - this.elements.wordBuilder.classList.add("hide"); - this.elements.lemmaBuilder.classList.add("hide"); - this.elements.ignoreCaseCheckbox.classList.add("hide"); - this.elements.inputOptions.classList.add("hide"); - this.elements.incidenceModifiersButton.classList.add("hide"); - this.elements.conditionContainer.classList.add("hide"); - this.elements.englishPosBuilder.classList.add("hide"); - this.elements.germanPosBuilder.classList.add("hide"); - this.elements.simplePosBuilder.classList.add("hide"); - this.elements.entityBuilder.classList.add("hide"); - this.elements.textAnnotationBuilder.classList.add("hide"); + this.elements.wordBuilder.classList.add('hide'); + this.elements.lemmaBuilder.classList.add('hide'); + this.elements.ignoreCaseCheckbox.classList.add('hide'); + this.elements.inputOptions.classList.add('hide'); + this.elements.incidenceModifiersButton.classList.add('hide'); + this.elements.conditionContainer.classList.add('hide'); + this.elements.englishPosBuilder.classList.add('hide'); + this.elements.germanPosBuilder.classList.add('hide'); + this.elements.simplePosBuilder.classList.add('hide'); + this.elements.entityBuilder.classList.add('hide'); + this.elements.textAnnotationBuilder.classList.add('hide'); } @@ -463,20 +473,16 @@ class ConcordanceQueryBuilder { let buttonElement; builderElement = document.createElement('div'); builderElement.innerHTML = ` -
+
${prettyText} - close + close
`; buttonElement = builderElement.firstElementChild; - buttonElement.addEventListener("click", () => {this.deleteTokenAttr(buttonElement);}); + buttonElement.addEventListener('click', () => {this.deleteTokenAttr(buttonElement);}); this.elements.tokenQuery.appendChild(buttonElement); } - deleteTokenAttr(attr){ - // let tokenQuery = this.elements.tokenQuery.childNodes; - // console.log(tokenQuery); - // console.log(this.elements.tokenQuery); - console.log(this.elements.tokenQuery.childNodes.length); + deleteTokenAttr(attr) { if (this.elements.tokenQuery.childNodes.length < 2) { this.elements.tokenQuery.removeChild(attr); this.wordBuilder(); @@ -488,30 +494,30 @@ class ConcordanceQueryBuilder { addToken() { let c; - let tokenQueryContent = ""; //for ButtonFactory(prettyText) - let tokenQueryText = ""; //for ButtonFactory(queryText) + let tokenQueryContent = ''; //for ButtonFactory(prettyText) + let tokenQueryText = ''; //for ButtonFactory(queryText) this.elements.cancelBool = false; let emptyTokenCheck = false; - if (this.elements.ignoreCase.checked){ + if (this.elements.ignoreCase.checked) { c = ' %c'; - }else{ + } else { c = ''; } - for (let element of this.elements.tokenQuery.childNodes){ + for (let element of this.elements.tokenQuery.childNodes) { tokenQueryContent += ' ' + element.firstChild.data + ' '; tokenQueryText += decodeURI(element.dataset.tokentext); - if (element.innerText.indexOf("empty token") !== -1){ + if (element.innerText.indexOf('empty token') !== -1) { emptyTokenCheck = true; } } - if (this.elements.tokenQueryFilled === false){ + if (this.elements.tokenQueryFilled === false) { switch (this.elements.positionalAttr.value) { - case "word": - if (this.elements.wordInput.value === "") { + case 'word': + if (this.elements.wordInput.value === '') { this.disableTokenSubmit(); } else { tokenQueryContent += `word=${this.elements.wordInput.value}${c}`; @@ -519,8 +525,8 @@ class ConcordanceQueryBuilder { this.elements.wordInput.value = ''; } break; - case "lemma": - if (this.elements.lemmaInput.value === "") { + case 'lemma': + if (this.elements.lemmaInput.value === '') { this.disableTokenSubmit(); } else { tokenQueryContent += `lemma=${this.elements.lemmaInput.value}${c}`; @@ -528,8 +534,8 @@ class ConcordanceQueryBuilder { this.elements.lemmaInput.value = ''; } break; - case "english-pos": - if (this.elements.englishPos.value === "default") { + case 'english-pos': + if (this.elements.englishPos.value === 'default') { this.disableTokenSubmit(); } else { tokenQueryContent += `pos=${this.elements.englishPos.value}`; @@ -537,8 +543,8 @@ class ConcordanceQueryBuilder { this.elements.englishPos.value = ''; } break; - case "german-pos": - if (this.elements.germanPos.value === "default") { + case 'german-pos': + if (this.elements.germanPos.value === 'default') { this.disableTokenSubmit(); } else { tokenQueryContent += `pos=${this.elements.germanPos.value}`; @@ -546,8 +552,8 @@ class ConcordanceQueryBuilder { this.elements.germanPos.value = ''; } break; - case "simple-pos-button": - if (this.elements.simplePos.value === "default") { + case 'simple-pos-button': + if (this.elements.simplePos.value === 'default') { this.disableTokenSubmit(); } else { tokenQueryContent += `simple_pos=${this.elements.simplePos.value}`; @@ -562,25 +568,25 @@ class ConcordanceQueryBuilder { } // cancelBool looks in disableTokenSubmit() whether a value is passed. If the input fields/dropdowns are empty (cancelBool === true), no token is added. - if (this.elements.cancelBool === false){ + if (this.elements.cancelBool === false) { // Square brackets are added only if it is not an empty token (where they are already present). if (emptyTokenCheck === false) { tokenQueryText = '[' + tokenQueryText + ']'; - } + } this.buttonfactory('token', tokenQueryContent, tokenQueryText); this.hideEverything(); this.elements.positionalAttrArea.classList.add('hide'); - this.elements.tokenQuery.innerHTML = ""; + this.elements.tokenQuery.innerHTML = ''; } } disableTokenSubmit() { this.elements.cancelBool = true; - this.elements.tokenSubmitButton.classList.add("red"); + this.elements.tokenSubmitButton.classList.add('red'); this.elements.noValueMessage.classList.remove('hide'); setTimeout(() => { - this.elements.tokenSubmitButton.classList.remove("red"); + this.elements.tokenSubmitButton.classList.remove('red'); }, 500); setTimeout(() => { this.elements.noValueMessage.classList.add('hide'); @@ -592,75 +598,75 @@ class ConcordanceQueryBuilder { //#region Dropdown Select Handler wordBuilder() { this.hideEverything(); - this.elements.wordInput.value = ""; - this.elements.wordBuilder.classList.remove("hide"); - this.elements.inputOptions.classList.remove("hide"); + this.elements.wordInput.value = ''; + this.elements.wordBuilder.classList.remove('hide'); + this.elements.inputOptions.classList.remove('hide'); this.elements.incidenceModifiersButton.classList.remove('hide'); - this.elements.conditionContainer.classList.remove("hide"); - this.elements.ignoreCaseCheckbox.classList.remove("hide"); + this.elements.conditionContainer.classList.remove('hide'); + this.elements.ignoreCaseCheckbox.classList.remove('hide'); // Resets materialize select field to default value let SelectInstance = M.FormSelect.getInstance(this.elements.positionalAttr); - SelectInstance.input.value = "word"; - this.elements.positionalAttr.value = "word"; + SelectInstance.input.value = 'word'; + this.elements.positionalAttr.value = 'word'; } lemmaBuilder() { this.hideEverything(); - this.elements.lemmaBuilder.classList.remove("hide"); - this.elements.inputOptions.classList.remove("hide"); + this.elements.lemmaBuilder.classList.remove('hide'); + this.elements.inputOptions.classList.remove('hide'); this.elements.incidenceModifiersButton.classList.remove('hide'); - this.elements.conditionContainer.classList.remove("hide"); - this.elements.ignoreCaseCheckbox.classList.remove("hide"); + this.elements.conditionContainer.classList.remove('hide'); + this.elements.ignoreCaseCheckbox.classList.remove('hide'); } englishPosHandler() { this.hideEverything(); - this.elements.englishPosBuilder.classList.remove("hide"); + this.elements.englishPosBuilder.classList.remove('hide'); this.elements.incidenceModifiersButton.classList.remove('hide'); - this.elements.conditionContainer.classList.remove("hide"); + this.elements.conditionContainer.classList.remove('hide'); // Resets materialize select dropdown let selectInstance = M.FormSelect.getInstance(this.elements.englishPos); - selectInstance.input.value = "English pos tagset"; - this.elements.englishPos.value = "default"; + selectInstance.input.value = 'English pos tagset'; + this.elements.englishPos.value = 'default'; } germanPosHandler() { this.hideEverything(); - this.elements.germanPosBuilder.classList.remove("hide"); + this.elements.germanPosBuilder.classList.remove('hide'); this.elements.incidenceModifiersButton.classList.remove('hide'); - this.elements.conditionContainer.classList.remove("hide"); + this.elements.conditionContainer.classList.remove('hide'); // Resets materialize select dropdown let selectInstance = M.FormSelect.getInstance(this.elements.germanPos); - selectInstance.input.value = "German pos tagset"; - this.elements.germanPos.value = "default"; + selectInstance.input.value = 'German pos tagset'; + this.elements.germanPos.value = 'default'; } simplePosBuilder() { this.hideEverything(); - this.elements.simplePosBuilder.classList.remove("hide"); + this.elements.simplePosBuilder.classList.remove('hide'); this.elements.incidenceModifiersButton.classList.remove('hide'); - this.elements.conditionContainer.classList.remove("hide"); + this.elements.conditionContainer.classList.remove('hide'); this.elements.simplePos.selectedIndex = 0; // Resets materialize select dropdown let selectInstance = M.FormSelect.getInstance(this.elements.simplePos); - selectInstance.input.value = "simple_pos tagset"; - this.elements.simplePos.value = "default"; + selectInstance.input.value = 'simple_pos tagset'; + this.elements.simplePos.value = 'default'; } emptyTokenHandler() { - this.tokenButtonfactory("empty token", "[]"); + this.tokenButtonfactory('empty token', '[]'); this.elements.tokenQueryFilled = true; this.hideEverything(); this.elements.incidenceModifiersButton.classList.remove('hide'); } //#endregion Dropdown Select Handler - //#region Options to edit your token - Wildcard Charakter, Option Group, Incidence Modifiers, Ignore Case, "and", "or" + //#region Options to edit your token - Wildcard Charakter, Option Group, Incidence Modifiers, Ignore Case, 'and', 'or' inputOptionHandler(elem) { let input; @@ -687,35 +693,35 @@ class ConcordanceQueryBuilder { instance.close(); switch (this.elements.positionalAttr.value) { - case "word": - this.elements.wordInput.value += " {" + this.elements.nInput.value + "}"; + case 'word': + this.elements.wordInput.value += ' {' + this.elements.nInput.value + '}'; break; - case "lemma": - this.elements.lemmaInput.value += " {" + this.elements.nInput.value + "}"; + case 'lemma': + this.elements.lemmaInput.value += ' {' + this.elements.nInput.value + '}'; break; - case "english-pos": + case 'english-pos': this.elements.tokenQueryFilled = true; this.tokenButtonfactory(`pos=${this.elements.englishPos.value}`, `pos="${this.elements.englishPos.value}"`); - this.tokenButtonfactory("{" + this.elements.nInput.value + "}", "{" + this.elements.nInput.value + "}"); - this.elements.englishPosBuilder.classList.add("hide"); - this.elements.incidenceModifiersButton.classList.add("hide"); + this.tokenButtonfactory('{' + this.elements.nInput.value + '}', '{' + this.elements.nInput.value + '}'); + this.elements.englishPosBuilder.classList.add('hide'); + this.elements.incidenceModifiersButton.classList.add('hide'); break; - case "german-pos": + case 'german-pos': this.elements.tokenQueryFilled = true; this.tokenButtonfactory(`pos=${this.elements.germanPos.value}`, `pos="${this.elements.germanPos.value}"`); - this.tokenButtonfactory("{" + this.elements.nInput.value + "}", "{" + this.elements.nInput.value + "}"); - this.elements.germanPosBuilder.classList.add("hide"); - this.elements.incidenceModifiersButton.classList.add("hide"); + this.tokenButtonfactory('{' + this.elements.nInput.value + '}', '{' + this.elements.nInput.value + '}'); + this.elements.germanPosBuilder.classList.add('hide'); + this.elements.incidenceModifiersButton.classList.add('hide'); break; - case "simple-pos-button": + case 'simple-pos-button': this.elements.tokenQueryFilled = true; this.tokenButtonfactory(`simple_pos=${this.elements.simplePos.value}`, `simple_pos="${this.elements.simplePos.value}"`); - this.tokenButtonfactory("{" + this.elements.nInput.value + "}", "{" + this.elements.nInput.value + "}"); - this.elements.simplePosBuilder.classList.add("hide"); - this.elements.incidenceModifiersButton.classList.add("hide"); + this.tokenButtonfactory('{' + this.elements.nInput.value + '}', '{' + this.elements.nInput.value + '}'); + this.elements.simplePosBuilder.classList.add('hide'); + this.elements.incidenceModifiersButton.classList.add('hide'); break; - case "empty-token": - this.tokenButtonfactory("{" + this.elements.nInput.value + "}", "{" + this.elements.nInput.value + "}"); + case 'empty-token': + this.tokenButtonfactory('{' + this.elements.nInput.value + '}', '{' + this.elements.nInput.value + '}'); break; default: break; @@ -728,34 +734,34 @@ class ConcordanceQueryBuilder { instance.close(); switch (this.elements.positionalAttr.value) { - case "word": + case 'word': this.elements.wordInput.value += `{${this.elements.nmInput.value}, ${this.elements.mInput.value}}`; break; - case "lemma": + case 'lemma': this.elements.lemmaInput.value += `{${this.elements.nmInput.value}, ${this.elements.mInput.value}}`; break; - case "english-pos": + case 'english-pos': this.elements.tokenQueryFilled = true; this.tokenButtonfactory(`pos=${this.elements.englishPos.value}`, `pos="${this.elements.englishPos.value}"`); this.tokenButtonfactory(`{${this.elements.nmInput.value}, ${this.elements.mInput.value}}`, `{${this.elements.nmInput.value}, ${this.elements.mInput.value}}`); - this.elements.englishPosBuilder.classList.add("hide"); - this.elements.incidenceModifiersButton.classList.add("hide"); + this.elements.englishPosBuilder.classList.add('hide'); + this.elements.incidenceModifiersButton.classList.add('hide'); break; - case "german-pos": + case 'german-pos': this.elements.tokenQueryFilled = true; this.tokenButtonfactory(`pos=${this.elements.germanPos.value}`, `pos="${this.elements.germanPos.value}"`); this.tokenButtonfactory(`{${this.elements.nmInput.value}, ${this.elements.mInput.value}}`, `{${this.elements.nmInput.value}, ${this.elements.mInput.value}}`); - this.elements.germanPosBuilder.classList.add("hide"); - this.elements.incidenceModifiersButton.classList.add("hide"); + this.elements.germanPosBuilder.classList.add('hide'); + this.elements.incidenceModifiersButton.classList.add('hide'); break; - case "simple-pos-button": + case 'simple-pos-button': this.elements.tokenQueryFilled = true; this.tokenButtonfactory(`simple_pos=${this.elements.simplePos.value}`, `simple_pos="${this.elements.simplePos.value}"`); this.tokenButtonfactory(`{${this.elements.nmInput.value}, ${this.elements.mInput.value}}`, `{${this.elements.nmInput.value}, ${this.elements.mInput.value}}`); - this.elements.simplePosBuilder.classList.add("hide"); - this.elements.incidenceModifiersButton.classList.add("hide"); + this.elements.simplePosBuilder.classList.add('hide'); + this.elements.incidenceModifiersButton.classList.add('hide'); break; - case "empty-token": + case 'empty-token': this.tokenButtonfactory(`{${this.elements.nmInput.value}, ${this.elements.mInput.value}}`, `{${this.elements.nmInput.value}, ${this.elements.mInput.value}}`); break; default: @@ -765,25 +771,25 @@ class ConcordanceQueryBuilder { incidenceModifiersHandler(elem) { // For word and lemma, the incidence modifiers are inserted in the input field. For the others, one or two chips are created which contain the respective value of the token and the incidence modifier. - if (this.elements.positionalAttr.value === "empty-token") { + if (this.elements.positionalAttr.value === 'empty-token') { this.tokenButtonfactory(elem.innerText, elem.dataset.token); - } else if (this.elements.positionalAttr.value === "english-pos") { + } else if (this.elements.positionalAttr.value === 'english-pos') { this.tokenButtonfactory(`pos=${this.elements.englishPos.value}`, `pos="${this.elements.englishPos.value}"`); this.tokenButtonfactory(elem.innerText, elem.dataset.token); - this.elements.englishPosBuilder.classList.add("hide"); - this.elements.incidenceModifiersButton.classList.add("hide"); + this.elements.englishPosBuilder.classList.add('hide'); + this.elements.incidenceModifiersButton.classList.add('hide'); this.elements.tokenQueryFilled = true; - } else if (this.elements.positionalAttr.value === "german-pos") { + } else if (this.elements.positionalAttr.value === 'german-pos') { this.tokenButtonfactory(`pos=${this.elements.germanPos.value}`, `pos="${this.elements.germanPos.value}"`); this.tokenButtonfactory(elem.innerText, elem.dataset.token); - this.elements.germanPosBuilder.classList.add("hide"); - this.elements.incidenceModifiersButton.classList.add("hide"); + this.elements.germanPosBuilder.classList.add('hide'); + this.elements.incidenceModifiersButton.classList.add('hide'); this.elements.tokenQueryFilled = true; - } else if (this.elements.positionalAttr.value === "simple-pos-button") { + } else if (this.elements.positionalAttr.value === 'simple-pos-button') { this.tokenButtonfactory(`simple_pos=${this.elements.simplePos.value}`, `simple_pos="${this.elements.simplePos.value}"`); this.tokenButtonfactory(elem.innerText, elem.dataset.token); - this.elements.simplePosBuilder.classList.add("hide"); - this.elements.incidenceModifiersButton.classList.add("hide"); + this.elements.simplePosBuilder.classList.add('hide'); + this.elements.incidenceModifiersButton.classList.add('hide'); this.elements.tokenQueryFilled = true; } else { let input; @@ -800,11 +806,11 @@ class ConcordanceQueryBuilder { } orHandler() { - this.conditionHandler("or", " | "); + this.conditionHandler('or', ' | '); } andHandler() { - this.conditionHandler("and", " & "); + this.conditionHandler('and', ' & '); } conditionHandler(conditionText, conditionQueryContent) { @@ -813,34 +819,34 @@ class ConcordanceQueryBuilder { let tokenQueryText; let c; - if (this.elements.ignoreCase.checked){ + if (this.elements.ignoreCase.checked) { c = ' %c'; - }else{ + } else { c = ''; } switch (this.elements.positionalAttr.value) { - case "word": + case 'word': tokenQueryContent = `word=${this.elements.wordInput.value}${c}`; tokenQueryText = `word="${this.elements.wordInput.value}"${c}`; this.elements.wordInput.value = ''; break; - case "lemma": + case 'lemma': tokenQueryContent = `lemma=${this.elements.lemmaInput.value}${c}`; - tokenQueryText = `word="${this.elements.lemmaInput.value}"${c}`; + tokenQueryText = `lemma="${this.elements.lemmaInput.value}"${c}`; this.elements.lemmaInput.value = ''; break; - case "english-pos": + case 'english-pos': tokenQueryContent = `pos=${this.elements.englishPos.value}`; tokenQueryText = `pos="${this.elements.englishPos.value}"`; this.elements.englishPos.value = ''; break; - case "german-pos": + case 'german-pos': tokenQueryContent = `pos=${this.elements.germanPos.value}`; tokenQueryText = `pos="${this.elements.germanPos.value}"`; this.elements.germanPos.value = ''; break; - case "simple-pos-button": + case 'simple-pos-button': tokenQueryContent = `simple_pos=${this.elements.simplePos.value}`; tokenQueryText = `simple_pos="${this.elements.simplePos.value}"`; this.elements.simplePos.value = ''; @@ -855,14 +861,19 @@ class ConcordanceQueryBuilder { this.wordBuilder(); } - //#endregion Options to edit your token - Wildcard Charakter, Option Group, Incidence Modifiers, Ignore Case, "and", "or" + //#endregion Options to edit your token - Wildcard Charakter, Option Group, Incidence Modifiers, Ignore Case, 'and', 'or' //#endregion Token Attribute Builder Functions + + // ########################################################################## + // ############ Structural Attribute Builder Functions ###################### + // ########################################################################## + //#region Structural Attribute Builder Functions addSentence() { this.hideEverything(); - if(this.elements.sentence.text === 'End Sentence') { + if (this.elements.sentence.text === 'End Sentence') { this.buttonfactory('end-sentence', 'Sentence End', ''); this.elements.sentence.innerHTML = 'Sentence'; } else { @@ -884,8 +895,8 @@ class ConcordanceQueryBuilder { this.elements.entity.innerHTML = 'Entity'; } else { this.hideEverything(); - this.elements.entityBuilder.classList.remove("hide"); - window.location.href = "#entity-builder"; + this.elements.entityBuilder.classList.remove('hide'); + window.location.href = '#entity-builder'; } } @@ -897,8 +908,8 @@ class ConcordanceQueryBuilder { // Resets materialize select dropdown let SelectInstance = M.FormSelect.getInstance(this.elements.englishEntType); - SelectInstance.input.value = "English ent_type"; - this.elements.englishEntType.value = "default"; + SelectInstance.input.value = 'English ent_type'; + this.elements.englishEntType.value = 'default'; } germanEntTypeHandler() { @@ -909,8 +920,8 @@ class ConcordanceQueryBuilder { // Resets materialize select dropdown let SelectInstance = M.FormSelect.getInstance(this.elements.germanEntType); - SelectInstance.input.value = "German ent_type"; - this.elements.germanEntType.value = "default"; + SelectInstance.input.value = 'German ent_type'; + this.elements.germanEntType.value = 'default'; } emptyEntityButton() { @@ -922,14 +933,14 @@ class ConcordanceQueryBuilder { addTextAnnotation() { this.hideEverything(); - this.elements.textAnnotationBuilder.classList.remove("hide"); - window.location.href = "#text-annotation-builder"; + this.elements.textAnnotationBuilder.classList.remove('hide'); + window.location.href = '#text-annotation-builder'; // Resets materialize select dropdown let SelectInstance = M.FormSelect.getInstance(this.elements.textAnnotationOptions); - SelectInstance.input.value = "address"; - this.elements.textAnnotationOptions.value = "address"; - this.elements.textAnnotationInput.value= ""; + SelectInstance.input.value = 'address'; + this.elements.textAnnotationOptions.value = 'address'; + this.elements.textAnnotationInput.value= ''; } textAnnotationSubmitHandler() { @@ -948,12 +959,6 @@ class ConcordanceQueryBuilder { this.hideEverything(); } } - - -//#endregion Structural Attribute Builder Functions - - - - + //#endregion Structural Attribute Builder Functions + } - diff --git a/app/templates/_styles.html.j2 b/app/templates/_styles.html.j2 index 2c1ea8f8..cb047f8f 100644 --- a/app/templates/_styles.html.j2 +++ b/app/templates/_styles.html.j2 @@ -4,6 +4,7 @@ + {%- assets filters='pyscss', output='gen/app.%(version)s.css', diff --git a/app/templates/contributions/contribute.html.j2 b/app/templates/contributions/contribute.html.j2 new file mode 100644 index 00000000..6789e1f8 --- /dev/null +++ b/app/templates/contributions/contribute.html.j2 @@ -0,0 +1,32 @@ +{% extends "base.html.j2" %} +{% import "materialize/wtf.html.j2" as wtf %} + + +{% block page_content %} +
+
+
+

{{ title }}

+

+ In order to add a new model, please fill in the form below. +

+ +
+
+ {{ form.hidden_tag() }} + {{ wtf.render_field(form.title) }} + {{ wtf.render_field(form.description) }} + {{ wtf.render_field(form.publisher) }} + {{ wtf.render_field(form.publisher_url) }} + {{ wtf.render_field(form.publishing_url) }} + {{ wtf.render_field(form.publishing_year) }} + {{ wtf.render_field(form.shared) }} + {{ wtf.render_field(form.version) }} + {{ wtf.render_field(form.compatible_service_versions) }} + {{ wtf.render_field(form.submit, class_='width-100', material_icon='send') }} + +
+
+
+
+{% endblock page_content %} \ No newline at end of file diff --git a/app/templates/corpora/analyse_corpus.concordance.html.j2 b/app/templates/corpora/analyse_corpus.concordance.html.j2 index 0fc14597..e19a3728 100644 --- a/app/templates/corpora/analyse_corpus.concordance.html.j2 +++ b/app/templates/corpora/analyse_corpus.concordance.html.j2 @@ -64,9 +64,9 @@

 

- + build - Query builder + Query builder (beta)
+ + + + + {% endblock modals %} {% block scripts %} @@ -256,6 +611,7 @@ const corpusAnalysisApp = new CorpusAnalysisApp({{ corpus.hashid|tojson }}); const corpusAnalysisConcordance = new CorpusAnalysisConcordance(corpusAnalysisApp); const corpusAnalysisReader = new CorpusAnalysisReader(corpusAnalysisApp); +const concordanceQueryBuilder = new ConcordanceQueryBuilder(); corpusAnalysisApp.init(); diff --git a/app/templates/main/manual/_09_query_builder.html.j2 b/app/templates/main/manual/_09_query_builder.html.j2 index f22e5369..ff3544eb 100644 --- a/app/templates/main/manual/_09_query_builder.html.j2 +++ b/app/templates/main/manual/_09_query_builder.html.j2 @@ -38,14 +38,14 @@ under the tab "Examples".

Submit button on the right. You can also use the options below to modify your token request before pressing the submit button. These options are explained further here.

- word and lemma explanation + word and lemma explanation

English pos, german pos or simple_pos

You can choose between the options "english pos", "german pos" and "simple_pos" to search for different parts-of-speech. You can find an overview of all tags under the "Tagsets" tab.

- part-of-speech-tag explanation + part-of-speech-tag explanation

Empty Token

Here you can search for an empty token. This selection should never stand @@ -75,7 +75,7 @@ under the tab "Examples".

With an option group you can search for different variants of a token. The variants are not limited, so you can manually enter more options in the same format. "Option1" and "option2" must be replaced accordingly.

- option group explanation + option group explanation


@@ -100,7 +100,7 @@ under the tab "Examples".

it will be displayed. Note that "and" is not responsible for lining up tokens in this case. For this you can simply string them together:
[word="I"] [word="will" & simple_pos="VERB"] [word="go"].

- part-of-speech-tag explanation + OR/AND explanation


@@ -134,7 +134,7 @@ under the tab "Examples".

the respective abbreviations under the tab "Tagsets".
You can also search for unspecified entities by selecting "Add entity of any type".

To close the entity query you started, you have to click the entity button one more time. This will make the
Entity End
element appear in your query. - entity explanation + entity explanation


@@ -142,7 +142,7 @@ under the tab "Examples".

With the meta data you can annotate your text and add specific conditions. You can select a category on the left and enter your desired value on the right. The selected metadata will apply to your entire request and will be added at the end.

- meta data explanation + meta data explanation


@@ -158,11 +158,11 @@ under the tab "Examples".

Deleting the elements

You can delete the added elements from the query by clicking the X behind the respective content.

- delete explanation + delete explanation

Move the elements of your query

You can drag and drop elements to customize your query.

- Drag&Drop explanation + Drag&Drop explanation diff --git a/app/templates/services/tesseract_ocr_pipeline.html.j2 b/app/templates/services/tesseract_ocr_pipeline.html.j2 index c38c3965..982265bc 100644 --- a/app/templates/services/tesseract_ocr_pipeline.html.j2 +++ b/app/templates/services/tesseract_ocr_pipeline.html.j2 @@ -160,8 +160,8 @@ - {% for m in tesseract_ocr_models %} - + {% for m in tesseract_ocr_pipeline_models %} + {{ m.title }} {% if m.description == '' %} Description is not available. diff --git a/app/templates/services/transkribus_htr_pipeline.html.j2 b/app/templates/services/transkribus_htr_pipeline.html.j2 index 7aedbd4f..d54d9906 100644 --- a/app/templates/services/transkribus_htr_pipeline.html.j2 +++ b/app/templates/services/transkribus_htr_pipeline.html.j2 @@ -156,15 +156,13 @@