sporada/nopaque
1
0
Fork 0
mirror of https://gitlab.ub.uni-bielefeld.de/sfb1288inf/nopaque.git synced 2024-12-25 10:54:18 +00:00
Code Issues Projects Releases Wiki Activity

Codestyle enhancements

Browse Source
This commit is contained in:
Patrick Jentsch 2024-11-07 08:57:32 +01:00
parent fcb49025e9
commit c223f07289
3 changed files with 27 additions and 32 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
app/jobs/__init__.py
View File

@ -1,2 +1,2 @@
from .handle_corpora import job as handle_corpora
from .handle_jobs import job as handle_jobs
from .handle_corpora import handle_corpora
from .handle_jobs import handle_jobs

29
app/jobs/handle_corpora.py
View File

@ -1,12 +1,12 @@
from app import db, docker_client, scheduler
from app.models import Corpus, CorpusStatus
from flask import current_app
import docker
import os
import shutil
from app import db, docker_client, scheduler
from app.models import Corpus, CorpusStatus
def job():
def handle_corpora():
with scheduler.app.app_context():
_handle_corpora()
@ -21,14 +21,14 @@ def _handle_corpora():
for corpus in [x for x in corpora if x.status == CorpusStatus.RUNNING_ANALYSIS_SESSION and x.num_analysis_sessions == 0]:
corpus.status = CorpusStatus.CANCELING_ANALYSIS_SESSION
for corpus in [x for x in corpora if x.status == CorpusStatus.RUNNING_ANALYSIS_SESSION]:
_checkout_analysing_corpus_container(corpus)
_checkout_cqpserver_container(corpus)
for corpus in [x for x in corpora if x.status == CorpusStatus.STARTING_ANALYSIS_SESSION]:
_create_cqpserver_container(corpus)
for corpus in [x for x in corpora if x.status == CorpusStatus.CANCELING_ANALYSIS_SESSION]:
_remove_cqpserver_container(corpus)
db.session.commit()
def _create_build_corpus_service(corpus):
def _create_build_corpus_service(corpus: Corpus):
''' # Docker service settings # '''
''' ## Command ## '''
command = ['bash', '-c']
@ -53,9 +53,7 @@ def _create_build_corpus_service(corpus):
image = f'{current_app.config["NOPAQUE_DOCKER_IMAGE_PREFIX"]}cwb:r1879'
''' ## Labels ## '''
labels = {
'origin': current_app.config['SERVER_NAME'],
'type': 'corpus.build',
'corpus_id': str(corpus.id)
'nopaque.server_name': current_app.config['SERVER_NAME']
}
''' ## Mounts ## '''
mounts = []
@ -100,7 +98,7 @@ def _create_build_corpus_service(corpus):
return
corpus.status = CorpusStatus.QUEUED
def _checkout_build_corpus_service(corpus):
def _checkout_build_corpus_service(corpus: Corpus):
service_name = f'build-corpus_{corpus.id}'
try:
service = docker_client.services.get(service_name)
@ -128,8 +126,7 @@ def _checkout_build_corpus_service(corpus):
except docker.errors.DockerException as e:
current_app.logger.error(f'Remove service "{service_name}" failed: {e}')
def _create_cqpserver_container(corpus):
''' # Docker container settings # '''
def _create_cqpserver_container(corpus: Corpus):
''' ## Command ## '''
command = []
command.append(
@ -146,7 +143,7 @@ def _create_cqpserver_container(corpus):
''' ## Image ## '''
image = f'{current_app.config["NOPAQUE_DOCKER_IMAGE_PREFIX"]}cwb:r1879'
''' ## Name ## '''
name = f'cqpserver_{corpus.id}'
name = f'nopaque-cqpserver-{corpus.id}'
''' ## Network ## '''
network = f'{current_app.config["NOPAQUE_DOCKER_NETWORK_NAME"]}'
''' ## Volumes ## '''
@ -203,8 +200,8 @@ def _create_cqpserver_container(corpus):
return
corpus.status = CorpusStatus.RUNNING_ANALYSIS_SESSION
def _checkout_analysing_corpus_container(corpus):
container_name = f'cqpserver_{corpus.id}'
def _checkout_cqpserver_container(corpus: Corpus):
container_name = f'nopaque-cqpserver-{corpus.id}'
try:
docker_client.containers.get(container_name)
except docker.errors.NotFound as e:
@ -214,8 +211,8 @@ def _checkout_analysing_corpus_container(corpus):
except docker.errors.DockerException as e:
current_app.logger.error(f'Get container "{container_name}" failed: {e}')
def _remove_cqpserver_container(corpus):
container_name = f'cqpserver_{corpus.id}'
def _remove_cqpserver_container(corpus: Corpus):
container_name = f'nopaque-cqpserver-{corpus.id}'
try:
container = docker_client.containers.get(container_name)
except docker.errors.NotFound:

26
app/jobs/handle_jobs.py
View File

@ -1,3 +1,10 @@
from datetime import datetime
from flask import current_app
from werkzeug.utils import secure_filename
import docker
import json
import os
import shutil
from app import db, docker_client, hashids, scheduler
from app.models import (
Job,
@ -6,16 +13,9 @@ from app.models import (
TesseractOCRPipelineModel,
SpaCyNLPPipelineModel
)
from datetime import datetime
from flask import current_app
from werkzeug.utils import secure_filename
import docker
import json
import os
import shutil
def job():
def handle_jobs():
with scheduler.app.app_context():
_handle_jobs()
@ -29,7 +29,7 @@ def _handle_jobs():
_remove_job_service(job)
db.session.commit()
def _create_job_service(job):
def _create_job_service(job: Job):
''' # Docker service settings # '''
''' ## Service specific settings ## '''
if job.service == 'file-setup-pipeline':
@ -86,9 +86,7 @@ def _create_job_service(job):
constraints = ['node.role==worker']
''' ## Labels ## '''
labels = {
'origin': current_app.config['SERVER_NAME'],
'type': 'job',
'job_id': str(job.id)
'origin': current_app.config['SERVER_NAME']
}
''' ## Mounts ## '''
mounts = []
@ -169,7 +167,7 @@ def _create_job_service(job):
return
job.status = JobStatus.QUEUED
def _checkout_job_service(job):
def _checkout_job_service(job: Job):
service_name = f'job_{job.id}'
try:
service = docker_client.services.get(service_name)
@ -218,7 +216,7 @@ def _checkout_job_service(job):
except docker.errors.DockerException as e:
current_app.logger.error(f'Remove service "{service_name}" failed: {e}')
def _remove_job_service(job):
def _remove_job_service(job: Job):
service_name = f'job_{job.id}'
try:
service = docker_client.services.get(service_name)