From c223f072894cde21dba3173031ccfc150bc9877a Mon Sep 17 00:00:00 2001 From: Patrick Jentsch Date: Thu, 7 Nov 2024 08:57:32 +0100 Subject: [PATCH] Codestyle enhancements --- app/jobs/__init__.py | 4 ++-- app/jobs/handle_corpora.py | 29 +++++++++++++---------------- app/jobs/handle_jobs.py | 26 ++++++++++++-------------- 3 files changed, 27 insertions(+), 32 deletions(-) diff --git a/app/jobs/__init__.py b/app/jobs/__init__.py index 26b42130..1d1bfdbf 100644 --- a/app/jobs/__init__.py +++ b/app/jobs/__init__.py @@ -1,2 +1,2 @@ -from .handle_corpora import job as handle_corpora -from .handle_jobs import job as handle_jobs +from .handle_corpora import handle_corpora +from .handle_jobs import handle_jobs diff --git a/app/jobs/handle_corpora.py b/app/jobs/handle_corpora.py index 8564d047..eece43e2 100644 --- a/app/jobs/handle_corpora.py +++ b/app/jobs/handle_corpora.py @@ -1,12 +1,12 @@ -from app import db, docker_client, scheduler -from app.models import Corpus, CorpusStatus from flask import current_app import docker import os import shutil +from app import db, docker_client, scheduler +from app.models import Corpus, CorpusStatus -def job(): +def handle_corpora(): with scheduler.app.app_context(): _handle_corpora() @@ -21,14 +21,14 @@ def _handle_corpora(): for corpus in [x for x in corpora if x.status == CorpusStatus.RUNNING_ANALYSIS_SESSION and x.num_analysis_sessions == 0]: corpus.status = CorpusStatus.CANCELING_ANALYSIS_SESSION for corpus in [x for x in corpora if x.status == CorpusStatus.RUNNING_ANALYSIS_SESSION]: - _checkout_analysing_corpus_container(corpus) + _checkout_cqpserver_container(corpus) for corpus in [x for x in corpora if x.status == CorpusStatus.STARTING_ANALYSIS_SESSION]: _create_cqpserver_container(corpus) for corpus in [x for x in corpora if x.status == CorpusStatus.CANCELING_ANALYSIS_SESSION]: _remove_cqpserver_container(corpus) db.session.commit() -def _create_build_corpus_service(corpus): +def _create_build_corpus_service(corpus: Corpus): ''' # Docker service settings # ''' ''' ## Command ## ''' command = ['bash', '-c'] @@ -53,9 +53,7 @@ def _create_build_corpus_service(corpus): image = f'{current_app.config["NOPAQUE_DOCKER_IMAGE_PREFIX"]}cwb:r1879' ''' ## Labels ## ''' labels = { - 'origin': current_app.config['SERVER_NAME'], - 'type': 'corpus.build', - 'corpus_id': str(corpus.id) + 'nopaque.server_name': current_app.config['SERVER_NAME'] } ''' ## Mounts ## ''' mounts = [] @@ -100,7 +98,7 @@ def _create_build_corpus_service(corpus): return corpus.status = CorpusStatus.QUEUED -def _checkout_build_corpus_service(corpus): +def _checkout_build_corpus_service(corpus: Corpus): service_name = f'build-corpus_{corpus.id}' try: service = docker_client.services.get(service_name) @@ -128,8 +126,7 @@ def _checkout_build_corpus_service(corpus): except docker.errors.DockerException as e: current_app.logger.error(f'Remove service "{service_name}" failed: {e}') -def _create_cqpserver_container(corpus): - ''' # Docker container settings # ''' +def _create_cqpserver_container(corpus: Corpus): ''' ## Command ## ''' command = [] command.append( @@ -146,7 +143,7 @@ def _create_cqpserver_container(corpus): ''' ## Image ## ''' image = f'{current_app.config["NOPAQUE_DOCKER_IMAGE_PREFIX"]}cwb:r1879' ''' ## Name ## ''' - name = f'cqpserver_{corpus.id}' + name = f'nopaque-cqpserver-{corpus.id}' ''' ## Network ## ''' network = f'{current_app.config["NOPAQUE_DOCKER_NETWORK_NAME"]}' ''' ## Volumes ## ''' @@ -203,8 +200,8 @@ def _create_cqpserver_container(corpus): return corpus.status = CorpusStatus.RUNNING_ANALYSIS_SESSION -def _checkout_analysing_corpus_container(corpus): - container_name = f'cqpserver_{corpus.id}' +def _checkout_cqpserver_container(corpus: Corpus): + container_name = f'nopaque-cqpserver-{corpus.id}' try: docker_client.containers.get(container_name) except docker.errors.NotFound as e: @@ -214,8 +211,8 @@ def _checkout_analysing_corpus_container(corpus): except docker.errors.DockerException as e: current_app.logger.error(f'Get container "{container_name}" failed: {e}') -def _remove_cqpserver_container(corpus): - container_name = f'cqpserver_{corpus.id}' +def _remove_cqpserver_container(corpus: Corpus): + container_name = f'nopaque-cqpserver-{corpus.id}' try: container = docker_client.containers.get(container_name) except docker.errors.NotFound: diff --git a/app/jobs/handle_jobs.py b/app/jobs/handle_jobs.py index 333ff8b5..da78ebb6 100644 --- a/app/jobs/handle_jobs.py +++ b/app/jobs/handle_jobs.py @@ -1,3 +1,10 @@ +from datetime import datetime +from flask import current_app +from werkzeug.utils import secure_filename +import docker +import json +import os +import shutil from app import db, docker_client, hashids, scheduler from app.models import ( Job, @@ -6,16 +13,9 @@ from app.models import ( TesseractOCRPipelineModel, SpaCyNLPPipelineModel ) -from datetime import datetime -from flask import current_app -from werkzeug.utils import secure_filename -import docker -import json -import os -import shutil -def job(): +def handle_jobs(): with scheduler.app.app_context(): _handle_jobs() @@ -29,7 +29,7 @@ def _handle_jobs(): _remove_job_service(job) db.session.commit() -def _create_job_service(job): +def _create_job_service(job: Job): ''' # Docker service settings # ''' ''' ## Service specific settings ## ''' if job.service == 'file-setup-pipeline': @@ -86,9 +86,7 @@ def _create_job_service(job): constraints = ['node.role==worker'] ''' ## Labels ## ''' labels = { - 'origin': current_app.config['SERVER_NAME'], - 'type': 'job', - 'job_id': str(job.id) + 'origin': current_app.config['SERVER_NAME'] } ''' ## Mounts ## ''' mounts = [] @@ -169,7 +167,7 @@ def _create_job_service(job): return job.status = JobStatus.QUEUED -def _checkout_job_service(job): +def _checkout_job_service(job: Job): service_name = f'job_{job.id}' try: service = docker_client.services.get(service_name) @@ -218,7 +216,7 @@ def _checkout_job_service(job): except docker.errors.DockerException as e: current_app.logger.error(f'Remove service "{service_name}" failed: {e}') -def _remove_job_service(job): +def _remove_job_service(job: Job): service_name = f'job_{job.id}' try: service = docker_client.services.get(service_name)