sporada/nopaque
1
0
Fork 0
mirror of https://gitlab.ub.uni-bielefeld.de/sfb1288inf/nopaque.git synced 2024-12-24 10:34:17 +00:00
Code Issues Projects Releases Wiki Activity

Codestyle update (less use of #noqa)

Browse Source
This commit is contained in:
Patrick Jentsch 2020-11-19 12:31:29 +01:00
parent 66887f21b8
commit ab5db4ae83
2 changed files with 110 additions and 67 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

105
web/app/tasks/corpus_utils.py
View File

@ -25,13 +25,16 @@ def create_build_corpus_service(corpus):
'name': 'build-corpus_{}'.format(corpus.id), 'name': 'build-corpus_{}'.format(corpus.id),
'restart_policy': docker.types.RestartPolicy() 'restart_policy': docker.types.RestartPolicy()
} }
service_image = 'gitlab.ub.uni-bielefeld.de:4567/sfb1288inf/cqpserver:latest' # noqa service_image = \
'gitlab.ub.uni-bielefeld.de:4567/sfb1288inf/cqpserver:latest'
try: try:
docker_client.services.create(service_image, **service_kwargs) docker_client.services.create(service_image, **service_kwargs)
except docker.errors.APIError as e: except docker.errors.APIError as e:
logging.error('Create "{}" service raised '.format(service_kwargs['name']) # noqa logging.error(
+ '[docker-APIError] The server returned an error. ' 'Create "{}" service raised '.format(service_kwargs['name'])
+ 'Details: {}'.format(e)) + '"docker.errors.APIError" The server returned an error. '
+ 'Details: {}'.format(e)
)
else: else:
corpus.status = 'queued' corpus.status = 'queued'
@ -41,18 +44,24 @@ def checkout_build_corpus_service(corpus):
try: try:
service = docker_client.services.get(service_name) service = docker_client.services.get(service_name)
except docker.errors.NotFound: except docker.errors.NotFound:
logging.error('Get "{}" service raised '.format(service_name) logging.error(
+ '[docker-NotFound] The service does not exist. ' 'Get "{}" service raised '.format(service_name)
+ '(corpus.status: {} -> failed)'.format(corpus.status)) + '"docker.errors.NotFound" The service does not exist. '
+ '(corpus.status: {} -> failed)'.format(corpus.status)
)
corpus.status = 'failed' corpus.status = 'failed'
except docker.errors.APIError as e: except docker.errors.APIError as e:
logging.error('Get "{}" service raised '.format(service_name) logging.error(
+ '[docker-APIError] The server returned an error. ' 'Get "{}" service raised '.format(service_name)
+ 'Details: {}'.format(e)) + '"docker.errors.APIError" The server returned an error. '
+ 'Details: {}'.format(e)
)
except docker.errors.InvalidVersion: except docker.errors.InvalidVersion:
logging.error('Get "{}" service raised '.format(service_name) logging.error(
+ '[docker-InvalidVersion] One of the arguments is ' 'Get "{}" service raised '.format(service_name)
+ 'not supported with the current API version.') + '"docker.errors.InvalidVersion" One of the arguments is '
+ 'not supported with the current API version.'
)
else: else:
service_tasks = service.tasks() service_tasks = service.tasks()
if not service_tasks: if not service_tasks:
@ -60,13 +69,16 @@ def checkout_build_corpus_service(corpus):
task_state = service_tasks[0].get('Status').get('State') task_state = service_tasks[0].get('Status').get('State')
if corpus.status == 'queued' and task_state != 'pending': if corpus.status == 'queued' and task_state != 'pending':
corpus.status = 'running' corpus.status = 'running'
elif corpus.status == 'running' and task_state in ['complete', 'failed']: # noqa elif (corpus.status == 'running'
and task_state in ['complete', 'failed']):
try: try:
service.remove() service.remove()
except docker.errors.APIError as e: except docker.errors.APIError as e:
logging.error('Remove "{}" service raised '.format(service_name) # noqa logging.error(
+ '[docker-APIError] The server returned an error. ' # noqa 'Remove "{}" service raised '.format(service_name)
+ 'Details: {}'.format(e)) + '"docker.errors.APIError" The server returned an error. '
+ 'Details: {}'.format(e)
)
return return
else: else:
corpus.status = 'prepared' if task_state == 'complete' \ corpus.status = 'prepared' if task_state == 'complete' \
@ -84,7 +96,8 @@ def create_cqpserver_container(corpus):
'name': 'cqpserver_{}'.format(corpus.id), 'name': 'cqpserver_{}'.format(corpus.id),
'network': 'nopaque_default' 'network': 'nopaque_default'
} }
container_image = 'gitlab.ub.uni-bielefeld.de:4567/sfb1288inf/cqpserver:latest' # noqa container_image = \
'gitlab.ub.uni-bielefeld.de:4567/sfb1288inf/cqpserver:latest'
# Check if a cqpserver container already exists. If this is the case, # Check if a cqpserver container already exists. If this is the case,
# remove it and create a new one # remove it and create a new one
try: try:
@ -92,35 +105,45 @@ def create_cqpserver_container(corpus):
except docker.errors.NotFound: except docker.errors.NotFound:
pass pass
except docker.errors.APIError as e: except docker.errors.APIError as e:
logging.error('Get "{}" container raised '.format(container_kwargs['name']) logging.error(
+ '[docker-APIError] The server returned an error. ' 'Get "{}" container raised '.format(container_kwargs['name'])
+ 'Details: {}'.format(e)) + '"docker.errors.APIError" The server returned an error. '
+ 'Details: {}'.format(e)
)
return return
else: else:
try: try:
container.remove(force=True) container.remove(force=True)
except docker.errors.APIError as e: except docker.errors.APIError as e:
logging.error('Remove "{}" container raised '.format(container_kwargs['name']) logging.error(
+ '[docker-APIError] The server returned an error. ' 'Remove "{}" container raised '.format(container_kwargs['name']) # noqa
+ 'Details: {}'.format(e)) + '"docker.errors.APIError" The server returned an error. '
+ 'Details: {}'.format(e)
)
return return
try: try:
docker_client.containers.run(container_image, **container_kwargs) docker_client.containers.run(container_image, **container_kwargs)
except docker.errors.ContainerError: except docker.errors.ContainerError:
# This case should not occur, because detach is True. # This case should not occur, because detach is True.
logging.error('Run "{}" container raised '.format(container_kwargs['name']) logging.error(
+ '[docker-ContainerError] The container exits with a ' 'Run "{}" container raised '.format(container_kwargs['name'])
+ 'non-zero exit code and detach is False.') + '"docker.errors.ContainerError" The container exits with a '
+ 'non-zero exit code and detach is False.'
)
corpus.status = 'failed' corpus.status = 'failed'
except docker.errors.ImageNotFound: except docker.errors.ImageNotFound:
logging.error('Run "{}" container raised '.format(container_kwargs['name']) logging.error(
+ '[docker-ImageNotFound] The specified image does not ' 'Run "{}" container raised '.format(container_kwargs['name'])
+ 'exist.') + '"docker.errors.ImageNotFound" The specified image does not '
+ 'exist.'
)
corpus.status = 'failed' corpus.status = 'failed'
except docker.errors.APIError as e: except docker.errors.APIError as e:
logging.error('Run "{}" container raised '.format(container_kwargs['name']) logging.error(
+ '[docker-APIError] The server returned an error. ' 'Run "{}" container raised '.format(container_kwargs['name'])
+ 'Details: {}'.format(e)) + '"docker.errors.APIError" The server returned an error. '
+ 'Details: {}'.format(e)
)
else: else:
corpus.status = 'analysing' corpus.status = 'analysing'
@ -132,16 +155,20 @@ def remove_cqpserver_container(corpus):
except docker.errors.NotFound: except docker.errors.NotFound:
pass pass
except docker.errors.APIError as e: except docker.errors.APIError as e:
logging.error('Get "{}" container raised '.format(container_name) logging.error(
+ '[docker-APIError] The server returned an error. ' 'Get "{}" container raised '.format(container_name)
+ 'Details: {}'.format(e)) + '"docker.errors.APIError" The server returned an error. '
+ 'Details: {}'.format(e)
)
return return
else: else:
try: try:
container.remove(force=True) container.remove(force=True)
except docker.errors.APIError as e: except docker.errors.APIError as e:
logging.error('Remove "{}" container raised '.format(container_name) logging.error(
+ '[docker-APIError] The server returned an error. ' 'Remove "{}" container raised '.format(container_name)
+ 'Details: {}'.format(e)) + '"docker.errors.APIError" The server returned an error. '
+ 'Details: {}'.format(e)
)
return return
corpus.status = 'prepared' corpus.status = 'prepared'

72
web/app/tasks/job_utils.py
View File

@ -29,14 +29,16 @@ def create_job_service(job):
mem_reservation=job.mem_mb * (10 ** 6) mem_reservation=job.mem_mb * (10 ** 6)
), ),
'restart_policy': docker.types.RestartPolicy()} 'restart_policy': docker.types.RestartPolicy()}
service_image = ('gitlab.ub.uni-bielefeld.de:4567/sfb1288inf/' service_image = 'gitlab.ub.uni-bielefeld.de:4567/sfb1288inf/{}:{}'.format(
+ job.service + ':' + job.service_version) job.service, job.service_version)
try: try:
docker_client.services.create(service_image, **service_kwargs) docker_client.services.create(service_image, **service_kwargs)
except docker.errors.APIError as e: except docker.errors.APIError as e:
logging.error('Create "{}" service raised '.format(service_kwargs['name']) # noqa logging.error(
+ '[docker-APIError] The server returned an error. ' 'Create "{}" service raised '.format(service_kwargs['name'])
+ 'Details: {}'.format(e)) + '"docker.errors.APIError" The server returned an error. '
+ 'Details: {}'.format(e)
)
return return
else: else:
job.status = 'queued' job.status = 'queued'
@ -50,18 +52,22 @@ def checkout_job_service(job):
service = docker_client.services.get(service_name) service = docker_client.services.get(service_name)
except docker.errors.NotFound: except docker.errors.NotFound:
logging.error('Get "{}" service raised '.format(service_name) logging.error('Get "{}" service raised '.format(service_name)
+ '[docker-NotFound] The service does not exist. ' + '"docker.errors.NotFound" The service does not exist. '
+ '(job.status: {} -> failed)'.format(job.status)) + '(job.status: {} -> failed)'.format(job.status))
job.status = 'failed' job.status = 'failed'
except docker.errors.APIError as e: except docker.errors.APIError as e:
logging.error('Get "{}" service raised '.format(service_name) logging.error(
+ '[docker-APIError] The server returned an error. ' 'Get "{}" service raised '.format(service_name)
+ 'Details: {}'.format(e)) + '"docker.errors.APIError" The server returned an error. '
+ 'Details: {}'.format(e)
)
return return
except docker.errors.InvalidVersion: except docker.errors.InvalidVersion:
logging.error('Get "{}" service raised '.format(service_name) logging.error(
+ '[docker-InvalidVersion] One of the arguments is ' 'Get "{}" service raised '.format(service_name)
+ 'not supported with the current API version.') + '"docker.errors.InvalidVersion" One of the arguments is '
+ 'not supported with the current API version.'
)
return return
else: else:
service_tasks = service.tasks() service_tasks = service.tasks()
@ -70,13 +76,15 @@ def checkout_job_service(job):
task_state = service_tasks[0].get('Status').get('State') task_state = service_tasks[0].get('Status').get('State')
if job.status == 'queued' and task_state != 'pending': if job.status == 'queued' and task_state != 'pending':
job.status = 'running' job.status = 'running'
elif job.status == 'running' and task_state in ['complete', 'failed']: # noqa elif job.status == 'running' and task_state in ['complete', 'failed']:
try: try:
service.remove() service.remove()
except docker.errors.APIError as e: except docker.errors.APIError as e:
logging.error('Remove "{}" service raised '.format(service_name) # noqa logging.error(
+ '[docker-APIError] The server returned an error. ' # noqa 'Remove "{}" service raised '.format(service_name)
+ 'Details: {}'.format(e)) + '"docker.errors.APIError" The server returned an error. '
+ 'Details: {}'.format(e)
)
return return
else: else:
if task_state == 'complete': if task_state == 'complete':
@ -99,29 +107,37 @@ def remove_job_service(job):
except docker.errors.NotFound: except docker.errors.NotFound:
job.status = 'canceled' job.status = 'canceled'
except docker.errors.APIError as e: except docker.errors.APIError as e:
logging.error('Get "{}" service raised '.format(service_name) logging.error(
+ '[docker-APIError] The server returned an error. ' 'Get "{}" service raised '.format(service_name)
+ 'Details: {}'.format(e)) + '"docker.errors.APIError" The server returned an error. '
+ 'Details: {}'.format(e)
)
return return
except docker.errors.InvalidVersion: except docker.errors.InvalidVersion:
logging.error('Get "{}" service raised '.format(service_name) logging.error(
+ '[docker-InvalidVersion] One of the arguments is ' 'Get "{}" service raised '.format(service_name)
+ 'not supported with the current API version.') + '"docker.errors.InvalidVersion" One of the arguments is '
+ 'not supported with the current API version.'
)
return return
else: else:
try: try:
service.update(mounts=None) service.update(mounts=None)
except docker.errors.APIError as e: except docker.errors.APIError as e:
logging.error('Update "{}" service raised '.format(service_name) # noqa logging.error(
+ '[docker-APIError] The server returned an error. ' # noqa 'Update "{}" service raised '.format(service_name)
+ 'Details: {}'.format(e)) + '"docker.errors.APIError" The server returned an error. '
+ 'Details: {}'.format(e)
)
return return
try: try:
service.remove() service.remove()
except docker.errors.APIError as e: except docker.errors.APIError as e:
logging.error('Remove "{}" service raised '.format(service_name) # noqa logging.error(
+ '[docker-APIError] The server returned an error. ' # noqa 'Remove "{}" service raised '.format(service_name)
+ 'Details: {}'.format(e)) + '"docker.errors.APIError" The server returned an error. '
+ 'Details: {}'.format(e)
)
def send_notification(job): def send_notification(job):