diff --git a/app/scheduler_functions.py b/app/scheduler_functions.py index 2a7d141d..8199922d 100644 --- a/app/scheduler_functions.py +++ b/app/scheduler_functions.py @@ -42,25 +42,22 @@ def checkout_jobs(): mem_reservation=job.mem_mb * (10 ** 6) ) _restart_policy = docker.types.RestartPolicy(condition='none') - ''' - ' Create the service with the prepared values. - ' - ' Note: A service reserves hardware ressources. In case no worker - ' node has the required ressources available (not reserved), - ' the service gets queued by the Docker engine until a node - ' is able to meet the requirements. - ''' - service = client.services.create( - _image, - command=_command, - constraints=_constraints, - labels=_labels, - mounts=_mounts, - name=_name, - resources=_resources, - restart_policy=_restart_policy - ) - job.status = 'scheduled' + try: + service = client.services.create( + _image, + command=_command, + constraints=_constraints, + labels=_labels, + mounts=_mounts, + name=_name, + resources=_resources, + restart_policy=_restart_policy + ) + job.status = 'scheduled' + except docker.errors.APIError: + job.status = 'failed' + print('[ERROR] {}: client.services.create raised APIError' + .format(job.id)) for job in jobs.filter(Job.status != 'complete', Job.status != 'failed', Job.status != 'submitted').all(): @@ -70,6 +67,16 @@ def checkout_jobs(): if job.status == 'complete' or job.status == 'failed': job.end_date = datetime.utcnow() service.remove() + except docker.errors.APIError: + job.status = 'failed' + print('[ERROR] {}: client.services.get raised APIError' + .format(job.id)) except docker.errors.NotFound: job.status = 'failed' + print('[ERROR] {}: client.services.get raised NotFound' + .format(job.id)) + except docker.errors.InvalidVersion: + job.status = 'failed' + print('[ERROR] {}: client.services.get raised InvalidVersion' + .format(job.id)) db.session.commit()