Rework a lot of daemon code. Add missing patch messages

This commit is contained in:
Patrick Jentsch 2021-08-04 17:07:59 +02:00
parent da350474fb
commit 575dc45242
3 changed files with 202 additions and 87 deletions

View File

@ -1,3 +1,4 @@
from flask import current_app
from ..models import Corpus from ..models import Corpus
import docker import docker
import logging import logging
@ -26,39 +27,69 @@ class CheckCorporaMixin:
self.remove_cqpserver_container(corpus) self.remove_cqpserver_container(corpus)
def create_build_corpus_service(self, corpus): def create_build_corpus_service(self, corpus):
corpus_data_dir = os.path.join(corpus.path, 'data') ''' # Docker service settings # '''
shutil.rmtree(corpus_data_dir, ignore_errors=True) ''' ## Command ## '''
os.mkdir(corpus_data_dir) command = 'docker-entrypoint.sh build-corpus'
corpus_registry_dir = os.path.join(corpus.path, 'registry') ''' ## Constraints ## '''
shutil.rmtree(corpus_registry_dir, ignore_errors=True) constraints = ['node.role==worker']
os.mkdir(corpus_registry_dir) ''' ## Image ## '''
corpus_file = os.path.join(corpus.path, 'merged', 'corpus.vrt') image = current_app.config['DOCKER_IMAGE_PREFIX'] + 'cqpserver:latest'
service_kwargs = { ''' ## Labels ## '''
'command': 'docker-entrypoint.sh build-corpus', labels = {
'constraints': ['node.role==worker'], 'origin': current_app.config['SERVER_NAME'],
'labels': {'origin': 'nopaque', 'type': 'build-corpus',
'type': 'corpus.build', 'corpus_id': str(corpus.id)
'corpus_id': str(corpus.id)},
'mounts': [corpus_file + ':/root/files/corpus.vrt:ro',
corpus_data_dir + ':/corpora/data:rw',
corpus_registry_dir + ':/usr/local/share/cwb/registry:rw'],
'name': 'build-corpus_{}'.format(corpus.id),
'restart_policy': docker.types.RestartPolicy()
} }
service_image = \ ''' ## Mounts ## '''
'gitlab.ub.uni-bielefeld.de:4567/sfb1288inf/cqpserver:latest' ''' ### Corpus file mount ### '''
corpus_file_source = os.path.join(corpus.path, 'merged', 'corpus.vrt')
corpus_file_target = '/root/files/corpus.vrt'
corpus_file_mount = \
corpus_file_source + ':' + corpus_file_target + ':ro'
''' ### Corpus data mount ### '''
corpus_data_source = os.path.join(corpus.path, 'data')
corpus_data_target = '/corpora/data'
corpus_data_mount = \
corpus_data_source + ':' + corpus_data_target + ':rw'
# Make sure that their is no data in the corpus data directory
shutil.rmtree(corpus_data_source, ignore_errors=True)
os.mkdir(corpus_data_source)
''' ### Corpus registry mount ### '''
corpus_registry_source = os.path.join(corpus.path, 'registry')
corpus_registry_target = '/usr/local/share/cwb/registry'
corpus_registry_mount = \
corpus_registry_source + ':' + corpus_registry_target + ':rw'
# Make sure that their is no data in the corpus registry directory
shutil.rmtree(corpus_registry_source, ignore_errors=True)
os.mkdir(corpus_registry_source)
mounts = [corpus_file_mount, corpus_data_mount, corpus_registry_mount]
''' ## Name ## '''
name = 'build-corpus_{}'.format(corpus.id)
''' ## Restart policy ## '''
restart_policy = docker.types.RestartPolicy()
try: try:
self.docker.services.create(service_image, **service_kwargs) self.docker.services.create(
image,
command=command,
constraints=constraints,
labels=labels,
mounts=mounts,
name=name,
restart_policy=restart_policy
)
except docker.errors.APIError as e: except docker.errors.APIError as e:
logging.error( logging.error(
'Create "{}" service raised '.format(service_kwargs['name']) 'Create "{}" service raised '.format(name)
+ '"docker.errors.APIError" The server returned an error. ' + '"docker.errors.APIError" The server returned an error. '
+ 'Details: {}'.format(e) + 'Details: {}'.format(e)
) )
else: else:
corpus.status = 'queued' corpus.status = 'queued'
patch_operation = { patch_operation = {
'op': 'replace', 'path': '/corpora/{}/status'.format(corpus.id), 'value': corpus.status} 'op': 'replace',
'path': '/corpora/{}/status'.format(corpus.id),
'value': corpus.status
}
self.buffer_user_patch_operation(corpus, patch_operation) self.buffer_user_patch_operation(corpus, patch_operation)
def checkout_build_corpus_service(self, corpus): def checkout_build_corpus_service(self, corpus):
@ -73,7 +104,10 @@ class CheckCorporaMixin:
) )
corpus.status = 'failed' corpus.status = 'failed'
patch_operation = { patch_operation = {
'op': 'replace', 'path': '/corpora/{}/status'.format(corpus.id), 'value': corpus.status} 'op': 'replace',
'path': '/corpora/{}/status'.format(corpus.id),
'value': corpus.status
}
self.buffer_user_patch_operation(corpus, patch_operation) self.buffer_user_patch_operation(corpus, patch_operation)
except docker.errors.APIError as e: except docker.errors.APIError as e:
logging.error( logging.error(
@ -95,7 +129,10 @@ class CheckCorporaMixin:
if corpus.status == 'queued' and task_state != 'pending': if corpus.status == 'queued' and task_state != 'pending':
corpus.status = 'running' corpus.status = 'running'
patch_operation = { patch_operation = {
'op': 'replace', 'path': '/corpora/{}/status'.format(corpus.id), 'value': corpus.status} 'op': 'replace',
'path': '/corpora/{}/status'.format(corpus.id),
'value': corpus.status
}
self.buffer_user_patch_operation(corpus, patch_operation) self.buffer_user_patch_operation(corpus, patch_operation)
elif (corpus.status == 'running' elif (corpus.status == 'running'
and task_state in ['complete', 'failed']): and task_state in ['complete', 'failed']):
@ -104,39 +141,53 @@ class CheckCorporaMixin:
except docker.errors.APIError as e: except docker.errors.APIError as e:
logging.error( logging.error(
'Remove "{}" service raised '.format(service_name) 'Remove "{}" service raised '.format(service_name)
+ '"docker.errors.APIError" The server returned an error. ' + '"docker.errors.APIError" The server returned an error. ' # noqa
+ 'Details: {}'.format(e) + 'Details: {}'.format(e)
) )
return return
else: else:
corpus.status = 'prepared' if task_state == 'complete' \ corpus.status = \
else 'failed' 'prepared' if task_state == 'complete' else 'failed'
patch_operation = { patch_operation = {
'op': 'replace', 'path': '/corpora/{}/status'.format(corpus.id), 'value': corpus.status} 'op': 'replace',
'path': '/corpora/{}/status'.format(corpus.id),
'value': corpus.status
}
self.buffer_user_patch_operation(corpus, patch_operation) self.buffer_user_patch_operation(corpus, patch_operation)
def create_cqpserver_container(self, corpus): def create_cqpserver_container(self, corpus):
corpus_data_dir = os.path.join(corpus.path, 'data') ''' # Docker container settings # '''
corpus_registry_dir = os.path.join(corpus.path, 'registry') ''' ## Command ## '''
container_kwargs = { command = 'cqpserver'
'command': 'cqpserver', ''' ## Detach ## '''
'detach': True, detach = True
'volumes': [corpus_data_dir + ':/corpora/data:rw', ''' ## Image ## '''
corpus_registry_dir + ':/usr/local/share/cwb/registry:rw'], image = current_app.config['DOCKER_IMAGE_PREFIX'] + 'cqpserver:latest'
'name': 'cqpserver_{}'.format(corpus.id), ''' ## Name ## '''
'network': 'nopaque_default' name = 'cqpserver_{}'.format(corpus.id),
} ''' ## Network ## '''
container_image = \ network = 'nopaque_default'
'gitlab.ub.uni-bielefeld.de:4567/sfb1288inf/cqpserver:latest' ''' ## Volumes ## '''
''' ### Corpus data volume ### '''
corpus_data_source = os.path.join(corpus.path, 'data')
corpus_data_target = '/corpora/data'
corpus_data_volume = \
corpus_data_source + ':' + corpus_data_target + ':rw'
''' ### Corpus registry volume ### '''
corpus_registry_source = os.path.join(corpus.path, 'registry')
corpus_registry_target = '/usr/local/share/cwb/registry'
corpus_registry_volume = \
corpus_registry_source + ':' + corpus_registry_target + ':rw'
volumes = [corpus_data_volume, corpus_registry_volume]
# Check if a cqpserver container already exists. If this is the case, # Check if a cqpserver container already exists. If this is the case,
# remove it and create a new one # remove it and create a new one
try: try:
container = self.docker.containers.get(container_kwargs['name']) container = self.docker.containers.get(name)
except docker.errors.NotFound: except docker.errors.NotFound:
pass pass
except docker.errors.APIError as e: except docker.errors.APIError as e:
logging.error( logging.error(
'Get "{}" container raised '.format(container_kwargs['name']) 'Get "{}" container raised '.format(name)
+ '"docker.errors.APIError" The server returned an error. ' + '"docker.errors.APIError" The server returned an error. '
+ 'Details: {}'.format(e) + 'Details: {}'.format(e)
) )
@ -146,45 +197,55 @@ class CheckCorporaMixin:
container.remove(force=True) container.remove(force=True)
except docker.errors.APIError as e: except docker.errors.APIError as e:
logging.error( logging.error(
'Remove "{}" container raised '.format( 'Remove "{}" container raised '.format(name)
container_kwargs['name'])
+ '"docker.errors.APIError" The server returned an error. ' + '"docker.errors.APIError" The server returned an error. '
+ 'Details: {}'.format(e) + 'Details: {}'.format(e)
) )
return return
try: try:
self.docker.containers.run(container_image, **container_kwargs) self.docker.containers.run(image, command=command, detach=detach,
volumes=volumes, name=name,
network=network)
except docker.errors.ContainerError: except docker.errors.ContainerError:
# This case should not occur, because detach is True. # This case should not occur, because detach is True.
logging.error( logging.error(
'Run "{}" container raised '.format(container_kwargs['name']) 'Run "{}" container raised '.format(name)
+ '"docker.errors.ContainerError" The container exits with a ' + '"docker.errors.ContainerError" The container exits with a '
+ 'non-zero exit code and detach is False.' + 'non-zero exit code and detach is False.'
) )
corpus.status = 'failed' corpus.status = 'failed'
patch_operation = { patch_operation = {
'op': 'replace', 'path': '/corpora/{}/status'.format(corpus.id), 'value': corpus.status} 'op': 'replace',
'path': '/corpora/{}/status'.format(corpus.id),
'value': corpus.status
}
self.buffer_user_patch_operation(corpus, patch_operation) self.buffer_user_patch_operation(corpus, patch_operation)
except docker.errors.ImageNotFound: except docker.errors.ImageNotFound:
logging.error( logging.error(
'Run "{}" container raised '.format(container_kwargs['name']) 'Run "{}" container raised '.format(name)
+ '"docker.errors.ImageNotFound" The specified image does not ' + '"docker.errors.ImageNotFound" The specified image does not '
+ 'exist.' + 'exist.'
) )
corpus.status = 'failed' corpus.status = 'failed'
patch_operation = { patch_operation = {
'op': 'replace', 'path': '/corpora/{}/status'.format(corpus.id), 'value': corpus.status} 'op': 'replace',
'path': '/corpora/{}/status'.format(corpus.id),
'value': corpus.status
}
self.buffer_user_patch_operation(corpus, patch_operation) self.buffer_user_patch_operation(corpus, patch_operation)
except docker.errors.APIError as e: except docker.errors.APIError as e:
logging.error( logging.error(
'Run "{}" container raised '.format(container_kwargs['name']) 'Run "{}" container raised '.format(name)
+ '"docker.errors.APIError" The server returned an error. ' + '"docker.errors.APIError" The server returned an error. '
+ 'Details: {}'.format(e) + 'Details: {}'.format(e)
) )
else: else:
corpus.status = 'analysing' corpus.status = 'analysing'
patch_operation = { patch_operation = {
'op': 'replace', 'path': '/corpora/{}/status'.format(corpus.id), 'value': corpus.status} 'op': 'replace',
'path': '/corpora/{}/status'.format(corpus.id),
'value': corpus.status
}
self.buffer_user_patch_operation(corpus, patch_operation) self.buffer_user_patch_operation(corpus, patch_operation)
def checkout_analysing_corpus_container(self, corpus): def checkout_analysing_corpus_container(self, corpus):
@ -194,6 +255,12 @@ class CheckCorporaMixin:
except docker.errors.NotFound: except docker.errors.NotFound:
logging.error('Could not find "{}" but the corpus state is "analysing".') # noqa logging.error('Could not find "{}" but the corpus state is "analysing".') # noqa
corpus.status = 'prepared' corpus.status = 'prepared'
patch_operation = {
'op': 'replace',
'path': '/corpora/{}/status'.format(corpus.id),
'value': corpus.status
}
self.buffer_user_patch_operation(corpus, patch_operation)
except docker.errors.APIError as e: except docker.errors.APIError as e:
logging.error( logging.error(
'Get "{}" container raised '.format(container_name) 'Get "{}" container raised '.format(container_name)
@ -227,5 +294,8 @@ class CheckCorporaMixin:
return return
corpus.status = 'prepared' corpus.status = 'prepared'
patch_operation = { patch_operation = {
'op': 'replace', 'path': '/corpora/{}/status'.format(corpus.id), 'value': corpus.status} 'op': 'replace',
'path': '/corpora/{}/status'.format(corpus.id),
'value': corpus.status
}
self.buffer_user_patch_operation(corpus, patch_operation) self.buffer_user_patch_operation(corpus, patch_operation)

View File

@ -1,16 +1,14 @@
from datetime import datetime from datetime import datetime
from flask import current_app
from werkzeug.utils import secure_filename from werkzeug.utils import secure_filename
from .. import db, mail from .. import db, mail
from ..email import create_message from ..email import create_message
from ..models import Job, JobResult from ..models import Job, JobResult
import docker import docker
import logging
import json import json
import logging
import os import os
import shutil
DOCKER_REGISTRY = 'gitlab.ub.uni-bielefeld.de:4567'
DOCKER_IMAGE_PREFIX = '{}/sfb1288inf/'.format(DOCKER_REGISTRY)
class CheckJobsMixin: class CheckJobsMixin:
@ -28,54 +26,64 @@ class CheckJobsMixin:
self.remove_job_service(job) self.remove_job_service(job)
def create_job_service(self, job): def create_job_service(self, job):
# Service specific settings ''' # Docker service settings # '''
''' ## Service specific settings ## '''
if job.service == 'file-setup': if job.service == 'file-setup':
mem_mb = 2048 mem_mb = 2048
n_cores = 2 n_cores = 2
executable = 'file-setup' executable = 'file-setup'
image = DOCKER_IMAGE_PREFIX + 'file-setup:' + job.service_version image = (current_app.config['DOCKER_IMAGE_PREFIX']
+ 'file-setup:' + job.service_version)
elif job.service == 'ocr': elif job.service == 'ocr':
mem_mb = 4096 mem_mb = 4096
n_cores = 4 n_cores = 4
executable = 'ocr' executable = 'ocr'
image = DOCKER_IMAGE_PREFIX + 'ocr:' + job.service_version image = (current_app.config['DOCKER_IMAGE_PREFIX']
+ 'ocr:' + job.service_version)
elif job.service == 'nlp': elif job.service == 'nlp':
mem_mb = 2048 mem_mb = 2048
n_cores = 2 n_cores = 2
executable = 'nlp' executable = 'nlp'
image = DOCKER_IMAGE_PREFIX + 'nlp:' + job.service_version image = (current_app.config['DOCKER_IMAGE_PREFIX']
# Command + 'nlp:' + job.service_version)
''' ## Command ## '''
command = '{} -i /input -o /output'.format(executable) command = '{} -i /input -o /output'.format(executable)
command += ' --log-dir /input' command += ' --log-dir /input'
command += ' --mem-mb {}'.format(mem_mb) command += ' --mem-mb {}'.format(mem_mb)
command += ' --n-cores {}'.format(n_cores) command += ' --n-cores {}'.format(n_cores)
command += ' --zip [' + job.service + ']_' + secure_filename(job.title) command += ' --zip [' + job.service + ']_' + secure_filename(job.title)
command += ' ' + ' '.join(json.loads(job.service_args)) command += ' ' + ' '.join(json.loads(job.service_args))
# Constraints ''' ## Constraints ## '''
constraints = ['node.role==worker'] constraints = ['node.role==worker']
# Labels ''' ## Labels ## '''
labels = {'origin': 'nopaque', 'type': 'job', 'job_id': str(job.id)} labels = {
# Mounts 'origin': current_app.config['SERVER_NAME'],
## Input mount 'type': 'job',
'job_id': str(job.id)
}
''' ## Mounts ## '''
''' ### Input mount ### '''
input_mount_source = job.path input_mount_source = job.path
input_mount_target = '/input' input_mount_target = '/input'
if job.service == 'file-setup': if job.service == 'file-setup':
input_mount_target += '/' + secure_filename(job.title) input_mount_target += '/' + secure_filename(job.title)
input_mount = input_mount_source + ':' + input_mount_target + ':rw' input_mount = input_mount_source + ':' + input_mount_target + ':rw'
## Output mount ''' ### Output mount ### '''
output_mount_source = os.path.join(job.path, 'output') output_mount_source = os.path.join(job.path, 'output')
output_mount_target = '/output' output_mount_target = '/output'
output_mount = output_mount_source + ':' + output_mount_target + ':rw' output_mount = output_mount_source + ':' + output_mount_target + ':rw'
# Make sure that their is no data in the output directory
shutil.rmtree(output_mount_source, ignore_errors=True)
os.makedirs(output_mount_source) os.makedirs(output_mount_source)
mounts = [input_mount, output_mount] mounts = [input_mount, output_mount]
# Name ''' ## Name ## '''
name = 'job_{}'.format(job.id) name = 'job_{}'.format(job.id)
# Resources ''' ## Resources ## '''
resources = docker.types.Resources( resources = docker.types.Resources(
cpu_reservation=n_cores * (10 ** 9), cpu_reservation=n_cores * (10 ** 9),
mem_reservation=mem_mb * (10 ** 6) mem_reservation=mem_mb * (10 ** 6)
) )
# Restart policy ''' ## Restart policy ## '''
restart_policy = docker.types.RestartPolicy() restart_policy = docker.types.RestartPolicy()
try: try:
self.docker.services.create( self.docker.services.create(
@ -90,14 +98,18 @@ class CheckJobsMixin:
) )
except docker.errors.APIError as e: except docker.errors.APIError as e:
logging.error( logging.error(
'Create "{}" service raised '.format(service_kwargs['name']) 'Create "{}" service raised '.format(name)
+ '"docker.errors.APIError" The server returned an error. ' + '"docker.errors.APIError" The server returned an error. '
+ 'Details: {}'.format(e) + 'Details: {}'.format(e)
) )
return return
else: else:
job.status = 'queued' job.status = 'queued'
patch_operation = {'op': 'replace', 'path': '/jobs/{}/status'.format(job.id), 'value': job.status} # noqa patch_operation = {
'op': 'replace',
'path': '/jobs/{}/status'.format(job.id),
'value': job.status
}
self.buffer_user_patch_operation(job, patch_operation) self.buffer_user_patch_operation(job, patch_operation)
finally: finally:
self.send_job_notification(job) self.send_job_notification(job)
@ -107,11 +119,17 @@ class CheckJobsMixin:
try: try:
service = self.docker.services.get(service_name) service = self.docker.services.get(service_name)
except docker.errors.NotFound: except docker.errors.NotFound:
logging.error('Get "{}" service raised '.format(service_name) logging.error(
'Get "{}" service raised '.format(service_name)
+ '"docker.errors.NotFound" The service does not exist. ' + '"docker.errors.NotFound" The service does not exist. '
+ '(job.status: {} -> failed)'.format(job.status)) + '(job.status: {} -> failed)'.format(job.status)
)
job.status = 'failed' job.status = 'failed'
patch_operation = {'op': 'replace', 'path': '/jobs/{}/status'.format(job.id), 'value': job.status} # noqa patch_operation = {
'op': 'replace',
'path': '/jobs/{}/status'.format(job.id),
'value': job.status
}
self.buffer_user_patch_operation(job, patch_operation) self.buffer_user_patch_operation(job, patch_operation)
except docker.errors.APIError as e: except docker.errors.APIError as e:
logging.error( logging.error(
@ -134,9 +152,13 @@ class CheckJobsMixin:
task_state = service_tasks[0].get('Status').get('State') task_state = service_tasks[0].get('Status').get('State')
if job.status == 'queued' and task_state != 'pending': if job.status == 'queued' and task_state != 'pending':
job.status = 'running' job.status = 'running'
patch_operation = {'op': 'replace', 'path': '/jobs/{}/status'.format(job.id), 'value': job.status} # noqa patch_operation = {
'op': 'replace',
'path': '/jobs/{}/status'.format(job.id),
'value': job.status
}
self.buffer_user_patch_operation(job, patch_operation) self.buffer_user_patch_operation(job, patch_operation)
elif job.status == 'running' and task_state in ['complete', 'failed']: elif job.status == 'running' and task_state in ['complete', 'failed']: # noqa
try: try:
service.remove() service.remove()
except docker.errors.APIError as e: except docker.errors.APIError as e:
@ -156,13 +178,25 @@ class CheckJobsMixin:
db.session.add(job_result) db.session.add(job_result)
db.session.flush() db.session.flush()
db.session.refresh(job_result) db.session.refresh(job_result)
patch_operation = {'op': 'add', 'path': '/jobs/{}/results/{}'.format(job.id, job_result.id), 'value': job_result.to_dict()} # noqa patch_operation = {
'op': 'add',
'path': '/jobs/{}/results/{}'.format(job.id, job_result.id), # noqa
'value': job_result.to_dict()
}
self.buffer_user_patch_operation(job, patch_operation) # noqa self.buffer_user_patch_operation(job, patch_operation) # noqa
job.end_date = datetime.utcnow() job.end_date = datetime.utcnow()
patch_operation = {'op': 'replace', 'path': '/jobs/{}/end_date'.format(job.id), 'value': job.end_date.timestamp()} # noqa patch_operation = {
'op': 'replace',
'path': '/jobs/{}/end_date'.format(job.id),
'value': job.end_date.timestamp()
}
self.buffer_user_patch_operation(job, patch_operation) self.buffer_user_patch_operation(job, patch_operation)
job.status = task_state job.status = task_state
patch_operation = {'op': 'replace', 'path': '/jobs/{}/status'.format(job.id), 'value': job.status} # noqa patch_operation = {
'op': 'replace',
'path': '/jobs/{}/status'.format(job.id),
'value': job.status
}
self.buffer_user_patch_operation(job, patch_operation) self.buffer_user_patch_operation(job, patch_operation)
finally: finally:
self.send_job_notification(job) self.send_job_notification(job)
@ -173,7 +207,11 @@ class CheckJobsMixin:
service = self.docker.services.get(service_name) service = self.docker.services.get(service_name)
except docker.errors.NotFound: except docker.errors.NotFound:
job.status = 'canceled' job.status = 'canceled'
patch_operation = {'op': 'replace', 'path': '/jobs/{}/status'.format(job.id), 'value': job.status} # noqa patch_operation = {
'op': 'replace',
'path': '/jobs/{}/status'.format(job.id),
'value': job.status
}
self.buffer_user_patch_operation(job, patch_operation) self.buffer_user_patch_operation(job, patch_operation)
except docker.errors.APIError as e: except docker.errors.APIError as e:
logging.error( logging.error(
@ -214,7 +252,10 @@ class CheckJobsMixin:
if (job.creator.setting_job_status_mail_notifications == 'end' if (job.creator.setting_job_status_mail_notifications == 'end'
and job.status not in ['complete', 'failed']): and job.status not in ['complete', 'failed']):
return return
msg = create_message(job.creator.email, msg = create_message(
job.creator.email,
'Status update for your Job "{}"'.format(job.title), # noqa 'Status update for your Job "{}"'.format(job.title), # noqa
'tasks/email/notification', job=job) 'tasks/email/notification',
job=job
)
mail.send(msg) mail.send(msg)

View File

@ -7,6 +7,10 @@ ROOT_DIR = os.path.abspath(os.path.dirname(__file__))
class Config: class Config:
''' # Docker # '''
DOCKER_REGISTRY = 'gitlab.ub.uni-bielefeld.de:4567'
DOCKER_IMAGE_PREFIX = DOCKER_REGISTRY + '/sfb1288inf/'
''' # Flask # ''' ''' # Flask # '''
PREFERRED_URL_SCHEME = os.environ.get('PREFERRED_URL_SCHEME', 'http') PREFERRED_URL_SCHEME = os.environ.get('PREFERRED_URL_SCHEME', 'http')
SECRET_KEY = os.environ.get('SECRET_KEY', 'hard to guess string') SECRET_KEY = os.environ.get('SECRET_KEY', 'hard to guess string')