Skip to content

Commit

Permalink
DWPF-1603 Codebase changes for platform migration (#503)
Browse files Browse the repository at this point in the history
  • Loading branch information
SamDudley authored May 17, 2024
1 parent 61dfd93 commit fc3cb7b
Show file tree
Hide file tree
Showing 16 changed files with 1,338 additions and 2,203 deletions.
6 changes: 3 additions & 3 deletions .env.ci
Original file line number Diff line number Diff line change
@@ -1,9 +1,9 @@
DJANGO_SETTINGS_MODULE=config.settings.ci
DATABASE_URL=psql://postgres:postgres@db:5432/fido
FLASK_CONFIG=testing
SECRET_KEY=used_for_testing
ALLOWED_HOSTS="*"
DEBUG=False
DATABASE_URL=psql://postgres:postgres@db:5432/fido
REDIS_ENDPOINT=redis://redis:6379
AUTHBROKER_CLIENT_ID=
AUTHBROKER_CLIENT_SECRET=
AUTHBROKER_URL=
DEBUG=False
2 changes: 2 additions & 0 deletions .env.example
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,8 @@ SECRET_KEY=super-secret-key
ALLOWED_HOSTS="*"
DEBUG=True
DISABLE_COLLECTSTATIC="0"
DATABASE_URL=psql://postgres:postgres@db:5432/fido
REDIS_ENDPOINT=redis://redis:6379

# Authbroker
AUTHBROKER_CLIENT_ID=copy-from-dev-environment
Expand Down
5 changes: 5 additions & 0 deletions config/celery.py
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,8 @@
import os

from celery import Celery
from dbt_copilot_python.celery_health_check import healthcheck
from dbt_copilot_python.utility import is_copilot


os.environ.setdefault("DJANGO_SETTINGS_MODULE", "config.settings.prod")
Expand All @@ -11,6 +13,9 @@
celery_app.config_from_object("django.conf:settings", namespace="CELERY")
celery_app.autodiscover_tasks()

if is_copilot():
celery_app = healthcheck.setup(celery_app)


@celery_app.task(bind=True)
def debug_task(self):
Expand Down
51 changes: 46 additions & 5 deletions config/settings/base.py
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,10 @@
from pathlib import Path

import environ
from dbt_copilot_python.utility import is_copilot
from django.urls import reverse_lazy
from django_log_formatter_asim import ASIMFormatter
from django_log_formatter_ecs import ECSFormatter


BASE_DIR = Path(__file__).resolve().parent.parent.parent
Expand Down Expand Up @@ -220,15 +223,16 @@ def FILTERS_VERBOSE_LOOKUPS():
if "redis" in VCAP_SERVICES:
credentials = VCAP_SERVICES["redis"][0]["credentials"]

CELERY_BROKER_URL = "rediss://:{}@{}:{}/0?ssl_cert_reqs=required".format(
REDIS_URL = "rediss://:{}@{}:{}/0?ssl_cert_reqs=required".format(
credentials["password"],
credentials["host"],
credentials["port"],
)
else:
CELERY_BROKER_URL = env("CELERY_BROKER_URL", default=None)
REDIS_URL = env.str("REDIS_ENDPOINT")

# celery
# Celery
CELERY_BROKER_URL = REDIS_URL
CELERY_ACCEPT_CONTENT = ["application/json"]
CELERY_RESULT_SERIALIZER = "json"

Expand Down Expand Up @@ -322,8 +326,9 @@ def FILTERS_VERBOSE_LOOKUPS():

CACHES = {
"default": {
"BACKEND": "django.core.cache.backends.db.DatabaseCache",
"LOCATION": "django_cache_table",
"BACKEND": "django.core.cache.backends.redis.RedisCache",
"LOCATION": REDIS_URL,
"KEY_PREFIX": "cache_",
}
}

Expand All @@ -334,3 +339,39 @@ def FILTERS_VERBOSE_LOOKUPS():

# Selenium (BDD tests)
USE_REMOTE_CHROME = env("USE_REMOTE_CHROME", default=True)

# Logging
LOGGING = {
"version": 1,
"disable_existing_loggers": False,
"formatters": {
"simple": {
"format": "{levelname} {message}",
"style": "{",
},
"asim": {
"()": ASIMFormatter,
},
# TODO (DWPF-1696): Remove ECS formatter
"ecs": {
"()": ECSFormatter,
},
},
"handlers": {
"console": {
"class": "logging.StreamHandler",
"formatter": "simple",
},
},
"root": {
"handlers": ["console"],
"level": "INFO",
},
"loggers": {
"django": {
"handlers": ["console"],
"level": "INFO",
"propagate": False,
},
},
}
39 changes: 1 addition & 38 deletions config/settings/dev.py
Original file line number Diff line number Diff line change
@@ -1,10 +1,7 @@
import sys

import sentry_sdk
from django_log_formatter_ecs import ECSFormatter
from sentry_sdk.integrations.django import DjangoIntegration

from .base import * # noqa
from .paas import * # noqa


MIDDLEWARE += [
Expand All @@ -24,40 +21,6 @@
# Django staff SSO user migration process requries the following
MIGRATE_EMAIL_USER_ON_LOGIN = True

CACHES = {
"default": {
"BACKEND": "django.core.cache.backends.redis.RedisCache",
"LOCATION": CELERY_BROKER_URL,
"KEY_PREFIX": "cache_",
}
}

LOGGING = {
"version": 1,
"disable_existing_loggers": False,
"formatters": {
"ecs_formatter": {
"()": ECSFormatter,
},
},
"handlers": {
"ecs": {
"class": "logging.StreamHandler",
"stream": sys.stdout,
"formatter": "ecs_formatter",
},
},
"loggers": {
"django.request": {
"handlers": [
"ecs",
],
"level": "INFO",
"propagate": True,
},
},
}

# Set async file uploading
ASYNC_FILE_UPLOAD = True

Expand Down
87 changes: 0 additions & 87 deletions config/settings/local.py
Original file line number Diff line number Diff line change
@@ -1,98 +1,11 @@
import logging
import sys

import requests
from django_log_formatter_ecs import ECSFormatter

from .base import * # noqa


CAN_ELEVATE_SSO_USER_PERMISSIONS = True
CAN_CREATE_TEST_USER = True

FRONT_END_SERVER = env(
"FRONT_END_SERVER",
default="http://localhost:3000",
)

AUTHENTICATION_BACKENDS += [
"user.backends.CustomAuthbrokerBackend",
]

ASYNC_FILE_UPLOAD = False

LOG_TO_ELK = env.bool("LOG_TO_ELK", default=False)
ELK_ADDRESS = env("ELK_ADDRESS", default=None)

if LOG_TO_ELK:

class LogstashHTTPHandler(logging.Handler):
def emit(self, record):
log_entry = self.format(record)

return requests.post(
ELK_ADDRESS,
data="{}".format(log_entry),
headers={"Content-type": "application/json"},
).content

LOGGING = {
"version": 1,
"disable_existing_loggers": False,
"formatters": {
"ecs_formatter": {
"()": ECSFormatter,
},
},
"handlers": {
"stdout": {
"class": "logging.StreamHandler",
"stream": sys.stdout,
},
"logstash": {
"()": LogstashHTTPHandler,
"formatter": "ecs_formatter",
},
},
"loggers": {
"django.request": {
"handlers": [
"stdout",
"logstash",
],
"level": "WARNING",
"propagate": True,
},
},
}
else:
LOGGING = {
"version": 1,
"disable_existing_loggers": False,
"handlers": {
"stdout": {
"class": "logging.StreamHandler",
"stream": sys.stdout,
},
},
"root": {
"handlers": ["stdout"],
"level": os.getenv("ROOT_LOG_LEVEL", "INFO"),
},
"loggers": {
"django": {
"handlers": [
"stdout",
],
"level": os.getenv("DJANGO_LOG_LEVEL", "INFO"),
"propagate": True,
},
"forecast.import_csv": {
"handlers": [
"stdout",
],
"level": "INFO",
"propagate": True,
},
},
}
19 changes: 19 additions & 0 deletions config/settings/paas.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,19 @@
import dj_database_url
from dbt_copilot_python.database import database_url_from_env
from dbt_copilot_python.network import setup_allowed_hosts
from dbt_copilot_python.utility import is_copilot

from .base import *


# TODO (DWPF-1696): Remove ECS formatter
LOGGING["handlers"]["console"]["formatter"] = "ecs"

if is_copilot():
ALLOWED_HOSTS = setup_allowed_hosts(ALLOWED_HOSTS)

DATABASES["default"] = dj_database_url.config(
default=database_url_from_env("DATABASE_CREDENTIALS")
)

LOGGING["handlers"]["console"]["formatter"] = "asim"
67 changes: 1 addition & 66 deletions config/settings/prod.py
Original file line number Diff line number Diff line change
@@ -1,10 +1,7 @@
import sys

import sentry_sdk
from django_log_formatter_ecs import ECSFormatter
from sentry_sdk.integrations.django import DjangoIntegration

from .base import * # noqa
from .paas import * # noqa


MIDDLEWARE += [
Expand All @@ -27,68 +24,6 @@
"nofollow",
]

CACHES = {
"default": {
"BACKEND": "django.core.cache.backends.redis.RedisCache",
"LOCATION": CELERY_BROKER_URL,
"KEY_PREFIX": "cache_",
}
}

LOGGING = {
"version": 1,
"disable_existing_loggers": False,
"formatters": {
"ecs_formatter": {
"()": ECSFormatter,
},
"simple": {"format": "%(levelname)s %(message)s"},
},
"handlers": {
"ecs": {
"class": "logging.StreamHandler",
"stream": sys.stdout,
"formatter": "ecs_formatter",
},
"stdout": {
"class": "logging.StreamHandler",
"stream": sys.stdout,
"formatter": "simple",
"level": "INFO",
},
},
"loggers": {
"django.request": {
"handlers": [
"ecs",
],
"level": "INFO",
"propagate": True,
},
"forecast.import_csv": {
"handlers": [
"stdout",
],
"level": "INFO",
"propagate": True,
},
"forecast.views.upload_file": {
"handlers": [
"stdout",
],
"level": "INFO",
"propagate": True,
},
"forecast.tasks": {
"handlers": [
"stdout",
],
"level": "INFO",
"propagate": True,
},
},
}

# Set async file uploading
ASYNC_FILE_UPLOAD = True

Expand Down
Loading

0 comments on commit fc3cb7b

Please sign in to comment.