enviPy-bayer/envipath/settings.py

"""
Django settings for envipath project.

Generated by 'django-admin startproject' using Django 4.2.17.

For more information on this file, see
https://docs.djangoproject.com/en/4.2/topics/settings/

For the full list of settings and their values, see
https://docs.djangoproject.com/en/4.2/ref/settings/
"""
import json
import os
from pathlib import Path

from dotenv import load_dotenv
from sklearn.ensemble import RandomForestClassifier
from sklearn.tree import DecisionTreeClassifier

# Build paths inside the project like this: BASE_DIR / 'subdir'.
BASE_DIR = Path(__file__).resolve().parent.parent

ENV_PATH = os.environ.get("ENV_PATH", BASE_DIR / ".env.dev")
print(f"Loading env from {ENV_PATH}")
load_dotenv(ENV_PATH, override=False)

# Quick-start development settings - unsuitable for production
# See https://docs.djangoproject.com/en/4.2/howto/deployment/checklist/

# SECURITY WARNING: keep the secret key used in production secret!
SECRET_KEY = os.environ.get("SECRET_KEY", "secret-key")

# SECURITY WARNING: don't run with debug turned on in production!
DEBUG = os.environ.get("DEBUG", "False") == "True"

ALLOWED_HOSTS = os.environ["ALLOWED_HOSTS"].split(",")


# Application definition
INSTALLED_APPS = [
    "django.contrib.admin",
    "django.contrib.auth",
    "django.contrib.contenttypes",
    "django.contrib.sessions",
    "django.contrib.messages",
    "django.contrib.staticfiles",
    "django.contrib.postgres",
    # 3rd party
    "django_extensions",
    "oauth2_provider",
    # Custom
    "epapi",  # API endpoints (v1, etc.)
    "epdb",
    "migration",
]

TENANT = os.environ.get("TENANT", "public")

if TENANT != "public":
    INSTALLED_APPS.append(TENANT)

EPDB_PACKAGE_MODEL = os.environ.get("EPDB_PACKAGE_MODEL", "epdb.Package")


def GET_PACKAGE_MODEL():
    from django.apps import apps

    return apps.get_model(EPDB_PACKAGE_MODEL)


AUTHENTICATION_BACKENDS = [
    "django.contrib.auth.backends.ModelBackend",
]

MIDDLEWARE = [
    "django.middleware.security.SecurityMiddleware",
    "whitenoise.middleware.WhiteNoiseMiddleware",
    "django.contrib.sessions.middleware.SessionMiddleware",
    "django.middleware.common.CommonMiddleware",
    "django.middleware.csrf.CsrfViewMiddleware",
    "django.contrib.auth.middleware.AuthenticationMiddleware",
    "django.contrib.messages.middleware.MessageMiddleware",
    "django.middleware.clickjacking.XFrameOptionsMiddleware",
    "oauth2_provider.middleware.OAuth2TokenMiddleware",
]

OAUTH2_PROVIDER = {
    "PKCE_REQUIRED": False,  # Accept PKCE requests but dont require them
}

if os.environ.get("REGISTRATION_MANDATORY", False) == "True":
    MIDDLEWARE.append("epdb.middleware.login_required_middleware.LoginRequiredMiddleware")

ROOT_URLCONF = "envipath.urls"

TEMPLATE_DIRS = [
    os.path.join(BASE_DIR, "templates"),
]

# If we have a non-public tenant, we might need to overwrite some templates
# search TENANT folder first...
if TENANT != "public":
    TEMPLATE_DIRS.insert(0, os.path.join(BASE_DIR, TENANT, "templates"))

TEMPLATES = [
    {
        "BACKEND": "django.template.backends.django.DjangoTemplates",
        "DIRS": TEMPLATE_DIRS,
        "APP_DIRS": True,
        "OPTIONS": {
            "context_processors": [
                "django.template.context_processors.debug",
                "django.template.context_processors.request",
                "django.contrib.auth.context_processors.auth",
                "django.contrib.messages.context_processors.messages",
                "epdb.context_processors.package_context",
            ],
        },
    },
]

ALLOWED_HTML_TAGS = {"b", "i", "u", "br", "em", "mark", "p", "s", "strong"}

WSGI_APPLICATION = "envipath.wsgi.application"

# Database
# https://docs.djangoproject.com/en/4.2/ref/settings/#databases

DATABASES = {
    "default": {
        "ENGINE": "django.db.backends.postgresql",
        "USER": os.environ["POSTGRES_USER"],
        "NAME": os.environ["POSTGRES_DB"],
        "PASSWORD": os.environ["POSTGRES_PASSWORD"],
        "HOST": os.environ["POSTGRES_SERVICE_NAME"],
        "PORT": os.environ["POSTGRES_PORT"],
    }
}

if os.environ.get("USE_TEMPLATE_DB", False) == "True":
    DATABASES["default"]["TEST"] = {
        "NAME": f"test_{os.environ['TEMPLATE_DB']}",
        "TEMPLATE": os.environ["TEMPLATE_DB"],
    }

CACHES = {
    "default": {
        "BACKEND": "django.core.cache.backends.locmem.LocMemCache",
        "LOCATION": "unique-snowflake",
    }
}

# Password validation
# https://docs.djangoproject.com/en/4.2/ref/settings/#auth-password-validators

AUTH_PASSWORD_VALIDATORS = [
    {"NAME": "django.contrib.auth.password_validation.UserAttributeSimilarityValidator"},
    {"NAME": "django.contrib.auth.password_validation.MinimumLengthValidator"},
    {"NAME": "django.contrib.auth.password_validation.CommonPasswordValidator"},
    {"NAME": "django.contrib.auth.password_validation.NumericPasswordValidator"},
]

# Internationalization
# https://docs.djangoproject.com/en/4.2/topics/i18n/

LANGUAGE_CODE = "en-us"

TIME_ZONE = "UTC"

USE_I18N = True

USE_TZ = True


# Default primary key field type
# https://docs.djangoproject.com/en/4.2/ref/settings/#default-auto-field

DEFAULT_AUTO_FIELD = "django.db.models.BigAutoField"

EMAIL_SUBJECT_PREFIX = "[enviPath] "
if DEBUG:
    EMAIL_BACKEND = "django.core.mail.backends.console.EmailBackend"
else:
    EMAIL_BACKEND = "django.core.mail.backends.smtp.EmailBackend"
    EMAIL_USE_TLS = True
    EMAIL_HOST = "mail.gandi.net"
    EMAIL_HOST_USER = os.environ["EMAIL_HOST_USER"]
    EMAIL_HOST_PASSWORD = os.environ["EMAIL_HOST_PASSWORD"]
    EMAIL_PORT = 587
    DEFAULT_FROM_EMAIL = os.environ["DEFAULT_FROM_EMAIL"]
    SERVER_EMAIL = os.environ["SERVER_EMAIL"]

AUTH_USER_MODEL = "epdb.User"
ADMIN_APPROVAL_REQUIRED = os.environ.get("ADMIN_APPROVAL_REQUIRED", "False") == "True"

# # SESAME
# SESAME_MAX_AGE = 300
# # TODO set to "home"
# LOGIN_REDIRECT_URL = "/"


SERVER_HOST = os.environ.get("SERVER_URL", "http://localhost:8000")
SERVER_PATH = os.environ.get("SERVER_PATH", "")

SERVER_URL = SERVER_HOST
if SERVER_PATH:
    SERVER_URL = os.path.join(SERVER_HOST, SERVER_PATH)


LOGIN_URL = "/login/"
if SERVER_PATH:
    LOGIN_URL = f"/{SERVER_PATH}/login/"

CSRF_TRUSTED_ORIGINS = [SERVER_HOST]

AMBIT_URL = "http://localhost:9001"
DEFAULT_VALUES = {"description": "no description"}

EP_DATA_DIR = os.environ["EP_DATA_DIR"]
if not os.path.exists(EP_DATA_DIR):
    os.mkdir(EP_DATA_DIR)

MODEL_DIR = os.path.join(EP_DATA_DIR, "models")
if not os.path.exists(MODEL_DIR):
    os.mkdir(MODEL_DIR)

STATIC_DIR = os.path.join(EP_DATA_DIR, "static")
if not os.path.exists(STATIC_DIR):
    os.mkdir(STATIC_DIR)

LOG_DIR = os.path.join(EP_DATA_DIR, "log")
if not os.path.exists(LOG_DIR):
    os.mkdir(LOG_DIR)

PLUGIN_DIR = os.path.join(EP_DATA_DIR, "plugins")
if not os.path.exists(PLUGIN_DIR):
    os.mkdir(PLUGIN_DIR)

API_PAGINATION_DEFAULT_PAGE_SIZE = int(os.environ.get("API_PAGINATION_DEFAULT_PAGE_SIZE", 50))
PAGINATION_MAX_PER_PAGE_SIZE = int(
    os.environ.get("API_PAGINATION_MAX_PAGE_SIZE", 100)
)  # Ninja override

# Set this as our static root dir
STATIC_ROOT = STATIC_DIR

STATIC_URL = "/static/"
if SERVER_PATH:
    STATIC_URL = f"/{SERVER_PATH}/static/"

# Where the sources are stored...
STATICFILES_DIRS = (BASE_DIR / "static",)

FIXTURE_DIRS = (BASE_DIR / "fixtures",)

# Logging
LOGGING = {
    "version": 1,
    "disable_existing_loggers": True,
    "formatters": {
        "simple": {
            "format": "[%(asctime)s] %(levelname)s %(module)s - %(message)s",
            "datefmt": "%Y-%m-%d %H:%M:%S",
        },
    },
    "handlers": {
        "console": {
            "level": "INFO",
            "class": "logging.StreamHandler",
            "formatter": "simple",
        },
        "file": {
            "level": "DEBUG",  # Or higher
            "class": "logging.FileHandler",
            "filename": os.path.join(LOG_DIR, "debug.log"),
            "formatter": "simple",
        },
        "auth_file": {
            "level": "INFO",  # Or higher
            "class": "logging.FileHandler",
            "filename": os.path.join(LOG_DIR, "auth.log"),
            "formatter": "simple",
        }
    },
    "loggers": {
        # For everything under epdb/ loaded via getlogger(__name__)
        "epdb": {
            "handlers": ["file"],  # "console",
            "propagate": True,
            "level": os.environ.get("LOG_LEVEL", "INFO"),
        },
        # For everything under envipath/ loaded via getlogger(__name__)
        "envipath": {
            "handlers": ["file", "console"],
            "propagate": True,
            "level": os.environ.get("LOG_LEVEL", "INFO"),
        },
        # For everything under utilities/ loaded via getlogger(__name__)
        "utilities": {
            "handlers": ["file", "console"],
            "propagate": True,
            "level": os.environ.get("LOG_LEVEL", "INFO"),
        },
        "auth": {
            "handlers": ["auth_file"],
            "propagate": True,
            "level": os.environ.get("LOG_LEVEL", "INFO"),
        }
    },
}

# Flags
ENVIFORMER_PRESENT = os.environ.get("ENVIFORMER_PRESENT", "False") == "True"
ENVIFORMER_DEVICE = os.environ.get("ENVIFORMER_DEVICE", "cpu")


# If celery is not present set always eager to true which will cause delayed tasks to block until finished
FLAG_CELERY_PRESENT = os.environ.get("FLAG_CELERY_PRESENT", "False") == "True"
if not FLAG_CELERY_PRESENT:
    CELERY_TASK_ALWAYS_EAGER = True

# Celery Configuration Options
CELERY_TIMEZONE = "Europe/Berlin"
CELERY_BROKER_URL = os.environ.get("CELERY_BROKER_URL", "redis://localhost:6379/0")
CELERY_RESULT_BACKEND = os.environ.get("CELERY_RESULT_BACKEND", "redis://localhost:6379/1")
CELERY_ACCEPT_CONTENT = ["json"]
CELERY_TASK_SERIALIZER = "json"

MODEL_BUILDING_ENABLED = os.environ.get("MODEL_BUILDING_ENABLED", "False") == "True"
APPLICABILITY_DOMAIN_ENABLED = os.environ.get("APPLICABILITY_DOMAIN_ENABLED", "False") == "True"
DEFAULT_RF_MODEL_PARAMS = {
    "base_clf": RandomForestClassifier(
        n_estimators=100,
        max_features="log2",
        random_state=42,
        criterion="entropy",
        ccp_alpha=0.0,
        max_depth=3,
        min_samples_leaf=1,
    ),
    "num_chains": 10,
}

DEFAULT_MODEL_PARAMS = {
    "base_clf": DecisionTreeClassifier(
        criterion="entropy",
        max_depth=3,
        min_samples_split=5,
        # min_samples_leaf=5,
        max_features="sqrt",
        # class_weight='balanced',
        random_state=42,
    ),
    "num_chains": 10,
}

DEFAULT_MAX_NUMBER_OF_NODES = 9999
DEFAULT_MAX_DEPTH = 8
DEFAULT_MODEL_THRESHOLD = 0.25

# Loading Plugins
PLUGINS_ENABLED = os.environ.get("PLUGINS_ENABLED", "False") == "True"
BASE_PLUGINS = os.environ.get("BASE_PLUGINS", None)
if BASE_PLUGINS:
    BASE_PLUGINS = BASE_PLUGINS.split(",")
else:
    BASE_PLUGINS = []

CLASSIFIER_PLUGINS = {}
PROPERTY_PLUGINS = {}
DESCRIPTOR_PLUGINS = {}

SENTRY_ENABLED = os.environ.get("SENTRY_ENABLED", "False") == "True"
if SENTRY_ENABLED:
    import sentry_sdk

    def before_send(event, hint):
        # Check if was a handled exception by one of our loggers
        if event.get("logger"):
            for log_path in LOGGING.get("loggers").keys():
                if event["logger"].startswith(log_path):
                    return None

        return event

    sentry_sdk.init(
        dsn=os.environ.get("SENTRY_DSN"),
        # Add data like request headers and IP for users,
        # see https://docs.sentry.io/platforms/python/data-management/data-collected/ for more info
        send_default_pii=True,
        environment=os.environ.get("SENTRY_ENVIRONMENT", "development"),
        before_send=before_send,
    )

IUCLID_EXPORT_ENABLED = os.environ.get("IUCLID_EXPORT_ENABLED", "False") == "True"
if IUCLID_EXPORT_ENABLED:
    INSTALLED_APPS.append("epiuclid")

# compile into digestible flags
FLAGS = {
    "MODEL_BUILDING": MODEL_BUILDING_ENABLED,
    "CELERY": FLAG_CELERY_PRESENT,
    "PLUGINS": PLUGINS_ENABLED,
    "SENTRY": SENTRY_ENABLED,
    "ENVIFORMER": ENVIFORMER_PRESENT,
    "APPLICABILITY_DOMAIN": APPLICABILITY_DOMAIN_ENABLED,
    "IUCLID_EXPORT": IUCLID_EXPORT_ENABLED,
}

# path of the URL are checked via "startswith"
# -> /password_reset/done is covered as well
LOGIN_EXEMPT_URLS = [
    "/register",
    "/api/v1/",  # Let API handle its own authentication
    "/api/legacy/",
    "/o/token/",
    "/o/userinfo/",
    "/password_reset/",
    "/reset/",
    "/terms",
    "/privacy",
    "/cookie-policy",
    "/about",
    "/contact",
    "/careers",
    "/cite",
    "/legal",
    "/entra/",
    "/auth/",
]

if SERVER_PATH:
    LOGIN_EXEMPT_URLS = [f"/{SERVER_PATH}{x}" for x in LOGIN_EXEMPT_URLS]

# MS AD/Entra
MS_ENTRA_ENABLED = os.environ.get("MS_ENTRA_ENABLED", "False") == "True"
if MS_ENTRA_ENABLED:
    # Add app to installed apps
    INSTALLED_APPS.append("epauth")
    # Set vars required by app
    MS_ENTRA_CLIENT_ID = os.environ["MS_CLIENT_ID"]
    MS_ENTRA_CLIENT_SECRET = os.environ["MS_CLIENT_SECRET"]
    MS_ENTRA_TENANT_ID = os.environ["MS_TENANT_ID"]
    MS_ENTRA_AUTHORITY = f"https://login.microsoftonline.com/{MS_ENTRA_TENANT_ID}"
    MS_ENTRA_REDIRECT_URI = os.environ["MS_REDIRECT_URI"]
    MS_ENTRA_SCOPES = os.environ.get("MS_SCOPES", "").split(",")

# Site ID 10 -> beta.envipath.org
MATOMO_SITE_ID = os.environ.get("MATOMO_SITE_ID", "10")

# CAP
CAP_ENABLED = os.environ.get("CAP_ENABLED", "False") == "True"
CAP_API_BASE = os.environ.get("CAP_API_BASE", None)
CAP_SITE_KEY = os.environ.get("CAP_SITE_KEY", None)
CAP_SECRET_KEY = os.environ.get("CAP_SECRET_KEY", None)

# Biotransformer
BIOTRANSFORMER_ENABLED = os.environ.get("BIOTRANSFORMER_ENABLED", "False") == "True"
FLAGS["BIOTRANSFORMER"] = BIOTRANSFORMER_ENABLED
if BIOTRANSFORMER_ENABLED:
    BIOTRANSFORMER_URL = os.environ.get("BIOTRANSFORMER_URL", None)

# PES
PES_API_MAPPING = os.environ.get("PES_API_MAPPING", None)
if PES_API_MAPPING:
    import json
    PES_API_MAPPING = json.loads(PES_API_MAPPING)
else:
    PES_API_MAPPING = {}

# Entra Groups
ENTRA_GROUPS = os.environ.get("ENTRA_GROUPS", None)
if ENTRA_GROUPS:
    import json
    ENTRA_GROUPS = json.loads(ENTRA_GROUPS)
else:
    ENTRA_GROUPS = {}

ENTRA_SECRET_GROUPS = os.environ.get("ENTRA_SECRET_GROUPS", None)
if ENTRA_SECRET_GROUPS:
    import json
    ENTRA_SECRET_GROUPS = json.loads(ENTRA_SECRET_GROUPS)
else:
    ENTRA_SECRET_GROUPS = {}

# PES Data Pools vs Entra Mapping
DATA_POOL_MAPPING = os.environ.get("DATA_POOL_MAPPING", None)
if DATA_POOL_MAPPING:
    import json
    DATA_POOL_MAPPING = json.loads(DATA_POOL_MAPPING)
else:
    DATA_POOL_MAPPING = {}

PROXIES = {}
if os.environ.get("HTTP_PROXY"):
    PROXIES["http"] = os.environ.get("HTTP_PROXY")
    PROXIES["https"] = os.environ.get("HTTPS_PROXY")

# BB4g
BB4G_URL = os.environ.get("BB4G_URL")
BB4G_TENANT_ID = os.environ.get("BB4G_TENANT_ID")
BB4G_CLIENT_ID = os.environ.get("BB4G_CLIENT_ID")
BB4G_CLIENT_SECRET = os.environ.get("BB4G_CLIENT_SECRET")
BB4G_SCOPE = os.environ.get("BB4G_SCOPE")

os.environ["NO_PROXY"] = "localhost,127.0.0.1,epbiotransformer3"