enviPy-bayer/envipath/settings.py

"""
Django settings for envipath project.

Generated by 'django-admin startproject' using Django 4.2.17.

For more information on this file, see
https://docs.djangoproject.com/en/4.2/topics/settings/

For the full list of settings and their values, see
https://docs.djangoproject.com/en/4.2/ref/settings/
"""
import os
from pathlib import Path

from dotenv import load_dotenv
from envipy_plugins import Classifier, Property, Descriptor
from sklearn.ensemble import RandomForestClassifier
from sklearn.tree import DecisionTreeClassifier

# Build paths inside the project like this: BASE_DIR / 'subdir'.
BASE_DIR = Path(__file__).resolve().parent.parent

load_dotenv(BASE_DIR / '.env', override=False)

# Quick-start development settings - unsuitable for production
# See https://docs.djangoproject.com/en/4.2/howto/deployment/checklist/

# SECURITY WARNING: keep the secret key used in production secret!
SECRET_KEY = '7!VTW`aZqg/UBLsM.P=m)2]lWqg>{+:xUgG1"WO@bCyaHR2Up8XW&g<*3.F4l2gi9c.E3}dHyA0D`&z?u#U%^7HYbj],eP"g_MS|3BNMD[mI>s#<i/%2ngZ~Oy+/w&@]'

# SECURITY WARNING: don't run with debug turned on in production!
DEBUG = os.environ.get('DEBUG', 'False') == 'True'
ALLOWED_HOSTS = os.environ['ALLOWED_HOSTS'].split(',')

# Application definition

INSTALLED_APPS = [
    'django.contrib.admin',
    'django.contrib.auth',
    'django.contrib.contenttypes',
    'django.contrib.sessions',
    'django.contrib.messages',
    'django.contrib.staticfiles',
    # 3rd party
    'django_extensions',
    # Custom
    'epdb',
    'migration',
]

AUTHENTICATION_BACKENDS = [
    "django.contrib.auth.backends.ModelBackend",
]

MIDDLEWARE = [
    'django.middleware.security.SecurityMiddleware',
    'django.contrib.sessions.middleware.SessionMiddleware',
    'django.middleware.common.CommonMiddleware',
    'django.middleware.csrf.CsrfViewMiddleware',
    'django.contrib.auth.middleware.AuthenticationMiddleware',
    'django.contrib.messages.middleware.MessageMiddleware',
    'django.middleware.clickjacking.XFrameOptionsMiddleware',
]

if os.environ.get('REGISTRATION_MANDATORY', False) == 'True':
    MIDDLEWARE.append('epdb.middleware.login_required_middleware.LoginRequiredMiddleware')

ROOT_URLCONF = 'envipath.urls'

TEMPLATES = [
    {
        'BACKEND': 'django.template.backends.django.DjangoTemplates',
        'DIRS': (os.path.join(BASE_DIR, 'templates'),),
        'APP_DIRS': True,
        'OPTIONS': {
            'context_processors': [
                'django.template.context_processors.debug',
                'django.template.context_processors.request',
                'django.contrib.auth.context_processors.auth',
                'django.contrib.messages.context_processors.messages',
            ],
        },
    },
]

WSGI_APPLICATION = 'envipath.wsgi.application'

# Database
# https://docs.djangoproject.com/en/4.2/ref/settings/#databases

DATABASES = {
    "default": {
        "ENGINE": "django.db.backends.postgresql",
        "USER": os.environ['POSTGRES_USER'],
        "NAME": os.environ['POSTGRES_DB'],
        "PASSWORD": os.environ['POSTGRES_PASSWORD'],
        "HOST": os.environ['POSTGRES_SERVICE_NAME'],
        "PORT": os.environ['POSTGRES_PORT']
    }
}

# Password validation
# https://docs.djangoproject.com/en/4.2/ref/settings/#auth-password-validators

AUTH_PASSWORD_VALIDATORS = [
    {
        'NAME': 'django.contrib.auth.password_validation.UserAttributeSimilarityValidator',
    },
    {
        'NAME': 'django.contrib.auth.password_validation.MinimumLengthValidator',
    },
    {
        'NAME': 'django.contrib.auth.password_validation.CommonPasswordValidator',
    },
    {
        'NAME': 'django.contrib.auth.password_validation.NumericPasswordValidator',
    },
]

# Internationalization
# https://docs.djangoproject.com/en/4.2/topics/i18n/

LANGUAGE_CODE = 'en-us'

TIME_ZONE = 'UTC'

USE_I18N = True

USE_TZ = True



# Default primary key field type
# https://docs.djangoproject.com/en/4.2/ref/settings/#default-auto-field

DEFAULT_AUTO_FIELD = 'django.db.models.BigAutoField'

# EMAIL
EMAIL_BACKEND = 'django.core.mail.backends.smtp.EmailBackend'
EMAIL_USE_TLS = True
EMAIL_HOST = 'mail.gandi.net'
EMAIL_HOST_USER = os.environ['EMAIL_HOST_USER']
EMAIL_HOST_PASSWORD = os.environ['EMAIL_HOST_PASSWORD']
EMAIL_PORT = 587

AUTH_USER_MODEL = "epdb.User"
ADMIN_APPROVAL_REQUIRED = os.environ.get('ADMIN_APPROVAL_REQUIRED', 'False') == 'True'

# # SESAME
# SESAME_MAX_AGE = 300
# # TODO set to "home"
# LOGIN_REDIRECT_URL = "/"
LOGIN_URL = '/login/'

SERVER_URL = os.environ.get('SERVER_URL', 'http://localhost:8000')

CSRF_TRUSTED_ORIGINS = [SERVER_URL]

AMBIT_URL = 'http://localhost:9001'
DEFAULT_VALUES = {
    'description': 'no description'
}

EP_DATA_DIR = os.environ['EP_DATA_DIR']
MODEL_DIR = os.path.join(EP_DATA_DIR, 'models')
if not os.path.exists(MODEL_DIR):
    os.mkdir(MODEL_DIR)

STATIC_DIR = os.path.join(EP_DATA_DIR, 'static')
if not os.path.exists(STATIC_DIR):
    os.mkdir(STATIC_DIR)

LOG_DIR = os.path.join(EP_DATA_DIR, 'log')
if not os.path.exists(LOG_DIR):
    os.mkdir(LOG_DIR)

PLUGIN_DIR = os.path.join(EP_DATA_DIR, 'plugins')
if not os.path.exists(PLUGIN_DIR):
    os.mkdir(PLUGIN_DIR)

# Set this as our static root dir
STATIC_ROOT = STATIC_DIR

STATIC_URL = '/static/'

# Where the sources are stored...
STATICFILES_DIRS = (
    BASE_DIR / 'static',
)

FIXTURE_DIRS = (
        BASE_DIR / 'fixtures',
)

# Logging
LOGGING = {
    "version": 1,
    "disable_existing_loggers": True,
    "formatters": {
        "simple": {
            'format': '[%(asctime)s] %(levelname)s %(module)s - %(message)s',
            'datefmt': '%Y-%m-%d %H:%M:%S',
        },
    },
    "handlers": {
        "console": {
            "level": "INFO",
            "class": "logging.StreamHandler",
            "formatter": "simple",
        },
        "file": {
            "level": "DEBUG",  # Or higher
            "class": "logging.FileHandler",
            "filename": os.path.join(LOG_DIR, "debug.log"),
            "formatter": "simple"
        },
    },
    "loggers": {
        # For everything under epdb/ loaded via getlogger(__name__)
        "epdb": {
            "handlers": ["file"],  # "console",
            "propagate": True,
            "level": os.environ.get('LOG_LEVEL', 'INFO')
        },
        # For everything under envipath/ loaded via getlogger(__name__)
        'envipath': {
            'handlers': ['file', 'console'],
            'propagate': True,
            'level': os.environ.get('LOG_LEVEL', 'INFO')
        },
        # For everything under utilities/ loaded via getlogger(__name__)
        'utilities': {
            'handlers': ['file', 'console'],
            'propagate': True,
            'level': os.environ.get('LOG_LEVEL', 'INFO')
        },
    },
}

# Flags
ENVIFORMER_PRESENT = os.environ.get('ENVIFORMER_PRESENT', 'False') == 'True'
if ENVIFORMER_PRESENT:
    print("Loading enviFormer")
    device = os.environ.get('ENVIFORMER_DEVICE', 'cpu')
    from enviformer import load
    ENVIFORMER_INSTANCE = load(device=device)
    print("loaded")


# If celery is not present set always eager to true which will cause delayed tasks to block until finished
FLAG_CELERY_PRESENT = os.environ.get('FLAG_CELERY_PRESENT', 'False') == 'True'
if not FLAG_CELERY_PRESENT:
    CELERY_TASK_ALWAYS_EAGER = True

# Celery Configuration Options
CELERY_TIMEZONE = "Europe/Berlin"
# Celery Configuration
CELERY_BROKER_URL = 'redis://localhost:6379/0'  # Use Redis as message broker
CELERY_RESULT_BACKEND = 'redis://localhost:6379/1'
CELERY_ACCEPT_CONTENT = ['json']
CELERY_TASK_SERIALIZER = 'json'

MODEL_BUILDING_ENABLED = os.environ.get('MODEL_BUILDING_ENABLED', 'False') == 'True'
APPLICABILITY_DOMAIN_ENABLED = os.environ.get('APPLICABILITY_DOMAIN_ENABLED', 'False') == 'True'
DEFAULT_RF_MODEL_PARAMS = {
    'base_clf': RandomForestClassifier(
        n_estimators=100,
        max_features='log2',
        random_state=42,
        criterion='entropy',
        ccp_alpha=0.0,
        max_depth=3,
        min_samples_leaf=1
    ),
    'num_chains': 10,
}

DEFAULT_MODEL_PARAMS = {
    'base_clf': DecisionTreeClassifier(
        criterion='entropy',
        max_depth=3,
        min_samples_split=5,
        # min_samples_leaf=5,
        max_features='sqrt',
        # class_weight='balanced',
        random_state=42
    ),
    'num_chains': 10,
}

DEFAULT_MAX_NUMBER_OF_NODES = 30
DEFAULT_MAX_DEPTH = 5
DEFAULT_MODEL_THRESHOLD = 0.25

# Loading Plugins
PLUGINS_ENABLED = os.environ.get('PLUGINS_ENABLED', 'False') == 'True'
if PLUGINS_ENABLED:
    from utilities.plugin import discover_plugins
    CLASSIFIER_PLUGINS = discover_plugins(_cls=Classifier)
    PROPERTY_PLUGINS = discover_plugins(_cls=Property)
    DESCRIPTOR_PLUGINS = discover_plugins(_cls=Descriptor)
else:
    CLASSIFIER_PLUGINS = {}
    PROPERTY_PLUGINS = {}
    DESCRIPTOR_PLUGINS = {}

SENTRY_ENABLED = os.environ.get('SENTRY_ENABLED', 'False') == 'True'
if SENTRY_ENABLED:
    import sentry_sdk

    def before_send(event, hint):
        # Check if was a handled exception by one of our loggers
        if event.get('logger'):
            for log_path in LOGGING.get('loggers').keys():
                if event['logger'].startswith(log_path):
                    return None

        return event

    sentry_sdk.init(
        dsn=os.environ.get('SENTRY_DSN'),
        # Add data like request headers and IP for users,
        # see https://docs.sentry.io/platforms/python/data-management/data-collected/ for more info
        send_default_pii=True,
        environment=os.environ.get('SENTRY_ENVIRONMENT', 'development'),
        before_send=before_send,
    )

# compile into digestible flags
FLAGS = {
    'MODEL_BUILDING': MODEL_BUILDING_ENABLED,
    'CELERY': FLAG_CELERY_PRESENT,
    'PLUGINS': PLUGINS_ENABLED,
    'SENTRY': SENTRY_ENABLED,
    'ENVIFORMER': ENVIFORMER_PRESENT,
    'APPLICABILITY_DOMAIN': APPLICABILITY_DOMAIN_ENABLED,
}