Current Dev State

envipath/__init__.py

@@ -0,0 +1,5 @@
+# This will make sure the app is always imported when
+# Django starts so that shared_task will use this app.
+from .celery import app as celery_app
+
+__all__ = ('celery_app',)

envipath/api.py

@@ -0,0 +1,7 @@
+from epdb.api import router as epdb_app_router
+from ninja import NinjaAPI
+
+api = NinjaAPI()
+
+api.add_router("/", epdb_app_router)
+

envipath/asgi.py

@@ -0,0 +1,16 @@
+"""
+ASGI config for envipath project.
+
+It exposes the ASGI callable as a module-level variable named ``application``.
+
+For more information on this file, see
+https://docs.djangoproject.com/en/4.2/howto/deployment/asgi/
+"""
+
+import os
+
+from django.core.asgi import get_asgi_application
+
+os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'envipath.settings')
+
+application = get_asgi_application()

envipath/celery.py

@@ -0,0 +1,27 @@
+import os
+
+from celery import Celery
+from celery.signals import setup_logging
+
+# Set the default Django settings module for the 'celery' program.
+os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'envipath.settings')
+
+app = Celery('envipath')
+
+# Using a string here means the worker doesn't have to serialize
+# the configuration object to child processes.
+# - namespace='CELERY' means all celery-related configuration keys
+#   should have a `CELERY_` prefix.
+app.config_from_object('django.conf:settings', namespace='CELERY')
+
+
+@setup_logging.connect
+def config_loggers(*args, **kwargs):
+    from logging.config import dictConfig
+    from django.conf import settings
+
+    dictConfig(settings.LOGGING)
+
+
+# Load task modules from all registered Django apps.
+app.autodiscover_tasks()

envipath/settings.py

@@ -0,0 +1,282 @@
+"""
+Django settings for envipath project.
+
+Generated by 'django-admin startproject' using Django 4.2.17.
+
+For more information on this file, see
+https://docs.djangoproject.com/en/4.2/topics/settings/
+
+For the full list of settings and their values, see
+https://docs.djangoproject.com/en/4.2/ref/settings/
+"""
+import os
+from pathlib import Path
+
+from dotenv import load_dotenv
+from envipy_plugins import Classifier, Property, Descriptor
+from sklearn.ensemble import RandomForestClassifier
+
+# Build paths inside the project like this: BASE_DIR / 'subdir'.
+BASE_DIR = Path(__file__).resolve().parent.parent
+
+load_dotenv(BASE_DIR / '.env', override=False)
+
+# Quick-start development settings - unsuitable for production
+# See https://docs.djangoproject.com/en/4.2/howto/deployment/checklist/
+
+# SECURITY WARNING: keep the secret key used in production secret!
+SECRET_KEY = '7!VTW`aZqg/UBLsM.P=m)2]lWqg>{+:xUgG1"WO@bCyaHR2Up8XW&g<*3.F4l2gi9c.E3}dHyA0D`&z?u#U%^7HYbj],eP"g_MS|3BNMD[mI>s#<i/%2ngZ~Oy+/w&@]'
+
+# SECURITY WARNING: don't run with debug turned on in production!
+DEBUG = os.environ.get('DEBUG', 'False') == 'True'
+ALLOWED_HOSTS = os.environ['ALLOWED_HOSTS'].split(',')
+
+# Application definition
+
+INSTALLED_APPS = [
+    'django.contrib.admin',
+    'django.contrib.auth',
+    'django.contrib.contenttypes',
+    'django.contrib.sessions',
+    'django.contrib.messages',
+    'django.contrib.staticfiles',
+    # 3rd party
+    'django_extensions',
+    # Custom
+    'epdb',
+    'migration',
+]
+
+AUTHENTICATION_BACKENDS = [
+    "django.contrib.auth.backends.ModelBackend",
+]
+
+MIDDLEWARE = [
+    'django.middleware.security.SecurityMiddleware',
+    'django.contrib.sessions.middleware.SessionMiddleware',
+    'django.middleware.common.CommonMiddleware',
+    'django.middleware.csrf.CsrfViewMiddleware',
+    'django.contrib.auth.middleware.AuthenticationMiddleware',
+    'django.contrib.messages.middleware.MessageMiddleware',
+    'django.middleware.clickjacking.XFrameOptionsMiddleware',
+]
+
+ROOT_URLCONF = 'envipath.urls'
+
+TEMPLATES = [
+    {
+        'BACKEND': 'django.template.backends.django.DjangoTemplates',
+        'DIRS': (os.path.join(BASE_DIR, 'templates'),),
+        'APP_DIRS': True,
+        'OPTIONS': {
+            'context_processors': [
+                'django.template.context_processors.debug',
+                'django.template.context_processors.request',
+                'django.contrib.auth.context_processors.auth',
+                'django.contrib.messages.context_processors.messages',
+            ],
+        },
+    },
+]
+
+WSGI_APPLICATION = 'envipath.wsgi.application'
+
+# Database
+# https://docs.djangoproject.com/en/4.2/ref/settings/#databases
+
+DATABASES = {
+    "default": {
+        "ENGINE": "django.db.backends.postgresql",
+        "USER": os.environ['POSTGRES_USER'],
+        "NAME": os.environ['POSTGRES_DB'],
+        "PASSWORD": os.environ['POSTGRES_PASSWORD'],
+        "HOST": os.environ['POSTGRES_SERVICE_NAME'],
+        "PORT": os.environ['POSTGRES_PORT']
+    }
+}
+
+# Password validation
+# https://docs.djangoproject.com/en/4.2/ref/settings/#auth-password-validators
+
+AUTH_PASSWORD_VALIDATORS = [
+    {
+        'NAME': 'django.contrib.auth.password_validation.UserAttributeSimilarityValidator',
+    },
+    {
+        'NAME': 'django.contrib.auth.password_validation.MinimumLengthValidator',
+    },
+    {
+        'NAME': 'django.contrib.auth.password_validation.CommonPasswordValidator',
+    },
+    {
+        'NAME': 'django.contrib.auth.password_validation.NumericPasswordValidator',
+    },
+]
+
+# Internationalization
+# https://docs.djangoproject.com/en/4.2/topics/i18n/
+
+LANGUAGE_CODE = 'en-us'
+
+TIME_ZONE = 'UTC'
+
+USE_I18N = True
+
+USE_TZ = True
+
+
+
+# Default primary key field type
+# https://docs.djangoproject.com/en/4.2/ref/settings/#default-auto-field
+
+DEFAULT_AUTO_FIELD = 'django.db.models.BigAutoField'
+
+# EMAIL
+EMAIL_BACKEND = 'django.core.mail.backends.smtp.EmailBackend'
+EMAIL_USE_TLS = True
+EMAIL_HOST = 'mail.gandi.net'
+EMAIL_HOST_USER = os.environ['EMAIL_HOST_USER']
+EMAIL_HOST_PASSWORD = os.environ['EMAIL_HOST_PASSWORD']
+EMAIL_PORT = 587
+
+AUTH_USER_MODEL = "epdb.User"
+
+# # SESAME
+# SESAME_MAX_AGE = 300
+# # TODO set to "home"
+# LOGIN_REDIRECT_URL = "/"
+# LOGIN_URL = '/login/'
+
+SERVER_URL = os.environ.get('SERVER_URL', 'http://localhost:8000')
+
+CSRF_TRUSTED_ORIGINS = [SERVER_URL]
+
+AMBIT_URL = 'http://localhost:9001'
+DEFAULT_VALUES = {
+    'description': 'no description'
+}
+
+EP_DATA_DIR = os.environ['EP_DATA_DIR']
+MODEL_DIR = os.path.join(EP_DATA_DIR, 'models')
+if not os.path.exists(MODEL_DIR):
+    os.mkdir(MODEL_DIR)
+
+STATIC_DIR = os.path.join(EP_DATA_DIR, 'static')
+if not os.path.exists(STATIC_DIR):
+    os.mkdir(STATIC_DIR)
+
+LOG_DIR = os.path.join(EP_DATA_DIR, 'log')
+if not os.path.exists(LOG_DIR):
+    os.mkdir(LOG_DIR)
+
+PLUGIN_DIR = os.path.join(EP_DATA_DIR, 'plugins')
+if not os.path.exists(PLUGIN_DIR):
+    os.mkdir(PLUGIN_DIR)
+
+# Set this as our static root dir
+STATIC_ROOT = STATIC_DIR
+
+STATIC_URL = '/static/'
+
+# Where the sources are stored...
+STATICFILES_DIRS = (
+    BASE_DIR / 'static',
+)
+
+FIXTURE_DIRS = (
+        BASE_DIR / 'fixtures',
+)
+
+# Logging
+LOGGING = {
+    "version": 1,
+    "disable_existing_loggers": True,
+    "formatters": {
+        "simple": {
+            'format': '[%(asctime)s] %(levelname)s %(module)s - %(message)s',
+            'datefmt': '%Y-%m-%d %H:%M:%S',
+        },
+    },
+    "handlers": {
+        "console": {
+            "level": "INFO",
+            "class": "logging.StreamHandler",
+            "formatter": "simple",
+        },
+        "file": {
+            "level": "DEBUG",  # Or higher
+            "class": "logging.FileHandler",
+            "filename": os.path.join(LOG_DIR, "debug.log"),
+            "formatter": "simple"
+        },
+    },
+    "loggers": {
+        # For everything under epdb/ loaded via getlogger(__name__)
+        "epdb": {
+            "handlers": ["file"],  # "console",
+            "propagate": True,
+            "level": os.environ.get('LOG_LEVEL', 'INFO')
+        },
+        # For everything under envipath/ loaded via getlogger(__name__)
+        'envipath': {
+            'handlers': ['file', 'console'],
+            'propagate': True,
+            'level': os.environ.get('LOG_LEVEL', 'INFO')
+        },
+        # For everything under utilities/ loaded via getlogger(__name__)
+        'utilities': {
+            'handlers': ['file', 'console'],
+            'propagate': True,
+            'level': os.environ.get('LOG_LEVEL', 'INFO')
+        },
+    },
+}
+
+# Flags
+ENVIFORMER_PRESENT = os.environ.get('ENVIFORMER_PRESENT', 'False') == 'True'
+if ENVIFORMER_PRESENT:
+    print("Loading enviFormer")
+    from enviformer import load
+    ENVIFORMER_INSTANCE = load()
+    print("loaded")
+
+
+# If celery is not present set always eager to true which will cause delayed tasks to block until finished
+FLAG_CELERY_PRESENT = os.environ.get('FLAG_CELERY_PRESENT', 'False') == 'True'
+if not FLAG_CELERY_PRESENT:
+    CELERY_TASK_ALWAYS_EAGER = True
+
+# Celery Configuration Options
+CELERY_TIMEZONE = "Europe/Berlin"
+# Celery Configuration
+CELERY_BROKER_URL = 'redis://localhost:6379/0'  # Use Redis as message broker
+CELERY_RESULT_BACKEND = 'redis://localhost:6379/1'
+CELERY_ACCEPT_CONTENT = ['json']
+CELERY_TASK_SERIALIZER = 'json'
+
+DEFAULT_MODELS_PARAMS = {
+    'base_clf': RandomForestClassifier(n_estimators=100,
+                                       max_features='log2',
+                                       random_state=42,
+                                       criterion='entropy',
+                                       ccp_alpha=0.0,
+                                       max_depth=3,
+                                       min_samples_leaf=1),
+    'num_chains': 10,
+}
+
+DEFAULT_MAX_NUMBER_OF_NODES = 30
+DEFAULT_MAX_DEPTH = 5
+DEFAULT_MODEL_THRESHOLD = 0.25
+
+# Loading Plugins
+PLUGINS_ENABLED = os.environ.get('PLUGINS_ENABLED', 'False') == 'True'
+if PLUGINS_ENABLED:
+    from utilities.plugin import discover_plugins
+    CLASSIFIER_PLUGINS = discover_plugins(_cls=Classifier)
+    PROPERTY_PLUGINS = discover_plugins(_cls=Property)
+    DESCRIPTOR_PLUGINS = discover_plugins(_cls=Descriptor)
+else:
+    CLASSIFIER_PLUGINS = {}
+    PROPERTY_PLUGINS = {}
+    DESCRIPTOR_PLUGINS = {}

envipath/urls.py

@@ -0,0 +1,27 @@
+"""
+URL configuration for envipath project.
+
+The `urlpatterns` list routes URLs to views. For more information please see:
+    https://docs.djangoproject.com/en/4.2/topics/http/urls/
+Examples:
+Function views
+    1. Add an import:  from my_app import views
+    2. Add a URL to urlpatterns:  path('', views.home, name='home')
+Class-based views
+    1. Add an import:  from other_app.views import Home
+    2. Add a URL to urlpatterns:  path('', Home.as_view(), name='home')
+Including another URLconf
+    1. Import the include() function: from django.urls import include, path
+    2. Add a URL to urlpatterns:  path('blog/', include('blog.urls'))
+"""
+from django.contrib import admin
+from django.urls import include, path
+
+from .api import api
+
+urlpatterns = [
+    path("", include("epdb.urls")),
+    path("", include("migration.urls")),
+    path("admin/", admin.site.urls),
+    path("api/", api.urls),
+]

envipath/wsgi.py

@@ -0,0 +1,16 @@
+"""
+WSGI config for envipath project.
+
+It exposes the WSGI callable as a module-level variable named ``application``.
+
+For more information on this file, see
+https://docs.djangoproject.com/en/4.2/howto/deployment/wsgi/
+"""
+
+import os
+
+from django.core.wsgi import get_wsgi_application
+
+os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'envipath.settings')
+
+application = get_wsgi_application()