[Fix] Simplify Depth adjustment (#386)

Co-authored-by: Tim Lorsbach <tim@lorsba.ch>
Reviewed-on: enviPath/enviPy#386

.dockerignore

@@ -0,0 +1,93 @@
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+*$py.class
+
+# uv / virtual environments
+.venv/
+venv/
+env/
+ENV/
+
+# uv cache
+.uv/
+uv.lock.bak
+
+# Distribution / packaging
+.Python
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Test / coverage
+htmlcov/
+.tox/
+.nox/
+.coverage
+.coverage.*
+.cache/
+pytest_cache/
+nosetests.xml
+coverage.xml
+*.cover
+*.py,cover
+
+# Type checkers
+.mypy_cache/
+.pyre/
+.pytype/
+
+# Jupyter Notebook
+.ipynb_checkpoints
+
+# Environment variables
+.env
+.env.*
+*.env
+
+# IDEs / editors
+.vscode/
+.idea/
+*.swp
+*.swo
+*~
+
+# OS files
+.DS_Store
+Thumbs.db
+
+# Git
+.git/
+.gitignore
+.gitea/
+
+# Docker
+Dockerfile
+docker-compose.yml
+docker-compose.yaml
+
+# Logs
+*.log
+
+# Temporary files
+tmp/
+temp/
+
+# Frontend stuff
+node_modules/

.env.local.example

@@ -20,3 +20,16 @@ LOG_LEVEL='INFO'
 SERVER_URL='http://localhost:8000'
 PLUGINS_ENABLED=True
 EP_DATA_DIR='data'
+EMAIL_HOST_USER='admin@envipath.com'
+EMAIL_HOST_PASSWORD='dummy-password'
+
+DEFAULT_FROM_EMAIL="test@test.com"
+SERVER_EMAIL='test@test.com'
+
+# Testing settings VScode
+DJANGO_SETTINGS_MODULE='envipath.settings'
+MANAGE_PY_PATH='./manage.py'
+
+APPLICABILITY_DOMAIN_ENABLED=True
+ENVIFORMER_PRESENT=True
+MODEL_BUILDING_ENABLED=True

.env.prod.example

@@ -3,10 +3,20 @@ EP_DATA_DIR=
 ALLOWED_HOSTS=
 DEBUG=
 LOG_LEVEL=
+MODEL_BUILDING_ENABLED=
+APPLICABILITY_DOMAIN_ENABLED=
 ENVIFORMER_PRESENT=
-FLAG_CELERY_PRESENT=
+ENVIFORMER_DEVICE=
-SERVER_URL=
 PLUGINS_ENABLED=
+SERVER_URL=
+SERVER_PATH=
+ADMIN_APPROVAL_REQUIRED=
+REGISTRATION_MANDATORY=
+LOG_DIR=
+# Celery
+FLAG_CELERY_PRESENT=
+CELERY_BROKER_URL=
+CELERY_RESULT_BACKEND=
 # DB
 POSTGRES_SERVICE_NAME=
 POSTGRES_DB=
@@ -16,5 +26,30 @@ POSTGRES_PORT=
 # MAIL
 EMAIL_HOST_USER=
 EMAIL_HOST_PASSWORD=
-# MATOMO
+DEFAULT_FROM_EMAIL=
-MATOMO_SITE_ID
+SERVER_EMAIL=
+# SENTRY
+SENTRY_ENABLED=
+SENTRY_DSN=
+SENTRY_ENVIRONMENT=
+# MS ENTRA
+MS_ENTRA_ENABLED=
+MS_CLIENT_ID=
+MS_CLIENT_SECRET=
+MS_TENANT_ID=
+MS_REDIRECT_URI=
+MS_SCOPES=
+# Tenant
+TENANT=
+EPDB_PACKAGE_MODEL=
+# Captcha
+CAP_ENABLED=
+CAP_API_BASE=
+CAP_SITE_KEY=
+CAP_SECRET_KEY=
+# QUARKUS (JAVA)
+ENVIRULE_ENABLED=
+FINGERPRINT_URL=
+# Biotransformer
+BIOTRANSFORMER_ENABLED=
+BIOTRANSFORMER_URL=

.gitea/actions/setup-envipy/action.yaml

@@ -0,0 +1,67 @@
+name: 'Setup enviPy Environment'
+description: 'Shared setup for enviPy CI - installs dependencies and prepares environment'
+
+inputs:
+  skip-frontend:
+    description: 'Skip frontend build steps (pnpm, tailwind)'
+    required: false
+    default: 'false'
+  skip-playwright:
+    description: 'Skip Playwright installation'
+    required: false
+    default: 'false'
+  ssh-private-key:
+    description: 'SSH private key for git access'
+    required: true
+  run-migrations:
+    description: 'Run Django migrations after setup'
+    required: false
+    default: 'true'
+
+runs:
+  using: "composite"
+  steps:
+    - name: Setup ssh
+      shell: bash
+      run: |
+        mkdir -p ~/.ssh
+        echo "${{ inputs.ssh-private-key }}" > ~/.ssh/id_ed25519
+        chmod 600 ~/.ssh/id_ed25519
+        ssh-keyscan git.envipath.com >> ~/.ssh/known_hosts
+        eval $(ssh-agent -s)
+        ssh-add ~/.ssh/id_ed25519
+
+    - name: Setup Python venv
+      shell: bash
+      run: |
+        uv sync --locked --all-extras --dev
+
+    - name: Install Playwright
+      if: inputs.skip-playwright == 'false'
+      shell: bash
+      run: |
+        source .venv/bin/activate
+        playwright install --with-deps
+
+    - name: Build Frontend
+      if: inputs.skip-frontend == 'false'
+      shell: bash
+      run: |
+        uv run python scripts/pnpm_wrapper.py install
+        uv run python scripts/pnpm_wrapper.py run build
+
+    - name: Wait for Postgres
+      shell: bash
+      run: |
+        until pg_isready -h postgres -U ${{ env.POSTGRES_USER }}; do
+          echo "Waiting for postgres..."
+          sleep 2
+        done
+        echo "Postgres is ready!"
+
+    - name: Run Django Migrations
+      if: inputs.run-migrations == 'true'
+      shell: bash
+      run: |
+        source .venv/bin/activate
+        python manage.py migrate --noinput

.gitea/docker/Dockerfile.ci

@@ -0,0 +1,53 @@
+# Custom CI Docker image for Gitea runners
+# Pre-installs Node.js 24, pnpm 10, uv, and system dependencies
+# to eliminate setup time in CI workflows
+
+FROM ubuntu:24.04
+
+# Prevent interactive prompts during package installation
+ENV DEBIAN_FRONTEND=noninteractive
+
+# Install system dependencies
+RUN apt-get update && \
+    apt-get install -y \
+      postgresql-client \
+      redis-tools \
+      openjdk-11-jre-headless \
+      curl \
+      ca-certificates \
+      gnupg \
+      lsb-release \
+      git \
+      ssh \
+      libxrender1 \
+      libxext6 \
+      libfontconfig1 \
+      libfreetype6 \
+      libcairo2 \
+      libglib2.0-0t64 \
+      && rm -rf /var/lib/apt/lists/*
+
+# Install Node.js 24 via NodeSource
+RUN curl -fsSL https://deb.nodesource.com/setup_24.x | bash - && \
+    apt-get install -y nodejs && \
+    rm -rf /var/lib/apt/lists/*
+
+# Enable corepack and install pnpm 10
+RUN corepack enable && \
+    corepack prepare pnpm@10 --activate
+
+# Install uv https://docs.astral.sh/uv/guides/integration/docker/#available-images
+COPY --from=ghcr.io/astral-sh/uv:latest /uv /uvx /bin/
+ENV PATH="/root/.cargo/bin:${PATH}"
+
+# Verify installations
+RUN node --version && \
+    npm --version && \
+    pnpm --version && \
+    uv --version && \
+    pg_isready --version && \
+    redis-cli --version && \
+    java -version
+
+# Set working directory
+WORKDIR /workspace

.gitea/workflows/api-ci.yaml

@@ -0,0 +1,86 @@
+name: API CI
+
+on:
+  pull_request:
+    branches:
+      - develop
+    paths:
+      - 'epapi/**'
+      - 'epdb/models.py'  # API depends on models
+      - 'epdb/logic.py'   # API depends on business logic
+      - 'tests/fixtures/**'  # API tests use fixtures
+  workflow_dispatch:
+
+jobs:
+  api-tests:
+    if: ${{ !contains(gitea.event.pull_request.title, 'WIP') }}
+    runs-on: ubuntu-latest
+    container:
+      image: git.envipath.com/envipath/envipy-ci:latest
+
+    services:
+      postgres:
+        image: postgres:16
+        env:
+          POSTGRES_USER: ${{ vars.POSTGRES_USER }}
+          POSTGRES_PASSWORD: ${{ secrets.POSTGRES_PASSWORD }}
+          POSTGRES_DB: ${{ vars.POSTGRES_DB }}
+        ports:
+          - ${{ vars.POSTGRES_PORT}}:5432
+        options: >-
+          --health-cmd="pg_isready -U postgres"
+          --health-interval=10s
+          --health-timeout=5s
+          --health-retries=5
+
+    env:
+      RUNNER_TOOL_CACHE: /toolcache
+      EP_DATA_DIR: /opt/enviPy/
+      ALLOWED_HOSTS: 127.0.0.1,localhost
+      DEBUG: True
+      LOG_LEVEL: INFO
+      MODEL_BUILDING_ENABLED: True
+      APPLICABILITY_DOMAIN_ENABLED: True
+      ENVIFORMER_PRESENT: True
+      ENVIFORMER_DEVICE: cpu
+      FLAG_CELERY_PRESENT: False
+      PLUGINS_ENABLED: True
+      SERVER_URL: http://localhost:8000
+      ADMIN_APPROVAL_REQUIRED: True
+      REGISTRATION_MANDATORY: True
+      LOG_DIR: ''
+      # DB
+      POSTGRES_SERVICE_NAME: postgres
+      POSTGRES_DB: ${{ vars.POSTGRES_DB }}
+      POSTGRES_USER: ${{ vars.POSTGRES_USER }}
+      POSTGRES_PASSWORD: ${{ secrets.POSTGRES_PASSWORD }}
+      POSTGRES_PORT: 5432
+      # SENTRY
+      SENTRY_ENABLED: False
+      # MS ENTRA
+      MS_ENTRA_ENABLED: False
+
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+
+      # Use shared setup action - skips frontend builds for API-only tests
+      - name: Setup enviPy Environment
+        uses: ./.gitea/actions/setup-envipy
+        with:
+          skip-frontend: 'true'
+          skip-playwright: 'false'
+          ssh-private-key: ${{ secrets.ENVIPY_CI_PRIVATE_KEY }}
+          run-migrations: 'true'
+
+      - name: Run API tests
+        run: |
+          .venv/bin/python manage.py test epapi -v 2
+
+      - name: Test API endpoints availability
+        run: |
+          .venv/bin/python manage.py runserver 0.0.0.0:8000 &
+          SERVER_PID=$!
+          sleep 5
+          curl -f http://localhost:8000/api/v1/docs || echo "API docs not available"
+          kill $SERVER_PID

.gitea/workflows/build-ci-image.yaml

@@ -0,0 +1,48 @@
+name: Build CI Docker Image
+
+on:
+  workflow_dispatch:
+  push:
+    branches:
+      - develop
+      - main
+    paths:
+      - '.gitea/docker/Dockerfile.ci'
+      - '.gitea/workflows/build-ci-image.yaml'
+
+jobs:
+  build-and-push:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+
+      - name: Log in to container registry
+        uses: docker/login-action@v3
+        with:
+          registry: git.envipath.com
+          username: ${{ secrets.CI_REGISTRY_USER }}
+          password: ${{ secrets.CI_REGISTRY_PASSWORD }}
+
+      - name: Extract metadata
+        id: meta
+        uses: docker/metadata-action@v5
+        with:
+          images: git.envipath.com/envipath/envipy-ci
+          tags: |
+            type=raw,value=latest
+            type=sha,prefix={{branch}}-
+
+      - name: Build and push Docker image
+        uses: docker/build-push-action@v5
+        with:
+          context: .
+          file: .gitea/docker/Dockerfile.ci
+          push: true
+          tags: ${{ steps.meta.outputs.tags }}
+          labels: ${{ steps.meta.outputs.labels }}
+          cache-from: type=registry,ref=git.envipath.com/envipath/envipy-ci:latest
+          cache-to: type=inline

.gitea/workflows/ci.yaml

@@ -8,7 +8,10 @@ on:
 
 jobs:
   test:
+    if: ${{ !contains(gitea.event.pull_request.title, 'WIP') }}
     runs-on: ubuntu-latest
+    container:
+      image: git.envipath.com/envipath/envipy-ci:latest
 
     services:
       postgres:
@@ -40,7 +43,7 @@ jobs:
       EP_DATA_DIR: /opt/enviPy/
       ALLOWED_HOSTS: 127.0.0.1,localhost
       DEBUG: True
-      LOG_LEVEL: DEBUG
+      LOG_LEVEL: INFO
       MODEL_BUILDING_ENABLED: True
       APPLICABILITY_DOMAIN_ENABLED: True
       ENVIFORMER_PRESENT: True
@@ -63,54 +66,22 @@ jobs:
       MS_ENTRA_ENABLED: False
 
     steps:
-
       - name: Checkout repository
         uses: actions/checkout@v4
 
-      - name: Install system tools via apt
+      # Use shared setup action - includes all dependencies and migrations
-        run: |
+      - name: Setup enviPy Environment
-          sudo apt-get update
+        uses: ./.gitea/actions/setup-envipy
-          sudo apt-get install -y postgresql-client redis-tools openjdk-11-jre-headless
-
-      - name: Setup ssh
-        run: |
-          echo "${{ secrets.ENVIPY_CI_PRIVATE_KEY }}" > ~/.ssh/id_ed25519
-          chmod 600 ~/.ssh/id_ed25519
-          ssh-keyscan git.envipath.com >> ~/.ssh/known_hosts
-          eval $(ssh-agent -s)
-          ssh-add ~/.ssh/id_ed25519
-
-      - name: Install pnpm
-        uses: pnpm/action-setup@v4
         with:
-          version: 10
+          skip-frontend: 'false'
+          skip-playwright: 'false'
+          ssh-private-key: ${{ secrets.ENVIPY_CI_PRIVATE_KEY }}
+          run-migrations: 'true'
 
-      - name: Use Node.js
+      - name: Run frontend tests
-        uses: actions/setup-node@v4
-        with:
-          node-version: 20
-          cache: "pnpm"
-
-      - name: Install uv
-        uses: astral-sh/setup-uv@v6
-        with:
-          enable-cache: true
-
-      - name: Setup venv
         run: |
-          uv sync --locked --all-extras --dev
+          .venv/bin/python manage.py test --tag frontend
-
-      - name: Wait for services
-        run: |
-          until pg_isready -h postgres -U postgres; do sleep 2; done
-          # until redis-cli -h redis ping; do sleep 2; done
-
-      - name: Run Django Migrations
-        run: |
-          source .venv/bin/activate
-          python manage.py migrate --noinput
 
       - name: Run Django tests
         run: |
-          source .venv/bin/activate
+          .venv/bin/python manage.py test tests --exclude-tag slow --exclude-tag frontend
-          python manage.py test tests --exclude-tag slow

.gitignore

@@ -1,17 +1,375 @@
-*.pyc
+
+
+
+### Python ###
+
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[codz]
+*$py.class
+
+# C extensions
+*.so
+
+# Distribution / packaging
+.Python
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+share/python-wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+MANIFEST
+
+# PyInstaller
+#   Usually these files are written by a python script from a template
+#   before PyInstaller builds the exe, so as to inject date/other infos into it.
+*.manifest
+*.spec
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.nox/
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*.cover
+*.py.cover
+.hypothesis/
+.pytest_cache/
+cover/
+
+# Translations
+*.mo
+*.pot
+
+# Django stuff:
+*.log
+local_settings.py
 db.sqlite3
-.idea/
+db.sqlite3-journal
 static/admin/
 static/django_extensions/
+
+# Flask stuff:
+instance/
+.webassets-cache
+
+# Scrapy stuff:
+.scrapy
+
+# Sphinx documentation
+docs/_build/
+
+# PyBuilder
+.pybuilder/
+target/
+
+# Jupyter Notebook
+.ipynb_checkpoints
+
+# IPython
+profile_default/
+ipython_config.py
+
+
+# pdm
+#   Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control.
+#   pdm recommends including project-wide configuration in pdm.toml, but excluding .pdm-python.
+#   https://pdm-project.org/en/latest/usage/project/#working-with-version-control
+# pdm.lock
+# pdm.toml
+.pdm-python
+.pdm-build/
+
+# pixi
+#   Similar to Pipfile.lock, it is generally recommended to include pixi.lock in version control.
+# pixi.lock
+#   Pixi creates a virtual environment in the .pixi directory, just like venv module creates one
+#   in the .venv directory. It is recommended not to include this directory in version control.
+.pixi
+
+# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm
+__pypackages__/
+
+# Celery stuff
+celerybeat-schedule
+celerybeat.pid
+
+# Redis
+*.rdb
+*.aof
+*.pid
+
+# RabbitMQ
+mnesia/
+rabbitmq/
+rabbitmq-data/
+
+# ActiveMQ
+activemq-data/
+
+# SageMath parsed files
+*.sage.py
+
+# Environments
 .env
+.envrc
+.venv
+env/
+venv/
+ENV/
+env.bak/
+venv.bak/
+
+# Spyder project settings
+.spyderproject
+.spyproject
+
+# Rope project settings
+.ropeproject
+
+# mkdocs documentation
+/site
+
+# mypy
+.mypy_cache/
+.dmypy.json
+dmypy.json
+
+# Pyre type checker
+.pyre/
+
+# pytype static type analyzer
+.pytype/
+
+# Cython debug symbols
+cython_debug/
+
+# PyCharm
+#   JetBrains specific template is maintained in a separate JetBrains.gitignore that can
+#   be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore
+#   and can be added to the global gitignore or merged into this file.  For a more nuclear
+#   option (not recommended) you can uncomment the following to ignore the entire idea folder.
+.idea/
+
+# Abstra
+#   Abstra is an AI-powered process automation framework.
+#   Ignore directories containing user credentials, local state, and settings.
+#   Learn more at https://abstra.io/docs
+.abstra/
+
+# Visual Studio Code
+#   Visual Studio Code specific template is maintained in a separate VisualStudioCode.gitignore
+#   that can be found at https://github.com/github/gitignore/blob/main/Global/VisualStudioCode.gitignore
+#   and can be added to the global gitignore or merged into this file. However, if you prefer,
+#   you could uncomment the following to ignore the entire vscode folder
+.vscode/
+*.code-workspace
+
+# Ruff stuff:
+.ruff_cache/
+
+# UV cache
+.uv-cache/
+
+# PyPI configuration file
+.pypirc
+
+# Marimo
+marimo/_static/
+marimo/_lsp/
+__marimo__/
+
+# Streamlit
+.streamlit/secrets.toml
+
+### Agents ###
+.claude/
+.codex/
+.cursor/
+.github/prompts/
+.junie/
+.windsurf/
+
+AGENTS.md
+CLAUDE.md
+GEMINI.md
+.aider.*
+
+### Node.js ###
+
+# Logs
+logs
+*.log
+npm-debug.log*
+yarn-debug.log*
+yarn-error.log*
+lerna-debug.log*
+
+# Diagnostic reports (https://nodejs.org/api/report.html)
+report.[0-9]*.[0-9]*.[0-9]*.[0-9]*.json
+
+# Runtime data
+pids
+*.pid
+*.seed
+*.pid.lock
+
+# Directory for instrumented libs generated by jscoverage/JSCover
+lib-cov
+
+# Coverage directory used by tools like istanbul
+coverage
+*.lcov
+
+# nyc test coverage
+.nyc_output
+
+# Grunt intermediate storage (https://gruntjs.com/creating-plugins#storing-task-files)
+.grunt
+
+# Bower dependency directory (https://bower.io/)
+bower_components
+
+# node-waf configuration
+.lock-wscript
+
+# Compiled binary addons (https://nodejs.org/api/addons.html)
+build/Release
+
+# Dependency directories
+node_modules/
+jspm_packages/
+
+# Snowpack dependency directory (https://snowpack.dev/)
+web_modules/
+
+# TypeScript cache
+*.tsbuildinfo
+
+# Optional npm cache directory
+.npm
+
+# Optional eslint cache
+.eslintcache
+
+# Optional stylelint cache
+.stylelintcache
+
+# Optional REPL history
+.node_repl_history
+
+# Output of 'npm pack'
+*.tgz
+
+# Yarn Integrity file
+.yarn-integrity
+
+# dotenv environment variable files
+.env
+.env.*
+!.env.example
+
+# parcel-bundler cache (https://parceljs.org/)
+.cache
+.parcel-cache
+
+# Next.js build output
+.next
+out
+
+# Nuxt.js build / generate output
+.nuxt
+dist
+.output
+
+# Gatsby files
+.cache/
+# Comment in the public line in if your project uses Gatsby and not Next.js
+# https://nextjs.org/blog/next-9-1#public-directory-support
+# public
+
+# vuepress build output
+.vuepress/dist
+
+# vuepress v2.x temp and cache directory
+.temp
+.cache
+
+# Sveltekit cache directory
+.svelte-kit/
+
+# vitepress build output
+**/.vitepress/dist
+
+# vitepress cache directory
+**/.vitepress/cache
+
+# Docusaurus cache and generated files
+.docusaurus
+
+# Serverless directories
+.serverless/
+
+# FuseBox cache
+.fusebox/
+
+# DynamoDB Local files
+.dynamodb/
+
+# Firebase cache directory
+.firebase/
+
+# TernJS port file
+.tern-port
+
+# Stores VSCode versions used for testing VSCode extensions
+.vscode-test
+
+# yarn v3
+.pnp.*
+.yarn/*
+!.yarn/patches
+!.yarn/plugins
+!.yarn/releases
+!.yarn/sdks
+!.yarn/versions
+
+# Vite files
+vite.config.js.timestamp-*
+vite.config.ts.timestamp-*
+.vite/
+
+### Custom ###
+
 debug.log
 scratches/
-
+test-results/
 data/
+*.arff
 
-.DS_Store
+# Auto generated
-
-node_modules/
 static/css/output.css
 
-*.code-workspace
+# macOS system files
+.DS_Store
+.Trashes
+._*

.pre-commit-config.yaml

@@ -5,10 +5,12 @@ repos:
     rev: v3.2.0
     hooks:
       - id: trailing-whitespace
+        exclude: epiuclid/schemas/
       - id: end-of-file-fixer
+        exclude: epiuclid/schemas/
       - id: check-yaml
       - id: check-added-large-files
-        exclude: ^static/images/
+        exclude: ^static/images/|^epiuclid/schemas/|^fixtures/
 
   - repo: https://github.com/astral-sh/ruff-pre-commit
     rev: v0.13.3

Dockerfile

@@ -0,0 +1,98 @@
+FROM python:3.12-slim AS builder
+
+ENV PYTHONDONTWRITEBYTECODE=1 \
+    PYTHONUNBUFFERED=1 \
+    UV_LINK_MODE=copy
+
+WORKDIR /app
+
+RUN apt-get update && apt-get install -y --no-install-recommends \
+    build-essential \
+    libpq-dev \
+    curl \
+    openssh-client \
+    git \
+    nodejs \
+    npm \
+    && rm -rf /var/lib/apt/lists/*
+
+# Install pnpm
+RUN npm install -g pnpm
+
+RUN curl -LsSf https://astral.sh/uv/install.sh | sh
+ENV PATH="/root/.local/bin:${PATH}"
+
+# Install dependencies first (cached layer — only invalidated when lockfile changes)
+COPY pyproject.toml uv.lock ./
+
+# Add key from git.envipath.com to known_hosts
+RUN mkdir -p -m 0700 /root/.ssh \
+    && ssh-keyscan git.envipath.com >> /root/.ssh/known_hosts
+
+# We'll need access to private repos, let docker make use of host ssh agent and use it like:
+# docker build --ssh default -t envipath/envipy:1.0 .
+RUN --mount=type=ssh \
+    uv sync --locked --extra ms-login --extra pepper-plugin
+
+# Now copy source and do a final sync to install the project itself
+# Ensure .dockerignore is reasonable
+COPY biotransformer biotransformer
+COPY bridge bridge
+COPY envipath envipath
+COPY epapi epapi
+COPY epauth epauth
+COPY epdb epdb
+COPY epiuclid epiuclid
+COPY fixtures fixtures
+COPY migration migration
+COPY pepper pepper
+COPY scripts scripts
+COPY static static
+COPY templates templates
+COPY tests tests
+COPY utilities utilities
+COPY manage.py .
+
+# Install frontend deps
+COPY package.json pnpm-lock.yaml pnpm-workspace.yaml ./
+
+# Build frontend assets
+RUN uv run python scripts/pnpm_wrapper.py install
+RUN uv run python scripts/pnpm_wrapper.py run build
+
+FROM python:3.12-slim AS production
+
+ENV PYTHONDONTWRITEBYTECODE=1 \
+    PYTHONUNBUFFERED=1 \
+    PATH="/app/.venv/bin:$PATH"
+
+WORKDIR /app
+
+RUN apt-get update && apt-get install -y --no-install-recommends \
+    libpq5 \
+    libxrender1 \
+    libxext6 \
+    libfontconfig1 \
+    nano \
+    && rm -rf /var/lib/apt/lists/*
+
+RUN useradd -ms /bin/bash django
+
+# Create directories in /opt and set ownership
+RUN mkdir -p /opt/enviPy \
+    && mkdir -p /opt/enviPy/celery \
+    && mkdir -p /opt/enviPy/log \
+    && mkdir -p /opt/enviPy/models \
+    && mkdir -p /opt/enviPy/plugins \
+    && mkdir -p /opt/enviPy/static \
+    && chown -R django:django /opt/enviPy
+
+COPY --from=builder --chown=django:django /app /app
+
+RUN touch /app/.env && chown -R django:django /app/.env
+
+USER django
+
+EXPOSE 8000
+
+CMD ["gunicorn", "envipath.wsgi:application", "--bind", "0.0.0.0:8000", "--workers", "3"]

README.md

@@ -8,13 +8,12 @@ These instructions will guide you through setting up the project for local devel
 
 - Python 3.11 or later
 - [uv](https://github.com/astral-sh/uv) - Python package manager
-- **Docker and Docker Compose** - Required for running PostgreSQL database
+- **Docker and Docker Compose** - Required for running PostgreSQL database and Redis (for async Celery tasks)
 - Git
 - Make
 
 > **Note:** This application requires PostgreSQL (uses `ArrayField`). Docker is the easiest way to run PostgreSQL locally.
 
-
 ### 1. Install Dependencies
 
 This project uses `uv` to manage dependencies and `poe-the-poet` for task running. First, [install `uv` if you don't have it yet](https://docs.astral.sh/uv/guides/install-python/).
@@ -79,25 +78,48 @@ uv run poe bootstrap     # Bootstrap data only
 uv run poe shell         # Open the Django shell
 uv run poe build         # Build frontend assets and collect static files
 uv run poe clean         # Remove database volumes (WARNING: destroys all data)
+uv run poe celery        # Start Celery worker for async task processing
+uv run poe celery-dev    # Start database and Celery worker
 ```
 
+### 4. Async Celery Setup (Optional)
+
+By default, Celery tasks run synchronously (`CELERY_TASK_ALWAYS_EAGER = True`), which means prediction tasks block the HTTP request until completion. To enable asynchronous task processing with live status updates on pathway pages:
+
+1. **Set the Celery flag in your `.env` file:**
+
+   ```bash
+   FLAG_CELERY_PRESENT=True
+   ```
+
+2. **Start Redis and Celery worker:**
+
+   ```bash
+   uv run poe celery-dev
+   ```
+
+3. **Start the development server** (in another terminal):
+   ```bash
+   uv run poe dev
+   ```
+
 ### Troubleshooting
 
-*   **Docker Connection Error:** If you see an error like `open //./pipe/dockerDesktopLinuxEngine: The system cannot find the file specified` (on Windows), it likely means your Docker Desktop application is not running. Please start Docker Desktop and try the command again.
+- **Docker Connection Error:** If you see an error like `open //./pipe/dockerDesktopLinuxEngine: The system cannot find the file specified` (on Windows), it likely means your Docker Desktop application is not running. Please start Docker Desktop and try the command again.
 
-*   **SSH Keys for Git Dependencies:** Some dependencies are installed from private git repositories and require SSH authentication. Ensure your SSH keys are configured correctly for Git.
+- **SSH Keys for Git Dependencies:** Some dependencies are installed from private git repositories and require SSH authentication. Ensure your SSH keys are configured correctly for Git.
-    *   For a general guide, see [GitHub's official documentation](https://docs.github.com/en/authentication/connecting-to-github-with-ssh/generating-a-new-ssh-key-and-adding-it-to-the-ssh-agent).
+  - For a general guide, see [GitHub's official documentation](https://docs.github.com/en/authentication/connecting-to-github-with-ssh/generating-a-new-ssh-key-and-adding-it-to-the-ssh-agent).
-    *   **Windows Users:** If `uv sync` hangs while fetching git dependencies, you may need to explicitly configure Git to use the Windows OpenSSH client and use the `ssh-agent` to manage your key's passphrase.
+  - **Windows Users:** If `uv sync` hangs while fetching git dependencies, you may need to explicitly configure Git to use the Windows OpenSSH client and use the `ssh-agent` to manage your key's passphrase.
+    1.  **Point Git to the correct SSH executable:**
+        ```powershell
+        git config --global core.sshCommand "C:/Windows/System32/OpenSSH/ssh.exe"
+        ```
+    2.  **Enable and use the SSH agent:**
 
-        1.  **Point Git to the correct SSH executable:**
+        ```powershell
-            ```powershell
+        # Run these commands in an administrator PowerShell
-            git config --global core.sshCommand "C:/Windows/System32/OpenSSH/ssh.exe"
+        Get-Service ssh-agent | Set-Service -StartupType Automatic -PassThru | Start-Service
-            ```
-        2.  **Enable and use the SSH agent:**
-            ```powershell
-            # Run these commands in an administrator PowerShell
-            Get-Service ssh-agent | Set-Service -StartupType Automatic -PassThru | Start-Service
 
-            # Add your key to the agent. It will prompt for the passphrase once.
+        # Add your key to the agent. It will prompt for the passphrase once.
-            ssh-add
+        ssh-add
-            ```
+        ```

biotransformer/__init__.py

@@ -0,0 +1,112 @@
+import enum
+from datetime import datetime
+from typing import List
+
+import requests
+from django.conf import settings as s
+
+# Once stable these will be exposed by enviPy-plugins lib
+from envipy_additional_information import EnviPyModel, UIConfig, WidgetType  # noqa: I001
+from envipy_additional_information import register  # noqa: I001
+
+from bridge.contracts import Classifier  # noqa: I001
+from bridge.dto import (
+    BuildResult,
+    EnviPyDTO,
+    EvaluationResult,
+    RunResult,
+    TransformationProductPrediction,
+)  # noqa: I001
+
+
+class BiotransformerEnvType(enum.Enum):
+    CYP450 = "CYP450"
+    ALLHUMAN = "ALLHUMAN"
+    ECBASED = "ECBASED"
+    HGUT = "HGUT"
+    PHASEII = "PHASEII"
+    ENV = "ENV"
+
+
+@register("biotransformerconfig")
+class BiotransformerConfig(EnviPyModel):
+    env_type: BiotransformerEnvType
+
+    class UI:
+        title = "Biotransformer Type"
+        env_type = UIConfig(widget=WidgetType.SELECT, label="Biotransformer Type", order=1)
+
+
+class Biotransformer(Classifier):
+    Config = BiotransformerConfig
+
+    def __init__(self, config: BiotransformerConfig | None = None):
+        super().__init__(config)
+        self.url = f"{s.BIOTRANSFORMER_URL}/biotransformer"
+
+    @classmethod
+    def requires_rule_packages(cls) -> bool:
+        return False
+
+    @classmethod
+    def requires_data_packages(cls) -> bool:
+        return False
+
+    @classmethod
+    def identifier(cls) -> str:
+        return "biotransformer3"
+
+    @classmethod
+    def name(cls) -> str:
+        return "Biotransformer 3.0"
+
+    @classmethod
+    def display(cls) -> str:
+        return "Biotransformer 3.0"
+
+    def build(self, eP: EnviPyDTO, *args, **kwargs) -> BuildResult | None:
+        return
+
+    def run(self, eP: EnviPyDTO, *args, **kwargs) -> RunResult:
+        smiles = [c.smiles for c in eP.get_compounds()]
+        preds = self._post(smiles)
+
+        results = []
+
+        for substrate in preds.keys():
+            results.append(
+                TransformationProductPrediction(
+                    substrate=substrate,
+                    products=preds[substrate],
+                )
+            )
+
+        return RunResult(
+            producer=eP.get_context().url,
+            description=f"Generated at {datetime.now()}",
+            result=results,
+        )
+
+    def evaluate(self, eP: EnviPyDTO, *args, **kwargs) -> EvaluationResult:
+        pass
+
+    def build_and_evaluate(self, eP: EnviPyDTO, *args, **kwargs) -> EvaluationResult:
+        pass
+
+    def _post(self, smiles: List[str]) -> dict[str, dict[str, float]]:
+        data = {"substrates": smiles, "mode": self.config.env_type.value}
+        res = requests.post(self.url, json=data)
+
+        res.raise_for_status()
+
+        # Example Response JSON:
+        # {
+        #     'products': {
+        #         'CN1C=NC2=C1C(=O)N(C(=O)N2C)C': {
+        #             'CN1C2=C(C(=O)N(C)C1=O)NC=N2': 0.5,
+        #             'CN1C=NC2=C1C(=O)N(C)C(=O)N2.CN1C=NC2=C1C(=O)NC(=O)N2C.CO': 0.5
+        #         }
+        #     }
+        # }
+
+        return res.json()["products"]

bridge/contracts.py

@@ -0,0 +1,400 @@
+import enum
+from abc import ABC, abstractmethod
+
+from envipy_additional_information import EnviPyModel
+
+from .dto import BuildResult, EnviPyDTO, EvaluationResult, RunResult
+
+
+class PropertyType(enum.Enum):
+    """
+    Enumeration representing different types of properties.
+
+    PropertyType is an Enum class that defines categories or types of properties
+    based on their weight or nature. It can typically be used when classifying
+    objects or entities by their weight classification, such as lightweight or heavy.
+    """
+
+    LIGHTWEIGHT = "lightweight"
+    HEAVY = "heavy"
+
+
+class Plugin(ABC):
+    """
+    Defines an abstract base class Plugin to serve as a blueprint for plugins.
+
+    This class establishes the structure that all plugin implementations must
+    follow. It enforces the presence of required methods to ensure consistent
+    functionality across all derived classes.
+
+    """
+
+    @classmethod
+    @abstractmethod
+    def identifier(cls) -> str:
+        pass
+
+    @classmethod
+    @abstractmethod
+    def name(cls) -> str:
+        """
+        Represents an abstract method that provides a contract for implementing a method
+        to return a name as a string. Must be implemented in subclasses.
+        Name must be unique across all plugins.
+
+        Methods
+        -------
+        name() -> str
+            Abstract method to be defined in subclasses, which returns a string
+            representing a name.
+        """
+        pass
+
+    @classmethod
+    @abstractmethod
+    def display(cls) -> str:
+        """
+        An abstract method that must be implemented by subclasses to display
+        specific information or behavior. The method ensures that all subclasses
+        provide their own implementation of the display functionality.
+
+        Raises:
+            NotImplementedError: Raises this error when the method is not implemented
+            in a subclass.
+
+        Returns:
+            str: A string used in dropdown menus or other user interfaces to display
+        """
+        pass
+
+
+class Property(Plugin):
+    @classmethod
+    @abstractmethod
+    def requires_rule_packages(cls) -> bool:
+        """
+        Defines an abstract method to determine whether rule packages are required.
+
+        This method should be implemented by subclasses to specify if they depend
+        on rule packages for their functioning.
+
+        Raises:
+            NotImplementedError: If the subclass has not implemented this method.
+
+        @return: A boolean indicating if rule packages are required.
+        """
+        pass
+
+    @classmethod
+    @abstractmethod
+    def requires_data_packages(cls) -> bool:
+        """
+        Defines an abstract method to determine whether data packages are required.
+
+        This method should be implemented by subclasses to specify if they depend
+        on data packages for their functioning.
+
+        Raises:
+            NotImplementedError: If the subclass has not implemented this method.
+
+        Returns:
+            bool: True if the service requires data packages, False otherwise.
+        """
+        pass
+
+    @abstractmethod
+    def get_type(self) -> PropertyType:
+        """
+        An abstract method that provides the type of property. This method must
+        be implemented by subclasses to specify the appropriate property type.
+
+        Raises:
+            NotImplementedError: If the method is not implemented by a subclass.
+
+        Returns:
+            PropertyType: The type of the property associated with the implementation.
+        """
+        pass
+
+    def is_heavy(self):
+        """
+        Determines if the current property type is heavy.
+
+        This method evaluates whether the property type returned from the `get_type()`
+        method is classified as `HEAVY`. It utilizes the `PropertyType.HEAVY` constant
+        for this comparison.
+
+        Raises:
+            AttributeError: If the `get_type()` method is not defined or does not return
+            a valid value.
+
+        Returns:
+            bool: True if the property type is `HEAVY`, otherwise False.
+        """
+        return self.get_type() == PropertyType.HEAVY
+
+    @abstractmethod
+    def build(self, eP: EnviPyDTO, *args, **kwargs) -> BuildResult | None:
+        """
+        Abstract method to prepare and construct a specific build process based on the provided
+        environment data transfer object (EnviPyDTO). This method should be implemented by
+        subclasses to handle the particular requirements of the environment.
+
+        Parameters:
+        eP : EnviPyDTO
+            The data transfer object containing environment details for the build process.
+
+        *args :
+            Additional positional arguments required for the build.
+
+        **kwargs :
+            Additional keyword arguments to offer flexibility and customization for
+            the build process.
+
+        Returns:
+        BuildResult | None
+            Returns a BuildResult instance if the build operation succeeds, else returns None.
+
+        Raises:
+        NotImplementedError
+            If the method is not implemented in a subclass.
+        """
+        pass
+
+    @abstractmethod
+    def run(self, eP: EnviPyDTO, *args, **kwargs) -> RunResult:
+        """
+        Represents an abstract base class for executing a generic process with
+        provided parameters and returning a standardized result.
+
+        Attributes:
+            None.
+
+        Methods:
+            run(eP, *args, **kwargs):
+                Executes a task with specified input parameters and optional
+                arguments, returning the outcome in the form of a RunResult object.
+                This is an abstract method and must be implemented in subclasses.
+
+        Raises:
+            NotImplementedError: If the subclass does not implement the abstract
+            method.
+
+        Parameters:
+            eP (EnviPyDTO): The primary object containing information or data required
+            for processing. Mandatory.
+            *args: Variable length argument list for additional positional arguments.
+            **kwargs: Arbitrary keyword arguments for additional options or settings.
+
+        Returns:
+            RunResult: A result object encapsulating the status, output, or details
+            of the process execution.
+
+        """
+        pass
+
+    @abstractmethod
+    def evaluate(self, eP: EnviPyDTO, *args, **kwargs) -> EvaluationResult:
+        """
+        Abstract method for evaluating data based on the given input and additional arguments.
+
+        This method is intended to be implemented by subclasses and provides
+        a mechanism to perform an evaluation procedure based on input encapsulated
+        in an EnviPyDTO object.
+
+        Parameters:
+        eP : EnviPyDTO
+            The data transfer object containing necessary input for evaluation.
+        *args : tuple
+            Additional positional arguments for the evaluation process.
+        **kwargs : dict
+            Additional keyword arguments for the evaluation process.
+
+        Returns:
+        EvaluationResult
+            The result of the evaluation performed by the method.
+
+        Raises:
+        NotImplementedError
+            If the method is not implemented in the subclass.
+        """
+        pass
+
+    @abstractmethod
+    def build_and_evaluate(self, eP: EnviPyDTO, *args, **kwargs) -> EvaluationResult:
+        """
+        An abstract method designed to build and evaluate a model or system using the provided
+        environmental parameters and additional optional arguments.
+
+        Args:
+            eP (EnviPyDTO): The environmental parameters required for building and evaluating.
+            *args: Additional positional arguments.
+            **kwargs: Additional keyword arguments.
+
+        Returns:
+            EvaluationResult: The result of the evaluation process.
+
+        Raises:
+            NotImplementedError: If the method is not implemented by a subclass.
+        """
+        pass
+
+
+class Classifier(Plugin):
+    Config: type[EnviPyModel] | None = None
+
+    def __init__(self, config: EnviPyModel | None = None):
+        self.config = config
+
+    @classmethod
+    def has_config(cls) -> bool:
+        return cls.Config is not None
+
+    @classmethod
+    def parse_config(cls, data: dict | None = None) -> EnviPyModel | None:
+        if cls.Config is None:
+            return None
+        return cls.Config(**(data or {}))
+
+    @classmethod
+    def create(cls, data: dict | None = None):
+        return cls(cls.parse_config(data))
+
+    @classmethod
+    @abstractmethod
+    def requires_rule_packages(cls) -> bool:
+        """
+        Defines an abstract method to determine whether rule packages are required.
+
+        This method should be implemented by subclasses to specify if they depend
+        on rule packages for their functioning.
+
+        Raises:
+            NotImplementedError: If the subclass has not implemented this method.
+
+        @return: A boolean indicating if rule packages are required.
+        """
+        pass
+
+    @classmethod
+    @abstractmethod
+    def requires_data_packages(cls) -> bool:
+        """
+        Defines an abstract method to determine whether data packages are required.
+
+        This method should be implemented by subclasses to specify if they depend
+        on data packages for their functioning.
+
+        Raises:
+            NotImplementedError: If the subclass has not implemented this method.
+
+        Returns:
+            bool: True if the service requires data packages, False otherwise.
+        """
+        pass
+
+    @abstractmethod
+    def build(self, eP: EnviPyDTO, *args, **kwargs) -> BuildResult | None:
+        """
+        Abstract method to prepare and construct a specific build process based on the provided
+        environment data transfer object (EnviPyDTO). This method should be implemented by
+        subclasses to handle the particular requirements of the environment.
+
+        Parameters:
+        eP : EnviPyDTO
+            The data transfer object containing environment details for the build process.
+
+        *args :
+            Additional positional arguments required for the build.
+
+        **kwargs :
+            Additional keyword arguments to offer flexibility and customization for
+            the build process.
+
+        Returns:
+        BuildResult | None
+            Returns a BuildResult instance if the build operation succeeds, else returns None.
+
+        Raises:
+        NotImplementedError
+            If the method is not implemented in a subclass.
+        """
+        pass
+
+    @abstractmethod
+    def run(self, eP: EnviPyDTO, *args, **kwargs) -> RunResult:
+        """
+        Represents an abstract base class for executing a generic process with
+        provided parameters and returning a standardized result.
+
+        Attributes:
+            None.
+
+        Methods:
+            run(eP, *args, **kwargs):
+                Executes a task with specified input parameters and optional
+                arguments, returning the outcome in the form of a RunResult object.
+                This is an abstract method and must be implemented in subclasses.
+
+        Raises:
+            NotImplementedError: If the subclass does not implement the abstract
+            method.
+
+        Parameters:
+            eP (EnviPyDTO): The primary object containing information or data required
+            for processing. Mandatory.
+            *args: Variable length argument list for additional positional arguments.
+            **kwargs: Arbitrary keyword arguments for additional options or settings.
+
+        Returns:
+            RunResult: A result object encapsulating the status, output, or details
+            of the process execution.
+
+        """
+        pass
+
+    @abstractmethod
+    def evaluate(self, eP: EnviPyDTO, *args, **kwargs) -> EvaluationResult | None:
+        """
+        Abstract method for evaluating data based on the given input and additional arguments.
+
+        This method is intended to be implemented by subclasses and provides
+        a mechanism to perform an evaluation procedure based on input encapsulated
+        in an EnviPyDTO object.
+
+        Parameters:
+        eP : EnviPyDTO
+            The data transfer object containing necessary input for evaluation.
+        *args : tuple
+            Additional positional arguments for the evaluation process.
+        **kwargs : dict
+            Additional keyword arguments for the evaluation process.
+
+        Returns:
+        EvaluationResult
+            The result of the evaluation performed by the method.
+
+        Raises:
+        NotImplementedError
+            If the method is not implemented in the subclass.
+        """
+        pass
+
+    @abstractmethod
+    def build_and_evaluate(self, eP: EnviPyDTO, *args, **kwargs) -> EvaluationResult | None:
+        """
+        An abstract method designed to build and evaluate a model or system using the provided
+        environmental parameters and additional optional arguments.
+
+        Args:
+            eP (EnviPyDTO): The environmental parameters required for building and evaluating.
+            *args: Additional positional arguments.
+            **kwargs: Additional keyword arguments.
+
+        Returns:
+            EvaluationResult: The result of the evaluation process.
+
+        Raises:
+            NotImplementedError: If the method is not implemented by a subclass.
+        """
+        pass

bridge/dto.py

@@ -0,0 +1,149 @@
+from dataclasses import dataclass
+from typing import Any, List, Optional, Protocol
+
+from envipy_additional_information import EnviPyModel, register
+from pydantic import HttpUrl
+
+from utilities.chem import FormatConverter, ProductSet
+
+
+@dataclass(frozen=True, slots=True)
+class Context:
+    uuid: str
+    url: str
+    work_dir: str
+
+
+class CompoundProto(Protocol):
+    url: str | None
+    name: str | None
+    smiles: str
+
+
+class RuleProto(Protocol):
+    url: str
+    name: str
+
+    def apply(self, smiles, *args, **kwargs): ...
+
+
+class ReactionProto(Protocol):
+    url: str
+    name: str
+    rules: List[RuleProto]
+
+
+class EnviPyDTO(Protocol):
+    def get_context(self) -> Context: ...
+
+    def get_compounds(self) -> List[CompoundProto]: ...
+
+    def get_reactions(self) -> List[ReactionProto]: ...
+
+    def get_rules(self) -> List[RuleProto]: ...
+
+    @staticmethod
+    def standardize(smiles, remove_stereo=False, canonicalize_tautomers=False): ...
+
+    @staticmethod
+    def apply(
+        smiles: str,
+        smirks: str,
+        preprocess_smiles: bool = True,
+        bracketize: bool = True,
+        standardize: bool = True,
+        kekulize: bool = True,
+        remove_stereo: bool = True,
+        reactant_filter_smarts: str | None = None,
+        product_filter_smarts: str | None = None,
+    ) -> List["ProductSet"]: ...
+
+
+class EnviPyPrediction(EnviPyModel):
+    pass
+
+
+class PropertyPrediction(EnviPyPrediction):
+    pass
+
+
+class TransformationProductPrediction(EnviPyPrediction):
+    substrate: str
+    products: dict[str, float]
+
+
+@register("buildresult")
+class BuildResult(EnviPyModel):
+    data: dict[str, Any] | List[dict[str, Any]] | None
+
+
+@register("runresult")
+class RunResult(EnviPyModel):
+    producer: HttpUrl
+    description: Optional[str] = None
+    result: EnviPyPrediction | List[EnviPyPrediction]
+
+
+@register("evaluationresult")
+class EvaluationResult(EnviPyModel):
+    data: dict[str, Any] | List[dict[str, Any]] | None
+
+
+class BaseDTO(EnviPyDTO):
+    def __init__(
+        self,
+        uuid: str,
+        url: str,
+        work_dir: str,
+        compounds: List[CompoundProto],
+        reactions: List[ReactionProto],
+        rules: List[RuleProto],
+    ):
+        self.uuid = uuid
+        self.url = url
+        self.work_dir = work_dir
+        self.compounds = compounds
+        self.reactions = reactions
+        self.rules = rules
+
+    def get_context(self) -> Context:
+        return Context(uuid=self.uuid, url=self.url, work_dir=self.work_dir)
+
+    def get_compounds(self) -> List[CompoundProto]:
+        return self.compounds
+
+    def get_reactions(self) -> List[ReactionProto]:
+        return self.reactions
+
+    def get_rules(self) -> List[RuleProto]:
+        return self.rules
+
+    @staticmethod
+    def standardize(smiles, remove_stereo=False, canonicalize_tautomers=False):
+        return FormatConverter.standardize(
+            smiles, remove_stereo=remove_stereo, canonicalize_tautomers=canonicalize_tautomers
+        )
+
+    @staticmethod
+    def apply(
+        smiles: str,
+        smirks: str,
+        preprocess_smiles: bool = True,
+        bracketize: bool = True,
+        standardize: bool = True,
+        kekulize: bool = True,
+        remove_stereo: bool = True,
+        reactant_filter_smarts: str | None = None,
+        product_filter_smarts: str | None = None,
+    ) -> List["ProductSet"]:
+        return FormatConverter.apply(
+            smiles,
+            smirks,
+            preprocess_smiles,
+            bracketize,
+            standardize,
+            kekulize,
+            remove_stereo,
+            reactant_filter_smarts,
+            product_filter_smarts,
+        )

docker-compose.dev.yml

@@ -1,6 +1,6 @@
 services:
   db:
-    image: postgres:15
+    image: postgres:18
     container_name: envipath-postgres
     environment:
       POSTGRES_USER: postgres
@@ -9,12 +9,18 @@ services:
     ports:
       - "5432:5432"
     volumes:
-      - postgres_data:/var/lib/postgresql/data
+      - postgres_data:/var/lib/postgresql
     healthcheck:
       test: ["CMD-SHELL", "pg_isready -U postgres"]
       interval: 5s
       timeout: 5s
       retries: 5
 
+  redis:
+    image: redis:7-alpine
+    container_name: envipath-redis
+    ports:
+      - "6379:6379"
+
 volumes:
   postgres_data:

docker-compose.yml

@@ -0,0 +1,50 @@
+services:
+  db:
+    image: postgres:18
+    container_name: eppostgres
+    environment:
+      POSTGRES_USER: ${POSTGRES_USER}
+      POSTGRES_PASSWORD: ${POSTGRES_PASSWORD}
+      POSTGRES_DB: ${POSTGRES_DB}
+    volumes:
+      - ep_postgres_data:/var/lib/postgresql
+    healthcheck:
+      test: [ "CMD-SHELL", "pg_isready -U postgres" ]
+      interval: 5s
+      timeout: 5s
+      retries: 5
+
+  redis:
+    image: redis:7-alpine
+    container_name: epredis
+    volumes:
+      - ep_redis_data:/data
+
+  biotransformer3:
+    image: envipath/biotransformer3:1.0
+    container_name: epbiotransformer3
+
+  web:
+    image: envipath/envipy:1.0
+    container_name: epdjango
+    ports:
+      - "127.0.0.1:8000:8000"
+    env_file:
+      - .env
+    command: gunicorn envipath.wsgi:application --bind 0.0.0.0:8000 --workers 3
+    volumes:
+      - ep_data:/opt/enviPy/
+
+  celery_worker:
+    image: envipath/envipy:1.0
+    container_name: epcelery
+    env_file:
+      - .env
+    command: celery -A envipath worker --concurrency=6 -Q model,predict,background --pool threads
+    volumes:
+      - ep_data:/opt/enviPy/
+
+volumes:
+  ep_postgres_data:
+  ep_redis_data:
+  ep_data:

envipath/api.py

@@ -1,4 +1,4 @@
-from epdb.api import router as epdb_app_router
+from epapi.v1.router import router as v1_router  # Refactored API from epdb.api_v2
 from epdb.legacy_api import router as epdb_legacy_app_router
 from ninja import NinjaAPI
 
@@ -8,5 +8,5 @@ api_v1 = NinjaAPI(title="API V1 Docs", urls_namespace="api-v1")
 api_legacy = NinjaAPI(title="Legacy API Docs", urls_namespace="api-legacy")
 
 # Add routers
-api_v1.add_router("/", epdb_app_router)
+api_v1.add_router("/", v1_router)
 api_legacy.add_router("/", epdb_legacy_app_router)

envipath/settings.py

@@ -14,14 +14,15 @@ import os
 from pathlib import Path
 
 from dotenv import load_dotenv
-from envipy_plugins import Classifier, Property, Descriptor
 from sklearn.ensemble import RandomForestClassifier
 from sklearn.tree import DecisionTreeClassifier
 
 # Build paths inside the project like this: BASE_DIR / 'subdir'.
 BASE_DIR = Path(__file__).resolve().parent.parent
 
-load_dotenv(BASE_DIR / ".env", override=False)
+ENV_PATH = os.environ.get("ENV_PATH", BASE_DIR / ".env")
+print(f"Loading env from {ENV_PATH}")
+load_dotenv(ENV_PATH, override=False)
 
 # Quick-start development settings - unsuitable for production
 # See https://docs.djangoproject.com/en/4.2/howto/deployment/checklist/
@@ -36,7 +37,6 @@ ALLOWED_HOSTS = os.environ["ALLOWED_HOSTS"].split(",")
 
 
 # Application definition
-
 INSTALLED_APPS = [
     "django.contrib.admin",
     "django.contrib.auth",
@@ -44,20 +44,37 @@ INSTALLED_APPS = [
     "django.contrib.sessions",
     "django.contrib.messages",
     "django.contrib.staticfiles",
+    "django.contrib.postgres",
     # 3rd party
     "django_extensions",
     "oauth2_provider",
     # Custom
+    "epapi",  # API endpoints (v1, etc.)
     "epdb",
     "migration",
 ]
 
+TENANT = os.environ.get("TENANT", "public")
+
+if TENANT != "public":
+    INSTALLED_APPS.append(TENANT)
+
+EPDB_PACKAGE_MODEL = os.environ.get("EPDB_PACKAGE_MODEL", "epdb.Package")
+
+
+def GET_PACKAGE_MODEL():
+    from django.apps import apps
+
+    return apps.get_model(EPDB_PACKAGE_MODEL)
+
+
 AUTHENTICATION_BACKENDS = [
     "django.contrib.auth.backends.ModelBackend",
 ]
 
 MIDDLEWARE = [
     "django.middleware.security.SecurityMiddleware",
+    "whitenoise.middleware.WhiteNoiseMiddleware",
     "django.contrib.sessions.middleware.SessionMiddleware",
     "django.middleware.common.CommonMiddleware",
     "django.middleware.csrf.CsrfViewMiddleware",
@@ -76,10 +93,19 @@ if os.environ.get("REGISTRATION_MANDATORY", False) == "True":
 
 ROOT_URLCONF = "envipath.urls"
 
+TEMPLATE_DIRS = [
+    os.path.join(BASE_DIR, "templates"),
+]
+
+# If we have a non-public tenant, we might need to overwrite some templates
+# search TENANT folder first...
+if TENANT != "public":
+    TEMPLATE_DIRS.insert(0, os.path.join(BASE_DIR, TENANT, "templates"))
+
 TEMPLATES = [
     {
         "BACKEND": "django.template.backends.django.DjangoTemplates",
-        "DIRS": (os.path.join(BASE_DIR, "templates"),),
+        "DIRS": TEMPLATE_DIRS,
         "APP_DIRS": True,
         "OPTIONS": {
             "context_processors": [
@@ -111,6 +137,13 @@ DATABASES = {
     }
 }
 
+if os.environ.get("USE_TEMPLATE_DB", False) == "True":
+    DATABASES["default"]["TEST"] = {
+        "NAME": f"test_{os.environ['TEMPLATE_DB']}",
+        "TEMPLATE": os.environ["TEMPLATE_DB"],
+    }
+
+
 # Password validation
 # https://docs.djangoproject.com/en/4.2/ref/settings/#auth-password-validators
 
@@ -158,11 +191,21 @@ ADMIN_APPROVAL_REQUIRED = os.environ.get("ADMIN_APPROVAL_REQUIRED", "False") ==
 # SESAME_MAX_AGE = 300
 # # TODO set to "home"
 # LOGIN_REDIRECT_URL = "/"
+
+
+SERVER_HOST = os.environ.get("SERVER_URL", "http://localhost:8000")
+SERVER_PATH = os.environ.get("SERVER_PATH", "")
+
+SERVER_URL = SERVER_HOST
+if SERVER_PATH:
+    SERVER_URL = os.path.join(SERVER_HOST, SERVER_PATH)
+
+
 LOGIN_URL = "/login/"
+if SERVER_PATH:
+    LOGIN_URL = f"/{SERVER_PATH}/login/"
 
-SERVER_URL = os.environ.get("SERVER_URL", "http://localhost:8000")
+CSRF_TRUSTED_ORIGINS = [SERVER_HOST]
-
-CSRF_TRUSTED_ORIGINS = [SERVER_URL]
 
 AMBIT_URL = "http://localhost:9001"
 DEFAULT_VALUES = {"description": "no description"}
@@ -187,10 +230,17 @@ PLUGIN_DIR = os.path.join(EP_DATA_DIR, "plugins")
 if not os.path.exists(PLUGIN_DIR):
     os.mkdir(PLUGIN_DIR)
 
+API_PAGINATION_DEFAULT_PAGE_SIZE = int(os.environ.get("API_PAGINATION_DEFAULT_PAGE_SIZE", 50))
+PAGINATION_MAX_PER_PAGE_SIZE = int(
+    os.environ.get("API_PAGINATION_MAX_PAGE_SIZE", 100)
+)  # Ninja override
+
 # Set this as our static root dir
 STATIC_ROOT = STATIC_DIR
 
 STATIC_URL = "/static/"
+if SERVER_PATH:
+    STATIC_URL = f"/{SERVER_PATH}/static/"
 
 # Where the sources are stored...
 STATICFILES_DIRS = (BASE_DIR / "static",)
@@ -254,9 +304,8 @@ if not FLAG_CELERY_PRESENT:
 
 # Celery Configuration Options
 CELERY_TIMEZONE = "Europe/Berlin"
-# Celery Configuration
+CELERY_BROKER_URL = os.environ.get("CELERY_BROKER_URL", "redis://localhost:6379/0")
-CELERY_BROKER_URL = "redis://localhost:6379/0"  # Use Redis as message broker
+CELERY_RESULT_BACKEND = os.environ.get("CELERY_RESULT_BACKEND", "redis://localhost:6379/1")
-CELERY_RESULT_BACKEND = "redis://localhost:6379/1"
 CELERY_ACCEPT_CONTENT = ["json"]
 CELERY_TASK_SERIALIZER = "json"
 
@@ -288,22 +337,21 @@ DEFAULT_MODEL_PARAMS = {
     "num_chains": 10,
 }
 
-DEFAULT_MAX_NUMBER_OF_NODES = 30
+DEFAULT_MAX_NUMBER_OF_NODES = 50
-DEFAULT_MAX_DEPTH = 5
+DEFAULT_MAX_DEPTH = 8
 DEFAULT_MODEL_THRESHOLD = 0.25
 
 # Loading Plugins
 PLUGINS_ENABLED = os.environ.get("PLUGINS_ENABLED", "False") == "True"
-if PLUGINS_ENABLED:
+BASE_PLUGINS = os.environ.get("BASE_PLUGINS", None)
-    from utilities.plugin import discover_plugins
+if BASE_PLUGINS:
-
+    BASE_PLUGINS = BASE_PLUGINS.split(",")
-    CLASSIFIER_PLUGINS = discover_plugins(_cls=Classifier)
-    PROPERTY_PLUGINS = discover_plugins(_cls=Property)
-    DESCRIPTOR_PLUGINS = discover_plugins(_cls=Descriptor)
 else:
-    CLASSIFIER_PLUGINS = {}
+    BASE_PLUGINS = []
-    PROPERTY_PLUGINS = {}
+
-    DESCRIPTOR_PLUGINS = {}
+CLASSIFIER_PLUGINS = {}
+PROPERTY_PLUGINS = {}
+DESCRIPTOR_PLUGINS = {}
 
 SENTRY_ENABLED = os.environ.get("SENTRY_ENABLED", "False") == "True"
 if SENTRY_ENABLED:
@@ -327,6 +375,10 @@ if SENTRY_ENABLED:
         before_send=before_send,
     )
 
+IUCLID_EXPORT_ENABLED = os.environ.get("IUCLID_EXPORT_ENABLED", "False") == "True"
+if IUCLID_EXPORT_ENABLED:
+    INSTALLED_APPS.append("epiuclid")
+
 # compile into digestible flags
 FLAGS = {
     "MODEL_BUILDING": MODEL_BUILDING_ENABLED,
@@ -335,28 +387,34 @@ FLAGS = {
     "SENTRY": SENTRY_ENABLED,
     "ENVIFORMER": ENVIFORMER_PRESENT,
     "APPLICABILITY_DOMAIN": APPLICABILITY_DOMAIN_ENABLED,
+    "IUCLID_EXPORT": IUCLID_EXPORT_ENABLED,
 }
 
 # path of the URL are checked via "startswith"
 # -> /password_reset/done is covered as well
 LOGIN_EXEMPT_URLS = [
     "/register",
+    "/api/v1/",  # Let API handle its own authentication
     "/api/legacy/",
     "/o/token/",
     "/o/userinfo/",
     "/password_reset/",
     "/reset/",
-    "/microsoft/",
     "/terms",
     "/privacy",
     "/cookie-policy",
     "/about",
     "/contact",
-    "/jobs",
+    "/careers",
     "/cite",
     "/legal",
+    "/entra/",
+    "/auth/",
 ]
 
+if SERVER_PATH:
+    LOGIN_EXEMPT_URLS = [f"/{SERVER_PATH}{x}" for x in LOGIN_EXEMPT_URLS]
+
 # MS AD/Entra
 MS_ENTRA_ENABLED = os.environ.get("MS_ENTRA_ENABLED", "False") == "True"
 if MS_ENTRA_ENABLED:
@@ -372,3 +430,15 @@ if MS_ENTRA_ENABLED:
 
 # Site ID 10 -> beta.envipath.org
 MATOMO_SITE_ID = os.environ.get("MATOMO_SITE_ID", "10")
+
+# CAP
+CAP_ENABLED = os.environ.get("CAP_ENABLED", "False") == "True"
+CAP_API_BASE = os.environ.get("CAP_API_BASE", None)
+CAP_SITE_KEY = os.environ.get("CAP_SITE_KEY", None)
+CAP_SECRET_KEY = os.environ.get("CAP_SECRET_KEY", None)
+
+# Biotransformer
+BIOTRANSFORMER_ENABLED = os.environ.get("BIOTRANSFORMER_ENABLED", "False") == "True"
+FLAGS["BIOTRANSFORMER"] = BIOTRANSFORMER_ENABLED
+if BIOTRANSFORMER_ENABLED:
+    BIOTRANSFORMER_URL = os.environ.get("BIOTRANSFORMER_URL", None)

envipath/urls.py

@@ -21,14 +21,30 @@ from django.urls import include, path
 
 from .api import api_v1, api_legacy
 
+PATH_PREFIX = s.SERVER_PATH
+if PATH_PREFIX and not PATH_PREFIX.endswith("/"):
+    PATH_PREFIX += "/"
+
+
 urlpatterns = [
-    path("", include("epdb.urls")),
+    path(f"{PATH_PREFIX}", include("epdb.urls")),
-    path("", include("migration.urls")),
+    path(f"{PATH_PREFIX}admin/", admin.site.urls),
-    path("admin/", admin.site.urls),
+    path(f"{PATH_PREFIX}api/v1/", api_v1.urls),
-    path("api/v1/", api_v1.urls),
+    path(f"{PATH_PREFIX}api/legacy/", api_legacy.urls),
-    path("api/legacy/", api_legacy.urls),
+    path(f"{PATH_PREFIX}o/", include("oauth2_provider.urls", namespace="oauth2_provider")),
-    path("o/", include("oauth2_provider.urls", namespace="oauth2_provider")),
 ]
 
+if "migration" in s.INSTALLED_APPS:
+    urlpatterns.append(path(f"{PATH_PREFIX}", include("migration.urls")))
+
 if s.MS_ENTRA_ENABLED:
-    urlpatterns.append(path("", include("epauth.urls")))
+    urlpatterns.append(path(f"{PATH_PREFIX}", include("epauth.urls")))
+
+if s.TENANT != "public":
+    urlpatterns.append(path(f"{PATH_PREFIX}", include(f"{s.TENANT}.urls")))
+
+# Custom error handlers
+handler400 = "epdb.views.handler400"
+handler403 = "epdb.views.handler403"
+handler404 = "epdb.views.handler404"
+handler500 = "epdb.views.handler500"

epapi/apps.py

@@ -0,0 +1,6 @@
+from django.apps import AppConfig
+
+
+class EpapiConfig(AppConfig):
+    default_auto_field = "django.db.models.BigAutoField"
+    name = "epapi"

epapi/tests/__init__.py

@@ -0,0 +1 @@
+# Tests for epapi app

epapi/tests/utils/__init__.py

@@ -0,0 +1 @@
+"""Tests for epapi utility modules."""

epapi/tests/utils/test_validation_errors.py

@@ -0,0 +1,218 @@
+"""
+Tests for validation error utilities.
+
+Tests the format_validation_error() and handle_validation_error() functions
+that transform Pydantic validation errors into user-friendly messages.
+"""
+
+from django.test import TestCase, tag
+import json
+from pydantic import BaseModel, ValidationError, field_validator
+from typing import Literal
+
+from ninja.errors import HttpError
+from epapi.utils.validation_errors import format_validation_error, handle_validation_error
+
+
+@tag("api", "utils")
+class ValidationErrorUtilityTests(TestCase):
+    """Test validation error utility functions."""
+
+    def test_format_missing_field_error(self):
+        """Test formatting of missing required field error."""
+
+        # Create a model with required field
+        class TestModel(BaseModel):
+            required_field: str
+
+        # Trigger validation error
+        try:
+            TestModel()
+        except ValidationError as e:
+            errors = e.errors()
+            self.assertEqual(len(errors), 1)
+            formatted = format_validation_error(errors[0])
+            self.assertEqual(formatted, "This field is required")
+
+    def test_format_enum_error(self):
+        """Test formatting of enum validation error."""
+
+        class TestModel(BaseModel):
+            status: Literal["active", "inactive"]
+
+        try:
+            TestModel(status="invalid")
+        except ValidationError as e:
+            errors = e.errors()
+            self.assertEqual(len(errors), 1)
+            formatted = format_validation_error(errors[0])
+            # Literal errors get formatted as "Please enter ..." with the valid options
+            self.assertIn("Please enter", formatted)
+            self.assertIn("active", formatted)
+            self.assertIn("inactive", formatted)
+
+    def test_format_type_errors(self):
+        """Test formatting of type validation errors (string, int, float)."""
+        test_cases = [
+            # (field_type, invalid_value, expected_message)
+            # Note: We don't check exact error_type as Pydantic may use different types
+            # (e.g., int_type vs int_parsing) but we verify the formatted message is correct
+            (str, 123, "Please enter a valid string"),
+            (int, "not_a_number", "Please enter a valid int"),
+            (float, "not_a_float", "Please enter a valid float"),
+        ]
+
+        for field_type, invalid_value, expected_message in test_cases:
+            with self.subTest(field_type=field_type.__name__):
+
+                class TestModel(BaseModel):
+                    field: field_type
+
+                try:
+                    TestModel(field=invalid_value)
+                except ValidationError as e:
+                    errors = e.errors()
+                    self.assertEqual(len(errors), 1)
+                    formatted = format_validation_error(errors[0])
+                    self.assertEqual(formatted, expected_message)
+
+    def test_format_value_error(self):
+        """Test formatting of value error from custom validator."""
+
+        class TestModel(BaseModel):
+            age: int
+
+            @field_validator("age")
+            @classmethod
+            def validate_age(cls, v):
+                if v < 0:
+                    raise ValueError("Age must be positive")
+                return v
+
+        try:
+            TestModel(age=-5)
+        except ValidationError as e:
+            errors = e.errors()
+            self.assertEqual(len(errors), 1)
+            formatted = format_validation_error(errors[0])
+            self.assertEqual(formatted, "Age must be positive")
+
+    def test_format_unknown_error_type_fallback(self):
+        """Test that unknown error types fall back to default formatting."""
+        # Mock an error with an unknown type
+        mock_error = {
+            "type": "unknown_custom_type",
+            "msg": "Input should be a valid email address",
+            "ctx": {},
+        }
+
+        formatted = format_validation_error(mock_error)
+        # Should use the else branch which does replacements on the message
+        self.assertEqual(formatted, "Please enter a valid email address")
+
+    def test_handle_validation_error_structure(self):
+        """Test that handle_validation_error raises HttpError with correct structure."""
+
+        class TestModel(BaseModel):
+            name: str
+            count: int
+
+        try:
+            TestModel(name=123, count="invalid")
+        except ValidationError as e:
+            # handle_validation_error should raise HttpError
+            with self.assertRaises(HttpError) as context:
+                handle_validation_error(e)
+
+            http_error = context.exception
+            self.assertEqual(http_error.status_code, 400)
+
+            # Parse the JSON from the error message
+            error_data = json.loads(http_error.message)
+
+            # Check structure
+            self.assertEqual(error_data["type"], "validation_error")
+            self.assertIn("field_errors", error_data)
+            self.assertIn("message", error_data)
+            self.assertEqual(error_data["message"], "Please correct the errors below")
+
+            # Check that both fields have errors
+            self.assertIn("name", error_data["field_errors"])
+            self.assertIn("count", error_data["field_errors"])
+
+    def test_handle_validation_error_no_pydantic_internals(self):
+        """Test that handle_validation_error doesn't expose Pydantic internals."""
+
+        class TestModel(BaseModel):
+            email: str
+
+        try:
+            TestModel(email=123)
+        except ValidationError as e:
+            with self.assertRaises(HttpError) as context:
+                handle_validation_error(e)
+
+            http_error = context.exception
+            error_data = json.loads(http_error.message)
+            error_str = json.dumps(error_data)
+
+            # Ensure no Pydantic internals are exposed
+            self.assertNotIn("pydantic", error_str.lower())
+            self.assertNotIn("https://errors.pydantic.dev", error_str)
+            self.assertNotIn("loc", error_str)
+
+    def test_handle_validation_error_user_friendly_messages(self):
+        """Test that all error messages are user-friendly."""
+
+        class TestModel(BaseModel):
+            name: str
+            age: int
+            status: Literal["active", "inactive"]
+
+        try:
+            TestModel(name=123, status="invalid")  # Multiple errors
+        except ValidationError as e:
+            with self.assertRaises(HttpError) as context:
+                handle_validation_error(e)
+
+            http_error = context.exception
+            error_data = json.loads(http_error.message)
+
+            # All messages should be user-friendly (contain "Please" or "This field")
+            for field, messages in error_data["field_errors"].items():
+                for message in messages:
+                    # User-friendly messages start with "Please" or "This field"
+                    self.assertTrue(
+                        message.startswith("Please") or message.startswith("This field"),
+                        f"Message '{message}' is not user-friendly",
+                    )
+
+    def test_handle_validation_error_multiple_errors_same_field(self):
+        """Test handling multiple validation errors for the same field."""
+
+        class TestModel(BaseModel):
+            value: int
+
+            @field_validator("value")
+            @classmethod
+            def validate_range(cls, v):
+                if v < 0:
+                    raise ValueError("Must be non-negative")
+                if v > 100:
+                    raise ValueError("Must be at most 100")
+                return v
+
+        # Test with string (type error) - this will fail before the validator runs
+        try:
+            TestModel(value="invalid")
+        except ValidationError as e:
+            with self.assertRaises(HttpError) as context:
+                handle_validation_error(e)
+
+            http_error = context.exception
+            error_data = json.loads(http_error.message)
+
+            # Should have error for 'value' field
+            self.assertIn("value", error_data["field_errors"])
+            self.assertIsInstance(error_data["field_errors"]["value"], list)
+            self.assertGreater(len(error_data["field_errors"]["value"]), 0)

epapi/tests/v1/__init__.py

@@ -0,0 +1 @@
+# Tests for epapi v1 API

epapi/tests/v1/test_additional_information.py

@@ -0,0 +1,446 @@
+"""
+Tests for Additional Information API endpoints.
+
+Tests CRUD operations on scenario additional information including the new PATCH endpoint.
+"""
+
+from django.test import TestCase, tag
+import json
+from uuid import uuid4
+
+from epdb.logic import PackageManager, UserManager
+from epdb.models import Scenario
+
+
+@tag("api", "additional_information")
+class AdditionalInformationAPITests(TestCase):
+    """Test additional information API endpoints."""
+
+    @classmethod
+    def setUpTestData(cls):
+        """Set up test data: user, package, and scenario."""
+        cls.user = UserManager.create_user(
+            "ai-test-user",
+            "ai-test@envipath.com",
+            "SuperSafe",
+            set_setting=False,
+            add_to_group=False,
+            is_active=True,
+        )
+        cls.other_user = UserManager.create_user(
+            "ai-other-user",
+            "ai-other@envipath.com",
+            "SuperSafe",
+            set_setting=False,
+            add_to_group=False,
+            is_active=True,
+        )
+        cls.package = PackageManager.create_package(
+            cls.user, "AI Test Package", "Test package for additional information"
+        )
+        # Package owned by other_user (no access for cls.user)
+        cls.other_package = PackageManager.create_package(
+            cls.other_user, "Other Package", "Package without access"
+        )
+        # Create a scenario for testing
+        cls.scenario = Scenario.objects.create(
+            package=cls.package,
+            name="Test Scenario",
+            description="Test scenario for additional information tests",
+            scenario_type="biodegradation",
+            scenario_date="2024-01-01",
+        )
+        cls.other_scenario = Scenario.objects.create(
+            package=cls.other_package,
+            name="Other Scenario",
+            description="Scenario in package without access",
+            scenario_type="biodegradation",
+            scenario_date="2024-01-01",
+        )
+
+    def test_list_all_schemas(self):
+        """Test GET /api/v1/information/schema/ returns all schemas."""
+        self.client.force_login(self.user)
+
+        response = self.client.get("/api/v1/information/schema/")
+
+        self.assertEqual(response.status_code, 200)
+        data = response.json()
+        self.assertIsInstance(data, dict)
+        # Should have multiple schemas
+        self.assertGreater(len(data), 0)
+        # Each schema should have RJSF format
+        for name, schema in data.items():
+            self.assertIn("schema", schema)
+            self.assertIn("uiSchema", schema)
+            self.assertIn("formData", schema)
+            self.assertIn("groups", schema)
+
+    def test_get_specific_schema(self):
+        """Test GET /api/v1/information/schema/{model_name}/ returns specific schema."""
+        self.client.force_login(self.user)
+
+        # Assuming 'temperature' is a valid model
+        response = self.client.get("/api/v1/information/schema/temperature/")
+
+        self.assertEqual(response.status_code, 200)
+        data = response.json()
+        self.assertIn("schema", data)
+        self.assertIn("uiSchema", data)
+
+    def test_get_nonexistent_schema_returns_404(self):
+        """Test GET for non-existent schema returns 404."""
+        self.client.force_login(self.user)
+
+        response = self.client.get("/api/v1/information/schema/nonexistent/")
+
+        self.assertEqual(response.status_code, 404)
+
+    def test_list_scenario_information_empty(self):
+        """Test GET /api/v1/scenario/{uuid}/information/ returns empty list initially."""
+        self.client.force_login(self.user)
+
+        response = self.client.get(f"/api/v1/scenario/{self.scenario.uuid}/information/")
+
+        self.assertEqual(response.status_code, 200)
+        data = response.json()
+        self.assertIsInstance(data, list)
+        self.assertEqual(len(data), 0)
+
+    def test_create_additional_information(self):
+        """Test POST creates additional information."""
+        self.client.force_login(self.user)
+
+        # Create temperature information (assuming temperature model exists)
+        payload = {"interval": {"start": 20, "end": 25}}
+        response = self.client.post(
+            f"/api/v1/scenario/{self.scenario.uuid}/information/temperature/",
+            data=json.dumps(payload),
+            content_type="application/json",
+        )
+
+        self.assertEqual(response.status_code, 200)
+        data = response.json()
+        self.assertEqual(data["status"], "created")
+        self.assertIn("uuid", data)
+        self.assertIsNotNone(data["uuid"])
+
+    def test_create_with_invalid_data_returns_400(self):
+        """Test POST with invalid data returns 400 with validation errors."""
+        self.client.force_login(self.user)
+
+        # Invalid data (missing required fields or wrong types)
+        payload = {"invalid_field": "value"}
+        response = self.client.post(
+            f"/api/v1/scenario/{self.scenario.uuid}/information/temperature/",
+            data=json.dumps(payload),
+            content_type="application/json",
+        )
+
+        self.assertEqual(response.status_code, 400)
+        data = response.json()
+        # Should have validation error details in 'detail' field
+        self.assertIn("detail", data)
+
+    def test_validation_errors_are_user_friendly(self):
+        """Test that validation errors are user-friendly and field-specific."""
+        self.client.force_login(self.user)
+
+        # Invalid data - wrong type (string instead of number in interval)
+        payload = {"interval": {"start": "not_a_number", "end": 25}}
+        response = self.client.post(
+            f"/api/v1/scenario/{self.scenario.uuid}/information/temperature/",
+            data=json.dumps(payload),
+            content_type="application/json",
+        )
+
+        self.assertEqual(response.status_code, 400)
+        data = response.json()
+
+        # Parse the error response - Django Ninja wraps errors in 'detail'
+        error_str = data.get("detail") or data.get("error")
+        self.assertIsNotNone(error_str, "Response should contain error details")
+
+        # Parse the JSON error string
+        error_data = json.loads(error_str)
+
+        # Check structure
+        self.assertEqual(error_data.get("type"), "validation_error")
+        self.assertIn("field_errors", error_data)
+        self.assertIn("message", error_data)
+
+        # Ensure error messages are user-friendly (no Pydantic URLs or technical jargon)
+        error_str = json.dumps(error_data)
+        self.assertNotIn("pydantic", error_str.lower())
+        self.assertNotIn("https://errors.pydantic.dev", error_str)
+        self.assertNotIn("loc", error_str)  # No technical field like 'loc'
+
+        # Check that error message is helpful
+        self.assertIn("Please", error_data["message"])  # User-friendly language
+
+    def test_patch_additional_information(self):
+        """Test PATCH updates existing additional information."""
+        self.client.force_login(self.user)
+
+        # First create an item
+        create_payload = {"interval": {"start": 20, "end": 25}}
+        create_response = self.client.post(
+            f"/api/v1/scenario/{self.scenario.uuid}/information/temperature/",
+            data=json.dumps(create_payload),
+            content_type="application/json",
+        )
+        item_uuid = create_response.json()["uuid"]
+
+        # Then update it with PATCH
+        update_payload = {"interval": {"start": 30, "end": 35}}
+        patch_response = self.client.patch(
+            f"/api/v1/scenario/{self.scenario.uuid}/information/item/{item_uuid}/",
+            data=json.dumps(update_payload),
+            content_type="application/json",
+        )
+
+        self.assertEqual(patch_response.status_code, 200)
+        data = patch_response.json()
+        self.assertEqual(data["status"], "updated")
+        self.assertEqual(data["uuid"], item_uuid)  # UUID preserved
+
+        # Verify the data was updated
+        list_response = self.client.get(f"/api/v1/scenario/{self.scenario.uuid}/information/")
+        items = list_response.json()
+        self.assertEqual(len(items), 1)
+        updated_item = items[0]
+        self.assertEqual(updated_item["uuid"], item_uuid)
+        self.assertEqual(updated_item["data"]["interval"]["start"], 30)
+        self.assertEqual(updated_item["data"]["interval"]["end"], 35)
+
+    def test_patch_nonexistent_item_returns_404(self):
+        """Test PATCH on non-existent item returns 404."""
+        self.client.force_login(self.user)
+
+        fake_uuid = str(uuid4())
+        payload = {"interval": {"start": 30, "end": 35}}
+        response = self.client.patch(
+            f"/api/v1/scenario/{self.scenario.uuid}/information/item/{fake_uuid}/",
+            data=json.dumps(payload),
+            content_type="application/json",
+        )
+
+        self.assertEqual(response.status_code, 404)
+
+    def test_patch_with_invalid_data_returns_400(self):
+        """Test PATCH with invalid data returns 400."""
+        self.client.force_login(self.user)
+
+        # First create an item
+        create_payload = {"interval": {"start": 20, "end": 25}}
+        create_response = self.client.post(
+            f"/api/v1/scenario/{self.scenario.uuid}/information/temperature/",
+            data=json.dumps(create_payload),
+            content_type="application/json",
+        )
+        item_uuid = create_response.json()["uuid"]
+
+        # Try to update with invalid data
+        invalid_payload = {"invalid_field": "value"}
+        patch_response = self.client.patch(
+            f"/api/v1/scenario/{self.scenario.uuid}/information/item/{item_uuid}/",
+            data=json.dumps(invalid_payload),
+            content_type="application/json",
+        )
+
+        self.assertEqual(patch_response.status_code, 400)
+
+    def test_patch_validation_errors_are_user_friendly(self):
+        """Test that PATCH validation errors are user-friendly and field-specific."""
+        self.client.force_login(self.user)
+
+        # First create an item
+        create_payload = {"interval": {"start": 20, "end": 25}}
+        create_response = self.client.post(
+            f"/api/v1/scenario/{self.scenario.uuid}/information/temperature/",
+            data=json.dumps(create_payload),
+            content_type="application/json",
+        )
+        item_uuid = create_response.json()["uuid"]
+
+        # Update with invalid data - wrong type (string instead of number in interval)
+        invalid_payload = {"interval": {"start": "not_a_number", "end": 25}}
+        patch_response = self.client.patch(
+            f"/api/v1/scenario/{self.scenario.uuid}/information/item/{item_uuid}/",
+            data=json.dumps(invalid_payload),
+            content_type="application/json",
+        )
+
+        self.assertEqual(patch_response.status_code, 400)
+        data = patch_response.json()
+
+        # Parse the error response - Django Ninja wraps errors in 'detail'
+        error_str = data.get("detail") or data.get("error")
+        self.assertIsNotNone(error_str, "Response should contain error details")
+
+        # Parse the JSON error string
+        error_data = json.loads(error_str)
+
+        # Check structure
+        self.assertEqual(error_data.get("type"), "validation_error")
+        self.assertIn("field_errors", error_data)
+        self.assertIn("message", error_data)
+
+        # Ensure error messages are user-friendly (no Pydantic URLs or technical jargon)
+        error_str = json.dumps(error_data)
+        self.assertNotIn("pydantic", error_str.lower())
+        self.assertNotIn("https://errors.pydantic.dev", error_str)
+        self.assertNotIn("loc", error_str)  # No technical field like 'loc'
+
+        # Check that error message is helpful
+        self.assertIn("Please", error_data["message"])  # User-friendly language
+
+    def test_delete_additional_information(self):
+        """Test DELETE removes additional information."""
+        self.client.force_login(self.user)
+
+        # Create an item
+        create_payload = {"interval": {"start": 20, "end": 25}}
+        create_response = self.client.post(
+            f"/api/v1/scenario/{self.scenario.uuid}/information/temperature/",
+            data=json.dumps(create_payload),
+            content_type="application/json",
+        )
+        item_uuid = create_response.json()["uuid"]
+
+        # Delete it
+        delete_response = self.client.delete(
+            f"/api/v1/scenario/{self.scenario.uuid}/information/item/{item_uuid}/"
+        )
+
+        self.assertEqual(delete_response.status_code, 200)
+        data = delete_response.json()
+        self.assertEqual(data["status"], "deleted")
+
+        # Verify deletion
+        list_response = self.client.get(f"/api/v1/scenario/{self.scenario.uuid}/information/")
+        items = list_response.json()
+        self.assertEqual(len(items), 0)
+
+    def test_delete_nonexistent_item_returns_404(self):
+        """Test DELETE on non-existent item returns 404."""
+        self.client.force_login(self.user)
+
+        fake_uuid = str(uuid4())
+        response = self.client.delete(
+            f"/api/v1/scenario/{self.scenario.uuid}/information/item/{fake_uuid}/"
+        )
+
+        self.assertEqual(response.status_code, 404)
+
+    def test_multiple_items_crud(self):
+        """Test creating, updating, and deleting multiple items."""
+        self.client.force_login(self.user)
+
+        # Create first item
+        item1_payload = {"interval": {"start": 20, "end": 25}}
+        response1 = self.client.post(
+            f"/api/v1/scenario/{self.scenario.uuid}/information/temperature/",
+            data=json.dumps(item1_payload),
+            content_type="application/json",
+        )
+        item1_uuid = response1.json()["uuid"]
+
+        # Create second item (different type if available, or same type)
+        item2_payload = {"interval": {"start": 30, "end": 35}}
+        response2 = self.client.post(
+            f"/api/v1/scenario/{self.scenario.uuid}/information/temperature/",
+            data=json.dumps(item2_payload),
+            content_type="application/json",
+        )
+        item2_uuid = response2.json()["uuid"]
+
+        # Verify both exist
+        list_response = self.client.get(f"/api/v1/scenario/{self.scenario.uuid}/information/")
+        items = list_response.json()
+        self.assertEqual(len(items), 2)
+
+        # Update first item
+        update_payload = {"interval": {"start": 15, "end": 20}}
+        self.client.patch(
+            f"/api/v1/scenario/{self.scenario.uuid}/information/item/{item1_uuid}/",
+            data=json.dumps(update_payload),
+            content_type="application/json",
+        )
+
+        # Delete second item
+        self.client.delete(f"/api/v1/scenario/{self.scenario.uuid}/information/item/{item2_uuid}/")
+
+        # Verify final state: one item with updated data
+        list_response = self.client.get(f"/api/v1/scenario/{self.scenario.uuid}/information/")
+        items = list_response.json()
+        self.assertEqual(len(items), 1)
+        self.assertEqual(items[0]["uuid"], item1_uuid)
+        self.assertEqual(items[0]["data"]["interval"]["start"], 15)
+
+    def test_list_info_denied_without_permission(self):
+        """User cannot list info for scenario in package they don't have access to"""
+        self.client.force_login(self.user)
+        response = self.client.get(f"/api/v1/scenario/{self.other_scenario.uuid}/information/")
+        self.assertEqual(response.status_code, 403)
+
+    def test_add_info_denied_without_permission(self):
+        """User cannot add info to scenario in package they don't have access to"""
+        self.client.force_login(self.user)
+        payload = {"interval": {"start": 25, "end": 30}}
+        response = self.client.post(
+            f"/api/v1/scenario/{self.other_scenario.uuid}/information/temperature/",
+            json.dumps(payload),
+            content_type="application/json",
+        )
+        self.assertEqual(response.status_code, 403)
+
+    def test_update_info_denied_without_permission(self):
+        """User cannot update info in scenario they don't have access to"""
+        self.client.force_login(self.other_user)
+        # First create an item as other_user
+        create_payload = {"interval": {"start": 20, "end": 25}}
+        create_response = self.client.post(
+            f"/api/v1/scenario/{self.other_scenario.uuid}/information/temperature/",
+            data=json.dumps(create_payload),
+            content_type="application/json",
+        )
+        item_uuid = create_response.json()["uuid"]
+
+        # Try to update as user (who doesn't have access)
+        self.client.force_login(self.user)
+        update_payload = {"interval": {"start": 30, "end": 35}}
+        response = self.client.patch(
+            f"/api/v1/scenario/{self.other_scenario.uuid}/information/item/{item_uuid}/",
+            data=json.dumps(update_payload),
+            content_type="application/json",
+        )
+        self.assertEqual(response.status_code, 403)
+
+    def test_delete_info_denied_without_permission(self):
+        """User cannot delete info from scenario they don't have access to"""
+        self.client.force_login(self.other_user)
+        # First create an item as other_user
+        create_payload = {"interval": {"start": 20, "end": 25}}
+        create_response = self.client.post(
+            f"/api/v1/scenario/{self.other_scenario.uuid}/information/temperature/",
+            data=json.dumps(create_payload),
+            content_type="application/json",
+        )
+        item_uuid = create_response.json()["uuid"]
+
+        # Try to delete as user (who doesn't have access)
+        self.client.force_login(self.user)
+        response = self.client.delete(
+            f"/api/v1/scenario/{self.other_scenario.uuid}/information/item/{item_uuid}/"
+        )
+        self.assertEqual(response.status_code, 403)
+
+    def test_nonexistent_scenario_returns_404(self):
+        """Test operations on non-existent scenario return 404."""
+        self.client.force_login(self.user)
+
+        fake_uuid = uuid4()
+        response = self.client.get(f"/api/v1/scenario/{fake_uuid}/information/")
+
+        self.assertEqual(response.status_code, 404)

epapi/tests/v1/test_api_permissions.py

@@ -0,0 +1,477 @@
+from django.test import TestCase, tag
+
+from epdb.logic import GroupManager, PackageManager, UserManager
+from epdb.models import (
+    Compound,
+    GroupPackagePermission,
+    Permission,
+    UserPackagePermission,
+)
+
+
+@tag("api", "end2end")
+class APIPermissionTestBase(TestCase):
+    """
+    Base class for API permission tests.
+
+    Sets up common test data:
+    - user1: Owner of packages
+    - user2: User with various permissions
+    - user3: User with no permissions
+    - reviewed_package: Public package (reviewed=True)
+    - unreviewed_package_owned: Unreviewed package owned by user1
+    - unreviewed_package_read: Unreviewed package with READ permission for user2
+    - unreviewed_package_write: Unreviewed package with WRITE permission for user2
+    - unreviewed_package_all: Unreviewed package with ALL permission for user2
+    - unreviewed_package_no_access: Unreviewed package with no permissions for user2/user3
+    - group_package: Unreviewed package accessible via group permission
+    - test_group: Group containing user2
+    """
+
+    @classmethod
+    def setUpTestData(cls):
+        # Create users
+        cls.user1 = UserManager.create_user(
+            "permission-user1",
+            "permission-user1@envipath.com",
+            "SuperSafe",
+            set_setting=False,
+            add_to_group=False,
+            is_active=True,
+        )
+        cls.user2 = UserManager.create_user(
+            "permission-user2",
+            "permission-user2@envipath.com",
+            "SuperSafe",
+            set_setting=False,
+            add_to_group=False,
+            is_active=True,
+        )
+        cls.user3 = UserManager.create_user(
+            "permission-user3",
+            "permission-user3@envipath.com",
+            "SuperSafe",
+            set_setting=False,
+            add_to_group=False,
+            is_active=True,
+        )
+
+        # Delete default packages to ensure clean test data
+        for user in [cls.user1, cls.user2, cls.user3]:
+            default_pkg = user.default_package
+            user.default_package = None
+            user.save()
+            if default_pkg:
+                default_pkg.delete()
+
+        # Create reviewed package (public)
+        cls.reviewed_package = PackageManager.create_package(
+            cls.user1, "Reviewed Package", "Public package"
+        )
+        cls.reviewed_package.reviewed = True
+        cls.reviewed_package.save()
+
+        # Create unreviewed packages with various permissions
+        cls.unreviewed_package_owned = PackageManager.create_package(
+            cls.user1, "User1 Owned Package", "Owned by user1"
+        )
+
+        cls.unreviewed_package_read = PackageManager.create_package(
+            cls.user1, "User2 Read Package", "User2 has READ permission"
+        )
+        UserPackagePermission.objects.create(
+            user=cls.user2, package=cls.unreviewed_package_read, permission=Permission.READ[0]
+        )
+
+        cls.unreviewed_package_write = PackageManager.create_package(
+            cls.user1, "User2 Write Package", "User2 has WRITE permission"
+        )
+        UserPackagePermission.objects.create(
+            user=cls.user2, package=cls.unreviewed_package_write, permission=Permission.WRITE[0]
+        )
+
+        cls.unreviewed_package_all = PackageManager.create_package(
+            cls.user1, "User2 All Package", "User2 has ALL permission"
+        )
+        UserPackagePermission.objects.create(
+            user=cls.user2, package=cls.unreviewed_package_all, permission=Permission.ALL[0]
+        )
+
+        cls.unreviewed_package_no_access = PackageManager.create_package(
+            cls.user1, "No Access Package", "No permissions for user2/user3"
+        )
+
+        # Create group and group package
+        cls.test_group = GroupManager.create_group(
+            cls.user1, "Test Group", "Group for permission testing"
+        )
+        cls.test_group.user_member.add(cls.user2)
+        cls.test_group.save()
+
+        cls.group_package = PackageManager.create_package(
+            cls.user1, "Group Package", "Accessible via group permission"
+        )
+        GroupPackagePermission.objects.create(
+            group=cls.test_group, package=cls.group_package, permission=Permission.READ[0]
+        )
+
+        # Create test compounds in each package
+        cls.reviewed_compound = Compound.create(
+            cls.reviewed_package, "C", "Reviewed Compound", "Test compound"
+        )
+        cls.owned_compound = Compound.create(
+            cls.unreviewed_package_owned, "CC", "Owned Compound", "Test compound"
+        )
+        cls.read_compound = Compound.create(
+            cls.unreviewed_package_read, "CCC", "Read Compound", "Test compound"
+        )
+        cls.write_compound = Compound.create(
+            cls.unreviewed_package_write, "CCCC", "Write Compound", "Test compound"
+        )
+        cls.all_compound = Compound.create(
+            cls.unreviewed_package_all, "CCCCC", "All Compound", "Test compound"
+        )
+        cls.no_access_compound = Compound.create(
+            cls.unreviewed_package_no_access, "CCCCCC", "No Access Compound", "Test compound"
+        )
+        cls.group_compound = Compound.create(
+            cls.group_package, "CCCCCCC", "Group Compound", "Test compound"
+        )
+
+
+@tag("api", "end2end")
+class PackageListPermissionTest(APIPermissionTestBase):
+    """
+    Test permissions for /api/v1/packages/ endpoint.
+
+    Special case: This endpoint allows anonymous access (auth=None)
+    """
+
+    ENDPOINT = "/api/v1/packages/"
+
+    def test_anonymous_user_sees_only_reviewed_packages(self):
+        """Anonymous users should only see reviewed packages."""
+        self.client.logout()
+        response = self.client.get(self.ENDPOINT)
+
+        self.assertEqual(response.status_code, 200)
+        payload = response.json()
+
+        # Should only see reviewed package
+        self.assertEqual(payload["total_items"], 1)
+        self.assertEqual(payload["items"][0]["uuid"], str(self.reviewed_package.uuid))
+        self.assertEqual(payload["items"][0]["review_status"], "reviewed")
+
+    def test_authenticated_user_sees_all_readable_packages(self):
+        """Authenticated users see reviewed + packages they have access to."""
+        self.client.force_login(self.user2)
+        response = self.client.get(self.ENDPOINT)
+
+        self.assertEqual(response.status_code, 200)
+        payload = response.json()
+
+        # user2 should see:
+        # - reviewed_package (public)
+        # - unreviewed_package_read (READ permission)
+        # - unreviewed_package_write (WRITE permission)
+        # - unreviewed_package_all (ALL permission)
+        # - group_package (via group membership)
+        # Total: 5 packages
+        self.assertEqual(payload["total_items"], 5)
+
+        visible_uuids = {item["uuid"] for item in payload["items"]}
+        expected_uuids = {
+            str(self.reviewed_package.uuid),
+            str(self.unreviewed_package_read.uuid),
+            str(self.unreviewed_package_write.uuid),
+            str(self.unreviewed_package_all.uuid),
+            str(self.group_package.uuid),
+        }
+        self.assertEqual(visible_uuids, expected_uuids)
+
+    def test_owner_sees_all_owned_packages(self):
+        """Package owner sees all packages they created."""
+        self.client.force_login(self.user1)
+        response = self.client.get(self.ENDPOINT)
+
+        self.assertEqual(response.status_code, 200)
+        payload = response.json()
+
+        # user1 owns all packages
+        # Total: 7 packages (all packages created in setUpTestData)
+        self.assertEqual(payload["total_items"], 7)
+
+    def test_filter_by_review_status_true(self):
+        """Filter to show only reviewed packages."""
+        self.client.force_login(self.user2)
+        response = self.client.get(self.ENDPOINT, {"review_status": True})
+
+        self.assertEqual(response.status_code, 200)
+        payload = response.json()
+
+        # Only reviewed_package
+        self.assertEqual(payload["total_items"], 1)
+        self.assertTrue(all(item["review_status"] == "reviewed" for item in payload["items"]))
+
+    def test_filter_by_review_status_false(self):
+        """Filter to show only unreviewed packages."""
+        self.client.force_login(self.user2)
+        response = self.client.get(self.ENDPOINT, {"review_status": False})
+
+        self.assertEqual(response.status_code, 200)
+        payload = response.json()
+
+        # user2's accessible unreviewed packages: 4
+        self.assertEqual(payload["total_items"], 4)
+        self.assertTrue(all(item["review_status"] == "unreviewed" for item in payload["items"]))
+
+    def test_anonymous_filter_unreviewed_returns_empty(self):
+        """Anonymous users get no results when filtering for unreviewed."""
+        self.client.logout()
+        response = self.client.get(self.ENDPOINT, {"review_status": False})
+
+        self.assertEqual(response.status_code, 200)
+        payload = response.json()
+
+        self.assertEqual(payload["total_items"], 0)
+
+
+@tag("api", "end2end")
+class GlobalCompoundListPermissionTest(APIPermissionTestBase):
+    """
+    Test permissions for /api/v1/compounds/ endpoint.
+
+    This endpoint requires authentication.
+    """
+
+    ENDPOINT = "/api/v1/compounds/"
+
+    def test_anonymous_user_cannot_access(self):
+        """Anonymous users should get 401 Unauthorized."""
+        self.client.logout()
+        response = self.client.get(self.ENDPOINT)
+
+        self.assertEqual(response.status_code, 401)
+
+    def test_authenticated_user_sees_compounds_from_readable_packages(self):
+        """Authenticated users see compounds from packages they can read."""
+        self.client.force_login(self.user2)
+        response = self.client.get(self.ENDPOINT)
+
+        self.assertEqual(response.status_code, 200)
+        payload = response.json()
+
+        self.assertEqual(payload["total_items"], 5)
+
+        visible_uuids = {item["uuid"] for item in payload["items"]}
+        expected_uuids = {
+            str(self.reviewed_compound.uuid),
+            str(self.read_compound.uuid),
+            str(self.write_compound.uuid),
+            str(self.all_compound.uuid),
+            str(self.group_compound.uuid),
+        }
+        self.assertEqual(visible_uuids, expected_uuids)
+
+    def test_user_without_permission_cannot_see_compound(self):
+        """User without permission to package cannot see its compounds."""
+        self.client.force_login(self.user3)
+        response = self.client.get(self.ENDPOINT)
+
+        self.assertEqual(response.status_code, 200)
+        payload = response.json()
+
+        # user3 should only see compounds from reviewed_package
+        self.assertEqual(payload["total_items"], 1)
+        self.assertEqual(payload["items"][0]["uuid"], str(self.reviewed_compound.uuid))
+
+    def test_owner_sees_all_compounds(self):
+        """Package owner sees all compounds they created."""
+        self.client.force_login(self.user1)
+        response = self.client.get(self.ENDPOINT)
+
+        self.assertEqual(response.status_code, 200)
+        payload = response.json()
+
+        # user1 owns all packages, so sees all compounds
+        self.assertEqual(payload["total_items"], 7)
+
+
+@tag("api", "end2end")
+class PackageScopedCompoundListPermissionTest(APIPermissionTestBase):
+    """
+    Test permissions for /api/v1/package/{uuid}/compound/ endpoint.
+
+    This endpoint requires authentication AND package access.
+    """
+
+    def test_anonymous_user_cannot_access_reviewed_package(self):
+        """Anonymous users should get 401 even for reviewed packages."""
+        self.client.logout()
+        endpoint = f"/api/v1/package/{self.reviewed_package.uuid}/compound/"
+        response = self.client.get(endpoint)
+
+        self.assertEqual(response.status_code, 401)
+
+    def test_authenticated_user_can_access_reviewed_package(self):
+        """Authenticated users can access reviewed packages."""
+        self.client.force_login(self.user3)
+        endpoint = f"/api/v1/package/{self.reviewed_package.uuid}/compound/"
+        response = self.client.get(endpoint)
+
+        self.assertEqual(response.status_code, 200)
+        payload = response.json()
+
+        self.assertEqual(payload["total_items"], 1)
+        self.assertEqual(payload["items"][0]["uuid"], str(self.reviewed_compound.uuid))
+
+    def test_user_can_access_package_with_read_permission(self):
+        """User with READ permission can access package-scoped endpoint."""
+        self.client.force_login(self.user2)
+        endpoint = f"/api/v1/package/{self.unreviewed_package_read.uuid}/compound/"
+        response = self.client.get(endpoint)
+
+        self.assertEqual(response.status_code, 200)
+        payload = response.json()
+
+        self.assertEqual(payload["total_items"], 1)
+        self.assertEqual(payload["items"][0]["uuid"], str(self.read_compound.uuid))
+
+    def test_user_can_access_package_with_write_permission(self):
+        """User with WRITE permission can access package-scoped endpoint."""
+        self.client.force_login(self.user2)
+        endpoint = f"/api/v1/package/{self.unreviewed_package_write.uuid}/compound/"
+        response = self.client.get(endpoint)
+
+        self.assertEqual(response.status_code, 200)
+        payload = response.json()
+
+        self.assertEqual(payload["total_items"], 1)
+        self.assertEqual(payload["items"][0]["uuid"], str(self.write_compound.uuid))
+
+    def test_user_can_access_package_with_all_permission(self):
+        """User with ALL permission can access package-scoped endpoint."""
+        self.client.force_login(self.user2)
+        endpoint = f"/api/v1/package/{self.unreviewed_package_all.uuid}/compound/"
+        response = self.client.get(endpoint)
+
+        self.assertEqual(response.status_code, 200)
+        payload = response.json()
+
+        self.assertEqual(payload["total_items"], 1)
+        self.assertEqual(payload["items"][0]["uuid"], str(self.all_compound.uuid))
+
+    def test_user_cannot_access_package_without_permission(self):
+        """User without permission gets 403 Forbidden."""
+        self.client.force_login(self.user2)
+        endpoint = f"/api/v1/package/{self.unreviewed_package_no_access.uuid}/compound/"
+        response = self.client.get(endpoint)
+
+        self.assertEqual(response.status_code, 403)
+
+    def test_nonexistent_package_returns_404(self):
+        """Request for non-existent package returns 404."""
+        self.client.force_login(self.user2)
+        fake_uuid = "00000000-0000-0000-0000-000000000000"
+        endpoint = f"/api/v1/package/{fake_uuid}/compound/"
+        response = self.client.get(endpoint)
+
+        self.assertEqual(response.status_code, 404)
+
+    def test_owner_can_access_owned_package(self):
+        """Package owner can access their package."""
+        self.client.force_login(self.user1)
+        endpoint = f"/api/v1/package/{self.unreviewed_package_owned.uuid}/compound/"
+        response = self.client.get(endpoint)
+
+        self.assertEqual(response.status_code, 200)
+        payload = response.json()
+
+        self.assertEqual(payload["total_items"], 1)
+        self.assertEqual(payload["items"][0]["uuid"], str(self.owned_compound.uuid))
+
+    def test_group_member_can_access_group_package(self):
+        """Group member can access package via group permission."""
+        self.client.force_login(self.user2)
+        endpoint = f"/api/v1/package/{self.group_package.uuid}/compound/"
+        response = self.client.get(endpoint)
+
+        self.assertEqual(response.status_code, 200)
+        payload = response.json()
+
+        self.assertEqual(payload["total_items"], 1)
+        self.assertEqual(payload["items"][0]["uuid"], str(self.group_compound.uuid))
+
+    def test_non_group_member_cannot_access_group_package(self):
+        """Non-group member cannot access package with only group permission."""
+        self.client.force_login(self.user3)
+        endpoint = f"/api/v1/package/{self.group_package.uuid}/compound/"
+        response = self.client.get(endpoint)
+
+        self.assertEqual(response.status_code, 403)
+
+
+@tag("api", "end2end")
+class MultiResourcePermissionTest(APIPermissionTestBase):
+    """
+    Test that permission system works consistently across all resource types.
+
+    Tests a sample of other endpoints to ensure permission logic is consistent.
+    """
+
+    def test_rules_endpoint_respects_permissions(self):
+        """Rules endpoint uses same permission logic."""
+        from epdb.models import SimpleAmbitRule
+
+        # Create rule in no-access package
+        rule = SimpleAmbitRule.create(
+            self.unreviewed_package_no_access, "Test Rule", "Test", "[C:1]>>[C:1]O"
+        )
+
+        self.client.force_login(self.user2)
+        response = self.client.get("/api/v1/rules/")
+
+        self.assertEqual(response.status_code, 200)
+        payload = response.json()
+
+        # user2 should not see the rule from no_access_package
+        rule_uuids = [item["uuid"] for item in payload["items"]]
+        self.assertNotIn(str(rule.uuid), rule_uuids)
+
+    def test_reactions_endpoint_respects_permissions(self):
+        """Reactions endpoint uses same permission logic."""
+        from epdb.models import Reaction
+
+        # Create reaction in no-access package
+        reaction = Reaction.create(
+            self.unreviewed_package_no_access, "Test Reaction", "Test", ["C"], ["CO"]
+        )
+
+        self.client.force_login(self.user2)
+        response = self.client.get("/api/v1/reactions/")
+
+        self.assertEqual(response.status_code, 200)
+        payload = response.json()
+
+        # user2 should not see the reaction from no_access_package
+        reaction_uuids = [item["uuid"] for item in payload["items"]]
+        self.assertNotIn(str(reaction.uuid), reaction_uuids)
+
+    def test_pathways_endpoint_respects_permissions(self):
+        """Pathways endpoint uses same permission logic."""
+        from epdb.models import Pathway
+
+        # Create pathway in no-access package
+        pathway = Pathway.objects.create(
+            package=self.unreviewed_package_no_access, name="Test Pathway", description="Test"
+        )
+
+        self.client.force_login(self.user2)
+        response = self.client.get("/api/v1/pathways/")
+
+        self.assertEqual(response.status_code, 200)
+        payload = response.json()
+
+        # user2 should not see the pathway from no_access_package
+        pathway_uuids = [item["uuid"] for item in payload["items"]]
+        self.assertNotIn(str(pathway.uuid), pathway_uuids)

epapi/tests/v1/test_contract_get_entities.py

@@ -0,0 +1,477 @@
+from django.test import TestCase, tag
+
+from epdb.logic import PackageManager, UserManager
+from epdb.models import Compound, Reaction, Pathway, EPModel, SimpleAmbitRule, Scenario
+
+
+class BaseTestAPIGetPaginated:
+    """
+    Mixin class for API pagination tests.
+
+    Subclasses must inherit from both this class and TestCase, e.g.:
+        class MyTest(BaseTestAPIGetPaginated, TestCase):
+            ...
+
+    Subclasses must define:
+    - resource_name: Singular name (e.g., "compound")
+    - resource_name_plural: Plural name (e.g., "compounds")
+    - global_endpoint: Global listing endpoint (e.g., "/api/v1/compounds/")
+    - package_endpoint_template: Template for package-scoped endpoint or None
+    - total_reviewed: Number of reviewed items to create
+    - total_unreviewed: Number of unreviewed items to create
+    - create_reviewed_resource(cls, package, idx): Factory method
+    - create_unreviewed_resource(cls, package, idx): Factory method
+    """
+
+    # Configuration to be overridden by subclasses
+    resource_name = None
+    resource_name_plural = None
+    global_endpoint = None
+    package_endpoint_template = None
+    total_reviewed = 50
+    total_unreviewed = 20
+    default_page_size = 50
+    max_page_size = 100
+
+    @classmethod
+    def setUpTestData(cls):
+        # Create test user
+        cls.user = UserManager.create_user(
+            f"{cls.resource_name}-user",
+            f"{cls.resource_name}-user@envipath.com",
+            "SuperSafe",
+            set_setting=False,
+            add_to_group=False,
+            is_active=True,
+        )
+
+        # Delete the auto-created default package to ensure clean test data
+        default_pkg = cls.user.default_package
+        cls.user.default_package = None
+        cls.user.save()
+        default_pkg.delete()
+
+        # Create reviewed package
+        cls.reviewed_package = PackageManager.create_package(
+            cls.user, "Reviewed Package", f"Reviewed package for {cls.resource_name} tests"
+        )
+        cls.reviewed_package.reviewed = True
+        cls.reviewed_package.save()
+
+        # Create unreviewed package
+        cls.unreviewed_package = PackageManager.create_package(
+            cls.user, "Draft Package", f"Unreviewed package for {cls.resource_name} tests"
+        )
+
+        # Create reviewed resources
+        for idx in range(cls.total_reviewed):
+            cls.create_reviewed_resource(cls.reviewed_package, idx)
+
+        # Create unreviewed resources
+        for idx in range(cls.total_unreviewed):
+            cls.create_unreviewed_resource(cls.unreviewed_package, idx)
+
+        # Set up package-scoped endpoints if applicable
+        if cls.package_endpoint_template:
+            cls.reviewed_package_endpoint = cls.package_endpoint_template.format(
+                uuid=cls.reviewed_package.uuid
+            )
+            cls.unreviewed_package_endpoint = cls.package_endpoint_template.format(
+                uuid=cls.unreviewed_package.uuid
+            )
+
+    @classmethod
+    def create_reviewed_resource(cls, package, idx):
+        """
+        Create a single reviewed resource.
+        Must be implemented by subclass.
+
+        Args:
+            package: The package to create the resource in
+            idx: Index of the resource (0-based)
+        """
+        raise NotImplementedError(f"{cls.__name__} must implement create_reviewed_resource()")
+
+    @classmethod
+    def create_unreviewed_resource(cls, package, idx):
+        """
+        Create a single unreviewed resource.
+        Must be implemented by subclass.
+
+        Args:
+            package: The package to create the resource in
+            idx: Index of the resource (0-based)
+        """
+        raise NotImplementedError(f"{cls.__name__} must implement create_unreviewed_resource()")
+
+    def setUp(self):
+        self.client.force_login(self.user)
+
+    def test_requires_session_authentication(self):
+        """Test that the global endpoint requires authentication."""
+        self.client.logout()
+        response = self.client.get(self.global_endpoint)
+        self.assertEqual(response.status_code, 401)
+
+    def test_global_listing_uses_default_page_size(self):
+        """Test that the global endpoint uses default pagination settings."""
+        response = self.client.get(self.global_endpoint, {"review_status": True})
+        self.assertEqual(response.status_code, 200)
+
+        payload = response.json()
+        self.assertEqual(payload["page"], 1)
+        self.assertEqual(payload["page_size"], self.default_page_size)
+        self.assertEqual(payload["total_items"], self.total_reviewed)
+
+        # Verify only reviewed items are returned
+        self.assertTrue(all(item["review_status"] == "reviewed" for item in payload["items"]))
+
+    def test_can_request_later_page(self):
+        """Test that pagination works for later pages."""
+        if self.total_reviewed <= self.default_page_size:
+            self.skipTest(
+                f"Not enough items to test pagination "
+                f"({self.total_reviewed} <= {self.default_page_size})"
+            )
+
+        response = self.client.get(self.global_endpoint, {"page": 2, "review_status": True})
+        self.assertEqual(response.status_code, 200)
+
+        payload = response.json()
+        self.assertEqual(payload["page"], 2)
+
+        # Calculate expected items on page 2
+        expected_items = min(self.default_page_size, self.total_reviewed - self.default_page_size)
+        self.assertEqual(len(payload["items"]), expected_items)
+
+        # Verify only reviewed items are returned
+        self.assertTrue(all(item["review_status"] == "reviewed" for item in payload["items"]))
+
+    def test_page_size_is_capped(self):
+        """Test that page size is capped at the maximum."""
+        if self.total_reviewed <= self.max_page_size:
+            self.skipTest(
+                f"Not enough items to test page size cap "
+                f"({self.total_reviewed} <= {self.max_page_size})"
+            )
+
+        response = self.client.get(self.global_endpoint, {"page_size": 150})
+        self.assertEqual(response.status_code, 200)
+
+        payload = response.json()
+        self.assertEqual(payload["page_size"], self.max_page_size)
+        self.assertEqual(len(payload["items"]), self.max_page_size)
+
+    def test_package_endpoint_for_reviewed_package(self):
+        """Test the package-scoped endpoint for reviewed packages."""
+        if not self.package_endpoint_template:
+            self.skipTest("No package endpoint for this resource")
+
+        response = self.client.get(self.reviewed_package_endpoint)
+        self.assertEqual(response.status_code, 200)
+
+        payload = response.json()
+        self.assertEqual(payload["total_items"], self.total_reviewed)
+
+        # Verify only reviewed items are returned
+        self.assertTrue(all(item["review_status"] == "reviewed" for item in payload["items"]))
+
+    def test_package_endpoint_for_unreviewed_package(self):
+        """Test the package-scoped endpoint for unreviewed packages."""
+        if not self.package_endpoint_template:
+            self.skipTest("No package endpoint for this resource")
+
+        response = self.client.get(self.unreviewed_package_endpoint)
+        self.assertEqual(response.status_code, 200)
+
+        payload = response.json()
+        self.assertEqual(payload["total_items"], self.total_unreviewed)
+
+        # Verify only unreviewed items are returned
+        self.assertTrue(all(item["review_status"] == "unreviewed" for item in payload["items"]))
+
+
+@tag("api", "end2end")
+class PackagePaginationAPITest(TestCase):
+    ENDPOINT = "/api/v1/packages/"
+
+    @classmethod
+    def setUpTestData(cls):
+        cls.user = UserManager.create_user(
+            "package-user",
+            "package-user@envipath.com",
+            "SuperSafe",
+            set_setting=False,
+            add_to_group=False,
+            is_active=True,
+        )
+
+        # Delete the auto-created default package to ensure clean test data
+        default_pkg = cls.user.default_package
+        cls.user.default_package = None
+        cls.user.save()
+        default_pkg.delete()
+
+        # Create reviewed packages
+        cls.total_reviewed = 25
+        for idx in range(cls.total_reviewed):
+            package = PackageManager.create_package(
+                cls.user, f"Reviewed Package {idx:03d}", "Reviewed package for tests"
+            )
+            package.reviewed = True
+            package.save()
+
+        # Create unreviewed packages
+        cls.total_unreviewed = 15
+        for idx in range(cls.total_unreviewed):
+            PackageManager.create_package(
+                cls.user, f"Draft Package {idx:03d}", "Unreviewed package for tests"
+            )
+
+    def setUp(self):
+        self.client.force_login(self.user)
+
+    def test_anonymous_can_access_reviewed_packages(self):
+        self.client.logout()
+        response = self.client.get(self.ENDPOINT)
+        self.assertEqual(response.status_code, 200)
+
+        payload = response.json()
+        # Anonymous users can only see reviewed packages
+        self.assertEqual(payload["total_items"], self.total_reviewed)
+        self.assertTrue(all(item["review_status"] == "reviewed" for item in payload["items"]))
+
+    def test_listing_uses_default_page_size(self):
+        response = self.client.get(self.ENDPOINT)
+        self.assertEqual(response.status_code, 200)
+
+        payload = response.json()
+        self.assertEqual(payload["page"], 1)
+        self.assertEqual(payload["page_size"], 50)
+        self.assertEqual(payload["total_items"], self.total_reviewed + self.total_unreviewed)
+
+    def test_reviewed_filter_true(self):
+        response = self.client.get(self.ENDPOINT, {"review_status": True})
+        self.assertEqual(response.status_code, 200)
+
+        payload = response.json()
+        self.assertEqual(payload["total_items"], self.total_reviewed)
+        self.assertTrue(all(item["review_status"] == "reviewed" for item in payload["items"]))
+
+    def test_reviewed_filter_false(self):
+        response = self.client.get(self.ENDPOINT, {"review_status": False})
+        self.assertEqual(response.status_code, 200)
+
+        payload = response.json()
+        self.assertEqual(payload["total_items"], self.total_unreviewed)
+        self.assertTrue(all(item["review_status"] == "unreviewed" for item in payload["items"]))
+
+    def test_reviewed_filter_false_anonymous(self):
+        self.client.logout()
+        response = self.client.get(self.ENDPOINT, {"review_status": False})
+        self.assertEqual(response.status_code, 200)
+
+        payload = response.json()
+        # Anonymous users cannot access unreviewed packages
+        self.assertEqual(payload["total_items"], 0)
+
+
+@tag("api", "end2end")
+class CompoundPaginationAPITest(BaseTestAPIGetPaginated, TestCase):
+    """Compound pagination tests using base class."""
+
+    resource_name = "compound"
+    resource_name_plural = "compounds"
+    global_endpoint = "/api/v1/compounds/"
+    package_endpoint_template = "/api/v1/package/{uuid}/compound/"
+    total_reviewed = 125
+    total_unreviewed = 35
+
+    @classmethod
+    def create_reviewed_resource(cls, package, idx):
+        simple_smiles = ["C", "CC", "CCC", "CCCC", "CCCCC"]
+        smiles = simple_smiles[idx % len(simple_smiles)] + ("O" * (idx // len(simple_smiles)))
+        return Compound.create(
+            package,
+            smiles,
+            f"Reviewed Compound {idx:03d}",
+            "Compound for pagination tests",
+        )
+
+    @classmethod
+    def create_unreviewed_resource(cls, package, idx):
+        simple_smiles = ["C", "CC", "CCC", "CCCC", "CCCCC"]
+        smiles = simple_smiles[idx % len(simple_smiles)] + ("N" * (idx // len(simple_smiles)))
+        return Compound.create(
+            package,
+            smiles,
+            f"Draft Compound {idx:03d}",
+            "Compound for pagination tests",
+        )
+
+
+@tag("api", "end2end")
+class RulePaginationAPITest(BaseTestAPIGetPaginated, TestCase):
+    """Rule pagination tests using base class."""
+
+    resource_name = "rule"
+    resource_name_plural = "rules"
+    global_endpoint = "/api/v1/rules/"
+    package_endpoint_template = "/api/v1/package/{uuid}/rule/"
+    total_reviewed = 125
+    total_unreviewed = 35
+
+    @classmethod
+    def create_reviewed_resource(cls, package, idx):
+        # Create unique SMIRKS by combining chain length and functional group variations
+        # This ensures each idx gets a truly unique SMIRKS pattern
+        carbon_chain = "C" * (idx + 1)  # C, CC, CCC, ... (grows with idx)
+        smirks = f"[{carbon_chain}:1]>>[{carbon_chain}:1]O"
+        return SimpleAmbitRule.create(
+            package,
+            f"Reviewed Rule {idx:03d}",
+            f"Rule {idx} for pagination tests",
+            smirks,
+        )
+
+    @classmethod
+    def create_unreviewed_resource(cls, package, idx):
+        # Create unique SMIRKS by varying the carbon chain length
+        carbon_chain = "C" * (idx + 1)  # C, CC, CCC, ... (grows with idx)
+        smirks = f"[{carbon_chain}:1]>>[{carbon_chain}:1]N"
+        return SimpleAmbitRule.create(
+            package,
+            f"Draft Rule {idx:03d}",
+            f"Rule {idx} for pagination tests",
+            smirks,
+        )
+
+
+@tag("api", "end2end")
+class ReactionPaginationAPITest(BaseTestAPIGetPaginated, TestCase):
+    """Reaction pagination tests using base class."""
+
+    resource_name = "reaction"
+    resource_name_plural = "reactions"
+    global_endpoint = "/api/v1/reactions/"
+    package_endpoint_template = "/api/v1/package/{uuid}/reaction/"
+    total_reviewed = 125
+    total_unreviewed = 35
+
+    @classmethod
+    def create_reviewed_resource(cls, package, idx):
+        # Generate unique SMILES with growing chain lengths to avoid duplicates
+        # Each idx gets a unique chain length
+        educt_smiles = "C" * (idx + 1)  # C, CC, CCC, ... (grows with idx)
+        product_smiles = educt_smiles + "O"
+        return Reaction.create(
+            package=package,
+            name=f"Reviewed Reaction {idx:03d}",
+            description="Reaction for pagination tests",
+            educts=[educt_smiles],
+            products=[product_smiles],
+        )
+
+    @classmethod
+    def create_unreviewed_resource(cls, package, idx):
+        # Generate unique SMILES with growing chain lengths to avoid duplicates
+        # Each idx gets a unique chain length
+        educt_smiles = "C" * (idx + 1)  # C, CC, CCC, ... (grows with idx)
+        product_smiles = educt_smiles + "N"
+        return Reaction.create(
+            package=package,
+            name=f"Draft Reaction {idx:03d}",
+            description="Reaction for pagination tests",
+            educts=[educt_smiles],
+            products=[product_smiles],
+        )
+
+
+@tag("api", "end2end")
+class PathwayPaginationAPITest(BaseTestAPIGetPaginated, TestCase):
+    """Pathway pagination tests using base class."""
+
+    resource_name = "pathway"
+    resource_name_plural = "pathways"
+    global_endpoint = "/api/v1/pathways/"
+    package_endpoint_template = "/api/v1/package/{uuid}/pathway/"
+    total_reviewed = 125
+    total_unreviewed = 35
+
+    @classmethod
+    def create_reviewed_resource(cls, package, idx):
+        return Pathway.objects.create(
+            package=package,
+            name=f"Reviewed Pathway {idx:03d}",
+            description="Pathway for pagination tests",
+        )
+
+    @classmethod
+    def create_unreviewed_resource(cls, package, idx):
+        return Pathway.objects.create(
+            package=package,
+            name=f"Draft Pathway {idx:03d}",
+            description="Pathway for pagination tests",
+        )
+
+
+@tag("api", "end2end")
+class ModelPaginationAPITest(BaseTestAPIGetPaginated, TestCase):
+    """Model pagination tests using base class."""
+
+    resource_name = "model"
+    resource_name_plural = "models"
+    global_endpoint = "/api/v1/models/"
+    package_endpoint_template = "/api/v1/package/{uuid}/model/"
+    total_reviewed = 125
+    total_unreviewed = 35
+
+    @classmethod
+    def create_reviewed_resource(cls, package, idx):
+        return EPModel.objects.create(
+            package=package,
+            name=f"Reviewed Model {idx:03d}",
+            description="Model for pagination tests",
+        )
+
+    @classmethod
+    def create_unreviewed_resource(cls, package, idx):
+        return EPModel.objects.create(
+            package=package,
+            name=f"Draft Model {idx:03d}",
+            description="Model for pagination tests",
+        )
+
+
+@tag("api", "end2end")
+class ScenarioPaginationAPITest(BaseTestAPIGetPaginated, TestCase):
+    """Scenario pagination tests using base class."""
+
+    resource_name = "scenario"
+    resource_name_plural = "scenarios"
+    global_endpoint = "/api/v1/scenarios/"
+    package_endpoint_template = "/api/v1/package/{uuid}/scenario/"
+    total_reviewed = 125
+    total_unreviewed = 35
+
+    @classmethod
+    def create_reviewed_resource(cls, package, idx):
+        return Scenario.create(
+            package,
+            f"Reviewed Scenario {idx:03d}",
+            "Scenario for pagination tests",
+            "2025-01-01",
+            "lab",
+            [],
+        )
+
+    @classmethod
+    def create_unreviewed_resource(cls, package, idx):
+        return Scenario.create(
+            package,
+            f"Draft Scenario {idx:03d}",
+            "Scenario for pagination tests",
+            "2025-01-01",
+            "field",
+            [],
+        )

epapi/tests/v1/test_scenario_creation.py

@@ -0,0 +1,301 @@
+"""
+Tests for Scenario Creation Endpoint Error Handling.
+
+Tests comprehensive error handling for POST /api/v1/package/{uuid}/scenario/
+including package not found, permission denied, validation errors, and database errors.
+"""
+
+from django.test import TestCase, tag
+import json
+from uuid import uuid4
+
+from epdb.logic import PackageManager, UserManager
+from epdb.models import Scenario
+
+
+@tag("api", "scenario_creation")
+class ScenarioCreationAPITests(TestCase):
+    """Test scenario creation endpoint error handling."""
+
+    @classmethod
+    def setUpTestData(cls):
+        """Set up test data: users and packages."""
+        cls.user = UserManager.create_user(
+            "scenario-test-user",
+            "scenario-test@envipath.com",
+            "SuperSafe",
+            set_setting=False,
+            add_to_group=False,
+            is_active=True,
+        )
+        cls.other_user = UserManager.create_user(
+            "other-user",
+            "other@envipath.com",
+            "SuperSafe",
+            set_setting=False,
+            add_to_group=False,
+            is_active=True,
+        )
+        cls.package = PackageManager.create_package(
+            cls.user, "Test Package", "Test package for scenario creation"
+        )
+
+    def test_create_scenario_package_not_found(self):
+        """Test that non-existent package UUID returns 404."""
+        self.client.force_login(self.user)
+
+        fake_uuid = uuid4()
+        payload = {
+            "name": "Test Scenario",
+            "description": "Test description",
+            "scenario_date": "2024-01-01",
+            "scenario_type": "biodegradation",
+            "additional_information": [],
+        }
+
+        response = self.client.post(
+            f"/api/v1/package/{fake_uuid}/scenario/",
+            data=json.dumps(payload),
+            content_type="application/json",
+        )
+
+        self.assertEqual(response.status_code, 404)
+        self.assertIn(f"Package with UUID {fake_uuid} not found", response.json()["detail"])
+
+    def test_create_scenario_insufficient_permissions(self):
+        """Test that unauthorized access returns 403."""
+        self.client.force_login(self.other_user)
+
+        payload = {
+            "name": "Test Scenario",
+            "description": "Test description",
+            "scenario_date": "2024-01-01",
+            "scenario_type": "biodegradation",
+            "additional_information": [],
+        }
+
+        response = self.client.post(
+            f"/api/v1/package/{self.package.uuid}/scenario/",
+            data=json.dumps(payload),
+            content_type="application/json",
+        )
+
+        self.assertEqual(response.status_code, 403)
+        self.assertIn("permission", response.json()["detail"].lower())
+
+    def test_create_scenario_invalid_ai_type(self):
+        """Test that unknown additional information type returns 400."""
+        self.client.force_login(self.user)
+
+        payload = {
+            "name": "Test Scenario",
+            "description": "Test description",
+            "scenario_date": "2024-01-01",
+            "scenario_type": "biodegradation",
+            "additional_information": [
+                {"type": "invalid_type_that_does_not_exist", "data": {"some_field": "some_value"}}
+            ],
+        }
+
+        response = self.client.post(
+            f"/api/v1/package/{self.package.uuid}/scenario/",
+            data=json.dumps(payload),
+            content_type="application/json",
+        )
+
+        self.assertEqual(response.status_code, 400)
+        response_data = response.json()
+        self.assertIn("Validation errors", response_data["detail"])
+
+    def test_create_scenario_validation_error(self):
+        """Test that invalid additional information data returns 400."""
+        self.client.force_login(self.user)
+
+        # Use malformed data structure for an actual AI type
+        payload = {
+            "name": "Test Scenario",
+            "description": "Test description",
+            "scenario_date": "2024-01-01",
+            "scenario_type": "biodegradation",
+            "additional_information": [
+                {
+                    "type": "invalid_type_name",
+                    "data": None,  # This should cause a validation error
+                }
+            ],
+        }
+
+        response = self.client.post(
+            f"/api/v1/package/{self.package.uuid}/scenario/",
+            data=json.dumps(payload),
+            content_type="application/json",
+        )
+
+        # Should return 422 for validation errors
+        self.assertEqual(response.status_code, 422)
+
+    def test_create_scenario_success(self):
+        """Test that valid scenario creation returns 200."""
+        self.client.force_login(self.user)
+
+        payload = {
+            "name": "Test Scenario",
+            "description": "Test description",
+            "scenario_date": "2024-01-01",
+            "scenario_type": "biodegradation",
+            "additional_information": [],
+        }
+
+        response = self.client.post(
+            f"/api/v1/package/{self.package.uuid}/scenario/",
+            data=json.dumps(payload),
+            content_type="application/json",
+        )
+
+        self.assertEqual(response.status_code, 200)
+        data = response.json()
+        self.assertEqual(data["name"], "Test Scenario")
+        self.assertEqual(data["description"], "Test description")
+
+        # Verify scenario was actually created
+        scenario = Scenario.objects.get(name="Test Scenario")
+        self.assertEqual(scenario.package, self.package)
+        self.assertEqual(scenario.scenario_type, "biodegradation")
+
+    def test_create_scenario_auto_name(self):
+        """Test that empty name triggers auto-generation."""
+        self.client.force_login(self.user)
+
+        payload = {
+            "name": "",  # Empty name should be auto-generated
+            "description": "Test description",
+            "scenario_date": "2024-01-01",
+            "scenario_type": "biodegradation",
+            "additional_information": [],
+        }
+
+        response = self.client.post(
+            f"/api/v1/package/{self.package.uuid}/scenario/",
+            data=json.dumps(payload),
+            content_type="application/json",
+        )
+
+        self.assertEqual(response.status_code, 200)
+        data = response.json()
+        # Auto-generated name should follow pattern "Scenario N"
+        self.assertTrue(data["name"].startswith("Scenario "))
+
+    def test_create_scenario_xss_protection(self):
+        """Test that XSS attempts are sanitized."""
+        self.client.force_login(self.user)
+
+        payload = {
+            "name": "<script>alert('xss')</script>Clean Name",
+            "description": "<img src=x onerror=alert('xss')>Description",
+            "scenario_date": "2024-01-01",
+            "scenario_type": "biodegradation",
+            "additional_information": [],
+        }
+
+        response = self.client.post(
+            f"/api/v1/package/{self.package.uuid}/scenario/",
+            data=json.dumps(payload),
+            content_type="application/json",
+        )
+
+        self.assertEqual(response.status_code, 200)
+        data = response.json()
+        # XSS should be cleaned out
+        self.assertNotIn("<script>", data["name"])
+        self.assertNotIn("onerror", data["description"])
+
+    def test_create_scenario_missing_required_field(self):
+        """Test that missing required fields returns validation error."""
+        self.client.force_login(self.user)
+
+        # Missing 'name' field entirely
+        payload = {
+            "description": "Test description",
+            "scenario_date": "2024-01-01",
+            "scenario_type": "biodegradation",
+            "additional_information": [],
+        }
+
+        response = self.client.post(
+            f"/api/v1/package/{self.package.uuid}/scenario/",
+            data=json.dumps(payload),
+            content_type="application/json",
+        )
+
+        # Should return 422 for schema validation errors
+        self.assertEqual(response.status_code, 422)
+
+    def test_create_scenario_type_error_in_ai(self):
+        """Test that TypeError in AI instantiation returns 400."""
+        self.client.force_login(self.user)
+
+        payload = {
+            "name": "Test Scenario",
+            "description": "Test description",
+            "scenario_date": "2024-01-01",
+            "scenario_type": "biodegradation",
+            "additional_information": [
+                {
+                    "type": "invalid_type_name",
+                    "data": "string instead of dict",  # Wrong type
+                }
+            ],
+        }
+
+        response = self.client.post(
+            f"/api/v1/package/{self.package.uuid}/scenario/",
+            data=json.dumps(payload),
+            content_type="application/json",
+        )
+
+        # Should return 422 for validation errors
+        self.assertEqual(response.status_code, 422)
+
+    def test_create_scenario_default_values(self):
+        """Test that default values are applied correctly."""
+        self.client.force_login(self.user)
+
+        # Minimal payload with only name
+        payload = {"name": "Minimal Scenario"}
+
+        response = self.client.post(
+            f"/api/v1/package/{self.package.uuid}/scenario/",
+            data=json.dumps(payload),
+            content_type="application/json",
+        )
+
+        self.assertEqual(response.status_code, 200)
+        data = response.json()
+        self.assertEqual(data["name"], "Minimal Scenario")
+        # Check defaults are applied
+        scenario = Scenario.objects.get(name="Minimal Scenario")
+        # Default description from model is "no description"
+        self.assertIn(scenario.description.lower(), ["", "no description"])
+
+    def test_create_scenario_unicode_characters(self):
+        """Test that unicode characters are handled properly."""
+        self.client.force_login(self.user)
+
+        payload = {
+            "name": "Test Scenario 测试 🧪",
+            "description": "Description with émojis and spëcial çhars",
+            "scenario_date": "2024-01-01",
+            "scenario_type": "biodegradation",
+            "additional_information": [],
+        }
+
+        response = self.client.post(
+            f"/api/v1/package/{self.package.uuid}/scenario/",
+            data=json.dumps(payload),
+            content_type="application/json",
+        )
+
+        self.assertEqual(response.status_code, 200)
+        data = response.json()
+        self.assertIn("测试", data["name"])
+        self.assertIn("émojis", data["description"])

epapi/tests/v1/test_schema_generation.py

@@ -0,0 +1,113 @@
+"""
+Property-based tests for schema generation.
+
+Tests that verify schema generation works correctly for all models,
+regardless of their structure.
+"""
+
+import pytest
+from typing import Type
+from pydantic import BaseModel
+
+from envipy_additional_information import registry, EnviPyModel
+from epapi.utils.schema_transformers import build_rjsf_output
+
+
+class TestSchemaGeneration:
+    """Test that all models can generate valid RJSF schemas."""
+
+    @pytest.mark.parametrize("model_name,model_cls", list(registry.list_models().items()))
+    def test_all_models_generate_rjsf(self, model_name: str, model_cls: Type[BaseModel]):
+        """Every model in the registry should generate valid RJSF format."""
+        # Skip non-EnviPyModel classes (parsers, etc.)
+        if not issubclass(model_cls, EnviPyModel):
+            pytest.skip(f"{model_name} is not an EnviPyModel")
+
+        # Should not raise exception
+        result = build_rjsf_output(model_cls)
+
+        # Verify structure
+        assert isinstance(result, dict), f"{model_name}: Result should be a dict"
+        assert "schema" in result, f"{model_name}: Missing 'schema' key"
+        assert "uiSchema" in result, f"{model_name}: Missing 'uiSchema' key"
+        assert "formData" in result, f"{model_name}: Missing 'formData' key"
+        assert "groups" in result, f"{model_name}: Missing 'groups' key"
+
+        # Verify types
+        assert isinstance(result["schema"], dict), f"{model_name}: schema should be dict"
+        assert isinstance(result["uiSchema"], dict), f"{model_name}: uiSchema should be dict"
+        assert isinstance(result["formData"], dict), f"{model_name}: formData should be dict"
+        assert isinstance(result["groups"], list), f"{model_name}: groups should be list"
+
+        # Verify schema has properties
+        assert "properties" in result["schema"], f"{model_name}: schema should have 'properties'"
+        assert isinstance(result["schema"]["properties"], dict), (
+            f"{model_name}: properties should be dict"
+        )
+
+    @pytest.mark.parametrize("model_name,model_cls", list(registry.list_models().items()))
+    def test_ui_schema_matches_schema_fields(self, model_name: str, model_cls: Type[BaseModel]):
+        """uiSchema keys should match schema properties (or be nested for intervals)."""
+        if not issubclass(model_cls, EnviPyModel):
+            pytest.skip(f"{model_name} is not an EnviPyModel")
+
+        result = build_rjsf_output(model_cls)
+        schema_props = set(result["schema"]["properties"].keys())
+        ui_schema_keys = set(result["uiSchema"].keys())
+
+        # uiSchema should have entries for all top-level properties
+        # (intervals may have nested start/end, but the main field should be present)
+        assert ui_schema_keys.issubset(schema_props), (
+            f"{model_name}: uiSchema has keys not in schema: {ui_schema_keys - schema_props}"
+        )
+
+    @pytest.mark.parametrize("model_name,model_cls", list(registry.list_models().items()))
+    def test_groups_is_list_of_strings(self, model_name: str, model_cls: Type[BaseModel]):
+        """Groups should be a list of strings."""
+        if not issubclass(model_cls, EnviPyModel):
+            pytest.skip(f"{model_name} is not an EnviPyModel")
+
+        result = build_rjsf_output(model_cls)
+        groups = result["groups"]
+
+        assert isinstance(groups, list), f"{model_name}: groups should be list"
+        assert all(isinstance(g, str) for g in groups), (
+            f"{model_name}: all groups should be strings, got {groups}"
+        )
+
+    @pytest.mark.parametrize("model_name,model_cls", list(registry.list_models().items()))
+    def test_form_data_matches_schema(self, model_name: str, model_cls: Type[BaseModel]):
+        """formData keys should match schema properties."""
+        if not issubclass(model_cls, EnviPyModel):
+            pytest.skip(f"{model_name} is not an EnviPyModel")
+
+        result = build_rjsf_output(model_cls)
+        schema_props = set(result["schema"]["properties"].keys())
+        form_data_keys = set(result["formData"].keys())
+
+        # formData should only contain keys that are in schema
+        assert form_data_keys.issubset(schema_props), (
+            f"{model_name}: formData has keys not in schema: {form_data_keys - schema_props}"
+        )
+
+
+class TestWidgetTypes:
+    """Test that widget types are valid."""
+
+    @pytest.mark.parametrize("model_name,model_cls", list(registry.list_models().items()))
+    def test_widget_types_are_valid(self, model_name: str, model_cls: Type[BaseModel]):
+        """All widget types in uiSchema should be valid WidgetType values."""
+        from envipy_additional_information.ui_config import WidgetType
+
+        if not issubclass(model_cls, EnviPyModel):
+            pytest.skip(f"{model_name} is not an EnviPyModel")
+
+        result = build_rjsf_output(model_cls)
+        valid_widgets = {wt.value for wt in WidgetType}
+
+        for field_name, ui_config in result["uiSchema"].items():
+            widget = ui_config.get("ui:widget")
+            if widget:
+                assert widget in valid_widgets, (
+                    f"{model_name}.{field_name}: Invalid widget '{widget}'. Valid: {valid_widgets}"
+                )

epapi/tests/v1/test_token_auth.py

@@ -0,0 +1,94 @@
+from datetime import timedelta
+
+from django.test import TestCase, tag
+from django.utils import timezone
+
+from epdb.logic import PackageManager, UserManager
+from epdb.models import APIToken
+
+
+@tag("api", "auth")
+class BearerTokenAuthTests(TestCase):
+    @classmethod
+    def setUpTestData(cls):
+        cls.user = UserManager.create_user(
+            "token-user",
+            "token-user@envipath.com",
+            "SuperSafe",
+            set_setting=False,
+            add_to_group=False,
+            is_active=True,
+        )
+
+        default_pkg = cls.user.default_package
+        cls.user.default_package = None
+        cls.user.save()
+        if default_pkg:
+            default_pkg.delete()
+
+        cls.unreviewed_package = PackageManager.create_package(
+            cls.user, "Token Auth Package", "Package for token auth tests"
+        )
+
+    def _auth_header(self, raw_token):
+        return {"HTTP_AUTHORIZATION": f"Bearer {raw_token}"}
+
+    def test_valid_token_allows_access(self):
+        _, raw_token = APIToken.create_token(self.user, name="Valid Token", expires_days=1)
+
+        response = self.client.get("/api/v1/compounds/", **self._auth_header(raw_token))
+
+        self.assertEqual(response.status_code, 200)
+
+    def test_expired_token_rejected(self):
+        token, raw_token = APIToken.create_token(self.user, name="Expired Token", expires_days=1)
+        token.expires_at = timezone.now() - timedelta(days=1)
+        token.save(update_fields=["expires_at"])
+
+        response = self.client.get("/api/v1/compounds/", **self._auth_header(raw_token))
+
+        self.assertEqual(response.status_code, 401)
+
+    def test_inactive_token_rejected(self):
+        token, raw_token = APIToken.create_token(self.user, name="Inactive Token", expires_days=1)
+        token.is_active = False
+        token.save(update_fields=["is_active"])
+
+        response = self.client.get("/api/v1/compounds/", **self._auth_header(raw_token))
+
+        self.assertEqual(response.status_code, 401)
+
+    def test_invalid_token_rejected(self):
+        response = self.client.get("/api/v1/compounds/", HTTP_AUTHORIZATION="Bearer invalid-token")
+
+        self.assertEqual(response.status_code, 401)
+
+    def test_no_token_rejected(self):
+        self.client.logout()
+        response = self.client.get("/api/v1/compounds/")
+
+        self.assertEqual(response.status_code, 401)
+
+    def test_bearer_populates_request_user_for_packages(self):
+        response = self.client.get("/api/v1/packages/")
+        self.assertEqual(response.status_code, 200)
+        payload = response.json()
+        uuids = {item["uuid"] for item in payload["items"]}
+        self.assertNotIn(str(self.unreviewed_package.uuid), uuids)
+
+        _, raw_token = APIToken.create_token(self.user, name="Package Token", expires_days=1)
+        response = self.client.get("/api/v1/packages/", **self._auth_header(raw_token))
+
+        self.assertEqual(response.status_code, 200)
+        payload = response.json()
+        uuids = {item["uuid"] for item in payload["items"]}
+        self.assertIn(str(self.unreviewed_package.uuid), uuids)
+
+    def test_session_auth_still_works_without_bearer(self):
+        self.client.force_login(self.user)
+        response = self.client.get("/api/v1/packages/")
+
+        self.assertEqual(response.status_code, 200)
+        payload = response.json()
+        uuids = {item["uuid"] for item in payload["items"]}
+        self.assertIn(str(self.unreviewed_package.uuid), uuids)

epapi/utils/schema_transformers.py

@@ -0,0 +1,181 @@
+"""
+Schema transformation utilities for converting Pydantic models to RJSF format.
+
+This module provides functions to extract UI configuration from Pydantic models
+and transform them into React JSON Schema Form (RJSF) compatible format.
+"""
+
+from typing import Type, Optional, Any
+
+import jsonref
+from pydantic import BaseModel
+
+from envipy_additional_information.ui_config import UIConfig
+from envipy_additional_information import registry
+
+
+def extract_groups(model_cls: Type[BaseModel]) -> list[str]:
+    """
+    Extract groups from registry-stored group information.
+
+    Args:
+        model_cls: The model class
+
+    Returns:
+        List of group names the model belongs to
+    """
+    return registry.get_groups(model_cls)
+
+
+def extract_ui_metadata(model_cls: Type[BaseModel]) -> dict[str, Any]:
+    """
+    Extract model-level UI metadata from UI class.
+
+    Returns metadata attributes that are NOT UIConfig instances.
+    Common metadata includes: unit, description, title.
+    """
+    metadata: dict[str, Any] = {}
+
+    if not hasattr(model_cls, "UI"):
+        return metadata
+
+    ui_class = getattr(model_cls, "UI")
+
+    # Iterate over all attributes in the UI class
+    for attr_name in dir(ui_class):
+        # Skip private attributes
+        if attr_name.startswith("_"):
+            continue
+
+        # Get the attribute value
+        try:
+            attr_value = getattr(ui_class, attr_name)
+        except AttributeError:
+            continue
+
+        # Skip callables but keep types/classes
+        if callable(attr_value) and not isinstance(attr_value, type):
+            continue
+
+        # Skip UIConfig instances (these are field-level configs, not metadata)
+        # This includes both UIConfig and IntervalConfig
+        if isinstance(attr_value, UIConfig):
+            continue
+
+        metadata[attr_name] = attr_value
+
+    return metadata
+
+
+def extract_ui_config_from_model(model_cls: Type[BaseModel]) -> dict[str, Any]:
+    """
+    Extract UI configuration from model's UI class.
+
+    Returns a dictionary mapping field names to their UI schema configurations.
+    Trusts the config classes to handle their own transformation logic.
+    """
+    ui_configs: dict[str, Any] = {}
+
+    if not hasattr(model_cls, "UI"):
+        return ui_configs
+
+    ui_class = getattr(model_cls, "UI")
+    schema = model_cls.model_json_schema()
+    field_names = schema.get("properties", {}).keys()
+
+    # Extract config for each field
+    for field_name in field_names:
+        # Skip if UI config doesn't exist for this field (field may be hidden from UI)
+        if not hasattr(ui_class, field_name):
+            continue
+
+        ui_config = getattr(ui_class, field_name)
+
+        if isinstance(ui_config, UIConfig):
+            ui_configs[field_name] = ui_config.to_ui_schema_field()
+
+    return ui_configs
+
+
+def build_ui_schema(model_cls: Type[BaseModel]) -> dict:
+    """Generate RJSF uiSchema from model's UI class."""
+    ui_schema = {}
+
+    # Extract field-level UI configs
+    field_configs = extract_ui_config_from_model(model_cls)
+
+    for field_name, config in field_configs.items():
+        ui_schema[field_name] = config
+
+    return ui_schema
+
+
+def build_schema(model_cls: Type[BaseModel]) -> dict[str, Any]:
+    """
+    Build JSON schema from Pydantic model, applying UI metadata.
+
+    Dereferences all $ref pointers to produce fully inlined schema.
+    This ensures the frontend receives schemas with enum values and nested
+    properties fully resolved, without needing client-side ref resolution.
+
+    Extracts model-level metadata from UI class (title, unit, etc.) and applies
+    it to the generated schema. This ensures UI metadata is the single source of truth.
+    """
+    schema = model_cls.model_json_schema()
+
+    # Dereference $ref pointers (inlines $defs) using jsonref
+    # This ensures the frontend receives schemas with enum values and nested
+    # properties fully resolved, currently necessary for client-side rendering.
+    # FIXME: This is a hack to get the schema to work with alpine schema-form.js replace once we migrate to client-side framework.
+    schema = jsonref.replace_refs(schema, proxies=False)
+
+    # Remove $defs section as all refs are now inlined
+    if "$defs" in schema:
+        del schema["$defs"]
+
+    # Extract and apply UI metadata (title, unit, description, etc.)
+    ui_metadata = extract_ui_metadata(model_cls)
+
+    # Apply all metadata consistently as custom properties with x- prefix
+    # This ensures consistency and avoids conflicts with standard JSON Schema properties
+    for key, value in ui_metadata.items():
+        if value is not None:
+            schema[f"x-{key}"] = value
+
+    # Set standard title property from UI metadata for JSON Schema compliance
+    if "title" in ui_metadata:
+        schema["title"] = ui_metadata["title"]
+    elif "label" in ui_metadata:
+        schema["title"] = ui_metadata["label"]
+
+    return schema
+
+
+def build_rjsf_output(model_cls: Type[BaseModel], initial_data: Optional[dict] = None) -> dict:
+    """
+    Main function that returns complete RJSF format.
+
+    Trusts the config classes to handle their own transformation logic.
+    No special-case handling - if a config knows how to transform itself, it will.
+
+    Returns:
+        dict with keys: schema, uiSchema, formData, groups
+    """
+    # Build schema with UI metadata applied
+    schema = build_schema(model_cls)
+
+    # Build UI schema - config classes handle their own transformation
+    ui_schema = build_ui_schema(model_cls)
+
+    # Extract groups from marker interfaces
+    groups = extract_groups(model_cls)
+
+    # Use provided initial_data or empty dict
+    form_data = initial_data if initial_data is not None else {}
+
+    return {
+        "schema": schema,
+        "uiSchema": ui_schema,
+        "formData": form_data,
+        "groups": groups,
+    }

epapi/utils/validation_errors.py

@@ -0,0 +1,82 @@
+"""Shared utilities for handling Pydantic validation errors."""
+
+import json
+from pydantic import ValidationError
+from pydantic_core import ErrorDetails
+from ninja.errors import HttpError
+
+
+def format_validation_error(error: ErrorDetails) -> str:
+    """Format a Pydantic validation error into a user-friendly message.
+
+    Args:
+        error: A Pydantic error details dictionary containing 'msg', 'type', 'ctx', etc.
+
+    Returns:
+        A user-friendly error message string.
+    """
+    msg = error.get("msg") or "Invalid value"
+    error_type = error.get("type") or ""
+
+    # Handle common validation types with friendly messages
+    if error_type == "enum":
+        ctx = error.get("ctx", {})
+        expected = ctx.get("expected", "") if ctx else ""
+        return f"Please select a valid option{': ' + expected if expected else ''}"
+    elif error_type == "literal_error":
+        # Literal errors (like Literal["active", "inactive"])
+        return msg.replace("Input should be ", "Please enter ")
+    elif error_type == "missing":
+        return "This field is required"
+    elif error_type == "string_type":
+        return "Please enter a valid string"
+    elif error_type == "int_type":
+        return "Please enter a valid int"
+    elif error_type == "int_parsing":
+        return "Please enter a valid int"
+    elif error_type == "float_type":
+        return "Please enter a valid float"
+    elif error_type == "float_parsing":
+        return "Please enter a valid float"
+    elif error_type == "value_error":
+        # Strip "Value error, " prefix from custom validator messages
+        return msg.replace("Value error, ", "")
+    else:
+        # Default: use the message from Pydantic but clean it up
+        return msg.replace("Input should be ", "Please enter ").replace("Value error, ", "")
+
+
+def handle_validation_error(e: ValidationError) -> None:
+    """Convert a Pydantic ValidationError into a structured HttpError.
+
+    This function transforms Pydantic validation errors into a JSON structure
+    that the frontend expects for displaying field-level errors.
+
+    Args:
+        e: The Pydantic ValidationError to handle.
+
+    Raises:
+        HttpError: Always raises a 400 error with structured JSON containing
+                   type, field_errors, and message fields.
+    """
+    # Transform Pydantic validation errors into user-friendly format
+    field_errors: dict[str, list[str]] = {}
+    for error in e.errors():
+        # Get the field name from location tuple
+        loc = error.get("loc", ())
+        field = str(loc[-1]) if loc else "root"
+
+        # Format the error message
+        friendly_msg = format_validation_error(error)
+
+        if field not in field_errors:
+            field_errors[field] = []
+        field_errors[field].append(friendly_msg)
+
+    # Return structured error for frontend parsing
+    error_response = {
+        "type": "validation_error",
+        "field_errors": field_errors,
+        "message": "Please correct the errors below",
+    }
+    raise HttpError(400, json.dumps(error_response))

epapi/v1/auth.py

@@ -0,0 +1,34 @@
+import hashlib
+
+from ninja.security import HttpBearer
+from ninja.errors import HttpError
+
+from epdb.models import APIToken
+
+
+class BearerTokenAuth(HttpBearer):
+    def authenticate(self, request, token):
+        if token is None:
+            return None
+
+        hashed_token = hashlib.sha256(token.encode()).hexdigest()
+        user = APIToken.authenticate(hashed_token, hashed=True)
+        if not user:
+            raise HttpError(401, "Invalid or expired token")
+
+        request.user = user
+        return user
+
+
+class OptionalBearerTokenAuth:
+    """Bearer auth that allows unauthenticated access.
+
+    Validates the Bearer token if present (401 on invalid token),
+    otherwise lets the request through for anonymous/session access.
+    """
+
+    def __init__(self):
+        self._bearer = BearerTokenAuth()
+
+    def __call__(self, request):
+        return self._bearer(request) or request.user

epapi/v1/dal.py

@@ -0,0 +1,141 @@
+from uuid import UUID
+
+from django.conf import settings as s
+from django.db.models import Model
+from epdb.logic import PackageManager
+from epdb.models import CompoundStructure, User, Compound, Scenario
+
+from .errors import EPAPINotFoundError, EPAPIPermissionDeniedError
+
+Package = s.GET_PACKAGE_MODEL()
+
+
+def get_compound_for_read(user, compound_uuid: UUID):
+    """
+    Get compound by UUID with permission check.
+    """
+    try:
+        compound = Compound.objects.get(uuid=compound_uuid)
+        package = compound.package
+    except Compound.DoesNotExist:
+        raise EPAPINotFoundError(f"Compound with UUID {compound_uuid} not found")
+
+    # FIXME: optimize package manager to exclusively work with UUIDs
+    if not user or user.is_anonymous or not PackageManager.readable(user, package):
+        raise EPAPIPermissionDeniedError("Insufficient permissions to access this compound.")
+
+    return compound
+
+
+def get_package_for_read(user, package_uuid: UUID):
+    """
+    Get package by UUID with permission check.
+    """
+
+    # FIXME: update package manager with custom exceptions to avoid manual checks here
+    try:
+        package = Package.objects.get(uuid=package_uuid)
+    except Package.DoesNotExist:
+        raise EPAPINotFoundError(f"Package with UUID {package_uuid} not found")
+
+    # FIXME: optimize package manager to exclusively work with UUIDs
+    if not user or user.is_anonymous or not PackageManager.readable(user, package):
+        raise EPAPIPermissionDeniedError("Insufficient permissions to access this package.")
+
+    return package
+
+
+def get_package_for_write(user, package_uuid: UUID):
+    """
+    Get package by UUID with permission check.
+    """
+
+    # FIXME: update package manager with custom exceptions to avoid manual checks here
+    try:
+        package = Package.objects.get(uuid=package_uuid)
+    except Package.DoesNotExist:
+        raise EPAPINotFoundError(f"Package with UUID {package_uuid} not found")
+
+    # FIXME: optimize package manager to exclusively work with UUIDs
+    if not user or user.is_anonymous or not PackageManager.writable(user, package):
+        raise EPAPIPermissionDeniedError("Insufficient permissions to access this package.")
+
+    return package
+
+
+def get_scenario_for_read(user, scenario_uuid: UUID):
+    """Get scenario by UUID with read permission check."""
+    try:
+        scenario = Scenario.objects.select_related("package").get(uuid=scenario_uuid)
+    except Scenario.DoesNotExist:
+        raise EPAPINotFoundError(f"Scenario with UUID {scenario_uuid} not found")
+
+    if not user or user.is_anonymous or not PackageManager.readable(user, scenario.package):
+        raise EPAPIPermissionDeniedError("Insufficient permissions to access this scenario.")
+
+    return scenario
+
+
+def get_scenario_for_write(user, scenario_uuid: UUID):
+    """Get scenario by UUID with write permission check."""
+    try:
+        scenario = Scenario.objects.select_related("package").get(uuid=scenario_uuid)
+    except Scenario.DoesNotExist:
+        raise EPAPINotFoundError(f"Scenario with UUID {scenario_uuid} not found")
+
+    if not user or user.is_anonymous or not PackageManager.writable(user, scenario.package):
+        raise EPAPIPermissionDeniedError("Insufficient permissions to modify this scenario.")
+
+    return scenario
+
+
+def get_user_packages_for_read(user: User | None):
+    """Get all packages readable by the user."""
+    if not user or user.is_anonymous:
+        return PackageManager.get_reviewed_packages()
+    return PackageManager.get_all_readable_packages(user, include_reviewed=True)
+
+
+def get_user_entities_for_read(model_class: Model, user: User | None):
+    """Build queryset for reviewed package entities."""
+
+    if not user or user.is_anonymous:
+        return model_class.objects.filter(package__reviewed=True).select_related("package")
+
+    qs = model_class.objects.filter(
+        package__in=PackageManager.get_all_readable_packages(user, include_reviewed=True)
+    ).select_related("package")
+    return qs
+
+
+def get_package_entities_for_read(model_class: Model, package_uuid: UUID, user: User | None = None):
+    """Build queryset for specific package entities."""
+    package = get_package_for_read(user, package_uuid)
+    qs = model_class.objects.filter(package=package).select_related("package")
+    return qs
+
+
+def get_user_structure_for_read(user: User | None):
+    """Build queryset for structures accessible to the user (via compound->package)."""
+
+    if not user or user.is_anonymous:
+        return CompoundStructure.objects.filter(compound__package__reviewed=True).select_related(
+            "compound__package"
+        )
+
+    qs = CompoundStructure.objects.filter(
+        compound__package__in=PackageManager.get_all_readable_packages(user, include_reviewed=True)
+    ).select_related("compound__package")
+    return qs
+
+
+def get_package_compound_structure_for_read(
+    package_uuid: UUID, compound_uuid: UUID, user: User | None = None
+):
+    """Build queryset for specific package compound structures."""
+
+    get_package_for_read(user, package_uuid)
+    compound = get_compound_for_read(user, compound_uuid)
+
+    qs = CompoundStructure.objects.filter(compound=compound).select_related("compound__package")
+    return qs

epapi/v1/endpoints/additional_information.py

@@ -0,0 +1,174 @@
+from ninja import Router, Body
+from ninja.errors import HttpError
+from uuid import UUID
+from pydantic import ValidationError
+from typing import Dict, Any
+import logging
+
+from envipy_additional_information import registry
+from envipy_additional_information.groups import GroupEnum
+from epapi.utils.schema_transformers import build_rjsf_output
+from epapi.utils.validation_errors import handle_validation_error
+from epdb.models import AdditionalInformation
+from ..dal import get_scenario_for_read, get_scenario_for_write
+
+logger = logging.getLogger(__name__)
+
+router = Router(tags=["Additional Information"])
+
+
+@router.get("/information/schema/")
+def list_all_schemas(request):
+    """Return all schemas in RJSF format with lowercase class names as keys."""
+    result = {}
+    for name, cls in registry.list_models().items():
+        try:
+            result[name] = build_rjsf_output(cls)
+        except Exception as e:
+            logger.warning(f"Failed to generate schema for {name}: {e}")
+            continue
+    return result
+
+
+@router.get("/information/schema/{model_name}/")
+def get_model_schema(request, model_name: str):
+    """Return RJSF schema for specific model."""
+    cls = registry.get_model(model_name.lower())
+    if not cls:
+        raise HttpError(404, f"Unknown model: {model_name}")
+    return build_rjsf_output(cls)
+
+
+@router.get("/scenario/{uuid:scenario_uuid}/information/")
+def list_scenario_info(request, scenario_uuid: UUID):
+    """List all additional information for a scenario"""
+    scenario = get_scenario_for_read(request.user, scenario_uuid)
+
+    result = []
+
+    for ai in AdditionalInformation.objects.filter(scenario=scenario):
+        result.append(
+            {
+                "type": ai.get().__class__.__name__,
+                "uuid": getattr(ai, "uuid", None),
+                "data": ai.data,
+                "attach_object": ai.content_object.simple_json() if ai.content_object else None,
+            }
+        )
+    return result
+
+
+@router.post("/scenario/{uuid:scenario_uuid}/information/{model_name}/")
+def add_scenario_info(
+    request, scenario_uuid: UUID, model_name: str, payload: Dict[str, Any] = Body(...)
+):
+    """Add new additional information to scenario"""
+    cls = registry.get_model(model_name.lower())
+    if not cls:
+        raise HttpError(404, f"Unknown model: {model_name}")
+
+    try:
+        instance = cls(**payload)  # Pydantic validates
+    except ValidationError as e:
+        handle_validation_error(e)
+
+    scenario = get_scenario_for_write(request.user, scenario_uuid)
+
+    # Model method now returns the UUID
+    created_uuid = scenario.add_additional_information(instance)
+
+    return {"status": "created", "uuid": created_uuid}
+
+
+@router.patch("/scenario/{uuid:scenario_uuid}/information/item/{uuid:ai_uuid}/")
+def update_scenario_info(
+    request, scenario_uuid: UUID, ai_uuid: UUID, payload: Dict[str, Any] = Body(...)
+):
+    """Update existing additional information for a scenario"""
+    scenario = get_scenario_for_write(request.user, scenario_uuid)
+    ai_uuid_str = str(ai_uuid)
+
+    ai = AdditionalInformation.objects.filter(uuid=ai_uuid_str, scenario=scenario)
+
+    if not ai.exists():
+        raise HttpError(404, f"Additional information with UUID {ai_uuid} not found")
+
+    ai = ai.first()
+
+    # Get the model class for validation
+    cls = registry.get_model(ai.type.lower())
+    if not cls:
+        raise HttpError(500, f"Unknown model type in data: {ai.type}")
+
+    # Validate the payload against the model
+    try:
+        instance = cls(**payload)
+    except ValidationError as e:
+        handle_validation_error(e)
+
+    # Use model method for update
+    try:
+        scenario.update_additional_information(ai_uuid_str, instance)
+    except ValueError as e:
+        raise HttpError(404, str(e))
+
+    return {"status": "updated", "uuid": ai_uuid_str}
+
+
+@router.delete("/scenario/{uuid:scenario_uuid}/information/item/{uuid:ai_uuid}/")
+def delete_scenario_info(request, scenario_uuid: UUID, ai_uuid: UUID):
+    """Delete additional information from scenario"""
+    scenario = get_scenario_for_write(request.user, scenario_uuid)
+
+    try:
+        scenario.remove_additional_information(str(ai_uuid))
+    except ValueError as e:
+        raise HttpError(404, str(e))
+
+    return {"status": "deleted"}
+
+
+@router.get("/information/groups/")
+def list_groups(request):
+    """Return list of available group names."""
+    return {"groups": GroupEnum.values()}
+
+
+@router.get("/information/groups/{group_name}/")
+def get_group_models(request, group_name: str):
+    """
+    Return models for a specific group organized by subcategory.
+
+    Args:
+        group_name: One of "sludge", "soil", or "sediment" (string)
+
+    Returns:
+        Dictionary with subcategories (exp, spike, comp, misc, or group name)
+        as keys and lists of model info as values
+    """
+    # Convert string to enum (raises ValueError if invalid)
+    try:
+        group_enum = GroupEnum(group_name)
+    except ValueError:
+        valid = ", ".join(GroupEnum.values())
+        raise HttpError(400, f"Invalid group '{group_name}'. Valid: {valid}")
+
+    try:
+        group_data = registry.collect_group(group_enum)
+    except (ValueError, TypeError) as e:
+        raise HttpError(400, str(e))
+
+    result = {}
+    for subcategory, models in group_data.items():
+        result[subcategory] = [
+            {
+                "name": cls.__name__.lower(),
+                "class": cls.__name__,
+                "title": getattr(cls.UI, "title", cls.__name__)
+                if hasattr(cls, "UI")
+                else cls.__name__,
+            }
+            for cls in models
+        ]
+
+    return result

epapi/v1/endpoints/compounds.py

@@ -0,0 +1,41 @@
+from django.conf import settings as s
+from ninja import Router
+from ninja_extra.pagination import paginate
+from uuid import UUID
+
+from epdb.models import Compound
+from ..pagination import EnhancedPageNumberPagination
+from ..schemas import CompoundOutSchema, ReviewStatusFilter
+from ..dal import get_user_entities_for_read, get_package_entities_for_read
+
+router = Router()
+
+
+@router.get("/compounds/", response=EnhancedPageNumberPagination.Output[CompoundOutSchema])
+@paginate(
+    EnhancedPageNumberPagination,
+    page_size=s.API_PAGINATION_DEFAULT_PAGE_SIZE,
+    filter_schema=ReviewStatusFilter,
+)
+def list_all_compounds(request):
+    """
+    List all compounds from reviewed packages.
+    """
+    return get_user_entities_for_read(Compound, request.user).order_by("name").all()
+
+
+@router.get(
+    "/package/{uuid:package_uuid}/compound/",
+    response=EnhancedPageNumberPagination.Output[CompoundOutSchema],
+)
+@paginate(
+    EnhancedPageNumberPagination,
+    page_size=s.API_PAGINATION_DEFAULT_PAGE_SIZE,
+    filter_schema=ReviewStatusFilter,
+)
+def list_package_compounds(request, package_uuid: UUID):
+    """
+    List all compounds for a specific package.
+    """
+    user = request.user
+    return get_package_entities_for_read(Compound, package_uuid, user).order_by("name").all()

epapi/v1/endpoints/groups.py

@@ -0,0 +1,23 @@
+from django.conf import settings as s
+from ninja import Router
+from ninja_extra.pagination import paginate
+
+from epdb.logic import GroupManager
+
+from ..pagination import EnhancedPageNumberPagination
+from ..schemas import GroupOutSchema
+
+router = Router()
+
+
+@router.get("/groups/", response=EnhancedPageNumberPagination.Output[GroupOutSchema])
+@paginate(
+    EnhancedPageNumberPagination,
+    page_size=s.API_PAGINATION_DEFAULT_PAGE_SIZE,
+)
+def list_all_groups(request):
+    """
+    List all groups the user has access to.
+    """
+    user = request.user
+    return GroupManager.get_groups(user)

epapi/v1/endpoints/models.py

@@ -0,0 +1,41 @@
+from django.conf import settings as s
+from ninja import Router
+from ninja_extra.pagination import paginate
+from uuid import UUID
+
+from epdb.models import EPModel
+from ..pagination import EnhancedPageNumberPagination
+from ..schemas import ModelOutSchema, ReviewStatusFilter
+from ..dal import get_user_entities_for_read, get_package_entities_for_read
+
+router = Router()
+
+
+@router.get("/models/", response=EnhancedPageNumberPagination.Output[ModelOutSchema])
+@paginate(
+    EnhancedPageNumberPagination,
+    page_size=s.API_PAGINATION_DEFAULT_PAGE_SIZE,
+    filter_schema=ReviewStatusFilter,
+)
+def list_all_models(request):
+    """
+    List all models from reviewed packages.
+    """
+    return get_user_entities_for_read(EPModel, request.user).order_by("name").all()
+
+
+@router.get(
+    "/package/{uuid:package_uuid}/model/",
+    response=EnhancedPageNumberPagination.Output[ModelOutSchema],
+)
+@paginate(
+    EnhancedPageNumberPagination,
+    page_size=s.API_PAGINATION_DEFAULT_PAGE_SIZE,
+    filter_schema=ReviewStatusFilter,
+)
+def list_package_models(request, package_uuid: UUID):
+    """
+    List all models for a specific package.
+    """
+    user = request.user
+    return get_package_entities_for_read(EPModel, package_uuid, user).order_by("name").all()

epapi/v1/endpoints/packages.py

@@ -0,0 +1,32 @@
+from django.conf import settings as s
+from ninja import Router
+from ninja_extra.pagination import paginate
+import logging
+
+from ..auth import OptionalBearerTokenAuth
+from ..dal import get_user_packages_for_read
+from ..pagination import EnhancedPageNumberPagination
+from ..schemas import PackageOutSchema, SelfReviewStatusFilter
+
+router = Router()
+logger = logging.getLogger(__name__)
+
+
+@router.get(
+    "/packages/",
+    response=EnhancedPageNumberPagination.Output[PackageOutSchema],
+    auth=OptionalBearerTokenAuth(),
+)
+@paginate(
+    EnhancedPageNumberPagination,
+    page_size=s.API_PAGINATION_DEFAULT_PAGE_SIZE,
+    filter_schema=SelfReviewStatusFilter,
+)
+def list_all_packages(request):
+    """
+    List packages accessible to the user.
+
+    """
+    user = request.user
+    qs = get_user_packages_for_read(user)
+    return qs.order_by("name").all()

epapi/v1/endpoints/pathways.py

@@ -0,0 +1,42 @@
+from django.conf import settings as s
+from ninja import Router
+from ninja_extra.pagination import paginate
+from uuid import UUID
+
+from epdb.models import Pathway
+from ..pagination import EnhancedPageNumberPagination
+from ..schemas import PathwayOutSchema, ReviewStatusFilter
+from ..dal import get_user_entities_for_read, get_package_entities_for_read
+
+router = Router()
+
+
+@router.get("/pathways/", response=EnhancedPageNumberPagination.Output[PathwayOutSchema])
+@paginate(
+    EnhancedPageNumberPagination,
+    page_size=s.API_PAGINATION_DEFAULT_PAGE_SIZE,
+    filter_schema=ReviewStatusFilter,
+)
+def list_all_pathways(request):
+    """
+    List all pathways from reviewed packages.
+    """
+    user = request.user
+    return get_user_entities_for_read(Pathway, user).order_by("name").all()
+
+
+@router.get(
+    "/package/{uuid:package_uuid}/pathway/",
+    response=EnhancedPageNumberPagination.Output[PathwayOutSchema],
+)
+@paginate(
+    EnhancedPageNumberPagination,
+    page_size=s.API_PAGINATION_DEFAULT_PAGE_SIZE,
+    filter_schema=ReviewStatusFilter,
+)
+def list_package_pathways(request, package_uuid: UUID):
+    """
+    List all pathways for a specific package.
+    """
+    user = request.user
+    return get_package_entities_for_read(Pathway, package_uuid, user).order_by("name").all()

epapi/v1/endpoints/reactions.py

@@ -0,0 +1,42 @@
+from django.conf import settings as s
+from ninja import Router
+from ninja_extra.pagination import paginate
+from uuid import UUID
+
+from epdb.models import Reaction
+from ..pagination import EnhancedPageNumberPagination
+from ..schemas import ReactionOutSchema, ReviewStatusFilter
+from ..dal import get_user_entities_for_read, get_package_entities_for_read
+
+router = Router()
+
+
+@router.get("/reactions/", response=EnhancedPageNumberPagination.Output[ReactionOutSchema])
+@paginate(
+    EnhancedPageNumberPagination,
+    page_size=s.API_PAGINATION_DEFAULT_PAGE_SIZE,
+    filter_schema=ReviewStatusFilter,
+)
+def list_all_reactions(request):
+    """
+    List all reactions from reviewed packages.
+    """
+    user = request.user
+    return get_user_entities_for_read(Reaction, user).order_by("name").all()
+
+
+@router.get(
+    "/package/{uuid:package_uuid}/reaction/",
+    response=EnhancedPageNumberPagination.Output[ReactionOutSchema],
+)
+@paginate(
+    EnhancedPageNumberPagination,
+    page_size=s.API_PAGINATION_DEFAULT_PAGE_SIZE,
+    filter_schema=ReviewStatusFilter,
+)
+def list_package_reactions(request, package_uuid: UUID):
+    """
+    List all reactions for a specific package.
+    """
+    user = request.user
+    return get_package_entities_for_read(Reaction, package_uuid, user).order_by("name").all()

epapi/v1/endpoints/rules.py

@@ -0,0 +1,42 @@
+from django.conf import settings as s
+from ninja import Router
+from ninja_extra.pagination import paginate
+from uuid import UUID
+
+from epdb.models import Rule
+from ..pagination import EnhancedPageNumberPagination
+from ..schemas import ReviewStatusFilter, RuleOutSchema
+from ..dal import get_user_entities_for_read, get_package_entities_for_read
+
+router = Router()
+
+
+@router.get("/rules/", response=EnhancedPageNumberPagination.Output[RuleOutSchema])
+@paginate(
+    EnhancedPageNumberPagination,
+    page_size=s.API_PAGINATION_DEFAULT_PAGE_SIZE,
+    filter_schema=ReviewStatusFilter,
+)
+def list_all_rules(request):
+    """
+    List all rules from reviewed packages.
+    """
+    user = request.user
+    return get_user_entities_for_read(Rule, user).order_by("name").all()
+
+
+@router.get(
+    "/package/{uuid:package_uuid}/rule/",
+    response=EnhancedPageNumberPagination.Output[RuleOutSchema],
+)
+@paginate(
+    EnhancedPageNumberPagination,
+    page_size=s.API_PAGINATION_DEFAULT_PAGE_SIZE,
+    filter_schema=ReviewStatusFilter,
+)
+def list_package_rules(request, package_uuid: UUID):
+    """
+    List all rules for a specific package.
+    """
+    user = request.user
+    return get_package_entities_for_read(Rule, package_uuid, user).order_by("name").all()

epapi/v1/endpoints/scenarios.py

@@ -0,0 +1,129 @@
+from django.conf import settings as s
+from django.db import IntegrityError, OperationalError, DatabaseError
+from ninja import Router, Body
+from ninja.errors import HttpError
+from ninja_extra.pagination import paginate
+from uuid import UUID
+from pydantic import ValidationError
+import logging
+import json
+
+from epdb.models import Scenario
+from epdb.views import _anonymous_or_real
+from ..pagination import EnhancedPageNumberPagination
+from ..schemas import (
+    ReviewStatusFilter,
+    ScenarioOutSchema,
+    ScenarioCreateSchema,
+)
+from ..dal import get_user_entities_for_read, get_package_entities_for_read, get_package_for_write
+from envipy_additional_information import registry
+
+logger = logging.getLogger(__name__)
+
+router = Router()
+
+
+@router.get("/scenarios/", response=EnhancedPageNumberPagination.Output[ScenarioOutSchema])
+@paginate(
+    EnhancedPageNumberPagination,
+    page_size=s.API_PAGINATION_DEFAULT_PAGE_SIZE,
+    filter_schema=ReviewStatusFilter,
+)
+def list_all_scenarios(request):
+    user = request.user
+    items = get_user_entities_for_read(Scenario, user)
+    return items.order_by("name").all()
+
+
+@router.get(
+    "/package/{uuid:package_uuid}/scenario/",
+    response=EnhancedPageNumberPagination.Output[ScenarioOutSchema],
+)
+@paginate(
+    EnhancedPageNumberPagination,
+    page_size=s.API_PAGINATION_DEFAULT_PAGE_SIZE,
+    filter_schema=ReviewStatusFilter,
+)
+def list_package_scenarios(request, package_uuid: UUID):
+    user = request.user
+    items = get_package_entities_for_read(Scenario, package_uuid, user)
+    return items.order_by("name").all()
+
+
+@router.post("/package/{uuid:package_uuid}/scenario/", response=ScenarioOutSchema)
+def create_scenario(request, package_uuid: UUID, payload: ScenarioCreateSchema = Body(...)):
+    """Create a new scenario with optional additional information."""
+    user = _anonymous_or_real(request)
+
+    try:
+        current_package = get_package_for_write(user, package_uuid)
+    except ValueError as e:
+        error_msg = str(e)
+        if "does not exist" in error_msg:
+            raise HttpError(404, f"Package not found: {package_uuid}")
+        elif "Insufficient permissions" in error_msg:
+            raise HttpError(403, "You do not have permission to access this package")
+        else:
+            logger.error(f"Unexpected ValueError from get_package_by_id: {error_msg}")
+            raise HttpError(400, "Invalid package request")
+
+    # Build additional information models from payload
+    additional_information_models = []
+    validation_errors = []
+
+    for ai_item in payload.additional_information:
+        # Get model class from registry
+        model_cls = registry.get_model(ai_item.type.lower())
+        if not model_cls:
+            validation_errors.append(f"Unknown additional information type: {ai_item.type}")
+            continue
+
+        try:
+            # Validate and create model instance
+            instance = model_cls(**ai_item.data)
+            additional_information_models.append(instance)
+        except ValidationError as e:
+            # Collect validation errors to return to user
+            error_messages = [err.get("msg", "Validation error") for err in e.errors()]
+            validation_errors.append(f"{ai_item.type}: {', '.join(error_messages)}")
+        except (TypeError, AttributeError, KeyError) as e:
+            logger.warning(f"Failed to instantiate {ai_item.type} model: {str(e)}")
+            validation_errors.append(f"{ai_item.type}: Invalid data structure - {str(e)}")
+        except Exception as e:
+            logger.error(f"Unexpected error instantiating {ai_item.type}: {str(e)}")
+            validation_errors.append(f"{ai_item.type}: Failed to process - please check your data")
+
+    # If there are validation errors, return them
+    if validation_errors:
+        raise HttpError(
+            400,
+            json.dumps(
+                {
+                    "error": "Validation errors in additional information",
+                    "details": validation_errors,
+                }
+            ),
+        )
+
+    # Create scenario using the existing Scenario.create method
+    try:
+        new_scenario = Scenario.create(
+            package=current_package,
+            name=payload.name,
+            description=payload.description,
+            scenario_date=payload.scenario_date,
+            scenario_type=payload.scenario_type,
+            additional_information=additional_information_models,
+        )
+    except IntegrityError as e:
+        logger.error(f"Database integrity error creating scenario: {str(e)}")
+        raise HttpError(400, "Scenario creation failed - data constraint violation")
+    except OperationalError as e:
+        logger.error(f"Database operational error creating scenario: {str(e)}")
+        raise HttpError(503, "Database temporarily unavailable - please try again")
+    except (DatabaseError, AttributeError) as e:
+        logger.error(f"Error creating scenario: {str(e)}")
+        raise HttpError(500, "Failed to create scenario due to database error")
+
+    return new_scenario

epapi/v1/endpoints/settings.py

@@ -0,0 +1,23 @@
+from django.conf import settings as s
+from ninja import Router
+from ninja_extra.pagination import paginate
+
+from epdb.logic import SettingManager
+
+from ..pagination import EnhancedPageNumberPagination
+from ..schemas import SettingOutSchema
+
+router = Router()
+
+
+@router.get("/settings/", response=EnhancedPageNumberPagination.Output[SettingOutSchema])
+@paginate(
+    EnhancedPageNumberPagination,
+    page_size=s.API_PAGINATION_DEFAULT_PAGE_SIZE,
+)
+def list_all_settings(request):
+    """
+    List all settings the user has access to.
+    """
+    user = request.user
+    return SettingManager.get_all_settings(user)

epapi/v1/endpoints/structure.py

@@ -0,0 +1,50 @@
+from django.conf import settings as s
+from ninja import Router
+from ninja_extra.pagination import paginate
+from uuid import UUID
+
+from ..pagination import EnhancedPageNumberPagination
+from ..schemas import CompoundStructureOutSchema, StructureReviewStatusFilter
+from ..dal import (
+    get_user_structure_for_read,
+    get_package_compound_structure_for_read,
+)
+
+router = Router()
+
+
+@router.get(
+    "/structures/", response=EnhancedPageNumberPagination.Output[CompoundStructureOutSchema]
+)
+@paginate(
+    EnhancedPageNumberPagination,
+    page_size=s.API_PAGINATION_DEFAULT_PAGE_SIZE,
+    filter_schema=StructureReviewStatusFilter,
+)
+def list_all_structures(request):
+    """
+    List all structures from all packages.
+    """
+    user = request.user
+    return get_user_structure_for_read(user).order_by("name").all()
+
+
+@router.get(
+    "/package/{uuid:package_uuid}/compound/{uuid:compound_uuid}/structure/",
+    response=EnhancedPageNumberPagination.Output[CompoundStructureOutSchema],
+)
+@paginate(
+    EnhancedPageNumberPagination,
+    page_size=s.API_PAGINATION_DEFAULT_PAGE_SIZE,
+    filter_schema=StructureReviewStatusFilter,
+)
+def list_package_structures(request, package_uuid: UUID, compound_uuid: UUID):
+    """
+    List all structures for a specific package and compound.
+    """
+    user = request.user
+    return (
+        get_package_compound_structure_for_read(package_uuid, compound_uuid, user)
+        .order_by("name")
+        .all()
+    )

epapi/v1/errors.py

@@ -0,0 +1,28 @@
+from ninja.errors import HttpError
+
+
+class EPAPIError(HttpError):
+    status_code: int = 500
+
+    def __init__(self, message: str) -> None:
+        super().__init__(status_code=self.status_code, message=message)
+
+    @classmethod
+    def from_exception(cls, exc: Exception):
+        return cls(message=str(exc))
+
+
+class EPAPIUnauthorizedError(EPAPIError):
+    status_code = 401
+
+
+class EPAPIPermissionDeniedError(EPAPIError):
+    status_code = 403
+
+
+class EPAPINotFoundError(EPAPIError):
+    status_code = 404
+
+
+class EPAPIValidationError(EPAPIError):
+    status_code = 422

epapi/v1/interfaces/__init__.py

@@ -0,0 +1,3 @@
+"""
+Service interfaces: each subdirectory defines the full boundary contract between enviPy and feature-flagged apps. DTOs and projections are shared concerns to avoid direct ORM access.
+"""

epapi/v1/interfaces/iuclid/dto.py

@@ -0,0 +1,58 @@
+from dataclasses import dataclass, field
+from uuid import UUID
+
+
+@dataclass(frozen=True)
+class PathwayCompoundDTO:
+    pk: int
+    name: str
+    smiles: str | None = None
+    cas_number: str | None = None
+    ec_number: str | None = None
+
+
+@dataclass(frozen=True)
+class PathwayScenarioDTO:
+    scenario_uuid: UUID
+    name: str
+    additional_info: list = field(default_factory=list)  # EnviPyModel instances
+
+
+@dataclass(frozen=True)
+class PathwayNodeDTO:
+    node_uuid: UUID
+    compound_pk: int
+    name: str
+    depth: int
+    smiles: str | None = None
+    cas_number: str | None = None
+    ec_number: str | None = None
+    additional_info: list = field(default_factory=list)  # EnviPyModel instances
+    scenarios: list[PathwayScenarioDTO] = field(default_factory=list)
+
+
+@dataclass(frozen=True)
+class PathwayEdgeDTO:
+    edge_uuid: UUID
+    start_compound_pks: list[int] = field(default_factory=list)
+    end_compound_pks: list[int] = field(default_factory=list)
+    probability: float | None = None
+
+
+@dataclass(frozen=True)
+class PathwayModelInfoDTO:
+    model_name: str | None = None
+    model_uuid: UUID | None = None
+    software_name: str | None = None
+    software_version: str | None = None
+
+
+@dataclass(frozen=True)
+class PathwayExportDTO:
+    pathway_uuid: UUID
+    pathway_name: str
+    compounds: list[PathwayCompoundDTO] = field(default_factory=list)
+    nodes: list[PathwayNodeDTO] = field(default_factory=list)
+    edges: list[PathwayEdgeDTO] = field(default_factory=list)
+    root_compound_pks: list[int] = field(default_factory=list)
+    model_info: PathwayModelInfoDTO | None = None

epapi/v1/interfaces/iuclid/projections.py

@@ -0,0 +1,142 @@
+from uuid import UUID
+
+from epdb.logic import PackageManager
+from epdb.models import Pathway
+
+from epapi.v1.errors import EPAPINotFoundError, EPAPIPermissionDeniedError
+
+from .dto import (
+    PathwayCompoundDTO,
+    PathwayEdgeDTO,
+    PathwayExportDTO,
+    PathwayModelInfoDTO,
+    PathwayNodeDTO,
+    PathwayScenarioDTO,
+)
+
+
+def get_pathway_for_iuclid_export(user, pathway_uuid: UUID) -> PathwayExportDTO:
+    """Return pathway data projected into DTOs for the IUCLID export consumer."""
+    try:
+        pathway = (
+            Pathway.objects.select_related("package", "setting", "setting__model")
+            .prefetch_related(
+                "node_set__default_node_label__compound__external_identifiers__database",
+                "node_set__scenarios",
+                "edge_set__start_nodes__default_node_label__compound",
+                "edge_set__end_nodes__default_node_label__compound",
+            )
+            .get(uuid=pathway_uuid)
+        )
+    except Pathway.DoesNotExist:
+        raise EPAPINotFoundError(f"Pathway with UUID {pathway_uuid} not found")
+
+    if not user or user.is_anonymous or not PackageManager.readable(user, pathway.package):
+        raise EPAPIPermissionDeniedError("Insufficient permissions to access this pathway.")
+
+    nodes: list[PathwayNodeDTO] = []
+    edges: list[PathwayEdgeDTO] = []
+    compounds_by_pk: dict[int, PathwayCompoundDTO] = {}
+    root_compound_pks: list[int] = []
+
+    for node in pathway.node_set.all().order_by("depth", "pk"):
+        cs = node.default_node_label
+        if cs is None:
+            continue
+        compound = cs.compound
+
+        cas_number = None
+        ec_number = None
+        for ext_id in compound.external_identifiers.all():
+            db_name = ext_id.database.name if ext_id.database else None
+            if db_name == "CAS" and cas_number is None:
+                cas_number = ext_id.identifier_value
+            elif db_name == "EC" and ec_number is None:
+                ec_number = ext_id.identifier_value
+
+        ai_for_node = []
+        scenario_entries: list[PathwayScenarioDTO] = []
+        for scenario in sorted(node.scenarios.all(), key=lambda item: item.pk):
+            ai_for_scenario = list(scenario.get_additional_information(direct_only=True))
+            ai_for_node.extend(ai_for_scenario)
+            scenario_entries.append(
+                PathwayScenarioDTO(
+                    scenario_uuid=scenario.uuid,
+                    name=scenario.name,
+                    additional_info=ai_for_scenario,
+                )
+            )
+
+        nodes.append(
+            PathwayNodeDTO(
+                node_uuid=node.uuid,
+                compound_pk=compound.pk,
+                name=compound.name,
+                depth=node.depth,
+                smiles=cs.smiles,
+                cas_number=cas_number,
+                ec_number=ec_number,
+                additional_info=ai_for_node,
+                scenarios=scenario_entries,
+            )
+        )
+
+        if node.depth == 0 and compound.pk not in root_compound_pks:
+            root_compound_pks.append(compound.pk)
+
+        if compound.pk not in compounds_by_pk:
+            compounds_by_pk[compound.pk] = PathwayCompoundDTO(
+                pk=compound.pk,
+                name=compound.name,
+                smiles=cs.smiles,
+                cas_number=cas_number,
+                ec_number=ec_number,
+            )
+
+    for edge in pathway.edge_set.all():
+        start_compounds = {
+            n.default_node_label.compound.pk
+            for n in edge.start_nodes.all()
+            if n.default_node_label is not None
+        }
+        end_compounds = {
+            n.default_node_label.compound.pk
+            for n in edge.end_nodes.all()
+            if n.default_node_label is not None
+        }
+
+        probability = None
+        if edge.kv and edge.kv.get("probability") is not None:
+            try:
+                probability = float(edge.kv.get("probability"))
+            except (TypeError, ValueError):
+                probability = None
+
+        edges.append(
+            PathwayEdgeDTO(
+                edge_uuid=edge.uuid,
+                start_compound_pks=sorted(start_compounds),
+                end_compound_pks=sorted(end_compounds),
+                probability=probability,
+            )
+        )
+
+    model_info = None
+    if pathway.setting and pathway.setting.model:
+        model = pathway.setting.model
+        model_info = PathwayModelInfoDTO(
+            model_name=model.get_name(),
+            model_uuid=model.uuid,
+            software_name="enviPath",
+            software_version=None,
+        )
+
+    return PathwayExportDTO(
+        pathway_uuid=pathway.uuid,
+        pathway_name=pathway.get_name(),
+        compounds=list(compounds_by_pk.values()),
+        nodes=nodes,
+        edges=edges,
+        root_compound_pks=root_compound_pks,
+        model_info=model_info,
+    )

epapi/v1/pagination.py

@@ -0,0 +1,60 @@
+import math
+from typing import Any, Generic, List, TypeVar
+
+from django.db.models import QuerySet
+from ninja import Schema
+from ninja.pagination import PageNumberPagination
+
+T = TypeVar("T")
+
+
+class EnhancedPageNumberPagination(PageNumberPagination):
+    class Output(Schema, Generic[T]):
+        items: List[T]
+        page: int
+        page_size: int
+        total_items: int
+        total_pages: int
+
+    def paginate_queryset(
+        self,
+        queryset: QuerySet,
+        pagination: PageNumberPagination.Input,
+        **params: Any,
+    ) -> Any:
+        page_size = self._get_page_size(pagination.page_size)
+        offset = (pagination.page - 1) * page_size
+        total_items = self._items_count(queryset)
+        total_pages = math.ceil(total_items / page_size) if page_size > 0 else 0
+
+        return {
+            "items": queryset[offset : offset + page_size],
+            "page": pagination.page,
+            "page_size": page_size,
+            "total_items": total_items,
+            "total_pages": total_pages,
+        }
+
+    async def apaginate_queryset(
+        self,
+        queryset: QuerySet,
+        pagination: PageNumberPagination.Input,
+        **params: Any,
+    ) -> Any:
+        page_size = self._get_page_size(pagination.page_size)
+        offset = (pagination.page - 1) * page_size
+        total_items = await self._aitems_count(queryset)
+        total_pages = math.ceil(total_items / page_size) if page_size > 0 else 0
+
+        if isinstance(queryset, QuerySet):
+            items = [obj async for obj in queryset[offset : offset + page_size]]
+        else:
+            items = queryset[offset : offset + page_size]
+
+        return {
+            "items": items,
+            "page": pagination.page,
+            "page_size": page_size,
+            "total_items": total_items,
+            "total_pages": total_pages,
+        }

epapi/v1/router.py

@@ -0,0 +1,44 @@
+from ninja import Router
+from ninja.security import SessionAuth
+
+from envipath import settings as s
+from .auth import BearerTokenAuth
+from .endpoints import (
+    packages,
+    scenarios,
+    compounds,
+    rules,
+    reactions,
+    pathways,
+    models,
+    structure,
+    additional_information,
+    settings,
+    groups,
+)
+
+# Main router with authentication
+router = Router(
+    auth=[
+        SessionAuth(),
+        BearerTokenAuth(),
+    ]
+)
+
+# Include all endpoint routers
+router.add_router("", packages.router)
+router.add_router("", scenarios.router)
+router.add_router("", compounds.router)
+router.add_router("", rules.router)
+router.add_router("", reactions.router)
+router.add_router("", pathways.router)
+router.add_router("", models.router)
+router.add_router("", structure.router)
+router.add_router("", additional_information.router)
+router.add_router("", settings.router)
+router.add_router("", groups.router)
+
+if s.IUCLID_EXPORT_ENABLED:
+    from epiuclid.api import router as iuclid_router
+
+    router.add_router("", iuclid_router)

epapi/v1/schemas.py

@@ -0,0 +1,135 @@
+from ninja import FilterSchema, FilterLookup, Schema
+from typing import Annotated, Optional, List, Dict, Any
+from uuid import UUID
+
+
+# Filter schema for query parameters
+class ReviewStatusFilter(FilterSchema):
+    """Filter schema for review_status query parameter."""
+
+    review_status: Annotated[Optional[bool], FilterLookup("package__reviewed")] = None
+
+
+class SelfReviewStatusFilter(FilterSchema):
+    """Filter schema for review_status query parameter on self-reviewed entities."""
+
+    review_status: Annotated[Optional[bool], FilterLookup("reviewed")] = None
+
+
+class StructureReviewStatusFilter(FilterSchema):
+    """Filter schema for review_status on structures (via compound->package)."""
+
+    review_status: Annotated[Optional[bool], FilterLookup("compound__package__reviewed")] = None
+
+
+# Base schema for all package-scoped entities
+class PackageEntityOutSchema(Schema):
+    """Base schema for entities belonging to a package."""
+
+    uuid: UUID
+    url: str = ""
+    name: str
+    description: str
+    review_status: str = ""
+    package: str = ""
+
+    @staticmethod
+    def resolve_url(obj):
+        return obj.url
+
+    @staticmethod
+    def resolve_package(obj):
+        return obj.package.url
+
+    @staticmethod
+    def resolve_review_status(obj):
+        return "reviewed" if obj.package.reviewed else "unreviewed"
+
+
+# All package-scoped entities inherit from base
+class ScenarioOutSchema(PackageEntityOutSchema):
+    pass
+
+
+class AdditionalInformationItemSchema(Schema):
+    """Schema for additional information item in scenario creation."""
+
+    type: str
+    data: Dict[str, Any]
+
+
+class ScenarioCreateSchema(Schema):
+    """Schema for creating a new scenario."""
+
+    name: str
+    description: str = ""
+    scenario_date: str = "No date"
+    scenario_type: str = "Not specified"
+    additional_information: List[AdditionalInformationItemSchema] = []
+
+
+class CompoundOutSchema(PackageEntityOutSchema):
+    pass
+
+
+class RuleOutSchema(PackageEntityOutSchema):
+    pass
+
+
+class ReactionOutSchema(PackageEntityOutSchema):
+    pass
+
+
+class PathwayOutSchema(PackageEntityOutSchema):
+    pass
+
+
+class ModelOutSchema(PackageEntityOutSchema):
+    pass
+
+
+class CompoundStructureOutSchema(PackageEntityOutSchema):
+    compound: str = ""
+
+    @staticmethod
+    def resolve_compound(obj):
+        return obj.compound.url
+
+    @staticmethod
+    def resolve_package(obj):
+        return obj.compound.package.url
+
+    @staticmethod
+    def resolve_review_status(obj):
+        return "reviewed" if obj.compound.package.reviewed else "unreviewed"
+
+
+# Package is special (no package FK)
+class PackageOutSchema(Schema):
+    uuid: UUID
+    url: str = ""
+    name: str
+    description: str
+    review_status: str = ""
+
+    @staticmethod
+    def resolve_url(obj):
+        return obj.url
+
+    @staticmethod
+    def resolve_review_status(obj):
+        return "reviewed" if obj.reviewed else "unreviewed"
+
+
+class SettingOutSchema(Schema):
+    uuid: UUID
+    url: str = ""
+    name: str
+    description: str
+
+
+class GroupOutSchema(Schema):
+    uuid: UUID
+    url: str = ""
+    name: str
+    description: str

epauth/urls.py

@@ -3,6 +3,6 @@ from django.urls import path
 from . import views
 
 urlpatterns = [
-    path("microsoft/login/", views.microsoft_login, name="microsoft_login"),
+    path("entra/login/", views.entra_login, name="entra_login"),
-    path("microsoft/callback/", views.microsoft_callback, name="microsoft_callback"),
+    path("auth/redirect/", views.entra_callback, name="entra_callback"),
 ]

epauth/views.py

@@ -1,34 +1,49 @@
 import msal
 from django.conf import settings as s
+from django.contrib.auth import get_user_model
 from django.contrib.auth import login
 from django.shortcuts import redirect
-from django.contrib.auth import get_user_model
 
 from epdb.logic import UserManager
 
 
-def microsoft_login(request):
+def get_msal_app_with_cache(request):
+    """
+    Create MSAL app with session-based token cache.
+    """
+    cache = msal.SerializableTokenCache()
+
+    # Load cache from session if it exists
+    if request.session.get("msal_token_cache"):
+        cache.deserialize(request.session["msal_token_cache"])
+
     msal_app = msal.ConfidentialClientApplication(
         client_id=s.MS_ENTRA_CLIENT_ID,
         client_credential=s.MS_ENTRA_CLIENT_SECRET,
-        authority=s.MS_ENTRA_AUTHORITY
+        authority=s.MS_ENTRA_AUTHORITY,
+        token_cache=cache,
+    )
+
+    return msal_app, cache
+
+
+def entra_login(request):
+    msal_app = msal.ConfidentialClientApplication(
+        client_id=s.MS_ENTRA_CLIENT_ID,
+        client_credential=s.MS_ENTRA_CLIENT_SECRET,
+        authority=s.MS_ENTRA_AUTHORITY,
     )
 
     flow = msal_app.initiate_auth_code_flow(
-        scopes=s.MS_ENTRA_SCOPES,
+        scopes=s.MS_ENTRA_SCOPES, redirect_uri=s.MS_ENTRA_REDIRECT_URI
-        redirect_uri=s.MS_ENTRA_REDIRECT_URI
     )
 
     request.session["msal_auth_flow"] = flow
     return redirect(flow["auth_uri"])
 
 
-def microsoft_callback(request):
+def entra_callback(request):
-    msal_app = msal.ConfidentialClientApplication(
+    msal_app, cache = get_msal_app_with_cache(request)
-        client_id=s.MS_ENTRA_CLIENT_ID,
-        client_credential=s.MS_ENTRA_CLIENT_SECRET,
-        authority=s.MS_ENTRA_AUTHORITY
-    )
 
     flow = request.session.pop("msal_auth_flow", None)
     if not flow:
@@ -37,30 +52,79 @@ def microsoft_callback(request):
     # Acquire token using the flow and callback request
     result = msal_app.acquire_token_by_auth_code_flow(flow, request.GET)
 
-    if "access_token" in result:
+    # Save the token cache to session
-        # Optional: Fetch user info from Microsoft Graph
+    if cache.has_state_changed:
-        import requests
+        request.session["msal_token_cache"] = cache.serialize()
-        resp = requests.get(
-            "https://graph.microsoft.com/v1.0/me",
-            headers={"Authorization": f"Bearer {result['access_token']}"}
-        )
-        user_info = resp.json()
 
-        user_name = user_info["displayName"]
+    claims = result["id_token_claims"]
-        user_email = user_info["mail"]
-        user_oid = user_info["id"]
 
-        # Get implementing class
+    user_name = claims.get("name")
-        User = get_user_model()
+    user_email = claims.get("emailaddress", claims.get("email"))
+    user_oid = claims.get("oid")
 
-        if User.objects.filter(uuid=user_oid).exists():
+    if not all([user_name, user_email, user_oid]):
-            login(request, User.objects.get(uuid=user_oid))
+        raise ValueError("Missing required claims in ID token")
-        else:
-            u = UserManager.create_user(user_name, user_email, None, uuid=user_oid, is_active=True)
-            login(request, u)
 
-        # TODO Group Sync
+    # Get implementing class
+    User = get_user_model()
 
-        return redirect("/")
+    if User.objects.filter(uuid=user_oid).exists():
+        u = User.objects.get(uuid=user_oid)
 
-    return redirect("/")  # Handle errors
+        if u.username != user_name:
+            u.username = user_name
+            u.save()
+
+    else:
+        u = UserManager.create_user(user_name, user_email, None, uuid=user_oid, is_active=True)
+
+    login(request, u)
+
+    return redirect(s.SERVER_URL)  # Handle errors
+
+
+def get_access_token_from_request(request, scopes=None):
+    """
+    Get an access token from the request using MSAL token cache.
+    """
+    if scopes is None:
+        scopes = s.MS_ENTRA_SCOPES
+
+    # Get user from request (must be authenticated)
+    if not request.user.is_authenticated:
+        return None
+
+    # Create MSAL app with persistent cache
+    msal_app, cache = get_msal_app_with_cache(request)
+
+    # Try to get accounts from cache
+    accounts = msal_app.get_accounts()
+
+    if not accounts:
+        return None
+
+    # Find the account that matches the current user
+    user_account = None
+    for account in accounts:
+        if account.get("local_account_id") == str(request.user.uuid):
+            user_account = account
+            break
+
+    # If no matching account found, use the first available account
+    if not user_account and accounts:
+        user_account = accounts[0]
+
+    if not user_account:
+        return None
+
+    # Try to acquire token silently from cache
+    result = msal_app.acquire_token_silent(scopes=scopes, account=user_account)
+
+    # Save cache changes back to session
+    if cache.has_state_changed:
+        request.session["msal_token_cache"] = cache.serialize()
+
+    if result and "access_token" in result:
+        return result
+
+    return None

epdb/admin.py

@@ -1,32 +1,161 @@
+import logging
+
+from django.conf import settings as s
 from django.contrib import admin
+from django.contrib import messages
 
 from .models import (
-    User,
+    AdditionalInformation,
-    UserPackagePermission,
+    ClassifierPluginModel,
-    Group,
-    GroupPackagePermission,
-    Package,
-    MLRelativeReasoning,
-    EnviFormer,
     Compound,
     CompoundStructure,
-    SimpleAmbitRule,
-    ParallelRule,
-    Reaction,
-    Pathway,
-    Node,
     Edge,
-    Scenario,
+    EnviFormer,
-    Setting,
     ExternalDatabase,
     ExternalIdentifier,
+    Group,
+    GroupPackagePermission,
     JobLog,
     License,
+    MLRelativeReasoning,
+    Node,
+    ParallelRule,
+    Pathway,
+    PropertyPluginModel,
+    Reaction,
+    Scenario,
+    Setting,
+    SimpleAmbitRule,
+    User,
+    UserPackagePermission,
 )
 
+Package = s.GET_PACKAGE_MODEL()
+
+logger = logging.getLogger(__name__)
+
+
+class AdditionalInformationAdmin(admin.ModelAdmin):
+    pass
+
 
 class UserAdmin(admin.ModelAdmin):
-    list_display = ["username", "email", "is_active"]
+    list_display = [
+        "username",
+        "email",
+        "is_active",
+        "is_staff",
+        "is_superuser",
+        "last_login",
+        "date_joined",
+    ]
+
+    actions = ["send_welcome_mail", "send_affiliation_mail"]
+
+    @admin.action(description="Send welcome mail")
+    def send_welcome_mail(self, request, queryset):
+        from django.core.mail import EmailMultiAlternatives
+
+        tpl = """Hello {username},
+
+Your account has been successfully activated.
+
+To log in, please visit
+https://envipath.org/password_reset/
+and request a new password.
+
+If you have any questions or feedback, feel free to visit our community forum at
+https://community.envipath.org/.
+You do not need to register again for the forum - you can log in using your enviPath account by clicking "Log In" and then "Log in with enviPath."
+
+Best regards,
+
+The enviPath Team"""
+
+        users = []
+
+        for user in queryset:
+            if user.is_active:
+                logger.info(f"{user.username} already active - not sending mail again")
+                continue
+            try:
+                msg = EmailMultiAlternatives(
+                    "Your enviPath Account Is Now Active",
+                    tpl.format(username=user.username),
+                    "admin@envipath.org",
+                    [user.email],
+                    bcc=["admin@envipath.org"],
+                )
+
+                msg.send(fail_silently=False)
+
+                user.is_active = True
+                user.password = "ASDF"
+                user.save()
+
+                users.append(user)
+                logger.info(f"{user.username} -> {user.email} mail sent")
+            except Exception as e:
+                logger.info(f"Error sending mail to {user.username}: {e}")
+
+        self.message_user(
+            request, f"Sent welcome mail to {[u.email for u in users]}", messages.SUCCESS
+        )
+
+    @admin.action(description="Send affiliation mail")
+    def send_affiliation_mail(self, request, queryset):
+        from django.core.mail import EmailMultiAlternatives
+
+        tpl = """Dear {username},
+
+Thank you for your interest in enviPath!
+
+Please note that the public enviPath system is intended for non-commercial use only.
+We see that you registered using the email address {email}.
+If possible, we kindly ask you to register using an official email address that reflects your affiliation (e.g., a university, NGO, or research organization).
+
+If you would like us to update your account, simply reply to this email and let us know which address we should use.
+We will then change it in our system, and you will receive a password reset email at the new address.
+
+If you are registering with a company email address and are interested in commercial use, you are very welcome to book a meeting with us so we can discuss how we can best support you.
+To book a meeting, please visit https://envipath.com/book
+
+If changing to an affiliation email address is not possible, please contact us at registration@envipath.org
+
+Best regards,
+
+enviPath team"""
+
+        users = []
+
+        for user in queryset:
+            if user.is_active or user.contacted:
+                logger.info(
+                    f"{user.username} already active or already contacted - not sending mail again"
+                )
+                continue
+            try:
+                msg = EmailMultiAlternatives(
+                    "Regarding your enviPath registration",
+                    tpl.format(username=user.username, email=user.email),
+                    "admin@envipath.org",
+                    [user.email],
+                    bcc=["admin@envipath.org"],
+                )
+
+                msg.send(fail_silently=False)
+
+                user.contacted = True
+                user.save()
+
+                users.append(user)
+                logger.info(f"{user.username} -> {user.email} affiliation mail sent")
+            except Exception as e:
+                logger.info(f"Error sending mail to {user.username}: {e}")
+
+        self.message_user(
+            request, f"Sent affiliation mail to {[u.email for u in users]}", messages.SUCCESS
+        )
 
 
 class UserPackagePermissionAdmin(admin.ModelAdmin):
@@ -46,7 +175,7 @@ class JobLogAdmin(admin.ModelAdmin):
 
 
 class EPAdmin(admin.ModelAdmin):
-    search_fields = ["name", "description"]
+    search_fields = ["name", "description", "url", "uuid"]
     list_display = ["name", "url", "created"]
     ordering = ["-created"]
 
@@ -63,6 +192,14 @@ class EnviFormerAdmin(EPAdmin):
     pass
 
 
+class PropertyPluginModelAdmin(admin.ModelAdmin):
+    pass
+
+
+class ClassifierPluginModelAdmin(admin.ModelAdmin):
+    pass
+
+
 class LicenseAdmin(admin.ModelAdmin):
     list_display = ["cc_string", "link", "image_link"]
 
@@ -115,6 +252,7 @@ class ExternalIdentifierAdmin(admin.ModelAdmin):
     pass
 
 
+admin.site.register(AdditionalInformation, AdditionalInformationAdmin)
 admin.site.register(User, UserAdmin)
 admin.site.register(UserPackagePermission, UserPackagePermissionAdmin)
 admin.site.register(Group, GroupAdmin)
@@ -123,7 +261,9 @@ admin.site.register(JobLog, JobLogAdmin)
 admin.site.register(Package, PackageAdmin)
 admin.site.register(MLRelativeReasoning, MLRelativeReasoningAdmin)
 admin.site.register(EnviFormer, EnviFormerAdmin)
+admin.site.register(PropertyPluginModel, PropertyPluginModelAdmin)
 admin.site.register(License, LicenseAdmin)
+admin.site.register(ClassifierPluginModel, ClassifierPluginModelAdmin)
 admin.site.register(Compound, CompoundAdmin)
 admin.site.register(CompoundStructure, CompoundStructureAdmin)
 admin.site.register(SimpleAmbitRule, SimpleAmbitRuleAdmin)

epdb/api.py

@@ -2,20 +2,12 @@ from typing import List
 
 from django.contrib.auth import get_user_model
 from ninja import Router, Schema, Field
-from ninja.errors import HttpError
 from ninja.pagination import paginate
-from ninja.security import HttpBearer
+
+from epapi.v1.auth import BearerTokenAuth
 
 from .logic import PackageManager
-from .models import User, Compound, APIToken
+from .models import User, Compound
-
-
-class BearerTokenAuth(HttpBearer):
-    def authenticate(self, request, token):
-        for token_obj in APIToken.objects.select_related("user").all():
-            if token_obj.check_token(token) and token_obj.is_valid():
-                return token_obj.user
-        raise HttpError(401, "Invalid or expired token")
 
 
 def _anonymous_or_real(request):

epdb/apps.py

@@ -1,4 +1,9 @@
+import logging
+
 from django.apps import AppConfig
+from django.conf import settings
+
+logger = logging.getLogger(__name__)
 
 
 class EPDBConfig(AppConfig):
@@ -7,3 +12,17 @@ class EPDBConfig(AppConfig):
 
     def ready(self):
         import epdb.signals  # noqa: F401
+
+        model_name = getattr(settings, "EPDB_PACKAGE_MODEL", "epdb.Package")
+        logger.info(f"Using Package model: {model_name}")
+
+        from .autodiscovery import autodiscover
+
+        autodiscover()
+
+        if settings.PLUGINS_ENABLED:
+            from bridge.contracts import Property, Classifier
+            from utilities.plugin import discover_plugins
+
+            settings.PROPERTY_PLUGINS.update(**discover_plugins(_cls=Property))
+            settings.CLASSIFIER_PLUGINS.update(**discover_plugins(_cls=Classifier))

epdb/autodiscovery.py

@@ -0,0 +1,5 @@
+from django.utils.module_loading import autodiscover_modules
+
+
+def autodiscover():
+    autodiscover_modules("epdb_hooks")

epdb/context_processors.py

@@ -5,7 +5,7 @@ Context processors automatically make variables available to all templates.
 """
 
 from .logic import PackageManager
-from .models import Package
+from django.conf import settings as s
 
 
 def package_context(request):
@@ -20,7 +20,7 @@ def package_context(request):
 
     reviewed_package_qs = PackageManager.get_reviewed_packages()
 
-    unreviewed_package_qs = Package.objects.none()
+    unreviewed_package_qs = s.GET_PACKAGE_MODEL().objects.none()
 
     # Only get user-specific packages if user is authenticated
     if current_user.is_authenticated:

epdb/logic.py

@@ -1,39 +1,42 @@
-import re
 import logging
-import json
+import re
-from typing import Union, List, Optional, Set, Dict, Any
+from typing import Any, Dict, List, Optional, Set, Union, Tuple
 from uuid import UUID
 
 import nh3
+from django.conf import settings as s
 from django.contrib.auth import get_user_model
 from django.db import transaction
-from django.conf import settings as s
 from pydantic import ValidationError
 
 from epdb.models import (
-    User,
+    AdditionalInformation,
-    Package,
-    UserPackagePermission,
-    GroupPackagePermission,
-    Permission,
-    Group,
-    Setting,
-    EPModel,
-    UserSettingPermission,
-    Rule,
-    Pathway,
-    Node,
-    Edge,
     Compound,
-    Reaction,
     CompoundStructure,
+    Edge,
     EnzymeLink,
+    EPModel,
+    ExpansionSchemeChoice,
+    Group,
+    GroupPackagePermission,
+    Node,
+    Pathway,
+    Permission,
+    PropertyPluginModel,
+    Reaction,
+    Rule,
+    Setting,
+    User,
+    UserPackagePermission,
+    UserSettingPermission,
 )
 from utilities.chem import FormatConverter
-from utilities.misc import PackageImporter, PackageExporter
+from utilities.misc import PackageExporter, PackageImporter
 
 logger = logging.getLogger(__name__)
 
+Package = s.GET_PACKAGE_MODEL()
+
 
 class EPDBURLParser:
     UUID_PATTERN = r"[a-f0-9]{8}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{12}"
@@ -192,8 +195,6 @@ class UserManager(object):
         if clean_username != username or clean_email != email:
             # This will be caught by the try in view.py/register
             raise ValueError("Invalid username or password")
-        # avoid circular import :S
-        from .tasks import send_registration_mail
 
         extra_fields = {"is_active": not s.ADMIN_APPROVAL_REQUIRED}
 
@@ -212,10 +213,6 @@ class UserManager(object):
         u.default_package = p
         u.save()
 
-        if not u.is_active:
-            # send email for verification
-            send_registration_mail.delay(u.pk)
-
         if set_setting:
             u.default_setting = Setting.objects.get(global_default=True)
             u.save()
@@ -267,8 +264,12 @@ class GroupManager(object):
         return bool(re.findall(GroupManager.group_pattern, url))
 
     @staticmethod
-    def create_group(current_user, name, description):
+    def create_group(current_user, name, description, *args, **kwargs):
         g = Group()
+
+        if "uuid" in kwargs:
+            g.uuid = kwargs["uuid"]
+
         # Clean for potential XSS
         g.name = nh3.clean(name, tags=s.ALLOWED_HTML_TAGS).strip()
         g.description = nh3.clean(description, tags=s.ALLOWED_HTML_TAGS).strip()
@@ -344,52 +345,17 @@ class PackageManager(object):
 
     @staticmethod
     def readable(user, package):
-        if (
+        return (
-            UserPackagePermission.objects.filter(package=package, user=user).exists()
+            PackageManager.has_package_permission(user, package, "read") | package.reviewed is True
-            or GroupPackagePermission.objects.filter(
+        )
-                package=package, group__in=GroupManager.get_groups(user)
-            )
-            or package.reviewed is True
-            or user.is_superuser
-        ):
-            return True
-
-        return False
 
     @staticmethod
     def writable(user, package):
-        if (
+        return PackageManager.has_package_permission(user, package, "write")
-            UserPackagePermission.objects.filter(
-                package=package, user=user, permission=Permission.WRITE[0]
-            ).exists()
-            or GroupPackagePermission.objects.filter(
-                package=package,
-                group__in=GroupManager.get_groups(user),
-                permission=Permission.WRITE[0],
-            ).exists()
-            or UserPackagePermission.objects.filter(
-                package=package, user=user, permission=Permission.ALL[0]
-            ).exists()
-            or user.is_superuser
-        ):
-            return True
-        return False
 
     @staticmethod
     def administrable(user, package):
-        if (
+        return PackageManager.has_package_permission(user, package, "all")
-            UserPackagePermission.objects.filter(
-                package=package, user=user, permission=Permission.ALL[0]
-            ).exists()
-            or GroupPackagePermission.objects.filter(
-                package=package,
-                group__in=GroupManager.get_groups(user),
-                permission=Permission.ALL[0],
-            ).exists()
-            or user.is_superuser
-        ):
-            return True
-        return False
 
     @staticmethod
     def has_package_permission(user: "User", package: Union[str, UUID, "Package"], permission: str):
@@ -442,6 +408,7 @@ class PackageManager(object):
             if PackageManager.readable(user, p):
                 return p
             else:
+                # FIXME: use custom exception to be translatable to 403 in API
                 raise ValueError(
                     "Insufficient permissions to access Package with ID {}".format(package_id)
                 )
@@ -472,7 +439,9 @@ class PackageManager(object):
             # remove package if user is owner and package is reviewed e.g. admin
             qs = qs.filter(reviewed=False)
 
-        return qs.distinct()
+        qs = qs.distinct()
+
+        return qs
 
     @staticmethod
     def get_all_writeable_packages(user):
@@ -516,7 +485,9 @@ class PackageManager(object):
 
         qs = qs.filter(reviewed=False)
 
-        return qs.distinct()
+        qs = qs.distinct()
+
+        return qs
 
     @staticmethod
     def get_packages():
@@ -578,30 +549,39 @@ class PackageManager(object):
         else:
             _ = perm_cls.objects.update_or_create(defaults={"permission": new_perm}, **data)
 
+    @staticmethod
+    def grant_read(caller: User, package: Package, grantee: Union[User, Group]):
+        PackageManager.update_permissions(caller, package, grantee, Permission.READ[0])
+
+    @staticmethod
+    def grant_write(caller: User, package: Package, grantee: Union[User, Group]):
+        PackageManager.update_permissions(caller, package, grantee, Permission.WRITE[0])
+
     @staticmethod
     @transaction.atomic
     def import_legacy_package(
         data: dict, owner: User, keep_ids=False, add_import_timestamp=True, trust_reviewed=False
     ):
-        from uuid import UUID, uuid4
-        from datetime import datetime
         from collections import defaultdict
+        from datetime import datetime
+        from uuid import UUID, uuid4
+
+        from envipy_additional_information import AdditionalInformationConverter
+
         from .models import (
-            Package,
             Compound,
             CompoundStructure,
-            SimpleRule,
+            Edge,
-            SimpleAmbitRule,
+            Node,
             ParallelRule,
+            Pathway,
+            Reaction,
+            Scenario,
             SequentialRule,
             SequentialRuleOrdering,
-            Reaction,
+            SimpleAmbitRule,
-            Pathway,
+            SimpleRule,
-            Node,
-            Edge,
-            Scenario,
         )
-        from envipy_additional_information import AdditionalInformationConverter
 
         pack = Package()
         pack.uuid = UUID(data["id"].split("/")[-1]) if keep_ids else uuid4()
@@ -627,15 +607,30 @@ class PackageManager(object):
 
         # Stores old_id to new_id
         mapping = {}
-        # Stores new_scen_id to old_parent_scen_id
-        parent_mapping = {}
         # Mapping old scen_id to old_obj_id
         scen_mapping = defaultdict(list)
         # Enzymelink Mapping rule_id to enzymelink objects
         enzyme_mapping = defaultdict(list)
 
+        # old_parent_id to child
+        postponed_scens = defaultdict(list)
+
         # Store Scenarios
         for scenario in data["scenarios"]:
+            skip_scen = False
+            # Check if parent exists and park this Scenario to convert it later into an
+            # AdditionalInformation object
+            for ex in scenario.get("additionalInformationCollection", {}).get(
+                "additionalInformation", []
+            ):
+                if ex["name"] == "referringscenario":
+                    postponed_scens[ex["data"]].append(scenario)
+                    skip_scen = True
+                    break
+
+            if skip_scen:
+                continue
+
             scen = Scenario()
             scen.package = pack
             scen.uuid = UUID(scenario["id"].split("/")[-1]) if keep_ids else uuid4()
@@ -648,19 +643,12 @@ class PackageManager(object):
 
             mapping[scenario["id"]] = scen.uuid
 
-            new_add_inf = defaultdict(list)
-            # TODO Store AI...
             for ex in scenario.get("additionalInformationCollection", {}).get(
                 "additionalInformation", []
             ):
                 name = ex["name"]
                 addinf_data = ex["data"]
 
-                # park the parent scen id for now and link it later
-                if name == "referringscenario":
-                    parent_mapping[scen.uuid] = addinf_data
-                    continue
-
                 # Broken eP Data
                 if name == "initialmasssediment" and addinf_data == "missing data":
                     continue
@@ -668,17 +656,11 @@ class PackageManager(object):
                     continue
 
                 try:
-                    res = AdditionalInformationConverter.convert(name, addinf_data)
+                    ai = AdditionalInformationConverter.convert(name, addinf_data)
-                    res_cls_name = res.__class__.__name__
+                    AdditionalInformation.create(pack, ai, scenario=scen)
-                    ai_data = json.loads(res.model_dump_json())
+                except (ValidationError, ValueError):
-                    ai_data["uuid"] = f"{uuid4()}"
-                    new_add_inf[res_cls_name].append(ai_data)
-                except ValidationError:
                     logger.error(f"Failed to convert {name} with {addinf_data}")
 
-            scen.additional_information = new_add_inf
-            scen.save()
-
         print("Scenarios imported...")
 
         # Store compounds and its structures
@@ -707,6 +689,10 @@ class PackageManager(object):
                 struc.description = structure["description"]
                 struc.aliases = structure.get("aliases", [])
                 struc.smiles = structure["smiles"]
+
+                if structure.get("molfile"):
+                    struc.molfile = structure["molfile"]
+
                 struc.save()
 
                 for scen in structure["scenarios"]:
@@ -918,14 +904,46 @@ class PackageManager(object):
 
         print("Pathways imported...")
 
-        # Linking Phase
+        for parent, children in postponed_scens.items():
-        for child, parent in parent_mapping.items():
+            for child in children:
-            child_obj = Scenario.objects.get(uuid=child)
+                for ex in child.get("additionalInformationCollection", {}).get(
-            parent_obj = Scenario.objects.get(uuid=mapping[parent])
+                    "additionalInformation", []
-            child_obj.parent = parent_obj
+                ):
-            child_obj.save()
+                    child_id = child["id"]
+                    name = ex["name"]
+                    addinf_data = ex["data"]
+
+                    if name == "referringscenario":
+                        continue
+                    # Broken eP Data
+                    if name == "initialmasssediment" and addinf_data == "missing data":
+                        continue
+                    if name == "columnheight" and addinf_data == "(2)-(2.5);(6)-(8)":
+                        continue
+
+                    ai = AdditionalInformationConverter.convert(name, addinf_data)
+
+                    if child_id not in scen_mapping:
+                        logger.info(
+                            f"{child_id} not found in scen_mapping. Seems like its not attached to any object"
+                        )
+                        print(
+                            f"{child_id} not found in scen_mapping. Seems like its not attached to any object"
+                        )
+
+                    scen = Scenario.objects.get(uuid=mapping[parent])
+                    mapping[child_id] = scen.uuid
+                    for obj in scen_mapping[child_id]:
+                        _ = AdditionalInformation.create(pack, ai, scen, content_object=obj)
 
         for scen_id, objects in scen_mapping.items():
+            new_id = mapping.get(scen_id)
+
+            if new_id is None:
+                logger.warning(f"Could not find mapping for {scen_id}")
+                print(f"Could not find mapping for {scen_id}")
+                continue
+
             scen = Scenario.objects.get(uuid=mapping[scen_id])
             for o in objects:
                 o.scenarios.add(scen)
@@ -958,6 +976,7 @@ class PackageManager(object):
                     matches = re.findall(r">(R[0-9]+)<", evidence["evidence"])
                     if not matches or len(matches) != 1:
                         logger.warning(f"Could not find reaction id in {evidence['evidence']}")
+                        print(f"Could not find reaction id in {evidence['evidence']}")
                         continue
 
                     e.add_kegg_reaction_id(matches[0])
@@ -976,55 +995,10 @@ class PackageManager(object):
 
         print("Fixing Node depths...")
         total_pws = Pathway.objects.filter(package=pack).count()
+
         for p, pw in enumerate(Pathway.objects.filter(package=pack)):
-            print(pw.url)
+            pw.update_depths()
-            in_count = defaultdict(lambda: 0)
+            print(f"{p + 1}/{total_pws} fixed.", end="\r")
-            out_count = defaultdict(lambda: 0)
-
-            for e in pw.edges:
-                # TODO check if this will remain
-                for react in e.start_nodes.all():
-                    out_count[str(react.uuid)] += 1
-
-                for prod in e.end_nodes.all():
-                    in_count[str(prod.uuid)] += 1
-
-            root_nodes = []
-            for n in pw.nodes:
-                num_parents = in_count[str(n.uuid)]
-                if num_parents == 0:
-                    # must be a root node or unconnected node
-                    if n.depth != 0:
-                        n.depth = 0
-                        n.save()
-
-                    # Only root node may have children
-                    if out_count[str(n.uuid)] > 0:
-                        root_nodes.append(n)
-
-            levels = [root_nodes]
-            seen = set()
-            # Do a bfs to determine depths starting with level 0 a.k.a. root nodes
-            for i, level_nodes in enumerate(levels):
-                new_level = []
-                for n in level_nodes:
-                    for e in n.out_edges.all():
-                        for prod in e.end_nodes.all():
-                            if str(prod.uuid) not in seen:
-                                old_depth = prod.depth
-                                if old_depth != i + 1:
-                                    print(f"updating depth from {old_depth} to {i + 1}")
-                                    prod.depth = i + 1
-                                    prod.save()
-
-                                new_level.append(prod)
-
-                    seen.add(str(n.uuid))
-
-                if new_level:
-                    levels.append(new_level)
-
-            print(f"{p + 1}/{total_pws} fixed.")
 
         return pack
 
@@ -1103,18 +1077,23 @@ class SettingManager(object):
         description: str = None,
         max_nodes: int = None,
         max_depth: int = None,
-        rule_packages: List[Package] = None,
+        rule_packages: List[Package] | None = None,
         model: EPModel = None,
         model_threshold: float = None,
+        expansion_scheme: ExpansionSchemeChoice = ExpansionSchemeChoice.BFS,
+        property_models: List["PropertyPluginModel"] | None = None,
     ):
         new_s = Setting()
+
         # Clean for potential XSS
         new_s.name = nh3.clean(name, tags=s.ALLOWED_HTML_TAGS).strip()
         new_s.description = nh3.clean(description, tags=s.ALLOWED_HTML_TAGS).strip()
+
         new_s.max_nodes = max_nodes
         new_s.max_depth = max_depth
         new_s.model = model
         new_s.model_threshold = model_threshold
+        new_s.expansion_scheme = expansion_scheme
 
         new_s.save()
 
@@ -1123,6 +1102,11 @@ class SettingManager(object):
                 new_s.rule_packages.add(r)
             new_s.save()
 
+        if property_models is not None:
+            for pm in property_models:
+                new_s.property_models.add(pm)
+            new_s.save()
+
         usp = UserSettingPermission()
         usp.user = user
         usp.setting = new_s
@@ -1388,6 +1372,9 @@ class SEdge(object):
         self.rule = rule
         self.probability = probability
 
+    def product_smiles(self):
+        return [p.smiles for p in self.products]
+
     def __hash__(self):
         full_hash = 0
 
@@ -1473,6 +1460,7 @@ class SPathway(object):
         self.smiles_to_node: Dict[str, SNode] = dict(**{n.smiles: n for n in self.root_nodes})
         self.edges: Set["SEdge"] = set()
         self.done = False
+        self.empty_due_to_threshold = False
 
     @staticmethod
     def from_pathway(pw: "Pathway", persist: bool = True):
@@ -1537,6 +1525,207 @@ class SPathway(object):
 
         return sorted(res, key=lambda x: hash(x))
 
+    def _expand(self, substrates: List[SNode]) -> Tuple[List[SNode], List[SEdge]]:
+        """
+        Expands the given substrates by generating new nodes and edges based on prediction settings.
+
+        This method processes a list of substrates and expands them into new nodes and edges using defined
+        rules and settings. It evaluates each substrate to determine its applicability domain, persists
+        domain assessments, and generates candidates for further processing. Newly created nodes and edges
+        are returned, and any applicable information is stored or updated internally during the process.
+
+        Parameters:
+            substrates (List[SNode]): A list of substrate nodes to be expanded.
+
+        Returns:
+            Tuple[List[SNode], List[SEdge]]:
+                A tuple containing:
+                - A list of new nodes generated during the expansion.
+                - A list of new edges representing connections between nodes based on candidate reactions.
+
+        Raises:
+            ValueError: If a node does not have an ID when it should have been saved already.
+        """
+        new_nodes: List[SNode] = []
+        new_edges: List[SEdge] = []
+
+        for sub in substrates:
+            # For App Domain we have to ensure that each Node is evaluated
+            if sub.app_domain_assessment is None:
+                if self.prediction_setting.model:
+                    if self.prediction_setting.model.app_domain:
+                        app_domain_assessment = self.prediction_setting.model.app_domain.assess(
+                            sub.smiles
+                        )
+
+                        if self.persist is not None:
+                            n = self.snode_persist_lookup[sub]
+
+                            if n.id is None:
+                                raise ValueError(f"Node {n} has no ID... aborting!")
+
+                            node_data = n.simple_json()
+                            node_data["image"] = f"{n.url}?image=svg"
+                            app_domain_assessment["assessment"]["node"] = node_data
+
+                            n.kv["app_domain_assessment"] = app_domain_assessment
+                            n.save()
+
+                        sub.app_domain_assessment = app_domain_assessment
+
+            expansion_result = self.prediction_setting.expand(self, sub)
+
+            # We don't have any substrate, but technically we have at least one rule that triggered.
+            # If our substrate is a root node a.k.a. depth == 0 store that info in SPathway
+            if (
+                len(expansion_result["transformations"]) == 0
+                and expansion_result["rule_triggered"]
+                and sub.depth == 0
+            ):
+                self.empty_due_to_threshold = True
+
+                # Emit directly
+                if self.persist is not None:
+                    self.persist.kv["empty_due_to_threshold"] = True
+                    self.persist.save()
+
+            # candidates is a List of PredictionResult. The length of the List is equal to the number of rules
+            for cand_set in expansion_result["transformations"]:
+                if cand_set:
+                    # cand_set is a PredictionResult object that can consist of multiple candidate reactions
+                    for cand in cand_set:
+                        cand_nodes = []
+                        # candidate reactions can have multiple fragments
+                        for c in cand:
+                            if c not in self.smiles_to_node:
+                                # For new nodes do an AppDomain Assessment if an AppDomain is attached
+                                app_domain_assessment = None
+                                if self.prediction_setting.model:
+                                    if self.prediction_setting.model.app_domain:
+                                        app_domain_assessment = (
+                                            self.prediction_setting.model.app_domain.assess(c)
+                                        )
+                                snode = SNode(c, sub.depth + 1, app_domain_assessment)
+                                self.smiles_to_node[c] = snode
+                                new_nodes.append(snode)
+
+                            node = self.smiles_to_node[c]
+                            cand_nodes.append(node)
+
+                        edge = SEdge(
+                            sub,
+                            cand_nodes,
+                            rule=cand_set.rule,
+                            probability=cand_set.probability,
+                        )
+                        self.edges.add(edge)
+                        new_edges.append(edge)
+
+        return new_nodes, new_edges
+
+    def predict(self):
+        """
+        Predicts outcomes based on a graph traversal algorithm using the specified expansion schema.
+
+        This method iteratively explores the nodes of a graph starting from the root nodes, propagating
+        probabilities through edges, and updating the probabilities of the connected nodes. The traversal
+        can follow one of three predefined expansion schemas: Depth-First Search (DFS), Breadth-First Search
+        (BFS), or a Greedy approach based on node probabilities. The methodology ensures that all reachable
+        nodes are processed systematically according to the specified schema.
+
+        Errors will be raised if the expansion schema is undefined or invalid. Additionally, this method
+        supports persisting changes by writing back data to the database when configured to do so.
+
+        Attributes
+        ----------
+        done : bool
+            A flag indicating whether the prediction process is completed.
+        persist : Any
+            An optional object that manages persistence operations for saving modifications.
+        root_nodes : List[SNode]
+            A collection of initial nodes in the graph from which traversal begins.
+        prediction_setting : Any
+            Configuration object specifying settings for graph traversal, such as the choice of
+            expansion schema.
+
+        Raises
+        ------
+        ValueError
+            If an invalid or unknown expansion schema is provided in `prediction_setting`.
+        """
+        # populate initial queue
+        queue = list(self.root_nodes)
+        processed = set()
+
+        # initial nodes have prob 1.0
+        node_probs: Dict[SNode, float] = {}
+        node_probs.update({n: 1.0 for n in queue})
+
+        while queue:
+            current = queue.pop(0)
+
+            if current in processed:
+                continue
+
+            processed.add(current)
+
+            new_nodes, new_edges = self._expand([current])
+
+            if new_nodes or new_edges:
+                # Check if we need to write back data to the database
+                if self.persist:
+                    self._sync_to_pathway()
+                    # call save to update the internal modified field
+                    self.persist.save()
+
+                if new_nodes:
+                    for edge in new_edges:
+                        # All edge have `current` as educt
+                        # Use `current` and adjust probs
+                        current_prob = node_probs[current]
+
+                        for prod in edge.products:
+                            # Either is a new product or a product and we found a path with a higher prob
+                            if (
+                                prod not in node_probs
+                                or current_prob * edge.probability > node_probs[prod]
+                            ):
+                                node_probs[prod] = current_prob * edge.probability
+
+                    # Update Queue to proceed
+                    if self.prediction_setting.expansion_scheme == "DFS":
+                        for n in new_nodes:
+                            if n not in processed:
+                                # We want to follow this path -> prepend queue
+                                queue.insert(0, n)
+                    elif self.prediction_setting.expansion_scheme == "BFS":
+                        for n in new_nodes:
+                            if n not in processed:
+                                # Add at the end, everything queued before will be processed
+                                # before new_nodese
+                                queue.append(n)
+                    elif self.prediction_setting.expansion_scheme == "GREEDY":
+                        # Simply add them, as we will re-order the queue later
+                        for n in new_nodes:
+                            if n not in processed:
+                                queue.append(n)
+
+                        node_and_probs = []
+                        for queued_val in queue:
+                            node_and_probs.append((queued_val, node_probs[queued_val]))
+
+                        # re-order the queue and only pick smiles
+                        queue = [
+                            n[0] for n in sorted(node_and_probs, key=lambda x: x[1], reverse=True)
+                        ]
+                    else:
+                        raise ValueError(
+                            f"Unknown expansion schema: {self.prediction_setting.expansion_scheme}"
+                        )
+
+        # Queue exhausted, we're done
+        self.done = True
+
     def predict_step(self, from_depth: int = None, from_node: "Node" = None):
         substrates: List[SNode] = []
 
@@ -1547,67 +1736,15 @@ class SPathway(object):
                 if from_node == v:
                     substrates = [k]
                     break
+            else:
+                raise ValueError(f"Node {from_node} not found in SPathway!")
         else:
             raise ValueError("Neither from_depth nor from_node_url specified")
 
         new_tp = False
         if substrates:
-            for sub in substrates:
+            new_nodes, _ = self._expand(substrates)
-                if sub.app_domain_assessment is None:
+            new_tp = len(new_nodes) > 0
-                    if self.prediction_setting.model:
-                        if self.prediction_setting.model.app_domain:
-                            app_domain_assessment = self.prediction_setting.model.app_domain.assess(
-                                sub.smiles
-                            )
-
-                            if self.persist is not None:
-                                n = self.snode_persist_lookup[sub]
-
-                                assert n.id is not None, (
-                                    "Node has no id! Should have been saved already... aborting!"
-                                )
-                                node_data = n.simple_json()
-                                node_data["image"] = f"{n.url}?image=svg"
-                                app_domain_assessment["assessment"]["node"] = node_data
-
-                                n.kv["app_domain_assessment"] = app_domain_assessment
-                                n.save()
-
-                            sub.app_domain_assessment = app_domain_assessment
-
-                candidates = self.prediction_setting.expand(self, sub)
-                # candidates is a List of PredictionResult. The length of the List is equal to the number of rules
-                for cand_set in candidates:
-                    if cand_set:
-                        new_tp = True
-                        # cand_set is a PredictionResult object that can consist of multiple candidate reactions
-                        for cand in cand_set:
-                            cand_nodes = []
-                            # candidate reactions can have multiple fragments
-                            for c in cand:
-                                if c not in self.smiles_to_node:
-                                    # For new nodes do an AppDomain Assessment if an AppDomain is attached
-                                    app_domain_assessment = None
-                                    if self.prediction_setting.model:
-                                        if self.prediction_setting.model.app_domain:
-                                            app_domain_assessment = (
-                                                self.prediction_setting.model.app_domain.assess(c)
-                                            )
-
-                                    self.smiles_to_node[c] = SNode(
-                                        c, sub.depth + 1, app_domain_assessment
-                                    )
-
-                                node = self.smiles_to_node[c]
-                                cand_nodes.append(node)
-
-                            edge = SEdge(
-                                sub,
-                                cand_nodes,
-                                rule=cand_set.rule,
-                                probability=cand_set.probability,
-                            )
-                            self.edges.add(edge)
 
         # In case no substrates are found, we're done.
         # For "predict from node" we're always done
@@ -1620,6 +1757,14 @@ class SPathway(object):
             # call save to update the internal modified field
             self.persist.save()
 
+    def get_edge_for_educt_smiles(self, smiles: str) -> List[SEdge]:
+        res = []
+        for e in self.edges:
+            for n in e.educts:
+                if n.smiles == smiles:
+                    res.append(e)
+        return res
+
     def _sync_to_pathway(self) -> None:
         logger.info("Updating Pathway with SPathway")
 
@@ -1683,11 +1828,6 @@ class SPathway(object):
                 "to": to_indices,
             }
 
-            # if edge.rule:
-            #     e['rule'] = {
-            #         'name': edge.rule.name,
-            #         'id': edge.rule.url,
-            #     }
             edges.append(e)
 
         return {

epdb/management/commands/bootstrap.py

@@ -8,7 +8,6 @@ from epdb.logic import UserManager, GroupManager, PackageManager, SettingManager
 from epdb.models import (
     UserSettingPermission,
     MLRelativeReasoning,
-    EnviFormer,
     Permission,
     User,
     ExternalDatabase,
@@ -231,7 +230,6 @@ class Command(BaseCommand):
             package=pack,
             rule_packages=[mapping["EAWAG-BBD"]],
             data_packages=[mapping["EAWAG-BBD"]],
-            eval_packages=[],
             threshold=0.5,
             name="ECC - BBD - T0.5",
             description="ML Relative Reasoning",
@@ -239,7 +237,3 @@ class Command(BaseCommand):
 
         ml_model.build_dataset()
         ml_model.build_model()
-
-        # If available, create EnviFormerModel
-        if s.ENVIFORMER_PRESENT:
-            EnviFormer.create(pack, "EnviFormer - T0.5", "EnviFormer Model with Threshold 0.5", 0.5)

epdb/management/commands/create_api_token.py

@@ -0,0 +1,92 @@
+from django.conf import settings as s
+from django.contrib.auth import get_user_model
+from django.core.management.base import BaseCommand, CommandError
+
+from epdb.models import APIToken
+
+
+class Command(BaseCommand):
+    help = "Create an API token for a user"
+
+    def add_arguments(self, parser):
+        parser.add_argument(
+            "--username",
+            required=True,
+            help="Username of the user who will own the token",
+        )
+        parser.add_argument(
+            "--name",
+            required=True,
+            help="Descriptive name for the token",
+        )
+        parser.add_argument(
+            "--expires-days",
+            type=int,
+            default=90,
+            help="Days until expiration (0 for no expiration)",
+        )
+        parser.add_argument(
+            "--inactive",
+            action="store_true",
+            help="Create the token as inactive",
+        )
+        parser.add_argument(
+            "--curl",
+            action="store_true",
+            help="Print a curl example using the token",
+        )
+        parser.add_argument(
+            "--base-url",
+            default=None,
+            help="Base URL for curl example (default SERVER_URL or http://localhost:8000)",
+        )
+        parser.add_argument(
+            "--endpoint",
+            default="/api/v1/compounds/",
+            help="Endpoint path for curl example",
+        )
+
+    def handle(self, *args, **options):
+        username = options["username"]
+        name = options["name"]
+        expires_days = options["expires_days"]
+
+        if expires_days < 0:
+            raise CommandError("--expires-days must be >= 0")
+
+        if expires_days == 0:
+            expires_days = None
+
+        user_model = get_user_model()
+        try:
+            user = user_model.objects.get(username=username)
+        except user_model.DoesNotExist as exc:
+            raise CommandError(f"User not found for username '{username}'") from exc
+
+        token, raw_token = APIToken.create_token(user, name=name, expires_days=expires_days)
+
+        if options["inactive"]:
+            token.is_active = False
+            token.save(update_fields=["is_active"])
+
+        self.stdout.write(f"User: {user.username} ({user.email})")
+        self.stdout.write(f"Token name: {token.name}")
+        self.stdout.write(f"Token id: {token.id}")
+        if token.expires_at:
+            self.stdout.write(f"Expires at: {token.expires_at.isoformat()}")
+        else:
+            self.stdout.write("Expires at: never")
+        self.stdout.write(f"Active: {token.is_active}")
+        self.stdout.write("Raw token:")
+        self.stdout.write(raw_token)
+
+        if options["curl"]:
+            base_url = (
+                options["base_url"] or getattr(s, "SERVER_URL", None) or "http://localhost:8000"
+            )
+            endpoint = options["endpoint"]
+            endpoint = endpoint if endpoint.startswith("/") else f"/{endpoint}"
+            url = f"{base_url.rstrip('/')}{endpoint}"
+            curl_cmd = f'curl -H "Authorization: Bearer {raw_token}" "{url}"'
+            self.stdout.write("Curl:")
+            self.stdout.write(curl_cmd)

epdb/management/commands/create_ml_models.py

@@ -2,7 +2,9 @@ from django.conf import settings as s
 from django.core.management.base import BaseCommand
 from django.db import transaction
 
-from epdb.models import MLRelativeReasoning, EnviFormer, Package
+from epdb.models import EnviFormer, MLRelativeReasoning
+
+Package = s.GET_PACKAGE_MODEL()
 
 
 class Command(BaseCommand):
@@ -75,11 +77,13 @@ class Command(BaseCommand):
             return packages
 
         # Iteratively create models in options["model_names"]
-        print(f"Creating models: {options['model_names']}\n"
+        print(
-              f"Data packages: {options['data_packages']}\n"
+            f"Creating models: {options['model_names']}\n"
-              f"Rule Packages (only for MLRR): {options['rule_packages']}\n"
+            f"Data packages: {options['data_packages']}\n"
-              f"Eval Packages: {options['eval_packages']}\n"
+            f"Rule Packages (only for MLRR): {options['rule_packages']}\n"
-              f"Threshold: {options['threshold']:.2f}")
+            f"Eval Packages: {options['eval_packages']}\n"
+            f"Threshold: {options['threshold']:.2f}"
+        )
         data_packages = decode_packages(options["data_packages"])
         eval_packages = decode_packages(options["eval_packages"])
         rule_packages = decode_packages(options["rule_packages"])
@@ -89,22 +93,20 @@ class Command(BaseCommand):
                 model = EnviFormer.create(
                     pack,
                     data_packages=data_packages,
-                    eval_packages=eval_packages,
+                    threshold=options["threshold"],
-                    threshold=options['threshold'],
                     name=f"EnviFormer - {', '.join(options['data_packages'])} - T{options['threshold']:.2f}",
                     description=f"EnviFormer transformer trained on {options['data_packages']} "
-                                f"evaluated on {options['eval_packages']}.",
+                    f"evaluated on {options['eval_packages']}.",
                 )
             elif model_name == "mlrr":
                 model = MLRelativeReasoning.create(
                     package=pack,
                     rule_packages=rule_packages,
                     data_packages=data_packages,
-                    eval_packages=eval_packages,
+                    threshold=options["threshold"],
-                    threshold=options['threshold'],
                     name=f"ECC - {', '.join(options['data_packages'])} - T{options['threshold']:.2f}",
                     description=f"ML Relative Reasoning trained on {options['data_packages']} with rules from "
-                                f"{options['rule_packages']} and evaluated on {options['eval_packages']}.",
+                    f"{options['rule_packages']} and evaluated on {options['eval_packages']}.",
                 )
             else:
                 raise ValueError(f"Cannot create model of type {model_name}, unknown model type")

epdb/management/commands/dump_enviformer.py

@@ -47,7 +47,7 @@ class Command(BaseCommand):
             "description": model.description,
             "kv": model.kv,
             "data_packages_uuids": [str(p.uuid) for p in model.data_packages.all()],
-            "eval_packages_uuids": [str(p.uuid) for p in model.data_packages.all()],
+            "eval_packages_uuids": [str(p.uuid) for p in model.eval_packages.all()],
             "threshold": model.threshold,
             "eval_results": model.eval_results,
             "multigen_eval": model.multigen_eval,

epdb/management/commands/load_enviformer.py

@@ -8,7 +8,9 @@ from django.conf import settings as s
 from django.core.management.base import BaseCommand
 from django.db import transaction
 
-from epdb.models import EnviFormer, Package
+from epdb.models import EnviFormer
+
+Package = s.GET_PACKAGE_MODEL()
 
 
 class Command(BaseCommand):

epdb/management/commands/localize_urls.py

@@ -1,8 +1,8 @@
 from django.apps import apps
+from django.conf import settings as s
 from django.core.management.base import BaseCommand
-
+from django.db.models import F, JSONField, TextField, Value
-from django.db.models import F, Value, TextField, JSONField
+from django.db.models.functions import Cast, Replace
-from django.db.models.functions import Replace, Cast
 
 from epdb.models import EnviPathModel
 
@@ -23,10 +23,12 @@ class Command(BaseCommand):
         )
 
     def handle(self, *args, **options):
+        Package = s.GET_PACKAGE_MODEL()
+        Package.objects.update(url=Replace(F("url"), Value(options["old"]), Value(options["new"])))
+
         MODELS = [
             "User",
             "Group",
-            "Package",
             "Compound",
             "CompoundStructure",
             "Pathway",
@@ -39,15 +41,12 @@ class Command(BaseCommand):
             "SequentialRule",
             "Scenario",
             "Setting",
-            "MLRelativeReasoning",
+            "EPModel",
-            "RuleBasedRelativeReasoning",
-            "EnviFormer",
             "ApplicabilityDomain",
             "EnzymeLink",
         ]
         for model in MODELS:
             obj_cls = apps.get_model("epdb", model)
-            print(f"Localizing urls for {model}")
             obj_cls.objects.update(
                 url=Replace(F("url"), Value(options["old"]), Value(options["new"]))
             )

epdb/management/commands/recreate_db.py

@@ -0,0 +1,83 @@
+import os
+import subprocess
+
+from django.conf import settings
+from django.core.management import call_command
+from django.core.management.base import BaseCommand
+
+
+class Command(BaseCommand):
+    def add_arguments(self, parser):
+        parser.add_argument(
+            "-n",
+            "--name",
+            type=str,
+            help="Name of the database to recreate. Default is 'appdb'",
+            default="appdb",
+        )
+
+        parser.add_argument(
+            "-d",
+            "--dump",
+            type=str,
+            help="Path to the dump file",
+            default="./fixtures/db.dump",
+        )
+
+        parser.add_argument(
+            "-ou",
+            "--oldurl",
+            type=str,
+            help="Old URL, e.g. https://envipath.org/",
+            default="https://envipath.org/",
+        )
+
+        parser.add_argument(
+            "-nu",
+            "--newurl",
+            type=str,
+            help="New URL, e.g. http://localhost:8000/",
+            default="http://localhost:8000/",
+        )
+
+    def handle(self, *args, **options):
+        dump_file = options["dump"]
+
+        if not os.path.exists(dump_file):
+            raise ValueError(f"Dump file {dump_file} does not exist")
+
+        db_name = options["name"]
+
+        print(f"Dropping database {db_name} y/n: ", end="")
+
+        if input() in "yY":
+            result = subprocess.run(
+                ["dropdb", db_name],
+                capture_output=True,
+                text=True,
+            )
+            print(result.stdout)
+        else:
+            raise ValueError("Aborted")
+
+        print(f"Creating database {db_name}")
+
+        result = subprocess.run(
+            ["createdb", db_name],
+            capture_output=True,
+            text=True,
+        )
+        print(result.stdout)
+        print(f"Restoring database {db_name} from {dump_file}")
+
+        result = subprocess.run(
+            ["pg_restore", "-d", db_name, dump_file, "--no-owner"],
+            capture_output=True,
+            text=True,
+        )
+        print(result.stdout)
+
+        if db_name == settings.DATABASES["default"]["NAME"]:
+            call_command("localize_urls", "--old", options["oldurl"], "--new", options["newurl"])
+        else:
+            print("Skipping localize_urls as database is not the default one.")

epdb/migrations/0012_node_stereo_removed_pathway_predicted.py

@@ -0,0 +1,22 @@
+# Generated by Django 5.2.7 on 2025-12-02 13:09
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ("epdb", "0011_auto_20251111_1413"),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name="node",
+            name="stereo_removed",
+            field=models.BooleanField(default=False),
+        ),
+        migrations.AddField(
+            model_name="pathway",
+            name="predicted",
+            field=models.BooleanField(default=False),
+        ),
+    ]

epdb/migrations/0013_setting_expansion_schema.py

@@ -0,0 +1,25 @@
+# Generated by Django 5.2.7 on 2025-12-14 11:30
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ("epdb", "0012_node_stereo_removed_pathway_predicted"),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name="setting",
+            name="expansion_schema",
+            field=models.CharField(
+                choices=[
+                    ("BFS", "Breadth First Search"),
+                    ("DFS", "Depth First Search"),
+                    ("GREEDY", "Greedy"),
+                ],
+                default="BFS",
+                max_length=20,
+            ),
+        ),
+    ]

epdb/migrations/0014_rename_expansion_schema_setting_expansion_scheme.py

@@ -0,0 +1,17 @@
+# Generated by Django 5.2.7 on 2025-12-14 16:02
+
+from django.db import migrations
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ("epdb", "0013_setting_expansion_schema"),
+    ]
+
+    operations = [
+        migrations.RenameField(
+            model_name="setting",
+            old_name="expansion_schema",
+            new_name="expansion_scheme",
+        ),
+    ]

epdb/migrations/0015_user_is_reviewer.py

@@ -0,0 +1,17 @@
+# Generated by Django 5.2.7 on 2026-01-19 19:26
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ("epdb", "0014_rename_expansion_schema_setting_expansion_scheme"),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name="user",
+            name="is_reviewer",
+            field=models.BooleanField(default=False),
+        ),
+    ]

epdb/migrations/0016_remove_enviformer_model_status_and_more.py

@@ -0,0 +1,179 @@
+# Generated by Django 5.2.7 on 2026-02-12 09:38
+
+import django.db.models.deletion
+from django.conf import settings
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ("epdb", "0015_user_is_reviewer"),
+    ]
+
+    operations = [
+        migrations.RemoveField(
+            model_name="enviformer",
+            name="model_status",
+        ),
+        migrations.RemoveField(
+            model_name="mlrelativereasoning",
+            name="model_status",
+        ),
+        migrations.RemoveField(
+            model_name="rulebasedrelativereasoning",
+            name="model_status",
+        ),
+        migrations.AddField(
+            model_name="epmodel",
+            name="model_status",
+            field=models.CharField(
+                choices=[
+                    ("INITIAL", "Initial"),
+                    ("INITIALIZING", "Model is initializing."),
+                    ("BUILDING", "Model is building."),
+                    (
+                        "BUILT_NOT_EVALUATED",
+                        "Model is built and can be used for predictions, Model is not evaluated yet.",
+                    ),
+                    ("EVALUATING", "Model is evaluating"),
+                    ("FINISHED", "Model has finished building and evaluation."),
+                    ("ERROR", "Model has failed."),
+                ],
+                default="INITIAL",
+            ),
+        ),
+        migrations.AlterField(
+            model_name="enviformer",
+            name="eval_packages",
+            field=models.ManyToManyField(
+                blank=True,
+                related_name="%(app_label)s_%(class)s_eval_packages",
+                to=settings.EPDB_PACKAGE_MODEL,
+                verbose_name="Evaluation Packages",
+            ),
+        ),
+        migrations.AlterField(
+            model_name="enviformer",
+            name="rule_packages",
+            field=models.ManyToManyField(
+                blank=True,
+                related_name="%(app_label)s_%(class)s_rule_packages",
+                to=settings.EPDB_PACKAGE_MODEL,
+                verbose_name="Rule Packages",
+            ),
+        ),
+        migrations.AlterField(
+            model_name="mlrelativereasoning",
+            name="eval_packages",
+            field=models.ManyToManyField(
+                blank=True,
+                related_name="%(app_label)s_%(class)s_eval_packages",
+                to=settings.EPDB_PACKAGE_MODEL,
+                verbose_name="Evaluation Packages",
+            ),
+        ),
+        migrations.AlterField(
+            model_name="mlrelativereasoning",
+            name="rule_packages",
+            field=models.ManyToManyField(
+                blank=True,
+                related_name="%(app_label)s_%(class)s_rule_packages",
+                to=settings.EPDB_PACKAGE_MODEL,
+                verbose_name="Rule Packages",
+            ),
+        ),
+        migrations.AlterField(
+            model_name="rulebasedrelativereasoning",
+            name="eval_packages",
+            field=models.ManyToManyField(
+                blank=True,
+                related_name="%(app_label)s_%(class)s_eval_packages",
+                to=settings.EPDB_PACKAGE_MODEL,
+                verbose_name="Evaluation Packages",
+            ),
+        ),
+        migrations.AlterField(
+            model_name="rulebasedrelativereasoning",
+            name="rule_packages",
+            field=models.ManyToManyField(
+                blank=True,
+                related_name="%(app_label)s_%(class)s_rule_packages",
+                to=settings.EPDB_PACKAGE_MODEL,
+                verbose_name="Rule Packages",
+            ),
+        ),
+        migrations.CreateModel(
+            name="PropertyPluginModel",
+            fields=[
+                (
+                    "epmodel_ptr",
+                    models.OneToOneField(
+                        auto_created=True,
+                        on_delete=django.db.models.deletion.CASCADE,
+                        parent_link=True,
+                        primary_key=True,
+                        serialize=False,
+                        to="epdb.epmodel",
+                    ),
+                ),
+                ("threshold", models.FloatField(default=0.5)),
+                ("eval_results", models.JSONField(blank=True, default=dict, null=True)),
+                ("multigen_eval", models.BooleanField(default=False)),
+                ("plugin_identifier", models.CharField(max_length=255)),
+                (
+                    "app_domain",
+                    models.ForeignKey(
+                        blank=True,
+                        default=None,
+                        null=True,
+                        on_delete=django.db.models.deletion.SET_NULL,
+                        to="epdb.applicabilitydomain",
+                    ),
+                ),
+                (
+                    "data_packages",
+                    models.ManyToManyField(
+                        blank=True,
+                        related_name="%(app_label)s_%(class)s_data_packages",
+                        to=settings.EPDB_PACKAGE_MODEL,
+                        verbose_name="Data Packages",
+                    ),
+                ),
+                (
+                    "eval_packages",
+                    models.ManyToManyField(
+                        blank=True,
+                        related_name="%(app_label)s_%(class)s_eval_packages",
+                        to=settings.EPDB_PACKAGE_MODEL,
+                        verbose_name="Evaluation Packages",
+                    ),
+                ),
+                (
+                    "rule_packages",
+                    models.ManyToManyField(
+                        blank=True,
+                        related_name="%(app_label)s_%(class)s_rule_packages",
+                        to=settings.EPDB_PACKAGE_MODEL,
+                        verbose_name="Rule Packages",
+                    ),
+                ),
+            ],
+            options={
+                "abstract": False,
+            },
+            bases=("epdb.epmodel",),
+        ),
+        migrations.AddField(
+            model_name="setting",
+            name="property_models",
+            field=models.ManyToManyField(
+                blank=True,
+                related_name="settings",
+                to="epdb.propertypluginmodel",
+                verbose_name="Setting Property Models",
+            ),
+        ),
+        migrations.DeleteModel(
+            name="PluginModel",
+        ),
+    ]

epdb/migrations/0017_additionalinformation.py

@@ -0,0 +1,93 @@
+# Generated by Django 5.2.7 on 2026-02-20 12:02
+
+import django.db.models.deletion
+import uuid
+from django.conf import settings
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ("contenttypes", "0002_remove_content_type_name"),
+        ("epdb", "0016_remove_enviformer_model_status_and_more"),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name="AdditionalInformation",
+            fields=[
+                (
+                    "id",
+                    models.BigAutoField(
+                        auto_created=True, primary_key=True, serialize=False, verbose_name="ID"
+                    ),
+                ),
+                ("uuid", models.UUIDField(default=uuid.uuid4, editable=False, unique=True)),
+                ("url", models.TextField(null=True, unique=True, verbose_name="URL")),
+                ("kv", models.JSONField(blank=True, default=dict, null=True)),
+                ("type", models.TextField(verbose_name="Additional Information Type")),
+                ("data", models.JSONField(blank=True, default=dict, null=True)),
+                ("object_id", models.PositiveBigIntegerField(blank=True, null=True)),
+                (
+                    "content_type",
+                    models.ForeignKey(
+                        blank=True,
+                        null=True,
+                        on_delete=django.db.models.deletion.CASCADE,
+                        to="contenttypes.contenttype",
+                    ),
+                ),
+                (
+                    "package",
+                    models.ForeignKey(
+                        on_delete=django.db.models.deletion.CASCADE,
+                        to=settings.EPDB_PACKAGE_MODEL,
+                        verbose_name="Package",
+                    ),
+                ),
+                (
+                    "scenario",
+                    models.ForeignKey(
+                        blank=True,
+                        null=True,
+                        on_delete=django.db.models.deletion.CASCADE,
+                        related_name="scenario_additional_information",
+                        to="epdb.scenario",
+                    ),
+                ),
+            ],
+            options={
+                "indexes": [
+                    models.Index(fields=["type"], name="epdb_additi_type_394349_idx"),
+                    models.Index(
+                        fields=["scenario", "type"], name="epdb_additi_scenari_a59edf_idx"
+                    ),
+                    models.Index(
+                        fields=["content_type", "object_id"], name="epdb_additi_content_44d4b4_idx"
+                    ),
+                    models.Index(
+                        fields=["scenario", "content_type", "object_id"],
+                        name="epdb_additi_scenari_ef2bf5_idx",
+                    ),
+                ],
+                "constraints": [
+                    models.CheckConstraint(
+                        condition=models.Q(
+                            models.Q(("content_type__isnull", True), ("object_id__isnull", True)),
+                            models.Q(("content_type__isnull", False), ("object_id__isnull", False)),
+                            _connector="OR",
+                        ),
+                        name="ck_addinfo_gfk_pair",
+                    ),
+                    models.CheckConstraint(
+                        condition=models.Q(
+                            ("scenario__isnull", False),
+                            ("content_type__isnull", False),
+                            _connector="OR",
+                        ),
+                        name="ck_addinfo_not_both_null",
+                    ),
+                ],
+            },
+        ),
+    ]

epdb/migrations/0018_auto_20260220_1203.py

@@ -0,0 +1,132 @@
+# Generated by Django 5.2.7 on 2026-02-20 12:03
+
+from django.db import migrations
+
+
+def get_additional_information(scenario):
+    from envipy_additional_information import registry
+    from envipy_additional_information.parsers import TypeOfAerationParser
+
+    for k, vals in scenario.additional_information.items():
+        if k == "enzyme":
+            continue
+
+        if k == "SpikeConentration":
+            k = "SpikeConcentration"
+
+        if k == "AerationType":
+            k = "TypeOfAeration"
+
+        for v in vals:
+            # Per default additional fields are ignored
+            MAPPING = {c.__name__: c for c in registry.list_models().values()}
+            try:
+                inst = MAPPING[k](**v)
+            except Exception:
+                if k == "TypeOfAeration":
+                    toa = TypeOfAerationParser()
+                    inst = toa.from_string(v["type"])
+
+            # Add uuid to uniquely identify objects for manipulation
+            if "uuid" in v:
+                inst.__dict__["uuid"] = v["uuid"]
+
+            yield inst
+
+
+def forward_func(apps, schema_editor):
+    Scenario = apps.get_model("epdb", "Scenario")
+    ContentType = apps.get_model("contenttypes", "ContentType")
+    AdditionalInformation = apps.get_model("epdb", "AdditionalInformation")
+
+    bulk = []
+    related = []
+    ctype = {o.model: o for o in ContentType.objects.all()}
+    parents = Scenario.objects.prefetch_related(
+        "compound_set",
+        "compoundstructure_set",
+        "reaction_set",
+        "rule_set",
+        "pathway_set",
+        "node_set",
+        "edge_set",
+    ).filter(parent__isnull=True)
+
+    for i, scenario in enumerate(parents):
+        print(f"{i + 1}/{len(parents)}", end="\r")
+        if scenario.parent is not None:
+            related.append(scenario.parent)
+            continue
+
+        for ai in get_additional_information(scenario):
+            bulk.append(
+                AdditionalInformation(
+                    package=scenario.package,
+                    scenario=scenario,
+                    type=ai.__class__.__name__,
+                    data=ai.model_dump(mode="json"),
+                )
+            )
+
+    print("\n", len(bulk))
+
+    related = Scenario.objects.prefetch_related(
+        "compound_set",
+        "compoundstructure_set",
+        "reaction_set",
+        "rule_set",
+        "pathway_set",
+        "node_set",
+        "edge_set",
+    ).filter(parent__isnull=False)
+
+    for i, scenario in enumerate(related):
+        print(f"{i + 1}/{len(related)}", end="\r")
+        parent = scenario.parent
+        # Check to which objects this scenario is attached to
+        for ai in get_additional_information(scenario):
+            rel_objs = [
+                "compound",
+                "compoundstructure",
+                "reaction",
+                "rule",
+                "pathway",
+                "node",
+                "edge",
+            ]
+            for rel_obj in rel_objs:
+                for o in getattr(scenario, f"{rel_obj}_set").all():
+                    bulk.append(
+                        AdditionalInformation(
+                            package=scenario.package,
+                            scenario=parent,
+                            type=ai.__class__.__name__,
+                            data=ai.model_dump(mode="json"),
+                            content_type=ctype[rel_obj],
+                            object_id=o.pk,
+                        )
+                    )
+
+    print("Start creating additional information objects...")
+    AdditionalInformation.objects.bulk_create(bulk)
+    print("Done!")
+    print(len(bulk))
+
+    Scenario.objects.filter(parent__isnull=False).delete()
+    # Call ai save to fix urls
+    ais = AdditionalInformation.objects.all()
+    total = ais.count()
+
+    for i, ai in enumerate(ais):
+        print(f"{i + 1}/{total}", end="\r")
+        ai.save()
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ("epdb", "0017_additionalinformation"),
+    ]
+
+    operations = [
+        migrations.RunPython(forward_func, reverse_code=migrations.RunPython.noop),
+    ]

epdb/migrations/0019_remove_scenario_additional_information_and_more.py

@@ -0,0 +1,20 @@
+# Generated by Django 5.2.7 on 2026-02-23 08:45
+
+from django.db import migrations
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ("epdb", "0018_auto_20260220_1203"),
+    ]
+
+    operations = [
+        migrations.RemoveField(
+            model_name="scenario",
+            name="additional_information",
+        ),
+        migrations.RemoveField(
+            model_name="scenario",
+            name="parent",
+        ),
+    ]

epdb/migrations/0020_alter_compoundstructure_options_and_more.py

@@ -0,0 +1,65 @@
+# Generated by Django 5.2.7 on 2026-03-09 10:41
+
+import django.db.models.deletion
+from django.db import migrations, models
+
+
+def populate_polymorphic_ctype(apps, schema_editor):
+    ContentType = apps.get_model("contenttypes", "ContentType")
+    Compound = apps.get_model("epdb", "Compound")
+    CompoundStructure = apps.get_model("epdb", "CompoundStructure")
+
+    # Update Compound records
+    compound_ct = ContentType.objects.get_for_model(Compound)
+    Compound.objects.filter(polymorphic_ctype__isnull=True).update(polymorphic_ctype=compound_ct)
+
+    # Update CompoundStructure records
+    compound_structure_ct = ContentType.objects.get_for_model(CompoundStructure)
+    CompoundStructure.objects.filter(polymorphic_ctype__isnull=True).update(
+        polymorphic_ctype=compound_structure_ct
+    )
+
+
+def reverse_populate_polymorphic_ctype(apps, schema_editor):
+    Compound = apps.get_model("epdb", "Compound")
+    CompoundStructure = apps.get_model("epdb", "CompoundStructure")
+
+    Compound.objects.all().update(polymorphic_ctype=None)
+    CompoundStructure.objects.all().update(polymorphic_ctype=None)
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ("contenttypes", "0002_remove_content_type_name"),
+        ("epdb", "0019_remove_scenario_additional_information_and_more"),
+    ]
+
+    operations = [
+        migrations.AlterModelOptions(
+            name="compoundstructure",
+            options={"base_manager_name": "objects"},
+        ),
+        migrations.AddField(
+            model_name="compound",
+            name="polymorphic_ctype",
+            field=models.ForeignKey(
+                editable=False,
+                null=True,
+                on_delete=django.db.models.deletion.CASCADE,
+                related_name="polymorphic_%(app_label)s.%(class)s_set+",
+                to="contenttypes.contenttype",
+            ),
+        ),
+        migrations.AddField(
+            model_name="compoundstructure",
+            name="polymorphic_ctype",
+            field=models.ForeignKey(
+                editable=False,
+                null=True,
+                on_delete=django.db.models.deletion.CASCADE,
+                related_name="polymorphic_%(app_label)s.%(class)s_set+",
+                to="contenttypes.contenttype",
+            ),
+        ),
+        migrations.RunPython(populate_polymorphic_ctype, reverse_populate_polymorphic_ctype),
+    ]

epdb/migrations/0021_classifierpluginmodel.py

@@ -0,0 +1,75 @@
+# Generated by Django 5.2.7 on 2026-03-25 11:44
+
+import django.db.models.deletion
+from django.conf import settings
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ("epdb", "0020_alter_compoundstructure_options_and_more"),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name="ClassifierPluginModel",
+            fields=[
+                (
+                    "epmodel_ptr",
+                    models.OneToOneField(
+                        auto_created=True,
+                        on_delete=django.db.models.deletion.CASCADE,
+                        parent_link=True,
+                        primary_key=True,
+                        serialize=False,
+                        to="epdb.epmodel",
+                    ),
+                ),
+                ("threshold", models.FloatField(default=0.5)),
+                ("eval_results", models.JSONField(blank=True, default=dict, null=True)),
+                ("multigen_eval", models.BooleanField(default=False)),
+                ("plugin_identifier", models.CharField(max_length=255)),
+                ("plugin_config", models.JSONField(blank=True, default=dict, null=True)),
+                (
+                    "app_domain",
+                    models.ForeignKey(
+                        blank=True,
+                        default=None,
+                        null=True,
+                        on_delete=django.db.models.deletion.SET_NULL,
+                        to="epdb.applicabilitydomain",
+                    ),
+                ),
+                (
+                    "data_packages",
+                    models.ManyToManyField(
+                        related_name="%(app_label)s_%(class)s_data_packages",
+                        to=settings.EPDB_PACKAGE_MODEL,
+                        verbose_name="Data Packages",
+                    ),
+                ),
+                (
+                    "eval_packages",
+                    models.ManyToManyField(
+                        blank=True,
+                        related_name="%(app_label)s_%(class)s_eval_packages",
+                        to=settings.EPDB_PACKAGE_MODEL,
+                        verbose_name="Evaluation Packages",
+                    ),
+                ),
+                (
+                    "rule_packages",
+                    models.ManyToManyField(
+                        blank=True,
+                        related_name="%(app_label)s_%(class)s_rule_packages",
+                        to=settings.EPDB_PACKAGE_MODEL,
+                        verbose_name="Rule Packages",
+                    ),
+                ),
+            ],
+            options={
+                "abstract": False,
+            },
+            bases=("epdb.epmodel",),
+        ),
+    ]

epdb/migrations/0022_alter_classifierpluginmodel_data_packages_and_more.py

@@ -0,0 +1,53 @@
+# Generated by Django 5.2.7 on 2026-03-25 11:56
+
+from django.conf import settings
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ("epdb", "0021_classifierpluginmodel"),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name="classifierpluginmodel",
+            name="data_packages",
+            field=models.ManyToManyField(
+                blank=True,
+                related_name="%(app_label)s_%(class)s_data_packages",
+                to=settings.EPDB_PACKAGE_MODEL,
+                verbose_name="Data Packages",
+            ),
+        ),
+        migrations.AlterField(
+            model_name="enviformer",
+            name="data_packages",
+            field=models.ManyToManyField(
+                blank=True,
+                related_name="%(app_label)s_%(class)s_data_packages",
+                to=settings.EPDB_PACKAGE_MODEL,
+                verbose_name="Data Packages",
+            ),
+        ),
+        migrations.AlterField(
+            model_name="mlrelativereasoning",
+            name="data_packages",
+            field=models.ManyToManyField(
+                blank=True,
+                related_name="%(app_label)s_%(class)s_data_packages",
+                to=settings.EPDB_PACKAGE_MODEL,
+                verbose_name="Data Packages",
+            ),
+        ),
+        migrations.AlterField(
+            model_name="rulebasedrelativereasoning",
+            name="data_packages",
+            field=models.ManyToManyField(
+                blank=True,
+                related_name="%(app_label)s_%(class)s_data_packages",
+                to=settings.EPDB_PACKAGE_MODEL,
+                verbose_name="Data Packages",
+            ),
+        ),
+    ]

epdb/migrations/0023_alter_compoundstructure_options_and_more.py

@@ -0,0 +1,49 @@
+# Generated by Django 6.0.3 on 2026-04-21 11:43
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ("epdb", "0022_alter_classifierpluginmodel_data_packages_and_more"),
+    ]
+
+    operations = [
+        migrations.AlterModelOptions(
+            name="compoundstructure",
+            options={},
+        ),
+        migrations.AlterModelOptions(
+            name="epmodel",
+            options={},
+        ),
+        migrations.AlterModelOptions(
+            name="parallelrule",
+            options={},
+        ),
+        migrations.AlterModelOptions(
+            name="rule",
+            options={},
+        ),
+        migrations.AlterModelOptions(
+            name="sequentialrule",
+            options={},
+        ),
+        migrations.AlterModelOptions(
+            name="simpleambitrule",
+            options={},
+        ),
+        migrations.AlterModelOptions(
+            name="simplerdkitrule",
+            options={},
+        ),
+        migrations.AlterModelOptions(
+            name="simplerule",
+            options={},
+        ),
+        migrations.AddField(
+            model_name="compoundstructure",
+            name="molfile",
+            field=models.TextField(blank=True, null=True, verbose_name="Molfile"),
+        ),
+    ]

epdb/migrations/0024_user_contacted.py

@@ -0,0 +1,17 @@
+# Generated by Django 6.0.3 on 2026-04-21 19:56
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ("epdb", "0023_alter_compoundstructure_options_and_more"),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name="user",
+            name="contacted",
+            field=models.BooleanField(blank=True, null=True),
+        ),
+    ]

epdb/migrations/0025_auto_20260511_2025.py

@@ -0,0 +1,56 @@
+# Generated by Django 6.0.3 on 2026-05-11 20:25
+
+from django.db import migrations
+from envipy_additional_information import HalfLife, HalfLifeModel, HalfLifeWS
+
+MAPPING = {
+    "": HalfLifeModel.OTHER,
+    "HS-SFO": HalfLifeModel.HS_SFO,
+    "FOMC": HalfLifeModel.FOMC,
+    "FOTC": HalfLifeModel.DFOP,
+    "FMOC": HalfLifeModel.FOMC,
+    "DFOP": HalfLifeModel.DFOP,
+    "SFO + SFO": HalfLifeModel.SFO_SFO,
+    "FOMC-SFO": HalfLifeModel.FOMC_SFO,
+    "first order kinetics": HalfLifeModel.SFO,
+    "SFO²": HalfLifeModel.SFO,
+    "HS": HalfLifeModel.HS,
+    "top down": HalfLifeModel.OTHER,
+    "SFO": HalfLifeModel.SFO,
+    "First Order": HalfLifeModel.SFO,
+    "SFO/SFO": HalfLifeModel.SFO_SFO,
+    "FOMC + SFO": HalfLifeModel.FOMC_SFO,
+    "true": HalfLifeModel.SFO,
+    "SFO-SFO": HalfLifeModel.SFO_SFO,
+    "DFOP-SFO": HalfLifeModel.DFOP_SFO,
+}
+
+
+def forward_func(apps, schema_editor):
+    AdditionalInformation = apps.get_model("epdb", "AdditionalInformation")
+
+    hls = AdditionalInformation.objects.filter(type="HalfLife")
+
+    for hl in hls:
+        data = hl.data
+        data["model"] = MAPPING[data["model"]].value
+        hl.data = HalfLife(**data).model_dump(mode="json")
+        hl.save()
+
+    hlws = AdditionalInformation.objects.filter(type="HalfLifeWS")
+
+    for hl in hlws:
+        data = hl.data
+        data["model"] = MAPPING[data["model"]].value
+        hl.data = HalfLifeWS(**data).model_dump(mode="json")
+        hl.save()
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ("epdb", "0024_user_contacted"),
+    ]
+
+    operations = [
+        migrations.RunPython(forward_func, reverse_code=migrations.RunPython.noop),
+    ]

epdb/tasks.py

@@ -1,19 +1,34 @@
 import csv
 import io
 import logging
-from datetime import datetime
 from typing import Any, Callable, List, Optional
 from uuid import uuid4
 
 from celery import shared_task
 from celery.utils.functional import LRUCache
+from django.conf import settings as s
+from django.core.mail import EmailMultiAlternatives
+from django.utils import timezone
 
 from epdb.logic import SPathway
-from epdb.models import EPModel, JobLog, Node, Package, Pathway, Rule, Setting, User, Edge
+from epdb.models import (
+    AdditionalInformation,
+    Edge,
+    EPModel,
+    JobLog,
+    Node,
+    Pathway,
+    Rule,
+    Setting,
+    User,
+)
+from utilities.chem import FormatConverter
 
 logger = logging.getLogger(__name__)
 ML_CACHE = LRUCache(3)  # Cache the three most recent ML models to reduce load times.
 
+Package = s.GET_PACKAGE_MODEL()
+
 
 def get_ml_model(model_pk: int):
     if model_pk not in ML_CACHE:
@@ -29,11 +44,11 @@ def dispatch_eager(user: "User", job: Callable, *args, **kwargs):
         log.task_id = uuid4()
         log.job_name = job.__name__
         log.status = "SUCCESS"
-        log.done_at = datetime.now()
+        log.done_at = timezone.now()
         log.task_result = str(x) if x else None
         log.save()
 
-        return x
+        return log, x
     except Exception as e:
         logger.exception(e)
         raise e
@@ -49,7 +64,7 @@ def dispatch(user: "User", job: Callable, *args, **kwargs):
         log.status = "INITIAL"
         log.save()
 
-        return x.result
+        return log
     except Exception as e:
         logger.exception(e)
         raise e
@@ -61,15 +76,39 @@ def mul(a, b):
 
 
 @shared_task(queue="predict")
-def predict_simple(model_pk: int, smiles: str):
+def predict_simple(model_pk: int, smiles: str, *args, **kwargs):
     mod = get_ml_model(model_pk)
-    res = mod.predict(smiles)
+    res = mod.predict(smiles, *args, **kwargs)
     return res
 
 
 @shared_task(queue="background")
 def send_registration_mail(user_pk: int):
-    pass
+    u = User.objects.get(id=user_pk)
+
+    tpl = """Welcome {username}!,
+
+Thank you for your interest in enviPath.
+
+The public system is intended for non-commercial use only.
+We will review your account details and usually activate your account within 24 hours.
+Once activated, you will be notified by email.
+
+If we have any questions, we will contact you at this email address.
+
+Best regards,
+
+enviPath team"""
+
+    msg = EmailMultiAlternatives(
+        "Your enviPath account",
+        tpl.format(username=u.username),
+        "admin@envipath.org",
+        [u.email],
+        bcc=["admin@envipath.org"],
+    )
+
+    msg.send(fail_silently=False)
 
 
 @shared_task(bind=True, queue="model")
@@ -136,14 +175,25 @@ def predict(
     pred_setting_pk: int,
     limit: Optional[int] = None,
     node_pk: Optional[int] = None,
+    setting_overrides: Optional[dict] = None,
 ) -> Pathway:
     pw = Pathway.objects.get(id=pw_pk)
     setting = Setting.objects.get(id=pred_setting_pk)
+
+    if setting_overrides:
+        for k, v in setting_overrides.items():
+            setattr(setting, k, v)
+
     # If the setting has a model add/restore it from the cache
     if setting.model is not None:
         setting.model = get_ml_model(setting.model.pk)
 
-    pw.kv.update(**{"status": "running"})
+    kv = {"status": "running"}
+
+    if setting_overrides:
+        kv["setting_overrides"] = setting_overrides
+
+    pw.kv.update(**kv)
     pw.save()
 
     if JobLog.objects.filter(task_id=self.request.id).exists():
@@ -168,10 +218,12 @@ def predict(
             spw = SPathway.from_pathway(pw)
             spw.predict_step(from_node=n)
         else:
-            raise ValueError("Neither limit nor node_pk given!")
+            spw = SPathway(prediction_setting=setting, persist=pw)
+            spw.predict()
 
     except Exception as e:
         pw.kv.update({"status": "failed"})
+        pw.kv.update(**{"error": str(e)})
         pw.save()
 
         if JobLog.objects.filter(task_id=self.request.id).exists():
@@ -187,9 +239,28 @@ def predict(
     if JobLog.objects.filter(task_id=self.request.id).exists():
         JobLog.objects.filter(task_id=self.request.id).update(status="SUCCESS", task_result=pw.url)
 
+    # dispatch property job
+    compute_properties.delay(pw_pk, pred_setting_pk)
+
     return pw.url
 
 
+@shared_task(bind=True, queue="background")
+def compute_properties(self, pathway_pk: int, setting_pk: int):
+    pw = Pathway.objects.get(id=pathway_pk)
+    setting = Setting.objects.get(id=setting_pk)
+
+    nodes = [n for n in pw.nodes]
+    smiles = [n.default_node_label.smiles for n in nodes]
+
+    for prop_mod in setting.property_models.all():
+        if prop_mod.instance().is_heavy():
+            rr = prop_mod.predict_batch(smiles)
+            for idx, pred in enumerate(rr.result):
+                n = nodes[idx]
+                _ = AdditionalInformation.create(pw.package, ai=pred, content_object=n)
+
+
 @shared_task(bind=True, queue="background")
 def identify_missing_rules(
     self,
@@ -281,3 +352,144 @@ def identify_missing_rules(
     buffer.seek(0)
 
     return buffer.getvalue()
+
+
+@shared_task(bind=True, queue="background")
+def engineer_pathways(self, pw_pks: List[int], setting_pk: int, target_package_pk: int):
+    from utilities.misc import PathwayUtils
+
+    setting = Setting.objects.get(pk=setting_pk)
+    # Temporarily set model_threshold to 0.0 to keep all tps
+    setting.model_threshold = 0.0
+
+    target = Package.objects.get(pk=target_package_pk)
+
+    intermediate_pathways = []
+    predicted_pathways = []
+
+    for pw in Pathway.objects.filter(pk__in=pw_pks):
+        pu = PathwayUtils(pw)
+
+        eng_pw, node_to_snode_mapping, intermediates = pu.engineer(setting)
+
+        # If we've found intermediates, do the following
+        # - Get a copy of the original pathway and add intermediates
+        # - Store the predicted pathway for further investigation
+        if len(intermediates):
+            copy_mapping = {}
+            copied_pw = pw.copy(target, copy_mapping)
+            copied_pw.name = f"{copied_pw.name} (Engineered)"
+            copied_pw.description = f"The original Pathway can be found here: {pw.url}"
+            copied_pw.save()
+
+            for inter in intermediates:
+                start = copy_mapping[inter[0]]
+                end = copy_mapping[inter[1]]
+                start_snode = inter[2]
+                end_snode = inter[3]
+                for idx, intermediate_edge in enumerate(inter[4]):
+                    smiles_to_node = {}
+
+                    snodes_to_create = list(
+                        set(intermediate_edge.educts + intermediate_edge.products)
+                    )
+
+                    for snode in snodes_to_create:
+                        if snode == start_snode or snode == end_snode:
+                            smiles_to_node[snode.smiles] = start if snode == start_snode else end
+                            continue
+
+                        if snode.smiles not in smiles_to_node:
+                            n = Node.create(copied_pw, smiles=snode.smiles, depth=snode.depth)
+                            # Used in viz to highlight intermediates
+                            n.kv.update({"is_engineered_intermediate": True})
+                            n.save()
+                            smiles_to_node[snode.smiles] = n
+
+                    Edge.create(
+                        copied_pw,
+                        [smiles_to_node[educt.smiles] for educt in intermediate_edge.educts],
+                        [smiles_to_node[product.smiles] for product in intermediate_edge.products],
+                        rule=intermediate_edge.rule,
+                    )
+
+            # Persist the predicted pathway
+            pred_pw = pu.spathway_to_pathway(target, eng_pw, name=f"{pw.name} (Predicted)")
+
+            intermediate_pathways.append(copied_pw.url)
+            predicted_pathways.append(pred_pw.url)
+
+    return intermediate_pathways, predicted_pathways
+
+
+@shared_task(bind=True, queue="background")
+def batch_predict(
+    self,
+    substrates: List[str] | List[List[str]],
+    prediction_setting_pk: int,
+    target_package_pk: int,
+    num_tps: int = 50,
+):
+    target_package = Package.objects.get(pk=target_package_pk)
+    prediction_setting = Setting.objects.get(pk=prediction_setting_pk)
+
+    if len(substrates) == 0:
+        raise ValueError("No substrates given!")
+
+    is_pair = isinstance(substrates[0], list)
+
+    substrate_and_names = []
+    if not is_pair:
+        for sub in substrates:
+            substrate_and_names.append([sub, None])
+    else:
+        substrate_and_names = substrates
+
+    # Check prerequisite that we can standardize all substrates
+    standardized_substrates_and_smiles = []
+    for substrate in substrate_and_names:
+        try:
+            stand_smiles = FormatConverter.standardize(substrate[0], remove_stereo=True)
+            standardized_substrates_and_smiles.append([stand_smiles, substrate[1]])
+        except ValueError:
+            raise ValueError(
+                f'Pathway prediction failed as standardization of SMILES "{substrate}" failed!'
+            )
+
+    pathways = []
+
+    for pair in standardized_substrates_and_smiles:
+        pw = Pathway.create(
+            target_package,
+            pair[0],
+            name=pair[1],
+            predicted=True,
+        )
+
+        # set mode and setting
+        pw.setting = prediction_setting
+        pw.kv.update({"mode": "predict"})
+        pw.save()
+
+        predict(
+            pw.pk,
+            prediction_setting.pk,
+            limit=None,
+            setting_overrides={
+                "max_nodes": num_tps,
+                "max_depth": num_tps,
+                "model_threshold": 0.001,
+            },
+        )
+
+        pathways.append(pw)
+
+    buffer = io.StringIO()
+
+    for idx, pw in enumerate(pathways):
+        # Carry out header only for the first pathway
+        buffer.write(pw.to_csv(include_header=idx == 0, include_pathway_url=True))
+
+    buffer.seek(0)
+
+    return buffer.getvalue()

epdb/template_registry.py

@@ -0,0 +1,17 @@
+from collections import defaultdict
+from threading import Lock
+
+_registry = defaultdict(list)
+_lock = Lock()
+
+
+def register_template(slot: str, template_name: str, *, order: int = 100):
+    item = (order, template_name)
+    with _lock:
+        if item not in _registry[slot]:
+            _registry[slot].append(item)
+            _registry[slot].sort(key=lambda x: x[0])
+
+
+def get_templates(slot: str):
+    return [template_name for _, template_name in _registry.get(slot, [])]

epdb/templatetags/envipytags.py

@@ -2,6 +2,8 @@ from django import template
 from pydantic import AnyHttpUrl, ValidationError
 from pydantic.type_adapter import TypeAdapter
 
+from epdb.template_registry import get_templates
+
 register = template.Library()
 
 url_adapter = TypeAdapter(AnyHttpUrl)
@@ -19,3 +21,8 @@ def is_url(value):
         return True
     except ValidationError:
         return False
+
+
+@register.simple_tag
+def epdb_slot_templates(slot):
+    return get_templates(slot)

epdb/urls.py

@@ -49,6 +49,7 @@ urlpatterns = [
     re_path(r"^group$", v.groups, name="groups"),
     re_path(r"^search$", v.search, name="search"),
     re_path(r"^predict$", v.predict_pathway, name="predict_pathway"),
+    re_path(r"^batch-predict$", v.batch_predict_pathway, name="batch_predict_pathway"),
     # User Detail
     re_path(rf"^user/(?P<user_uuid>{UUID})", v.user, name="user"),
     # Group Detail
@@ -142,6 +143,11 @@ urlpatterns = [
         v.package_pathway,
         name="package pathway detail",
     ),
+    re_path(
+        rf"^package/(?P<package_uuid>{UUID})/predict$",
+        v.package_predict_pathway,
+        name="package predict pathway",
+    ),
     # Pathway Nodes
     re_path(
         rf"^package/(?P<package_uuid>{UUID})/pathway/(?P<pathway_uuid>{UUID})/node$",
@@ -191,7 +197,8 @@ urlpatterns = [
     re_path(r"^indigo/dearomatize$", v.dearomatize, name="indigo_dearomatize"),
     re_path(r"^indigo/layout$", v.layout, name="indigo_layout"),
     re_path(r"^depict$", v.depict, name="depict"),
-    re_path(r"^jobs", v.jobs, name="jobs"),
+    path("jobs", v.jobs, name="jobs"),
+    path("jobs/<uuid:job_uuid>", v.job, name="job detail"),
     # OAuth Stuff
     path("o/userinfo/", v.userinfo, name="oauth_userinfo"),
     # Static Pages
@@ -202,5 +209,4 @@ urlpatterns = [
     re_path(r"^contact$", v.static_contact_support, name="contact_support"),
     re_path(r"^careers$", v.static_careers, name="careers"),
     re_path(r"^cite$", v.static_cite, name="cite"),
-    re_path(r"^legal$", v.static_legal, name="legal"),
 ]

epiuclid/api.py

@@ -0,0 +1,22 @@
+from uuid import UUID
+
+from django.http import HttpResponse
+from ninja import Router
+from epapi.v1.interfaces.iuclid.projections import get_pathway_for_iuclid_export
+
+from .serializers.i6z import I6ZSerializer
+from .serializers.pathway_mapper import PathwayMapper
+
+router = Router(tags=["iuclid"])
+
+
+@router.get("/pathway/{uuid:pathway_uuid}/export/iuclid")
+def export_pathway_iuclid(request, pathway_uuid: UUID):
+    export = get_pathway_for_iuclid_export(request.user, pathway_uuid)
+    bundle = PathwayMapper().map(export)
+    i6z_bytes = I6ZSerializer().serialize(bundle)
+    return HttpResponse(
+        i6z_bytes,
+        content_type="application/zip",
+        headers={"Content-Disposition": f'attachment; filename="pathway-{pathway_uuid}.i6z"'},
+    )

epiuclid/apps.py

@@ -0,0 +1,6 @@
+from django.apps import AppConfig
+
+
+class EpiuclidConfig(AppConfig):
+    default_auto_field = "django.db.models.BigAutoField"
+    name = "epiuclid"