[Feature] Server pagination implementation (#243)

## Major Changes
- Implement a REST style API app in epapi
- Currently implements a GET method for all entity types in the browse menu (both package level and global)
- Provides paginated results per default with query style filtering for reviewed vs unreviewed.
- Provides new paginated templates with thin wrappers per entity types for easier maintainability
- Implements e2e tests for the API

## Minor changes
- Added more comprehensive gitignore to cover coverage reports and other test/node.js etc. data.
- Add additional CI file for API tests that only gets triggered on API relevant changes.

## ⚠️ Currently only works with session-based authentication. Token based will be added in new PR.

Co-authored-by: Tim Lorsbach <tim@lorsba.ch>
Co-authored-by: jebus <lorsbach@envipath.com>
Reviewed-on: enviPath/enviPy#243
Co-authored-by: Tobias O <tobias.olenyi@envipath.com>
Co-committed-by: Tobias O <tobias.olenyi@envipath.com>

epapi/v1/auth.py

@@ -0,0 +1,8 @@
+from ninja.security import HttpBearer
+from ninja.errors import HttpError
+
+
+class BearerTokenAuth(HttpBearer):
+    def authenticate(self, request, token):
+        # FIXME: placeholder; implement it in O(1) time
+        raise HttpError(401, "Invalid or expired token")

epapi/v1/dal.py

@@ -0,0 +1,95 @@
+from django.db.models import Model
+from epdb.logic import PackageManager
+from epdb.models import CompoundStructure, User, Package, Compound
+from uuid import UUID
+
+from .errors import EPAPINotFoundError, EPAPIPermissionDeniedError
+
+
+def get_compound_or_error(user, compound_uuid: UUID):
+    """
+    Get compound by UUID with permission check.
+    """
+    try:
+        compound = Compound.objects.get(uuid=compound_uuid)
+        package = compound.package
+    except Compound.DoesNotExist:
+        raise EPAPINotFoundError(f"Compound with UUID {compound_uuid} not found")
+
+    # FIXME: optimize package manager to exclusively work with UUIDs
+    if not user or user.is_anonymous or not PackageManager.readable(user, package):
+        raise EPAPIPermissionDeniedError("Insufficient permissions to access this compound.")
+
+    return compound
+
+
+def get_package_or_error(user, package_uuid: UUID):
+    """
+    Get package by UUID with permission check.
+    """
+
+    # FIXME: update package manager with custom exceptions to avoid manual checks here
+    try:
+        package = Package.objects.get(uuid=package_uuid)
+    except Package.DoesNotExist:
+        raise EPAPINotFoundError(f"Package with UUID {package_uuid} not found")
+
+    # FIXME: optimize package manager to exclusively work with UUIDs
+    if not user or user.is_anonymous or not PackageManager.readable(user, package):
+        raise EPAPIPermissionDeniedError("Insufficient permissions to access this package.")
+
+    return package
+
+
+def get_user_packages_qs(user: User | None):
+    """Get all packages readable by the user."""
+    if not user or user.is_anonymous:
+        return PackageManager.get_reviewed_packages()
+    return PackageManager.get_all_readable_packages(user, include_reviewed=True)
+
+
+def get_user_entities_qs(model_class: Model, user: User | None):
+    """Build queryset for reviewed package entities."""
+
+    if not user or user.is_anonymous:
+        return model_class.objects.filter(package__reviewed=True).select_related("package")
+
+    qs = model_class.objects.filter(
+        package__in=PackageManager.get_all_readable_packages(user, include_reviewed=True)
+    ).select_related("package")
+    return qs
+
+
+def get_package_scoped_entities_qs(
+    model_class: Model, package_uuid: UUID, user: User | None = None
+):
+    """Build queryset for specific package entities."""
+    package = get_package_or_error(user, package_uuid)
+    qs = model_class.objects.filter(package=package).select_related("package")
+    return qs
+
+
+def get_user_structures_qs(user: User | None):
+    """Build queryset for structures accessible to the user (via compound->package)."""
+
+    if not user or user.is_anonymous:
+        return CompoundStructure.objects.filter(compound__package__reviewed=True).select_related(
+            "compound__package"
+        )
+
+    qs = CompoundStructure.objects.filter(
+        compound__package__in=PackageManager.get_all_readable_packages(user, include_reviewed=True)
+    ).select_related("compound__package")
+    return qs
+
+
+def get_package_compound_scoped_structure_qs(
+    package_uuid: UUID, compound_uuid: UUID, user: User | None = None
+):
+    """Build queryset for specific package compound structures."""
+
+    get_package_or_error(user, package_uuid)
+    compound = get_compound_or_error(user, compound_uuid)
+
+    qs = CompoundStructure.objects.filter(compound=compound).select_related("compound__package")
+    return qs

epapi/v1/endpoints/compounds.py

@@ -0,0 +1,41 @@
+from django.conf import settings as s
+from ninja import Router
+from ninja_extra.pagination import paginate
+from uuid import UUID
+
+from epdb.models import Compound
+from ..pagination import EnhancedPageNumberPagination
+from ..schemas import CompoundOutSchema, ReviewStatusFilter
+from ..dal import get_user_entities_qs, get_package_scoped_entities_qs
+
+router = Router()
+
+
+@router.get("/compounds/", response=EnhancedPageNumberPagination.Output[CompoundOutSchema])
+@paginate(
+    EnhancedPageNumberPagination,
+    page_size=s.API_PAGINATION_DEFAULT_PAGE_SIZE,
+    filter_schema=ReviewStatusFilter,
+)
+def list_all_compounds(request):
+    """
+    List all compounds from reviewed packages.
+    """
+    return get_user_entities_qs(Compound, request.user).order_by("name").all()
+
+
+@router.get(
+    "/package/{uuid:package_uuid}/compound/",
+    response=EnhancedPageNumberPagination.Output[CompoundOutSchema],
+)
+@paginate(
+    EnhancedPageNumberPagination,
+    page_size=s.API_PAGINATION_DEFAULT_PAGE_SIZE,
+    filter_schema=ReviewStatusFilter,
+)
+def list_package_compounds(request, package_uuid: UUID):
+    """
+    List all compounds for a specific package.
+    """
+    user = request.user
+    return get_package_scoped_entities_qs(Compound, package_uuid, user).order_by("name").all()

epapi/v1/endpoints/models.py

@@ -0,0 +1,41 @@
+from django.conf import settings as s
+from ninja import Router
+from ninja_extra.pagination import paginate
+from uuid import UUID
+
+from epdb.models import EPModel
+from ..pagination import EnhancedPageNumberPagination
+from ..schemas import ModelOutSchema, ReviewStatusFilter
+from ..dal import get_user_entities_qs, get_package_scoped_entities_qs
+
+router = Router()
+
+
+@router.get("/models/", response=EnhancedPageNumberPagination.Output[ModelOutSchema])
+@paginate(
+    EnhancedPageNumberPagination,
+    page_size=s.API_PAGINATION_DEFAULT_PAGE_SIZE,
+    filter_schema=ReviewStatusFilter,
+)
+def list_all_models(request):
+    """
+    List all models from reviewed packages.
+    """
+    return get_user_entities_qs(EPModel, request.user).order_by("name").all()
+
+
+@router.get(
+    "/package/{uuid:package_uuid}/model/",
+    response=EnhancedPageNumberPagination.Output[ModelOutSchema],
+)
+@paginate(
+    EnhancedPageNumberPagination,
+    page_size=s.API_PAGINATION_DEFAULT_PAGE_SIZE,
+    filter_schema=ReviewStatusFilter,
+)
+def list_package_models(request, package_uuid: UUID):
+    """
+    List all models for a specific package.
+    """
+    user = request.user
+    return get_package_scoped_entities_qs(EPModel, package_uuid, user).order_by("name").all()

epapi/v1/endpoints/packages.py

@@ -0,0 +1,27 @@
+from django.conf import settings as s
+from ninja import Router
+from ninja_extra.pagination import paginate
+import logging
+
+from ..dal import get_user_packages_qs
+from ..pagination import EnhancedPageNumberPagination
+from ..schemas import PackageOutSchema, SelfReviewStatusFilter
+
+router = Router()
+logger = logging.getLogger(__name__)
+
+
+@router.get("/packages/", response=EnhancedPageNumberPagination.Output[PackageOutSchema], auth=None)
+@paginate(
+    EnhancedPageNumberPagination,
+    page_size=s.API_PAGINATION_DEFAULT_PAGE_SIZE,
+    filter_schema=SelfReviewStatusFilter,
+)
+def list_all_packages(request):
+    """
+    List packages accessible to the user.
+
+    """
+    user = request.user
+    qs = get_user_packages_qs(user)
+    return qs.order_by("name").all()

epapi/v1/endpoints/pathways.py

@@ -0,0 +1,42 @@
+from django.conf import settings as s
+from ninja import Router
+from ninja_extra.pagination import paginate
+from uuid import UUID
+
+from epdb.models import Pathway
+from ..pagination import EnhancedPageNumberPagination
+from ..schemas import PathwayOutSchema, ReviewStatusFilter
+from ..dal import get_user_entities_qs, get_package_scoped_entities_qs
+
+router = Router()
+
+
+@router.get("/pathways/", response=EnhancedPageNumberPagination.Output[PathwayOutSchema])
+@paginate(
+    EnhancedPageNumberPagination,
+    page_size=s.API_PAGINATION_DEFAULT_PAGE_SIZE,
+    filter_schema=ReviewStatusFilter,
+)
+def list_all_pathways(request):
+    """
+    List all pathways from reviewed packages.
+    """
+    user = request.user
+    return get_user_entities_qs(Pathway, user).order_by("name").all()
+
+
+@router.get(
+    "/package/{uuid:package_uuid}/pathway/",
+    response=EnhancedPageNumberPagination.Output[PathwayOutSchema],
+)
+@paginate(
+    EnhancedPageNumberPagination,
+    page_size=s.API_PAGINATION_DEFAULT_PAGE_SIZE,
+    filter_schema=ReviewStatusFilter,
+)
+def list_package_pathways(request, package_uuid: UUID):
+    """
+    List all pathways for a specific package.
+    """
+    user = request.user
+    return get_package_scoped_entities_qs(Pathway, package_uuid, user).order_by("name").all()

epapi/v1/endpoints/reactions.py

@@ -0,0 +1,42 @@
+from django.conf import settings as s
+from ninja import Router
+from ninja_extra.pagination import paginate
+from uuid import UUID
+
+from epdb.models import Reaction
+from ..pagination import EnhancedPageNumberPagination
+from ..schemas import ReactionOutSchema, ReviewStatusFilter
+from ..dal import get_user_entities_qs, get_package_scoped_entities_qs
+
+router = Router()
+
+
+@router.get("/reactions/", response=EnhancedPageNumberPagination.Output[ReactionOutSchema])
+@paginate(
+    EnhancedPageNumberPagination,
+    page_size=s.API_PAGINATION_DEFAULT_PAGE_SIZE,
+    filter_schema=ReviewStatusFilter,
+)
+def list_all_reactions(request):
+    """
+    List all reactions from reviewed packages.
+    """
+    user = request.user
+    return get_user_entities_qs(Reaction, user).order_by("name").all()
+
+
+@router.get(
+    "/package/{uuid:package_uuid}/reaction/",
+    response=EnhancedPageNumberPagination.Output[ReactionOutSchema],
+)
+@paginate(
+    EnhancedPageNumberPagination,
+    page_size=s.API_PAGINATION_DEFAULT_PAGE_SIZE,
+    filter_schema=ReviewStatusFilter,
+)
+def list_package_reactions(request, package_uuid: UUID):
+    """
+    List all reactions for a specific package.
+    """
+    user = request.user
+    return get_package_scoped_entities_qs(Reaction, package_uuid, user).order_by("name").all()

epapi/v1/endpoints/rules.py

@@ -0,0 +1,42 @@
+from django.conf import settings as s
+from ninja import Router
+from ninja_extra.pagination import paginate
+from uuid import UUID
+
+from epdb.models import Rule
+from ..pagination import EnhancedPageNumberPagination
+from ..schemas import ReviewStatusFilter, RuleOutSchema
+from ..dal import get_user_entities_qs, get_package_scoped_entities_qs
+
+router = Router()
+
+
+@router.get("/rules/", response=EnhancedPageNumberPagination.Output[RuleOutSchema])
+@paginate(
+    EnhancedPageNumberPagination,
+    page_size=s.API_PAGINATION_DEFAULT_PAGE_SIZE,
+    filter_schema=ReviewStatusFilter,
+)
+def list_all_rules(request):
+    """
+    List all rules from reviewed packages.
+    """
+    user = request.user
+    return get_user_entities_qs(Rule, user).order_by("name").all()
+
+
+@router.get(
+    "/package/{uuid:package_uuid}/rule/",
+    response=EnhancedPageNumberPagination.Output[RuleOutSchema],
+)
+@paginate(
+    EnhancedPageNumberPagination,
+    page_size=s.API_PAGINATION_DEFAULT_PAGE_SIZE,
+    filter_schema=ReviewStatusFilter,
+)
+def list_package_rules(request, package_uuid: UUID):
+    """
+    List all rules for a specific package.
+    """
+    user = request.user
+    return get_package_scoped_entities_qs(Rule, package_uuid, user).order_by("name").all()

epapi/v1/endpoints/scenarios.py

@@ -0,0 +1,36 @@
+from django.conf import settings as s
+from ninja import Router
+from ninja_extra.pagination import paginate
+from uuid import UUID
+
+from epdb.models import Scenario
+from ..pagination import EnhancedPageNumberPagination
+from ..schemas import ReviewStatusFilter, ScenarioOutSchema
+from ..dal import get_user_entities_qs, get_package_scoped_entities_qs
+
+router = Router()
+
+
+@router.get("/scenarios/", response=EnhancedPageNumberPagination.Output[ScenarioOutSchema])
+@paginate(
+    EnhancedPageNumberPagination,
+    page_size=s.API_PAGINATION_DEFAULT_PAGE_SIZE,
+    filter_schema=ReviewStatusFilter,
+)
+def list_all_scenarios(request):
+    user = request.user
+    return get_user_entities_qs(Scenario, user).order_by("name").all()
+
+
+@router.get(
+    "/package/{uuid:package_uuid}/scenario/",
+    response=EnhancedPageNumberPagination.Output[ScenarioOutSchema],
+)
+@paginate(
+    EnhancedPageNumberPagination,
+    page_size=s.API_PAGINATION_DEFAULT_PAGE_SIZE,
+    filter_schema=ReviewStatusFilter,
+)
+def list_package_scenarios(request, package_uuid: UUID):
+    user = request.user
+    return get_package_scoped_entities_qs(Scenario, package_uuid, user).order_by("name").all()

epapi/v1/endpoints/structure.py

@@ -0,0 +1,50 @@
+from django.conf import settings as s
+from ninja import Router
+from ninja_extra.pagination import paginate
+from uuid import UUID
+
+from ..pagination import EnhancedPageNumberPagination
+from ..schemas import CompoundStructureOutSchema, StructureReviewStatusFilter
+from ..dal import (
+    get_user_structures_qs,
+    get_package_compound_scoped_structure_qs,
+)
+
+router = Router()
+
+
+@router.get(
+    "/structures/", response=EnhancedPageNumberPagination.Output[CompoundStructureOutSchema]
+)
+@paginate(
+    EnhancedPageNumberPagination,
+    page_size=s.API_PAGINATION_DEFAULT_PAGE_SIZE,
+    filter_schema=StructureReviewStatusFilter,
+)
+def list_all_structures(request):
+    """
+    List all structures from all packages.
+    """
+    user = request.user
+    return get_user_structures_qs(user).order_by("name").all()
+
+
+@router.get(
+    "/package/{uuid:package_uuid}/compound/{uuid:compound_uuid}/structure/",
+    response=EnhancedPageNumberPagination.Output[CompoundStructureOutSchema],
+)
+@paginate(
+    EnhancedPageNumberPagination,
+    page_size=s.API_PAGINATION_DEFAULT_PAGE_SIZE,
+    filter_schema=StructureReviewStatusFilter,
+)
+def list_package_structures(request, package_uuid: UUID, compound_uuid: UUID):
+    """
+    List all structures for a specific package and compound.
+    """
+    user = request.user
+    return (
+        get_package_compound_scoped_structure_qs(package_uuid, compound_uuid, user)
+        .order_by("name")
+        .all()
+    )

epapi/v1/errors.py

@@ -0,0 +1,28 @@
+from ninja.errors import HttpError
+
+
+class EPAPIError(HttpError):
+    status_code: int = 500
+
+    def __init__(self, message: str) -> None:
+        super().__init__(status_code=self.status_code, message=message)
+
+    @classmethod
+    def from_exception(cls, exc: Exception):
+        return cls(message=str(exc))
+
+
+class EPAPIUnauthorizedError(EPAPIError):
+    status_code = 401
+
+
+class EPAPIPermissionDeniedError(EPAPIError):
+    status_code = 403
+
+
+class EPAPINotFoundError(EPAPIError):
+    status_code = 404
+
+
+class EPAPIValidationError(EPAPIError):
+    status_code = 422

epapi/v1/pagination.py

@@ -0,0 +1,60 @@
+import math
+from typing import Any, Generic, List, TypeVar
+
+from django.db.models import QuerySet
+from ninja import Schema
+from ninja.pagination import PageNumberPagination
+
+T = TypeVar("T")
+
+
+class EnhancedPageNumberPagination(PageNumberPagination):
+    class Output(Schema, Generic[T]):
+        items: List[T]
+        page: int
+        page_size: int
+        total_items: int
+        total_pages: int
+
+    def paginate_queryset(
+        self,
+        queryset: QuerySet,
+        pagination: PageNumberPagination.Input,
+        **params: Any,
+    ) -> Any:
+        page_size = self._get_page_size(pagination.page_size)
+        offset = (pagination.page - 1) * page_size
+        total_items = self._items_count(queryset)
+        total_pages = math.ceil(total_items / page_size) if page_size > 0 else 0
+
+        return {
+            "items": queryset[offset : offset + page_size],
+            "page": pagination.page,
+            "page_size": page_size,
+            "total_items": total_items,
+            "total_pages": total_pages,
+        }
+
+    async def apaginate_queryset(
+        self,
+        queryset: QuerySet,
+        pagination: PageNumberPagination.Input,
+        **params: Any,
+    ) -> Any:
+        page_size = self._get_page_size(pagination.page_size)
+        offset = (pagination.page - 1) * page_size
+        total_items = await self._aitems_count(queryset)
+        total_pages = math.ceil(total_items / page_size) if page_size > 0 else 0
+
+        if isinstance(queryset, QuerySet):
+            items = [obj async for obj in queryset[offset : offset + page_size]]
+        else:
+            items = queryset[offset : offset + page_size]
+
+        return {
+            "items": items,
+            "page": pagination.page,
+            "page_size": page_size,
+            "total_items": total_items,
+            "total_pages": total_pages,
+        }

epapi/v1/router.py

@@ -0,0 +1,22 @@
+from ninja import Router
+from ninja.security import SessionAuth
+from .auth import BearerTokenAuth
+from .endpoints import packages, scenarios, compounds, rules, reactions, pathways, models, structure
+
+# Main router with authentication
+router = Router(
+    auth=[
+        SessionAuth(),
+        BearerTokenAuth(),
+    ]
+)
+
+# Include all endpoint routers
+router.add_router("", packages.router)
+router.add_router("", scenarios.router)
+router.add_router("", compounds.router)
+router.add_router("", rules.router)
+router.add_router("", reactions.router)
+router.add_router("", pathways.router)
+router.add_router("", models.router)
+router.add_router("", structure.router)

epapi/v1/schemas.py

@@ -0,0 +1,104 @@
+from ninja import FilterSchema, FilterLookup, Schema
+from typing import Annotated, Optional
+from uuid import UUID
+
+
+# Filter schema for query parameters
+class ReviewStatusFilter(FilterSchema):
+    """Filter schema for review_status query parameter."""
+
+    review_status: Annotated[Optional[bool], FilterLookup("package__reviewed")] = None
+
+
+class SelfReviewStatusFilter(FilterSchema):
+    """Filter schema for review_status query parameter on self-reviewed entities."""
+
+    review_status: Annotated[Optional[bool], FilterLookup("reviewed")] = None
+
+
+class StructureReviewStatusFilter(FilterSchema):
+    """Filter schema for review_status on structures (via compound->package)."""
+
+    review_status: Annotated[Optional[bool], FilterLookup("compound__package__reviewed")] = None
+
+
+# Base schema for all package-scoped entities
+class PackageEntityOutSchema(Schema):
+    """Base schema for entities belonging to a package."""
+
+    uuid: UUID
+    url: str = ""
+    name: str
+    description: str
+    review_status: str = ""
+    package: str = ""
+
+    @staticmethod
+    def resolve_url(obj):
+        return obj.url
+
+    @staticmethod
+    def resolve_package(obj):
+        return obj.package.url
+
+    @staticmethod
+    def resolve_review_status(obj):
+        return "reviewed" if obj.package.reviewed else "unreviewed"
+
+
+# All package-scoped entities inherit from base
+class ScenarioOutSchema(PackageEntityOutSchema):
+    pass
+
+
+class CompoundOutSchema(PackageEntityOutSchema):
+    pass
+
+
+class RuleOutSchema(PackageEntityOutSchema):
+    pass
+
+
+class ReactionOutSchema(PackageEntityOutSchema):
+    pass
+
+
+class PathwayOutSchema(PackageEntityOutSchema):
+    pass
+
+
+class ModelOutSchema(PackageEntityOutSchema):
+    pass
+
+
+class CompoundStructureOutSchema(PackageEntityOutSchema):
+    compound: str = ""
+
+    @staticmethod
+    def resolve_compound(obj):
+        return obj.compound.url
+
+    @staticmethod
+    def resolve_package(obj):
+        return obj.compound.package.url
+
+    @staticmethod
+    def resolve_review_status(obj):
+        return "reviewed" if obj.compound.package.reviewed else "unreviewed"
+
+
+# Package is special (no package FK)
+class PackageOutSchema(Schema):
+    uuid: UUID
+    url: str = ""
+    name: str
+    description: str
+    review_status: str = ""
+
+    @staticmethod
+    def resolve_url(obj):
+        return obj.url
+
+    @staticmethod
+    def resolve_review_status(obj):
+        return "reviewed" if obj.reviewed else "unreviewed"