From 2babe7f7e2ff22b6f6dfba6ddacae34099bf995a Mon Sep 17 00:00:00 2001 From: jebus Date: Tue, 2 Sep 2025 08:06:18 +1200 Subject: [PATCH] [Feature] Scenario Creation (#78) Co-authored-by: Tim Lorsbach Reviewed-on: https://git.envipath.com/enviPath/enviPy/pulls/78 --- epdb/logic.py | 7 +- epdb/models.py | 96 ++++++++-- epdb/views.py | 113 +++++++++++- pyproject.toml | 2 +- templates/actions/collections/scenario.html | 2 +- templates/actions/objects/scenario.html | 12 ++ .../collections/new_scenario_modal.html | 146 +++++++-------- .../add_additional_information_modal.html | 61 +++++++ ...scenario_additional_information_modal.html | 39 ++++ templates/objects/scenario.html | 44 ++++- templates/tables/scenario.html | 72 -------- utilities/dataclasses.py | 0 utilities/misc.py | 168 ++++++++++++++++++ uv.lock | 4 +- 14 files changed, 583 insertions(+), 183 deletions(-) create mode 100644 templates/modals/objects/add_additional_information_modal.html create mode 100644 templates/modals/objects/update_scenario_additional_information_modal.html delete mode 100644 templates/tables/scenario.html delete mode 100644 utilities/dataclasses.py create mode 100644 utilities/misc.py diff --git a/epdb/logic.py b/epdb/logic.py index a5f7ded4..6d5b930f 100644 --- a/epdb/logic.py +++ b/epdb/logic.py @@ -1,5 +1,6 @@ import re import logging +import json from typing import Union, List, Optional, Set, Dict, Any from django.contrib.auth import get_user_model @@ -552,11 +553,13 @@ class PackageManager(object): try: res = AdditionalInformationConverter.convert(name, addinf_data) + res_cls_name = res.__class__.__name__ + ai_data = json.loads(res.model_dump_json()) + ai_data['uuid'] = f"{uuid4()}" + new_add_inf[res_cls_name].append(ai_data) except: logger.error(f"Failed to convert {name} with {addinf_data}") - new_add_inf[name].append(res.model_dump_json()) - scen.additional_information = new_add_inf scen.save() diff --git a/epdb/models.py b/epdb/models.py index 6ce37918..37d73c61 100644 --- a/epdb/models.py +++ b/epdb/models.py @@ -20,6 +20,7 @@ from django.db import models, transaction from django.db.models import JSONField, Count, Q, QuerySet from django.utils import timezone from django.utils.functional import cached_property +from envipy_additional_information import EnviPyModel from model_utils.models import TimeStampedModel from polymorphic.models import PolymorphicModel from sklearn.metrics import precision_score, recall_score, jaccard_score @@ -2292,7 +2293,8 @@ class ApplicabilityDomain(EnviPathModel): transformation = { 'rule': rule_data, 'reliability': rule_reliabilities[rule_idx], - # TODO + # We're setting it here to False, as we don't know whether "assess" is called during pathway + # prediction or from Model Page. For persisted Nodes this field will be overwritten at runtime 'is_predicted': False, 'local_compatibility': local_compatibilities[rule_idx], 'probability': preds[rule_idx].probability, @@ -2407,27 +2409,88 @@ class Scenario(EnviPathModel): @staticmethod @transaction.atomic - def create(package, name, description, date, type, additional_information): + def create(package: 'Package', name:str, description:str, scenario_date:str, scenario_type:str, additional_information: List['EnviPyModel']): s = Scenario() s.package = package + + if name is None or name.strip() == '': + name = f"Scenario {Scenario.objects.filter(package=package).count() + 1}" + s.name = name - s.description = description - s.date = date - s.type = type - s.additional_information = additional_information + + if description is not None and description.strip() != '': + s.description = description + + if scenario_date is not None and scenario_date.strip() != '': + s.scenario_date = scenario_date + + if scenario_type is not None and scenario_type.strip() != '': + s.scenario_type = scenario_type + + add_inf = defaultdict(list) + + for info in additional_information: + cls_name = info.__class__.__name__ + ai_data = json.loads(info.model_dump_json()) + ai_data['uuid'] = f"{uuid4()}" + add_inf[cls_name].append(ai_data) + + + s.additional_information = add_inf s.save() return s - def add_additional_information(self, data): - pass + @transaction.atomic + def add_additional_information(self, data: 'EnviPyModel'): + cls_name = data.__class__.__name__ + ai_data = json.loads(data.model_dump_json()) + ai_data['uuid'] = f"{uuid4()}" - def remove_additional_information(self, data): - pass + if cls_name not in self.additional_information: + self.additional_information[cls_name] = [] - def set_additional_information(self, data): - pass + self.additional_information[cls_name].append(ai_data) + self.save() + + + @transaction.atomic + def remove_additional_information(self, ai_uuid): + found_type = None + found_idx = -1 + + for k, vals in self.additional_information.items(): + for i, v in enumerate(vals): + if v['uuid'] == ai_uuid: + found_type = k + found_idx = i + break + + if found_type is not None and found_idx >= 0: + if len(self.additional_information[found_type]) == 1: + del self.additional_information[k] + else: + self.additional_information[k].pop(found_idx) + self.save() + else: + raise ValueError(f"Could not find additional information with uuid {ai_uuid}") + + @transaction.atomic + def set_additional_information(self, data: Dict[str, 'EnviPyModel']): + new_ais = defaultdict(list) + for k, vals in data.items(): + for v in vals: + ai_data = json.loads(v.model_dump_json()) + if hasattr(v, 'uuid'): + ai_data['uuid'] = str(v.uuid) + else: + ai_data['uuid'] = str(uuid4()) + + new_ais[k].append(ai_data) + + self.additional_information = new_ais + self.save() def get_additional_information(self): from envipy_additional_information import NAME_MAPPING @@ -2437,7 +2500,14 @@ class Scenario(EnviPathModel): continue for v in vals: - yield NAME_MAPPING[k](**json.loads(v)) + # Per default additional fields are ignored + MAPPING = {c.__name__: c for c in NAME_MAPPING.values()} + inst = MAPPING[k](**v) + # Add uuid to uniquely identify objects for manipulation + if 'uuid' in v: + inst.__dict__['uuid'] = v['uuid'] + + yield inst class UserSettingPermission(Permission): diff --git a/epdb/views.py b/epdb/views.py index 72ff30b8..d9c8fed9 100644 --- a/epdb/views.py +++ b/epdb/views.py @@ -4,15 +4,14 @@ from typing import List, Dict, Any from django.conf import settings as s from django.contrib.auth import get_user_model -from django.db.models import F, Value -from django.db.models.fields import CharField -from django.db.models.functions import Concat from django.http import JsonResponse, HttpResponse, HttpResponseNotAllowed, HttpResponseBadRequest from django.shortcuts import render, redirect from django.views.decorators.csrf import csrf_exempt +from envipy_additional_information import NAME_MAPPING from utilities.chem import FormatConverter, IndigoUtils from utilities.decorators import package_permission_required +from utilities.misc import HTMLGenerator from .logic import GroupManager, PackageManager, UserManager, SettingManager, SearchManager, EPDBURLParser from .models import Package, GroupPackagePermission, Group, CompoundStructure, Compound, Reaction, Rule, Pathway, Node, \ EPModel, EnviFormer, MLRelativeReasoning, RuleBaseRelativeReasoning, Scenario, SimpleAmbitRule, APIToken, \ @@ -1726,7 +1725,7 @@ def package_scenarios(request, package_uuid): if request.method == 'GET': - if 'application/json' in request.META.get('HTTP_ACCEPT') and not request.GET.get('all', None): + if 'application/json' in request.META.get('HTTP_ACCEPT') and not request.GET.get('all', False): scens = Scenario.objects.filter(package=current_package).order_by('name') res = [{'name': s.name, 'url': s.url, 'uuid': s.uuid} for s in scens] return JsonResponse(res, safe=False) @@ -1757,8 +1756,57 @@ def package_scenarios(request, package_uuid): context['reviewed_objects'] = reviewed_scenario_qs context['unreviewed_objects'] = unreviewed_scenario_qs - return render(request, 'collections/objects_list.html', context) + from envipy_additional_information import SLUDGE_ADDITIONAL_INFORMATION, SOIL_ADDITIONAL_INFORMATION, \ + SEDIMENT_ADDITIONAL_INFORMATION + context['scenario_types'] = { + 'Soil Data': { + 'name': 'soil', + 'widgets': [HTMLGenerator.generate_html(ai, prefix=f'soil_{0}') for ai in + [x for s in SOIL_ADDITIONAL_INFORMATION.values() for x in s]] + }, + 'Sludge Data': { + 'name': 'sludge', + 'widgets': [HTMLGenerator.generate_html(ai, prefix=f'sludge_{0}') for ai in + [x for s in SLUDGE_ADDITIONAL_INFORMATION.values() for x in s]] + }, + 'Water-Sediment System Data': { + 'name': 'sediment', + 'widgets': [HTMLGenerator.generate_html(ai, prefix=f'sediment_{0}') for ai in + [x for s in SEDIMENT_ADDITIONAL_INFORMATION.values() for x in s]] + } + } + context['sludge_additional_information'] = SLUDGE_ADDITIONAL_INFORMATION + context['soil_additional_information'] = SOIL_ADDITIONAL_INFORMATION + context['sediment_additional_information'] = SEDIMENT_ADDITIONAL_INFORMATION + + return render(request, 'collections/objects_list.html', context) + elif request.method == 'POST': + + log_post_params(request) + + scenario_name = request.POST.get('scenario-name') + scenario_description = request.POST.get('scenario-description') + scenario_date_year = request.POST.get('scenario-date-year') + scenario_date_month = request.POST.get('scenario-date-month') + scenario_date_day = request.POST.get('scenario-date-day') + + scenario_date = scenario_date_year + if scenario_date_month is not None and scenario_date_month.strip() != '': + scenario_date += f'-{int(scenario_date_month):02d}' + if scenario_date_day is not None and scenario_date_day.strip() != '': + scenario_date += f'-{int(scenario_date_day):02d}' + + scenario_type = request.POST.get('scenario-type') + + additional_information = HTMLGenerator.build_models(request.POST.dict()) + additional_information = [x for s in additional_information.values() for x in s] + + s = Scenario.create(current_package, name=scenario_name, description=scenario_description, + scenario_date=scenario_date, scenario_type=scenario_type, + additional_information=additional_information) + + return redirect(s.url) else: return HttpResponseNotAllowed(['GET', ]) @@ -1779,10 +1827,63 @@ def package_scenario(request, package_uuid, scenario_uuid): context['scenario'] = current_scenario + available_add_infs = [] + for add_inf in NAME_MAPPING.values(): + available_add_infs.append({ + 'display_name': add_inf.property_name(None), + 'name': add_inf.__name__, + 'widget': HTMLGenerator.generate_html(add_inf, prefix=f'{0}') + }) + context['available_additional_information'] = available_add_infs + + context['update_widgets'] = [HTMLGenerator.generate_html(ai, prefix=f'{i}') for i, ai in enumerate(current_scenario.get_additional_information())] + return render(request, 'objects/scenario.html', context) + elif request.method == 'POST': + + log_post_params(request) + + if hidden := request.POST.get('hidden', None): + + if hidden == 'delete': + current_scenario.delete() + return redirect(current_package.url + '/scenario') + elif hidden == 'delete-additional-information': + uuid = request.POST.get('uuid') + current_scenario.remove_additional_information(uuid) + return redirect(current_scenario.url) + elif hidden == 'delete-all-additional-information': + current_scenario.additional_information = dict() + current_scenario.save() + return redirect(current_scenario.url) + elif hidden == 'set-additional-information': + ais = HTMLGenerator.build_models(request.POST.dict()) + + if s.DEBUG: + logger.info(ais) + + current_scenario.set_additional_information(ais) + return redirect(current_scenario.url) + elif hidden == 'add-additional-information': + ais = HTMLGenerator.build_models(request.POST.dict()) + + if len(ais.keys()) != 1: + raise ValueError('Only one additional information field can be added at a time.') + + ai = list(ais.values())[0][0] + + if s.DEBUG: + logger.info(ais) + + current_scenario.add_additional_information(ai) + return redirect(current_scenario.url) + else: + return HttpResponseBadRequest() + else: + return HttpResponseBadRequest() else: - return HttpResponseNotAllowed(['GET', ]) + return HttpResponseNotAllowed(['GET', 'POST']) ############## diff --git a/pyproject.toml b/pyproject.toml index c1d11d46..0402bbae 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -29,4 +29,4 @@ dependencies = [ [tool.uv.sources] enviformer = { git = "ssh://git@git.envipath.com/enviPath/enviformer.git", rev = "v0.1.0" } envipy-plugins = { git = "ssh://git@git.envipath.com/enviPath/enviPy-plugins.git", rev = "v0.1.0" } -envipy-additional-information = { git = "ssh://git@git.envipath.com/enviPath/enviPy-additional-information.git" } +envipy-additional-information = { git = "ssh://git@git.envipath.com/enviPath/enviPy-additional-information.git", rev = "v0.1.4"} diff --git a/templates/actions/collections/scenario.html b/templates/actions/collections/scenario.html index f9ccc08e..25b46b7f 100644 --- a/templates/actions/collections/scenario.html +++ b/templates/actions/collections/scenario.html @@ -1,6 +1,6 @@ {% if meta.can_edit %}
  • - + New Scenario
    • {% endif %} \ No newline at end of file diff --git a/templates/actions/objects/scenario.html b/templates/actions/objects/scenario.html index c9ddab01..ddc79ed4 100644 --- a/templates/actions/objects/scenario.html +++ b/templates/actions/objects/scenario.html @@ -1,2 +1,14 @@ {% if meta.can_edit %} +
  • + + Add Additional Information +
    • +
  • + + Set Additional Information +
    • +
  • + + Delete Scenario +
    • {% endif %} \ No newline at end of file diff --git a/templates/modals/collections/new_scenario_modal.html b/templates/modals/collections/new_scenario_modal.html index 9f2c8343..1a8d4266 100644 --- a/templates/modals/collections/new_scenario_modal.html +++ b/templates/modals/collections/new_scenario_modal.html @@ -1,115 +1,95 @@ - + +
    +
    Description
    +
    + {{ scenario.description }} +
    + {{ scenario.scenario_type }} +
    + Reported {{ scenario.scenario_date }} +
    +
    +
    @@ -31,18 +45,42 @@ + {% if meta.can_edit %} + {% endif %} {% for ai in scenario.get_additional_information %} - + - + {% if meta.can_edit %} + + {% endif %} {% endfor %} - + {% if meta.can_edit %} + + + + + + + {% endif %}
    Property Value UnitRemove
    {{ ai.property_name|safe }} {{ ai.property_name|safe }} {{ ai.property_data|safe }} {{ ai.property_unit|safe }} +
    + {% csrf_token %} + + + +
    +
    Delete all +
    + {% csrf_token %} + + +
    +
    diff --git a/templates/tables/scenario.html b/templates/tables/scenario.html deleted file mode 100644 index 62f5df53..00000000 --- a/templates/tables/scenario.html +++ /dev/null @@ -1,72 +0,0 @@ -{% for obj in available_additional_information.types %} -
    - - - - - - - {% for ai in available_additional_information.ais %} - - {% if obj.type in ai.types and ai.sub_type is not defined %} - - - {% for c in "1 2 3"|make_list %} - - {% endfor %} - {% endif %} - {% endfor %} - - {% for subtype in available_additional_information.subtypes %} - - - - - {% for ai in available_additional_information.ais %} - - {% if obj.type in ai.types and subtype.type == ai.sub_type %} - - - {% for c in "1 2 3"|make_list %} - - {% endfor %} - {% endif %} - - {% endfor %} - {% endfor %} - -

    {{obj.title}}

    {{ ai.name }} - {% for form in ai.forms %} - - {% if form.type == 'select' %} - - {% else %} - - {% endif %} - {% endfor %} -

    {{subtype.title}}

    {{ ai.name }} - {% for form in ai.forms %} - - {% if form.type == 'select' %} - - {% else %} - - {% endif %} - {% endfor %} -
    -
    -{% endfor %} diff --git a/utilities/dataclasses.py b/utilities/dataclasses.py deleted file mode 100644 index e69de29b..00000000 diff --git a/utilities/misc.py b/utilities/misc.py new file mode 100644 index 00000000..9933f20a --- /dev/null +++ b/utilities/misc.py @@ -0,0 +1,168 @@ +import html +import logging +from collections import defaultdict +from enum import Enum +from types import NoneType +from typing import Dict, List, Any + +from envipy_additional_information import Interval, EnviPyModel +from envipy_additional_information import NAME_MAPPING +from pydantic import BaseModel, HttpUrl + +logger = logging.getLogger(__name__) + + +class HTMLGenerator: + registry = {x.__name__: x for x in NAME_MAPPING.values()} + + @staticmethod + def generate_html(additional_information: 'EnviPyModel', prefix='') -> str: + from typing import get_origin, get_args, Union + + if isinstance(additional_information, type): + clz_name = additional_information.__name__ + else: + clz_name = additional_information.__class__.__name__ + + widget = f'

    {clz_name}

    ' + + if hasattr(additional_information, 'uuid'): + uuid = additional_information.uuid + widget += f'' + + for name, field in additional_information.model_fields.items(): + value = getattr(additional_information, name, None) + full_name = f"{clz_name}__{prefix}__{name}" + annotation = field.annotation + base_type = get_origin(annotation) or annotation + + # Optional[Interval[float]] alias for Union[X, None] + if base_type is Union: + for arg in get_args(annotation): + if arg is not NoneType: + field_type = arg + break + else: + field_type = base_type + + is_interval_float = ( + field_type == Interval[float] or + str(field_type) == str(Interval[float]) or + 'Interval[float]' in str(field_type) + ) + + if is_interval_float: + widget += f""" +
    +
    + + +
    +
    + + +
    +
    + """ + elif issubclass(field_type, Enum): + options: str = '' + for e in field_type: + options += f'' + + widget += f""" +
    + + +
    + """ + else: + if field_type == str or field_type == HttpUrl: + input_type = 'text' + elif field_type == float or field_type == int: + input_type = 'number' + elif field_type == bool: + input_type = 'checkbox' + else: + raise ValueError(f"Could not parse field type {field_type} for {name}") + + value_to_use = value if value and field_type != bool else '' + + widget += f""" +
    + + +
    + """ + + return widget + "
    " + + @staticmethod + def build_models(params) -> Dict[str, List['EnviPyModel']]: + + def has_non_none(d): + """ + Recursively checks if any value in a (possibly nested) dict is not None. + """ + for value in d.values(): + if isinstance(value, dict): + if has_non_none(value): # recursive check + return True + elif value is not None: + return True + return False + + """ + Build Pydantic model instances from flattened HTML parameters. + + Args: + params: dict of {param_name: value}, e.g. form data + model_registry: mapping of class names (strings) to Pydantic model classes + + Returns: + dict: {ClassName: [list of model instances]} + """ + grouped: Dict[str, Dict[str, Dict[str, Any]]] = {} + + # Step 1: group fields by ClassName and Number + for key, value in params.items(): + if value == '': + value = None + + parts = key.split("__") + if len(parts) < 3: + continue # skip invalid keys + + class_name, number, *field_parts = parts + grouped.setdefault(class_name, {}).setdefault(number, {}) + + # handle nested fields like interval__start + target = grouped[class_name][number] + current = target + for p in field_parts[:-1]: + current = current.setdefault(p, {}) + current[field_parts[-1]] = value + + # Step 2: instantiate Pydantic models + instances: Dict[str, List[BaseModel]] = defaultdict(list) + for class_name, number_dict in grouped.items(): + model_cls = HTMLGenerator.registry.get(class_name) + + if not model_cls: + logger.info(f"Could not find model class for {class_name}") + continue + + for number, fields in number_dict.items(): + if not has_non_none(fields): + print(f"Skipping empty {class_name} {number} {fields}") + continue + + uuid = fields.pop('uuid', None) + instance = model_cls(**fields) + if uuid: + instance.__dict__['uuid'] = uuid + instances[class_name].append(instance) + + return instances diff --git a/uv.lock b/uv.lock index b07eebad..88ff783a 100644 --- a/uv.lock +++ b/uv.lock @@ -426,7 +426,7 @@ requires-dist = [ { name = "django-ninja", specifier = ">=1.4.1" }, { name = "django-polymorphic", specifier = ">=4.1.0" }, { name = "enviformer", git = "ssh://git@git.envipath.com/enviPath/enviformer.git?rev=v0.1.0" }, - { name = "envipy-additional-information", git = "ssh://git@git.envipath.com/enviPath/enviPy-additional-information.git" }, + { name = "envipy-additional-information", git = "ssh://git@git.envipath.com/enviPath/enviPy-additional-information.git?rev=v0.1.4" }, { name = "envipy-plugins", git = "ssh://git@git.envipath.com/enviPath/enviPy-plugins.git?rev=v0.1.0" }, { name = "epam-indigo", specifier = ">=1.30.1" }, { name = "gunicorn", specifier = ">=23.0.0" }, @@ -443,7 +443,7 @@ requires-dist = [ [[package]] name = "envipy-additional-information" version = "0.1.0" -source = { git = "ssh://git@git.envipath.com/enviPath/enviPy-additional-information.git#4804b24b3479bed6108a49e4401bff8947c03cbd" } +source = { git = "ssh://git@git.envipath.com/enviPath/enviPy-additional-information.git?rev=v0.1.4#4da604090bf7cf1f3f552d69485472dbc623030a" } dependencies = [ { name = "pydantic" }, ]