[Feature] Threshold Warning + Cosmetics (#277)

Co-authored-by: Tim Lorsbach <tim@lorsba.ch>
Reviewed-on: enviPath/enviPy#277

.gitea/workflows/api-ci.yaml

@@ -15,6 +15,8 @@ jobs:
   api-tests:
     if: ${{ !contains(gitea.event.pull_request.title, 'WIP') }}
     runs-on: ubuntu-latest
+    container:
+      image: git.envipath.com/envipath/envipy-ci:latest
 
     services:
       postgres:
@@ -67,19 +69,17 @@ jobs:
         uses: ./.gitea/actions/setup-envipy
         with:
           skip-frontend: 'true'
-          skip-playwright: 'true'
+          skip-playwright: 'false'
           ssh-private-key: ${{ secrets.ENVIPY_CI_PRIVATE_KEY }}
           run-migrations: 'true'
 
       - name: Run API tests
         run: |
-          source .venv/bin/activate
+          .venv/bin/python manage.py test epapi -v 2
-          python manage.py test epapi -v 2
 
       - name: Test API endpoints availability
         run: |
-          source .venv/bin/activate
+          .venv/bin/python manage.py runserver 0.0.0.0:8000 &
-          python manage.py runserver 0.0.0.0:8000 &
           SERVER_PID=$!
           sleep 5
           curl -f http://localhost:8000/api/v1/docs || echo "API docs not available"

.gitea/workflows/ci.yaml

@@ -12,9 +12,6 @@ jobs:
     runs-on: ubuntu-latest
     container:
       image: git.envipath.com/envipath/envipy-ci:latest
-      credentials:
-        username: ${{ secrets.CI_REGISTRY_USER }}
-        password: ${{ secrets.CI_REGISTRY_PASSWORD }}
 
     services:
       postgres:

epdb/logic.py

@@ -1489,6 +1489,7 @@ class SPathway(object):
         self.smiles_to_node: Dict[str, SNode] = dict(**{n.smiles: n for n in self.root_nodes})
         self.edges: Set["SEdge"] = set()
         self.done = False
+        self.empty_due_to_threshold = False
 
     @staticmethod
     def from_pathway(pw: "Pathway", persist: bool = True):
@@ -1601,9 +1602,24 @@ class SPathway(object):
 
                         sub.app_domain_assessment = app_domain_assessment
 
-            candidates = self.prediction_setting.expand(self, sub)
+            expansion_result = self.prediction_setting.expand(self, sub)
+
+            # We don't have any substrate, but technically we have at least one rule that triggered.
+            # If our substrate is a root node a.k.a. depth == 0 store that info in SPathway
+            if (
+                len(expansion_result["transformations"]) == 0
+                and expansion_result["rule_triggered"]
+                and sub.depth == 0
+            ):
+                self.empty_due_to_threshold = True
+
+                # Emit directly
+                if self.persist is not None:
+                    self.persist.kv["empty_due_to_threshold"] = True
+                    self.persist.save()
+
             # candidates is a List of PredictionResult. The length of the List is equal to the number of rules
-            for cand_set in candidates:
+            for cand_set in expansion_result["transformations"]:
                 if cand_set:
                     # cand_set is a PredictionResult object that can consist of multiple candidate reactions
                     for cand in cand_set:
@@ -1727,10 +1743,6 @@ class SPathway(object):
                         for queued_val in queue:
                             node_and_probs.append((queued_val, node_probs[queued_val]))
 
-                        from pprint import pprint
-
-                        pprint(node_and_probs)
-
                         # re-order the queue and only pick smiles
                         queue = [
                             n[0] for n in sorted(node_and_probs, key=lambda x: x[1], reverse=True)

epdb/models.py

@@ -23,7 +23,7 @@ from django.db import models, transaction
 from django.db.models import Count, JSONField, Q, QuerySet
 from django.utils import timezone
 from django.utils.functional import cached_property
-from envipy_additional_information import EnviPyModel
+from envipy_additional_information import EnviPyModel, HalfLife
 from model_utils.models import TimeStampedModel
 from polymorphic.models import PolymorphicModel
 from sklearn.metrics import jaccard_score, precision_score, recall_score
@@ -795,9 +795,7 @@ class Compound(EnviPathModel, AliasMixin, ScenarioMixin, ChemicalIdentifierMixin
 
     @property
     def related_pathways(self):
-        pathways = Node.objects.filter(node_labels__in=[self.default_structure]).values_list(
+        pathways = self.related_nodes.values_list("pathway", flat=True)
-            "pathway", flat=True
-        )
         return Pathway.objects.filter(package=self.package, id__in=set(pathways)).order_by("name")
 
     @property
@@ -807,6 +805,12 @@ class Compound(EnviPathModel, AliasMixin, ScenarioMixin, ChemicalIdentifierMixin
             | Reaction.objects.filter(package=self.package, products__in=[self.default_structure])
         ).order_by("name")
 
+    @property
+    def related_nodes(self):
+        return Node.objects.filter(
+            node_labels__in=[self.default_structure], pathway__package=self.package
+        )
+
     @staticmethod
     @transaction.atomic
     def create(
@@ -1042,6 +1046,17 @@ class Compound(EnviPathModel, AliasMixin, ScenarioMixin, ChemicalIdentifierMixin
 
         return new_compound
 
+    def half_lifes(self):
+        hls: Dict[Scenario, List[HalfLife]] = defaultdict(list)
+
+        for n in self.related_nodes:
+            for scen in n.scenarios.all().order_by("name"):
+                for ai in scen.get_additional_information():
+                    if isinstance(ai, HalfLife):
+                        hls[scen].append(ai)
+
+        return dict(hls)
+
     class Meta:
         unique_together = [("uuid", "package")]
 
@@ -1780,6 +1795,9 @@ class Pathway(EnviPathModel, AliasMixin, ScenarioMixin):
     def failed(self):
         return self.status() == "failed"
 
+    def empty_due_to_threshold(self):
+        return self.kv.get("empty_due_to_threshold", False)
+
     def d3_json(self):
         # Ideally it would be something like this but
         # to reduce crossing in edges do a DFS
@@ -1887,7 +1905,7 @@ class Pathway(EnviPathModel, AliasMixin, ScenarioMixin):
             "status": self.status(),
         }
 
-        return json.dumps(res)
+        return res
 
     def to_csv(self, include_header=True, include_pathway_url=False) -> str:
         import csv
@@ -3887,33 +3905,48 @@ class Setting(EnviPathModel):
         rules = sorted(rules, key=lambda x: x.url)
         return rules
 
-    def expand(self, pathway, current_node):
+    def expand(self, pathway, current_node) -> Dict[str, Any]:
+        res: Dict[str, Any] = defaultdict(list)
+
         """Decision Method whether to expand on a certain Node or not"""
         if pathway.num_nodes() >= self.max_nodes:
             logger.info(
                 f"Pathway has {pathway.num_nodes()} Nodes which exceeds the limit of {self.max_nodes}"
             )
-            return []
+            res["expansion_skipped"] = True
+            return res
 
         if pathway.depth() >= self.max_depth:
             logger.info(
                 f"Pathway has reached depth {pathway.depth()} which exceeds the limit of {self.max_depth}"
             )
-            return []
+            res["expansion_skipped"] = True
+            return res
 
-        transformations = []
         if self.model is not None:
             pred_results = self.model.predict(current_node.smiles)
+
+            # Store whether there are results that may be removed as they are below
+            # the given threshold
+            if len(pred_results):
+                res["rule_triggered"] = True
+
             for pred_result in pred_results:
-                if pred_result.probability >= self.model_threshold:
+                if (
-                    transformations.append(pred_result)
+                    len(pred_result.product_sets)
+                    and pred_result.probability >= self.model_threshold
+                ):
+                    res["transformations"].append(pred_result)
         else:
             for rule in self.applicable_rules:
                 tmp_products = rule.apply(current_node.smiles)
                 if tmp_products:
-                    transformations.append(PredictionResult(tmp_products, 1.0, rule))
+                    res["transformations"].append(PredictionResult(tmp_products, 1.0, rule))
 
-        return transformations
+            if len(res["transformations"]):
+                res["rule_triggered"] = True
+
+        return res
 
     @transaction.atomic
     def make_global_default(self):

epdb/views.py

@@ -1937,6 +1937,7 @@ def package_pathway(request, package_uuid, pathway_uuid):
                 {
                     "status": current_pathway.status(),
                     "modified": current_pathway.modified.strftime("%Y-%m-%d %H:%M:%S"),
+                    "emptyDueToThreshold": current_pathway.empty_due_to_threshold(),
                 }
             )
 

static/js/alpine/pathway.js

@@ -22,13 +22,16 @@ document.addEventListener('alpine:init', () => {
     status: config.status,
     modified: config.modified,
     statusUrl: config.statusUrl,
+    emptyDueToThreshold: config.emptyDueToThreshold === "True",
     showUpdateNotice: false,
+    showEmptyDueToThresholdNotice: false,
+    emptyDueToThresholdMessage: 'The Pathway is empty due to the selected threshold. Please try a different threshold.',
     updateMessage: '',
     pollInterval: null,
 
     get statusTooltip() {
       const tooltips = {
-        'completed': 'Pathway prediction complete.',
+        'completed': 'Pathway prediction completed.',
         'failed': 'Pathway prediction failed.',
         'running': 'Pathway prediction running.'
       };
@@ -39,9 +42,17 @@ document.addEventListener('alpine:init', () => {
       if (this.status === 'running') {
         this.startPolling();
       }
+
+      if (this.emptyDueToThreshold) {
+          this.showEmptyDueToThresholdNotice = true;
+      }
+
     },
 
     startPolling() {
+      if (this.pollInterval) {
+        return;
+      }
       this.pollInterval = setInterval(() => this.checkStatus(), 5000);
     },
 
@@ -50,10 +61,17 @@ document.addEventListener('alpine:init', () => {
         const response = await fetch(this.statusUrl);
         const data = await response.json();
 
+        if (data.emptyDueToThreshold) {
+          this.emptyDueToThreshold = true;
+          this.showEmptyDueToThresholdNotice = true;
+        }
+
         if (data.modified > this.modified) {
+          if (!this.emptyDueToThreshold) {
             this.showUpdateNotice = true;
             this.updateMessage = this.getUpdateMessage(data.status);
           }
+        }
 
         if (data.status !== 'running') {
           this.status = data.status;

templates/modals/search_modal.html

@@ -1,11 +1,10 @@
-{% load static %}
 <dialog
   id="search_modal"
-  class="modal @max-sm:modal-top justify-center"
+  class="modal items-start sm:items-center"
   x-data="searchModal()"
   @close="reset()"
 >
-  <div class="modal-box h-full w-lvw p-1 sm:h-8/12 sm:w-[85vw] sm:max-w-5xl">
+  <div class="modal-box mt-4 sm:mt-0 p-1 sm:h-8/12 sm:w-[85vw] sm:max-w-5xl">
     <!-- Search Input and Mode Selector -->
     <div class="form-control mb-4 w-full shrink-0">
       <div class="join m-0 w-full items-center p-3">
@@ -43,7 +42,7 @@
             type="button"
             tabindex="0"
             popovertarget="search_dropdown_menu"
-            style="anchor-name: --1"
+            style="anchor-name: --anchor-mode"
             class="btn join-item btn-ghost"
           >
             <span x-text="searchModeLabel"></span>
@@ -67,7 +66,7 @@
             popover
             x-ref="modeDropdown"
             id="search_dropdown_menu"
-            style="position-anchor: --anchor-2"
+            style="position-anchor: --anchor-mode"
           >
             <li class="menu-title">Text</li>
             <li>
@@ -495,8 +494,7 @@
     </div>
   </div>
 
-  <!-- Backdrop to close -->
   <form method="dialog" class="modal-backdrop">
-    <button>close</button>
+    <button aria-label="close"></button>
   </form>
 </dialog>

templates/objects/compound.html

@@ -181,6 +181,55 @@
       </div>
     {% endif %}
 
+    {% if compound.half_lifes %}
+      <div class="collapse-arrow bg-base-200 collapse">
+        <input type="checkbox" />
+        <div class="collapse-title text-xl font-medium">Half-lives</div>
+        <div class="collapse-content">
+          <div class="overflow-x-auto">
+            <table class="table-zebra table">
+              <thead>
+                <tr>
+                  <th>Scenario</th>
+                  <th>Values</th>
+                </tr>
+              </thead>
+              <tbody>
+                {% for scenario, half_lifes in compound.half_lifes.items %}
+                  <tr>
+                    <td>
+                      <a href="{{ scenario.url }}" class="hover:bg-base-200"
+                        >{{ scenario.name }}
+                        <i>({{ scenario.package.name }})</i></a
+                      >
+                    </td>
+                    <td>
+                      <table class="table-zebra table">
+                        <tbody>
+                          <tr>
+                            <td>Scenario Type</td>
+                            <td>{{ scenario.scenario_type }}</td>
+                          </tr>
+                          <tr>
+                            <td>Half-life (days)</td>
+                            <td>{{ half_lifes.0.dt50 }}</td>
+                          </tr>
+                          <tr>
+                            <td>Model</td>
+                            <td>{{ half_lifes.0.model }}</td>
+                          </tr>
+                        </tbody>
+                      </table>
+                    </td>
+                  </tr>
+                {% endfor %}
+              </tbody>
+            </table>
+          </div>
+        </div>
+      </div>
+    {% endif %}
+
     <!-- External Identifiers -->
     {% if compound.get_external_identifiers %}
       <div class="collapse-arrow bg-base-200 collapse">

templates/objects/pathway.html

@@ -216,10 +216,12 @@
           x-data="pathwayViewer({
           status: '{{ pathway.status }}',
           modified: '{{ pathway.modified|date:"Y-m-d H:i:s" }}',
-          statusUrl: '{{ pathway.url }}?status=true'
+          statusUrl: '{{ pathway.url }}?status=true',
+          emptyDueToThreshold: '{{ pathway.empty_due_to_threshold }}'
         })"
           x-init="init()"
         >
+          {% if pathway.predicted %}
             <!-- Status Display -->
             <div class="tooltip tooltip-left absolute top-4 right-4 z-10">
               <div class="tooltip-content" x-text="statusTooltip"></div>
@@ -241,6 +243,7 @@
                     <path d="M20 6 9 17l-5-5" />
                   </svg>
                 </template>
+
                 <!-- Failed icon -->
                 <template x-if="status === 'failed'">
                   <svg
@@ -268,7 +271,7 @@
                 </div>
               </div>
             </div>
-
+          {% endif %}
           <!-- Update Notice -->
           <div
             x-show="showUpdateNotice"
@@ -280,6 +283,15 @@
               Reload page
             </button>
           </div>
+          <!-- Empty due to Threshold notice -->
+          <div
+            x-show="showEmptyDueToThresholdNotice"
+            x-cloak
+            class="alert alert-info absolute right-4 bottom-4 left-4 z-10"
+          >
+            <span x-html="emptyDueToThresholdMessage"></span>
+          </div>
+
           <svg id="pwsvg">
             <defs>
               <marker
@@ -455,8 +467,7 @@
       </div>
     {% endif %}
   </div>
-  {# prettier-ignore-start #}
+  {{ pathway.d3_json|json_script:"pathway" }}
-  {# FIXME: This is a hack to get the pathway data into the JavaScript code. #}
 
   <script>
     // Global switch for app domain view
@@ -472,63 +483,81 @@
     }
 
     function transformReferences(text) {
-            return text.replace(/\[\s*(http[^\]|]+)\s*\|\s*([^\]]+)\s*\]/g, '<a target="parent" href="$1">$2</a>');
+      return text.replace(
+        /\[\s*(http[^\]|]+)\s*\|\s*([^\]]+)\s*\]/g,
+        '<a target="parent" href="$1">$2</a>',
+      );
     }
 
+    var pathway = JSON.parse(document.getElementById("pathway").textContent);
 
-        var pathway = {{ pathway.d3_json | safe }};
+    document.addEventListener("DOMContentLoaded", function () {
-
+      draw(pathway, "vizdiv");
-        document.addEventListener('DOMContentLoaded', function() {
-            draw(pathway, 'vizdiv');
 
       // Transform references in description
-            const descContent = document.getElementById('DescriptionContent');
+      const descContent = document.getElementById("DescriptionContent");
       if (descContent) {
         const newDesc = transformReferences(descContent.innerText);
         descContent.innerHTML = newDesc;
       }
 
       // App domain toggle
-            const appDomainBtn = document.getElementById('app-domain-toggle-button');
+      const appDomainBtn = document.getElementById("app-domain-toggle-button");
       if (appDomainBtn) {
-                appDomainBtn.addEventListener('click', function() {
+        appDomainBtn.addEventListener("click", function () {
           appDomainViewEnabled = !appDomainViewEnabled;
-                    const icon = document.getElementById('app-domain-icon');
+          const icon = document.getElementById("app-domain-icon");
 
           if (appDomainViewEnabled) {
             // Change to eye-off icon
-                        icon.innerHTML = '<path d="M9.88 9.88a3 3 0 1 0 4.24 4.24"/><path d="M10.73 5.08A10.43 10.43 0 0 1 12 5c7 0 10 7 10 7a13.16 13.16 0 0 1-1.67 2.68"/><path d="M6.61 6.61A13.526 13.526 0 0 0 2 12s3 7 10 7a9.74 9.74 0 0 0 5.39-1.61"/><line x1="2" x2="22" y1="2" y2="22"/>';
+            icon.innerHTML =
+              '<path d="M9.88 9.88a3 3 0 1 0 4.24 4.24"/><path d="M10.73 5.08A10.43 10.43 0 0 1 12 5c7 0 10 7 10 7a13.16 13.16 0 0 1-1.67 2.68"/><path d="M6.61 6.61A13.526 13.526 0 0 0 2 12s3 7 10 7a9.74 9.74 0 0 0 5.39-1.61"/><line x1="2" x2="22" y1="2" y2="22"/>';
 
             nodes.forEach((x) => {
               if (x.app_domain) {
                 if (x.app_domain.inside_app_domain) {
-                                    d3.select(x.el).select("circle").classed("inside_app_domain", true);
+                  d3.select(x.el)
+                    .select("circle")
+                    .classed("inside_app_domain", true);
                 } else {
-                                    d3.select(x.el).select("circle").classed("outside_app_domain", true);
+                  d3.select(x.el)
+                    .select("circle")
+                    .classed("outside_app_domain", true);
                 }
               }
             });
             links.forEach((x) => {
               if (x.app_domain) {
                 if (x.app_domain.passes_app_domain) {
-                                    d3.select(x.el).attr("marker-end", d => d.target.pseudo ? "" : "url(#arrow_passes_app_domain)");
+                  d3.select(x.el).attr("marker-end", (d) =>
+                    d.target.pseudo ? "" : "url(#arrow_passes_app_domain)",
+                  );
                   d3.select(x.el).classed("passes_app_domain", true);
                 } else {
-                                    d3.select(x.el).attr("marker-end", d => d.target.pseudo ? "" : "url(#arrow_fails_app_domain)");
+                  d3.select(x.el).attr("marker-end", (d) =>
+                    d.target.pseudo ? "" : "url(#arrow_fails_app_domain)",
+                  );
                   d3.select(x.el).classed("fails_app_domain", true);
                 }
               }
             });
           } else {
             // Change back to eye icon
-                        icon.innerHTML = '<path d="M2 12s3-7 10-7 10 7 10 7-3 7-10 7-10-7-10-7Z"/><circle cx="12" cy="12" r="3"/>';
+            icon.innerHTML =
+              '<path d="M2 12s3-7 10-7 10 7 10 7-3 7-10 7-10-7-10-7Z"/><circle cx="12" cy="12" r="3"/>';
 
             nodes.forEach((x) => {
-                            d3.select(x.el).select("circle").classed("inside_app_domain", false);
+              d3.select(x.el)
-                            d3.select(x.el).select("circle").classed("outside_app_domain", false);
+                .select("circle")
+                .classed("inside_app_domain", false);
+              d3.select(x.el)
+                .select("circle")
+                .classed("outside_app_domain", false);
             });
             links.forEach((x) => {
-                            d3.select(x.el).attr("marker-end", d => d.target.pseudo ? "" : "url(#arrow)");
+              d3.select(x.el).attr("marker-end", (d) =>
+                d.target.pseudo ? "" : "url(#arrow)",
+              );
               d3.select(x.el).classed("passes_app_domain", false);
               d3.select(x.el).classed("fails_app_domain", false);
             });
@@ -543,7 +572,5 @@
         actionsButton?.classList.remove("hidden");
       }
     });
-
   </script>
-    {# prettier-ignore-end #}
 {% endblock content %}

tests/test_sobjects.py

@@ -51,7 +51,9 @@ class SPathwayTest(TestCase):
             None,
         )
 
-        with patch.object(self.mock_setting, "expand", return_value=[pr]):
+        mock_val = {"rule_triggered": True, "transformations": [pr]}
+
+        with patch.object(self.mock_setting, "expand", return_value=mock_val):
             spw.predict_step(from_depth=0)
 
         self.assertEqual(len(spw.smiles_to_node.keys()), 4)
@@ -72,7 +74,9 @@ class SPathwayTest(TestCase):
             None,
         )
 
-        with patch.object(self.mock_setting, "expand", return_value=[pr]):
+        mock_val = {"rule_triggered": True, "transformations": [pr]}
+
+        with patch.object(self.mock_setting, "expand", return_value=mock_val):
             spw.predict_step(from_depth=0)
 
         self.assertEqual(len(spw.smiles_to_node.keys()), 4)