Basic System (#31)

Co-authored-by: Tim Lorsbach <tim@lorsba.ch>
Reviewed-on: enviPath/enviPy#31

epdb/models.py

@@ -14,7 +14,7 @@ from django.contrib.auth.hashers import make_password, check_password
 from django.contrib.auth.models import AbstractUser
 from django.contrib.postgres.fields import ArrayField
 from django.db import models, transaction
-from django.db.models import JSONField
+from django.db.models import JSONField, Count, Q
 from django.utils import timezone
 from django.utils.functional import cached_property
 from model_utils.models import TimeStampedModel
@@ -43,8 +43,6 @@ class User(AbstractUser):
                                       on_delete=models.SET_NULL, related_name='default_group')
     default_setting = models.ForeignKey('epdb.Setting', on_delete=models.SET_NULL,
                                         verbose_name='The users default settings', null=True, blank=False)
-    # TODO remove
-    groups = models.ManyToManyField("Group", verbose_name='groups')
 
     USERNAME_FIELD = "email"
     REQUIRED_FIELDS = ['username']
@@ -87,6 +85,7 @@ class Group(TimeStampedModel):
     uuid = models.UUIDField(null=False, blank=False, verbose_name='UUID of this object', unique=True, default=uuid4)
     name = models.TextField(blank=False, null=False, verbose_name='Group name')
     owner = models.ForeignKey("User", verbose_name='Group Owner', on_delete=models.CASCADE)
+    public = models.BooleanField(verbose_name='Public Group', default=False)
     description = models.TextField(blank=False, null=False, verbose_name='Descriptions', default='no description')
     user_member = models.ManyToManyField("User", verbose_name='User members', related_name='users_in_group')
     group_member = models.ManyToManyField("Group", verbose_name='Group member', related_name='groups_in_group')
@@ -314,7 +313,7 @@ class Compound(EnviPathModel, AliasMixin, ScenarioMixin):
     @transaction.atomic
     def create(package: Package, smiles: str, name: str = None, description: str = None, *args, **kwargs) -> 'Compound':
 
-        if smiles is None or smiles == '':
+        if smiles is None or smiles.strip() == '':
             raise ValueError('SMILES is required')
 
         smiles = smiles.strip()
@@ -338,12 +337,14 @@ class Compound(EnviPathModel, AliasMixin, ScenarioMixin):
         c = Compound()
         c.package = package
 
-        # For name and description we have defaults so only set them if they carry a real value
-        if name is not None and name != '':
-            c.name = name
+        if name is None or name.strip() == '':
+            name = f"Compound {Compound.objects.filter(package=package).count() + 1}"
 
-        if description is not None and description != '':
-            c.description = description
+        c.name = name
+
+        # We have a default here only set the value if it carries some payload
+        if description is not None and description.strip() != '':
+            c.description = description.strip()
 
         c.save()
 
@@ -403,25 +404,27 @@ class Compound(EnviPathModel, AliasMixin, ScenarioMixin):
 class CompoundStructure(EnviPathModel, AliasMixin, ScenarioMixin):
     compound = models.ForeignKey('epdb.Compound', on_delete=models.CASCADE, db_index=True)
     smiles = models.TextField(blank=False, null=False, verbose_name='SMILES')
+    canonical_smiles = models.TextField(blank=False, null=False, verbose_name='Canonical SMILES')
+    inchikey = models.TextField(max_length=27, blank=False, null=False, verbose_name="InChIKey")
     normalized_structure = models.BooleanField(null=False, blank=False, default=False)
 
+    def save(self, *args, **kwargs):
+        # Compute these fields only on initial save call
+        if self.pk is None:
+            try:
+                # Generate canonical SMILES
+                self.canonical_smiles = FormatConverter.canonicalize(self.smiles)
+                # Generate InChIKey
+                self.inchikey = FormatConverter.InChIKey(self.smiles)
+            except Exception as e:
+                logger.error(f"Could compute canonical SMILES/InChIKey from {self.smiles}, error: {e}")
+
+        super().save(*args, **kwargs)
+
     @property
     def url(self):
         return '{}/structure/{}'.format(self.compound.url, self.uuid)
 
-    # @property
-    # def related_pathways(self):
-    #     pathways = Node.objects.filter(node_labels__in=[self]).values_list('pathway', flat=True)
-    #     return Pathway.objects.filter(package=self.compound.package, id__in=set(pathways)).order_by('name')
-
-    # @property
-    # def related_reactions(self):
-    #     return (
-    #             Reaction.objects.filter(package=self.compound.package, educts__in=[self])
-    #             |
-    #             Reaction.objects.filter(package=self.compound.package, products__in=[self])
-    #     ).order_by('name')
-
     @staticmethod
     @transaction.atomic
     def create(compound: Compound, smiles: str, name: str = None, description: str = None, *args, **kwargs):
@@ -448,19 +451,9 @@ class CompoundStructure(EnviPathModel, AliasMixin, ScenarioMixin):
 
         return cs
 
-    # TODO add find method
-
     @property
-    def InChIKey(self):
-        return FormatConverter.InChIKey(self.smiles)
-
-    @property
-    def canonical_smiles(self):
-        return FormatConverter.canonicalize(self.smiles)
-
-    @property
-    def as_svg(self):
-        return IndigoUtils.mol_to_svg(self.smiles)
+    def as_svg(self, width: int = 800, height: int = 400):
+        return IndigoUtils.mol_to_svg(self.smiles, width=width, height=height)
 
 
 class Rule(PolymorphicModel, EnviPathModel, AliasMixin, ScenarioMixin):
@@ -492,19 +485,9 @@ class Rule(PolymorphicModel, EnviPathModel, AliasMixin, ScenarioMixin):
 
     @staticmethod
     @transaction.atomic
-    def create(package: Package, rule_type: str, name: str = None, description: str = None, *args, **kwargs):
-        r = Rule.cls_for_type(rule_type)()
-        r.package = package
-        r.name = name
-        r.description = description
-
-        # As we are setting params this way the "k" has to match the property name
-        for k, v in kwargs.items():
-            setattr(r, k, v)
-
-        r.save()
-        return r
-
+    def create(rule_type: str, *args, **kwargs):
+        cls = Rule.cls_for_type(rule_type)
+        return cls.create(*args, **kwargs)
 
 #
 #     @property
@@ -533,6 +516,54 @@ class SimpleAmbitRule(SimpleRule):
     reactant_filter_smarts = models.TextField(null=True, verbose_name='Reactant Filter SMARTS')
     product_filter_smarts = models.TextField(null=True, verbose_name='Product Filter SMARTS')
 
+    @staticmethod
+    @transaction.atomic
+    def create(package: Package, name: str = None, description: str = None, smirks: str = None,
+               reactant_filter_smarts: str = None, product_filter_smarts: str = None):
+
+        if smirks is None or smirks.strip() == '':
+            raise ValueError('SMIRKS is required!')
+
+        smirks = smirks.strip()
+
+        if not FormatConverter.is_valid_smirks(smirks):
+            raise ValueError(f'SMIRKS "{smirks}" is invalid!')
+
+        query = SimpleAmbitRule.objects.filter(package=package, smirks=smirks)
+
+        if reactant_filter_smarts is not None and reactant_filter_smarts.strip() != '':
+            query = query.filter(reactant_filter_smarts=reactant_filter_smarts)
+
+        if product_filter_smarts is not None and product_filter_smarts.strip() != '':
+            query = query.filter(product_filter_smarts=product_filter_smarts)
+
+        if query.exists():
+            if query.count() > 1:
+                logger.error(f'More than one rule matched this one! {query}')
+            return query.first()
+
+        r = SimpleAmbitRule()
+        r.package = package
+
+        if name is None or name.strip() == '':
+            name = f'Rule {Rule.objects.filter(package=package).count() + 1}'
+
+        r.name = name
+
+        if description is not None and description.strip() != '':
+            r.description = description
+
+        r.smirks = smirks
+
+        if reactant_filter_smarts is not None and reactant_filter_smarts.strip() != '':
+            r.reactant_filter_smarts = reactant_filter_smarts
+
+        if product_filter_smarts is not None and product_filter_smarts.strip() != '':
+            r.product_filter_smarts = product_filter_smarts
+
+        r.save()
+        return r
+
     @property
     def url(self):
         return '{}/simple-ambit-rule/{}'.format(self.package.url, self.uuid)
@@ -642,7 +673,7 @@ class Reaction(EnviPathModel, AliasMixin, ScenarioMixin):
     def create(package: Package, name: str = None, description: str = None,
                educts: Union[List[str], List[CompoundStructure]] = None,
                products: Union[List[str], List[CompoundStructure]] = None,
-               rule: Rule = None, multi_step: bool = True):
+               rules: Union[Rule|List[Rule]] = None, multi_step: bool = True):
 
         _educts = []
         _products = []
@@ -662,18 +693,61 @@ class Reaction(EnviPathModel, AliasMixin, ScenarioMixin):
             _products += products
 
         else:
-            raise ValueError("")
+            raise ValueError("Found mixed types for educts and/or products!")
+
+        if len(_educts) == 0 or len(_products) == 0:
+            raise ValueError("No educts or products specified!")
+
+        if rules is None:
+            rules = []
+
+        if isinstance(rules, Rule):
+            rules = [rules]
+
+
+        query = Reaction.objects.annotate(
+            educt_count=Count('educts', filter=Q(educts__in=_educts), distinct=True),
+            product_count=Count('products', filter=Q(products__in=_products), distinct=True),
+        )
+
+        # The annotate/filter wont work if rules is an empty list
+        if rules:
+            query = query.annotate(
+                rule_count=Count('rules', filter=Q(rules__in=rules), distinct=True)
+            ).filter(rule_count=len(rules))
+        else:
+            query = query.annotate(
+                rule_count=Count('rules', distinct=True)
+            ).filter(rule_count=0)
+
+        existing_reaction_qs = query.filter(
+            educt_count=len(_educts),
+            product_count=len(_products),
+            multi_step=multi_step,
+            package=package
+        )
+
+        if existing_reaction_qs.exists():
+            if existing_reaction_qs.count() > 1:
+                logger.error(f'Found more than one reaction for given input! {existing_reaction_qs}')
+            return existing_reaction_qs.first()
 
         r = Reaction()
         r.package = package
-        r.name = name
-        r.description = description
+
+        if name is not None and name.strip() != '':
+            r.name = name
+
+        if description is not None and name.strip() != '':
+            r.description = description
+
         r.multi_step = multi_step
 
         r.save()
 
-        if rule:
-            r.rules.add(rule)
+        if rules:
+            for rule in rules:
+                r.rules.add(rule)
 
         for educt in _educts:
             r.educts.add(educt)
@@ -700,6 +774,7 @@ class Reaction(EnviPathModel, AliasMixin, ScenarioMixin):
 
 class Pathway(EnviPathModel, AliasMixin, ScenarioMixin):
     package = models.ForeignKey('epdb.Package', verbose_name='Package', on_delete=models.CASCADE, db_index=True)
+    setting = models.ForeignKey('epdb.Setting', verbose_name='Setting', on_delete=models.CASCADE, null=True, blank=True)
 
     @property
     def root_nodes(self):
@@ -709,6 +784,12 @@ class Pathway(EnviPathModel, AliasMixin, ScenarioMixin):
     def nodes(self):
         return Node.objects.filter(pathway=self)
 
+    def get_node(self, node_url):
+        for n in self.nodes:
+            if n.url == node_url:
+                return n
+        return None
+
     @property
     def edges(self):
         return Edge.objects.filter(pathway=self)
@@ -717,6 +798,26 @@ class Pathway(EnviPathModel, AliasMixin, ScenarioMixin):
     def url(self):
         return '{}/pathway/{}'.format(self.package.url, self.uuid)
 
+    # Mode
+    def is_built(self):
+        return self.kv.get('mode', 'build') == 'predicted'
+
+    def is_predicted(self):
+        return self.kv.get('mode', 'build') == 'predicted'
+
+    def is_predicted(self):
+        return self.kv.get('mode', 'build') == 'predicted'
+
+    # Status
+    def completed(self):
+        return self.kv.get('status', 'completed') == 'completed'
+
+    def running(self):
+        return self.kv.get('status', 'completed') == 'running'
+
+    def failed(self):
+        return self.kv.get('status', 'completed') == 'failed'
+
     def d3_json(self):
         # Ideally it would be something like this but
         # to reduce crossing in edges do a DFS
@@ -770,7 +871,11 @@ class Pathway(EnviPathModel, AliasMixin, ScenarioMixin):
                 new_link = {
                     'name': link['name'],
                     'id': link['id'],
+                    'url': link['url'],
+                    'image': link['image'],
                     'reaction': link['reaction'],
+                    'reaction_probability': link['reaction_probability'],
+                    'scenarios': link['scenarios'],
                     'source': node_url_to_idx[link['start_node_urls'][0]],
                     'target': pseudo_idx
                 }
@@ -781,7 +886,11 @@ class Pathway(EnviPathModel, AliasMixin, ScenarioMixin):
                     new_link = {
                         'name': link['name'],
                         'id': link['id'],
+                        'url': link['url'],
+                        'image': link['image'],
                         'reaction': link['reaction'],
+                        'reaction_probability': link['reaction_probability'],
+                        'scenarios': link['scenarios'],
                         'source': pseudo_idx,
                         'target': node_url_to_idx[target]
                     }
@@ -797,9 +906,9 @@ class Pathway(EnviPathModel, AliasMixin, ScenarioMixin):
             "completed": "true",
             "description": self.description,
             "id": self.url,
-            "isIncremental": False,
-            "isPredicted": False,
-            "lastModified": 1447842835894,
+            "isIncremental": self.kv.get('mode') == 'incremental',
+            "isPredicted": self.kv.get('mode') == 'predicted',
+            "lastModified": self.modified.strftime('%Y-%m-%d %H:%M:%S'),
             "pathwayName": self.name,
             "reviewStatus": "reviewed" if self.package.reviewed else 'unreviewed',
             "scenarios": [],
@@ -813,18 +922,38 @@ class Pathway(EnviPathModel, AliasMixin, ScenarioMixin):
 
     @staticmethod
     @transaction.atomic
-    def create(package, name, description, smiles):
+    def create(package: 'Package', smiles: str, name: Optional[str] = None, description: Optional[str] = None):
         pw = Pathway()
         pw.package = package
-        pw.name = name
-        pw.description = description
-        pw.save()
 
-        # create root node
-        Node.create(pw, smiles, 0)
+        if name is None:
+            name = f"Pathway {Pathway.objects.filter(package=package).count() + 1}"
+
+        pw.name = name
+
+        if description is not None:
+            pw.description = description
+
+        pw.save()
+        try:
+            # create root node
+            Node.create(pw, smiles, 0)
+        except ValueError as e:
+            # Node creation failed, most likely due to an invalid smiles
+            # delete this pathway...
+            pw.delete()
+            raise e
 
         return pw
 
+    @transaction.atomic
+    def add_node(self, smiles: str, name: Optional[str] = None, description: Optional[str] = None):
+        return Node.create(self, smiles, 0)
+
+    @transaction.atomic
+    def add_edge(self, start_nodes: List['Node'], end_nodes: List['Node'], rule: Optional['Rule'] = None,
+                 name: Optional[str] = None, description: Optional[str] = None):
+        return Edge.create(self, start_nodes, end_nodes, rule, name=name, description=description)
 
 class Node(EnviPathModel, AliasMixin, ScenarioMixin):
     pathway = models.ForeignKey('epdb.Pathway', verbose_name='belongs to', on_delete=models.CASCADE, db_index=True)
@@ -848,14 +977,14 @@ class Node(EnviPathModel, AliasMixin, ScenarioMixin):
             "imageSize": 490,  # TODO
             "name": self.default_node_label.name,
             "smiles": self.default_node_label.smiles,
+            "scenarios": [{'name': s.name, 'url': s.url} for s in self.scenarios.all()],
         }
 
     @staticmethod
-    def create(pathway, smiles, depth):
-        c = Compound.create(pathway.package, smiles)
+    def create(pathway: 'Pathway', smiles: str, depth: int, name: Optional[str] = None, description: Optional[str] = None):
+        c = Compound.create(pathway.package, smiles, name=name, description=description)
 
         if Node.objects.filter(pathway=pathway, default_node_label=c.default_structure).exists():
-            print("found node")
             return Node.objects.get(pathway=pathway, default_node_label=c.default_structure)
 
         n = Node()
@@ -886,34 +1015,21 @@ class Edge(PolymorphicModel, EnviPathModel, AliasMixin, ScenarioMixin):
         return '{}/edge/{}'.format(self.pathway.url, self.uuid)
 
     def d3_json(self):
-        # {
-        #     "ecNumbers": [
-        #         {
-        #             "ecName": "DDT 2,3-dioxygenase",
-        #             "ecNumber": "1.14.12.-"
-        #         }
-        #     ],
-        #     "id": "https://envipath.org/package/32de3cf4-e3e6-4168-956e-32fa5ddb0ce1/pathway/3f58e4d4-1c63-4b30-bf31-7ae4b98899fe/edge/ff193e7b-f010-43d4-acb3-45f34d938824",
-        #     "idreaction": "https://envipath.org/package/32de3cf4-e3e6-4168-956e-32fa5ddb0ce1/reaction/e11419cd-6b46-470b-8a06-a08d62281734",
-        #     "multistep": "false",
-        #     "name": "Eawag BBD reaction r0450",
-        #     "pseudo": False,
-        #     "scenarios": [],
-        #     "source": 0,
-        #     "target": 4
-        # }
-
         return {
             'name': self.name,
             'id': self.url,
-            'reaction': self.edge_label.url if self.edge_label else None,
+            'url': self.url,
+            'image': self.url + '?image=svg',
+            'reaction': {'name': self.edge_label.name, 'url': self.edge_label.url } if self.edge_label else None,
+            'reaction_probability': self.kv.get('probability'),
             # TODO
             'start_node_urls': [x.url for x in self.start_nodes.all()],
             'end_node_urls': [x.url for x in self.end_nodes.all()],
+            "scenarios": [{'name': s.name, 'url': s.url} for s in self.scenarios.all()],
         }
 
     @staticmethod
-    def create(pathway, start_nodes, end_nodes, rule: Optional[Rule] = None, name: Optional[str] = None,
+    def create(pathway, start_nodes: List[Node], end_nodes: List[Node], rule: Optional[Rule] = None, name: Optional[str] = None,
                description: Optional[str] = None):
         e = Edge()
         e.pathway = pathway
@@ -934,13 +1050,17 @@ class Edge(PolymorphicModel, EnviPathModel, AliasMixin, ScenarioMixin):
         r = Reaction.create(pathway.package, name=name, description=description,
                             educts=[n.default_node_label for n in e.start_nodes.all()],
                             products=[n.default_node_label for n in e.end_nodes.all()],
-                            rule=rule, multi_step=False
+                            rules=rule, multi_step=False
                             )
 
         e.edge_label = r
         e.save()
         return e
 
+    @property
+    def as_svg(self):
+        return self.edge_label.as_svg if self.edge_label else None
+
 
 class EPModel(PolymorphicModel, EnviPathModel):
     package = models.ForeignKey('epdb.Package', verbose_name='Package', on_delete=models.CASCADE, db_index=True)
@@ -976,6 +1096,9 @@ class MLRelativeReasoning(EPModel):
 
     eval_results = JSONField(null=True, blank=True, default=dict)
 
+    def status(self):
+        return self.PROGRESS_STATUS_CHOICES[self.model_status]
+
     @staticmethod
     @transaction.atomic
     def create(package, name, description, rule_packages, data_packages, eval_packages, threshold):
@@ -1201,7 +1324,7 @@ class MLRelativeReasoning(EPModel):
         self.save()
 
         mod = SparseLabelECC(
-            **s.DEFAULT_MODELS_PARAMS
+            **s.DEFAULT_DT_MODEL_PARAMS
         )
 
         mod.fit(X, y)
@@ -1247,7 +1370,7 @@ class MLRelativeReasoning(EPModel):
             y_train, y_test = y[train_index], y[test_index]
 
             model = SparseLabelECC(
-                **s.DEFAULT_MODELS_PARAMS
+                **s.DEFAULT_DT_MODEL_PARAMS
             )
             model.fit(X_train, y_train)
 
@@ -1500,11 +1623,15 @@ class Setting(EnviPathModel):
     max_nodes = models.IntegerField(null=False, blank=False, verbose_name='Setting Max Number of Nodes', default=30)
 
     rule_packages = models.ManyToManyField("Package", verbose_name="Setting Rule Packages",
-                                           related_name="setting_rule_packages")
+                                           related_name="setting_rule_packages", blank=True)
     model = models.ForeignKey('EPModel', verbose_name='Setting EPModel', on_delete=models.SET_NULL, null=True,
                               blank=True)
     model_threshold = models.FloatField(null=True, blank=True, verbose_name='Setting Model Threshold', default=0.25)
 
+    @property
+    def url(self):
+        return '{}/setting/{}'.format(s.SERVER_URL, self.uuid)
+
     @cached_property
     def applicable_rules(self):
         """