Basic System (#31)

Co-authored-by: Tim Lorsbach <tim@lorsba.ch>
Reviewed-on: enviPath/enviPy#31

epdb/logic.py

@@ -7,13 +7,21 @@ from django.db import transaction
 from django.conf import settings as s
 
 from epdb.models import User, Package, UserPackagePermission, GroupPackagePermission, Permission, Group, Setting, \
-    EPModel, UserSettingPermission, Rule, Pathway, Node, Edge
+    EPModel, UserSettingPermission, Rule, Pathway, Node, Edge, Compound, Reaction, CompoundStructure
+from utilities.chem import FormatConverter
 
 logger = logging.getLogger(__name__)
 
+
 class UserManager(object):
+    user_pattern = re.compile(r".*/user/[a-f0-9]{8}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{12}")
+
     @staticmethod
-    def create_user(username, email, password, *args, **kwargs):
+    def is_user_url(url: str):
+        return bool(re.findall(UserManager.user_pattern, url))
+
+    @staticmethod
+    def create_user(username, email, password, set_setting=True, add_to_group=True, *args, **kwargs):
         # avoid circular import :S
         from .tasks import send_registration_mail
 
@@ -34,6 +42,17 @@ class UserManager(object):
             # send email for verification
             send_registration_mail.delay(u.pk)
 
+        if set_setting:
+            u.default_setting = Setting.objects.get(global_default=True)
+            u.save()
+
+        if add_to_group:
+            g = Group.objects.get(public=True, name='enviPath Users')
+            g.user_member.add(u)
+            g.save()
+            u.default_group = g
+            u.save()
+
         return u
 
     @staticmethod
@@ -54,7 +73,16 @@ class UserManager(object):
         uuid = user_url.strip().split('/')[-1]
         return get_user_model().objects.get(uuid=uuid)
 
+    @staticmethod
+    def writable(current_user, user):
+        return (current_user == user) or user.is_superuser
+
 class GroupManager(object):
+    group_pattern = re.compile(r".*/group/[a-f0-9]{8}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{12}")
+
+    @staticmethod
+    def is_group_url(url: str):
+        return bool(re.findall(GroupManager.group_pattern, url))
 
     @staticmethod
     def create_group(current_user, name, description):
@@ -110,9 +138,17 @@ class GroupManager(object):
 
         group.save()
 
+    @staticmethod
+    def writable(user, group):
+        return (user == group.owner) or user.is_superuser
+
 
 class PackageManager(object):
-    package_pattern = re.compile(r".*/package/[a-f0-9]{8}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{12}$")
+    package_pattern = re.compile(r".*/package/[a-f0-9]{8}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{12}")
+
+    @staticmethod
+    def is_package_url(url: str):
+        return bool(re.findall(PackageManager.package_pattern, url))
 
     @staticmethod
     def get_reviewed_packages():
@@ -120,7 +156,6 @@ class PackageManager(object):
 
     @staticmethod
     def readable(user, package):
-        # TODO Owner!
         if UserPackagePermission.objects.filter(package=package, user=user).exists() or \
                 GroupPackagePermission.objects.filter(package=package, group__in=GroupManager.get_groups(user)) or \
                 package.reviewed is True or \
@@ -131,14 +166,21 @@ class PackageManager(object):
 
     @staticmethod
     def writable(user, package):
-        # TODO Owner!
-        if UserPackagePermission.objects.filter(package=package, user=user, permission=Permission.WRITE).exists() or \
-                GroupPackagePermission.objects.filter(package=package, group__in=GroupManager.get_groups(user),
-                                                      permission=Permission.WRITE) or \
+        if UserPackagePermission.objects.filter(package=package, user=user, permission=Permission.WRITE[0]).exists() or \
+                GroupPackagePermission.objects.filter(package=package, group__in=GroupManager.get_groups(user), permission=Permission.WRITE[0]).exists() or \
+                UserPackagePermission.objects.filter(package=package, user=user, permission=Permission.ALL[0]).exists() or \
                 user.is_superuser:
             return True
         return False
 
+    @staticmethod
+    def get_package_lp(package_url):
+        match = re.findall(PackageManager.package_pattern, package_url)
+        if match:
+            package_id = match[0].split('/')[-1]
+            return Package.objects.get(uuid=package_id)
+        return None
+
     @staticmethod
     def get_package_by_url(user, package_url):
         match = re.findall(PackageManager.package_pattern, package_url)
@@ -229,8 +271,12 @@ class PackageManager(object):
     @staticmethod
     @transaction.atomic
     def update_permissions(caller: User, package: Package, grantee: Union[User, Group], new_perm: Optional[str]):
-        if not PackageManager.writable(caller, package):
-            raise ValueError(f"User {caller} is not allowed to modify permissions on {package}")
+        caller_perm = None
+        if not caller.is_superuser:
+            caller_perm = UserPackagePermission.objects.get(user=caller, package=package).permission
+
+        if caller_perm != Permission.ALL[0] and not caller.is_superuser:
+            raise ValueError(f"Only owner are allowed to modify permissions")
 
         data = {
             'package': package,
@@ -629,8 +675,6 @@ class PackageManager(object):
 
         return pack
 
-
-
 class SettingManager(object):
     setting_pattern = re.compile(r".*/setting/[a-f0-9]{8}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{12}$")
 
@@ -648,7 +692,8 @@ class SettingManager(object):
     def get_setting_by_id(user, setting_id):
         s = Setting.objects.get(uuid=setting_id)
 
-        if s.global_default or s.public or s.owner == user or user.is_superuser:
+        if s.global_default or s.public or  user.is_superuser or \
+                UserSettingPermission.objects.filter(user=user, setting=s).exists():
             return s
 
         raise ValueError(
@@ -697,6 +742,116 @@ class SettingManager(object):
     def set_default_setting(user: User, setting: Setting):
         pass
 
+class SearchManager(object):
+
+
+    @staticmethod
+    def search(packages: Union[Package, List[Package]], searchterm: str, mode: str):
+        match mode:
+            case 'text':
+                return SearchManager._search_text(packages, searchterm)
+            case 'default':
+                return SearchManager._search_default_smiles(packages, searchterm)
+            case 'exact':
+                return SearchManager._search_exact_smiles(packages, searchterm)
+            case 'canonical':
+                return SearchManager._search_canonical_smiles(packages, searchterm)
+            case 'inchikey':
+                return SearchManager._search_inchikey(packages, searchterm)
+            case _:
+                raise ValueError(f"Unknown search mode {mode}!")
+
+    @staticmethod
+    def _search_inchikey(packages: Union[Package, List[Package]], searchterm: str):
+        from django.db.models import Q
+
+        search_cond = Q(inchikey=searchterm)
+        compound_qs = Compound.objects.filter(Q(package__in=packages) & Q(compoundstructure__inchikey=searchterm)).distinct()
+        compound_structure_qs = CompoundStructure.objects.filter(Q(compound__package__in=packages) & search_cond)
+        reactions_qs = Reaction.objects.filter(Q(package__in=packages) & (Q(educts__inchikey=searchterm) | Q(products__inchikey=searchterm))).distinct()
+        pathway_qs = Pathway.objects.filter(Q(package__in=packages) & (Q(edge__edge_label__educts__inchikey=searchterm) | Q(edge__edge_label__products__inchikey=searchterm))).distinct()
+
+        return {
+            'Compounds': [{'name': c.name, 'description': c.description, 'url': c.url} for c in compound_qs],
+            'Compound Structures': [{'name': c.name, 'description': c.description, 'url': c.url} for c in compound_structure_qs],
+            'Reactions': [{'name': r.name, 'description': r.description, 'url': r.url} for r in reactions_qs],
+            'Pathways': [{'name': p.name, 'description': p.description, 'url': p.url} for p in pathway_qs],
+        }
+
+    @staticmethod
+    def _search_exact_smiles(packages: Union[Package, List[Package]], searchterm: str):
+        from django.db.models import Q
+
+        search_cond = Q(smiles=searchterm)
+        compound_qs = Compound.objects.filter(Q(package__in=packages) & Q(compoundstructure__smiles=searchterm)).distinct()
+        compound_structure_qs = CompoundStructure.objects.filter(Q(compound__package__in=packages) & search_cond)
+        reactions_qs = Reaction.objects.filter(Q(package__in=packages) & (Q(educts__smiles=searchterm) | Q(products__smiles=searchterm))).distinct()
+        pathway_qs = Pathway.objects.filter(Q(package__in=packages) & (Q(edge__edge_label__educts__smiles=searchterm) | Q(edge__edge_label__products__smiles=searchterm))).distinct()
+
+        return {
+            'Compounds': [{'name': c.name, 'description': c.description, 'url': c.url} for c in compound_qs],
+            'Compound Structures': [{'name': c.name, 'description': c.description, 'url': c.url} for c in compound_structure_qs],
+            'Reactions': [{'name': r.name, 'description': r.description, 'url': r.url} for r in reactions_qs],
+            'Pathways': [{'name': p.name, 'description': p.description, 'url': p.url} for p in pathway_qs],
+        }
+
+    @staticmethod
+    def _search_default_smiles(packages: Union[Package, List[Package]], searchterm: str):
+        from django.db.models import Q
+
+        inchi_front = FormatConverter.InChIKey(searchterm)[:14]
+
+        search_cond = Q(inchikey__startswith=inchi_front)
+        compound_qs = Compound.objects.filter(Q(package__in=packages) & Q(compoundstructure__inchikey__startswith=inchi_front)).distinct()
+        compound_structure_qs = CompoundStructure.objects.filter(Q(compound__package__in=packages) & search_cond)
+        reactions_qs = Reaction.objects.filter(Q(package__in=packages) & (Q(educts__inchikey__startswith=inchi_front) | Q(products__inchikey__startswith=inchi_front))).distinct()
+        pathway_qs = Pathway.objects.filter(Q(package__in=packages) & (Q(edge__edge_label__educts__inchikey__startswith=inchi_front) | Q(edge__edge_label__products__inchikey__startswith=inchi_front))).distinct()
+
+        return {
+            'Compounds': [{'name': c.name, 'description': c.description, 'url': c.url} for c in compound_qs],
+            'Compound Structures': [{'name': c.name, 'description': c.description, 'url': c.url} for c in compound_structure_qs],
+            'Reactions': [{'name': r.name, 'description': r.description, 'url': r.url} for r in reactions_qs],
+            'Pathways': [{'name': p.name, 'description': p.description, 'url': p.url} for p in pathway_qs],
+        }
+
+    @staticmethod
+    def _search_canonical_smiles(packages: Union[Package, List[Package]], searchterm: str):
+        from django.db.models import Q
+
+        search_cond = Q(canonical_smiles=searchterm)
+        compound_qs = Compound.objects.filter(Q(package__in=packages) & Q(compoundstructure__canonical_smiles=searchterm)).distinct()
+        compound_structure_qs = CompoundStructure.objects.filter(Q(compound__package__in=packages) & search_cond)
+        reactions_qs = Reaction.objects.filter(Q(package__in=packages) & (Q(educts__canonical_smiles=searchterm) | Q(products__canonical_smiles=searchterm))).distinct()
+        pathway_qs = Pathway.objects.filter(Q(package__in=packages) & (Q(edge__edge_label__educts__canonical_smiles=searchterm) | Q(edge__edge_label__products__canonical_smiles=searchterm))).distinct()
+
+        return {
+            'Compounds': [{'name': c.name, 'description': c.description, 'url': c.url} for c in compound_qs],
+            'Compound Structures': [{'name': c.name, 'description': c.description, 'url': c.url} for c in compound_structure_qs],
+            'Reactions': [{'name': r.name, 'description': r.description, 'url': r.url} for r in reactions_qs],
+            'Pathways': [{'name': p.name, 'description': p.description, 'url': p.url} for p in pathway_qs],
+        }
+
+    @staticmethod
+    def _search_text(packages: Union[Package, List[Package]], searchterm: str):
+        from django.db.models import Q
+
+        search_cond = (Q(name__icontains=searchterm) | Q(description__icontains=searchterm))
+        cond = Q(package__in=packages) & search_cond
+        compound_qs = Compound.objects.filter(cond)
+        compound_structure_qs = CompoundStructure.objects.filter(Q(compound__package__in=packages) & search_cond)
+        rule_qs = Rule.objects.filter(cond)
+        reactions_qs = Reaction.objects.filter(cond)
+        pathway_qs = Pathway.objects.filter(cond)
+
+        res = {
+            'Compounds': [{'name': c.name, 'description': c.description, 'url': c.url} for c in compound_qs],
+            'Compound Structures': [{'name': c.name, 'description': c.description, 'url': c.url} for c in compound_structure_qs],
+            'Rules': [{'name': r.name, 'description': r.description, 'url': r.url} for r in rule_qs],
+            'Reactions': [{'name': r.name, 'description': r.description, 'url': r.url} for r in reactions_qs],
+            'Pathways': [{'name': p.name, 'description': p.description, 'url': p.url} for p in pathway_qs],
+        }
+        return res
+
 
 class SNode(object):
 
@@ -719,7 +874,7 @@ class SNode(object):
 class SEdge(object):
 
     def __init__(self, educts: Union[SNode, List[SNode]], products: Union[SNode | List[SNode]],
-                 rule: Optional['Rule'] = None):
+                 rule: Optional['Rule'] = None, probability: Optional[float] = None):
 
         if not isinstance(educts, list):
             educts = [educts]
@@ -727,6 +882,7 @@ class SEdge(object):
         self.educts = educts
         self.products = products
         self.rule = rule
+        self.probability = probability
 
     def __hash__(self):
         full_hash = 0
@@ -799,11 +955,45 @@ class SPathway(object):
                 elif isinstance(n, SNode):
                     self.root_nodes.append(n)
 
-        self.queue = list()
         self.smiles_to_node: Dict[str, SNode] = dict(**{n.smiles: n for n in self.root_nodes})
         self.edges: Set['SEdge'] = set()
         self.done = False
 
+    @staticmethod
+    def from_pathway(pw: 'Pathway', persist: bool = True):
+        """ Initializes a SPathway with a state given by a Pathway """
+        spw = SPathway(root_nodes=pw.root_nodes, persist=pw if persist else None, prediction_setting=pw.setting)
+        # root_nodes are already added in __init__, add remaining nodes
+        for n in pw.nodes:
+            snode = SNode(n.default_node_label.smiles, n.depth)
+            if snode.smiles not in spw.smiles_to_node:
+                spw.smiles_to_node[snode.smiles] = snode
+                spw.snode_persist_lookup[snode] = n
+
+        for e in pw.edges:
+            sub = []
+            prod = []
+
+            for n in e.start_nodes.all():
+                sub.append(spw.smiles_to_node[n.default_node_label.smiles])
+
+            for n in e.end_nodes.all():
+                prod.append(spw.smiles_to_node[n.default_node_label.smiles])
+
+            rule = None
+            if e.edge_label.rules.all():
+                rule = e.edge_label.rules.all().first()
+
+            prob = None
+            if e.kv.get('probability'):
+                prob = float(e.kv['probability'])
+
+            sedge = SEdge(sub, prod, rule=rule, probability=prob)
+            spw.edges.add(sedge)
+            spw.sedge_persist_lookup[sedge] = e
+
+        return spw
+
     def num_nodes(self):
         return len(self.smiles_to_node.keys())
 
@@ -830,8 +1020,18 @@ class SPathway(object):
 
         return sorted(res, key=lambda x: hash(x))
 
-    def predict_step(self, from_depth: int = 0):
-        substrates = self._get_nodes_for_depth(from_depth)
+    def predict_step(self, from_depth: int = None, from_node: 'Node' = None):
+        substrates: List[SNode] = []
+
+        if from_depth is not None:
+            substrates = self._get_nodes_for_depth(from_depth)
+        elif from_node is not None:
+            for k,v in self.snode_persist_lookup.items():
+                if from_node == v:
+                    substrates = [k]
+                    break
+        else:
+            raise ValueError("Neither from_depth nor from_node_url specified")
 
         new_tp = False
         if substrates:
@@ -849,13 +1049,19 @@ class SPathway(object):
                                 node = self.smiles_to_node[c]
                                 cand_nodes.append(node)
 
-                            edge = SEdge(sub, cand_nodes, cand_set.rule)
+                            edge = SEdge(sub, cand_nodes, rule=cand_set.rule, probability=cand_set.probability)
                             self.edges.add(edge)
-        else:
+
+        # In case no substrates are found, we're done.
+        # For "predict from node" we're always done
+        if len(substrates) == 0 or from_node is not None:
             self.done = True
 
+        # Check if we need to write back data to database
         if new_tp and self.persist:
             self._sync_to_pathway()
+            # call save to update internal modified field
+            self.persist.save()
 
     def _sync_to_pathway(self):
         logger.info("Updating Pathway with SPathway")
@@ -876,6 +1082,11 @@ class SPathway(object):
                     product_nodes.append(self.snode_persist_lookup[snode])
 
                 e = Edge.create(self.persist, educt_nodes, product_nodes, sedge.rule)
+
+                if sedge.probability:
+                    e.kv.update({'probability': sedge.probability})
+                    e.save()
+
                 self.sedge_persist_lookup[sedge] = e
 
         logger.info("Update done!")

epdb/management/commands/bootstrap.py

@@ -13,12 +13,14 @@ class Command(BaseCommand):
     def create_users(self):
 
         if not User.objects.filter(email='anon@lorsba.ch').exists():
-            anon = UserManager.create_user("anonymous", "anon@lorsba.ch", "SuperSafe", is_active=True)
+            anon = UserManager.create_user("anonymous", "anon@lorsba.ch", "SuperSafe", is_active=True,
+                                           add_to_group=False, set_setting=False)
         else:
             anon = User.objects.get(email='anon@lorsba.ch')
 
         if not User.objects.filter(email='admin@lorsba.ch').exists():
-            admin = UserManager.create_user("admin", "admin@lorsba.ch", "SuperSafe", is_active=True)
+            admin = UserManager.create_user("admin", "admin@lorsba.ch", "SuperSafe", is_active=True, add_to_group=False,
+                                            set_setting=False)
             admin.is_staff = True
             admin.is_superuser = True
             admin.save()
@@ -26,6 +28,9 @@ class Command(BaseCommand):
             admin = User.objects.get(email='admin@lorsba.ch')
 
         g = GroupManager.create_group(admin, 'enviPath Users', 'All enviPath Users')
+        g.public = True
+        g.save()
+
         g.user_member.add(anon)
         g.save()
 
@@ -36,7 +41,8 @@ class Command(BaseCommand):
         admin.save()
 
         if not User.objects.filter(email='jebus@lorsba.ch').exists():
-            jebus = UserManager.create_user("jebus", "jebus@lorsba.ch", "SuperSafe", is_active=True)
+            jebus = UserManager.create_user("jebus", "jebus@lorsba.ch", "SuperSafe", is_active=True, add_to_group=False,
+                                            set_setting=False)
             jebus.is_staff = True
             jebus.is_superuser = True
             jebus.save()
@@ -94,6 +100,9 @@ class Command(BaseCommand):
         setting.make_global_default()
 
         for u in [anon, jebus]:
+            u.default_setting = setting
+            u.save()
+
             usp = UserSettingPermission()
             usp.user = u
             usp.setting = setting
@@ -119,7 +128,7 @@ class Command(BaseCommand):
 
         X, y = ml_model.build_dataset()
         ml_model.build_model(X, y)
-        ml_model.evaluate_model()
+        # ml_model.evaluate_model()
 
         # If available create EnviFormerModel
         if s.ENVIFORMER_PRESENT:

epdb/models.py

@@ -14,7 +14,7 @@ from django.contrib.auth.hashers import make_password, check_password
 from django.contrib.auth.models import AbstractUser
 from django.contrib.postgres.fields import ArrayField
 from django.db import models, transaction
-from django.db.models import JSONField
+from django.db.models import JSONField, Count, Q
 from django.utils import timezone
 from django.utils.functional import cached_property
 from model_utils.models import TimeStampedModel
@@ -43,8 +43,6 @@ class User(AbstractUser):
                                       on_delete=models.SET_NULL, related_name='default_group')
     default_setting = models.ForeignKey('epdb.Setting', on_delete=models.SET_NULL,
                                         verbose_name='The users default settings', null=True, blank=False)
-    # TODO remove
-    groups = models.ManyToManyField("Group", verbose_name='groups')
 
     USERNAME_FIELD = "email"
     REQUIRED_FIELDS = ['username']
@@ -87,6 +85,7 @@ class Group(TimeStampedModel):
     uuid = models.UUIDField(null=False, blank=False, verbose_name='UUID of this object', unique=True, default=uuid4)
     name = models.TextField(blank=False, null=False, verbose_name='Group name')
     owner = models.ForeignKey("User", verbose_name='Group Owner', on_delete=models.CASCADE)
+    public = models.BooleanField(verbose_name='Public Group', default=False)
     description = models.TextField(blank=False, null=False, verbose_name='Descriptions', default='no description')
     user_member = models.ManyToManyField("User", verbose_name='User members', related_name='users_in_group')
     group_member = models.ManyToManyField("Group", verbose_name='Group member', related_name='groups_in_group')
@@ -314,7 +313,7 @@ class Compound(EnviPathModel, AliasMixin, ScenarioMixin):
     @transaction.atomic
     def create(package: Package, smiles: str, name: str = None, description: str = None, *args, **kwargs) -> 'Compound':
 
-        if smiles is None or smiles == '':
+        if smiles is None or smiles.strip() == '':
             raise ValueError('SMILES is required')
 
         smiles = smiles.strip()
@@ -338,12 +337,14 @@ class Compound(EnviPathModel, AliasMixin, ScenarioMixin):
         c = Compound()
         c.package = package
 
-        # For name and description we have defaults so only set them if they carry a real value
-        if name is not None and name != '':
-            c.name = name
+        if name is None or name.strip() == '':
+            name = f"Compound {Compound.objects.filter(package=package).count() + 1}"
 
-        if description is not None and description != '':
-            c.description = description
+        c.name = name
+
+        # We have a default here only set the value if it carries some payload
+        if description is not None and description.strip() != '':
+            c.description = description.strip()
 
         c.save()
 
@@ -403,25 +404,27 @@ class Compound(EnviPathModel, AliasMixin, ScenarioMixin):
 class CompoundStructure(EnviPathModel, AliasMixin, ScenarioMixin):
     compound = models.ForeignKey('epdb.Compound', on_delete=models.CASCADE, db_index=True)
     smiles = models.TextField(blank=False, null=False, verbose_name='SMILES')
+    canonical_smiles = models.TextField(blank=False, null=False, verbose_name='Canonical SMILES')
+    inchikey = models.TextField(max_length=27, blank=False, null=False, verbose_name="InChIKey")
     normalized_structure = models.BooleanField(null=False, blank=False, default=False)
 
+    def save(self, *args, **kwargs):
+        # Compute these fields only on initial save call
+        if self.pk is None:
+            try:
+                # Generate canonical SMILES
+                self.canonical_smiles = FormatConverter.canonicalize(self.smiles)
+                # Generate InChIKey
+                self.inchikey = FormatConverter.InChIKey(self.smiles)
+            except Exception as e:
+                logger.error(f"Could compute canonical SMILES/InChIKey from {self.smiles}, error: {e}")
+
+        super().save(*args, **kwargs)
+
     @property
     def url(self):
         return '{}/structure/{}'.format(self.compound.url, self.uuid)
 
-    # @property
-    # def related_pathways(self):
-    #     pathways = Node.objects.filter(node_labels__in=[self]).values_list('pathway', flat=True)
-    #     return Pathway.objects.filter(package=self.compound.package, id__in=set(pathways)).order_by('name')
-
-    # @property
-    # def related_reactions(self):
-    #     return (
-    #             Reaction.objects.filter(package=self.compound.package, educts__in=[self])
-    #             |
-    #             Reaction.objects.filter(package=self.compound.package, products__in=[self])
-    #     ).order_by('name')
-
     @staticmethod
     @transaction.atomic
     def create(compound: Compound, smiles: str, name: str = None, description: str = None, *args, **kwargs):
@@ -448,19 +451,9 @@ class CompoundStructure(EnviPathModel, AliasMixin, ScenarioMixin):
 
         return cs
 
-    # TODO add find method
-
     @property
-    def InChIKey(self):
-        return FormatConverter.InChIKey(self.smiles)
-
-    @property
-    def canonical_smiles(self):
-        return FormatConverter.canonicalize(self.smiles)
-
-    @property
-    def as_svg(self):
-        return IndigoUtils.mol_to_svg(self.smiles)
+    def as_svg(self, width: int = 800, height: int = 400):
+        return IndigoUtils.mol_to_svg(self.smiles, width=width, height=height)
 
 
 class Rule(PolymorphicModel, EnviPathModel, AliasMixin, ScenarioMixin):
@@ -492,19 +485,9 @@ class Rule(PolymorphicModel, EnviPathModel, AliasMixin, ScenarioMixin):
 
     @staticmethod
     @transaction.atomic
-    def create(package: Package, rule_type: str, name: str = None, description: str = None, *args, **kwargs):
-        r = Rule.cls_for_type(rule_type)()
-        r.package = package
-        r.name = name
-        r.description = description
-
-        # As we are setting params this way the "k" has to match the property name
-        for k, v in kwargs.items():
-            setattr(r, k, v)
-
-        r.save()
-        return r
-
+    def create(rule_type: str, *args, **kwargs):
+        cls = Rule.cls_for_type(rule_type)
+        return cls.create(*args, **kwargs)
 
 #
 #     @property
@@ -533,6 +516,54 @@ class SimpleAmbitRule(SimpleRule):
     reactant_filter_smarts = models.TextField(null=True, verbose_name='Reactant Filter SMARTS')
     product_filter_smarts = models.TextField(null=True, verbose_name='Product Filter SMARTS')
 
+    @staticmethod
+    @transaction.atomic
+    def create(package: Package, name: str = None, description: str = None, smirks: str = None,
+               reactant_filter_smarts: str = None, product_filter_smarts: str = None):
+
+        if smirks is None or smirks.strip() == '':
+            raise ValueError('SMIRKS is required!')
+
+        smirks = smirks.strip()
+
+        if not FormatConverter.is_valid_smirks(smirks):
+            raise ValueError(f'SMIRKS "{smirks}" is invalid!')
+
+        query = SimpleAmbitRule.objects.filter(package=package, smirks=smirks)
+
+        if reactant_filter_smarts is not None and reactant_filter_smarts.strip() != '':
+            query = query.filter(reactant_filter_smarts=reactant_filter_smarts)
+
+        if product_filter_smarts is not None and product_filter_smarts.strip() != '':
+            query = query.filter(product_filter_smarts=product_filter_smarts)
+
+        if query.exists():
+            if query.count() > 1:
+                logger.error(f'More than one rule matched this one! {query}')
+            return query.first()
+
+        r = SimpleAmbitRule()
+        r.package = package
+
+        if name is None or name.strip() == '':
+            name = f'Rule {Rule.objects.filter(package=package).count() + 1}'
+
+        r.name = name
+
+        if description is not None and description.strip() != '':
+            r.description = description
+
+        r.smirks = smirks
+
+        if reactant_filter_smarts is not None and reactant_filter_smarts.strip() != '':
+            r.reactant_filter_smarts = reactant_filter_smarts
+
+        if product_filter_smarts is not None and product_filter_smarts.strip() != '':
+            r.product_filter_smarts = product_filter_smarts
+
+        r.save()
+        return r
+
     @property
     def url(self):
         return '{}/simple-ambit-rule/{}'.format(self.package.url, self.uuid)
@@ -642,7 +673,7 @@ class Reaction(EnviPathModel, AliasMixin, ScenarioMixin):
     def create(package: Package, name: str = None, description: str = None,
                educts: Union[List[str], List[CompoundStructure]] = None,
                products: Union[List[str], List[CompoundStructure]] = None,
-               rule: Rule = None, multi_step: bool = True):
+               rules: Union[Rule|List[Rule]] = None, multi_step: bool = True):
 
         _educts = []
         _products = []
@@ -662,18 +693,61 @@ class Reaction(EnviPathModel, AliasMixin, ScenarioMixin):
             _products += products
 
         else:
-            raise ValueError("")
+            raise ValueError("Found mixed types for educts and/or products!")
+
+        if len(_educts) == 0 or len(_products) == 0:
+            raise ValueError("No educts or products specified!")
+
+        if rules is None:
+            rules = []
+
+        if isinstance(rules, Rule):
+            rules = [rules]
+
+
+        query = Reaction.objects.annotate(
+            educt_count=Count('educts', filter=Q(educts__in=_educts), distinct=True),
+            product_count=Count('products', filter=Q(products__in=_products), distinct=True),
+        )
+
+        # The annotate/filter wont work if rules is an empty list
+        if rules:
+            query = query.annotate(
+                rule_count=Count('rules', filter=Q(rules__in=rules), distinct=True)
+            ).filter(rule_count=len(rules))
+        else:
+            query = query.annotate(
+                rule_count=Count('rules', distinct=True)
+            ).filter(rule_count=0)
+
+        existing_reaction_qs = query.filter(
+            educt_count=len(_educts),
+            product_count=len(_products),
+            multi_step=multi_step,
+            package=package
+        )
+
+        if existing_reaction_qs.exists():
+            if existing_reaction_qs.count() > 1:
+                logger.error(f'Found more than one reaction for given input! {existing_reaction_qs}')
+            return existing_reaction_qs.first()
 
         r = Reaction()
         r.package = package
-        r.name = name
-        r.description = description
+
+        if name is not None and name.strip() != '':
+            r.name = name
+
+        if description is not None and name.strip() != '':
+            r.description = description
+
         r.multi_step = multi_step
 
         r.save()
 
-        if rule:
-            r.rules.add(rule)
+        if rules:
+            for rule in rules:
+                r.rules.add(rule)
 
         for educt in _educts:
             r.educts.add(educt)
@@ -700,6 +774,7 @@ class Reaction(EnviPathModel, AliasMixin, ScenarioMixin):
 
 class Pathway(EnviPathModel, AliasMixin, ScenarioMixin):
     package = models.ForeignKey('epdb.Package', verbose_name='Package', on_delete=models.CASCADE, db_index=True)
+    setting = models.ForeignKey('epdb.Setting', verbose_name='Setting', on_delete=models.CASCADE, null=True, blank=True)
 
     @property
     def root_nodes(self):
@@ -709,6 +784,12 @@ class Pathway(EnviPathModel, AliasMixin, ScenarioMixin):
     def nodes(self):
         return Node.objects.filter(pathway=self)
 
+    def get_node(self, node_url):
+        for n in self.nodes:
+            if n.url == node_url:
+                return n
+        return None
+
     @property
     def edges(self):
         return Edge.objects.filter(pathway=self)
@@ -717,6 +798,26 @@ class Pathway(EnviPathModel, AliasMixin, ScenarioMixin):
     def url(self):
         return '{}/pathway/{}'.format(self.package.url, self.uuid)
 
+    # Mode
+    def is_built(self):
+        return self.kv.get('mode', 'build') == 'predicted'
+
+    def is_predicted(self):
+        return self.kv.get('mode', 'build') == 'predicted'
+
+    def is_predicted(self):
+        return self.kv.get('mode', 'build') == 'predicted'
+
+    # Status
+    def completed(self):
+        return self.kv.get('status', 'completed') == 'completed'
+
+    def running(self):
+        return self.kv.get('status', 'completed') == 'running'
+
+    def failed(self):
+        return self.kv.get('status', 'completed') == 'failed'
+
     def d3_json(self):
         # Ideally it would be something like this but
         # to reduce crossing in edges do a DFS
@@ -770,7 +871,11 @@ class Pathway(EnviPathModel, AliasMixin, ScenarioMixin):
                 new_link = {
                     'name': link['name'],
                     'id': link['id'],
+                    'url': link['url'],
+                    'image': link['image'],
                     'reaction': link['reaction'],
+                    'reaction_probability': link['reaction_probability'],
+                    'scenarios': link['scenarios'],
                     'source': node_url_to_idx[link['start_node_urls'][0]],
                     'target': pseudo_idx
                 }
@@ -781,7 +886,11 @@ class Pathway(EnviPathModel, AliasMixin, ScenarioMixin):
                     new_link = {
                         'name': link['name'],
                         'id': link['id'],
+                        'url': link['url'],
+                        'image': link['image'],
                         'reaction': link['reaction'],
+                        'reaction_probability': link['reaction_probability'],
+                        'scenarios': link['scenarios'],
                         'source': pseudo_idx,
                         'target': node_url_to_idx[target]
                     }
@@ -797,9 +906,9 @@ class Pathway(EnviPathModel, AliasMixin, ScenarioMixin):
             "completed": "true",
             "description": self.description,
             "id": self.url,
-            "isIncremental": False,
-            "isPredicted": False,
-            "lastModified": 1447842835894,
+            "isIncremental": self.kv.get('mode') == 'incremental',
+            "isPredicted": self.kv.get('mode') == 'predicted',
+            "lastModified": self.modified.strftime('%Y-%m-%d %H:%M:%S'),
             "pathwayName": self.name,
             "reviewStatus": "reviewed" if self.package.reviewed else 'unreviewed',
             "scenarios": [],
@@ -813,18 +922,38 @@ class Pathway(EnviPathModel, AliasMixin, ScenarioMixin):
 
     @staticmethod
     @transaction.atomic
-    def create(package, name, description, smiles):
+    def create(package: 'Package', smiles: str, name: Optional[str] = None, description: Optional[str] = None):
         pw = Pathway()
         pw.package = package
-        pw.name = name
-        pw.description = description
-        pw.save()
 
-        # create root node
-        Node.create(pw, smiles, 0)
+        if name is None:
+            name = f"Pathway {Pathway.objects.filter(package=package).count() + 1}"
+
+        pw.name = name
+
+        if description is not None:
+            pw.description = description
+
+        pw.save()
+        try:
+            # create root node
+            Node.create(pw, smiles, 0)
+        except ValueError as e:
+            # Node creation failed, most likely due to an invalid smiles
+            # delete this pathway...
+            pw.delete()
+            raise e
 
         return pw
 
+    @transaction.atomic
+    def add_node(self, smiles: str, name: Optional[str] = None, description: Optional[str] = None):
+        return Node.create(self, smiles, 0)
+
+    @transaction.atomic
+    def add_edge(self, start_nodes: List['Node'], end_nodes: List['Node'], rule: Optional['Rule'] = None,
+                 name: Optional[str] = None, description: Optional[str] = None):
+        return Edge.create(self, start_nodes, end_nodes, rule, name=name, description=description)
 
 class Node(EnviPathModel, AliasMixin, ScenarioMixin):
     pathway = models.ForeignKey('epdb.Pathway', verbose_name='belongs to', on_delete=models.CASCADE, db_index=True)
@@ -848,14 +977,14 @@ class Node(EnviPathModel, AliasMixin, ScenarioMixin):
             "imageSize": 490,  # TODO
             "name": self.default_node_label.name,
             "smiles": self.default_node_label.smiles,
+            "scenarios": [{'name': s.name, 'url': s.url} for s in self.scenarios.all()],
         }
 
     @staticmethod
-    def create(pathway, smiles, depth):
-        c = Compound.create(pathway.package, smiles)
+    def create(pathway: 'Pathway', smiles: str, depth: int, name: Optional[str] = None, description: Optional[str] = None):
+        c = Compound.create(pathway.package, smiles, name=name, description=description)
 
         if Node.objects.filter(pathway=pathway, default_node_label=c.default_structure).exists():
-            print("found node")
             return Node.objects.get(pathway=pathway, default_node_label=c.default_structure)
 
         n = Node()
@@ -886,34 +1015,21 @@ class Edge(PolymorphicModel, EnviPathModel, AliasMixin, ScenarioMixin):
         return '{}/edge/{}'.format(self.pathway.url, self.uuid)
 
     def d3_json(self):
-        # {
-        #     "ecNumbers": [
-        #         {
-        #             "ecName": "DDT 2,3-dioxygenase",
-        #             "ecNumber": "1.14.12.-"
-        #         }
-        #     ],
-        #     "id": "https://envipath.org/package/32de3cf4-e3e6-4168-956e-32fa5ddb0ce1/pathway/3f58e4d4-1c63-4b30-bf31-7ae4b98899fe/edge/ff193e7b-f010-43d4-acb3-45f34d938824",
-        #     "idreaction": "https://envipath.org/package/32de3cf4-e3e6-4168-956e-32fa5ddb0ce1/reaction/e11419cd-6b46-470b-8a06-a08d62281734",
-        #     "multistep": "false",
-        #     "name": "Eawag BBD reaction r0450",
-        #     "pseudo": False,
-        #     "scenarios": [],
-        #     "source": 0,
-        #     "target": 4
-        # }
-
         return {
             'name': self.name,
             'id': self.url,
-            'reaction': self.edge_label.url if self.edge_label else None,
+            'url': self.url,
+            'image': self.url + '?image=svg',
+            'reaction': {'name': self.edge_label.name, 'url': self.edge_label.url } if self.edge_label else None,
+            'reaction_probability': self.kv.get('probability'),
             # TODO
             'start_node_urls': [x.url for x in self.start_nodes.all()],
             'end_node_urls': [x.url for x in self.end_nodes.all()],
+            "scenarios": [{'name': s.name, 'url': s.url} for s in self.scenarios.all()],
         }
 
     @staticmethod
-    def create(pathway, start_nodes, end_nodes, rule: Optional[Rule] = None, name: Optional[str] = None,
+    def create(pathway, start_nodes: List[Node], end_nodes: List[Node], rule: Optional[Rule] = None, name: Optional[str] = None,
                description: Optional[str] = None):
         e = Edge()
         e.pathway = pathway
@@ -934,13 +1050,17 @@ class Edge(PolymorphicModel, EnviPathModel, AliasMixin, ScenarioMixin):
         r = Reaction.create(pathway.package, name=name, description=description,
                             educts=[n.default_node_label for n in e.start_nodes.all()],
                             products=[n.default_node_label for n in e.end_nodes.all()],
-                            rule=rule, multi_step=False
+                            rules=rule, multi_step=False
                             )
 
         e.edge_label = r
         e.save()
         return e
 
+    @property
+    def as_svg(self):
+        return self.edge_label.as_svg if self.edge_label else None
+
 
 class EPModel(PolymorphicModel, EnviPathModel):
     package = models.ForeignKey('epdb.Package', verbose_name='Package', on_delete=models.CASCADE, db_index=True)
@@ -976,6 +1096,9 @@ class MLRelativeReasoning(EPModel):
 
     eval_results = JSONField(null=True, blank=True, default=dict)
 
+    def status(self):
+        return self.PROGRESS_STATUS_CHOICES[self.model_status]
+
     @staticmethod
     @transaction.atomic
     def create(package, name, description, rule_packages, data_packages, eval_packages, threshold):
@@ -1201,7 +1324,7 @@ class MLRelativeReasoning(EPModel):
         self.save()
 
         mod = SparseLabelECC(
-            **s.DEFAULT_MODELS_PARAMS
+            **s.DEFAULT_DT_MODEL_PARAMS
         )
 
         mod.fit(X, y)
@@ -1247,7 +1370,7 @@ class MLRelativeReasoning(EPModel):
             y_train, y_test = y[train_index], y[test_index]
 
             model = SparseLabelECC(
-                **s.DEFAULT_MODELS_PARAMS
+                **s.DEFAULT_DT_MODEL_PARAMS
             )
             model.fit(X_train, y_train)
 
@@ -1500,11 +1623,15 @@ class Setting(EnviPathModel):
     max_nodes = models.IntegerField(null=False, blank=False, verbose_name='Setting Max Number of Nodes', default=30)
 
     rule_packages = models.ManyToManyField("Package", verbose_name="Setting Rule Packages",
-                                           related_name="setting_rule_packages")
+                                           related_name="setting_rule_packages", blank=True)
     model = models.ForeignKey('EPModel', verbose_name='Setting EPModel', on_delete=models.SET_NULL, null=True,
                               blank=True)
     model_threshold = models.FloatField(null=True, blank=True, verbose_name='Setting Model Threshold', default=0.25)
 
+    @property
+    def url(self):
+        return '{}/setting/{}'.format(s.SERVER_URL, self.uuid)
+
     @cached_property
     def applicable_rules(self):
         """

epdb/tasks.py

@@ -1,4 +1,6 @@
 import logging
+from typing import Optional
+
 from celery.signals import worker_process_init
 from celery import shared_task
 from epdb.models import Pathway, Node, Edge, EPModel, Setting
@@ -40,11 +42,40 @@ def evaluate_model(model_pk: int):
 
 
 @shared_task(queue='predict')
-def predict(pw_pk: int, pred_setting_pk: int):
+def predict(pw_pk: int, pred_setting_pk: int, limit: Optional[int] = None, node_pk: Optional[int] = None) -> Pathway:
     pw = Pathway.objects.get(id=pw_pk)
     setting = Setting.objects.get(id=pred_setting_pk)
-    spw = SPathway(prediction_setting=setting, persist=pw)
-    level = 0
-    while not spw.done:
-        spw.predict_step(from_depth=level)
-        level += 1
+
+    pw.kv.update(**{'status': 'running'})
+    pw.save()
+
+    try:
+        # regular prediction
+        if limit is not None:
+            spw = SPathway(prediction_setting=setting, persist=pw)
+            level = 0
+            while not spw.done:
+                spw.predict_step(from_depth=level)
+                level += 1
+
+                # break in case we are in incremental model
+                if limit != -1:
+                    if level >= limit:
+                        break
+
+        elif node_pk is not None:
+            n = Node.objects.get(id=node_pk, pathway=pw)
+            spw = SPathway.from_pathway(pw)
+            spw.predict_step(from_node=n)
+        else:
+            raise ValueError("Neither limit nor node_pk given!")
+
+
+
+    except Exception as e:
+        pw.kv.update({'status': 'failed'})
+        pw.save()
+        raise e
+
+    pw.kv.update(**{'status': 'completed'})
+    pw.save()

epdb/templatetags/envipytags.py

@@ -0,0 +1,7 @@
+from django import template
+
+register = template.Library()
+
+@register.filter
+def classname(obj):
+    return obj.__class__.__name__

epdb/urls.py

@@ -13,6 +13,8 @@ urlpatterns = [
     # Home
     re_path(r'^$', v.index, name='index'),
 
+    # re_path(r'^login', v.login, name='login'),
+
     # Top level urls
     re_path(r'^package$', v.packages, name='packages'),
     re_path(r'^compound$', v.compounds, name='compounds'),
@@ -53,11 +55,11 @@ urlpatterns = [
     re_path(rf'^package/(?P<package_uuid>{UUID})/pathway$', v.package_pathways, name='package pathway list'),
     re_path(rf'^package/(?P<package_uuid>{UUID})/pathway/(?P<pathway_uuid>{UUID})$', v.package_pathway, name='package pathway detail'),
     # Pathway Nodes
-    # re_path(rf'^package/(?P<package_uuid>{UUID})/pathway(?P<pathway_uuid>{UUID})/node$', v.package_pathway_nodes, name='package pathway node list'),
+    re_path(rf'^package/(?P<package_uuid>{UUID})/pathway/(?P<pathway_uuid>{UUID})/node$', v.package_pathway_nodes, name='package pathway node list'),
     re_path(rf'^package/(?P<package_uuid>{UUID})/pathway/(?P<pathway_uuid>{UUID})/node/(?P<node_uuid>{UUID})$', v.package_pathway_node, name='package pathway node detail'),
     # Pathway Edges
-    # re_path(rf'^package/(?P<package_uuid>{UUID})/pathway(?P<pathway_uuid>{UUID})/edge$', v.package_pathway_edges, name='package pathway edge list'),
-    # re_path(rf'^package/(?P<package_uuid>{UUID})/pathway(?P<pathway_uuid>{UUID})/edge/(?P<edge_uuid>{UUID})$', v.package_pathway_edge, name='package pathway edge detail'),
+    re_path(rf'^package/(?P<package_uuid>{UUID})/pathway/(?P<pathway_uuid>{UUID})/edge$', v.package_pathway_edges, name='package pathway edge list'),
+    re_path(rf'^package/(?P<package_uuid>{UUID})/pathway/(?P<pathway_uuid>{UUID})/edge/(?P<edge_uuid>{UUID})$', v.package_pathway_edge, name='package pathway edge detail'),
     # Scenario
     re_path(rf'^package/(?P<package_uuid>{UUID})/scenario$', v.package_scenarios, name='package scenario list'),
     re_path(rf'^package/(?P<package_uuid>{UUID})/scenario/(?P<scenario_uuid>{UUID})$', v.package_scenario, name='package scenario detail'),

epdb/views.py

@@ -1,5 +1,6 @@
 import json
 import logging
+from functools import wraps
 from typing import List, Dict, Any
 
 from django.conf import settings as s
@@ -7,29 +8,71 @@ from django.contrib.auth import get_user_model
 from django.http import JsonResponse, HttpResponse, HttpResponseNotAllowed, HttpResponseBadRequest
 from django.shortcuts import render, redirect
 from django.views.decorators.csrf import csrf_exempt
+from django.contrib.auth.decorators import login_required
 
 from utilities.chem import FormatConverter, IndigoUtils
-from .logic import GroupManager, PackageManager, UserManager, SettingManager
+from .logic import GroupManager, PackageManager, UserManager, SettingManager, SearchManager
 from .models import Package, GroupPackagePermission, Group, CompoundStructure, Compound, Reaction, Rule, Pathway, Node, \
     EPModel, EnviFormer, MLRelativeReasoning, RuleBaseRelativeReasoning, Scenario, SimpleAmbitRule, APIToken, \
-    UserPackagePermission, Permission, License, User
+    UserPackagePermission, Permission, License, User, Edge
 
 logger = logging.getLogger(__name__)
 
 def log_post_params(request):
-    for k, v in request.POST.items():
-        logger.debug(f"{k}\t{v}")
+    if s.DEBUG:
+        for k, v in request.POST.items():
+            logger.debug(f"{k}\t{v}")
+
+
+def catch_exceptions(view_func):
+    @wraps(view_func)
+    def _wrapped_view(request, *args, **kwargs):
+        try:
+            return view_func(request, *args, **kwargs)
+        except Exception as e:
+            # Optionally return JSON or plain HttpResponse
+            if request.headers.get('Accept') == 'application/json':
+                return JsonResponse(
+                    {'error': 'Internal server error. Please try again later.'},
+                    status=500
+                )
+            else:
+                return render(request, 'errors/error.html', get_base_context(request))
+    return _wrapped_view
+
+
+def editable(request, user):
+    url = request.build_absolute_uri(request.path)
+    if PackageManager.is_package_url(url):
+        _package = PackageManager.get_package_lp(request.build_absolute_uri())
+        return PackageManager.writable(user, _package)
+    elif GroupManager.is_group_url(url):
+        _group = GroupManager.get_group_lp(request.build_absolute_uri())
+        return GroupManager.writable(user, _group)
+    elif UserManager.is_user_url(url):
+        _user = UserManager.get_user_lp(request.build_absolute_uri())
+        return UserManager.writable(user, _user)
+    elif url in [s.SERVER_URL, f"{s.SERVER_URL}/", f"{s.SERVER_URL}/package", f"{s.SERVER_URL}/user",
+                 f"{s.SERVER_URL}/group", f"{s.SERVER_URL}/search"]:
+        return True
+    else:
+        print(f"Unknown url: {url}")
+        return False
+
 
 
 def get_base_context(request) -> Dict[str, Any]:
     current_user = _anonymous_or_real(request)
 
+    can_edit = editable(request, current_user)
+
     ctx = {
         'title': 'enviPath',
         'meta': {
             'version': '0.0.1',
             'server_url': s.SERVER_URL,
             'user': current_user,
+            'can_edit': can_edit,
             'readable_packages': PackageManager.get_all_readable_packages(current_user, include_reviewed=True),
             'writeable_packages': PackageManager.get_all_writeable_packages(current_user),
             'available_groups': GroupManager.get_groups(current_user),
@@ -65,8 +108,9 @@ def breadcrumbs(first_level_object=None, second_level_namespace=None, second_lev
     return bread
 
 
+# @catch_exceptions
+
 def index(request):
-    current_user = _anonymous_or_real(request)
     context = get_base_context(request)
     context['title'] = 'enviPath - Home'
     context['meta']['current_package'] = context['meta']['user'].default_package
@@ -77,6 +121,16 @@ def index(request):
     return render(request, 'index/index.html', context)
 
 
+# def login(request):
+#     current_user = _anonymous_or_real(request)
+#     if request.method == 'GET':
+#         context = get_base_context(request)
+#         context['title'] = 'enviPath'
+#         return render(request, 'login.html', context)
+#     else:
+#         return HttpResponseBadRequest()
+#
+# @login_required(login_url='/login')
 def packages(request):
     current_user = _anonymous_or_real(request)
 
@@ -86,9 +140,10 @@ def packages(request):
 
         context['object_type'] = 'package'
         context['meta']['current_package'] = context['meta']['user'].default_package
+        context['meta']['can_edit'] = True
 
-        reviewed_package_qs = Package.objects.filter(reviewed=True)
-        unreviewed_package_qs = PackageManager.get_all_readable_packages(current_user)
+        reviewed_package_qs = Package.objects.filter(reviewed=True).order_by('created')
+        unreviewed_package_qs = PackageManager.get_all_readable_packages(current_user).order_by('name')
 
         context['reviewed_objects'] = reviewed_package_qs
         context['unreviewed_objects'] = unreviewed_package_qs
@@ -103,7 +158,6 @@ def packages(request):
         else:
             package_name = request.POST.get('package-name')
             package_description = request.POST.get('package-description', s.DEFAULT_VALUES['description'])
-            # group = GroupManager.get_group_by_url(request.user, request.POST.get('package-group'))
 
             created_package = PackageManager.create_package(current_user, package_name, package_description)
 
@@ -342,7 +396,23 @@ def models(request):
 
 
 def search(request):
+    current_user = _anonymous_or_real(request)
+
     if request.method == 'GET':
+        package_urls = request.GET.getlist('packages')
+        searchterm = request.GET.get('search')
+        mode = request.GET.get('mode')
+
+        # add HTTP_ACCEPT check to differentiate between index and ajax call
+        if 'application/json' in request.META.get('HTTP_ACCEPT') and all([searchterm, mode]):
+            if package_urls:
+                packages = [PackageManager.get_package_by_url(current_user, p) for p in package_urls]
+            else:
+                packages = PackageManager.get_reviewed_packages()
+
+            search_result = SearchManager.search(packages, searchterm, mode)
+            return JsonResponse(search_result, safe=False)
+
         context = get_base_context(request)
         context['title'] = 'enviPath - Search'
 
@@ -352,13 +422,21 @@ def search(request):
             {'Home': s.SERVER_URL},
             {'Search': s.SERVER_URL + '/search'},
         ]
-        # TODO perm
-        reviewed_package_qs = Package.objects.filter(reviewed=True)
-        unreviewed_package_qs = Package.objects.filter(reviewed=False)
+
+        reviewed_package_qs = PackageManager.get_reviewed_packages()
+        unreviewed_package_qs = PackageManager.get_all_readable_packages(current_user)
 
         context['reviewed_objects'] = reviewed_package_qs
         context['unreviewed_objects'] = unreviewed_package_qs
 
+        if all([searchterm, mode]):
+            if package_urls:
+                packages = [PackageManager.get_package_by_url(current_user, p) for p in package_urls]
+            else:
+                packages = PackageManager.get_reviewed_packages()
+
+            context['search_result'] = SearchManager.search(packages, searchterm, mode)
+
         return render(request, 'search.html', context)
 
 
@@ -487,7 +565,7 @@ def package_model(request, package_uuid, model_uuid):
 
         return render(request, 'objects/model.html', context)
 
-    if request.method == 'POST':
+    elif request.method == 'POST':
         if hidden := request.POST.get('hidden', None):
             if hidden == 'delete-model':
                 current_model.delete()
@@ -496,21 +574,9 @@ def package_model(request, package_uuid, model_uuid):
                 return HttpResponseBadRequest()
         else:
             return HttpResponseBadRequest()
-        #
-        # new_compound_name = request.POST.get('compound-name')
-        # new_compound_description = request.POST.get('compound-description')
-        #
-        # if new_compound_name:
-        #     current_compound.name = new_compound_name
-        #
-        # if new_compound_description:
-        #     current_compound.description = new_compound_description
-        #
-        # if any([new_compound_name, new_compound_description]):
-        #     current_compound.save()
-        #     return redirect(current_compound.url)
-        # else:
-        #     return HttpResponseBadRequest()
+
+    else:
+        return HttpResponseBadRequest()
 
 
 def package(request, package_uuid):
@@ -803,8 +869,7 @@ def package_rules(request, package_uuid):
 
     elif request.method == 'POST':
 
-        for k, v in request.POST.items():
-            print(k, v)
+        log_post_params(request)
 
         # Generic params
         rule_name = request.POST.get('rule-name')
@@ -817,8 +882,8 @@ def package_rules(request, package_uuid):
         # Obtain parameters as required by rule type
         if rule_type == 'SimpleAmbitRule':
             params['smirks'] = request.POST.get('rule-smirks')
-            params['reactant_smarts'] = request.POST.get('rule-reactant-smarts')
-            params['product_smarts'] = request.POST.get('rule-product-smarts')
+            params['reactant_filter_smarts'] = request.POST.get('rule-reactant-smarts')
+            params['product_filter_smarts'] = request.POST.get('rule-product-smarts')
         elif rule_type == 'SimpleRDKitRule':
             params['reaction_smarts'] = request.POST.get('rule-reaction-smarts')
         elif rule_type == 'ParallelRule':
@@ -828,7 +893,7 @@ def package_rules(request, package_uuid):
         else:
             return HttpResponseBadRequest()
 
-        r = Rule.create(current_package, rule_type, name=rule_name, description=rule_description, **params)
+        r = Rule.create(rule_type=rule_type, package=current_package, name=rule_name, description=rule_description, **params)
         return redirect(r.url)
 
     else:
@@ -854,7 +919,7 @@ def package_rule(request, package_uuid, rule_uuid):
         else:  # isinstance(current_rule, ParallelRule) or isinstance(current_rule, SequentialRule):
             return render(request, 'objects/composite_rule.html', context)
 
-    if request.method == 'POST':
+    elif request.method == 'POST':
         if hidden := request.POST.get('hidden', None):
             if hidden == 'delete-rule':
                 current_rule.delete()
@@ -862,8 +927,23 @@ def package_rule(request, package_uuid, rule_uuid):
             else:
                 return HttpResponseBadRequest()
 
-        # TODO update!
+        rule_name = request.POST.get('rule-name', '').strip()
+        rule_description = request.POST.get('rule-description', '').strip()
 
+        if rule_name:
+            current_rule.name = rule_name
+
+        if rule_description:
+            current_rule.description = rule_description
+
+        if any([rule_name, rule_description]):
+            current_rule.save()
+            return redirect(current_rule.url)
+        else:
+            return HttpResponseBadRequest()
+
+    else:
+        return HttpResponseBadRequest()
 
 # https://envipath.org/package/<id>/reaction
 def package_reactions(request, package_uuid):
@@ -912,6 +992,8 @@ def package_reactions(request, package_uuid):
 
         return redirect(r.url)
 
+    else:
+        return HttpResponseBadRequest()
 
 # https://envipath.org/package/<id>/reaction/<id>
 def package_reaction(request, package_uuid, reaction_uuid):
@@ -931,7 +1013,7 @@ def package_reaction(request, package_uuid, reaction_uuid):
 
         return render(request, 'objects/reaction.html', context)
 
-    if request.method == 'POST':
+    elif request.method == 'POST':
         if hidden := request.POST.get('hidden', None):
             if hidden == 'delete-reaction':
                 current_reaction.delete()
@@ -954,6 +1036,8 @@ def package_reaction(request, package_uuid, reaction_uuid):
         else:
             return HttpResponseBadRequest()
 
+    else:
+        return HttpResponseBadRequest()
 
 # https://envipath.org/package/<id>/pathway
 def package_pathways(request, package_uuid):
@@ -989,7 +1073,7 @@ def package_pathways(request, package_uuid):
 
         return render(request, 'collections/objects_list.html', context)
 
-    if request.method == 'POST':
+    elif request.method == 'POST':
 
         log_post_params(request)
 
@@ -1002,15 +1086,34 @@ def package_pathways(request, package_uuid):
             return HttpResponseBadRequest()
 
         stand_smiles = FormatConverter.standardize(smiles)
-        pw = Pathway.create(current_package, name, description, stand_smiles)
 
-        if pw_mode != 'build':
+        if pw_mode not in ['predict', 'build', 'incremental']:
+            return HttpResponseBadRequest()
+
+        pw = Pathway.create(current_package, name, description, stand_smiles)
+        # set mode
+        pw.kv.update({'mode': pw_mode})
+        pw.save()
+
+        if pw_mode == 'predict' or pw_mode == 'incremental':
+            # unlimited pred (will be handled by setting)
+            limit = -1
+
+            # For incremental predict first level and return
+            if pw_mode == 'incremental':
+                limit = 1
+
             pred_setting = current_user.prediction_settings()
+            pw.setting = pred_setting
+            pw.save()
+
             from .tasks import predict
-            predict.delay(pw.pk, pred_setting.pk)
+            predict.delay(pw.pk, pred_setting.pk, limit=limit)
 
         return redirect(pw.url)
 
+    else:
+        return HttpResponseBadRequest()
 
 # https://envipath.org/package/<id>/pathway/<id>
 def package_pathway(request, package_uuid, pathway_uuid):
@@ -1043,7 +1146,7 @@ def package_pathway(request, package_uuid, pathway_uuid):
         return render(request, 'objects/pathway.html', context)
         # return render(request, 'pathway_playground2.html', context)
 
-    if request.method == 'POST':
+    elif request.method == 'POST':
         if hidden := request.POST.get('hidden', None):
             if hidden == 'delete-pathway':
                 current_pathway.delete()
@@ -1051,30 +1154,92 @@ def package_pathway(request, package_uuid, pathway_uuid):
             else:
                 return HttpResponseBadRequest()
 
+        pathway_name = request.POST.get('pathway-name')
+        pathway_description = request.POST.get('pathway-description')
 
-#
-#
-#
-# def package_relative_reasonings(request, package_id):
-#     if request.method == 'GET':
-#         pass
-#
-#
-# def package_relative_reasoning(request, package_id, relative_reasoning_id):
-#     current_user = _anonymous_or_real(request)
-#
-#     if request.method == 'GET':
-#         pass
-#     elif request.method == 'POST':
-#         pass
-#
-# #
-# #
-# # # https://envipath.org/package/<id>/pathway/<id>/node
-# # def package_pathway_nodes(request, package_id, pathway_id):
-# #     pass
-# #
-# #
+        if any([pathway_name, pathway_description]):
+            if pathway_name is not None and pathway_name.strip() != '':
+                pathway_name = pathway_name.strip()
+
+                current_pathway.name = pathway_name
+
+            if pathway_description is not None and pathway_description.strip() != '':
+                pathway_description = pathway_description.strip()
+
+                current_pathway.description = pathway_description
+
+            current_pathway.save()
+            return redirect(current_pathway.url)
+
+        node_url = request.POST.get('node')
+
+        if node_url:
+            n = current_pathway.get_node(node_url)
+
+            from .tasks import predict
+            # Dont delay?
+            predict(current_pathway.pk, current_pathway.setting.pk, node_pk=n.pk)
+            return JsonResponse({'success': current_pathway.url})
+
+        return HttpResponseBadRequest()
+
+    else:
+        return HttpResponseBadRequest()
+
+# https://envipath.org/package/<id>/pathway/<id>/node
+def package_pathway_nodes(request, package_uuid, pathway_uuid):
+    current_user = _anonymous_or_real(request)
+    current_package = PackageManager.get_package_by_id(current_user, package_uuid)
+    current_pathway = Pathway.objects.get(package=current_package, uuid=pathway_uuid)
+
+    if request.method == 'GET':
+        context = get_base_context(request)
+        context['title'] = f'enviPath - {current_package.name} - {current_pathway.name} - Nodes'
+
+        context['meta']['current_package'] = current_package
+        context['object_type'] = 'node'
+        context['breadcrumbs'] = [
+            {'Home': s.SERVER_URL},
+            {'Package': s.SERVER_URL + '/package'},
+            {current_package.name: current_package.url},
+            {'Pathway': current_package.url + '/pathway'},
+            {current_pathway.name: current_pathway.url},
+            {'Node': current_pathway.url + '/node'},
+        ]
+
+        reviewed_node_qs = Node.objects.none()
+        unreviewed_node_qs = Node.objects.none()
+
+        if current_package.reviewed:
+            reviewed_node_qs = Node.objects.filter(pathway=current_pathway).order_by('name')
+        else:
+            unreviewed_node_qs = Node.objects.filter(pathway=current_pathway).order_by('name')
+
+        if request.GET.get('all'):
+            return JsonResponse({
+                "objects": [
+                    {"name": pw.name, "url": pw.url, "reviewed": current_package.reviewed}
+                    for pw in (reviewed_node_qs if current_package.reviewed else unreviewed_node_qs)
+                ]
+            })
+
+        context['reviewed_objects'] = reviewed_node_qs
+        context['unreviewed_objects'] = unreviewed_node_qs
+
+        return render(request, 'collections/objects_list.html', context)
+
+    elif request.method == 'POST':
+
+        node_name = request.POST.get('node-name')
+        node_description = request.POST.get('node-description')
+        node_smiles = request.POST.get('node-smiles')
+
+        current_pathway.add_node(node_smiles, name=node_name, description=node_description)
+
+        return redirect(current_pathway.url)
+
+    else:
+        return HttpResponseBadRequest()
 
 
 # https://envipath.org/package/<id>/pathway/<id>/node/<id>
@@ -1091,19 +1256,129 @@ def package_pathway_node(request, package_uuid, pathway_uuid, node_uuid):
                 svg_data = current_node.as_svg
                 return HttpResponse(svg_data, content_type="image/svg+xml")
 
+        context = get_base_context(request)
+        context['title'] = f'enviPath - {current_package.name} - {current_pathway.name}'
+
+        context['meta']['current_package'] = current_package
+        context['object_type'] = 'pathway'
+
+        context['breadcrumbs'] = [
+            {'Home': s.SERVER_URL},
+            {'Package': s.SERVER_URL + '/package'},
+            {current_package.name: current_package.url},
+            {'Pathway': current_package.url + '/pathway'},
+            {current_pathway.name: current_pathway.url},
+            {'Node': current_pathway.url + '/node'},
+            {current_node.name: current_node.url},
+        ]
+
+        context['node'] = current_node
+
+        return render(request, 'objects/node.html', context)
+
+    elif request.method == 'POST':
+        if s.DEBUG:
+            for k, v in request.POST.items():
+                print(k, v)
+
+        if hidden := request.POST.get('hidden', None):
+            if hidden == 'delete-node':
+                current_node.delete()
+                return redirect(current_pathway.url)
+
+    else:
+        return HttpResponseBadRequest()
+
+
+# https://envipath.org/package/<id>/pathway/<id>/edge
+def package_pathway_edges(request, package_uuid, pathway_uuid):
+    current_user = _anonymous_or_real(request)
+    current_package = PackageManager.get_package_by_id(current_user, package_uuid)
+    current_pathway = Pathway.objects.get(package=current_package, uuid=pathway_uuid)
+
+    if request.method == 'GET':
+        context = get_base_context(request)
+        context['title'] = f'enviPath - {current_package.name} - {current_pathway.name} - Edges'
+
+        context['meta']['current_package'] = current_package
+        context['object_type'] = 'edge'
+        context['breadcrumbs'] = [
+            {'Home': s.SERVER_URL},
+            {'Package': s.SERVER_URL + '/package'},
+            {current_package.name: current_package.url},
+            {'Pathway': current_package.url + '/pathway'},
+            {current_pathway.name: current_pathway.url},
+            {'Edge': current_pathway.url + '/edge'},
+        ]
+
+        reviewed_edge_qs = Edge.objects.none()
+        unreviewed_edge_qs = Edge.objects.none()
+
+        if current_package.reviewed:
+            reviewed_edge_qs = Edge.objects.filter(pathway=current_pathway).order_by('name')
+        else:
+            unreviewed_edge_qs = Edge.objects.filter(pathway=current_pathway).order_by('name')
+
+        if request.GET.get('all'):
+            return JsonResponse({
+                "objects": [
+                    {"name": pw.name, "url": pw.url, "reviewed": current_package.reviewed}
+                    for pw in (reviewed_edge_qs if current_package.reviewed else unreviewed_edge_qs)
+                ]
+            })
+
+        context['reviewed_objects'] = reviewed_edge_qs
+        context['unreviewed_objects'] = unreviewed_edge_qs
+
+        return render(request, 'collections/objects_list.html', context)
+
+    elif request.method == 'POST':
+
+        edge_name = request.POST.get('edge-name')
+        edge_description = request.POST.get('edge-description')
+        edge_substrates = request.POST.getlist('edge-substrates')
+        edge_products = request.POST.getlist('edge-products')
+
+        substrate_nodes = [current_pathway.get_node(url) for url in edge_substrates]
+        product_nodes = [current_pathway.get_node(url) for url in edge_products]
+
+        # TODO in the future consider Rules here?
+        current_pathway.add_edge(substrate_nodes, product_nodes, name=edge_name, description=edge_description)
+
+        return redirect(current_pathway.url)
+
+    else:
+        return HttpResponseBadRequest()
+
+
+# https://envipath.org/package/<id>/pathway/<id>/edge/<id>
+def package_pathway_edge(request, package_uuid, pathway_uuid, edge_uuid):
+    current_user = _anonymous_or_real(request)
+    current_package = PackageManager.get_package_by_id(current_user, package_uuid)
+    current_pathway = Pathway.objects.get(package=current_package, uuid=pathway_uuid)
+    current_edge = Edge.objects.get(pathway=current_pathway, uuid=edge_uuid)
+
+    if request.method == 'GET':
+        is_image_request = request.GET.get('image')
+        if is_image_request:
+            if is_image_request == 'svg':
+                svg_data = current_edge.as_svg
+                return HttpResponse(svg_data, content_type="image/svg+xml")
+
+    elif request.method == 'POST':
+        if s.DEBUG:
+            for k, v in request.POST.items():
+                print(k, v)
+
+        if hidden := request.POST.get('hidden', None):
+            if hidden == 'delete-edge':
+                current_edge.delete()
+                return redirect(current_pathway.url)
+
+    else:
+        return HttpResponseBadRequest()
+
 
-# #
-# #
-# # # https://envipath.org/package/<id>/pathway/<id>/edge
-# # def package_pathway_edges(request, package_id, pathway_id):
-# #     pass
-# #
-# #
-# # # https://envipath.org/package/<id>/pathway/<id>/edge/<id>
-# # def package_pathway_edge(request, package_id, pathway_id, edge_id):
-# #     pass
-# #
-# #
 # https://envipath.org/package/<id>/scenario
 def package_scenarios(request, package_uuid):
     current_user = _anonymous_or_real(request)
@@ -1158,11 +1433,6 @@ def package_scenario(request, package_uuid, scenario_uuid):
         return render(request, 'objects/scenario.html', context)
 
 
-
-
-### END UNTESTED
-
-
 ##############
 # User/Group #
 ##############
@@ -1201,7 +1471,7 @@ def users(request):
                     return render(request, 'errors/user_account_inactive.html', status=403)
 
                 email = temp_user.email
-            except get_user_model().DoesNotExists:
+            except get_user_model().DoesNotExist:
                 return HttpResponseBadRequest()
 
             user = authenticate(username=email, password=password)
@@ -1289,6 +1559,18 @@ def user(request, user_uuid):
             logout(request)
             return redirect(s.SERVER_URL)
 
+        default_package = request.POST.get('default-package')
+        default_group = request.POST.get('default-group')
+        default_prediction_setting = request.POST.get('default-prediction-setting')
+
+        if any([default_package, default_group, default_prediction_setting]):
+            current_user.default_package = PackageManager.get_package_by_url(current_user, default_package)
+            current_user.default_group = GroupManager.get_group_by_url(current_user, default_group)
+            current_user.default_setting = SettingManager.get_setting_by_url(current_user, default_prediction_setting)
+            current_user.save()
+
+            return redirect(current_user.url)
+
         prediction_model_pk = request.POST.get('model')
         prediction_threshold = request.POST.get('threshold')
         prediction_max_nodes = request.POST.get('max_nodes')
@@ -1509,3 +1791,9 @@ def layout(request):
 def depict(request):
     if smiles := request.GET.get('smiles'):
         return HttpResponse(IndigoUtils.mol_to_svg(smiles), content_type='image/svg+xml')
+
+    elif smirks := request.GET.get('smirks'):
+        query_smirks = request.GET.get('is_query_smirks', False) == 'true'
+        return HttpResponse(IndigoUtils.smirks_to_svg(smirks, query_smirks), content_type='image/svg+xml')
+    else:
+        return HttpResponseBadRequest()