jebus/enviPy-bayer
1
0
Fork 0
forked from enviPath/enviPy
Code Issues Pull Requests Actions Releases Activity

[Feature] Package Export/Import (#116)

Browse Source 
Fixes #90
Fixes #91
Fixes #115
Fixes #104

Co-authored-by: Tim Lorsbach <tim@lorsba.ch>
Reviewed-on: enviPath/enviPy#116
This commit is contained in:
jebus
2025-09-16 02:41:10 +12:00
parent ce349a287b
commit 762a6b7baf
32 changed files with 2500683 additions and 145 deletions
Download Patch File Download Diff File Expand all files Collapse all files

36
epdb/logic.py
View File

@ -10,6 +10,7 @@ from django.conf import settings as s
from epdb.models import User, Package, UserPackagePermission, GroupPackagePermission, Permission, Group, Setting, \
EPModel, UserSettingPermission, Rule, Pathway, Node, Edge, Compound, Reaction, CompoundStructure
from utilities.chem import FormatConverter
from utilities.misc import PackageImporter, PackageExporter
logger = logging.getLogger(__name__)
@ -324,17 +325,6 @@ class PackageManager(object):
return True
return False
# @staticmethod
# def get_package_permission(user: 'User', package: Union[str, 'Package']):
# if PackageManager.administrable(user, package):
# return Permission.ALL[0]
# elif PackageManager.writable(user, package):
# return Permission.WRITE[0]
# elif PackageManager.readable(user, package):
# return Permission.READ[0]
# else:
# return None
@staticmethod
def has_package_permission(user: 'User', package: Union[str, 'Package'], permission: str):
@ -491,7 +481,7 @@ class PackageManager(object):
@staticmethod
@transaction.atomic
def import_package(data: dict, owner: User, keep_ids=False, add_import_timestamp=True, trust_reviewed=False):
def import_legacy_package(data: dict, owner: User, keep_ids=False, add_import_timestamp=True, trust_reviewed=False):
from uuid import UUID, uuid4
from datetime import datetime
from collections import defaultdict
@ -872,6 +862,28 @@ class PackageManager(object):
return pack
@staticmethod
@transaction.atomic
def import_pacakge(data: Dict[str, Any], owner: User, preserve_uuids=False, add_import_timestamp=True,
trust_reviewed=False) -> Package:
importer = PackageImporter(data, preserve_uuids, add_import_timestamp, trust_reviewed)
imported_package = importer.do_import()
up = UserPackagePermission()
up.user = owner
up.package = imported_package
up.permission = up.ALL[0]
up.save()
return imported_package
@staticmethod
def export_package(package: Package, include_models: bool = False,
include_external_identifiers: bool = True) -> Dict[str, Any]:
return PackageExporter(package).do_export()
class SettingManager(object):
setting_pattern = re.compile(r".*/setting/[a-f0-9]{8}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{12}$")

65
epdb/management/commands/bootstrap.py
View File

@ -12,21 +12,24 @@ class Command(BaseCommand):
def create_users(self):
if not User.objects.filter(email='anon@lorsba.ch').exists():
anon = UserManager.create_user("anonymous", "anon@lorsba.ch", "SuperSafe", is_active=True,
add_to_group=False, set_setting=False)
# Anonymous User
if not User.objects.filter(email='anon@envipath.com').exists():
anon = UserManager.create_user("anonymous", "anon@envipath.com", "SuperSafe",
is_active=True, add_to_group=False, set_setting=False)
else:
anon = User.objects.get(email='anon@lorsba.ch')
anon = User.objects.get(email='anon@envipath.com')
if not User.objects.filter(email='admin@lorsba.ch').exists():
admin = UserManager.create_user("admin", "admin@lorsba.ch", "SuperSafe", is_active=True, add_to_group=False,
set_setting=False)
# Admin User
if not User.objects.filter(email='admin@envipath.com').exists():
admin = UserManager.create_user("admin", "admin@envipath.com", "SuperSafe",
is_active=True, add_to_group=False, set_setting=False)
admin.is_staff = True
admin.is_superuser = True
admin.save()
else:
admin = User.objects.get(email='admin@lorsba.ch')
admin = User.objects.get(email='admin@envipath.com')
# System Group
g = GroupManager.create_group(admin, 'enviPath Users', 'All enviPath Users')
g.public = True
g.save()
@ -40,25 +43,25 @@ class Command(BaseCommand):
admin.default_group = g
admin.save()
if not User.objects.filter(email='jebus@lorsba.ch').exists():
jebus = UserManager.create_user("jebus", "jebus@lorsba.ch", "SuperSafe", is_active=True, add_to_group=False,
set_setting=False)
jebus.is_staff = True
jebus.is_superuser = True
jebus.save()
if not User.objects.filter(email='user0@envipath.com').exists():
user0 = UserManager.create_user("user0", "user0@envipath.com", "SuperSafe",
is_active=True, add_to_group=False, set_setting=False)
user0.is_staff = True
user0.is_superuser = True
user0.save()
else:
jebus = User.objects.get(email='jebus@lorsba.ch')
user0 = User.objects.get(email='user0@envipath.com')
g.user_member.add(jebus)
g.user_member.add(user0)
g.save()
jebus.default_group = g
jebus.save()
user0.default_group = g
user0.save()
return anon, admin, g, jebus
return anon, admin, g, user0
def import_package(self, data, owner):
return PackageManager.import_package(data, owner, keep_ids=True, add_import_timestamp=False, trust_reviewed=True)
return PackageManager.import_legacy_package(data, owner, keep_ids=True, add_import_timestamp=False, trust_reviewed=True)
def create_default_setting(self, owner, packages):
s = SettingManager.create_setting(
@ -108,13 +111,6 @@ class Command(BaseCommand):
'base_url': 'https://www.rhea-db.org',
'url_pattern': 'https://www.rhea-db.org/rhea/{id}'
},
{
'name': 'CAS',
'full_name': 'Chemical Abstracts Service Registry',
'description': 'Registry of chemical substances',
'base_url': 'https://www.cas.org',
'url_pattern': None # CAS doesn't have a free public URL pattern
},
{
'name': 'KEGG Reaction',
'full_name': 'KEGG Reaction Database',
@ -122,13 +118,6 @@ class Command(BaseCommand):
'base_url': 'https://www.genome.jp',
'url_pattern': 'https://www.genome.jp/entry/reaction+{id}'
},
{
'name': 'MetaCyc',
'full_name': 'MetaCyc Metabolic Pathway Database',
'description': 'Database of metabolic pathways and enzymes',
'base_url': 'https://metacyc.org',
'url_pattern': None
},
{
'name': 'UniProt',
'full_name': 'MetaCyc Metabolic Pathway Database',
@ -147,7 +136,9 @@ class Command(BaseCommand):
@transaction.atomic
def handle(self, *args, **options):
# Create users
anon, admin, g, jebus = self.create_users()
anon, admin, g, user0 = self.create_users()
self.populate_common_external_databases()
# Import Packages
packages = [
@ -169,7 +160,7 @@ class Command(BaseCommand):
setting.save()
setting.make_global_default()
for u in [anon, jebus]:
for u in [anon, user0]:
u.default_setting = setting
u.save()
@ -200,6 +191,6 @@ class Command(BaseCommand):
ml_model.build_model()
# ml_model.evaluate_model()
# If available create EnviFormerModel
# If available, create EnviFormerModel
if s.ENVIFORMER_PRESENT:
enviFormer_model = EnviFormer.create(pack, 'EnviFormer - T0.5', 'EnviFormer Model with Threshold 0.5', 0.5)

2
epdb/management/commands/import_package.py → epdb/management/commands/import_legacy_package.py
View File

@ -24,4 +24,4 @@ class Command(BaseCommand):
def handle(self, *args, **options):
owner = User.objects.get(username=options['owner'])
package_data = json.load(open(options['data']))
PackageManager.import_package(package_data, owner)
PackageManager.import_legacy_package(package_data, owner)

22
epdb/models.py
View File

@ -578,32 +578,38 @@ class Package(EnviPathModel):
license = models.ForeignKey('epdb.License', on_delete=models.SET_NULL, blank=True, null=True,
verbose_name='License')
def delete(self, *args, **kwargs):
# explicitly handle related Rules
for r in self.rules.all():
r.delete()
super().delete(*args, **kwargs)
def __str__(self):
return f"{self.name} (pk={self.pk})"
@property
def compounds(self):
return Compound.objects.filter(package=self)
return self.compound_set.all()
@property
def rules(self):
return Rule.objects.filter(package=self)
return self.rule_set.all()
@property
def reactions(self):
return Reaction.objects.filter(package=self)
return self.reaction_set.all()
@property
def pathways(self) -> 'Pathway':
return Pathway.objects.filter(package=self)
return self.pathway_set.all()
@property
def scenarios(self):
return Scenario.objects.filter(package=self)
return self.scenario_set.all()
@property
def models(self):
return EPModel.objects.filter(package=self)
return self.epmodel_set.all()
def _url(self):
return '{}/package/{}'.format(s.SERVER_URL, self.uuid)
@ -911,7 +917,6 @@ class CompoundStructure(EnviPathModel, AliasMixin, ScenarioMixin, ChemicalIdenti
class Rule(PolymorphicModel, EnviPathModel, AliasMixin, ScenarioMixin):
package = models.ForeignKey('epdb.Package', verbose_name='Package', on_delete=models.CASCADE, db_index=True)
# I think this only affects Django Admin which we are barely using
# # https://github.com/django-polymorphic/django-polymorphic/issues/229
# _non_polymorphic = models.Manager()
#
@ -1128,6 +1133,7 @@ class ParallelRule(Rule):
return res
class SequentialRule(Rule):
simple_rules = models.ManyToManyField('epdb.SimpleRule', verbose_name='Simple rules',
through='SequentialRuleOrdering')
@ -1959,7 +1965,7 @@ class RuleBasedRelativeReasoning(PackageBasedModel):
rbrr.package = package
if name is None or name.strip() == '':
name = f"MLRelativeReasoning {RuleBasedRelativeReasoning.objects.filter(package=package).count() + 1}"
name = f"RuleBasedRelativeReasoning {RuleBasedRelativeReasoning.objects.filter(package=package).count() + 1}"
rbrr.name = name

22
epdb/signals.py
View File

@ -1,8 +1,15 @@
import logging
import os
import shutil
from django.conf import settings as s
from django.db import transaction
from django.db.models.signals import pre_delete
from django.db.models.signals import pre_delete, post_delete
from django.dispatch import receiver
from epdb.models import Node, Edge
from epdb.models import Node, Edge, EPModel
logger = logging.getLogger(__name__)
@receiver(pre_delete, sender=Node)
@ -18,3 +25,14 @@ def delete_orphan_edges(sender, instance, **kwargs):
# check if the node that is about to be deleted is the only start node
if edge.end_nodes.count() == 1:
edge.delete()
@receiver(post_delete, sender=EPModel)
def delete_epmodel_files(sender, instance, **kwargs):
# Delete the files on disk for the deleted model
mod_uuid = str(instance.uuid)
for f in os.listdir(s.MODEL_DIR):
if f.startswith(mod_uuid):
logger.info(f"Deleting {os.path.join(s.MODEL_DIR, f)}")
shutil.rmtree(os.path.join(s.MODEL_DIR, f))

18
epdb/views.py
View File

@ -284,18 +284,21 @@ def packages(request):
if hidden := request.POST.get('hidden', None):
if hidden == 'import-legacy-package-json':
if hidden in ['import-legacy-package-json', 'import-package-json']:
f = request.FILES['file']
try:
file_data = f.read().decode("utf-8")
data = json.loads(file_data)
pack = PackageManager.import_package(data, current_user)
if hidden == 'import-legacy-package-json':
pack = PackageManager.import_legacy_package(data, current_user)
else:
pack = PackageManager.import_pacakge(data, current_user)
return redirect(pack.url)
except UnicodeDecodeError:
return error(request, 'Invalid encoding.', f'Invalid encoding, must be UTF-8')
else:
return HttpResponseBadRequest()
else:
@ -799,6 +802,15 @@ def package(request, package_uuid):
if request.method == 'GET':
if request.GET.get("export", False) == "true":
filename = f"{current_package.name.replace(' ', '_')}_{current_package.uuid}.json"
pack_json = PackageManager.export_package(current_package, include_models=False,
include_external_identifiers=False)
response = JsonResponse(pack_json, content_type='application/json')
response['Content-Disposition'] = f'attachment; filename="{filename}"'
return response
context = get_base_context(request)
context['title'] = f'enviPath - {current_package.name}'