jebus/enviPy-bayer
1
0
Fork 0
forked from enviPath/enviPy
Code Issues Pull Requests Actions Releases Activity

[Chore] Linted Files (#150)

Browse Source 
Co-authored-by: Tim Lorsbach <tim@lorsba.ch>
Reviewed-on: enviPath/enviPy#150
This commit is contained in:
jebus
2025-10-09 07:25:13 +13:00
parent 22f0bbe10b
commit afeb56622c
50 changed files with 5616 additions and 4408 deletions
Download Patch File Download Diff File Expand all files Collapse all files

168
epdb/management/commands/bootstrap.py
View File

@ -5,32 +5,49 @@ from django.core.management.base import BaseCommand
from django.db import transaction
from epdb.logic import UserManager, GroupManager, PackageManager, SettingManager
from epdb.models import UserSettingPermission, MLRelativeReasoning, EnviFormer, Permission, User, ExternalDatabase
from epdb.models import (
UserSettingPermission,
MLRelativeReasoning,
EnviFormer,
Permission,
User,
ExternalDatabase,
)
class Command(BaseCommand):
def create_users(self):
# Anonymous User
if not User.objects.filter(email='anon@envipath.com').exists():
anon = UserManager.create_user("anonymous", "anon@envipath.com", "SuperSafe",
is_active=True, add_to_group=False, set_setting=False)
if not User.objects.filter(email="anon@envipath.com").exists():
anon = UserManager.create_user(
"anonymous",
"anon@envipath.com",
"SuperSafe",
is_active=True,
add_to_group=False,
set_setting=False,
)
else:
anon = User.objects.get(email='anon@envipath.com')
anon = User.objects.get(email="anon@envipath.com")
# Admin User
if not User.objects.filter(email='admin@envipath.com').exists():
admin = UserManager.create_user("admin", "admin@envipath.com", "SuperSafe",
is_active=True, add_to_group=False, set_setting=False)
if not User.objects.filter(email="admin@envipath.com").exists():
admin = UserManager.create_user(
"admin",
"admin@envipath.com",
"SuperSafe",
is_active=True,
add_to_group=False,
set_setting=False,
)
admin.is_staff = True
admin.is_superuser = True
admin.save()
else:
admin = User.objects.get(email='admin@envipath.com')
admin = User.objects.get(email="admin@envipath.com")
# System Group
g = GroupManager.create_group(admin, 'enviPath Users', 'All enviPath Users')
g = GroupManager.create_group(admin, "enviPath Users", "All enviPath Users")
g.public = True
g.save()
@ -43,14 +60,20 @@ class Command(BaseCommand):
admin.default_group = g
admin.save()
if not User.objects.filter(email='user0@envipath.com').exists():
user0 = UserManager.create_user("user0", "user0@envipath.com", "SuperSafe",
is_active=True, add_to_group=False, set_setting=False)
if not User.objects.filter(email="user0@envipath.com").exists():
user0 = UserManager.create_user(
"user0",
"user0@envipath.com",
"SuperSafe",
is_active=True,
add_to_group=False,
set_setting=False,
)
user0.is_staff = True
user0.is_superuser = True
user0.save()
else:
user0 = User.objects.get(email='user0@envipath.com')
user0 = User.objects.get(email="user0@envipath.com")
g.user_member.add(user0)
g.save()
@ -61,18 +84,20 @@ class Command(BaseCommand):
return anon, admin, g, user0
def import_package(self, data, owner):
return PackageManager.import_legacy_package(data, owner, keep_ids=True, add_import_timestamp=False, trust_reviewed=True)
return PackageManager.import_legacy_package(
data, owner, keep_ids=True, add_import_timestamp=False, trust_reviewed=True
)
def create_default_setting(self, owner, packages):
s = SettingManager.create_setting(
owner,
name='Global Default Setting',
description='Global Default Setting containing BBD Rules and Max 30 Nodes and Max Depth of 8',
name="Global Default Setting",
description="Global Default Setting containing BBD Rules and Max 30 Nodes and Max Depth of 8",
max_nodes=30,
max_depth=5,
rule_packages=packages,
model=None,
model_threshold=None
model_threshold=None,
)
return s
@ -84,54 +109,51 @@ class Command(BaseCommand):
"""
databases = [
{
'name': 'PubChem Compound',
'full_name': 'PubChem Compound Database',
'description': 'Chemical database of small organic molecules',
'base_url': 'https://pubchem.ncbi.nlm.nih.gov',
'url_pattern': 'https://pubchem.ncbi.nlm.nih.gov/compound/{id}'
"name": "PubChem Compound",
"full_name": "PubChem Compound Database",
"description": "Chemical database of small organic molecules",
"base_url": "https://pubchem.ncbi.nlm.nih.gov",
"url_pattern": "https://pubchem.ncbi.nlm.nih.gov/compound/{id}",
},
{
'name': 'PubChem Substance',
'full_name': 'PubChem Substance Database',
'description': 'Database of chemical substances',
'base_url': 'https://pubchem.ncbi.nlm.nih.gov',
'url_pattern': 'https://pubchem.ncbi.nlm.nih.gov/substance/{id}'
"name": "PubChem Substance",
"full_name": "PubChem Substance Database",
"description": "Database of chemical substances",
"base_url": "https://pubchem.ncbi.nlm.nih.gov",
"url_pattern": "https://pubchem.ncbi.nlm.nih.gov/substance/{id}",
},
{
'name': 'ChEBI',
'full_name': 'Chemical Entities of Biological Interest',
'description': 'Dictionary of molecular entities',
'base_url': 'https://www.ebi.ac.uk/chebi',
'url_pattern': 'https://www.ebi.ac.uk/chebi/searchId.do?chebiId=CHEBI:{id}'
"name": "ChEBI",
"full_name": "Chemical Entities of Biological Interest",
"description": "Dictionary of molecular entities",
"base_url": "https://www.ebi.ac.uk/chebi",
"url_pattern": "https://www.ebi.ac.uk/chebi/searchId.do?chebiId=CHEBI:{id}",
},
{
'name': 'RHEA',
'full_name': 'RHEA Reaction Database',
'description': 'Comprehensive resource of biochemical reactions',
'base_url': 'https://www.rhea-db.org',
'url_pattern': 'https://www.rhea-db.org/rhea/{id}'
"name": "RHEA",
"full_name": "RHEA Reaction Database",
"description": "Comprehensive resource of biochemical reactions",
"base_url": "https://www.rhea-db.org",
"url_pattern": "https://www.rhea-db.org/rhea/{id}",
},
{
'name': 'KEGG Reaction',
'full_name': 'KEGG Reaction Database',
'description': 'Database of biochemical reactions',
'base_url': 'https://www.genome.jp',
'url_pattern': 'https://www.genome.jp/entry/{id}'
"name": "KEGG Reaction",
"full_name": "KEGG Reaction Database",
"description": "Database of biochemical reactions",
"base_url": "https://www.genome.jp",
"url_pattern": "https://www.genome.jp/entry/{id}",
},
{
'name': 'UniProt',
'full_name': 'MetaCyc Metabolic Pathway Database',
'description': 'UniProt is a freely accessible database of protein sequence and functional information',
'base_url': 'https://www.uniprot.org',
'url_pattern': 'https://www.uniprot.org/uniprotkb?query="{id}"'
}
"name": "UniProt",
"full_name": "MetaCyc Metabolic Pathway Database",
"description": "UniProt is a freely accessible database of protein sequence and functional information",
"base_url": "https://www.uniprot.org",
"url_pattern": 'https://www.uniprot.org/uniprotkb?query="{id}"',
},
]
for db_info in databases:
ExternalDatabase.objects.get_or_create(
name=db_info['name'],
defaults=db_info
)
ExternalDatabase.objects.get_or_create(name=db_info["name"], defaults=db_info)
@transaction.atomic
def handle(self, *args, **options):
@ -142,20 +164,24 @@ class Command(BaseCommand):
# Import Packages
packages = [
'EAWAG-BBD.json',
'EAWAG-SOIL.json',
'EAWAG-SLUDGE.json',
'EAWAG-SEDIMENT.json',
"EAWAG-BBD.json",
"EAWAG-SOIL.json",
"EAWAG-SLUDGE.json",
"EAWAG-SEDIMENT.json",
]
mapping = {}
for p in packages:
print(f"Importing {p}...")
package_data = json.loads(open(s.BASE_DIR / 'fixtures' / 'packages' / '2025-07-18' / p, encoding='utf-8').read())
package_data = json.loads(
open(
s.BASE_DIR / "fixtures" / "packages" / "2025-07-18" / p, encoding="utf-8"
).read()
)
imported_package = self.import_package(package_data, admin)
mapping[p.replace('.json', '')] = imported_package
mapping[p.replace(".json", "")] = imported_package
setting = self.create_default_setting(admin, [mapping['EAWAG-BBD']])
setting = self.create_default_setting(admin, [mapping["EAWAG-BBD"]])
setting.public = True
setting.save()
setting.make_global_default()
@ -171,26 +197,28 @@ class Command(BaseCommand):
usp.save()
# Create Model Package
pack = PackageManager.create_package(admin, "Public Prediction Models",
"Package to make Prediction Models publicly available")
pack = PackageManager.create_package(
admin,
"Public Prediction Models",
"Package to make Prediction Models publicly available",
)
pack.reviewed = True
pack.save()
# Create RR
ml_model = MLRelativeReasoning.create(
package=pack,
rule_packages=[mapping['EAWAG-BBD']],
data_packages=[mapping['EAWAG-BBD']],
rule_packages=[mapping["EAWAG-BBD"]],
data_packages=[mapping["EAWAG-BBD"]],
eval_packages=[],
threshold=0.5,
name='ECC - BBD - T0.5',
description='ML Relative Reasoning',
name="ECC - BBD - T0.5",
description="ML Relative Reasoning",
)
ml_model.build_dataset()
ml_model.build_model()
# ml_model.evaluate_model()
# If available, create EnviFormerModel
if s.ENVIFORMER_PRESENT:
enviFormer_model = EnviFormer.create(pack, 'EnviFormer - T0.5', 'EnviFormer Model with Threshold 0.5', 0.5)
EnviFormer.create(pack, "EnviFormer - T0.5", "EnviFormer Model with Threshold 0.5", 0.5)

53
epdb/management/commands/create_ml_models.py
View File

@ -12,11 +12,28 @@ class Command(BaseCommand):
the below command would be used:
`python manage.py create_ml_models enviformer mlrr -d bbd soil -e sludge
"""
def add_arguments(self, parser):
parser.add_argument("model_names", nargs="+", type=str, help="The names of models to train. Options are: enviformer, mlrr")
parser.add_argument("-d", "--data-packages", nargs="+", type=str, help="Packages for training")
parser.add_argument("-e", "--eval-packages", nargs="*", type=str, help="Packages for evaluation", default=[])
parser.add_argument("-r", "--rule-packages", nargs="*", type=str, help="Rule Packages mandatory for MLRR", default=[])
parser.add_argument(
"model_names",
nargs="+",
type=str,
help="The names of models to train. Options are: enviformer, mlrr",
)
parser.add_argument(
"-d", "--data-packages", nargs="+", type=str, help="Packages for training"
)
parser.add_argument(
"-e", "--eval-packages", nargs="*", type=str, help="Packages for evaluation", default=[]
)
parser.add_argument(
"-r",
"--rule-packages",
nargs="*",
type=str,
help="Rule Packages mandatory for MLRR",
default=[],
)
@transaction.atomic
def handle(self, *args, **options):
@ -28,7 +45,9 @@ class Command(BaseCommand):
sludge = Package.objects.filter(name="EAWAG-SLUDGE")[0]
sediment = Package.objects.filter(name="EAWAG-SEDIMENT")[0]
except IndexError:
raise IndexError("Can't find correct packages. They should be created with the bootstrap command")
raise IndexError(
"Can't find correct packages. They should be created with the bootstrap command"
)
def decode_packages(package_list):
"""Decode package strings into their respective packages"""
@ -52,15 +71,27 @@ class Command(BaseCommand):
data_packages = decode_packages(options["data_packages"])
eval_packages = decode_packages(options["eval_packages"])
rule_packages = decode_packages(options["rule_packages"])
for model_name in options['model_names']:
for model_name in options["model_names"]:
model_name = model_name.lower()
if model_name == "enviformer" and s.ENVIFORMER_PRESENT:
model = EnviFormer.create(pack, data_packages=data_packages, eval_packages=eval_packages, threshold=0.5,
name="EnviFormer - T0.5", description="EnviFormer transformer")
model = EnviFormer.create(
pack,
data_packages=data_packages,
eval_packages=eval_packages,
threshold=0.5,
name="EnviFormer - T0.5",
description="EnviFormer transformer",
)
elif model_name == "mlrr":
model = MLRelativeReasoning.create(package=pack, rule_packages=rule_packages,
data_packages=data_packages, eval_packages=eval_packages, threshold=0.5,
name='ECC - BBD - T0.5', description='ML Relative Reasoning')
model = MLRelativeReasoning.create(
package=pack,
rule_packages=rule_packages,
data_packages=data_packages,
eval_packages=eval_packages,
threshold=0.5,
name="ECC - BBD - T0.5",
description="ML Relative Reasoning",
)
else:
raise ValueError(f"Cannot create model of type {model_name}, unknown model type")
# Build the dataset for the model, train it, evaluate it and save it

45
epdb/management/commands/import_external_identifiers.py
View File

@ -1,57 +1,58 @@
from csv import DictReader
from django.core.management.base import BaseCommand
from django.db import transaction
from epdb.models import *
from epdb.models import Compound, CompoundStructure, Reaction, ExternalDatabase, ExternalIdentifier
class Command(BaseCommand):
STR_TO_MODEL = {
'Compound': Compound,
'CompoundStructure': CompoundStructure,
'Reaction': Reaction,
"Compound": Compound,
"CompoundStructure": CompoundStructure,
"Reaction": Reaction,
}
STR_TO_DATABASE = {
'ChEBI': ExternalDatabase.objects.get(name='ChEBI'),
'RHEA': ExternalDatabase.objects.get(name='RHEA'),
'KEGG Reaction': ExternalDatabase.objects.get(name='KEGG Reaction'),
'PubChem Compound': ExternalDatabase.objects.get(name='PubChem Compound'),
'PubChem Substance': ExternalDatabase.objects.get(name='PubChem Substance'),
"ChEBI": ExternalDatabase.objects.get(name="ChEBI"),
"RHEA": ExternalDatabase.objects.get(name="RHEA"),
"KEGG Reaction": ExternalDatabase.objects.get(name="KEGG Reaction"),
"PubChem Compound": ExternalDatabase.objects.get(name="PubChem Compound"),
"PubChem Substance": ExternalDatabase.objects.get(name="PubChem Substance"),
}
def add_arguments(self, parser):
parser.add_argument(
'--data',
"--data",
type=str,
help='Path of the ID Mapping file.',
help="Path of the ID Mapping file.",
required=True,
)
parser.add_argument(
'--replace-host',
"--replace-host",
type=str,
help='Replace https://envipath.org/ with this host, e.g. http://localhost:8000/',
help="Replace https://envipath.org/ with this host, e.g. http://localhost:8000/",
)
@transaction.atomic
def handle(self, *args, **options):
with open(options['data']) as fh:
with open(options["data"]) as fh:
reader = DictReader(fh)
for row in reader:
clz = self.STR_TO_MODEL[row['model']]
clz = self.STR_TO_MODEL[row["model"]]
url = row['url']
if options['replace_host']:
url = url.replace('https://envipath.org/', options['replace_host'])
url = row["url"]
if options["replace_host"]:
url = url.replace("https://envipath.org/", options["replace_host"])
instance = clz.objects.get(url=url)
db = self.STR_TO_DATABASE[row['identifier_type']]
db = self.STR_TO_DATABASE[row["identifier_type"]]
ExternalIdentifier.objects.create(
content_object=instance,
database=db,
identifier_value=row['identifier_value'],
url=db.url_pattern.format(id=row['identifier_value']),
is_primary=False
identifier_value=row["identifier_value"],
url=db.url_pattern.format(id=row["identifier_value"]),
is_primary=False,
)

18
epdb/management/commands/import_legacy_package.py
View File

@ -1,27 +1,29 @@
import json
from django.core.management.base import BaseCommand
from django.db import transaction
from epdb.logic import PackageManager
from epdb.models import *
from epdb.models import User
class Command(BaseCommand):
def add_arguments(self, parser):
parser.add_argument(
'--data',
"--data",
type=str,
help='Path of the Package to import.',
help="Path of the Package to import.",
required=True,
)
parser.add_argument(
'--owner',
"--owner",
type=str,
help='Username of the desired Owner.',
help="Username of the desired Owner.",
required=True,
)
@transaction.atomic
def handle(self, *args, **options):
owner = User.objects.get(username=options['owner'])
package_data = json.load(open(options['data']))
owner = User.objects.get(username=options["owner"])
package_data = json.load(open(options["data"]))
PackageManager.import_legacy_package(package_data, owner)

49
epdb/management/commands/localize_urls.py
View File

@ -6,46 +6,45 @@ from django.db.models.functions import Replace
class Command(BaseCommand):
def add_arguments(self, parser):
parser.add_argument(
'--old',
"--old",
type=str,
help='Old Host, most likely https://envipath.org/',
help="Old Host, most likely https://envipath.org/",
required=True,
)
parser.add_argument(
'--new',
"--new",
type=str,
help='New Host, most likely http://localhost:8000/',
help="New Host, most likely http://localhost:8000/",
required=True,
)
def handle(self, *args, **options):
MODELS = [
'User',
'Group',
'Package',
'Compound',
'CompoundStructure',
'Pathway',
'Edge',
'Node',
'Reaction',
'SimpleAmbitRule',
'SimpleRDKitRule',
'ParallelRule',
'SequentialRule',
'Scenario',
'Setting',
'MLRelativeReasoning',
'RuleBasedRelativeReasoning',
'EnviFormer',
'ApplicabilityDomain',
"User",
"Group",
"Package",
"Compound",
"CompoundStructure",
"Pathway",
"Edge",
"Node",
"Reaction",
"SimpleAmbitRule",
"SimpleRDKitRule",
"ParallelRule",
"SequentialRule",
"Scenario",
"Setting",
"MLRelativeReasoning",
"RuleBasedRelativeReasoning",
"EnviFormer",
"ApplicabilityDomain",
]
for model in MODELS:
obj_cls = apps.get_model("epdb", model)
print(f"Localizing urls for {model}")
obj_cls.objects.update(
url=Replace(F('url'), Value(options['old']), Value(options['new']))
url=Replace(F("url"), Value(options["old"]), Value(options["new"]))
)