[Feature] Package Export/Import (#116)

Fixes #90
Fixes #91
Fixes #115
Fixes #104

Co-authored-by: Tim Lorsbach <tim@lorsba.ch>
Reviewed-on: enviPath/enviPy#116
This commit is contained in:
2025-09-16 02:41:10 +12:00
parent ce349a287b
commit 762a6b7baf
32 changed files with 2500683 additions and 145 deletions

View File

@ -5,7 +5,7 @@ from epdb.models import Compound, User, CompoundStructure
class CompoundTest(TestCase):
fixtures = ["test_fixture.cleaned.json"]
fixtures = ["test_fixtures.json.gz"]
def setUp(self):
pass
@ -16,13 +16,6 @@ class CompoundTest(TestCase):
cls.user = User.objects.get(username='anonymous')
cls.package = PackageManager.create_package(cls.user, 'Anon Test Package', 'No Desc')
@classmethod
def tearDownClass(cls):
pass
def tearDown(self):
pass
def test_smoke(self):
c = Compound.create(
self.package,
@ -78,7 +71,7 @@ class CompoundTest(TestCase):
smiles='C1C(=NOC1(C2=CC(=CC(=C2)Cl)C(F)(F)F)C(F)(F)F)C3=CC=C(C4=CC=CC=C43)C(=O)NCC(=O)NCC(F)(F)F',
)
self.assertEqual(c.name, 'no name')
self.assertEqual(c.name, 'Compound 1')
self.assertEqual(c.description, 'no description')
def test_empty_name_and_description_are_ignored(self):
@ -89,7 +82,7 @@ class CompoundTest(TestCase):
description='',
)
self.assertEqual(c.name, 'no name')
self.assertEqual(c.name, 'Compound 1')
self.assertEqual(c.description, 'no description')
def test_deduplication(self):

View File

@ -1,16 +1,13 @@
import json
from django.test import TestCase
from django.test import TestCase
from epdb.logic import PackageManager
from epdb.models import Compound, User, CompoundStructure, Reaction, Rule, MLRelativeReasoning, Pathway
from epdb.models import Compound, User, Reaction
class CopyTest(TestCase):
fixtures = ["test_fixture.cleaned.json"]
def setUp(self):
pass
fixtures = ["test_fixtures.json.gz"]
@classmethod
def setUpClass(cls):
@ -61,15 +58,6 @@ class CopyTest(TestCase):
multi_step=False
)
@classmethod
def tearDownClass(cls):
pass
def tearDown(self):
pass
def test_compound_copy_basic(self):
"""Test basic compound copying functionality"""
mapping = dict()
@ -175,13 +163,12 @@ class CopyTest(TestCase):
self.assertEqual(self.REACTION.multi_step, copied_reaction.multi_step)
self.assertEqual(copied_reaction.package, self.target_package)
self.assertEqual(self.REACTION.package, self.package)
def test_reaction_copy_structures(self):
"""Test basic reaction copying functionality"""
mapping = dict()
copied_reaction = self.REACTION.copy(self.target_package, mapping)
for orig_educt, copy_educt in zip(self.REACTION.educts.all(), copied_reaction.educts.all()):
self.assertNotEqual(orig_educt.uuid, copy_educt.uuid)
self.assertEqual(orig_educt.name, copy_educt.name)
@ -197,4 +184,3 @@ class CopyTest(TestCase):
self.assertEqual(copy_product.compound.package, self.target_package)
self.assertEqual(orig_product.compound.package, self.package)
self.assertEqual(orig_product.smiles, copy_product.smiles)

View File

@ -6,7 +6,7 @@ from utilities.ml import Dataset
class DatasetTest(TestCase):
fixtures = ["test_fixture.cleaned.json"]
fixtures = ["test_fixtures.json.gz"]
def setUp(self):
self.cs1 = Compound.create(
@ -38,7 +38,7 @@ class DatasetTest(TestCase):
@classmethod
def setUpClass(cls):
super(DatasetGeneratorTest, cls).setUpClass()
super(DatasetTest, cls).setUpClass()
cls.user = User.objects.get(username='anonymous')
cls.package = PackageManager.create_package(cls.user, 'Anon Test Package', 'No Desc')

View File

@ -5,9 +5,6 @@ from utilities.chem import FormatConverter
class FormatConverterTestCase(TestCase):
def setUp(self):
pass
def test_standardization(self):
smiles = 'C[n+]1c([N-](C))cccc1'
standardized_smiles = FormatConverter.standardize(smiles)

View File

@ -1,38 +1,27 @@
import json
from django.test import TestCase
from django.test import TestCase
from epdb.logic import PackageManager
from epdb.models import Compound, User, CompoundStructure, Reaction, Rule, MLRelativeReasoning
from epdb.models import User, MLRelativeReasoning, Package
class ModelTest(TestCase):
fixtures = ["test_fixture.cleaned.json"]
def setUp(self):
pass
fixtures = ["test_fixtures.json.gz"]
@classmethod
def setUpClass(cls):
super(ModelTest, cls).setUpClass()
cls.user = User.objects.get(username='anonymous')
cls.package = PackageManager.create_package(cls.user, 'Anon Test Package', 'No Desc')
bbd_data = json.load(open('fixtures/packages/2025-07-18/EAWAG-BBD.json'))
cls.BBD = PackageManager.import_package(bbd_data, cls.user)
@classmethod
def tearDownClass(cls):
pass
def tearDown(self):
pass
cls.BBD_SUBSET = Package.objects.get(name='Fixtures')
def test_smoke(self):
threshold = float(0.5)
# get Package objects from urls
rule_package_objs = [self.BBD]
data_package_objs = [self.BBD]
rule_package_objs = [self.BBD_SUBSET]
data_package_objs = [self.BBD_SUBSET]
eval_packages_objs = []
mod = MLRelativeReasoning.create(
@ -44,8 +33,8 @@ class ModelTest(TestCase):
'ECC - BBD - 0.5',
'Created MLRelativeReasoning in Testcase',
)
ds = mod.load_dataset()
mod.build_dataset()
mod.build_model()
print("Model built!")
mod.evaluate_model()

View File

@ -1,14 +1,11 @@
from django.test import TestCase
from epdb.logic import PackageManager
from epdb.models import Compound, User, CompoundStructure, Reaction, Rule
from epdb.models import Compound, User, Reaction, Rule
class ReactionTest(TestCase):
fixtures = ["test_fixture.cleaned.json"]
def setUp(self):
pass
fixtures = ["test_fixtures.json.gz"]
@classmethod
def setUpClass(cls):
@ -16,13 +13,6 @@ class ReactionTest(TestCase):
cls.user = User.objects.get(username='anonymous')
cls.package = PackageManager.create_package(cls.user, 'Anon Test Package', 'No Desc')
@classmethod
def tearDownClass(cls):
pass
def tearDown(self):
pass
def test_smoke(self):
educt = Compound.create(
self.package,
@ -50,8 +40,6 @@ class ReactionTest(TestCase):
self.assertEqual(r.name, 'Eawag BBD reaction r0001')
self.assertEqual(r.description, 'no description')
def test_string_educts_and_products(self):
r = Reaction.create(
package=self.package,

View File

@ -5,7 +5,7 @@ from epdb.models import Rule, User
class RuleTest(TestCase):
fixtures = ["test_fixture.cleaned.json"]
fixtures = ["test_fixtures.json.gz"]
def setUp(self):
pass
@ -16,13 +16,6 @@ class RuleTest(TestCase):
cls.user = User.objects.get(username='anonymous')
cls.package = PackageManager.create_package(cls.user, 'Anon Test Package', 'No Desc')
@classmethod
def tearDownClass(cls):
pass
def tearDown(self):
pass
def test_smoke(self):
r = Rule.create(
rule_type='SimpleAmbitRule',

View File

@ -2,10 +2,10 @@ import gzip
import json
from django.conf import settings as s
from django.test import TestCase
from django.test import TestCase, tag
from utilities.chem import FormatConverter
@tag("slow")
class RuleApplicationTest(TestCase):
def setUp(self):
@ -19,10 +19,12 @@ class RuleApplicationTest(TestCase):
@classmethod
def tearDownClass(cls):
super().tearDownClass()
print(f"\nTotal Errors across Rules {len(cls.error_smiles)}")
# print(cls.error_smiles)
def tearDown(self):
super().tearDown()
print(f"\nTotal errors {self.total_errors}")
def run_bt_test(self, bt_rule_name):

View File

@ -0,0 +1,362 @@
from unittest.mock import patch, MagicMock, PropertyMock
from django.test import TestCase
from epdb.logic import PackageManager
from epdb.models import User, SimpleAmbitRule
class SimpleAmbitRuleTest(TestCase):
fixtures = ["test_fixtures.json.gz"]
@classmethod
def setUpClass(cls):
super(SimpleAmbitRuleTest, cls).setUpClass()
cls.user = User.objects.get(username='anonymous')
cls.package = PackageManager.create_package(cls.user, 'Simple Ambit Rule Test Package',
'Test Package for SimpleAmbitRule')
def test_create_basic_rule(self):
"""Test creating a basic SimpleAmbitRule with minimal parameters."""
smirks = '[H:5][C:1]([#6:6])([#1,#9,#17,#35,#53:4])[#9,#17,#35,#53]>>[H:5][C:1]([#6:6])([#8])[#1,#9,#17,#35,#53:4]'
rule = SimpleAmbitRule.create(
package=self.package,
smirks=smirks
)
self.assertIsInstance(rule, SimpleAmbitRule)
self.assertEqual(rule.smirks, smirks)
self.assertEqual(rule.package, self.package)
self.assertRegex(rule.name, r'Rule \d+')
self.assertEqual(rule.description, 'no description')
self.assertIsNone(rule.reactant_filter_smarts)
self.assertIsNone(rule.product_filter_smarts)
def test_create_with_all_parameters(self):
"""Test creating SimpleAmbitRule with all parameters."""
smirks = '[H:5][C:1]([#6:6])([#1,#9,#17,#35,#53:4])[#9,#17,#35,#53]>>[H:5][C:1]([#6:6])([#8])[#1,#9,#17,#35,#53:4]'
name = 'Test Rule'
description = 'A test biotransformation rule'
reactant_filter = '[CH2X4]'
product_filter = '[OH]'
rule = SimpleAmbitRule.create(
package=self.package,
name=name,
description=description,
smirks=smirks,
reactant_filter_smarts=reactant_filter,
product_filter_smarts=product_filter
)
self.assertEqual(rule.name, name)
self.assertEqual(rule.description, description)
self.assertEqual(rule.smirks, smirks)
self.assertEqual(rule.reactant_filter_smarts, reactant_filter)
self.assertEqual(rule.product_filter_smarts, product_filter)
def test_smirks_required(self):
"""Test that SMIRKS is required for rule creation."""
with self.assertRaises(ValueError) as cm:
SimpleAmbitRule.create(package=self.package, smirks=None)
self.assertIn('SMIRKS is required', str(cm.exception))
with self.assertRaises(ValueError) as cm:
SimpleAmbitRule.create(package=self.package, smirks='')
self.assertIn('SMIRKS is required', str(cm.exception))
with self.assertRaises(ValueError) as cm:
SimpleAmbitRule.create(package=self.package, smirks=' ')
self.assertIn('SMIRKS is required', str(cm.exception))
@patch('epdb.models.FormatConverter.is_valid_smirks')
def test_invalid_smirks_validation(self, mock_is_valid):
"""Test validation of SMIRKS format."""
mock_is_valid.return_value = False
invalid_smirks = 'invalid_smirks_string'
with self.assertRaises(ValueError) as cm:
SimpleAmbitRule.create(
package=self.package,
smirks=invalid_smirks
)
self.assertIn(f'SMIRKS "{invalid_smirks}" is invalid', str(cm.exception))
mock_is_valid.assert_called_once_with(invalid_smirks)
def test_smirks_trimming(self):
"""Test that SMIRKS strings are trimmed during creation."""
smirks = '[H:5][C:1]([#6:6])([#1,#9,#17,#35,#53:4])[#9,#17,#35,#53]>>[H:5][C:1]([#6:6])([#8])[#1,#9,#17,#35,#53:4]'
smirks_with_whitespace = f' {smirks} '
rule = SimpleAmbitRule.create(
package=self.package,
smirks=smirks_with_whitespace
)
self.assertEqual(rule.smirks, smirks)
def test_empty_name_and_description_handling(self):
"""Test that empty name and description are handled appropriately."""
smirks = '[H:5][C:1]([#6:6])([#1,#9,#17,#35,#53:4])[#9,#17,#35,#53]>>[H:5][C:1]([#6:6])([#8])[#1,#9,#17,#35,#53:4]'
rule = SimpleAmbitRule.create(
package=self.package,
smirks=smirks,
name='',
description=' '
)
self.assertRegex(rule.name, r'Rule \d+')
self.assertEqual(rule.description, 'no description')
def test_deduplication_basic(self):
"""Test that identical rules are deduplicated."""
smirks = '[H:5][C:1]([#6:6])([#1,#9,#17,#35,#53:4])[#9,#17,#35,#53]>>[H:5][C:1]([#6:6])([#8])[#1,#9,#17,#35,#53:4]'
rule1 = SimpleAmbitRule.create(
package=self.package,
smirks=smirks,
name='Rule 1'
)
rule2 = SimpleAmbitRule.create(
package=self.package,
smirks=smirks,
name='Rule 2' # Different name, but same SMIRKS
)
self.assertEqual(rule1.pk, rule2.pk)
self.assertEqual(SimpleAmbitRule.objects.filter(package=self.package, smirks=smirks).count(), 1)
def test_deduplication_with_filters(self):
"""Test deduplication with filter SMARTS."""
smirks = '[H:5][C:1]([#6:6])([#1,#9,#17,#35,#53:4])[#9,#17,#35,#53]>>[H:5][C:1]([#6:6])([#8])[#1,#9,#17,#35,#53:4]'
reactant_filter = '[CH2X4]'
product_filter = '[OH]'
rule1 = SimpleAmbitRule.create(
package=self.package,
smirks=smirks,
reactant_filter_smarts=reactant_filter,
product_filter_smarts=product_filter
)
rule2 = SimpleAmbitRule.create(
package=self.package,
smirks=smirks,
reactant_filter_smarts=reactant_filter,
product_filter_smarts=product_filter
)
self.assertEqual(rule1.pk, rule2.pk)
def test_no_deduplication_different_filters(self):
"""Test that rules with different filters are not deduplicated."""
smirks = '[H:5][C:1]([#6:6])([#1,#9,#17,#35,#53:4])[#9,#17,#35,#53]>>[H:5][C:1]([#6:6])([#8])[#1,#9,#17,#35,#53:4]'
rule1 = SimpleAmbitRule.create(
package=self.package,
smirks=smirks,
reactant_filter_smarts='[CH2X4]'
)
rule2 = SimpleAmbitRule.create(
package=self.package,
smirks=smirks,
reactant_filter_smarts='[CH3X4]'
)
self.assertNotEqual(rule1.pk, rule2.pk)
self.assertEqual(SimpleAmbitRule.objects.filter(package=self.package, smirks=smirks).count(), 2)
def test_filter_smarts_trimming(self):
"""Test that filter SMARTS are trimmed and handled correctly."""
smirks = '[H:5][C:1]([#6:6])([#1,#9,#17,#35,#53:4])[#9,#17,#35,#53]>>[H:5][C:1]([#6:6])([#8])[#1,#9,#17,#35,#53:4]'
# Test with whitespace-only filters (should be treated as None)
rule = SimpleAmbitRule.create(
package=self.package,
smirks=smirks,
reactant_filter_smarts=' ',
product_filter_smarts=' '
)
self.assertIsNone(rule.reactant_filter_smarts)
self.assertIsNone(rule.product_filter_smarts)
def test_url_property(self):
"""Test the URL property generation."""
rule = SimpleAmbitRule.create(
package=self.package,
smirks='[H:1][C:2]>>[H:1][O:2]'
)
expected_url = f'{self.package.url}/simple-ambit-rule/{rule.uuid}'
self.assertEqual(rule.url, expected_url)
@patch('epdb.models.FormatConverter.apply')
def test_apply_method(self, mock_apply):
"""Test the apply method delegates to FormatConverter."""
mock_apply.return_value = ['product1', 'product2']
rule = SimpleAmbitRule.create(
package=self.package,
smirks='[H:1][C:2]>>[H:1][O:2]'
)
test_smiles = 'CCO'
result = rule.apply(test_smiles)
mock_apply.assert_called_once_with(test_smiles, rule.smirks)
self.assertEqual(result, ['product1', 'product2'])
def test_reactants_smarts_property(self):
"""Test reactants_smarts property extracts correct part of SMIRKS."""
smirks = '[H:5][C:1]([#6:6])([#1,#9,#17,#35,#53:4])[#9,#17,#35,#53]>>[H:5][C:1]([#6:6])([#8])[#1,#9,#17,#35,#53:4]'
expected_reactants = '[H:5][C:1]([#6:6])([#1,#9,#17,#35,#53:4])[#9,#17,#35,#53]'
rule = SimpleAmbitRule.create(
package=self.package,
smirks=smirks
)
self.assertEqual(rule.reactants_smarts, expected_reactants)
def test_products_smarts_property(self):
"""Test products_smarts property extracts correct part of SMIRKS."""
smirks = '[H:5][C:1]([#6:6])([#1,#9,#17,#35,#53:4])[#9,#17,#35,#53]>>[H:5][C:1]([#6:6])([#8])[#1,#9,#17,#35,#53:4]'
expected_products = '[H:5][C:1]([#6:6])([#8])[#1,#9,#17,#35,#53:4]'
rule = SimpleAmbitRule.create(
package=self.package,
smirks=smirks
)
self.assertEqual(rule.products_smarts, expected_products)
@patch('epdb.models.Package.objects')
def test_related_reactions_property(self, mock_package_objects):
"""Test related_reactions property returns correct queryset."""
mock_qs = MagicMock()
mock_package_objects.filter.return_value = mock_qs
rule = SimpleAmbitRule.create(
package=self.package,
smirks='[H:1][C:2]>>[H:1][O:2]'
)
# Instead of directly assigning, patch the property or use with patch.object
with patch.object(type(rule), 'reaction_rule', new_callable=PropertyMock) as mock_reaction_rule:
mock_reaction_rule.return_value.filter.return_value.order_by.return_value = ['reaction1', 'reaction2']
result = rule.related_reactions
mock_package_objects.filter.assert_called_once_with(reviewed=True)
mock_reaction_rule.return_value.filter.assert_called_once_with(package__in=mock_qs)
mock_reaction_rule.return_value.filter.return_value.order_by.assert_called_once_with('name')
self.assertEqual(result, ['reaction1', 'reaction2'])
@patch('epdb.models.Pathway.objects')
@patch('epdb.models.Edge.objects')
def test_related_pathways_property(self, mock_edge_objects, mock_pathway_objects):
"""Test related_pathways property returns correct queryset."""
mock_related_reactions = ['reaction1', 'reaction2']
with patch.object(SimpleAmbitRule, "related_reactions", new_callable=PropertyMock) as mock_prop:
mock_prop.return_value = mock_related_reactions
rule = SimpleAmbitRule.create(
package=self.package,
smirks='[H:1][C:2]>>[H:1][O:2]'
)
# Mock Edge objects query
mock_edge_values = MagicMock()
mock_edge_values.values.return_value = ['pathway_id1', 'pathway_id2']
mock_edge_objects.filter.return_value = mock_edge_values
# Mock Pathway objects query
mock_pathway_qs = MagicMock()
mock_pathway_objects.filter.return_value.order_by.return_value = mock_pathway_qs
result = rule.related_pathways
mock_edge_objects.filter.assert_called_once_with(edge_label__in=mock_related_reactions)
mock_edge_values.values.assert_called_once_with('pathway_id')
mock_pathway_objects.filter.assert_called_once()
self.assertEqual(result, mock_pathway_qs)
@patch('epdb.models.IndigoUtils.smirks_to_svg')
def test_as_svg_property(self, mock_smirks_to_svg):
"""Test as_svg property calls IndigoUtils correctly."""
mock_smirks_to_svg.return_value = '<svg>test_svg</svg>'
rule = SimpleAmbitRule.create(
package=self.package,
smirks='[H:1][C:2]>>[H:1][O:2]'
)
result = rule.as_svg
mock_smirks_to_svg.assert_called_once_with(rule.smirks, True, width=800, height=400)
self.assertEqual(result, '<svg>test_svg</svg>')
def test_atomic_transaction(self):
"""Test that rule creation is atomic."""
smirks = '[H:1][C:2]>>[H:1][O:2]'
# This should work normally
rule = SimpleAmbitRule.create(package=self.package, smirks=smirks)
self.assertIsInstance(rule, SimpleAmbitRule)
# Test transaction rollback on error
with patch('epdb.models.SimpleAmbitRule.save', side_effect=Exception('Database error')):
with self.assertRaises(Exception):
SimpleAmbitRule.create(package=self.package, smirks='[H:3][C:4]>>[H:3][O:4]')
# Verify no partial data was saved
self.assertEqual(SimpleAmbitRule.objects.filter(package=self.package).count(), 1)
def test_multiple_duplicate_warning(self):
"""Test logging when multiple duplicates are found."""
smirks = '[H:1][C:2]>>[H:1][O:2]'
# Create first rule
rule1 = SimpleAmbitRule.create(package=self.package, smirks=smirks)
# Manually create a duplicate to simulate the error condition
rule2 = SimpleAmbitRule(package=self.package, smirks=smirks, name='Manual Rule')
rule2.save()
with patch('epdb.models.logger') as mock_logger:
# This should find the existing rule and log an error about multiple matches
result = SimpleAmbitRule.create(package=self.package, smirks=smirks)
# Should return the first matching rule
self.assertEqual(result.pk, rule1.pk)
# Should log an error about multiple matches
mock_logger.error.assert_called()
self.assertIn('More than one rule matched', mock_logger.error.call_args[0][0])
def test_model_fields(self):
"""Test model field properties."""
rule = SimpleAmbitRule.create(
package=self.package,
smirks='[H:1][C:2]>>[H:1][O:2]',
reactant_filter_smarts='[CH3]',
product_filter_smarts='[OH]'
)
# Test field properties
self.assertFalse(rule._meta.get_field('smirks').blank)
self.assertFalse(rule._meta.get_field('smirks').null)
self.assertTrue(rule._meta.get_field('reactant_filter_smarts').null)
self.assertTrue(rule._meta.get_field('product_filter_smarts').null)
# Test verbose names
self.assertEqual(rule._meta.get_field('smirks').verbose_name, 'SMIRKS')
self.assertEqual(rule._meta.get_field('reactant_filter_smarts').verbose_name, 'Reactant Filter SMARTS')
self.assertEqual(rule._meta.get_field('product_filter_smarts').verbose_name, 'Product Filter SMARTS')