diff --git a/bridge/contracts.py b/bridge/contracts.py index e6329867..bd1364e1 100644 --- a/bridge/contracts.py +++ b/bridge/contracts.py @@ -1,4 +1,5 @@ import enum +from typing import Any, Dict from abc import ABC, abstractmethod from envipy_additional_information import EnviPyModel @@ -69,6 +70,12 @@ class Plugin(ABC): class Property(Plugin): + def parameters(self) -> Dict[str, Any]: + """ + Returns the parameters of the PropertyPlugin. + """ + return {} + @classmethod @abstractmethod def requires_rule_packages(cls) -> bool: @@ -300,6 +307,12 @@ class Classifier(Plugin): """ pass + def parameters(self) -> Dict[str, Any]: + """ + Returns the parameters of the ClassifierPlugin. + """ + return {} + @abstractmethod def build(self, eP: EnviPyDTO, *args, **kwargs) -> BuildResult | None: """ diff --git a/epapi/v1/interfaces/iuclid/projections.py b/epapi/v1/interfaces/iuclid/projections.py index 1f663987..f201baba 100644 --- a/epapi/v1/interfaces/iuclid/projections.py +++ b/epapi/v1/interfaces/iuclid/projections.py @@ -56,7 +56,7 @@ def get_pathway_for_iuclid_export(user, pathway_uuid: UUID) -> PathwayExportDTO: ai_for_node = [] scenario_entries: list[PathwayScenarioDTO] = [] - for scenario in sorted(node.scenarios.all(), key=lambda item: item.pk): + for scenario in sorted(node.get_scenarios(), key=lambda item: item.pk): ai_for_scenario = list(scenario.get_additional_information(direct_only=True)) ai_for_node.extend(ai_for_scenario) scenario_entries.append( diff --git a/epdb/models.py b/epdb/models.py index 6cb7eff5..60597d3c 100644 --- a/epdb/models.py +++ b/epdb/models.py @@ -2162,7 +2162,7 @@ class Pathway(EnviPathModel, AliasMixin, ScenarioMixin, AdditionalInformationMix row += [cs.smiles, cs.get_name(), n.depth] - edges = self.edges.filter(end_nodes__in=[n]) + edges = self.edges.filter(end_nodes=n) if len(edges): for e in edges: _row = row.copy() @@ -2847,6 +2847,58 @@ class PackageBasedModel(EPModel): return res + def parameters(self): + params = { + "Model Evaluation Threshold": f"{self.threshold:.2f}", + "Multi Gen Evaluation": "Yes" if self.multigen_eval else "No", + } + + if self.app_domain: + params["Applicability Domain Num Neighbors"] = f"{self.app_domain.num_neighbours:.2f}" + params["Applicability Domain Reliability Threshold"] = ( + f"{self.app_domain.reliability_threshold:.2f}" + ) + params["Applicability Domain Local Compatibility Threshold"] = ( + f"{self.app_domain.local_compatibilty_threshold:.2f}" + ) + + return params + + def statistics(self): + from sklearn.metrics import auc + + recall = list(self.eval_results["average_recall_per_threshold"].values()) + precision = list(self.eval_results["average_precision_per_threshold"].values()) + mg_recall = list( + self.eval_results.get("multigen_average_recall_per_threshold", {}).values() + ) + mg_precision = list( + self.eval_results.get("multigen_average_precision_per_threshold", {}).values() + ) + + return { + "accuracy": [ + self.eval_results["average_accuracy"], + self.eval_results.get("multigen_average_accuracy"), + ], + "precision": [ + self.eval_results["average_precision_per_threshold"][f"{self.threshold:.2f}"], + self.eval_results.get("multigen_average_precision_per_threshold", {}).get( + f"{self.threshold:.2f}" + ), + ], + "recall": [ + self.eval_results["average_recall_per_threshold"][f"{self.threshold:.2f}"], + self.eval_results.get("multigen_average_recall_per_threshold", {}).get( + f"{self.threshold:.2f}" + ), + ], + "Area under PR Curve": [ + auc(recall, precision), + auc(mg_recall, mg_precision) if self.multigen_eval else None, + ], + } + @cached_property def applicable_rules(self) -> List["Rule"]: """ @@ -3027,8 +3079,6 @@ class PackageBasedModel(EPModel): thresholds.append(np.float64(threshold)) thresholds.sort() - logger.info(f"Thresholds: {thresholds}") - precision = {f"{t:.2f}": [] for t in thresholds} recall = {f"{t:.2f}": [] for t in thresholds} @@ -3077,14 +3127,17 @@ class PackageBasedModel(EPModel): for t in thresholds: for true, pred in zip(pathways, pred_pathways): acc, pre, rec = multigen_eval(true, pred, t) - if abs(t - threshold) < 0.01: - mg_acc = acc + + if f"{t:.2f}" == f"{threshold:.2f}": + mg_acc += acc + precision[f"{t:.2f}"].append(pre) recall[f"{t:.2f}"].append(rec) + avg_mg_acc = mg_acc / len(root_compounds) precision = {k: sum(v) / len(v) if len(v) > 0 else 0 for k, v in precision.items()} recall = {k: sum(v) / len(v) if len(v) > 0 else 0 for k, v in recall.items()} - return mg_acc, precision, recall + return avg_mg_acc, precision, recall # If there are eval packages perform single generation evaluation on them instead of random splits if self.eval_packages.count() > 0: @@ -4216,6 +4269,9 @@ class ClassifierPluginModel(PackageBasedModel): instance = impl(conf) return instance + def parameters(self): + return self.instance().parameters() + def build_dataset(self): """ Required by general model contract but actual implementation resides in plugin. @@ -4436,6 +4492,9 @@ class PropertyPluginModel(PackageBasedModel): instance = impl() return instance + def parameters(self): + return self.instance().parameters() + def build_dataset(self): """ Required by general model contract but actual implementation resides in plugin. diff --git a/epdb/tasks.py b/epdb/tasks.py index 48eb9689..4e9bcee3 100644 --- a/epdb/tasks.py +++ b/epdb/tasks.py @@ -477,8 +477,7 @@ def batch_predict( limit=None, setting_overrides={ "max_nodes": num_tps, - "max_depth": num_tps, - "model_threshold": 0.001, + "model_threshold": 0.0, }, ) diff --git a/epiuclid/serializers/pathway_mapper.py b/epiuclid/serializers/pathway_mapper.py index 14ce91ef..2bd638e8 100644 --- a/epiuclid/serializers/pathway_mapper.py +++ b/epiuclid/serializers/pathway_mapper.py @@ -351,7 +351,8 @@ class PathwayMapper: props = SoilPropertiesData() - for ai in ai_list: + for ai_obj in ai_list: + ai = ai_obj.get() if isinstance(ai, SoilTexture1) and props.soil_type is None: props.soil_type = ai.type.value elif isinstance(ai, SoilTexture2): diff --git a/templates/objects/model/_model_base.html b/templates/objects/model/_model_base.html index c6640108..97f31f78 100644 --- a/templates/objects/model/_model_base.html +++ b/templates/objects/model/_model_base.html @@ -117,6 +117,39 @@ {% endif %} + + {% if model.parameters %} + +
+ +
Model Parameters
+
+
+
+ + + + + + + + + {% for param, value in model.parameters.items %} + + + + + {% endfor %} + +
ParameterValue
{{ param }}{{ value }}
+
+
+
+
+ {% endif %} {% endblock %} {% block usemodel %} diff --git a/templates/objects/model/classification_model.html b/templates/objects/model/classification_model.html index 9a484ff4..a68c3109 100644 --- a/templates/objects/model/classification_model.html +++ b/templates/objects/model/classification_model.html @@ -313,6 +313,44 @@ {% endif %} + + +
+ +
Model Statistics for threshold {{ model.threshold }}
+
+
+
+ + + + + + {% if model.multigen_eval %} + + {% endif %} + + + + {% for metric, value in model.statistics.items %} + + + + {% if model.multigen_eval %} + + {% endif %} + + {% endfor %} + +
MetricSingle Gen ValueMulti Gen Value
{{ metric|upper }}{{ value.0|floatformat:3 }}{{ value.1|floatformat:3 }}
+
+
+
+
+ {% endif %}