# ============================================================== # predict.py – How to use the trained Walnut Multimodal Models # ============================================================== import torch import torch.nn as nn from torchvision import models from torchvision.transforms import functional as TF from PIL import Image from pathlib import Path import pandas as pd DEVICE = torch.device("cuda" if torch.cuda.is_available() else "cpu") # ---------------------------------------------------------- # Model definitions (must match training) # ---------------------------------------------------------- class HistoEncoder(nn.Module): def __init__(self, embed_dim=512): super().__init__() backbone = models.densenet121(weights=None) self.features = backbone.features self.pool = nn.AdaptiveAvgPool2d(1) self.proj = nn.Sequential( nn.Linear(1024, embed_dim), nn.ReLU(), nn.LayerNorm(embed_dim) ) def forward(self, x): x = self.features(x) x = self.pool(x).flatten(1) return self.proj(x) class IHCEncoder(nn.Module): def __init__(self, embed_dim=512): super().__init__() backbone = models.densenet121(weights=None) self.features = backbone.features self.pool = nn.AdaptiveAvgPool2d(1) self.proj = nn.Sequential( nn.Linear(1024, embed_dim), nn.ReLU(), nn.LayerNorm(embed_dim) ) def forward(self, x): x = self.features(x) x = self.pool(x).flatten(1) return self.proj(x) class MolecularMLP(nn.Module): def __init__(self, in_dim=5, embed_dim=128): super().__init__() self.encoder = nn.Sequential( nn.Linear(in_dim, 64), nn.ReLU(), nn.BatchNorm1d(64), nn.Dropout(0.3), nn.Linear(64, embed_dim), nn.ReLU(), nn.LayerNorm(embed_dim) ) self.classifier = nn.Sequential( nn.Linear(embed_dim, 64), nn.ReLU(), nn.Dropout(0.3), nn.Linear(64, 3) ) def forward(self, x): return self.classifier(self.encoder(x)) def extract_features(self, x): return self.encoder(x) class FusionModel(nn.Module): def __init__(self): super().__init__() self.histo_enc = HistoEncoder(512) self.ihc_enc = IHCEncoder(512) self.mol_enc = MolecularMLP(5, 128) self.fusion = nn.Sequential( nn.Linear(512+512+128, 256), nn.ReLU(), nn.Dropout(0.4), nn.Linear(256, 64), nn.ReLU(), nn.Dropout(0.3), nn.Linear(64, 3) ) def forward(self, histo, ihc, mol): h = self.histo_enc(histo) i = self.ihc_enc(ihc) m = self.mol_enc.extract_features(mol) x = torch.cat([h, i, m], dim=1) return self.fusion(x) # ---------------------------------------------------------- # Preprocessing # ---------------------------------------------------------- def preprocess_image(img_path): img = Image.open(img_path).convert("RGB") img = TF.resize(img, [224, 224]) img = TF.to_tensor(img) img = TF.normalize(img, [0.485, 0.456, 0.406], [0.229, 0.224, 0.225]) return img.unsqueeze(0) # add batch dimension # ---------------------------------------------------------- # Load models # ---------------------------------------------------------- def load_models(model_dir="."): model_dir = Path(model_dir) # Molecular only mol_model = MolecularMLP().to(DEVICE) mol_ckpt = torch.load(model_dir / "best_molecular_mlp.pt", map_location=DEVICE) mol_model.load_state_dict(mol_ckpt["model_state"]) mol_model.eval() # Fusion model fusion_model = FusionModel().to(DEVICE) fusion_path = model_dir / "best_fusion_model.pt" if fusion_path.exists(): fusion_model.load_state_dict(torch.load(fusion_path, map_location=DEVICE)) fusion_model.eval() else: fusion_model = None print("Fusion model not found.") return mol_model, fusion_model # ---------------------------------------------------------- # Predict function # ---------------------------------------------------------- STATE_NAMES = { 0: "Normal", 1: "Partial_Restoration", 2: "Induced_Neoplastic" } def predict_molecular(mol_model, flavonoid, alkaloid, terpenoid, phenol, steroid): x = torch.tensor([[flavonoid, alkaloid, terpenoid, phenol, steroid]], dtype=torch.float32).to(DEVICE) with torch.no_grad(): logits = mol_model(x) probs = torch.softmax(logits, dim=1)[0] pred = logits.argmax(dim=1).item() return { "predicted_state": STATE_NAMES[pred], "state_id": pred, "probabilities": { "Normal": probs[0].item(), "Partial_Restoration": probs[1].item(), "Induced_Neoplastic": probs[2].item() } } def predict_fusion(fusion_model, histo_path, ihc_path, flavonoid, alkaloid, terpenoid, phenol, steroid): histo = preprocess_image(histo_path).to(DEVICE) ihc = preprocess_image(ihc_path).to(DEVICE) mol = torch.tensor([[flavonoid, alkaloid, terpenoid, phenol, steroid]], dtype=torch.float32).to(DEVICE) with torch.no_grad(): logits = fusion_model(histo, ihc, mol) probs = torch.softmax(logits, dim=1)[0] pred = logits.argmax(dim=1).item() return { "predicted_state": STATE_NAMES[pred], "state_id": pred, "probabilities": { "Normal": probs[0].item(), "Partial_Restoration": probs[1].item(), "Induced_Neoplastic": probs[2].item() } } # ---------------------------------------------------------- # Example usage # ---------------------------------------------------------- if __name__ == "__main__": print("Loading models...") mol_model, fusion_model = load_models("/kaggle/working") # or path to downloaded models # Example 1: Molecular only prediction print("\n--- Molecular Only Prediction ---") result = predict_molecular( mol_model, flavonoid=14.30, alkaloid=4.07, terpenoid=2.57, phenol=1.80, steroid=0.22 ) print(result) # Example 2: Full Fusion prediction (if you have images) # result = predict_fusion( # fusion_model, # histo_path="path/to/H&E.png", # ihc_path="path/to/p63.png", # flavonoid=14.30, alkaloid=4.07, terpenoid=2.57, phenol=1.80, steroid=0.22 # ) # print(result)