|
@@ -1,3 +1,4 @@
|
|
|
|
|
+import math
|
|
|
import pathlib as pl
|
|
import pathlib as pl
|
|
|
import random
|
|
import random
|
|
|
import re
|
|
import re
|
|
@@ -10,6 +11,15 @@ import torch.utils.data as data
|
|
|
from jaxtyping import Float
|
|
from jaxtyping import Float
|
|
|
from torch.utils.data import DataLoader, Subset
|
|
from torch.utils.data import DataLoader, Subset
|
|
|
|
|
|
|
|
|
|
+from util.seeding import torch_generator
|
|
|
|
|
+
|
|
|
|
|
+# Tokens that appear in MRI filenames to indicate the diagnostic class.
|
|
|
|
|
+# On disk the cognitively-normal class is spelled "NL" (not "CN").
|
|
|
|
|
+CLASS_TOKENS: Dict[str, torch.Tensor] = {
|
|
|
|
|
+ "AD": torch.tensor([1.0, 0.0]),
|
|
|
|
|
+ "NL": torch.tensor([0.0, 1.0]),
|
|
|
|
|
+}
|
|
|
|
|
+
|
|
|
|
|
|
|
|
def _row_to_float_tensor(row: pd.DataFrame, *, image_id: int) -> torch.Tensor:
|
|
def _row_to_float_tensor(row: pd.DataFrame, *, image_id: int) -> torch.Tensor:
|
|
|
values = row.drop(columns=["Image Data ID"]).iloc[0]
|
|
values = row.drop(columns=["Image Data ID"]).iloc[0]
|
|
@@ -28,12 +38,18 @@ def xls_pre(df: pd.DataFrame) -> pd.DataFrame:
|
|
|
"""
|
|
"""
|
|
|
Preprocess the Excel DataFrame.
|
|
Preprocess the Excel DataFrame.
|
|
|
This function can be customized to filter or modify the DataFrame as needed.
|
|
This function can be customized to filter or modify the DataFrame as needed.
|
|
|
|
|
+
|
|
|
|
|
+ Returns a DataFrame with columns [Image Data ID, Sex, Age (current)] where
|
|
|
|
|
+ Sex is encoded 0/1. "Image Data ID" is deliberately kept as a column (not an
|
|
|
|
|
+ index) because downstream lookups filter on it directly
|
|
|
|
|
+ (``xls_values["Image Data ID"] == img_id``).
|
|
|
"""
|
|
"""
|
|
|
|
|
|
|
|
- data: pd.DataFrame = df[["Image Data ID", "Sex", "Age (current)"]] # pyright: ignore
|
|
|
|
|
- data["Sex"] = data["Sex"].str.strip() # type: ignore
|
|
|
|
|
- data = data.replace({"M": 0, "F": 1}) # type: ignore
|
|
|
|
|
- data.set_index("Image Data ID") # type: ignore
|
|
|
|
|
|
|
+ # Work on an explicit copy so column assignment writes back reliably instead
|
|
|
|
|
+ # of raising SettingWithCopyWarning against a view of ``df``.
|
|
|
|
|
+ data = df[["Image Data ID", "Sex", "Age (current)"]].copy()
|
|
|
|
|
+ data["Sex"] = data["Sex"].str.strip()
|
|
|
|
|
+ data = data.replace({"M": 0, "F": 1})
|
|
|
|
|
|
|
|
return data
|
|
return data
|
|
|
|
|
|
|
@@ -136,16 +152,18 @@ def load_adni_data_from_file(
|
|
|
|
|
|
|
|
file_mri_data = torch.from_numpy(nib.load(file).get_fdata()) # type: ignore # type checking does not work well with nibabel
|
|
file_mri_data = torch.from_numpy(nib.load(file).get_fdata()) # type: ignore # type checking does not work well with nibabel
|
|
|
|
|
|
|
|
- # Read the filename to determine the expected class
|
|
|
|
|
- file_expected_class = torch.tensor([0.0, 0.0]) # Default to a tensor of zeros
|
|
|
|
|
|
|
+ # Read the filename to determine the expected class. On disk the classes
|
|
|
|
|
+ # are spelled "AD" and "NL"; see CLASS_TOKENS.
|
|
|
|
|
+ file_expected_class: torch.Tensor | None = None
|
|
|
|
|
+ for token, one_hot in CLASS_TOKENS.items():
|
|
|
|
|
+ if token in filename:
|
|
|
|
|
+ file_expected_class = one_hot.clone()
|
|
|
|
|
+ break
|
|
|
|
|
|
|
|
- if "AD" in filename:
|
|
|
|
|
- file_expected_class = torch.tensor([1.0, 0.0])
|
|
|
|
|
- elif "NL" in filename:
|
|
|
|
|
- file_expected_class = torch.tensor([0.0, 1.0])
|
|
|
|
|
- else:
|
|
|
|
|
|
|
+ if file_expected_class is None:
|
|
|
raise ValueError(
|
|
raise ValueError(
|
|
|
- f"Filename {filename} does not contain a valid class identifier (AD or CN)."
|
|
|
|
|
|
|
+ f"Filename {filename} does not contain a valid class identifier "
|
|
|
|
|
+ f"({' or '.join(CLASS_TOKENS)})."
|
|
|
)
|
|
)
|
|
|
|
|
|
|
|
mri_data_unstacked.append(file_mri_data)
|
|
mri_data_unstacked.append(file_mri_data)
|
|
@@ -196,8 +214,8 @@ def divide_dataset(
|
|
|
Returns:
|
|
Returns:
|
|
|
Result[List[data.Subset[ADNIDataset]], str]: A Result object containing the subsets or an error message.
|
|
Result[List[data.Subset[ADNIDataset]], str]: A Result object containing the subsets or an error message.
|
|
|
"""
|
|
"""
|
|
|
- if sum(ratios) != 1.0:
|
|
|
|
|
- raise ValueError(f"Ratios must sum to 1.0, got {ratios}.")
|
|
|
|
|
|
|
+ if not math.isclose(sum(ratios), 1.0, rel_tol=1e-9, abs_tol=1e-9):
|
|
|
|
|
+ raise ValueError(f"Ratios must sum to 1.0, got {ratios} (sum={sum(ratios)}).")
|
|
|
|
|
|
|
|
# Set the random seed for reproducibility
|
|
# Set the random seed for reproducibility
|
|
|
gen = torch.Generator().manual_seed(seed)
|
|
gen = torch.Generator().manual_seed(seed)
|
|
@@ -207,27 +225,52 @@ def divide_dataset(
|
|
|
def initalize_dataloaders(
|
|
def initalize_dataloaders(
|
|
|
datasets: List[Subset[ADNIDataset]],
|
|
datasets: List[Subset[ADNIDataset]],
|
|
|
batch_size: int = 64,
|
|
batch_size: int = 64,
|
|
|
|
|
+ seed: int | None = None,
|
|
|
) -> List[DataLoader[ADNIDataset]]:
|
|
) -> List[DataLoader[ADNIDataset]]:
|
|
|
"""
|
|
"""
|
|
|
- Initializes the DataLoader for the given datasets.
|
|
|
|
|
|
|
+ Initializes the DataLoaders for the [train, val, test] datasets.
|
|
|
|
|
+
|
|
|
|
|
+ The three splits are configured differently:
|
|
|
|
|
+ - train: shuffled (for SGD) with a seeded generator for reproducibility,
|
|
|
|
|
+ and ``drop_last=True`` so a size-1 final batch cannot crash BatchNorm1d.
|
|
|
|
|
+ - val / test: NOT shuffled, so sample order is stable. Stable order is
|
|
|
|
|
+ required for evaluation, where saved model outputs must align with a
|
|
|
|
|
+ fixed sample axis (see the netCDF evaluation schema).
|
|
|
|
|
|
|
|
Args:
|
|
Args:
|
|
|
- datasets (List[Subset[ADNIDataset]]): List of datasets to create DataLoaders for.
|
|
|
|
|
- batch_size (int): The batch size for the DataLoader.
|
|
|
|
|
|
|
+ datasets: Subsets in ``[train, val, test]`` order.
|
|
|
|
|
+ batch_size: The batch size for the DataLoaders.
|
|
|
|
|
+ seed: Seed for the training shuffle generator. ``None`` leaves shuffling
|
|
|
|
|
+ to global RNG state.
|
|
|
|
|
|
|
|
Returns:
|
|
Returns:
|
|
|
- List[DataLoader[ADNIDataset]]: A list of DataLoaders for the datasets.
|
|
|
|
|
|
|
+ List[DataLoader[ADNIDataset]]: DataLoaders in the same order as ``datasets``.
|
|
|
"""
|
|
"""
|
|
|
|
|
+ loader_configs = [
|
|
|
|
|
+ {"shuffle": True, "drop_last": True}, # train
|
|
|
|
|
+ {"shuffle": False, "drop_last": False}, # val
|
|
|
|
|
+ {"shuffle": False, "drop_last": False}, # test
|
|
|
|
|
+ ]
|
|
|
|
|
+ if len(datasets) != len(loader_configs):
|
|
|
|
|
+ raise ValueError(
|
|
|
|
|
+ f"Expected 3 datasets ([train, val, test]), got {len(datasets)}."
|
|
|
|
|
+ )
|
|
|
|
|
+
|
|
|
pin_memory = torch.cuda.is_available()
|
|
pin_memory = torch.cuda.is_available()
|
|
|
- return [
|
|
|
|
|
- DataLoader(
|
|
|
|
|
- dataset,
|
|
|
|
|
- batch_size=batch_size,
|
|
|
|
|
- shuffle=True,
|
|
|
|
|
- pin_memory=pin_memory,
|
|
|
|
|
|
|
+ loaders: List[DataLoader[ADNIDataset]] = []
|
|
|
|
|
+ for dataset, cfg in zip(datasets, loader_configs):
|
|
|
|
|
+ generator = torch_generator(seed) if cfg["shuffle"] else None
|
|
|
|
|
+ loaders.append(
|
|
|
|
|
+ DataLoader(
|
|
|
|
|
+ dataset,
|
|
|
|
|
+ batch_size=batch_size,
|
|
|
|
|
+ shuffle=cfg["shuffle"],
|
|
|
|
|
+ drop_last=cfg["drop_last"],
|
|
|
|
|
+ pin_memory=pin_memory,
|
|
|
|
|
+ generator=generator,
|
|
|
|
|
+ )
|
|
|
)
|
|
)
|
|
|
- for dataset in datasets
|
|
|
|
|
- ]
|
|
|
|
|
|
|
+ return loaders
|
|
|
|
|
|
|
|
|
|
|
|
|
def divide_dataset_by_patient_id(
|
|
def divide_dataset_by_patient_id(
|
|
@@ -253,8 +296,8 @@ def divide_dataset_by_patient_id(
|
|
|
This split is grouped by PTID, so all images from the same patient are assigned
|
|
This split is grouped by PTID, so all images from the same patient are assigned
|
|
|
to exactly one partition to avoid patient-level leakage across train/val/test.
|
|
to exactly one partition to avoid patient-level leakage across train/val/test.
|
|
|
"""
|
|
"""
|
|
|
- if sum(ratios) != 1.0:
|
|
|
|
|
- raise ValueError(f"Ratios must sum to 1.0, got {ratios}.")
|
|
|
|
|
|
|
+ if not math.isclose(sum(ratios), 1.0, rel_tol=1e-9, abs_tol=1e-9):
|
|
|
|
|
+ raise ValueError(f"Ratios must sum to 1.0, got {ratios} (sum={sum(ratios)}).")
|
|
|
|
|
|
|
|
if not ptids:
|
|
if not ptids:
|
|
|
raise ValueError("ptids list cannot be empty.")
|
|
raise ValueError("ptids list cannot be empty.")
|