|
|
@@ -9,6 +9,13 @@ netCDF file in the locked-down evaluation schema (:mod:`evaluation.schema`).
|
|
|
step 5 evaluate_bayesian -> Bayesian ensemble, noise=[0.0], K=mc_passes
|
|
|
step 6 evaluate_noisy -> both ensembles across config noise_levels
|
|
|
|
|
|
+Every evaluator is parameterized by dataset *split*, so the same code produces
|
|
|
+test-split and validation-split evaluations (``*_val_task`` variants). Both
|
|
|
+loaders are unshuffled, so sample order is stable in each. Finally,
|
|
|
+``combine_evaluations_task`` pools the per-split files into ``combined_*.nc``
|
|
|
+(test + val by default) -- the splits are patient-disjoint, so pooling simply
|
|
|
+enlarges the evaluation set that analysis draws on.
|
|
|
+
|
|
|
Determinism
|
|
|
-----------
|
|
|
The test loader is unshuffled, so sample order is stable and aligns with the
|
|
|
@@ -31,6 +38,8 @@ import torch
|
|
|
from evaluation.schema import (
|
|
|
CLASS_NAMES,
|
|
|
EvaluationAccumulator,
|
|
|
+ concat_evaluations,
|
|
|
+ load_evaluation,
|
|
|
save_evaluation,
|
|
|
)
|
|
|
from model.factory import KIND_BAYESIAN, KIND_NORMAL, load_model
|
|
|
@@ -45,6 +54,20 @@ _NOISE_SEED_STREAM = 2
|
|
|
# Subdirectory (under work_dir) where evaluation netCDF files are written.
|
|
|
_EVAL_SUBDIR = "evaluations"
|
|
|
|
|
|
+# Evaluable dataset splits -> the ``state`` key holding their DataLoader. Both
|
|
|
+# val and test loaders are unshuffled, so their sample order is stable and can be
|
|
|
+# aligned with the netCDF ``sample`` axis. (``train`` is available for
|
|
|
+# completeness but is NOT part of the default pipeline: its samples were fit on,
|
|
|
+# so its accuracy is optimistic and not a generalization estimate.)
|
|
|
+SPLIT_TEST = "test"
|
|
|
+SPLIT_VAL = "val"
|
|
|
+SPLIT_TRAIN = "train"
|
|
|
+_SPLIT_LOADERS = {
|
|
|
+ SPLIT_TEST: "test_loader",
|
|
|
+ SPLIT_VAL: "val_loader",
|
|
|
+ SPLIT_TRAIN: "train_loader",
|
|
|
+}
|
|
|
+
|
|
|
# Fallbacks if a config predates the [evaluation] section.
|
|
|
_DEFAULT_NOISE_LEVELS: List[float] = [0.0, 0.02, 0.05, 0.1, 0.2]
|
|
|
_DEFAULT_MC_PASSES = 30
|
|
|
@@ -145,6 +168,7 @@ def _evaluate_ensemble(
|
|
|
noise_levels: Sequence[float],
|
|
|
n_passes: int,
|
|
|
out_path: pl.Path,
|
|
|
+ split: str = SPLIT_TEST,
|
|
|
) -> None:
|
|
|
"""Evaluate one ensemble and write a schema-compliant netCDF to ``out_path``."""
|
|
|
device = config["training"]["device"]
|
|
|
@@ -152,13 +176,14 @@ def _evaluate_ensemble(
|
|
|
base_seed = int(config["data"]["seed"])
|
|
|
stop_event, pause_event = _events(state)
|
|
|
|
|
|
- test_loader = state["test_loader"]
|
|
|
+ loader = state[_SPLIT_LOADERS[split]]
|
|
|
image_to_ptid: Dict[int, str] = state.get("image_to_ptid", {})
|
|
|
|
|
|
- image_ids, ptids, true_classes = _collect_sample_metadata(test_loader, image_to_ptid)
|
|
|
+ image_ids, ptids, true_classes = _collect_sample_metadata(loader, image_to_ptid)
|
|
|
log.info(
|
|
|
- f"Evaluating {len(model_paths)} {kind} model(s) on {len(image_ids)} samples, "
|
|
|
- f"{len(noise_levels)} noise level(s), {n_passes} MC pass(es) each."
|
|
|
+ f"Evaluating {len(model_paths)} {kind} model(s) on {len(image_ids)} "
|
|
|
+ f"{split} samples, {len(noise_levels)} noise level(s), "
|
|
|
+ f"{n_passes} MC pass(es) each."
|
|
|
)
|
|
|
|
|
|
accumulator = EvaluationAccumulator(
|
|
|
@@ -177,7 +202,7 @@ def _evaluate_ensemble(
|
|
|
noise_seed = derive_seed(base_seed, level_index, stream=_NOISE_SEED_STREAM)
|
|
|
mc_probs = _run_mc_passes(
|
|
|
model=model,
|
|
|
- loader=test_loader,
|
|
|
+ loader=loader,
|
|
|
device=device,
|
|
|
noise_level=float(noise),
|
|
|
n_passes=n_passes,
|
|
|
@@ -204,13 +229,14 @@ def _evaluate_ensemble(
|
|
|
"seed": base_seed,
|
|
|
"n_mc": n_passes,
|
|
|
"ensemble_kind": kind,
|
|
|
+ "split": split,
|
|
|
"config": _config_snapshot(config),
|
|
|
}
|
|
|
)
|
|
|
out_path.parent.mkdir(parents=True, exist_ok=True)
|
|
|
save_evaluation(dataset, str(out_path))
|
|
|
log.info(
|
|
|
- f"Saved {kind} evaluation to {out_path} "
|
|
|
+ f"Saved {kind} {split} evaluation to {out_path} "
|
|
|
f"(models={dataset.sizes['model']}, samples={dataset.sizes['sample']}, "
|
|
|
f"noise_levels={dataset.sizes['noise_level']})."
|
|
|
)
|
|
|
@@ -234,99 +260,187 @@ def _out_path(config: Dict[str, Any], name: str) -> pl.Path:
|
|
|
return pl.Path(config["work_dir"]) / _EVAL_SUBDIR / name
|
|
|
|
|
|
|
|
|
-# ---------------------------------------------------------------------------
|
|
|
-# Pipeline task entry points
|
|
|
-# ---------------------------------------------------------------------------
|
|
|
-def evaluate_normal_task(
|
|
|
- track: ProgressTracker,
|
|
|
- log: PipelineLogger,
|
|
|
- config: Dict[str, Any],
|
|
|
- state: Dict[str, Any],
|
|
|
-) -> None:
|
|
|
- """Step 4: evaluate the normal ensemble on clean data (noise=0.0, K=1)."""
|
|
|
- paths = state.get("normal_model_paths") or []
|
|
|
- if not paths:
|
|
|
- log.error("No normal models to evaluate (run training/load_models first).")
|
|
|
- return
|
|
|
- _evaluate_ensemble(
|
|
|
- model_paths=paths,
|
|
|
- kind=KIND_NORMAL,
|
|
|
- config=config,
|
|
|
- state=state,
|
|
|
- log=log,
|
|
|
- track=track,
|
|
|
- noise_levels=[0.0],
|
|
|
- n_passes=1,
|
|
|
- out_path=_out_path(config, "normal.nc"),
|
|
|
- )
|
|
|
+def eval_filename(split: str, kind: str, noisy: bool = False) -> str:
|
|
|
+ """Canonical evaluation filename for a (split, kind, noisy) combination.
|
|
|
+
|
|
|
+ The test split keeps its historical un-prefixed names (``normal.nc``,
|
|
|
+ ``noisy_bayesian.nc``); other splits are prefixed (``val_normal.nc``).
|
|
|
+ """
|
|
|
+ prefix = "" if split == SPLIT_TEST else f"{split}_"
|
|
|
+ return f"{prefix}{'noisy_' if noisy else ''}{kind}.nc"
|
|
|
|
|
|
|
|
|
-def evaluate_bayesian_task(
|
|
|
+def _evaluate_clean(
|
|
|
track: ProgressTracker,
|
|
|
log: PipelineLogger,
|
|
|
config: Dict[str, Any],
|
|
|
state: Dict[str, Any],
|
|
|
+ kind: str,
|
|
|
+ split: str,
|
|
|
) -> None:
|
|
|
- """Step 5: evaluate the Bayesian ensemble on clean data with MC uncertainty."""
|
|
|
- paths = state.get("bayesian_model_paths") or []
|
|
|
+ """Evaluate one ensemble on clean data for ``split`` (steps 4/5)."""
|
|
|
+ key = "normal_model_paths" if kind == KIND_NORMAL else "bayesian_model_paths"
|
|
|
+ paths = state.get(key) or []
|
|
|
if not paths:
|
|
|
- log.error("No Bayesian models to evaluate (run training/load_models first).")
|
|
|
+ log.error(f"No {kind} models to evaluate (run training/load_models first).")
|
|
|
return
|
|
|
- _, mc_passes = _eval_cfg(config)
|
|
|
+ n_passes = 1 if kind == KIND_NORMAL else _eval_cfg(config)[1]
|
|
|
_evaluate_ensemble(
|
|
|
model_paths=paths,
|
|
|
- kind=KIND_BAYESIAN,
|
|
|
+ kind=kind,
|
|
|
config=config,
|
|
|
state=state,
|
|
|
log=log,
|
|
|
track=track,
|
|
|
noise_levels=[0.0],
|
|
|
- n_passes=mc_passes,
|
|
|
- out_path=_out_path(config, "bayesian.nc"),
|
|
|
+ n_passes=n_passes,
|
|
|
+ out_path=_out_path(config, eval_filename(split, kind)),
|
|
|
+ split=split,
|
|
|
)
|
|
|
|
|
|
|
|
|
-def evaluate_noisy_task(
|
|
|
+def _evaluate_noisy(
|
|
|
track: ProgressTracker,
|
|
|
log: PipelineLogger,
|
|
|
config: Dict[str, Any],
|
|
|
state: Dict[str, Any],
|
|
|
+ split: str,
|
|
|
) -> None:
|
|
|
- """Step 6: evaluate both ensembles across the configured Gaussian noise levels."""
|
|
|
+ """Evaluate both ensembles across the noise sweep for ``split`` (step 6)."""
|
|
|
noise_levels, mc_passes = _eval_cfg(config)
|
|
|
- normal_paths = state.get("normal_model_paths") or []
|
|
|
- bayesian_paths = state.get("bayesian_model_paths") or []
|
|
|
-
|
|
|
- if not normal_paths and not bayesian_paths:
|
|
|
- log.error("No models to evaluate on noised data.")
|
|
|
- return
|
|
|
-
|
|
|
- if normal_paths:
|
|
|
+ any_models = False
|
|
|
+ for kind, key, n_passes in (
|
|
|
+ (KIND_NORMAL, "normal_model_paths", 1),
|
|
|
+ (KIND_BAYESIAN, "bayesian_model_paths", mc_passes),
|
|
|
+ ):
|
|
|
+ paths = state.get(key) or []
|
|
|
+ if not paths:
|
|
|
+ log.info(f"No {kind} models found; skipping {kind} noisy evaluation.")
|
|
|
+ continue
|
|
|
+ any_models = True
|
|
|
_evaluate_ensemble(
|
|
|
- model_paths=normal_paths,
|
|
|
- kind=KIND_NORMAL,
|
|
|
+ model_paths=paths,
|
|
|
+ kind=kind,
|
|
|
config=config,
|
|
|
state=state,
|
|
|
log=log,
|
|
|
track=track,
|
|
|
noise_levels=noise_levels,
|
|
|
- n_passes=1,
|
|
|
- out_path=_out_path(config, "noisy_normal.nc"),
|
|
|
+ n_passes=n_passes,
|
|
|
+ out_path=_out_path(config, eval_filename(split, kind, noisy=True)),
|
|
|
+ split=split,
|
|
|
)
|
|
|
- else:
|
|
|
- log.info("No normal models found; skipping normal noisy evaluation.")
|
|
|
+ if not any_models:
|
|
|
+ log.error("No models to evaluate on noised data.")
|
|
|
|
|
|
- if bayesian_paths:
|
|
|
- _evaluate_ensemble(
|
|
|
- model_paths=bayesian_paths,
|
|
|
- kind=KIND_BAYESIAN,
|
|
|
- config=config,
|
|
|
- state=state,
|
|
|
- log=log,
|
|
|
- track=track,
|
|
|
- noise_levels=noise_levels,
|
|
|
- n_passes=mc_passes,
|
|
|
- out_path=_out_path(config, "noisy_bayesian.nc"),
|
|
|
+
|
|
|
+# ---------------------------------------------------------------------------
|
|
|
+# Pipeline task entry points
|
|
|
+# ---------------------------------------------------------------------------
|
|
|
+def evaluate_normal_task(track, log, config, state) -> None:
|
|
|
+ """Step 4: normal ensemble on the clean TEST split."""
|
|
|
+ _evaluate_clean(track, log, config, state, KIND_NORMAL, SPLIT_TEST)
|
|
|
+
|
|
|
+
|
|
|
+def evaluate_bayesian_task(track, log, config, state) -> None:
|
|
|
+ """Step 5: Bayesian ensemble (MC uncertainty) on the clean TEST split."""
|
|
|
+ _evaluate_clean(track, log, config, state, KIND_BAYESIAN, SPLIT_TEST)
|
|
|
+
|
|
|
+
|
|
|
+def evaluate_noisy_task(track, log, config, state) -> None:
|
|
|
+ """Step 6: both ensembles across the noise sweep on the TEST split."""
|
|
|
+ _evaluate_noisy(track, log, config, state, SPLIT_TEST)
|
|
|
+
|
|
|
+
|
|
|
+def evaluate_normal_val_task(track, log, config, state) -> None:
|
|
|
+ """Normal ensemble on the clean VALIDATION split."""
|
|
|
+ _evaluate_clean(track, log, config, state, KIND_NORMAL, SPLIT_VAL)
|
|
|
+
|
|
|
+
|
|
|
+def evaluate_bayesian_val_task(track, log, config, state) -> None:
|
|
|
+ """Bayesian ensemble (MC uncertainty) on the clean VALIDATION split."""
|
|
|
+ _evaluate_clean(track, log, config, state, KIND_BAYESIAN, SPLIT_VAL)
|
|
|
+
|
|
|
+
|
|
|
+def evaluate_noisy_val_task(track, log, config, state) -> None:
|
|
|
+ """Both ensembles across the noise sweep on the VALIDATION split."""
|
|
|
+ _evaluate_noisy(track, log, config, state, SPLIT_VAL)
|
|
|
+
|
|
|
+
|
|
|
+def combine_evaluations_task(
|
|
|
+ track: ProgressTracker,
|
|
|
+ log: PipelineLogger,
|
|
|
+ config: Dict[str, Any],
|
|
|
+ state: Dict[str, Any],
|
|
|
+) -> None:
|
|
|
+ """Pool per-split evaluations into ``combined_*.nc`` for analysis.
|
|
|
+
|
|
|
+ Concatenates the configured splits (default test + val) along the ``sample``
|
|
|
+ axis. The splits are patient-disjoint by construction, so pooling them yields
|
|
|
+ a larger evaluation set with no duplicated samples -- which is the point:
|
|
|
+ more evaluation samples means tighter accuracy/uncertainty estimates.
|
|
|
+ Analysis prefers these pooled files over the single-split ones.
|
|
|
+ """
|
|
|
+ eval_dir = pl.Path(config["work_dir"]) / _EVAL_SUBDIR
|
|
|
+ splits = list(
|
|
|
+ config.get("evaluation", {}).get("combine_splits", [SPLIT_TEST, SPLIT_VAL])
|
|
|
+ )
|
|
|
+ if len(splits) < 2:
|
|
|
+ log.info(
|
|
|
+ f"combine_evaluations: only {splits} configured; nothing to pool."
|
|
|
+ )
|
|
|
+ return
|
|
|
+
|
|
|
+ written = 0
|
|
|
+ for kind in (KIND_NORMAL, KIND_BAYESIAN):
|
|
|
+ for noisy in (False, True):
|
|
|
+ paths = [
|
|
|
+ eval_dir / eval_filename(split, kind, noisy=noisy) for split in splits
|
|
|
+ ]
|
|
|
+ present = [p for p in paths if p.exists()]
|
|
|
+ if len(present) < 2:
|
|
|
+ continue
|
|
|
+
|
|
|
+ parts = [load_evaluation(str(p)) for p in present]
|
|
|
+ try:
|
|
|
+ # Guard against a mis-specified split: the same patient must not
|
|
|
+ # appear in two pooled parts (that would double-count them).
|
|
|
+ seen: set[str] = set()
|
|
|
+ overlap: set[str] = set()
|
|
|
+ for part in parts:
|
|
|
+ part_ptids = {str(p) for p in np.atleast_1d(part["ptid"].values)}
|
|
|
+ overlap |= seen & part_ptids
|
|
|
+ seen |= part_ptids
|
|
|
+ if overlap:
|
|
|
+ log.error(
|
|
|
+ f"combine_evaluations: {len(overlap)} patient(s) appear in "
|
|
|
+ f"more than one split ({sorted(overlap)[:3]}...); refusing "
|
|
|
+ f"to pool {kind} (noisy={noisy})."
|
|
|
+ )
|
|
|
+ continue
|
|
|
+
|
|
|
+ combined = concat_evaluations(
|
|
|
+ parts,
|
|
|
+ attrs={
|
|
|
+ "split": "+".join(splits),
|
|
|
+ "combined_from": ", ".join(p.name for p in present),
|
|
|
+ "n_sources": len(present),
|
|
|
+ },
|
|
|
+ )
|
|
|
+ out_name = f"combined_{'noisy_' if noisy else ''}{kind}.nc"
|
|
|
+ save_evaluation(combined, str(eval_dir / out_name))
|
|
|
+ written += 1
|
|
|
+ log.info(
|
|
|
+ f"combine_evaluations: wrote {out_name} "
|
|
|
+ f"({combined.sizes['sample']} samples from "
|
|
|
+ f"{', '.join(p.name for p in present)})."
|
|
|
+ )
|
|
|
+ finally:
|
|
|
+ for part in parts:
|
|
|
+ part.close()
|
|
|
+
|
|
|
+ if written == 0:
|
|
|
+ log.error(
|
|
|
+ "combine_evaluations: found no split pairs to pool. Run the test and "
|
|
|
+ "validation evaluation steps first."
|
|
|
)
|
|
|
- else:
|
|
|
- log.info("No Bayesian models found; skipping Bayesian noisy evaluation.")
|
Nicholas Schense