Put aside K-fold for now and added Val + test evaluation

alnn_rewrite.log

@@ -4842,3 +4842,4 @@ RuntimeError: Error(s) in loading state_dict for CNN3D:
 2026-07-15 10:24:47,258 - INFO - [Analyze Evaluations] Analysis artifacts written to outputs/exp1/analysis/.
 2026-07-15 10:24:47,259 - INFO - [Analyze Evaluations] Task completed.
 2026-07-15 10:24:47,261 - INFO - [SYSTEM] All tasks finished successfully!
+2026-07-15 11:53:52,291 - INFO - [SYSTEM] Application initialized and ready.

config.toml

@@ -1,10 +1,10 @@
 # Pipline Scenario to run
-# "scen_train_all" - Train and evaluate model, with noise
-# "scen_load_all"  - Load and evaluate model, with noise
-# "scen_load_eval" - Load and evaluate model, without noise
+# "scen_train_all" - Train, evaluate (test+val), noise, combine, analyze
+# "scen_load_all"  - Load, evaluate (test+val), noise, combine, analyze
+# "scen_load_eval" - Load, evaluate (test+val), combine, analyze (no noise)
 # "scen_analyze"   - Analyze existing evaluation files only (no training/eval)
 # "scen_kfold"     - Patient-grouped k-fold CV: train+eval per fold, then OOF + analyze
-scenario = "scen_train_all"
+scenario = "scen_kfold"
 
 
 [run]
@@ -43,12 +43,17 @@ noise_levels = [0.0, 0.05, 0.1, 0.2, 0.3, 0.5, 0.7, 1.0, 1.5, 2.0, 3.0, 5.0]
 # Monte-Carlo forward passes used to estimate Bayesian predictive/model
 # uncertainty (step 5/6). Ignored for the deterministic ensemble.
 mc_passes = 30
+# Splits pooled into the combined_*.nc files that analysis prefers. The splits
+# are patient-disjoint, so pooling just enlarges the evaluation set (tighter
+# accuracy/uncertainty estimates). Set to ["test"] for a test-only headline
+# number -- note val is not a pure held-out set if it ever informed tuning.
+combine_splits = ["test", "val"]
 
 
 [kfold]
 # Used only by the "scen_kfold" scenario. Each fold trains a full ensemble on
 # ~(k-1)/k of the patients and evaluates on the held-out 1/k; out-of-fold results
 # are pooled to cover every sample. NOTE: cost is ~k * a normal train+eval run.
-k_folds = 4          # number of patient-grouped folds (each ~1/k held out)
+k_folds = 5          # number of patient-grouped folds (each ~1/k held out)
 val_fraction = 0.15  # fraction of each fold's non-test patients used for validation
 include_noise = false # also run the noise sweep within each fold (much slower)

evaluation/__init__.py

@@ -14,6 +14,7 @@ from evaluation.schema import (
     EvaluationAccumulator,
     build_evaluation_dataset,
     compute_uncertainty,
+    concat_evaluations,
     load_evaluation,
     save_evaluation,
 )
@@ -26,6 +27,7 @@ __all__ = [
     "EvaluationAccumulator",
     "build_evaluation_dataset",
     "compute_uncertainty",
+    "concat_evaluations",
     "load_evaluation",
     "save_evaluation",
 ]

evaluation/schema.py

@@ -305,6 +305,33 @@ def load_evaluation(path: str) -> xr.Dataset:
     return xr.open_dataset(path, engine=_NETCDF_ENGINE)
 
 
+def concat_evaluations(
+    datasets: Sequence[xr.Dataset], attrs: Optional[Mapping[str, Any]] = None
+) -> xr.Dataset:
+    """Pool schema-compliant evaluations along the ``sample`` axis.
+
+    Used to merge evaluations of DISJOINT sample sets (e.g. the test and
+    validation splits, or k-fold held-out slices) into one dataset covering all
+    of them. The parts must share the ``model``, ``noise_level`` and
+    ``class_name`` axes; only ``sample`` grows. The per-part ``sample`` index is
+    re-based to a contiguous range, while ``image_id``/``ptid``/``true_class``
+    travel with their rows.
+
+    Args:
+        datasets: Evaluations to pool, each over a distinct set of samples.
+        attrs: Attributes merged over the result (e.g. ``{"split": "test+val"}``).
+    """
+    parts = list(datasets)
+    if not parts:
+        raise ValueError("concat_evaluations requires at least one dataset.")
+
+    combined = xr.concat(parts, dim="sample")
+    combined = combined.assign_coords(sample=np.arange(combined.sizes["sample"]))
+    if attrs:
+        combined.attrs.update(dict(attrs))
+    return combined
+
+
 # ---------------------------------------------------------------------------
 # Incremental builder
 # ---------------------------------------------------------------------------

tasks/__init__.py

@@ -8,10 +8,11 @@ Pipeline stages (see ai/ARCHITECTURE.md):
     1. load_data        -> implemented
     2. train_regular    -> implemented
     3. train_bayesian   -> implemented
-    4. evaluate_regular -> implemented
-    5. evaluate_bayesian-> implemented
-    6. evaluate_noisy   -> implemented
-    7. run_analysis     -> skeleton (step 7); reads evaluations, writes analysis/.
+    4. evaluate_regular -> implemented (TEST split; *_val variants do VALIDATION)
+    5. evaluate_bayesian-> implemented (TEST split; *_val variants do VALIDATION)
+    6. evaluate_noisy   -> implemented (TEST split; *_val variant does VALIDATION)
+       combine_evaluations -> pools the per-split evaluations into combined_*.nc
+    7. run_analysis     -> reads evaluations (prefers combined_*), writes analysis/.
     load_models         -> implemented; discovers saved .pt ensembles on disk and
                            records their paths in ``state`` so evaluation is
                            decoupled from training (training frees models from VRAM
@@ -69,6 +70,22 @@ PIPELINE_TASKS: Dict[str, TaskEntry] = {
         "task_name": "Evaluate Models on Noised Data",
         "task_func": evaluate.evaluate_noisy_task,
     },
+    "evaluate_regular_val": {
+        "task_name": "Evaluate Regular Models (Validation)",
+        "task_func": evaluate.evaluate_normal_val_task,
+    },
+    "evaluate_bayesian_val": {
+        "task_name": "Evaluate Bayesian Models (Validation)",
+        "task_func": evaluate.evaluate_bayesian_val_task,
+    },
+    "evaluate_noisy_val": {
+        "task_name": "Evaluate Models on Noised Data (Validation)",
+        "task_func": evaluate.evaluate_noisy_val_task,
+    },
+    "combine_evaluations": {
+        "task_name": "Combine Split Evaluations",
+        "task_func": evaluate.combine_evaluations_task,
+    },
     "run_analysis": {
         "task_name": "Analyze Evaluations",
         "task_func": analysis.run_analysis_task,
@@ -123,6 +140,10 @@ TASK_WRITES = {
     "evaluate_regular": ("evaluations",),
     "evaluate_bayesian": ("evaluations",),
     "evaluate_noisy": ("evaluations",),
+    "evaluate_regular_val": ("evaluations",),
+    "evaluate_bayesian_val": ("evaluations",),
+    "evaluate_noisy_val": ("evaluations",),
+    "combine_evaluations": ("evaluations",),
     "run_analysis": ("analysis",),
     # k-fold writes per-fold models+evals under folds/ and the aggregated OOF
     # files under evaluations/; both are protected (slow to regenerate).
@@ -167,36 +188,49 @@ def describe_protected_conflicts(work_dir: str, scenario_id: str) -> List[str]:
         lines.append(f"{subdir}/ — {desc}")
     return lines
 
+# Evaluation of both held-out splits (test + val), pooled for analysis. Shared by
+# the train and load scenarios so they stay in lockstep.
+_EVAL_ALL_SPLITS = [
+    "evaluate_regular",
+    "evaluate_bayesian",
+    "evaluate_regular_val",
+    "evaluate_bayesian_val",
+]
+_EVAL_ALL_SPLITS_NOISY = ["evaluate_noisy", "evaluate_noisy_val"]
+
 SCENARIOS: Dict[str, ScenarioEntry] = {
     "scen_train_all": {
-        "label": "1. Train, Evaluate, & Noise Analysis",
+        "label": "1. Train, Evaluate (test+val), Noise, & Analyze",
         "tasks": [
             "load_data",
             "train_regular",
             "train_bayesian",
             "load_models",
-            "evaluate_regular",
-            "evaluate_bayesian",
-            "evaluate_noisy",
+            *_EVAL_ALL_SPLITS,
+            *_EVAL_ALL_SPLITS_NOISY,
+            "combine_evaluations",
+            "run_analysis",
         ],
     },
     "scen_load_all": {
-        "label": "2. Load, Evaluate, & Noise Analysis",
+        "label": "2. Load, Evaluate (test+val), Noise, & Analyze",
         "tasks": [
             "load_data",
             "load_models",
-            "evaluate_regular",
-            "evaluate_bayesian",
-            "evaluate_noisy",
+            *_EVAL_ALL_SPLITS,
+            *_EVAL_ALL_SPLITS_NOISY,
+            "combine_evaluations",
+            "run_analysis",
         ],
     },
     "scen_load_eval": {
-        "label": "3. Load & Evaluate (Skip Noise)",
+        "label": "3. Load & Evaluate (test+val, Skip Noise) & Analyze",
         "tasks": [
             "load_data",
             "load_models",
-            "evaluate_regular",
-            "evaluate_bayesian",
+            *_EVAL_ALL_SPLITS,
+            "combine_evaluations",
+            "run_analysis",
         ],
     },
     "scen_analyze": {
@@ -205,6 +239,8 @@ SCENARIOS: Dict[str, ScenarioEntry] = {
             "run_analysis",
         ],
     },
+    # Set aside for now (kept working, not part of the default flow): full
+    # patient-grouped cross-validation. Costs ~k x a normal train+eval run.
     "scen_kfold": {
         "label": "5. K-Fold Cross-Validation (train + OOF eval + analyze)",
         "tasks": [

tasks/analysis.py

@@ -107,6 +107,7 @@ def _family_metrics(kind: str, source: str, ds: xr.Dataset) -> Dict[str, Any]:
         "name": _FAMILY_LABELS.get(kind, kind),
         "kind": kind,
         "source": source,
+        "split": str(ds.attrs.get("split", "unknown")),
         "noise_sigma": float(np.asarray(ds["noise_level"].values, dtype=float)[baseline]),
         "n_models": len(models),
         "n_samples": int(ds.sizes["sample"]),
@@ -118,17 +119,37 @@ def _family_metrics(kind: str, source: str, ds: xr.Dataset) -> Dict[str, Any]:
     }
 
 
+def _source_rank(stem: str, ds: xr.Dataset) -> Tuple[int, int, str]:
+    """Sort key deciding which evaluation file represents a model family.
+
+    Prefers, in order: clean over noisy; then the widest sample coverage --
+    pooled files (``combined_*`` = test+val, ``oof_*`` = k-fold out-of-fold)
+    over a single split, and test over val when only single splits exist.
+    """
+    split = str(ds.attrs.get("split", ""))
+    if stem.startswith(("combined_", "oof_")) or "+" in split:
+        coverage = 0  # pooled: the most evaluation samples
+    elif split == "test":
+        coverage = 1
+    elif split == "val":
+        coverage = 2
+    else:
+        coverage = 3  # unlabeled (pre-split-attr files)
+    return ("noisy" in stem, coverage, stem)
+
+
 def _family_sources(
     datasets: Dict[str, xr.Dataset]
 ) -> Dict[str, Tuple[str, xr.Dataset]]:
-    """Pick one evaluation dataset per model family, preferring clean files.
+    """Pick one evaluation dataset per model family.
 
-    ``normal.nc`` and ``noisy_normal.nc`` share the same members at sigma=0, so
-    only the first source seen per family is kept (non-noisy stems sort first) to
-    avoid analyzing a family twice.
+    A family appears in several files (``normal.nc``, ``val_normal.nc``,
+    ``combined_normal.nc``, ``noisy_normal.nc``) that all describe the same
+    ensemble; only the best-ranked one is analyzed so a family is never counted
+    twice. See :func:`_source_rank` for the preference order.
     """
     chosen: Dict[str, Tuple[str, xr.Dataset]] = {}
-    for stem in sorted(datasets, key=lambda s: ("noisy" in s, s)):
+    for stem in sorted(datasets, key=lambda s: _source_rank(s, datasets[s])):
         ds = datasets[stem]
         if "model_kind" not in ds.coords:
             continue

tasks/evaluate.py

@@ -9,6 +9,13 @@ netCDF file in the locked-down evaluation schema (:mod:`evaluation.schema`).
     step 5  evaluate_bayesian -> Bayesian ensemble, noise=[0.0], K=mc_passes
     step 6  evaluate_noisy    -> both ensembles across config noise_levels
 
+Every evaluator is parameterized by dataset *split*, so the same code produces
+test-split and validation-split evaluations (``*_val_task`` variants). Both
+loaders are unshuffled, so sample order is stable in each. Finally,
+``combine_evaluations_task`` pools the per-split files into ``combined_*.nc``
+(test + val by default) -- the splits are patient-disjoint, so pooling simply
+enlarges the evaluation set that analysis draws on.
+
 Determinism
 -----------
 The test loader is unshuffled, so sample order is stable and aligns with the
@@ -31,6 +38,8 @@ import torch
 from evaluation.schema import (
     CLASS_NAMES,
     EvaluationAccumulator,
+    concat_evaluations,
+    load_evaluation,
     save_evaluation,
 )
 from model.factory import KIND_BAYESIAN, KIND_NORMAL, load_model
@@ -45,6 +54,20 @@ _NOISE_SEED_STREAM = 2
 # Subdirectory (under work_dir) where evaluation netCDF files are written.
 _EVAL_SUBDIR = "evaluations"
 
+# Evaluable dataset splits -> the ``state`` key holding their DataLoader. Both
+# val and test loaders are unshuffled, so their sample order is stable and can be
+# aligned with the netCDF ``sample`` axis. (``train`` is available for
+# completeness but is NOT part of the default pipeline: its samples were fit on,
+# so its accuracy is optimistic and not a generalization estimate.)
+SPLIT_TEST = "test"
+SPLIT_VAL = "val"
+SPLIT_TRAIN = "train"
+_SPLIT_LOADERS = {
+    SPLIT_TEST: "test_loader",
+    SPLIT_VAL: "val_loader",
+    SPLIT_TRAIN: "train_loader",
+}
+
 # Fallbacks if a config predates the [evaluation] section.
 _DEFAULT_NOISE_LEVELS: List[float] = [0.0, 0.02, 0.05, 0.1, 0.2]
 _DEFAULT_MC_PASSES = 30
@@ -145,6 +168,7 @@ def _evaluate_ensemble(
     noise_levels: Sequence[float],
     n_passes: int,
     out_path: pl.Path,
+    split: str = SPLIT_TEST,
 ) -> None:
     """Evaluate one ensemble and write a schema-compliant netCDF to ``out_path``."""
     device = config["training"]["device"]
@@ -152,13 +176,14 @@ def _evaluate_ensemble(
     base_seed = int(config["data"]["seed"])
     stop_event, pause_event = _events(state)
 
-    test_loader = state["test_loader"]
+    loader = state[_SPLIT_LOADERS[split]]
     image_to_ptid: Dict[int, str] = state.get("image_to_ptid", {})
 
-    image_ids, ptids, true_classes = _collect_sample_metadata(test_loader, image_to_ptid)
+    image_ids, ptids, true_classes = _collect_sample_metadata(loader, image_to_ptid)
     log.info(
-        f"Evaluating {len(model_paths)} {kind} model(s) on {len(image_ids)} samples, "
-        f"{len(noise_levels)} noise level(s), {n_passes} MC pass(es) each."
+        f"Evaluating {len(model_paths)} {kind} model(s) on {len(image_ids)} "
+        f"{split} samples, {len(noise_levels)} noise level(s), "
+        f"{n_passes} MC pass(es) each."
     )
 
     accumulator = EvaluationAccumulator(
@@ -177,7 +202,7 @@ def _evaluate_ensemble(
             noise_seed = derive_seed(base_seed, level_index, stream=_NOISE_SEED_STREAM)
             mc_probs = _run_mc_passes(
                 model=model,
-                loader=test_loader,
+                loader=loader,
                 device=device,
                 noise_level=float(noise),
                 n_passes=n_passes,
@@ -204,13 +229,14 @@ def _evaluate_ensemble(
             "seed": base_seed,
             "n_mc": n_passes,
             "ensemble_kind": kind,
+            "split": split,
             "config": _config_snapshot(config),
         }
     )
     out_path.parent.mkdir(parents=True, exist_ok=True)
     save_evaluation(dataset, str(out_path))
     log.info(
-        f"Saved {kind} evaluation to {out_path} "
+        f"Saved {kind} {split} evaluation to {out_path} "
         f"(models={dataset.sizes['model']}, samples={dataset.sizes['sample']}, "
         f"noise_levels={dataset.sizes['noise_level']})."
     )
@@ -234,99 +260,187 @@ def _out_path(config: Dict[str, Any], name: str) -> pl.Path:
     return pl.Path(config["work_dir"]) / _EVAL_SUBDIR / name
 
 
-# ---------------------------------------------------------------------------
-# Pipeline task entry points
-# ---------------------------------------------------------------------------
-def evaluate_normal_task(
-    track: ProgressTracker,
-    log: PipelineLogger,
-    config: Dict[str, Any],
-    state: Dict[str, Any],
-) -> None:
-    """Step 4: evaluate the normal ensemble on clean data (noise=0.0, K=1)."""
-    paths = state.get("normal_model_paths") or []
-    if not paths:
-        log.error("No normal models to evaluate (run training/load_models first).")
-        return
-    _evaluate_ensemble(
-        model_paths=paths,
-        kind=KIND_NORMAL,
-        config=config,
-        state=state,
-        log=log,
-        track=track,
-        noise_levels=[0.0],
-        n_passes=1,
-        out_path=_out_path(config, "normal.nc"),
-    )
+def eval_filename(split: str, kind: str, noisy: bool = False) -> str:
+    """Canonical evaluation filename for a (split, kind, noisy) combination.
+
+    The test split keeps its historical un-prefixed names (``normal.nc``,
+    ``noisy_bayesian.nc``); other splits are prefixed (``val_normal.nc``).
+    """
+    prefix = "" if split == SPLIT_TEST else f"{split}_"
+    return f"{prefix}{'noisy_' if noisy else ''}{kind}.nc"
 
 
-def evaluate_bayesian_task(
+def _evaluate_clean(
     track: ProgressTracker,
     log: PipelineLogger,
     config: Dict[str, Any],
     state: Dict[str, Any],
+    kind: str,
+    split: str,
 ) -> None:
-    """Step 5: evaluate the Bayesian ensemble on clean data with MC uncertainty."""
-    paths = state.get("bayesian_model_paths") or []
+    """Evaluate one ensemble on clean data for ``split`` (steps 4/5)."""
+    key = "normal_model_paths" if kind == KIND_NORMAL else "bayesian_model_paths"
+    paths = state.get(key) or []
     if not paths:
-        log.error("No Bayesian models to evaluate (run training/load_models first).")
+        log.error(f"No {kind} models to evaluate (run training/load_models first).")
         return
-    _, mc_passes = _eval_cfg(config)
+    n_passes = 1 if kind == KIND_NORMAL else _eval_cfg(config)[1]
     _evaluate_ensemble(
         model_paths=paths,
-        kind=KIND_BAYESIAN,
+        kind=kind,
         config=config,
         state=state,
         log=log,
         track=track,
         noise_levels=[0.0],
-        n_passes=mc_passes,
-        out_path=_out_path(config, "bayesian.nc"),
+        n_passes=n_passes,
+        out_path=_out_path(config, eval_filename(split, kind)),
+        split=split,
     )
 
 
-def evaluate_noisy_task(
+def _evaluate_noisy(
     track: ProgressTracker,
     log: PipelineLogger,
     config: Dict[str, Any],
     state: Dict[str, Any],
+    split: str,
 ) -> None:
-    """Step 6: evaluate both ensembles across the configured Gaussian noise levels."""
+    """Evaluate both ensembles across the noise sweep for ``split`` (step 6)."""
     noise_levels, mc_passes = _eval_cfg(config)
-    normal_paths = state.get("normal_model_paths") or []
-    bayesian_paths = state.get("bayesian_model_paths") or []
-
-    if not normal_paths and not bayesian_paths:
-        log.error("No models to evaluate on noised data.")
-        return
-
-    if normal_paths:
+    any_models = False
+    for kind, key, n_passes in (
+        (KIND_NORMAL, "normal_model_paths", 1),
+        (KIND_BAYESIAN, "bayesian_model_paths", mc_passes),
+    ):
+        paths = state.get(key) or []
+        if not paths:
+            log.info(f"No {kind} models found; skipping {kind} noisy evaluation.")
+            continue
+        any_models = True
         _evaluate_ensemble(
-            model_paths=normal_paths,
-            kind=KIND_NORMAL,
+            model_paths=paths,
+            kind=kind,
             config=config,
             state=state,
             log=log,
             track=track,
             noise_levels=noise_levels,
-            n_passes=1,
-            out_path=_out_path(config, "noisy_normal.nc"),
+            n_passes=n_passes,
+            out_path=_out_path(config, eval_filename(split, kind, noisy=True)),
+            split=split,
         )
-    else:
-        log.info("No normal models found; skipping normal noisy evaluation.")
+    if not any_models:
+        log.error("No models to evaluate on noised data.")
 
-    if bayesian_paths:
-        _evaluate_ensemble(
-            model_paths=bayesian_paths,
-            kind=KIND_BAYESIAN,
-            config=config,
-            state=state,
-            log=log,
-            track=track,
-            noise_levels=noise_levels,
-            n_passes=mc_passes,
-            out_path=_out_path(config, "noisy_bayesian.nc"),
+
+# ---------------------------------------------------------------------------
+# Pipeline task entry points
+# ---------------------------------------------------------------------------
+def evaluate_normal_task(track, log, config, state) -> None:
+    """Step 4: normal ensemble on the clean TEST split."""
+    _evaluate_clean(track, log, config, state, KIND_NORMAL, SPLIT_TEST)
+
+
+def evaluate_bayesian_task(track, log, config, state) -> None:
+    """Step 5: Bayesian ensemble (MC uncertainty) on the clean TEST split."""
+    _evaluate_clean(track, log, config, state, KIND_BAYESIAN, SPLIT_TEST)
+
+
+def evaluate_noisy_task(track, log, config, state) -> None:
+    """Step 6: both ensembles across the noise sweep on the TEST split."""
+    _evaluate_noisy(track, log, config, state, SPLIT_TEST)
+
+
+def evaluate_normal_val_task(track, log, config, state) -> None:
+    """Normal ensemble on the clean VALIDATION split."""
+    _evaluate_clean(track, log, config, state, KIND_NORMAL, SPLIT_VAL)
+
+
+def evaluate_bayesian_val_task(track, log, config, state) -> None:
+    """Bayesian ensemble (MC uncertainty) on the clean VALIDATION split."""
+    _evaluate_clean(track, log, config, state, KIND_BAYESIAN, SPLIT_VAL)
+
+
+def evaluate_noisy_val_task(track, log, config, state) -> None:
+    """Both ensembles across the noise sweep on the VALIDATION split."""
+    _evaluate_noisy(track, log, config, state, SPLIT_VAL)
+
+
+def combine_evaluations_task(
+    track: ProgressTracker,
+    log: PipelineLogger,
+    config: Dict[str, Any],
+    state: Dict[str, Any],
+) -> None:
+    """Pool per-split evaluations into ``combined_*.nc`` for analysis.
+
+    Concatenates the configured splits (default test + val) along the ``sample``
+    axis. The splits are patient-disjoint by construction, so pooling them yields
+    a larger evaluation set with no duplicated samples -- which is the point:
+    more evaluation samples means tighter accuracy/uncertainty estimates.
+    Analysis prefers these pooled files over the single-split ones.
+    """
+    eval_dir = pl.Path(config["work_dir"]) / _EVAL_SUBDIR
+    splits = list(
+        config.get("evaluation", {}).get("combine_splits", [SPLIT_TEST, SPLIT_VAL])
+    )
+    if len(splits) < 2:
+        log.info(
+            f"combine_evaluations: only {splits} configured; nothing to pool."
+        )
+        return
+
+    written = 0
+    for kind in (KIND_NORMAL, KIND_BAYESIAN):
+        for noisy in (False, True):
+            paths = [
+                eval_dir / eval_filename(split, kind, noisy=noisy) for split in splits
+            ]
+            present = [p for p in paths if p.exists()]
+            if len(present) < 2:
+                continue
+
+            parts = [load_evaluation(str(p)) for p in present]
+            try:
+                # Guard against a mis-specified split: the same patient must not
+                # appear in two pooled parts (that would double-count them).
+                seen: set[str] = set()
+                overlap: set[str] = set()
+                for part in parts:
+                    part_ptids = {str(p) for p in np.atleast_1d(part["ptid"].values)}
+                    overlap |= seen & part_ptids
+                    seen |= part_ptids
+                if overlap:
+                    log.error(
+                        f"combine_evaluations: {len(overlap)} patient(s) appear in "
+                        f"more than one split ({sorted(overlap)[:3]}...); refusing "
+                        f"to pool {kind} (noisy={noisy})."
+                    )
+                    continue
+
+                combined = concat_evaluations(
+                    parts,
+                    attrs={
+                        "split": "+".join(splits),
+                        "combined_from": ", ".join(p.name for p in present),
+                        "n_sources": len(present),
+                    },
+                )
+                out_name = f"combined_{'noisy_' if noisy else ''}{kind}.nc"
+                save_evaluation(combined, str(eval_dir / out_name))
+                written += 1
+                log.info(
+                    f"combine_evaluations: wrote {out_name} "
+                    f"({combined.sizes['sample']} samples from "
+                    f"{', '.join(p.name for p in present)})."
+                )
+            finally:
+                for part in parts:
+                    part.close()
+
+    if written == 0:
+        log.error(
+            "combine_evaluations: found no split pairs to pool. Run the test and "
+            "validation evaluation steps first."
         )
-    else:
-        log.info("No Bayesian models found; skipping Bayesian noisy evaluation.")

tasks/kfold.py

@@ -27,16 +27,26 @@ from util.control import check_control_events
 from util.progress import ProgressTracker
 from util.ui_logger import PipelineLogger
 
-from . import evaluate, load_data, load_models, train_bayesian, train_normal
+def _inner_tasks(include_noise: bool):
+    """Per-fold pipeline (load_data is handled by the driver; noise optional).
+
+    Sibling task modules are imported lazily so this module never needs the
+    ``tasks`` package to be fully initialized at import time (``tasks/__init__``
+    imports this module in order to register the k-fold task).
+    """
+    from . import evaluate, load_models, train_bayesian, train_normal
+
+    tasks = [
+        ("Train Regular", train_normal.train_normal_task),
+        ("Train Bayesian", train_bayesian.train_bayesian_task),
+        ("Load Models", load_models.load_models_task),
+        ("Evaluate Regular", evaluate.evaluate_normal_task),
+        ("Evaluate Bayesian", evaluate.evaluate_bayesian_task),
+    ]
+    if include_noise:
+        tasks.append(("Evaluate Noisy", evaluate.evaluate_noisy_task))
+    return tasks
 
-# Inner per-fold pipeline (load_data is handled by the driver; noise optional).
-_BASE_INNER_TASKS = [
-    ("Train Regular", train_normal.train_normal_task),
-    ("Train Bayesian", train_bayesian.train_bayesian_task),
-    ("Load Models", load_models.load_models_task),
-    ("Evaluate Regular", evaluate.evaluate_normal_task),
-    ("Evaluate Bayesian", evaluate.evaluate_bayesian_task),
-]
 
 _FOLDS_SUBDIR = "folds"
 _EVAL_SUBDIR = "evaluations"
@@ -95,15 +105,15 @@ def run_kfold_task(
     stop_event = events.get("stop") if isinstance(events, dict) else None
     pause_event = events.get("pause") if isinstance(events, dict) else None
 
+    from . import load_data  # lazy, as in _inner_tasks
+
     base_seed = load_data.seed_pipeline(config, log)
     k_folds, val_fraction, include_noise = _kfold_cfg(config)
 
     dataset, ptids, image_to_ptid = load_data.load_full_dataset(config, log)
     state["image_to_ptid"] = image_to_ptid
 
-    inner_tasks = list(_BASE_INNER_TASKS)
-    if include_noise:
-        inner_tasks.append(("Evaluate Noisy", evaluate.evaluate_noisy_task))
+    inner_tasks = _inner_tasks(include_noise)
 
     work_dir = pl.Path(config["work_dir"])
     fold_dirs: List[pl.Path] = []

tasks/templates/model_report.typ

@@ -36,8 +36,9 @@
 #let family-block(fam) = {
   heading(level: 2, fam.name)
   text(9pt, fill: luma(90))[
-    Source: #raw(fam.source + ".nc") · noise σ = #fmt-num(fam.noise_sigma) ·
-    #fam.n_models model(s) · #fam.n_samples test samples · #fam.n_mc MC pass(es)
+    Source: #raw(fam.source + ".nc") · split #raw(fam.split) ·
+    noise σ = #fmt-num(fam.noise_sigma) ·
+    #fam.n_models model(s) · #fam.n_samples samples · #fam.n_mc MC pass(es)
   ]
   v(4pt)
   block(fill: luma(245), inset: 8pt, radius: 4pt, width: 100%)[