alnn_rewrite/evaluation/confidence_percentile.py

import xarray as xr
import numpy as np
import sys
import os

import matplotlib.pyplot as plt


sys.path.append(
    os.path.abspath(os.path.join(os.path.dirname(__file__), ".."))
)  # to allow imports from parent directory
from utils.config import config
import pathlib as pl

import colorama as clr

model_dataset_path = pl.Path(config["output"]["path"]) / "model_evaluation_results.nc"
array = xr.open_dataset(model_dataset_path)  # type: ignore


predictions: xr.DataArray = array["predictions"]
labels: xr.DataArray = array["labels"]

# Make plots directory if it doesn't exist
plots_dir = pl.Path(config["output"]["path"]) / "plots"
plots_dir.mkdir(parents=True, exist_ok=True)

# This script calculates and plots accuracy vs minimum confidence percentile threshold

# Average predictions across models
avg_predictions = predictions.mean(dim="model")
# Get confidence scores for the positive class
confidence_scores = avg_predictions.sel(img_class=1).values
true_labels = labels.sel(label=1).values


# Calculate accuracy at different confidence percentiles
percentiles = np.linspace(0, 100, num=21)
accuracies = []
for p in percentiles:
    absolute_confidences = 2 * np.abs(confidence_scores - 0.5)
    threshold = np.percentile(absolute_confidences, p)

    # Filter the predictions such that only those with absolute confidence above the threshold are considered
    selected_indices = np.where(absolute_confidences >= threshold)[0]
    if len(selected_indices) == 0:
        accuracies.append(0.0)
        continue
    selected_confidences = confidence_scores[selected_indices]
    selected_true_labels = true_labels[selected_indices]

    predicted_positive = selected_confidences >= 0.5
    true_positive = selected_true_labels == 1

    correct_predictions = (predicted_positive == true_positive).sum().item()
    total_predictions = len(selected_confidences)
    accuracy = correct_predictions / total_predictions if total_predictions > 0 else 0.0
    accuracies.append(accuracy)

# Plot accuracy vs confidence percentile threshold
plt.figure(figsize=(10, 6))
plt.plot(percentiles, accuracies, marker="o")
plt.title("Accuracy vs Confidence Percentile Threshold")
plt.xlabel("Confidence Percentile Threshold")
plt.ylabel("Accuracy")
plt.grid()
plt.xticks(percentiles)

plt.savefig(
    pl.Path(config["output"]["path"])
    / "plots"
    / "accuracy_vs_confidence_percentile.png"
)