|
|
@@ -1,153 +0,0 @@
|
|
|
-{
|
|
|
- "cells": [
|
|
|
- {
|
|
|
- "cell_type": "code",
|
|
|
- "execution_count": null,
|
|
|
- "id": "8f50efc2",
|
|
|
- "metadata": {},
|
|
|
- "outputs": [],
|
|
|
- "source": [
|
|
|
- "import numpy as np\n",
|
|
|
- "import matplotlib.pyplot as plt\n",
|
|
|
- "import seaborn as sns\n",
|
|
|
- "from scipy import io\n",
|
|
|
- "from sklearn.linear_model import LogisticRegression\n",
|
|
|
- "from sklearn.preprocessing import StandardScaler # , PolynomialFeatures\n",
|
|
|
- "from sklearn.model_selection import StratifiedKFold, cross_val_score\n",
|
|
|
- "from sklearn.metrics import roc_curve, auc\n",
|
|
|
- "\n",
|
|
|
- "# ---------------------------\n",
|
|
|
- "# Load the data from .mat files\n",
|
|
|
- "# ---------------------------\n",
|
|
|
- "data_path = \"../data/\"\n",
|
|
|
- "suv_file = data_path + \"suv_percentilesSLOthenUWM.mat\"\n",
|
|
|
- "flags_file = data_path + \"flags_combined.mat\"\n",
|
|
|
- "normal_range_file = data_path + \"normal_range.mat\"\n",
|
|
|
- "\n",
|
|
|
- "# Load .mat files\n",
|
|
|
- "suv_dict = io.loadmat(suv_file)\n",
|
|
|
- "flags_dict = io.loadmat(flags_file)\n",
|
|
|
- "normal_range_dict = io.loadmat(normal_range_file)\n",
|
|
|
- "\n",
|
|
|
- "# Extract relevant data\n",
|
|
|
- "# Adjust the slicing as needed based on your data dimensions.\n",
|
|
|
- "suv = suv_dict['lung_SUVperc_COMBINED'][0:58, :, :]\n",
|
|
|
- "flags = flags_dict['flags'][0:58, 3]\n",
|
|
|
- "\n",
|
|
|
- "# ---------------------------\n",
|
|
|
- "# Feature Engineering\n",
|
|
|
- "# ---------------------------\n",
|
|
|
- "# Choose the percentile of interest. For lung and bowel, p = 94 (0-indexed)\n",
|
|
|
- "p = 94\n",
|
|
|
- "\n",
|
|
|
- "# Compute a set of summary statistics for the chosen percentile.\n",
|
|
|
- "feature_max = np.nanmax(suv[:, :, p], axis=1)\n",
|
|
|
- "feature_mean = np.nanmean(suv[:, :, p], axis=1)\n",
|
|
|
- "feature_std = np.nanstd(suv[:, :, p], axis=1)\n",
|
|
|
- "\n",
|
|
|
- "# Stack the features so that each row corresponds to one sample and\n",
|
|
|
- "# each column is a different feature.\n",
|
|
|
- "X = np.column_stack((feature_max, feature_mean, feature_std))\n",
|
|
|
- "\n",
|
|
|
- "# Handle missing values by selecting only rows with complete data.\n",
|
|
|
- "valid_indices = ~np.isnan(X).any(axis=1)\n",
|
|
|
- "X = X[valid_indices]\n",
|
|
|
- "flags = flags[valid_indices]\n",
|
|
|
- "\n",
|
|
|
- "# ---------------------------\n",
|
|
|
- "# Preprocessing: Feature Scaling\n",
|
|
|
- "# ---------------------------\n",
|
|
|
- "scaler = StandardScaler()\n",
|
|
|
- "X_scaled = scaler.fit_transform(X)\n",
|
|
|
- "\n",
|
|
|
- "# --------------\n",
|
|
|
- "# (Optional) Experiment with Polynomial Features for non-linearity\n",
|
|
|
- "# --------------\n",
|
|
|
- "# Uncomment below to include non-linear interactions\n",
|
|
|
- "# from sklearn.preprocessing import PolynomialFeatures\n",
|
|
|
- "# poly = PolynomialFeatures(degree=2, include_bias=False)\n",
|
|
|
- "# X_scaled = poly.fit_transform(X_scaled)\n",
|
|
|
- "\n",
|
|
|
- "# ---------------------------\n",
|
|
|
- "# Model: Regularized Logistic Regression\n",
|
|
|
- "# ---------------------------\n",
|
|
|
- "# We use L2 regularization; you can change penalty to 'l1' to test Lasso.\n",
|
|
|
- "clf = LogisticRegression(penalty='l2', solver='liblinear', random_state=42)\n",
|
|
|
- "clf.fit(X_scaled, flags)\n",
|
|
|
- "\n",
|
|
|
- "# Display model coefficients (note: these are in the scaled feature space)\n",
|
|
|
- "print(\"Intercept:\", clf.intercept_)\n",
|
|
|
- "print(\"Coefficients:\", clf.coef_)\n",
|
|
|
- "\n",
|
|
|
- "# ---------------------------\n",
|
|
|
- "# Model Evaluation: Cross-Validation Accuracy\n",
|
|
|
- "# ---------------------------\n",
|
|
|
- "cv = StratifiedKFold(n_splits=5, shuffle=True, random_state=42)\n",
|
|
|
- "accuracy_scores = cross_val_score(clf, X_scaled, flags, cv=cv, scoring='accuracy')\n",
|
|
|
- "print(\"Cross-validated accuracy scores:\", accuracy_scores)\n",
|
|
|
- "print(\"Mean cross-validated accuracy:\", np.mean(accuracy_scores))\n",
|
|
|
- "\n",
|
|
|
- "# ---------------------------\n",
|
|
|
- "# Additional Evaluation: ROC Curve and AUC\n",
|
|
|
- "# ---------------------------\n",
|
|
|
- "# Get predicted probabilities for the positive class\n",
|
|
|
- "y_scores = clf.predict_proba(X_scaled)[:, 1]\n",
|
|
|
- "\n",
|
|
|
- "# Compute ROC curve\n",
|
|
|
- "fpr, tpr, thresholds = roc_curve(flags, y_scores)\n",
|
|
|
- "roc_auc = auc(fpr, tpr)\n",
|
|
|
- "print(\"AUC:\", roc_auc)\n",
|
|
|
- "\n",
|
|
|
- "# Plot ROC curve\n",
|
|
|
- "plt.figure(figsize=(8, 6))\n",
|
|
|
- "plt.plot(fpr, tpr, color='darkorange', lw=2, label='ROC curve (AUC = %0.2f)' % roc_auc)\n",
|
|
|
- "plt.plot([0, 1], [0, 1], color='navy', lw=2, linestyle='--')\n",
|
|
|
- "plt.xlim([0.0, 1.0])\n",
|
|
|
- "plt.ylim([0.0, 1.05])\n",
|
|
|
- "plt.xlabel('False Positive Rate')\n",
|
|
|
- "plt.ylabel('True Positive Rate')\n",
|
|
|
- "plt.title('Receiver Operating Characteristic (ROC)')\n",
|
|
|
- "plt.legend(loc=\"lower right\")\n",
|
|
|
- "plt.show()\n",
|
|
|
- "\n",
|
|
|
- "# ---------------------------\n",
|
|
|
- "# Plotting Logistic Curve for Primary Feature (maxSUV)\n",
|
|
|
- "# ---------------------------\n",
|
|
|
- "# For visualization, we pick the feature corresponding to maxSUV (first column).\n",
|
|
|
- "feature_index = 0\n",
|
|
|
- "x_range = np.linspace(X[:, feature_index].min(), X[:, feature_index].max(), 100)\n",
|
|
|
- "\n",
|
|
|
- "# For the additional features (mean and std), we set them to their average values.\n",
|
|
|
- "avg_mean = np.mean(X[:, 1])\n",
|
|
|
- "avg_std = np.mean(X[:, 2])\n",
|
|
|
- "X_plot = np.column_stack((x_range, np.repeat(avg_mean, 100), np.repeat(avg_std, 100)))\n",
|
|
|
- "\n",
|
|
|
- "# Scale the plot data using the same scaler.\n",
|
|
|
- "X_plot_scaled = scaler.transform(X_plot)\n",
|
|
|
- "\n",
|
|
|
- "# Compute predicted probabilities along the x_range.\n",
|
|
|
- "pred_probs = clf.predict_proba(X_plot_scaled)[:, 1]\n",
|
|
|
- "\n",
|
|
|
- "plt.figure(figsize=(10, 6))\n",
|
|
|
- "# Scatter plot of observed data along maxSUV (using flags to indicate classes).\n",
|
|
|
- "plt.scatter(X[flags == 0, feature_index], flags[flags == 0], label=\"NC\", color='blue', alpha=0.6)\n",
|
|
|
- "plt.scatter(X[flags == 1, feature_index], flags[flags == 1], label=\"AE\", color='red', alpha=0.6)\n",
|
|
|
- "# Plot the logistic regression prediction curve.\n",
|
|
|
- "plt.plot(x_range, pred_probs, color='green', lw=2, label=\"Regularized Logistic Regression\")\n",
|
|
|
- "plt.title(\"Prediction Probability of AE Depending on maxSUV\")\n",
|
|
|
- "plt.xlabel(\"maxSUV\")\n",
|
|
|
- "plt.ylabel(\"P(AE)\")\n",
|
|
|
- "plt.legend(loc='best')\n",
|
|
|
- "plt.grid(True)\n",
|
|
|
- "plt.show()\n"
|
|
|
- ]
|
|
|
- }
|
|
|
- ],
|
|
|
- "metadata": {
|
|
|
- "language_info": {
|
|
|
- "name": "python"
|
|
|
- }
|
|
|
- },
|
|
|
- "nbformat": 4,
|
|
|
- "nbformat_minor": 5
|
|
|
-}
|
