{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "aa8e6177",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "c:\\Users\\zahra\\anaconda3\\Lib\\site-packages\\xgboost\\core.py:158: UserWarning: [19:41:32] WARNING: C:\\buildkite-agent\\builds\\buildkite-windows-cpu-autoscaling-group-i-0c55ff5f71b100e98-1\\xgboost\\xgboost-ci-windows\\src\\learner.cc:740: \n",
      "Parameters: { \"use_label_encoder\" } are not used.\n",
      "\n",
      "  warnings.warn(smsg, UserWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "AUC (XGBoost): 0.6875\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "c:\\Users\\zahra\\anaconda3\\Lib\\site-packages\\xgboost\\core.py:158: UserWarning: [19:41:32] WARNING: C:\\buildkite-agent\\builds\\buildkite-windows-cpu-autoscaling-group-i-0c55ff5f71b100e98-1\\xgboost\\xgboost-ci-windows\\src\\learner.cc:740: \n",
      "Parameters: { \"use_label_encoder\" } are not used.\n",
      "\n",
      "  warnings.warn(smsg, UserWarning)\n",
      "c:\\Users\\zahra\\anaconda3\\Lib\\site-packages\\xgboost\\core.py:158: UserWarning: [19:41:33] WARNING: C:\\buildkite-agent\\builds\\buildkite-windows-cpu-autoscaling-group-i-0c55ff5f71b100e98-1\\xgboost\\xgboost-ci-windows\\src\\learner.cc:740: \n",
      "Parameters: { \"use_label_encoder\" } are not used.\n",
      "\n",
      "  warnings.warn(smsg, UserWarning)\n",
      "c:\\Users\\zahra\\anaconda3\\Lib\\site-packages\\xgboost\\core.py:158: UserWarning: [19:41:34] WARNING: C:\\buildkite-agent\\builds\\buildkite-windows-cpu-autoscaling-group-i-0c55ff5f71b100e98-1\\xgboost\\xgboost-ci-windows\\src\\learner.cc:740: \n",
      "Parameters: { \"use_label_encoder\" } are not used.\n",
      "\n",
      "  warnings.warn(smsg, UserWarning)\n",
      "c:\\Users\\zahra\\anaconda3\\Lib\\site-packages\\xgboost\\core.py:158: UserWarning: [19:41:35] WARNING: C:\\buildkite-agent\\builds\\buildkite-windows-cpu-autoscaling-group-i-0c55ff5f71b100e98-1\\xgboost\\xgboost-ci-windows\\src\\learner.cc:740: \n",
      "Parameters: { \"use_label_encoder\" } are not used.\n",
      "\n",
      "  warnings.warn(smsg, UserWarning)\n",
      "c:\\Users\\zahra\\anaconda3\\Lib\\site-packages\\xgboost\\core.py:158: UserWarning: [19:41:36] WARNING: C:\\buildkite-agent\\builds\\buildkite-windows-cpu-autoscaling-group-i-0c55ff5f71b100e98-1\\xgboost\\xgboost-ci-windows\\src\\learner.cc:740: \n",
      "Parameters: { \"use_label_encoder\" } are not used.\n",
      "\n",
      "  warnings.warn(smsg, UserWarning)\n",
      "c:\\Users\\zahra\\anaconda3\\Lib\\site-packages\\xgboost\\core.py:158: UserWarning: [19:41:37] WARNING: C:\\buildkite-agent\\builds\\buildkite-windows-cpu-autoscaling-group-i-0c55ff5f71b100e98-1\\xgboost\\xgboost-ci-windows\\src\\learner.cc:740: \n",
      "Parameters: { \"use_label_encoder\" } are not used.\n",
      "\n",
      "  warnings.warn(smsg, UserWarning)\n",
      "c:\\Users\\zahra\\anaconda3\\Lib\\site-packages\\xgboost\\core.py:158: UserWarning: [19:41:38] WARNING: C:\\buildkite-agent\\builds\\buildkite-windows-cpu-autoscaling-group-i-0c55ff5f71b100e98-1\\xgboost\\xgboost-ci-windows\\src\\learner.cc:740: \n",
      "Parameters: { \"use_label_encoder\" } are not used.\n",
      "\n",
      "  warnings.warn(smsg, UserWarning)\n",
      "c:\\Users\\zahra\\anaconda3\\Lib\\site-packages\\xgboost\\core.py:158: UserWarning: [19:41:39] WARNING: C:\\buildkite-agent\\builds\\buildkite-windows-cpu-autoscaling-group-i-0c55ff5f71b100e98-1\\xgboost\\xgboost-ci-windows\\src\\learner.cc:740: \n",
      "Parameters: { \"use_label_encoder\" } are not used.\n",
      "\n",
      "  warnings.warn(smsg, UserWarning)\n",
      "c:\\Users\\zahra\\anaconda3\\Lib\\site-packages\\xgboost\\core.py:158: UserWarning: [19:41:40] WARNING: C:\\buildkite-agent\\builds\\buildkite-windows-cpu-autoscaling-group-i-0c55ff5f71b100e98-1\\xgboost\\xgboost-ci-windows\\src\\learner.cc:740: \n",
      "Parameters: { \"use_label_encoder\" } are not used.\n",
      "\n",
      "  warnings.warn(smsg, UserWarning)\n",
      "c:\\Users\\zahra\\anaconda3\\Lib\\site-packages\\xgboost\\core.py:158: UserWarning: [19:41:41] WARNING: C:\\buildkite-agent\\builds\\buildkite-windows-cpu-autoscaling-group-i-0c55ff5f71b100e98-1\\xgboost\\xgboost-ci-windows\\src\\learner.cc:740: \n",
      "Parameters: { \"use_label_encoder\" } are not used.\n",
      "\n",
      "  warnings.warn(smsg, UserWarning)\n",
      "c:\\Users\\zahra\\anaconda3\\Lib\\site-packages\\xgboost\\core.py:158: UserWarning: [19:41:42] WARNING: C:\\buildkite-agent\\builds\\buildkite-windows-cpu-autoscaling-group-i-0c55ff5f71b100e98-1\\xgboost\\xgboost-ci-windows\\src\\learner.cc:740: \n",
      "Parameters: { \"use_label_encoder\" } are not used.\n",
      "\n",
      "  warnings.warn(smsg, UserWarning)\n",
      "c:\\Users\\zahra\\anaconda3\\Lib\\site-packages\\xgboost\\core.py:158: UserWarning: [19:41:43] WARNING: C:\\buildkite-agent\\builds\\buildkite-windows-cpu-autoscaling-group-i-0c55ff5f71b100e98-1\\xgboost\\xgboost-ci-windows\\src\\learner.cc:740: \n",
      "Parameters: { \"use_label_encoder\" } are not used.\n",
      "\n",
      "  warnings.warn(smsg, UserWarning)\n",
      "c:\\Users\\zahra\\anaconda3\\Lib\\site-packages\\xgboost\\core.py:158: UserWarning: [19:41:44] WARNING: C:\\buildkite-agent\\builds\\buildkite-windows-cpu-autoscaling-group-i-0c55ff5f71b100e98-1\\xgboost\\xgboost-ci-windows\\src\\learner.cc:740: \n",
      "Parameters: { \"use_label_encoder\" } are not used.\n",
      "\n",
      "  warnings.warn(smsg, UserWarning)\n",
      "c:\\Users\\zahra\\anaconda3\\Lib\\site-packages\\xgboost\\core.py:158: UserWarning: [19:41:45] WARNING: C:\\buildkite-agent\\builds\\buildkite-windows-cpu-autoscaling-group-i-0c55ff5f71b100e98-1\\xgboost\\xgboost-ci-windows\\src\\learner.cc:740: \n",
      "Parameters: { \"use_label_encoder\" } are not used.\n",
      "\n",
      "  warnings.warn(smsg, UserWarning)\n",
      "c:\\Users\\zahra\\anaconda3\\Lib\\site-packages\\xgboost\\core.py:158: UserWarning: [19:41:46] WARNING: C:\\buildkite-agent\\builds\\buildkite-windows-cpu-autoscaling-group-i-0c55ff5f71b100e98-1\\xgboost\\xgboost-ci-windows\\src\\learner.cc:740: \n",
      "Parameters: { \"use_label_encoder\" } are not used.\n",
      "\n",
      "  warnings.warn(smsg, UserWarning)\n",
      "c:\\Users\\zahra\\anaconda3\\Lib\\site-packages\\xgboost\\core.py:158: UserWarning: [19:41:47] WARNING: C:\\buildkite-agent\\builds\\buildkite-windows-cpu-autoscaling-group-i-0c55ff5f71b100e98-1\\xgboost\\xgboost-ci-windows\\src\\learner.cc:740: \n",
      "Parameters: { \"use_label_encoder\" } are not used.\n",
      "\n",
      "  warnings.warn(smsg, UserWarning)\n",
      "c:\\Users\\zahra\\anaconda3\\Lib\\site-packages\\xgboost\\core.py:158: UserWarning: [19:41:48] WARNING: C:\\buildkite-agent\\builds\\buildkite-windows-cpu-autoscaling-group-i-0c55ff5f71b100e98-1\\xgboost\\xgboost-ci-windows\\src\\learner.cc:740: \n",
      "Parameters: { \"use_label_encoder\" } are not used.\n",
      "\n",
      "  warnings.warn(smsg, UserWarning)\n",
      "c:\\Users\\zahra\\anaconda3\\Lib\\site-packages\\xgboost\\core.py:158: UserWarning: [19:41:49] WARNING: C:\\buildkite-agent\\builds\\buildkite-windows-cpu-autoscaling-group-i-0c55ff5f71b100e98-1\\xgboost\\xgboost-ci-windows\\src\\learner.cc:740: \n",
      "Parameters: { \"use_label_encoder\" } are not used.\n",
      "\n",
      "  warnings.warn(smsg, UserWarning)\n",
      "c:\\Users\\zahra\\anaconda3\\Lib\\site-packages\\xgboost\\core.py:158: UserWarning: [19:41:50] WARNING: C:\\buildkite-agent\\builds\\buildkite-windows-cpu-autoscaling-group-i-0c55ff5f71b100e98-1\\xgboost\\xgboost-ci-windows\\src\\learner.cc:740: \n",
      "Parameters: { \"use_label_encoder\" } are not used.\n",
      "\n",
      "  warnings.warn(smsg, UserWarning)\n",
      "c:\\Users\\zahra\\anaconda3\\Lib\\site-packages\\xgboost\\core.py:158: UserWarning: [19:41:51] WARNING: C:\\buildkite-agent\\builds\\buildkite-windows-cpu-autoscaling-group-i-0c55ff5f71b100e98-1\\xgboost\\xgboost-ci-windows\\src\\learner.cc:740: \n",
      "Parameters: { \"use_label_encoder\" } are not used.\n",
      "\n",
      "  warnings.warn(smsg, UserWarning)\n",
      "c:\\Users\\zahra\\anaconda3\\Lib\\site-packages\\xgboost\\core.py:158: UserWarning: [19:41:52] WARNING: C:\\buildkite-agent\\builds\\buildkite-windows-cpu-autoscaling-group-i-0c55ff5f71b100e98-1\\xgboost\\xgboost-ci-windows\\src\\learner.cc:740: \n",
      "Parameters: { \"use_label_encoder\" } are not used.\n",
      "\n",
      "  warnings.warn(smsg, UserWarning)\n",
      "c:\\Users\\zahra\\anaconda3\\Lib\\site-packages\\xgboost\\core.py:158: UserWarning: [19:41:53] WARNING: C:\\buildkite-agent\\builds\\buildkite-windows-cpu-autoscaling-group-i-0c55ff5f71b100e98-1\\xgboost\\xgboost-ci-windows\\src\\learner.cc:740: \n",
      "Parameters: { \"use_label_encoder\" } are not used.\n",
      "\n",
      "  warnings.warn(smsg, UserWarning)\n",
      "c:\\Users\\zahra\\anaconda3\\Lib\\site-packages\\xgboost\\core.py:158: UserWarning: [19:41:54] WARNING: C:\\buildkite-agent\\builds\\buildkite-windows-cpu-autoscaling-group-i-0c55ff5f71b100e98-1\\xgboost\\xgboost-ci-windows\\src\\learner.cc:740: \n",
      "Parameters: { \"use_label_encoder\" } are not used.\n",
      "\n",
      "  warnings.warn(smsg, UserWarning)\n",
      "c:\\Users\\zahra\\anaconda3\\Lib\\site-packages\\xgboost\\core.py:158: UserWarning: [19:41:55] WARNING: C:\\buildkite-agent\\builds\\buildkite-windows-cpu-autoscaling-group-i-0c55ff5f71b100e98-1\\xgboost\\xgboost-ci-windows\\src\\learner.cc:740: \n",
      "Parameters: { \"use_label_encoder\" } are not used.\n",
      "\n",
      "  warnings.warn(smsg, UserWarning)\n",
      "c:\\Users\\zahra\\anaconda3\\Lib\\site-packages\\xgboost\\core.py:158: UserWarning: [19:41:56] WARNING: C:\\buildkite-agent\\builds\\buildkite-windows-cpu-autoscaling-group-i-0c55ff5f71b100e98-1\\xgboost\\xgboost-ci-windows\\src\\learner.cc:740: \n",
      "Parameters: { \"use_label_encoder\" } are not used.\n",
      "\n",
      "  warnings.warn(smsg, UserWarning)\n",
      "c:\\Users\\zahra\\anaconda3\\Lib\\site-packages\\xgboost\\core.py:158: UserWarning: [19:41:57] WARNING: C:\\buildkite-agent\\builds\\buildkite-windows-cpu-autoscaling-group-i-0c55ff5f71b100e98-1\\xgboost\\xgboost-ci-windows\\src\\learner.cc:740: \n",
      "Parameters: { \"use_label_encoder\" } are not used.\n",
      "\n",
      "  warnings.warn(smsg, UserWarning)\n",
      "c:\\Users\\zahra\\anaconda3\\Lib\\site-packages\\xgboost\\core.py:158: UserWarning: [19:41:58] WARNING: C:\\buildkite-agent\\builds\\buildkite-windows-cpu-autoscaling-group-i-0c55ff5f71b100e98-1\\xgboost\\xgboost-ci-windows\\src\\learner.cc:740: \n",
      "Parameters: { \"use_label_encoder\" } are not used.\n",
      "\n",
      "  warnings.warn(smsg, UserWarning)\n",
      "c:\\Users\\zahra\\anaconda3\\Lib\\site-packages\\xgboost\\core.py:158: UserWarning: [19:41:59] WARNING: C:\\buildkite-agent\\builds\\buildkite-windows-cpu-autoscaling-group-i-0c55ff5f71b100e98-1\\xgboost\\xgboost-ci-windows\\src\\learner.cc:740: \n",
      "Parameters: { \"use_label_encoder\" } are not used.\n",
      "\n",
      "  warnings.warn(smsg, UserWarning)\n",
      "c:\\Users\\zahra\\anaconda3\\Lib\\site-packages\\xgboost\\core.py:158: UserWarning: [19:42:00] WARNING: C:\\buildkite-agent\\builds\\buildkite-windows-cpu-autoscaling-group-i-0c55ff5f71b100e98-1\\xgboost\\xgboost-ci-windows\\src\\learner.cc:740: \n",
      "Parameters: { \"use_label_encoder\" } are not used.\n",
      "\n",
      "  warnings.warn(smsg, UserWarning)\n",
      "c:\\Users\\zahra\\anaconda3\\Lib\\site-packages\\xgboost\\core.py:158: UserWarning: [19:42:01] WARNING: C:\\buildkite-agent\\builds\\buildkite-windows-cpu-autoscaling-group-i-0c55ff5f71b100e98-1\\xgboost\\xgboost-ci-windows\\src\\learner.cc:740: \n",
      "Parameters: { \"use_label_encoder\" } are not used.\n",
      "\n",
      "  warnings.warn(smsg, UserWarning)\n",
      "c:\\Users\\zahra\\anaconda3\\Lib\\site-packages\\xgboost\\core.py:158: UserWarning: [19:42:02] WARNING: C:\\buildkite-agent\\builds\\buildkite-windows-cpu-autoscaling-group-i-0c55ff5f71b100e98-1\\xgboost\\xgboost-ci-windows\\src\\learner.cc:740: \n",
      "Parameters: { \"use_label_encoder\" } are not used.\n",
      "\n",
      "  warnings.warn(smsg, UserWarning)\n",
      "c:\\Users\\zahra\\anaconda3\\Lib\\site-packages\\xgboost\\core.py:158: UserWarning: [19:42:03] WARNING: C:\\buildkite-agent\\builds\\buildkite-windows-cpu-autoscaling-group-i-0c55ff5f71b100e98-1\\xgboost\\xgboost-ci-windows\\src\\learner.cc:740: \n",
      "Parameters: { \"use_label_encoder\" } are not used.\n",
      "\n",
      "  warnings.warn(smsg, UserWarning)\n",
      "c:\\Users\\zahra\\anaconda3\\Lib\\site-packages\\xgboost\\core.py:158: UserWarning: [19:42:04] WARNING: C:\\buildkite-agent\\builds\\buildkite-windows-cpu-autoscaling-group-i-0c55ff5f71b100e98-1\\xgboost\\xgboost-ci-windows\\src\\learner.cc:740: \n",
      "Parameters: { \"use_label_encoder\" } are not used.\n",
      "\n",
      "  warnings.warn(smsg, UserWarning)\n",
      "c:\\Users\\zahra\\anaconda3\\Lib\\site-packages\\xgboost\\core.py:158: UserWarning: [19:42:05] WARNING: C:\\buildkite-agent\\builds\\buildkite-windows-cpu-autoscaling-group-i-0c55ff5f71b100e98-1\\xgboost\\xgboost-ci-windows\\src\\learner.cc:740: \n",
      "Parameters: { \"use_label_encoder\" } are not used.\n",
      "\n",
      "  warnings.warn(smsg, UserWarning)\n",
      "c:\\Users\\zahra\\anaconda3\\Lib\\site-packages\\xgboost\\core.py:158: UserWarning: [19:42:06] WARNING: C:\\buildkite-agent\\builds\\buildkite-windows-cpu-autoscaling-group-i-0c55ff5f71b100e98-1\\xgboost\\xgboost-ci-windows\\src\\learner.cc:740: \n",
      "Parameters: { \"use_label_encoder\" } are not used.\n",
      "\n",
      "  warnings.warn(smsg, UserWarning)\n",
      "c:\\Users\\zahra\\anaconda3\\Lib\\site-packages\\xgboost\\core.py:158: UserWarning: [19:42:07] WARNING: C:\\buildkite-agent\\builds\\buildkite-windows-cpu-autoscaling-group-i-0c55ff5f71b100e98-1\\xgboost\\xgboost-ci-windows\\src\\learner.cc:740: \n",
      "Parameters: { \"use_label_encoder\" } are not used.\n",
      "\n",
      "  warnings.warn(smsg, UserWarning)\n",
      "c:\\Users\\zahra\\anaconda3\\Lib\\site-packages\\xgboost\\core.py:158: UserWarning: [19:42:08] WARNING: C:\\buildkite-agent\\builds\\buildkite-windows-cpu-autoscaling-group-i-0c55ff5f71b100e98-1\\xgboost\\xgboost-ci-windows\\src\\learner.cc:740: \n",
      "Parameters: { \"use_label_encoder\" } are not used.\n",
      "\n",
      "  warnings.warn(smsg, UserWarning)\n",
      "c:\\Users\\zahra\\anaconda3\\Lib\\site-packages\\xgboost\\core.py:158: UserWarning: [19:42:09] WARNING: C:\\buildkite-agent\\builds\\buildkite-windows-cpu-autoscaling-group-i-0c55ff5f71b100e98-1\\xgboost\\xgboost-ci-windows\\src\\learner.cc:740: \n",
      "Parameters: { \"use_label_encoder\" } are not used.\n",
      "\n",
      "  warnings.warn(smsg, UserWarning)\n",
      "c:\\Users\\zahra\\anaconda3\\Lib\\site-packages\\xgboost\\core.py:158: UserWarning: [19:42:10] WARNING: C:\\buildkite-agent\\builds\\buildkite-windows-cpu-autoscaling-group-i-0c55ff5f71b100e98-1\\xgboost\\xgboost-ci-windows\\src\\learner.cc:740: \n",
      "Parameters: { \"use_label_encoder\" } are not used.\n",
      "\n",
      "  warnings.warn(smsg, UserWarning)\n",
      "c:\\Users\\zahra\\anaconda3\\Lib\\site-packages\\xgboost\\core.py:158: UserWarning: [19:42:11] WARNING: C:\\buildkite-agent\\builds\\buildkite-windows-cpu-autoscaling-group-i-0c55ff5f71b100e98-1\\xgboost\\xgboost-ci-windows\\src\\learner.cc:740: \n",
      "Parameters: { \"use_label_encoder\" } are not used.\n",
      "\n",
      "  warnings.warn(smsg, UserWarning)\n",
      "c:\\Users\\zahra\\anaconda3\\Lib\\site-packages\\xgboost\\core.py:158: UserWarning: [19:42:12] WARNING: C:\\buildkite-agent\\builds\\buildkite-windows-cpu-autoscaling-group-i-0c55ff5f71b100e98-1\\xgboost\\xgboost-ci-windows\\src\\learner.cc:740: \n",
      "Parameters: { \"use_label_encoder\" } are not used.\n",
      "\n",
      "  warnings.warn(smsg, UserWarning)\n",
      "c:\\Users\\zahra\\anaconda3\\Lib\\site-packages\\xgboost\\core.py:158: UserWarning: [19:42:13] WARNING: C:\\buildkite-agent\\builds\\buildkite-windows-cpu-autoscaling-group-i-0c55ff5f71b100e98-1\\xgboost\\xgboost-ci-windows\\src\\learner.cc:740: \n",
      "Parameters: { \"use_label_encoder\" } are not used.\n",
      "\n",
      "  warnings.warn(smsg, UserWarning)\n",
      "c:\\Users\\zahra\\anaconda3\\Lib\\site-packages\\xgboost\\core.py:158: UserWarning: [19:42:14] WARNING: C:\\buildkite-agent\\builds\\buildkite-windows-cpu-autoscaling-group-i-0c55ff5f71b100e98-1\\xgboost\\xgboost-ci-windows\\src\\learner.cc:740: \n",
      "Parameters: { \"use_label_encoder\" } are not used.\n",
      "\n",
      "  warnings.warn(smsg, UserWarning)\n",
      "c:\\Users\\zahra\\anaconda3\\Lib\\site-packages\\xgboost\\core.py:158: UserWarning: [19:42:15] WARNING: C:\\buildkite-agent\\builds\\buildkite-windows-cpu-autoscaling-group-i-0c55ff5f71b100e98-1\\xgboost\\xgboost-ci-windows\\src\\learner.cc:740: \n",
      "Parameters: { \"use_label_encoder\" } are not used.\n",
      "\n",
      "  warnings.warn(smsg, UserWarning)\n",
      "c:\\Users\\zahra\\anaconda3\\Lib\\site-packages\\xgboost\\core.py:158: UserWarning: [19:42:16] WARNING: C:\\buildkite-agent\\builds\\buildkite-windows-cpu-autoscaling-group-i-0c55ff5f71b100e98-1\\xgboost\\xgboost-ci-windows\\src\\learner.cc:740: \n",
      "Parameters: { \"use_label_encoder\" } are not used.\n",
      "\n",
      "  warnings.warn(smsg, UserWarning)\n",
      "c:\\Users\\zahra\\anaconda3\\Lib\\site-packages\\xgboost\\core.py:158: UserWarning: [19:42:17] WARNING: C:\\buildkite-agent\\builds\\buildkite-windows-cpu-autoscaling-group-i-0c55ff5f71b100e98-1\\xgboost\\xgboost-ci-windows\\src\\learner.cc:740: \n",
      "Parameters: { \"use_label_encoder\" } are not used.\n",
      "\n",
      "  warnings.warn(smsg, UserWarning)\n"
     ]
    },
    {
     "data": {
      "image/png": "[encoded data, 46124 characters]",
      "text/plain": [
       "<Figure size 1000x600 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "import numpy as np\n",
    "from scipy import io\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.preprocessing import StandardScaler\n",
    "from sklearn.metrics import roc_auc_score\n",
    "from sklearn.impute import SimpleImputer\n",
    "from sklearn.utils import resample\n",
    "import xgboost as xgb\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "# Step 1: Load the data\n",
    "data_path = \"../data/\"\n",
    "suv_file = data_path + \"suv_percentilesSLOthenUWM.mat\"\n",
    "flags_file = data_path + \"flags_combined.mat\"\n",
    "\n",
    "# Load .mat files\n",
    "suv_dict = io.loadmat(suv_file)\n",
    "flags_dict = io.loadmat(flags_file)\n",
    "\n",
    "# Extract relevant data\n",
    "suv = suv_dict['lung_SUVperc_COMBINED'][0:58, :, :]\n",
    "flags = flags_dict['flags'][0:58, 3]\n",
    "\n",
    "# Step 2: Flatten and clean the data\n",
    "suv_flattened = suv.reshape(suv.shape[0], -1)\n",
    "\n",
    "# Impute missing values\n",
    "imputer = SimpleImputer(strategy='mean')\n",
    "suv_imputed = imputer.fit_transform(suv_flattened)\n",
    "\n",
    "# Standardize features\n",
    "scaler = StandardScaler()\n",
    "X_data = scaler.fit_transform(suv_imputed)\n",
    "\n",
    "# Prepare labels\n",
    "y = flags.astype(int).ravel()\n",
    "\n",
    "# Step 3: Split into train and test sets\n",
    "X_train, X_test, y_train, y_test = train_test_split(X_data, y, test_size=0.3, stratify=y, random_state=42)\n",
    "\n",
    "# Step 4: Define and train the XGBoost model\n",
    "model = xgb.XGBClassifier(\n",
    "    n_estimators=1000,\n",
    "    learning_rate=0.01,\n",
    "    max_depth=4,\n",
    "    subsample=0.8,\n",
    "    colsample_bytree=0.8,\n",
    "    reg_alpha=1,\n",
    "    reg_lambda=1,\n",
    "    use_label_encoder=False,\n",
    "    eval_metric='logloss',\n",
    "    random_state=42\n",
    ")\n",
    "model.fit(X_train, y_train)\n",
    "\n",
    "# Step 5: Evaluate AUC\n",
    "y_pred_prob = model.predict_proba(X_test)[:, 1]\n",
    "auc = roc_auc_score(y_test, y_pred_prob)\n",
    "print(f\"AUC (XGBoost): {auc:.4f}\")\n",
    "\n",
    "# Step 6: Bootstrap-based Confidence Intervals for predictions\n",
    "n_bootstraps = 100\n",
    "y_pred_distributions = []\n",
    "\n",
    "for i in range(n_bootstraps):\n",
    "    X_train_boot, y_train_boot = resample(X_train, y_train, random_state=i)\n",
    "    model_boot = xgb.XGBClassifier(\n",
    "        n_estimators=1000,\n",
    "        learning_rate=0.01,\n",
    "        max_depth=4,\n",
    "        subsample=0.8,\n",
    "        colsample_bytree=0.8,\n",
    "        reg_alpha=1,\n",
    "        reg_lambda=1,\n",
    "        use_label_encoder=False,\n",
    "        eval_metric='logloss',\n",
    "        random_state=42\n",
    "    )\n",
    "    model_boot.fit(X_train_boot, y_train_boot)\n",
    "    y_pred_boot = model_boot.predict_proba(X_test)[:, 1]\n",
    "    y_pred_distributions.append(y_pred_boot)\n",
    "\n",
    "# Convert to numpy array\n",
    "y_pred_distributions = np.array(y_pred_distributions)\n",
    "mean_preds = y_pred_distributions.mean(axis=0)\n",
    "lower_ci = np.percentile(y_pred_distributions, 2.5, axis=0)\n",
    "upper_ci = np.percentile(y_pred_distributions, 97.5, axis=0)\n",
    "\n",
    "# Step 7: Plot prediction intervals\n",
    "plt.figure(figsize=(10, 6))\n",
    "plt.errorbar(range(len(mean_preds)), mean_preds,\n",
    "             yerr=[mean_preds - lower_ci, upper_ci - mean_preds],\n",
    "             fmt='o', ecolor='gray', alpha=0.6, capsize=3)\n",
    "plt.title(\"XGBoost Predicted Probabilities with 95% Confidence Intervals\")\n",
    "plt.xlabel(\"Test Sample Index\")\n",
    "plt.ylabel(\"Predicted Probability (Adverse Event)\")\n",
    "plt.axhline(0.5, color='red', linestyle='--', alpha=0.7)\n",
    "plt.tight_layout()\n",
    "plt.show()\n"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "base",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.12.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
