{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "078900c6-d71d-44f0-9e76-cc5ce66f0ca9",
   "metadata": {},
   "source": [
    "# Logistic regression: exploring bootstrapping\n",
    "\n",
    "Exploring different bootstrapping idea:\n",
    "\n",
    "* non-parametric with minimal constraint (no empty group)\n",
    "* stratified non-parametric\n",
    "* parametric boostrapping \n",
    "* comparing distribution of boostrapped parameters with MLE asymptotic approx\n",
    "\n",
    "Ref:\n",
    "  * https://www.scirp.org/journal/paperinformation?paperid=70962"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "e18cec5e",
   "metadata": {},
   "source": [
    "## Common"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "3b351af7",
   "metadata": {},
   "outputs": [],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "import numpy as np\n",
    "import scipy\n",
    "\n",
    "import pandas as pd\n",
    "import seaborn as sns\n",
    "import os\n",
    "\n",
    "from sklearn import linear_model\n",
    "\n",
    "# our libs\n",
    "import data_utils\n",
    "import logit_utils\n",
    "\n",
    "# testing multivariate normal\n",
    "import pingouin as pg\n",
    "import mvn"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "e0de7f02-942b-42ff-874c-3c0e2ac1e0dc",
   "metadata": {},
   "source": [
    "## Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "3373f87f",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "results_path = \"./results\"\n",
    "if not os.path.exists(results_path): os.makedirs(results_path)\n",
    "\n",
    "data_path = \"../../data/\"\n",
    "suv_filename = os.path.join(data_path, \"suv_percentilesSLOthenUWM.mat\")\n",
    "flags_filename = os.path.join(data_path,\"flags_combined.mat\")\n",
    "normal_range_filename = os.path.join(data_path, \"normal_range.mat\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "811d990d",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "\"\"\"\n",
    "Loading all data\n",
    "\"\"\"\n",
    "suv_dict = scipy.io.loadmat(suv_filename)\n",
    "flags_dict = scipy.io.loadmat(flags_filename)\n",
    "normal_range_dict = scipy.io.loadmat(normal_range_filename)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "4a292c31",
   "metadata": {},
   "outputs": [],
   "source": [
    "# get data\n",
    "perc = 95\n",
    "organ = \"lung\"\n",
    "x, y = data_utils.get_data(organ, perc, suv_dict, flags_dict)\n",
    "\n",
    "# number of boostrapping samples\n",
    "n_boots = 10000"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "35e9bb7b-5a50-4cae-9af4-261c11453dd4",
   "metadata": {},
   "source": [
    "## Fit\n",
    "\n",
    "Ref: logistic regression documentation\n",
    "* https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
    "* https://stats.stackexchange.com/questions/186830/what-is-scikit-learns-logisticregression-minimizing"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "239ae789",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "pars = array([-11.68319035,   5.68708926])\n"
     ]
    }
   ],
   "source": [
    "# fits\n",
    "logr = linear_model.LogisticRegression(penalty = None)\n",
    "pars = logit_utils.logit_poly_fit(logr, x, y, degree = 1)\n",
    "print(f\"{pars = }\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "9cf6fec1",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "cov_pars = array([[12.31854544, -6.71118734],\n",
      "       [-6.71118734,  3.81820749]])\n"
     ]
    }
   ],
   "source": [
    "# asymptotic covariance matrix of parameters\n",
    "cov_pars = logit_utils.logit_poly_cov(x, pars)\n",
    "print(f\"{cov_pars = }\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "2359d3c1",
   "metadata": {},
   "outputs": [],
   "source": [
    "# defining twosided probabilities\n",
    "alpha = 0.05   # significance level\n",
    "probs = [alpha/2, 1 - alpha/2]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "4ba4d5d4",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "pars_CI = array([[-18.5622299 ,   1.85727375],\n",
      "       [ -4.80415081,   9.51690477]])\n"
     ]
    }
   ],
   "source": [
    "# CI of params (assuming asymptotic distr of parameters)\n",
    "pars_CI = logit_utils.logit_poly_pars_quantiles_normal(probs, pars, cov_pars)\n",
    "print(f\"{pars_CI = }\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "96acc3ab",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.microsoft.datawrangler.viewer.v0+json": {
       "columns": [
        {
         "name": "index",
         "rawType": "object",
         "type": "string"
        },
        {
         "name": "value",
         "rawType": "float64",
         "type": "float"
        },
        {
         "name": "LCL",
         "rawType": "float64",
         "type": "float"
        },
        {
         "name": "UCL",
         "rawType": "float64",
         "type": "float"
        }
       ],
       "ref": "bcdc415d-9a04-45c2-b0d7-baa2474bb441",
       "rows": [
        [
         "b0",
         "-11.683190353932442",
         "-18.562229897321515",
         "-4.804150810543373"
        ],
        [
         "b1",
         "5.687089258005262",
         "1.8572737464039784",
         "9.516904769606544"
        ]
       ],
       "shape": {
        "columns": 3,
        "rows": 2
       }
      },
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>value</th>\n",
       "      <th>LCL</th>\n",
       "      <th>UCL</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>b0</th>\n",
       "      <td>-11.683190</td>\n",
       "      <td>-18.562230</td>\n",
       "      <td>-4.804151</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>b1</th>\n",
       "      <td>5.687089</td>\n",
       "      <td>1.857274</td>\n",
       "      <td>9.516905</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "        value        LCL       UCL\n",
       "b0 -11.683190 -18.562230 -4.804151\n",
       "b1   5.687089   1.857274  9.516905"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# present table of results\n",
    "pd.DataFrame({\"value\": pars, \"LCL\": pars_CI[0], \"UCL\": pars_CI[1]}, index = [f\"b{i}\" for i in range(len(pars))])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "6ef6f46b",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.microsoft.datawrangler.viewer.v0+json": {
       "columns": [
        {
         "name": "index",
         "rawType": "int64",
         "type": "integer"
        },
        {
         "name": "LLF",
         "rawType": "float64",
         "type": "float"
        },
        {
         "name": "AIC",
         "rawType": "float64",
         "type": "float"
        },
        {
         "name": "BIC",
         "rawType": "float64",
         "type": "float"
        },
        {
         "name": "A",
         "rawType": "float64",
         "type": "float"
        },
        {
         "name": "chi2",
         "rawType": "float64",
         "type": "float"
        },
        {
         "name": "p-value(chi2)",
         "rawType": "float64",
         "type": "float"
        },
        {
         "name": "n",
         "rawType": "int64",
         "type": "integer"
        },
        {
         "name": "k",
         "rawType": "int64",
         "type": "integer"
        },
        {
         "name": "dof",
         "rawType": "int64",
         "type": "integer"
        }
       ],
       "ref": "4623adb0-db65-4fb1-8902-19b5937f3808",
       "rows": [
        [
         "0",
         "-7.394195827272766",
         "18.78839165454553",
         "22.90927767563837",
         "0.9655172413793104",
         "23.105937357608315",
         "0.9999705652971257",
         "58",
         "2",
         "56"
        ]
       ],
       "shape": {
        "columns": 9,
        "rows": 1
       }
      },
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>LLF</th>\n",
       "      <th>AIC</th>\n",
       "      <th>BIC</th>\n",
       "      <th>A</th>\n",
       "      <th>chi2</th>\n",
       "      <th>p-value(chi2)</th>\n",
       "      <th>n</th>\n",
       "      <th>k</th>\n",
       "      <th>dof</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>-7.394196</td>\n",
       "      <td>18.788392</td>\n",
       "      <td>22.909278</td>\n",
       "      <td>0.965517</td>\n",
       "      <td>23.105937</td>\n",
       "      <td>0.999971</td>\n",
       "      <td>58</td>\n",
       "      <td>2</td>\n",
       "      <td>56</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "        LLF        AIC        BIC         A       chi2  p-value(chi2)   n  k  \\\n",
       "0 -7.394196  18.788392  22.909278  0.965517  23.105937       0.999971  58  2   \n",
       "\n",
       "   dof  \n",
       "0   56  "
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# goodness of fit measures\n",
    "pd.DataFrame([logit_utils.logit_poly_goodness_of_fit(x, y, pars)])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "54f463d0",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "image/png": "[encoded data, 51452 characters]",
      "text/plain": [
       "<Figure size 600x500 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# plotting results\n",
    "fig, ax = plt.subplots(figsize = (6, 5))\n",
    "\n",
    "ax.set_title(\"cond. prob. for AE\")\n",
    "\n",
    "ax.set_xlabel(r\"$x = max_{visit} SUV(visit, p)$\")\n",
    "ax.set_ylabel(\"P(AE|X = x)\")\n",
    "\n",
    "# plotting data points\n",
    "for i, lab in enumerate([\"NC\", \"AE\"]): ax.scatter(x[y == i], y[y == i],label = lab)  \n",
    "\n",
    "# plotting fitted model \n",
    "xp = np.linspace(0, 6, 100)\n",
    "yp = logit_utils.logit_poly_model(xp, pars)\n",
    "ax.plot(xp, yp, label = \"logistic reg\")\n",
    "\n",
    "for lab, c in zip([\"normal\", \"delta\"], [\"red\", \"green\"]):\n",
    "    \n",
    "    # define quantile model\n",
    "    qm = eval(f\"logit_utils.logit_poly_model_quantiles_{lab}\")\n",
    "    \n",
    "    # get result of model quantiles\n",
    "    res = qm(xp, probs, pars, cov_pars)\n",
    "\n",
    "    # make plot of quantiles\n",
    "    ax.fill_between(xp, *res, color = c, alpha = 0.5, label = f\"CI:{lab}\")\n",
    "\n",
    "ax.legend(loc='center right')\n",
    "\n",
    "plt.show()\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "6a1a899c-ac8a-44e8-bad6-04262e429913",
   "metadata": {},
   "source": [
    "## Nonpar. bootstrapping"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "f10f2fe4",
   "metadata": {},
   "outputs": [],
   "source": [
    "\"\"\"\n",
    "    Plotting density plot of data1 and sub-selection in data2\n",
    "\n",
    "    Input:\n",
    "        data1 \n",
    "        data2 : zoomed data1\n",
    "\n",
    "        file1: fig filename for data1\n",
    "        file2: fig filename for data2\n",
    "\"\"\"\n",
    "def plot_pdf(data1, data2, file1, file2):\n",
    "    \n",
    "    sns.set_theme(rc={\"figure.figsize\": (6, 4), \"font.size\": 14})\n",
    "        \n",
    "    #\n",
    "    # FIRST\n",
    "    #\n",
    "\n",
    "    # plotting distribution of parameters\n",
    "    ax = sns.jointplot(x = data1[:,0], y = data1[:,1], kind ='kde', fill=True)\n",
    "    ax.set_axis_labels(xlabel= f\"$\\\\beta_0$\", ylabel=f\"$\\\\beta_1$\")\n",
    "\n",
    "    # adding label\n",
    "    plt.text(0.04, 0.9, \"(a)\", fontsize=16, horizontalalignment = 'center', \n",
    "             verticalalignment = 'center', transform=ax.figure.transFigure)\n",
    "\n",
    "    plt.savefig(file1, bbox_inches='tight')\n",
    "    plt.show()\n",
    "\n",
    "    #\n",
    "    # SECOND\n",
    "    #\n",
    "        \n",
    "    # plotting distribution of parameters\n",
    "    ax = sns.jointplot(x = data2[:,0], y = data2[:,1], kind ='kde', fill=True)\n",
    "    ax.set_axis_labels(xlabel = f\"$\\\\beta_0$\", ylabel = f\"$\\\\beta_1$\")\n",
    "\n",
    "    # MLE fit\n",
    "    plt.plot([pars[0]], [pars[1]], marker = \"o\", markersize = 10, \n",
    "             markeredgecolor = \"red\", markerfacecolor = \"red\")\n",
    "\n",
    "    # adding normal distribution curves on margins\n",
    "    x_vals = np.linspace(min(data2[:,0]), max(data2[:,0]), 100)\n",
    "    ax.ax_marg_x.plot(x_vals, scipy.stats.norm.pdf(x_vals, pars[0], np.sqrt(cov_pars[0][0])), \"r\" )\n",
    "\n",
    "    y_vals = np.linspace(min(data2[:,1]), max(data2[:,1]),100)\n",
    "    ax.ax_marg_y.plot(scipy.stats.norm.pdf(y_vals, pars[1], np.sqrt(cov_pars[1][1])), y_vals, \"r\" )\n",
    "\n",
    "    # adding label\n",
    "    plt.text(0.04, 0.9, \"(b)\", fontsize = 16, horizontalalignment = 'center',\n",
    "            verticalalignment = 'center', transform = ax.figure.transFigure)\n",
    "\n",
    "    plt.savefig(file2, bbox_inches = 'tight')\n",
    "    plt.show()\n",
    "    sns.reset_orig()\n",
    "\n",
    "\"\"\"\n",
    "    Calculate mean vector and sample covariance matrix\n",
    "\n",
    "    Input:\n",
    "        x: array nxm for floats\n",
    "    \n",
    "    Return:\n",
    "        (mean, cov)\n",
    "\"\"\"\n",
    "def mean_and_cov(x):\n",
    "    \n",
    "    mean_x = np.mean(x, axis = 0)\n",
    "    dx = x - mean_x\n",
    "    \n",
    "    cov_x = dx.T @ dx/(len(dx)-1)\n",
    "    \n",
    "    return mean_x, cov_x"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "2a860703",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "bpars_nonpar.shape = (10000, 2)\n",
      "size_selected: 7851\n"
     ]
    }
   ],
   "source": [
    "# performing bootstrapping\n",
    "m = n_boots\n",
    "bpars_nonpar = logit_utils.get_nonparam_boots_pars(logr, x, y, m)\n",
    "\n",
    "print(f\"{bpars_nonpar.shape = }\")\n",
    "\n",
    "# select a zoom windows of parameters with the largest peak in the distribution\n",
    "window =  np.array([[-50,0], [0, 20]])\n",
    "\n",
    "mask_sel = data_utils.within_bounds(bpars_nonpar, window)\n",
    "bpars_nonpar_sel = bpars_nonpar[mask_sel]\n",
    "\n",
    "print(\"size_selected:\", len(bpars_nonpar_sel))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "c99b4ec3",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "[encoded data, 32548 characters]",
      "text/plain": [
       "<Figure size 600x600 with 3 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "[encoded data, 40748 characters]",
      "text/plain": [
       "<Figure size 600x600 with 3 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "plot_pdf(bpars_nonpar, bpars_nonpar_sel, \n",
    "         os.path.join(results_path, \"logit_nonpar_boots.pdf\"), \n",
    "         os.path.join(results_path, \"logit_nonpar_boots_sel.pdf\"))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "4c2f7492-6caa-4f05-8a2f-a5a1eac28d30",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "== asymp beta ==\n",
      "mean:\n",
      "[-11.68319035   5.68708926]\n",
      "sample_cov:\n",
      "[[12.31854544 -6.71118734]\n",
      " [-6.71118734  3.81820749]]\n",
      "\n",
      "== betas in window ==\n",
      "mean:[-13.80656613   6.82934266]\n",
      "sample_cov:\n",
      "[[ 39.79769147 -24.21182354]\n",
      " [-24.21182354  15.36205015]]\n",
      "\n",
      "== all betas ==\n",
      "mean:\n",
      "[-31.80230595  17.59449002]\n",
      "sample_cov:\n",
      "[[ 1720.40403215 -1063.13618435]\n",
      " [-1063.13618435   660.43890903]]\n"
     ]
    }
   ],
   "source": [
    "print(\"== asymp beta ==\\nmean:\\n{}\\nsample_cov:\\n{}\\n\".format(pars, cov_pars))\n",
    "print(\"== betas in window ==\\nmean:{}\\nsample_cov:\\n{}\\n\".format(*mean_and_cov(bpars_nonpar_sel)))\n",
    "print(\"== all betas ==\\nmean:\\n{}\\nsample_cov:\\n{}\".format(*mean_and_cov(bpars_nonpar)))\n",
    "\n",
    "# Conclusion: due to multiple peaks the empirical values are very far from MLE"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "5160ac22-29e7-4c06-8b60-1214231fbc43",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "case 1:\n",
      "\tmvn.mardia:(4.8335732870755725, -1.2779303906191906, 0.30480406700810814, 0.20127397132431946)\n",
      "\tmvn.hz:(1.021869672905809, 0.11258362256748744)\n",
      "\tmvn.royston:(9.031869973807346, 0.060307801029024555)\n",
      "\tpg.hz:HZResults(hz=1.021869672905809, pval=0.11258362256748755, normal=True)\n",
      "case 2: HZResults(hz=8.064710154435106, pval=1.655033985375614e-19, normal=False)\n",
      "our case: HZResults(hz=302.3193258436826, pval=7.641311146456192e-113, normal=False)\n"
     ]
    }
   ],
   "source": [
    "# testing multivariate_normality\n",
    "# Ref: \n",
    "#  * https://www.statology.org/multivariate-normality-test-python/\n",
    "#  * https://pingouin-stats.org/build/html/generated/pingouin.multivariate_normality.html\n",
    "#  * https://www.sfu.ca/sasdoc/sashtml/ets/chap14/sect38.htm\n",
    "\n",
    "# random generate\n",
    "#rng = np.random.default_rng(2011)  # this does not work !!!!!\n",
    "rng = np.random.default_rng(2001)\n",
    "\n",
    "# test case: expecting p>> 0.05 value => True\n",
    "data1 = rng.multivariate_normal([1, 2], [[2, 0.25], [0.25, 2]], size = 1000)\n",
    "test1 = {\"mvn.mardia\": mvn.mardia_test(data1), \n",
    "         \"mvn.hz\": mvn.hz_test(data1),\n",
    "         \"mvn.royston\" : mvn.royston_test(data1),\n",
    "         \"pg.hz\": pg.multivariate_normality(data1, 0.05)}\n",
    "\n",
    "print(\"case 1:\")\n",
    "for key, val in test1.items():\n",
    "    print(f\"\\t{key}:{val}\")\n",
    "\n",
    "# test case: expecting p << 0.05 value,  => False\n",
    "data2 = rng.multivariate_normal([3, 5], [[1, 0.5], [0.5, 1]], size = 1000)\n",
    "test2 = pg.multivariate_normality(np.log(np.abs(data2)), alpha = 0.05)\n",
    "print(\"case 2:\", test2)\n",
    "\n",
    "# testing our parameters\n",
    "our_case = pg.multivariate_normality(bpars_nonpar[:1000], alpha = 0.05)\n",
    "print(\"our case:\", our_case)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "98ad2d95-d89b-4f18-a2f7-c7fc17473135",
   "metadata": {},
   "source": [
    "Conclusion: Statistically looking, the distribution of parameters is far from multivariate Gaussian for these sizes of samples, but is captures essential properties."
   ]
  },
  {
   "cell_type": "markdown",
   "id": "a8e54912",
   "metadata": {},
   "source": [
    "## Nonpar. stratified bootstrapped\n",
    "\n",
    "The number of points in each group is the same as in the original case."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "b0e015ea",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "bpars_nonpar_strat.shape= (10001, 2)\n"
     ]
    }
   ],
   "source": [
    "# performing stratified bootstrapping\n",
    "m = n_boots\n",
    "bpars_nonpar_strat = logit_utils.get_nonparam_stratified_boots_pars(logr, x, y, m)\n",
    "\n",
    "print(f\"{bpars_nonpar_strat.shape= }\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "c53978ca",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "size_selected: 7867\n"
     ]
    }
   ],
   "source": [
    "# select a zoom windows of parameters with the largest peak in the distribution\n",
    "window = np.array([[-100,0], [0, 20]])\n",
    "\n",
    "mask_sel = data_utils.within_bounds(bpars_nonpar_strat, window)\n",
    "bpars_nonpar_strat_sel = bpars_nonpar_strat[mask_sel]\n",
    "\n",
    "print(\"size_selected:\", len(bpars_nonpar_strat_sel))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "a53543fc",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "[encoded data, 32624 characters]",
      "text/plain": [
       "<Figure size 600x600 with 3 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "[encoded data, 41272 characters]",
      "text/plain": [
       "<Figure size 600x600 with 3 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "plot_pdf(bpars_nonpar_strat, bpars_nonpar_strat_sel, \n",
    "         os.path.join(results_path, \"logit_nonpar_boots_strat.pdf\"), \n",
    "         os.path.join(results_path, \"logit_nonpar_boots_strat_sel.pdf\"))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "298ff85e",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "== asymp beta ==\n",
      "mean:\n",
      "[-11.68319035   5.68708926]\n",
      "sample_cov:\n",
      "[[12.31854544 -6.71118734]\n",
      " [-6.71118734  3.81820749]]\n",
      "\n",
      "== betas in windows ==\n",
      "mean:[-13.70249663   6.88927173]\n",
      "sample_cov:\n",
      "[[ 38.66266221 -24.47770901]\n",
      " [-24.47770901  15.59124051]]\n",
      "\n",
      "== all betas ==\n",
      "mean:\n",
      "[-32.5778551   18.20748471]\n",
      "sample_cov:\n",
      "[[ 1876.4222611  -1161.35737777]\n",
      " [-1161.35737777   721.86322217]]\n"
     ]
    }
   ],
   "source": [
    "print(\"== asymp beta ==\\nmean:\\n{}\\nsample_cov:\\n{}\\n\".format(pars, cov_pars))\n",
    "print(\"== betas in windows ==\\nmean:{}\\nsample_cov:\\n{}\\n\".format(*mean_and_cov(bpars_nonpar_strat_sel)))\n",
    "print(\"== all betas ==\\nmean:\\n{}\\nsample_cov:\\n{}\".format(*mean_and_cov(bpars_nonpar_strat)))\n",
    "\n",
    "# Conclusion: due to multiple peaks the empirical values are very far from MLE"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "d1a90c82",
   "metadata": {},
   "source": [
    "## Parametric boostrapping"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "88415c37",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "bpars_param.shape= (10000, 2)\n"
     ]
    }
   ],
   "source": [
    "# performing stratified bootstrapping\n",
    "m = n_boots\n",
    "bpars_param = logit_utils.get_parametric_boots_pars(logr, x, y, m)\n",
    "\n",
    "print(f\"{bpars_param.shape= }\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "id": "c157ea73",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "size_selected: 7832\n"
     ]
    }
   ],
   "source": [
    "# select a zoom windows of parameters with the largest peak in the distribution\n",
    "window = np.array([[-100,0], [0, 20]])\n",
    "\n",
    "mask_sel = data_utils.within_bounds(bpars_param, window)\n",
    "bpars_param_sel = bpars_param[mask_sel]\n",
    "\n",
    "print(\"size_selected:\", len(bpars_param_sel))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "id": "74a623bf",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "[encoded data, 28992 characters]",
      "text/plain": [
       "<Figure size 600x600 with 3 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "[encoded data, 48500 characters]",
      "text/plain": [
       "<Figure size 600x600 with 3 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "plot_pdf(bpars_param, bpars_param_sel, \n",
    "         os.path.join(results_path, \"logit_param_boots.pdf\"), \n",
    "         os.path.join(results_path, \"logit_param_boots_sel.pdf\"))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "3e9780e4",
   "metadata": {},
   "source": [
    "## Compare model CI results"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "id": "924d6463",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/horvat/Documents/work/medfiz/git/uncertainty_study/python/logistic/logit_utils.py:34: RuntimeWarning: overflow encountered in exp\n",
      "  return 1/(1 + np.exp(-F))\n"
     ]
    },
    {
     "data": {
      "image/png": "[encoded data, 90952 characters]",
      "text/plain": [
       "<Figure size 800x500 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# plotting results\n",
    "fig, ax = plt.subplots(figsize = (8, 5))\n",
    "\n",
    "ax.set_title(\"cond. prob. for AE\")\n",
    "\n",
    "ax.set_xlabel(r\"$x = max_{visit} SUV(visit, p)$\")\n",
    "ax.set_ylabel(\"P(AE|X = x)\")\n",
    "\n",
    "# plotting data points\n",
    "for i, lab in enumerate([\"NC\", \"AE\"]): ax.scatter(x[y == i], y[y == i],label = lab)  \n",
    "\n",
    "# plotting fitted model \n",
    "xp = np.linspace(min(x), max(x), 100)\n",
    "yp = logit_utils.logit_poly_model(xp, pars)\n",
    "ax.plot(xp, yp, label = \"logistic reg\")\n",
    "\n",
    "# CI based on asymptotic MLE distribution\n",
    "for lab, c in zip([\"normal\", \"delta\"], \n",
    "                  [\"red\", \"green\"]):\n",
    "    \n",
    "    # define quantile model\n",
    "    qm = eval(f\"logit_utils.logit_poly_model_quantiles_{lab}\")\n",
    "    \n",
    "    # get result of model quantiles\n",
    "    res = qm(xp, probs, pars, cov_pars)\n",
    "\n",
    "    # make plot of quantiles\n",
    "    ax.fill_between(xp, *res, color = c, alpha = 0.2, label = f\"CI:{lab}\")\n",
    "\n",
    "# CI based on boostrapping\n",
    "for lab, bpars, c in zip([\"nonpar\", \"nonpar-strat\",\"param\"], \n",
    "                         [bpars_nonpar, bpars_nonpar_strat, bpars_param], \n",
    "                         [\"green\", \"orange\", \"purple\"] ):\n",
    "\n",
    "    # calculate model values of for all bootstrapped parameters\n",
    "    model_values = [logit_utils.logit_poly_model(xp, p) for p in bpars]\n",
    "    res = np.quantile(model_values, probs, axis = 0)\n",
    "\n",
    "    # make plot of quantiles\n",
    "    ax.fill_between(xp, *res, color = c, alpha = 0.2, label = f\"CI:{lab}\")\n",
    "\n",
    "\n",
    "ax.legend(loc='lower right')\n",
    "\n",
    "plt.savefig(os.path.join(results_path, \"logit_cmp_CI.pdf\"))\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ab46c5ab",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "base",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.12.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
