"
]
},
"metadata": {},
"output_type": "display_data"
}
],
"source": [
"# Contingency table \n",
"print(\"Counts F =\")\n",
"display(pd.DataFrame(rTZ[\"counts\"][\"F\"]))\n",
"\n",
"# Probabilities\n",
"print(\"Probabilities P(T,Z) =\")\n",
"prob = rTZ[\"probabilities\"]\n",
"display(pd.DataFrame(prob[\"P_TZ\"]))\n",
"\n",
"# Create the heatmap for 2x2 Contingency Table\n",
"plt.figure(figsize=(6, 5))\n",
"sns.heatmap(rTZ[\"counts\"][\"F\"], \n",
" annot=True, # Show the numbers in the cells\n",
" fmt='d', # Use integer formatting\n",
" cmap='YlGnBu', # Color scheme (Yellow-Green-Blue)\n",
" xticklabels= [\"0\", \"1\"], \n",
" yticklabels= [\"0\", \"1\"],)\n",
"\n",
"plt.xlabel('T')\n",
"plt.ylabel('Z')\n",
"plt.show()"
]
},
{
"cell_type": "code",
"execution_count": 8,
"id": "d9d2aa8c",
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"P(T) = [0.9137931 0.0862069]\n",
"P(Z) = [0.87931034 0.12068966]\n"
]
}
],
"source": [
"# Marginal probabilities \n",
"print(f\"P(T) = {prob['P_T']}\")\n",
"print(f\"P(Z) = {prob['P_Z']}\")"
]
},
{
"cell_type": "code",
"execution_count": 9,
"id": "42f7d99e",
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"P(T|Z)=\n"
]
},
{
"data": {
"text/html": [
"\n",
"\n",
"
\n",
" \n",
" \n",
" | \n",
" 0 | \n",
" 1 | \n",
"
\n",
" \n",
" \n",
" \n",
" | 0 | \n",
" 0.921569 | \n",
" 0.857143 | \n",
"
\n",
" \n",
" | 1 | \n",
" 0.078431 | \n",
" 0.142857 | \n",
"
\n",
" \n",
"
\n",
"
\n",
"\n",
"
\n",
" \n",
" \n",
" | \n",
" 0 | \n",
" 1 | \n",
"
\n",
" \n",
" \n",
" \n",
" | 0 | \n",
" 0.007776 | \n",
" -0.05665 | \n",
"
\n",
" \n",
" | 1 | \n",
" -0.007776 | \n",
" 0.05665 | \n",
"
\n",
" \n",
"
\n",
"
\n",
"\n",
"
\n",
" \n",
" \n",
" | \n",
" 0 | \n",
" 1 | \n",
"
\n",
" \n",
" \n",
" \n",
" | 0 | \n",
" 0.008509 | \n",
" -0.061995 | \n",
"
\n",
" \n",
" | 1 | \n",
" -0.090196 | \n",
" 0.657143 | \n",
"
\n",
" \n",
"
\n",
"
{\n",
"│ 'stat': 0.3243168965699486,\n",
"│ 'dof': 1,\n",
"│ 'pvalue': 0.569024824759762,\n",
"│ 'expected': array([[46.60344828, 6.39655172],\n",
"│ [ 4.39655172, 0.60344828]]),\n",
"│ 'reject_alpha': False\n",
"}\n",
"\n"
],
"text/plain": [
"\u001b[1m{\u001b[0m\n",
"\u001b[2;32m│ \u001b[0m\u001b[32m'stat'\u001b[0m: \u001b[1;36m0.3243168965699486\u001b[0m,\n",
"\u001b[2;32m│ \u001b[0m\u001b[32m'dof'\u001b[0m: \u001b[1;36m1\u001b[0m,\n",
"\u001b[2;32m│ \u001b[0m\u001b[32m'pvalue'\u001b[0m: \u001b[1;36m0.569024824759762\u001b[0m,\n",
"\u001b[2;32m│ \u001b[0m\u001b[32m'expected'\u001b[0m: \u001b[1;35marray\u001b[0m\u001b[1m(\u001b[0m\u001b[1m[\u001b[0m\u001b[1m[\u001b[0m\u001b[1;36m46.60344828\u001b[0m, \u001b[1;36m6.39655172\u001b[0m\u001b[1m]\u001b[0m,\n",
"\u001b[2;32m│ \u001b[0m\u001b[1m[\u001b[0m \u001b[1;36m4.39655172\u001b[0m, \u001b[1;36m0.60344828\u001b[0m\u001b[1m]\u001b[0m\u001b[1m]\u001b[0m\u001b[1m)\u001b[0m,\n",
"\u001b[2;32m│ \u001b[0m\u001b[32m'reject_alpha'\u001b[0m: \u001b[3;91mFalse\u001b[0m\n",
"\u001b[1m}\u001b[0m\n"
]
},
"metadata": {},
"output_type": "display_data"
}
],
"source": [
"# Pearson chi-square test\n",
"pprint(rTZ[\"global_tests\"][\"chi2\"])"
]
},
{
"cell_type": "code",
"execution_count": 12,
"id": "eca15f98",
"metadata": {},
"outputs": [
{
"data": {
"text/html": [
"{\n",
"│ 'odds_ratio': 1.9583333333333333,\n",
"│ 'pvalue': 0.48734165612568525,\n",
"│ 'reject_alpha': False\n",
"}\n",
"\n"
],
"text/plain": [
"\u001b[1m{\u001b[0m\n",
"\u001b[2;32m│ \u001b[0m\u001b[32m'odds_ratio'\u001b[0m: \u001b[1;36m1.9583333333333333\u001b[0m,\n",
"\u001b[2;32m│ \u001b[0m\u001b[32m'pvalue'\u001b[0m: \u001b[1;36m0.48734165612568525\u001b[0m,\n",
"\u001b[2;32m│ \u001b[0m\u001b[32m'reject_alpha'\u001b[0m: \u001b[3;91mFalse\u001b[0m\n",
"\u001b[1m}\u001b[0m\n"
]
},
"metadata": {},
"output_type": "display_data"
}
],
"source": [
"# Fishers exact test\n",
"pprint(rTZ[\"global_tests\"][\"fisher\"], expand_all=True)"
]
},
{
"cell_type": "markdown",
"id": "c1076374-3f45-47b6-961b-7d5f6897ff80",
"metadata": {},
"source": [
"### Conclusion"
]
},
{
"cell_type": "markdown",
"id": "089bf384",
"metadata": {},
"source": [
"- Both tests have p-values larger than $0.05$ (chi-square: $p = 0.569$, Fisher: $p = 0.487$), so `reject_alpha: False` means you **do not reject** the null hypothesis $H_0$ at significance level $\\alpha = 0.05$.\n",
"\n",
"- Here $H_0$ is **independence** of $T$ and $Z$, i.e. for $z \\in \\{0,1\\}$:\n",
" $$\n",
" P(T \\mid Z=z) = P(T).\n",
" $$\n",
" So your conclusion is: **you do not have statistically significant evidence that $T$ depends on $Z$** (you cannot claim $P(T\\mid Z)$ differs from $P(T)$ based on this sample).\n",
"\n",
"- Also, one expected count is very small (about $0.603$), so the chi-square approximation may be unreliable; **Fisher’s exact test** is the safer one here, and it is also not significant."
]
},
{
"cell_type": "markdown",
"id": "6b46ab90",
"metadata": {},
"source": [
"## Prentice Criteria (PC) evaluation framework\n",
"\n",
"It was by Google AI. In practice, these are tested using a nested series of regression models:\n",
"\n",
"1. **Model for PC 1 (Effect of $Z$):**\n",
"\n",
"$$\n",
" \\operatorname{logit}(P(T=1)) = \\mu_1 + \\beta Z\n",
"$$ \n",
"\n",
"*Requirement:* $\\beta \\neq 0$\n",
"\n",
"3. **Model for PC 2 (Effect of $Z$ on $S$):**\n",
"\n",
"$$\n",
" E[S] = \\mu_2 + \\alpha Z\n",
"$$ \n",
"\n",
"*Requirement:* $\\alpha \\neq 0$\n",
"\n",
"5. **Model for PC 3 (The Full Model):**\n",
"\n",
"$$\n",
" \\operatorname{logit}(P(T=1)) = \\mu_3 + \\gamma S + \\beta_S Z\n",
"$$\n",
"\n",
"*Requirement (Criterion 4):* $\\gamma \\neq 0$ and $\\beta_S \\to 0$\n",
"\n",
"**Note:** PC 4 is notoriously difficult to satisfy perfectly. Researchers often calculate the **Proportion Explained (PE)**:\n",
"$$\n",
" PE = \\frac{\\beta - \\beta_S}{\\beta}\n",
"$$"
]
},
{
"cell_type": "code",
"execution_count": 13,
"id": "2242fb0d",
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"--- Prentice Criteria Evaluation ---\n",
"\n",
"1. Treatment effect on T: p = 0.5752 (Fail)\n",
"2. Treatment effect on S: p = 0.9909 (Fail)\n",
"3. Surrogate effect on T: p = 0.0041 (Pass)\n",
"4. Full Mediation (Z effect given S): p = 0.6442\n",
" RESULT: Pass (Treatment effect is fully captured by Surrogate)\n",
"\n",
"Proportion of Treatment Effect Explained (PE): -53.90%\n"
]
}
],
"source": [
"import statsmodels.api as sm\n",
"import statsmodels.formula.api as smf\n",
"\n",
"df = df_work.copy()\n",
"\n",
"print(\"--- Prentice Criteria Evaluation ---\\n\")\n",
"\n",
"# Criterion 1: Treatment effect on True Endpoint (T ~ Z)\n",
"model1 = smf.logit(\"T ~ Z\", data=df).fit(disp=0)\n",
"p1 = model1.pvalues['Z']\n",
"print(f\"1. Treatment effect on T: p = {p1:.4f} ({'Pass' if p1 < 0.05 else 'Fail'})\")\n",
"\n",
"# Criterion 2: Treatment effect on Surrogate (S ~ Z)\n",
"model2 = smf.ols(\"S ~ Z\", data=df).fit()\n",
"p2 = model2.pvalues['Z']\n",
"print(f\"2. Treatment effect on S: p = {p2:.4f} ({'Pass' if p2 < 0.05 else 'Fail'})\")\n",
"\n",
"# Criterion 3: Surrogate effect on True Endpoint (T ~ S)\n",
"model3 = smf.logit(\"T ~ S\", data=df).fit(disp=0)\n",
"p3 = model3.pvalues['S']\n",
"print(f\"3. Surrogate effect on T: p = {p3:.4f} ({'Pass' if p3 < 0.05 else 'Fail'})\")\n",
"\n",
"# Criterion 4: Full Mediation (T ~ S + Z)\n",
"# The treatment effect (Z) should become non-significant when S is included\n",
"model4 = smf.logit(\"T ~ S + Z\", data=df).fit(disp=0)\n",
"p4_z = model4.pvalues['Z']\n",
"p4_s = model4.pvalues['S']\n",
"\n",
"print(f\"4. Full Mediation (Z effect given S): p = {p4_z:.4f}\")\n",
"if p4_z > 0.05 and p4_s < 0.05:\n",
" print(\" RESULT: Pass (Treatment effect is fully captured by Surrogate)\")\n",
"else:\n",
" print(\" RESULT: Fail (Significant direct treatment effect remains)\")\n",
"\n",
"# Calculate Proportion Explained (PE)\n",
"beta_unadj = model1.params['Z']\n",
"beta_adj = model4.params['Z']\n",
"pe = (beta_unadj - beta_adj) / beta_unadj\n",
"print(f\"\\nProportion of Treatment Effect Explained (PE): {pe:.2%}\")\n"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "30841dae",
"metadata": {},
"outputs": [],
"source": []
}
],
"metadata": {
"kernelspec": {
"display_name": "Python 3 (ipykernel)",
"language": "python",
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.10.12"
}
},
"nbformat": 4,
"nbformat_minor": 5
}