|
@@ -12,7 +12,43 @@ import utils.metrics as met
|
|
|
import itertools as it
|
|
|
import matplotlib.ticker as ticker
|
|
|
|
|
|
-RUN = True
|
|
|
+
|
|
|
+# Define plotting helper function
|
|
|
+def plot_coverage(
|
|
|
+ percentiles,
|
|
|
+ ensemble_results,
|
|
|
+ individual_results,
|
|
|
+ title,
|
|
|
+ x_lablel,
|
|
|
+ y_label,
|
|
|
+ save_path,
|
|
|
+ flip=False,
|
|
|
+):
|
|
|
+ fig, ax = plt.subplots()
|
|
|
+ plt.plot(
|
|
|
+ percentiles,
|
|
|
+ ensemble_results,
|
|
|
+ 'ob',
|
|
|
+ label='Ensemble',
|
|
|
+ )
|
|
|
+ plt.plot(
|
|
|
+ percentiles,
|
|
|
+ individual_results,
|
|
|
+ 'xr',
|
|
|
+ label='Individual (on entire dataset)',
|
|
|
+ )
|
|
|
+ plt.xlabel(x_lablel)
|
|
|
+ plt.ylabel(y_label)
|
|
|
+ plt.title(title)
|
|
|
+ plt.legend()
|
|
|
+ if flip:
|
|
|
+ plt.gca().invert_xaxis()
|
|
|
+ ax.xaxis.set_major_formatter(ticker.PercentFormatter(xmax=1.0))
|
|
|
+ plt.savefig(save_path)
|
|
|
+ plt.close()
|
|
|
+
|
|
|
+
|
|
|
+RUN = False
|
|
|
|
|
|
# CONFIGURATION
|
|
|
if os.getenv('ADL_CONFIG_PATH') is None:
|
|
@@ -109,7 +145,6 @@ def get_predictions(config):
|
|
|
predicted_class = np.argmax(mean)
|
|
|
|
|
|
# Get the confidence and standard deviation of the predicted class
|
|
|
- print(stdev)
|
|
|
pc_stdev = np.squeeze(stdev)[predicted_class]
|
|
|
# Get the individual classes
|
|
|
class_1 = mean[0][0]
|
|
@@ -156,6 +191,7 @@ else:
|
|
|
entropies_df = pd.read_csv(f'{V2_PATH}/ensemble_entropies.csv')
|
|
|
indv_df = pd.read_csv(f'{V2_PATH}/individual_results.csv')
|
|
|
|
|
|
+
|
|
|
# Plot confidence vs standard deviation, and change color of dots based on if they are correct
|
|
|
correct_conf = confs_df[confs_df['predicted_class'] == confs_df['true_label']]
|
|
|
incorrect_conf = confs_df[confs_df['predicted_class'] != confs_df['true_label']]
|
|
@@ -163,6 +199,13 @@ incorrect_conf = confs_df[confs_df['predicted_class'] != confs_df['true_label']]
|
|
|
correct_stdev = stdevs_df[stdevs_df['predicted_class'] == stdevs_df['true_label']]
|
|
|
incorrect_stdev = stdevs_df[stdevs_df['predicted_class'] != stdevs_df['true_label']]
|
|
|
|
|
|
+correct_ent = entropies_df[
|
|
|
+ entropies_df['predicted_class'] == entropies_df['true_label']
|
|
|
+]
|
|
|
+incorrect_ent = entropies_df[
|
|
|
+ entropies_df['predicted_class'] != entropies_df['true_label']
|
|
|
+]
|
|
|
+
|
|
|
plot, ax = plt.subplots()
|
|
|
plt.scatter(
|
|
|
correct_conf['confidence'],
|
|
@@ -184,8 +227,30 @@ plt.savefig(f'{V2_PATH}/confidence_vs_stdev.png')
|
|
|
|
|
|
plt.close()
|
|
|
|
|
|
+# Do the same for confidence vs entropy
|
|
|
+plot, ax = plt.subplots()
|
|
|
+plt.scatter(
|
|
|
+ correct_conf['confidence'],
|
|
|
+ correct_ent['entropy'],
|
|
|
+ color='green',
|
|
|
+ label='Correct Prediction',
|
|
|
+)
|
|
|
+plt.scatter(
|
|
|
+ incorrect_conf['confidence'],
|
|
|
+ incorrect_ent['entropy'],
|
|
|
+ color='red',
|
|
|
+ label='Incorrect Prediction',
|
|
|
+)
|
|
|
+plt.xlabel('Confidence (Raw Value)')
|
|
|
+plt.ylabel('Entropy (Raw Value)')
|
|
|
+plt.title('Confidence vs Entropy')
|
|
|
+plt.legend()
|
|
|
+plt.savefig(f'{V2_PATH}/confidence_vs_entropy.png')
|
|
|
+
|
|
|
+plt.close()
|
|
|
+
|
|
|
|
|
|
-# Calculate individual model accuracy
|
|
|
+# Calculate individual model accuracy and entropy
|
|
|
# Determine predicted class
|
|
|
indv_df['predicted_class'] = indv_df[['class_1', 'class_2']].idxmax(axis=1)
|
|
|
indv_df['predicted_class'] = indv_df['predicted_class'].apply(
|
|
@@ -199,6 +264,9 @@ f1_indv = met.F1(
|
|
|
auc_indv = metrics.roc_auc_score(
|
|
|
indv_df['true_label'].to_numpy(), indv_df['class_2'].to_numpy()
|
|
|
)
|
|
|
+indv_df['entropy'] = -1 * indv_df[['class_1', 'class_2']].apply(
|
|
|
+ lambda x: x * np.log(x), axis=0
|
|
|
+).sum(axis=1)
|
|
|
|
|
|
# Calculate percentiles for confidence and standard deviation
|
|
|
quantiles_conf = confs_df.quantile(np.linspace(0, 1, 11), interpolation='lower')[
|
|
@@ -208,6 +276,18 @@ quantiles_stdev = stdevs_df.quantile(np.linspace(0, 1, 11), interpolation='lower
|
|
|
'stdev'
|
|
|
]
|
|
|
|
|
|
+# Additionally for individual confidence
|
|
|
+quantiles_indv_conf = indv_df.quantile(np.linspace(0, 1, 11), interpolation='lower')[
|
|
|
+ 'class_2'
|
|
|
+]
|
|
|
+
|
|
|
+# For indivual entropy
|
|
|
+quantiles_indv_entropy = indv_df.quantile(np.linspace(0, 1, 11), interpolation='lower')[
|
|
|
+ 'entropy'
|
|
|
+]
|
|
|
+
|
|
|
+#
|
|
|
+
|
|
|
accuracies_conf = []
|
|
|
# Use the quantiles to calculate the coverage
|
|
|
iter_conf = it.islice(quantiles_conf.items(), 0, None)
|
|
@@ -224,40 +304,56 @@ for quantile in iter_conf:
|
|
|
|
|
|
accuracies_df = pd.DataFrame(accuracies_conf)
|
|
|
|
|
|
-# Plot the coverage
|
|
|
-fig, ax = plt.subplots()
|
|
|
-plt.plot(accuracies_df['percentile'], accuracies_df['accuracy'], 'ob', label='Ensemble')
|
|
|
-plt.plot(
|
|
|
+indv_conf = []
|
|
|
+# Use the quantiles to calculate the coverage
|
|
|
+iter_conf = it.islice(quantiles_indv_conf.items(), 0, None)
|
|
|
+for quantile in iter_conf:
|
|
|
+ percentile = quantile[0]
|
|
|
+
|
|
|
+ filt = indv_df[indv_df['class_2'] >= quantile[1]]
|
|
|
+ accuracy = filt['correct'].mean()
|
|
|
+ f1 = met.F1(filt['predicted_class'].to_numpy(), filt['true_label'].to_numpy())
|
|
|
+
|
|
|
+ indv_conf.append({'percentile': percentile, 'accuracy': accuracy, 'f1': f1})
|
|
|
+
|
|
|
+indv_conf_df = pd.DataFrame(indv_conf)
|
|
|
+
|
|
|
+# Do the same for entropy
|
|
|
+indv_entropy = []
|
|
|
+iter_entropy = it.islice(quantiles_indv_entropy.items(), 0, None)
|
|
|
+for quantile in iter_entropy:
|
|
|
+ percentile = quantile[0]
|
|
|
+
|
|
|
+ filt = indv_df[indv_df['entropy'] <= quantile[1]]
|
|
|
+ accuracy = filt['correct'].mean()
|
|
|
+ f1 = met.F1(filt['predicted_class'].to_numpy(), filt['true_label'].to_numpy())
|
|
|
+
|
|
|
+ indv_entropy.append({'percentile': percentile, 'accuracy': accuracy, 'f1': f1})
|
|
|
+
|
|
|
+indv_entropy_df = pd.DataFrame(indv_entropy)
|
|
|
+
|
|
|
+
|
|
|
+# Plot the coverage for confidence and accuracy
|
|
|
+plot_coverage(
|
|
|
accuracies_df['percentile'],
|
|
|
- [accuracy_indv] * len(accuracies_df['percentile']),
|
|
|
- 'xr',
|
|
|
- label='Individual (on entire dataset)',
|
|
|
+ accuracies_df['accuracy'],
|
|
|
+ indv_conf_df['accuracy'],
|
|
|
+ 'Confidence Accuracy Coverage Plot',
|
|
|
+ 'Minimum Confidence Percentile (Low to High)',
|
|
|
+ 'Accuracy',
|
|
|
+ f'{V2_PATH}/coverage_conf.png',
|
|
|
)
|
|
|
-plt.xlabel('Minimum Confidence Percentile (Low to High)')
|
|
|
-plt.ylabel('Accuracy')
|
|
|
-plt.title('Confidence Accuracy Coverage Plot')
|
|
|
-plt.legend()
|
|
|
-ax.xaxis.set_major_formatter(ticker.PercentFormatter(xmax=1.0))
|
|
|
-plt.savefig(f'{V2_PATH}/coverage_conf.png')
|
|
|
-plt.close()
|
|
|
|
|
|
-# Plot coverage vs F1 for confidence
|
|
|
-fig, ax = plt.subplots()
|
|
|
-plt.plot(accuracies_df['percentile'], accuracies_df['f1'], 'ob', label='Ensemble')
|
|
|
-plt.plot(
|
|
|
+# Plot the coverage for confidence and F1
|
|
|
+plot_coverage(
|
|
|
accuracies_df['percentile'],
|
|
|
- [f1_indv] * len(accuracies_df['percentile']),
|
|
|
- 'xr',
|
|
|
- label='Individual (on entire dataset)',
|
|
|
+ accuracies_df['f1'],
|
|
|
+ indv_conf_df['f1'],
|
|
|
+ 'Confidence F1 Coverage Plot',
|
|
|
+ 'Minimum Confidence Percentile (Low to High)',
|
|
|
+ 'F1',
|
|
|
+ f'{V2_PATH}/f1_coverage_conf.png',
|
|
|
)
|
|
|
-plt.xlabel('Minimum Confidence Percentile (Low to High)')
|
|
|
-plt.ylabel('F1')
|
|
|
-plt.title('Confidence F1 Coverage Plot')
|
|
|
-plt.legend()
|
|
|
-ax.xaxis.set_major_formatter(ticker.PercentFormatter(xmax=1.0))
|
|
|
-plt.savefig(f'{V2_PATH}/coverage_f1_conf.png')
|
|
|
-plt.close()
|
|
|
-
|
|
|
|
|
|
# Repeat for standard deviation
|
|
|
accuracies_stdev = []
|
|
@@ -275,7 +371,6 @@ for quantile in iter_stdev:
|
|
|
|
|
|
accuracies_stdev_df = pd.DataFrame(accuracies_stdev)
|
|
|
|
|
|
-# Plot the coverage
|
|
|
fig, ax = plt.subplots()
|
|
|
plt.plot(
|
|
|
accuracies_stdev_df['percentile'],
|
|
@@ -369,50 +464,26 @@ for quantile in iter_entropy:
|
|
|
|
|
|
accuracies_entropy_df = pd.DataFrame(accuracies_entropy)
|
|
|
|
|
|
-# Plot the coverage
|
|
|
-fig, ax = plt.subplots()
|
|
|
-plt.plot(
|
|
|
+
|
|
|
+# Plot the coverage for entropy and accuracy
|
|
|
+plot_coverage(
|
|
|
accuracies_entropy_df['percentile'],
|
|
|
accuracies_entropy_df['accuracy'],
|
|
|
- 'ob',
|
|
|
- label='Ensemble',
|
|
|
+ indv_entropy_df['accuracy'],
|
|
|
+ 'Entropy Accuracy Coverage Plot',
|
|
|
+ 'Minimum Entropy Percentile (Low to High)',
|
|
|
+ 'Accuracy',
|
|
|
+ f'{V2_PATH}/coverage_entropy.png',
|
|
|
)
|
|
|
-plt.plot(
|
|
|
- accuracies_entropy_df['percentile'],
|
|
|
- [accuracy_indv] * len(accuracies_entropy_df['percentile']),
|
|
|
- 'xr',
|
|
|
- label='Individual (on entire dataset)',
|
|
|
-)
|
|
|
-plt.xlabel('Maximum Entropy Percentile (High to Low)')
|
|
|
-plt.ylabel('Accuracy')
|
|
|
-plt.title('Entropy Accuracy Coverage Plot')
|
|
|
-plt.legend()
|
|
|
-plt.gca().invert_xaxis()
|
|
|
-ax.xaxis.set_major_formatter(ticker.PercentFormatter(xmax=1.0))
|
|
|
-plt.savefig(f'{V2_PATH}/coverage_entropy.png')
|
|
|
-plt.close()
|
|
|
|
|
|
-# Plot coverage vs F1 for entropy
|
|
|
-fig, ax = plt.subplots()
|
|
|
-plt.plot(
|
|
|
+# Plot the coverage for entropy and F1
|
|
|
+plot_coverage(
|
|
|
accuracies_entropy_df['percentile'],
|
|
|
accuracies_entropy_df['f1'],
|
|
|
- 'ob',
|
|
|
- label='Ensemble',
|
|
|
+ indv_entropy_df['f1'],
|
|
|
+ 'Entropy F1 Coverage Plot',
|
|
|
+ 'Maximum Entropy Percentile (High to Low)',
|
|
|
+ 'F1',
|
|
|
+ f'{V2_PATH}/f1_coverage_entropy.png',
|
|
|
+ flip=True,
|
|
|
)
|
|
|
-plt.plot(
|
|
|
- accuracies_entropy_df['percentile'],
|
|
|
- [f1_indv] * len(accuracies_entropy_df['percentile']),
|
|
|
- 'xr',
|
|
|
- label='Individual (on entire dataset)',
|
|
|
-)
|
|
|
-plt.xlabel('Maximum Entropy Percentile (High to Low)')
|
|
|
-plt.ylabel('F1')
|
|
|
-plt.title('Entropy F1 Coverage Plot')
|
|
|
-plt.legend()
|
|
|
-plt.gca().invert_xaxis()
|
|
|
-ax.xaxis.set_major_formatter(ticker.PercentFormatter(xmax=1.0))
|
|
|
-plt.savefig(f'{V2_PATH}/coverage_f1_entropy.png')
|
|
|
-
|
|
|
-plt.close()
|
|
|
-
|