clean CNN + k-fold.py implementation finished and working

cnn_net_data.csv

@@ -1,21 +0,0 @@
-,Epoch,Avg_loss,Time,Val_loss
-0,1.0,0.5493318528226279,63.193450927734375,0.6318140625953674
-1,2.0,0.5052056489060226,126.88481521606445,0.615243136882782
-2,3.0,0.5018373664143017,191.29927468299866,0.5647260546684265
-3,4.0,0.48469717265332785,256.35769629478455,0.7602677941322327
-4,5.0,0.49741162923933235,321.498055934906,0.5531934499740601
-5,6.0,0.4794707558687451,386.7943546772003,0.5435163974761963
-6,7.0,0.48868317627212376,452.08525347709656,0.6170985102653503
-7,8.0,0.4829676952755567,517.441656589508,0.5604787468910217
-8,9.0,0.5030312086771993,582.736382484436,0.7099340558052063
-9,10.0,0.4912947150110041,647.9085173606873,0.5861038565635681
-10,11.0,0.4996078980779185,712.9769625663757,0.6088337898254395
-11,12.0,0.4908207041545979,778.0235903263092,0.6034520268440247
-12,13.0,0.4944904490003308,843.1272113323212,0.7052374482154846
-13,14.0,0.4979538926221792,908.1915690898895,0.5868995785713196
-14,15.0,0.48077325710972535,973.2567093372345,0.6026476621627808
-15,16.0,0.495152342956043,1038.3403534889221,0.6178974509239197
-16,17.0,0.48088199594645825,1103.4229621887207,0.5096692442893982
-17,18.0,0.48285331656631914,1168.5112218856812,0.5497284531593323
-18,19.0,0.48006295116202347,1233.6090314388275,0.5811787247657776
-19,20.0,0.47963200437212455,1298.728411436081,0.6435700058937073

figures/cnn_net_data.csv

@@ -0,0 +1,21 @@
+,Epoch,Avg_loss,Time,Val_loss
+0,1,0.6564611077308655,6.120804786682129,0.6717021465301514
+0,2,0.5484287977218628,9.014472484588623,0.6452324390411377
+0,3,0.51495161652565,11.7736234664917,0.6303113698959351
+0,4,0.4871969699859619,14.532826662063599,0.6158974766731262
+0,5,0.4686302661895752,17.292651653289795,0.6022759675979614
+0,6,0.47578913569450376,20.05824065208435,0.5884113311767578
+0,7,0.4836656928062439,22.82177448272705,0.567466139793396
+0,8,0.5069188416004181,25.59386396408081,0.550148606300354
+0,9,0.5358751595020295,28.37803363800049,0.543449878692627
+0,10,0.4405584990978241,31.168226957321167,0.5247213840484619
+0,11,0.4511557579040527,33.95251703262329,0.5243819952011108
+0,12,0.4626104533672333,36.74565577507019,0.5283139944076538
+0,13,0.43378773927688596,39.53649187088013,0.5321415662765503
+0,14,0.42303248047828673,42.320810317993164,0.5408555269241333
+0,15,0.4470825731754303,45.113035917282104,0.5399429202079773
+0,16,0.48105289340019225,47.912266969680786,0.5447285175323486
+0,17,0.48474061489105225,50.71782732009888,0.524436354637146
+0,18,0.43532639741897583,53.5210497379303,0.5256407260894775
+0,19,0.5061911702156067,56.32426357269287,0.5219452381134033
+0,20,0.43608837127685546,59.14072513580322,0.5256292819976807

main.py

@@ -6,13 +6,17 @@ from utils.train_methods import train, load, evaluate, predict
 from utils.CNN import CNN_Net
 from torch.utils.data import DataLoader
 from torchvision import datasets
+from sklearn.model_selection import KFold
 
 # GENERAL PURPOSE
 import pandas as pd
 import numpy as np
 import matplotlib.pyplot as plt
 import platform
+import time
+current_time = time.localtime()
 
+print(time.strftime("%Y-%m-%d_%H:%M", current_time))
 print("--- RUNNING ---")
 print("Pytorch Version: " + torch. __version__)
 print("Python Version: " + platform.python_version())
@@ -34,7 +38,9 @@ properties = {
 
 model_filepath = '/data/data_wnx1/rschuurs/Pytorch_CNN-RNN'
 CNN_filepath = '/data/data_wnx1/rschuurs/Pytorch_CNN-RNN/cnn_net.pth'       # cnn_net.pth
+# small dataset
 # mri_datapath = '/data/data_wnx1/rschuurs/Pytorch_CNN-RNN/PET_volumes_customtemplate_float32/'   # Small Test
+# big dataset
 mri_datapath = '/data/data_wnx1/_Data/AlzheimersDL/CNN+RNN-2class-1cnn+data/PET_volumes_customtemplate_float32/'   # Real data
 annotations_datapath = './data/data_wnx1/rschuurs/Pytorch_CNN-RNN/LP_ADNIMERGE.csv'
 
@@ -74,7 +80,7 @@ CNN = CNN_Net(prps=properties, final_layer_size=2)
 CNN.cuda()
 
 train(CNN, train_dataloader, test_dataloader, CNN_filepath, epochs, graphs=True)
-load(CNN, CNN_filepath)
+# load(CNN, CNN_filepath)
 evaluate(CNN, val_dataloader)
 predict(CNN, val_dataloader)
 

original_model/innvestigate/analyzer/base.py

@@ -546,7 +546,7 @@ class ReverseAnalyzerBase(AnalyzerNetworkBase):
       reverse mapping.
     * :func:`_head_mapping` defines how the outputs of the model
       should be instantiated before the are passed to the reversed
-      network.
+      model.
 
     Furthermore other parameters of the function
     :func:`innvestigate.utils.keras.graph.reverse_model` can
@@ -555,14 +555,14 @@ class ReverseAnalyzerBase(AnalyzerNetworkBase):
 
     :param reverse_verbose: Print information on the reverse process.
     :param reverse_clip_values: Clip the values that are passed along
-      the reverted network. Expects tuple (min, max).
+      the reverted model. Expects tuple (min, max).
     :param reverse_project_bottleneck_layers: Project the value range
-      of bottleneck tensors in the reverse network into another range.
+      of bottleneck tensors in the reverse model into another range.
     :param reverse_check_min_max_values: Print the min/max values
-      observed in each tensor along the reverse network whenever
+      observed in each tensor along the reverse model whenever
       :func:`analyze` is called.
     :param reverse_check_finite: Check if values passed along the
-      reverse network are finite.
+      reverse model are finite.
     :param reverse_keep_tensors: Keeps the tensors created in the
       backward pass and stores them in the attribute
       :attr:`_reversed_tensors`.
@@ -677,7 +677,7 @@ class ReverseAnalyzerBase(AnalyzerNetworkBase):
     def _head_mapping(self, X):
         """
         Map output tensors to new values before passing
-        them into the reverted network.
+        them into the reverted model.
         """
         return X
 

original_model/innvestigate/analyzer/gradient_based.py

@@ -93,7 +93,7 @@ class Gradient(base.ReverseAnalyzerBase):
     """Gradient analyzer.
 
     Returns as analysis the function value with respect to the input.
-    The gradient is computed via the librarie's network reverting.
+    The gradient is computed via the librarie's model reverting.
 
     :param model: A Keras model.
     """

original_model/innvestigate/applications/imagenet.py

@@ -131,7 +131,7 @@ def _prepare_keras_net(netname,
         try:
             pattern_info = _get_patterns_info(netname, load_patterns)
         except KeyError:
-            warnings.warn("There are no patterns for network '%s'." % netname)
+            warnings.warn("There are no patterns for model '%s'." % netname)
         else:
             patterns_path = keras.utils.data_utils.get_file(
                 pattern_info["file_name"],

original_model/innvestigate/tools/pattern.py

@@ -411,7 +411,7 @@ class PatternComputer(object):
             # This does not work with containers!
             # They should be replaced by trace_model_execution.
             if kchecks.is_network(layer):
-                raise Exception("Network in network is not suppored!")
+                raise Exception("Network in model is not suppored!")
             for pattern_type, clazz in six.iteritems(self.pattern_types):
                 pinstance = clazz(self.model, layer,
                                   model_tensors=model_tensors,

original_model/innvestigate/utils/keras/checks.py

@@ -259,7 +259,7 @@ def only_relu_activation(layer):
 
 def is_network(layer):
     """
-    Is network in network?
+    Is model in model?
     """
     return isinstance(layer, keras.engine.topology.Network)
 
@@ -297,7 +297,7 @@ def is_dense_layer(layer, *args, **kwargs):
 
 
 def is_convnet_layer(layer):
-    """Checks if layer is from a convolutional network."""
+    """Checks if layer is from a convolutional model."""
     # Inside function to not break import if Keras changes.
     CONVNET_LAYERS = (
         keras.engine.topology.InputLayer,
@@ -367,7 +367,7 @@ def is_convnet_layer(layer):
 
 
 def is_relu_convnet_layer(layer):
-    """Checks if layer is from a convolutional network with ReLUs."""
+    """Checks if layer is from a convolutional model with ReLUs."""
     return (is_convnet_layer(layer) and only_relu_activation(layer))
 
 

original_model/innvestigate/utils/keras/graph.py

@@ -828,7 +828,7 @@ def print_model_execution_graph(graph):
 def get_bottleneck_nodes(inputs, outputs, execution_list):
     """
     Given an execution list this function returns all nodes that
-    are a bottleneck in the network, i.e., "all information" must pass
+    are a bottleneck in the model, i.e., "all information" must pass
     through this node.
     """
 
@@ -875,7 +875,7 @@ def get_bottleneck_nodes(inputs, outputs, execution_list):
 def get_bottleneck_tensors(inputs, outputs, execution_list):
     """
     Given an execution list this function returns all tensors that
-    are a bottleneck in the network, i.e., "all information" must pass
+    are a bottleneck in the model, i.e., "all information" must pass
     through this tensor.
     """
 
@@ -933,7 +933,7 @@ def reverse_model(model, reverse_mappings,
     :param default_reverse_mapping: A function that reverses layers for
       which no mapping was given by param "reverse_mappings".
     :param head_mapping: Map output tensors to new values before passing
-      them into the reverted network.
+      them into the reverted model.
     :param stop_mapping_at_tensors: Tensors at which to stop the mapping.
       Similar to stop_gradient parameters for gradient computation.
     :param verbose: Print what's going on.

original_model/innvestigate/utils/tests/dryrun.py

@@ -48,7 +48,7 @@ class BaseLayerTestCase(unittest.TestCase):
     """
     A dryrun test on various networks for an analyzing method.
 
-    For each network the test check that the generated network
+    For each model the test check that the generated model
     has the right output shape, can be compiled
     and executed with random inputs.
     """

original_model/utils_old/models.py

@@ -274,7 +274,7 @@ class RNN_Net ():
 
 def XAlex3D(w_regularizer = None, drop_rate = 0., final_layer_size = 50) : 
   
-    #3D Multi-modal deep learning neural network (refer to fig. 4 for chain graph of architecture)
+    #3D Multi-modal deep learning neural model (refer to fig. 4 for chain graph of architecture)
     ###Create the CNN architecture
     def f(mri_volume, mri_volume_jacobian, clinical_inputs):
     
@@ -322,11 +322,11 @@ def XAlex3D(w_regularizer = None, drop_rate = 0., final_layer_size = 50) :
         #                     strides = (1,1,1), kernel_initializer="he_normal",
         #                     padding="same", kernel_regularizer = w_regularizer)(conv6_concat)
     
-        #Flatten 3D conv network representations
+        #Flatten 3D conv model representations
         flat_conv_6 = Reshape((np.prod(K.int_shape(conv6_concat)[1:]),))(conv6_concat)
         print(flat_conv_6.shape)
 
-        #2-layer Dense network for clinical features
+        #2-layer Dense model for clinical features
         vol_fc1 = _fc_bn_relu_drop(64,  w_regularizer = w_regularizer,
                                drop_rate = drop_rate)(clinical_inputs)
 

utils/CNN.py

@@ -1,16 +1,7 @@
 from torch import device, cuda
-import torch
-from torch import add
 import torch.nn as nn
 import utils.CNN_Layers as CustomLayers
-import torch.nn.functional as F
-import torch.optim as optim
-import pandas as pd
-import matplotlib.pyplot as plt
-import time
-import numpy as np
-from sklearn.metrics import roc_curve, auc, confusion_matrix, classification_report
-import seaborn as sns
+
 
 class CNN_Net(nn.Module):
     def __init__(self, prps, final_layer_size=5):

utils/K-fold.py

@@ -1,13 +1,16 @@
 import os
 import torch
+from utils.train_methods import train, evaluate
 from utils.CNN import CNN_Net
 from torch import nn
 from torch.utils.data import DataLoader, ConcatDataset
 from torchvision import transforms
 from sklearn.model_selection import KFold, StratifiedKFold
+from sklearn.metrics import roc_curve, auc, confusion_matrix, classification_report
 from utils.preprocess import prepare_datasets, prepare_predict
 import numpy as np
 import matplotlib.pyplot as plt
+import time
 
 
 def reset_weights(m):
@@ -20,12 +23,20 @@ def reset_weights(m):
             print(f'Reset trainable parameters of layer = {layer}')
             layer.reset_parameters()
 
+
 if __name__ == '__main__':
 
-    # Might have to replace datapaths or separate between training and testing
+    print("--- RUNNING K-FOLD ---")
+    print("Pytorch Version: " + torch.__version__)
+    current_time = time.localtime()
+    print(time.strftime("%Y-%m-%d_%H:%M", current_time))
+
+    # might have to replace datapaths or separate between training and testing
     model_filepath = '/data/data_wnx1/rschuurs/Pytorch_CNN-RNN'
     CNN_filepath = '/data/data_wnx1/rschuurs/Pytorch_CNN-RNN/cnn_net.pth'  # cnn_net.pth
+    # small dataset
     # mri_datapath = '/data/data_wnx1/rschuurs/Pytorch_CNN-RNN/PET_volumes_customtemplate_float32/'   # Small Test
+    # big dataset
     mri_datapath = '/data/data_wnx1/_Data/AlzheimersDL/CNN+RNN-2class-1cnn+data/PET_volumes_customtemplate_float32/'  # Real data
     annotations_datapath = './data/data_wnx1/rschuurs/Pytorch_CNN-RNN/LP_ADNIMERGE.csv'
 
@@ -40,15 +51,15 @@ if __name__ == '__main__':
     }
 
     # Configuration options
-    k_folds = 5     # TODO
-    num_epochs = 1
+    k_folds = 5
+    num_epochs = 10
     loss_function = nn.CrossEntropyLoss()
 
     # For fold results
     results = {}
 
     # Set fixed random number seed
-    torch.manual_seed(42)
+    torch.manual_seed(42) # todo
 
     training_data, val_data, test_data = prepare_datasets(mri_datapath, val_split=0.2, seed=12)
 
@@ -79,49 +90,13 @@ if __name__ == '__main__':
             dataset,
             batch_size=10, sampler=test_subsampler)
 
-        # Init the neural network
-        network = CNN_Net(prps=properties, final_layer_size=2)
-        network.apply(reset_weights)
-
-        # Initialize optimizer
-        optimizer = torch.optim.Adam(network.parameters(), lr=1e-5)
+        # Init the neural model
+        model = CNN_Net(prps=properties, final_layer_size=2)
+        model.apply(reset_weights)
+        model.cuda()
 
         # Run the training loop for defined number of epochs
-        for epoch in range(0, num_epochs):
-
-            # Print epoch
-            print(f'Starting epoch {epoch + 1}')
-
-            # Set current loss value
-            current_loss = 0.0
-
-            # Iterate over the DataLoader for training data
-            for i, data in enumerate(trainloader, 0):
-
-                # Get inputs
-                inputs, targets = data
-
-                # Zero the gradients
-                optimizer.zero_grad()
-
-                # Perform forward pass
-                outputs = network(inputs)
-
-                # Compute loss
-                loss = loss_function(outputs, targets)
-
-                # Perform backward pass
-                loss.backward()
-
-                # Perform optimization
-                optimizer.step()
-
-                # Print statistics
-                current_loss += loss.item()
-                if i % 500 == 499:
-                    print('Loss after mini-batch %5d: %.3f' %
-                          (i + 1, current_loss / 500))
-                    current_loss = 0.0
+        train(model, trainloader, testloader, CNN_filepath, epochs=num_epochs, graphs=True)
 
         # Process is complete.
         print('Training process has finished. Saving trained model.')
@@ -131,81 +106,31 @@ if __name__ == '__main__':
 
         # Saving the model
         save_path = f'./model-fold-{fold}.pth'
-        torch.save(network.state_dict(), save_path)
+        torch.save(model.state_dict(), save_path)
 
         # Evaluation for this fold
-        correct, total = 0, 0
-        with torch.no_grad():
-
-            predictions = []
-            true_labels = []
-
-            # Iterate over the test data and generate predictions
-            for i, data in enumerate(testloader, 0):
-                # Get inputs
-                inputs, targets = data
+        results = evaluate(model, testloader, graphs=True, k_folds=k_folds, fold=fold, results=results)
 
-                # Generate outputs
-                outputs = network(inputs)
 
-                # Set total and correct
-                _, predicted = torch.max(outputs.data, 1)
-                total += targets.size(0)
-                correct += (predicted == targets).sum().item()
-
-                predictions.extend(outputs.data[:, 1].cpu().numpy())  # Grabs probability of positive
-                true_labels.extend(targets.cpu().numpy())
-
-            # Print accuracy
-            print('Accuracy for fold %d: %d %%' % (fold, 100.0 * correct / total))
-            print('--------------------------------')
-            results[fold] = 100.0 * (correct / total)
-
-
-        # MAKES ROC CURVE
-        thresholds = np.linspace(0, 1, num=50)
-        tpr = []
-        fpr = []
-        acc = []
-
-        true_labels = np.array(true_labels)
-
-        for threshold in thresholds:
-            # Thresholding the predictions (meaning all predictions above threshold are considered positive)
-            thresholded_predictions = (predictions >= threshold).astype(int)
-
-            # Calculating true positives, false positives, true negatives, false negatives
-            true_positives = np.sum((thresholded_predictions == 1) & (true_labels == 1))
-            false_positives = np.sum((thresholded_predictions == 1) & (true_labels == 0))
-            true_negatives = np.sum((thresholded_predictions == 0) & (true_labels == 0))
-            false_negatives = np.sum((thresholded_predictions == 0) & (true_labels == 1))
-
-            accuracy = (true_positives + true_negatives) / (
-                        true_positives + false_positives + true_negatives + false_negatives)
-
-            # Calculate TPR and FPR
-            tpr.append(true_positives / (true_positives + false_negatives))
-            fpr.append(false_positives / (false_positives + true_negatives))
-            acc.append(accuracy)
-
-        plt.plot(fpr, tpr, lw=2, label=f'ROC Fold {fold}')
-        plt.plot([0, 1], [0, 1], color='navy', lw=2, linestyle='--')
-        plt.xlim([0.0, 1.0])
-        plt.ylim([0.0, 1.0])
-
-        plt.xlabel('False Positive Rate (1 - Specificity)')
-        plt.ylabel('True Positive Rate (Sensitivity)')
-        plt.title('Receiver Operating Characteristic (ROC) Curve')
-        plt.legend(loc="lower right")
-
-    plt.savefig(f'./ROC_{k_folds}_Folds.png')
-    plt.show()
-
-    # Print fold results
-    print(f'K-FOLD CROSS VALIDATION RESULTS FOR {k_folds} FOLDS')
-    print('--------------------------------')
-    sum = 0.0
-    for key, value in results.items():
-        print(f'Fold {key}: {value} %')
-        sum += value
-    print(f'Average: {sum / len(results.items())} %')
+        # Print fold results
+        print(f'K-FOLD CROSS VALIDATION RESULTS FOR {k_folds} FOLDS')
+        print('--------------------------------')
+        sum = 0.0
+        for key, value in results.items():
+            print(f'Fold {key}: {value} %')
+            sum += value
+        print(f'Average: {sum / len(results.items())} %')
+
+        # Saves to .txt if last one
+        if(fold==k_folds-1):
+            time_string = time.strftime("%Y-%m-%d_%H:%M", current_time)
+            txt = open(f"{k_folds}_folds_{time_string}.txt", "w")
+            txt.write('--------------------------------')
+            txt.write(f'K-FOLD CROSS VALIDATION RESULTS FOR {k_folds} FOLDS')
+            txt.write('--------------------------------')
+            sum = 0.0
+            for key, value in results.items():
+                txt.write(f'Fold {key}: {value} %')
+                sum += value
+            txt.write(f'Average: {sum / len(results.items())} %')
+            txt.close()

utils/preprocess.py

@@ -8,7 +8,6 @@ import torchvision.transforms as transforms
 import re
 
 
-
 '''
 Prepares CustomDatasets for training, validating, and testing CNN
 '''

utils/train_methods.py

@@ -58,13 +58,15 @@ def train(model, train_data, test_data, CNN_filepath, epochs=20, graphs=True):
         print(f"Avg. loss: {avg_loss}")
 
         # loss on validation
-        val_loss = evaluate(test_data, graphs)
-
-        losses = losses.append({'Epoch':int(epoch), 'Avg_loss':avg_loss, 'Val_loss':val_loss, 'Time':time.time() - start_time}, ignore_index=True)
+        val_loss = evaluate(model, test_data, graphs=False)
 
+        losses = pd.concat([losses, pd.DataFrame([{'Epoch':int(epoch), 'Avg_loss':avg_loss, 'Val_loss':val_loss, 'Time':time.time() - start_time}])])
 
     print('Finished Training')
-    losses.to_csv('./cnn_net_data.csv')
+
+    start_time = time.localtime()
+    time_string = time.strftime("%Y-%m-%d_%H:%M", start_time)
+    losses.to_csv(f'./cnn_net_data_{time_string}.csv')
 
     if(graphs):
         # MAKES EPOCH VS AVG LOSS GRAPH
@@ -75,8 +77,8 @@ def train(model, train_data, test_data, CNN_filepath, epochs=20, graphs=True):
 
         # MAKES EPOCH VS VALIDATION LOSS GRAPH
         plt.plot(losses['Epoch'], losses['Val_loss'], label="Loss on Validation")
-        plt.savefig('./avgloss_epoch_curve.png')
-        plt.show()
+        plt.savefig(f"./avgloss_epoch_curve_{time_string}.png")
+        # plt.show()
 
         torch.save(model.state_dict(), CNN_filepath)
         print("Model saved")
@@ -87,14 +89,11 @@ def load(model, filepath):
     model.load_state_dict(torch.load(filepath))
 
 
-def evaluate(model, val_data, graphs=True):
-# EVALUATE MODEL
-    correct = 0
-    total = 0
+def evaluate(model, val_data, graphs=True, k_folds=None, fold=None, results=None):
+    start_time = time.localtime()  # seconds
 
-    predictionsLabels = []
-    predictionsProbabilities = []
-    true_labels = []
+    correct, total = 0, 0
+    predictionsLabels, predictionsProbabilities, true_labels = [], [], []
 
     criterion = nn.CrossEntropyLoss(reduction='mean')
     model.eval()
@@ -102,7 +101,7 @@ def evaluate(model, val_data, graphs=True):
     with torch.no_grad():
         for data in val_data:
             images, labels = data[0].to(model.device), data[1].to(model.device)
-            # calculate outputs by running images through the network
+            # calculate outputs by running images through the model
             outputs = model.forward(images)
             # the class with the highest energy is what we choose as prediction
 
@@ -119,10 +118,45 @@ def evaluate(model, val_data, graphs=True):
                 predictionsProbabilities.extend(outputs.data[:, 1].cpu().numpy())     # Grabs probability of positive
                 true_labels.extend(labels.cpu().numpy())
 
+
+    # K-FOLD MODE
+    if(fold!=None):
+        # Print accuracy
+        print('Accuracy for fold %d: %d %%' % (fold, 100.0 * correct / total))
+        print('--------------------------------')
+        results[fold] = 100.0 * (correct / total)
+
+        true_labels = np.array(true_labels)
+
+        # ROC
+        # Calculate TPR and FPR
+        fpr, tpr, thresholds = roc_curve(true_labels, predictionsProbabilities)
+        time_string = time.strftime("%Y-%m-%d_%H:%M", start_time)
+
+        # Calculate AUC
+        roc_auc = auc(fpr, tpr)
+
+        plt.plot(fpr, tpr, lw=2, label=f'ROC Fold {fold} (AUC: {roc_auc})')
+        plt.plot([0, 1], [0, 1], color='navy', lw=2, linestyle='--')
+        plt.xlim([0.0, 1.005])
+        plt.ylim([0.0, 1.005])
+
+        plt.xlabel('False Positive Rate (1 - Specificity)')
+        plt.ylabel('True Positive Rate (Sensitivity)')
+        plt.title('Receiver Operating Characteristic (ROC) Curve')
+        plt.legend(loc="lower right")
+
+        plt.savefig(f'./ROC_{k_folds}_Folds_{time_string}.png')
+
+        return results
+
+
+    # NORMAL EVALUATION
     print(f'Accuracy of the network on {total} scans: {100 * correct // total}%')
 
     if(not graphs): print(f'Validation loss: {loss.item()}')
     else:
+        time_string = time.strftime("%Y-%m-%d_%H:%M", start_time)
         # ROC
         # Calculate TPR and FPR
         fpr, tpr, thresholds = roc_curve(true_labels, predictionsProbabilities)
@@ -139,8 +173,8 @@ def evaluate(model, val_data, graphs=True):
         plt.ylabel('True Positive Rate (Sensitivity)')
         plt.title('Receiver Operating Characteristic (ROC) Curve')
         plt.legend(loc="lower right")
-        plt.savefig('./ROC.png')
-        plt.show()
+        plt.savefig(f'./ROC_{time_string}.png')
+        # plt.show()
 
 
         # Calculate confusion matrix
@@ -152,8 +186,8 @@ def evaluate(model, val_data, graphs=True):
         plt.xlabel('Predicted labels')
         plt.ylabel('True labels')
         plt.title('Confusion Matrix')
-        plt.savefig('./confusion_matrix.png')
-        plt.show()
+        plt.savefig(f'./confusion_matrix_{time_string}.png')
+        # plt.show()
 
         # Classification Report
         report = classification_report(true_labels, predictionsLabels)