Code of replication study

Code of replication study in a "A Systematic Review and Replication study of Deep Learning Techniques for Tuberculosis Bacilli Detection From Sputum Smear Microscopic Images"

1. Load libraries, functions and settings

# ==============================================================================================================
# ==============================================================================================================
# Load libraries, functions and settings
# ==============================================================================================================
# ==============================================================================================================

print("\n* Loading libraries... ", end="")

import sys
import os  
import numpy as np
import time
import json
import psutil 
import cv2
import cpuinfo
import gc
import psutil
import pandas as pd
 
from PIL import Image, ImageDraw, ImageOps
from io import StringIO 
from datetime import datetime
from matplotlib import pyplot 
 
import tensorflow as tf
import tensorflow.keras.backend as tfback
from tensorflow.keras.models import Sequential
from tensorflow.keras.layers import Conv2D, MaxPooling2D, Flatten, Dense, Dropout, AveragePooling2D, Input
from tensorflow.keras.layers import BatchNormalization
from tensorflow.keras.preprocessing.image import ImageDataGenerator
from tensorflow.keras.preprocessing import image
from tensorflow.keras.utils import to_categorical 
from tensorflow.keras.optimizers import SGD  
from tensorflow.keras.layers import Activation   
from tensorflow.keras import activations 
from tensorflow.keras.utils import plot_model
from tensorflow.python.client import device_lib 
from tensorflow.keras.layers import Reshape   
  
from sklearn.metrics import confusion_matrix 
from sklearn.metrics import classification_report, confusion_matrix
from sklearn.metrics import plot_confusion_matrix  
from sklearn.metrics import roc_curve 
from sklearn.metrics import auc 
from sklearn.model_selection import train_test_split 
from sklearn.utils.multiclass import unique_labels  
from sklearn import metrics 

import matplotlib.pyplot as plt
  
tf.config.threading.set_inter_op_parallelism_threads(4)
tf.config.threading.set_intra_op_parallelism_threads(12) 
    
###------------------------------------------------
type_exec = "GPU"
#type_exec = "CPU"
### To use CPU only device 
if type_exec == "CPU":
    tf.config.set_visible_devices([], 'GPU') 
###------------------------------------------------
### To use GPU  
if type_exec == "GPU":
    from tensorflow.compat.v1 import ConfigProto
    from tensorflow.compat.v1 import InteractiveSession
    config = ConfigProto()
    config.gpu_options.per_process_gpu_memory_fraction = 0.8
    config.gpu_options.allow_growth = True
    session = InteractiveSession(config=config)  
    
    #https://medium.com/ibm-data-ai/memory-hygiene-with-tensorflow-during-model-training-and-deployment-for-inference-45cf49a15688
    gpus = tf.config.list_physical_devices('GPU')
    if gpus:
        for gpu in gpus:
            tf.config.experimental.set_memory_growth(gpu,True)
###------------------------------------------------
print("Ok")
# ==============================================================================================================
# ==============================================================================================================

* Loading libraries... Ok

1.1. Execution parameters and data

# ==============================================================================================================
# ==============================================================================================================
# Execution parameters and data
# ==============================================================================================================
# ==============================================================================================================
 
#os.chdir("/content/drive/My Drive/UFMG/data_tb/")

 
PATH_DATASET    = "D:/tuberculosis-phonecamera/dataset_1/data/"
#PATH_DATASET   = "D:/ZNSM-iDB/dataset_2/data/"
OUT_PATH        = "D:/data_tb/temps/"
BATCH_SIZE      = 25  
EPOCHS_SIZE     = 100 
TARGET_SIZE_IMG = 20 #20 #224 #148 #100

print("\n* Parameters")
print("  - PATH_DATASET:",PATH_DATASET)
print("  - OUT_PATH    :",OUT_PATH)
print("  - BATCH_SIZE  :",BATCH_SIZE)
print("  - EPOCHS_SIZE :",EPOCHS_SIZE)
print("  - RUN IN      :",type_exec)
print("  - SIZE_IMG    :",TARGET_SIZE_IMG)

date_time    = datetime.now().strftime("%d-%m-%Y_%H%M%S") 
FILE_ID_RUN  = "result-execution__"+date_time+"__"     
# ==============================================================================================================
# ==============================================================================================================

* Parameters
  - PATH_DATASET: D:/tuberculosis-phonecamera/dataset_1/data/
  - OUT_PATH    : D:/data_tb/temps/
  - BATCH_SIZE  : 25
  - EPOCHS_SIZE : 100
  - RUN IN      : GPU
  - SIZE_IMG    : 20

1.2. Execution parameters and data

# ==============================================================================================================
# ==============================================================================================================
# Recover system information
# ==============================================================================================================
# ==============================================================================================================
print("\n* System information")

print("  - sys.version:", sys.version) 
print("  - tf version:", tf.__version__)
print("  - tf keras version:", tf.keras.__version__)

print('  - CPU')
print("    -",cpuinfo.get_cpu_info()['brand_raw'])
print("    - Physical cores:", psutil.cpu_count(logical=False))
print("    - Total cores:", psutil.cpu_count(logical=True))
cpufreq = psutil.cpu_freq()
#print(f"   - Max Frequency: {cpufreq.max:.2f}Mhz")
#print(f"   - Min Frequency: {cpufreq.min:.2f}Mhz")

def _get_available_gpus(): 
    #global _LOCAL_DEVICES
    if tfback._LOCAL_DEVICES is None:
        devices = tf.config.list_logical_devices()
        tfback._LOCAL_DEVICES = [x.name for x in devices]
    return tfback._LOCAL_DEVICES 
 
tfback._get_available_gpus = _get_available_gpus

print("  - Devices:",device_lib.list_local_devices())
print()
 
device_name = tf.test.gpu_device_name()
#if device_name != '/device:GPU:0':
#  raise SystemError('GPU device not found')
print('  - Found GPU at: {}'.format(device_name))

print('  - tf threading')
print("    - inter_op_parallelism_threads:", tf.config.threading.get_inter_op_parallelism_threads() )
print("    - intra_op_parallelism_threads:", tf.config.threading.get_intra_op_parallelism_threads() )
# ==============================================================================================================
# ==============================================================================================================

* System information
  - sys.version: 3.8.5 (default, Sep  3 2020, 21:29:08) [MSC v.1916 64 bit (AMD64)]
  - tf version: 2.4.1
  - tf keras version: 2.4.0
  - CPU
    - AMD Ryzen 9 5900X 12-Core Processor
    - Physical cores: 12
    - Total cores: 24
  - Devices: [name: "/device:CPU:0"
device_type: "CPU"
memory_limit: 268435456
locality {
}
incarnation: 6959145621907510632
, name: "/device:GPU:0"
device_type: "GPU"
memory_limit: 10307397222
locality {
  bus_id: 1
  links {
  }
}
incarnation: 8684742116349477979
physical_device_desc: "device: 0, name: NVIDIA GeForce RTX 3060, pci bus id: 0000:06:00.0, compute capability: 8.6"
]

  - Found GPU at: /device:GPU:0
  - tf threading
    - inter_op_parallelism_threads: 4
    - intra_op_parallelism_threads: 12

1.3. Useful and diverse functions

# ==============================================================================================================
# ==============================================================================================================
# Useful and diverse functions
# ==============================================================================================================
# ==============================================================================================================
 
# -----------------------------------------------------------------------------
# plot diagnostic learning curves
def summarize_diagnostics(history, out_filename): 
  # plot loss
  pyplot.subplot(211)
  pyplot.title('Cross Entropy Loss')
  pyplot.plot(history.history['loss'], color='blue', label='train')
  pyplot.plot(history.history['val_loss'], color='orange', label='test')
  pyplot.suptitle('_', fontsize=16)
  # plot accuracy
  pyplot.subplot(212)
  pyplot.title('Classification Accuracy')
  pyplot.plot(history.history['accuracy'], color='blue', label='train')
  pyplot.plot(history.history['val_accuracy'], color='orange', label='test')
  pyplot.suptitle('_', fontsize=16)
  pyplot.subplots_adjust(left=0.125, bottom=0.1,  right=0.9,  top=0.9,  wspace=0.2,  hspace=0.35)
  # save plot to file 
  pyplot.savefig( out_filename)
  #pyplot.show()
  pyplot.close()
# -----------------------------------------------------------------------------

# -----------------------------------------------------------------------------
def my_imshow(img,scale_percent):  
    width = int(img.shape[1] * scale_percent / 100)
    height = int(img.shape[0] * scale_percent / 100) 
    dsize = (width, height) 
    frame2 = cv2.resize(img, dsize)

    #frame2 = cv2.resize(img, (800, 600)) 
    cv2.imshow('frame1',frame2) 
    k = cv2.waitKey(0) & 0xFF
    if k == 27:         # wait for ESC key to exit
        cv2.destroyAllWindows()
# -----------------------------------------------------------------------------

# -----------------------------------------------------------------------------        
def show(img, label):
    pyplot.figure()
    pyplot.imshow(img)
    pyplot.title(label)
    #plt.axis('off')
# -----------------------------------------------------------------------------

# ----------------------------------------------------------------------------- 
class MyCustomCallback(tf.keras.callbacks.Callback): 
    def on_epoch_end(self, epoch, logs=None):
        print("-----------------")
        tf.keras.backend.clear_session() 
        tf.compat.v1.reset_default_graph()
        tf.compat.v1.get_default_graph().finalize()
        gc.collect() 
        print("Memory Ram: ",round(psutil.virtual_memory().used/1024**3,2), " (Free: ",round(psutil.virtual_memory().available * 100 / psutil.virtual_memory().total,2),"%)",sep="")
        print("-----------------")
# -----------------------------------------------------------------------------

# -----------------------------------------------------------------------------
def plot_confusion_matrix(y_true, y_pred, classes,
                          normalize=False,
                          title=None,
                          cmap=pyplot.cm.Blues):
    """
    This function prints and plots the confusion matrix.
    Normalization can be applied by setting `normalize=True`.
    """
    if not title:
        if normalize:
            title = 'Normalized confusion matrix'
        else:
            title = 'Confusion matrix, without normalization'

    # Compute confusion matrix
    cm = confusion_matrix(y_true, y_pred)
    # Only use the labels that appear in the data
    classes = classes[unique_labels(y_true, y_pred)]
    if normalize:
        cm = cm.astype('float') / cm.sum(axis=1)[:, np.newaxis]
    #    print("Normalized confusion matrix")
    #else:
    #    print('Confusion matrix, without normalization')

    #print(cm)

    fig, ax = pyplot.subplots()
    im = ax.imshow(cm, interpolation='nearest', cmap=cmap)
    ax.figure.colorbar(im, ax=ax)
    # We want to show all ticks...
    ax.set(xticks=np.arange(cm.shape[1]),
           yticks=np.arange(cm.shape[0]),
           # ... and label them with the respective list entries
           xticklabels=classes, yticklabels=classes,
           title=title,
           ylabel='True label',
           xlabel='Predicted label')

    # Rotate the tick labels and set their alignment.
    pyplot.setp(ax.get_xticklabels(), rotation=45, ha="right", rotation_mode="anchor")

    # Loop over data dimensions and create text annotations.
    fmt = '.2f' if normalize else 'd'
    thresh = cm.max() / 2.
    for i in range(cm.shape[0]):
        for j in range(cm.shape[1]):
            ax.text(j, i, format(cm[i, j], fmt),
                    ha="center", va="center",
                    color="white" if cm[i, j] > thresh else "black")
    fig.tight_layout()
    #plt.xlim(-0.5, len(np.unique(y))-0.5)
    #plt.ylim(len(np.unique(y))-0.5, -0.5)
    return ax
# -----------------------------------------------------------------------------        
# ==============================================================================================================
# ==============================================================================================================

2. Load the data

# ==============================================================================================================
# ==============================================================================================================
# Load the data
# ==============================================================================================================
# ==============================================================================================================
print("\n* Loading dataset")


#create ImageDataGenerator
train_datagen       = ImageDataGenerator(rescale = 1./255)
test_datagen        = ImageDataGenerator(rescale = 1./255)
validation_datagen  = ImageDataGenerator(rescale = 1./255)


print("    - Reading files folders" )
train_datagen      = train_datagen.flow_from_directory(     PATH_DATASET+'/train/',      target_size = (TARGET_SIZE_IMG, TARGET_SIZE_IMG), batch_size = BATCH_SIZE, class_mode = 'binary', seed=42)#Defines a seed to perform training with shuffling the order of the image
validation_datagen = validation_datagen.flow_from_directory(PATH_DATASET+'/validation/', target_size = (TARGET_SIZE_IMG, TARGET_SIZE_IMG), batch_size = BATCH_SIZE, class_mode = 'binary', seed=42)
test_datagen       = test_datagen.flow_from_directory(      PATH_DATASET+'/test/',       target_size = (TARGET_SIZE_IMG, TARGET_SIZE_IMG), batch_size = BATCH_SIZE, class_mode = 'binary', shuffle=False)

validation_datagen_labels = validation_datagen.classes
test_datagen_labels       = test_datagen.classes
class_indices             = train_datagen.class_indices

print("   - train (",round((len(train_datagen.classes) * 100) / (len(train_datagen.classes) + len(validation_datagen.classes) + len(test_datagen.classes)),2),"%)", sep="")
print("      -", train_datagen.class_indices) 
print("      - 0:", np.count_nonzero(train_datagen.classes == 0))
print("      - 1:", np.count_nonzero(train_datagen.classes == 1))

print("   - Validation (",round((len(validation_datagen.classes) * 100) / (len(train_datagen.classes) + len(validation_datagen.classes) + len(test_datagen.classes)),2),"%)", sep="")
print("      -", validation_datagen.class_indices) 
print("      - 0:", np.count_nonzero(validation_datagen.classes == 0))
print("      - 1:", np.count_nonzero(validation_datagen.classes == 1))

print("   - test (",round((len(test_datagen.classes) * 100) / (len(train_datagen.classes) + len(validation_datagen.classes) + len(test_datagen.classes)),2),"%)", sep="")
print("      -", test_datagen.class_indices) 
print("      - 0:", np.count_nonzero(test_datagen.classes == 0))
print("      - 1:", np.count_nonzero(test_datagen.classes == 1))


plt.figure(figsize=(16,5))
plt.suptitle('Some images from the training dataset', fontsize=16)
for i in range(24):
    plt.subplot(2,12,i+1)
    for x,y in train_datagen:
        plt.imshow((x[0]))
        plt.title('{} ({})'.format( list(class_indices.keys())[list(class_indices.values()).index(y[0])] , int(y[0]) )     )
        plt.axis('off')
        break
plt.tight_layout()
plt.show()


train_datagen.reset() 
gc.collect()
# ==============================================================================================================
# ==============================================================================================================

* Loading dataset
    - Reading files folders
Found 111488 images belonging to 2 classes.
Found 15216 images belonging to 2 classes.
Found 31280 images belonging to 2 classes.
   - train (70.57%)
      - {'bacillus': 0, 'no-bacillus': 1}
      - 0: 55744
      - 1: 55744
   - Validation (9.63%)
      - {'bacillus': 0, 'no-bacillus': 1}
      - 0: 7608
      - 1: 7608
   - test (19.8%)
      - {'bacillus': 0, 'no-bacillus': 1}
      - 0: 15640
      - 1: 15640

62756

3. layer architecture structure to generate the CNN models

# ==============================================================================================================
# ==============================================================================================================
# layer architecture structure to generate the CNN models
# ==============================================================================================================
# ==============================================================================================================
model = Sequential()

 
#-------------------------------------------------------------------------------------------------
# Quinn et al. (2016)
model.add(Conv2D(7, (3,3), input_shape = (20, 20, 3), activation = 'relu')) 
model.add(MaxPooling2D(pool_size = (2,2))) 
model.add(Conv2D(12, (3,3), activation = 'relu', kernel_initializer='he_uniform', padding='same'))   
model.add(Flatten()) 
model.add(Dense(units = 500, activation = 'relu'))
model.add(Dense(units = 1, activation = 'sigmoid')) 
#-------------------------------------------------------------------------------------------------
  

'''    
#-------------------------------------------------------------------------------------------------
# Panicker et al. (2018) 
model.add(Conv2D(32, (3,3), input_shape = (224, 224, 3), activation = 'relu'))
model.add(BatchNormalization())
model.add(Activation(activations.relu))
model.add(MaxPooling2D(pool_size = (2,2)))

model.add(Conv2D(32, (3,3), activation = 'relu' ))
model.add(BatchNormalization())
model.add(Activation(activations.relu))
model.add(MaxPooling2D(pool_size = (2,2)))

model.add(Conv2D(64, (3,3), activation = 'relu' ))
model.add(BatchNormalization())
model.add(Activation(activations.relu))
model.add(MaxPooling2D(pool_size = (2,2)))

model.add(Conv2D(64, (3,3), activation = 'relu' ))
model.add(BatchNormalization())
model.add(Activation(activations.relu))
model.add(MaxPooling2D(pool_size = (2,2)))

model.add(Conv2D(64, (3,3), activation = 'relu' ))
model.add(BatchNormalization())
model.add(Activation(activations.relu))
model.add(MaxPooling2D(pool_size = (2,2)))
 
model.add(Conv2D(128, (3,3), activation = 'relu' ))
model.add(BatchNormalization())
model.add(Activation(activations.relu))
model.add(MaxPooling2D(pool_size = (2,2)))
model.add(Flatten()) 
model.add(Dense(units = 128, activation = 'relu'))
model.add(Dropout(0.05))

model.add(Dense(units = 1, activation = 'sigmoid'))  
#-------------------------------------------------------------------------------------------------
'''    

''' 
#-------------------------------------------------------------------------------------------------
# Udegova et al. (2021)
model.add( Input([148, 148, 3]) )
model.add(BatchNormalization()) 
#model.add( tf.keras.layers.experimental.preprocessing.Resizing(148,148) )

model.add(Conv2D(32, (1,1), activation = 'relu' ))
model.add(MaxPooling2D(pool_size = (2,2)))
   
model.add(Conv2D(32, (1,1), activation = 'relu' ))
model.add(MaxPooling2D(pool_size = (11,11), strides=(2, 2))) 

model.add(Conv2D(32, (1,1), activation = 'relu' ))
model.add(AveragePooling2D())

model.add(Flatten()) 
model.add(Dropout(0.5))
model.add(Dense(units = 512, activation = 'relu'))
model.add(Dropout(0.5))
model.add(Dense(units = 254, activation = 'relu'))
model.add(Dropout(0.4))
model.add(Dense(units = 64, activation = 'relu'))
model.add(Dropout(0.3))
model.add(Dense(units = 2, activation = 'relu')) 
model.add(Dense(units = 1, activation = 'sigmoid')) 
#-------------------------------------------------------------------------------------------------
'''

''' 
#-------------------------------------------------------------------------------------------------
#  Simon et al (2019)
model = Sequential()
model.add( Input([20, 20, 3]) )
model.add(Conv2D(7, (3,3),   activation = 'relu')) 
model.add(MaxPooling2D(pool_size = (2,2))) 
model.add(tf.keras.layers.Reshape((1,9,9,7)))  
model.add( tf.keras.layers.ConvLSTM2D(20, (3,3) ) )
model.add(Flatten()) 
model.add(Dense(units = 1, activation = 'sigmoid')) 
#-------------------------------------------------------------------------------------------------
'''  
          
opt = SGD(lr=0.001, momentum=0.9)
model.compile(optimizer = opt, loss = 'binary_crossentropy', 
              metrics = ['accuracy',
                          tf.keras.metrics.Precision(),  
                          tf.keras.metrics.Recall(),     
                          tf.keras.metrics.MeanSquaredError(),
                          tf.keras.metrics.AUC() ] ,run_eagerly=True)


print("* Summarize Model: ")
print(model.summary())
plot_model(model, to_file='model_plot.png', show_shapes=True, show_layer_names=True)
# ==============================================================================================================
# ==============================================================================================================

* Summarize Model: 
Model: "sequential"
_________________________________________________________________
Layer (type)                 Output Shape              Param #   
=================================================================
conv2d (Conv2D)              (None, 18, 18, 7)         196       
_________________________________________________________________
max_pooling2d (MaxPooling2D) (None, 9, 9, 7)           0         
_________________________________________________________________
conv2d_1 (Conv2D)            (None, 9, 9, 12)          768       
_________________________________________________________________
flatten (Flatten)            (None, 972)               0         
_________________________________________________________________
dense (Dense)                (None, 500)               486500    
_________________________________________________________________
dense_1 (Dense)              (None, 1)                 501       
=================================================================
Total params: 487,965
Trainable params: 487,965
Non-trainable params: 0
_________________________________________________________________
None

4. Training

# ==============================================================================================================
# ==============================================================================================================
# Perform the training and generate the model
# ==============================================================================================================
# ==============================================================================================================
 
print("\n* Conducting the training and generating the model...")
time1 = time.time()

print('  - Starting' )
# Conduct the training       
history = model.fit_generator(train_datagen, steps_per_epoch=len(train_datagen), 
                                                            validation_data = validation_datagen, validation_steps= len(validation_datagen), 
                                                            epochs=EPOCHS_SIZE, verbose=1  
                                                            ,callbacks=[MyCustomCallback()] )
 
print('  - Finished' )
time2 = time.time()
time_model_train = round( (((time2-time1)*1000.0)/1000), 2)
time_model_train = str(time_model_train)+'s  ('+str(time_model_train/60)+'m)('+str(time_model_train/60/60)+'h)'
print('  - Time: ',time_model_train )


plt.title('History Train')
plt.plot(history.history['accuracy'], label='accuracy ('+ "{:.4f}".format( history.history['accuracy'][-1])+')' )
plt.plot(history.history['precision'], label='precision ('+ "{:.4f}".format( history.history['precision'][-1])+')' )
plt.plot(history.history['recall'], label='recall ('+ "{:.4f}".format( history.history['recall'][-1])+')' )
plt.plot(history.history['mean_squared_error'], label='mean_squared_error ('+ "{:.4f}".format( history.history['mean_squared_error'][-1])+')' )
plt.plot(history.history['auc'], label='auc ('+ "{:.4f}".format( history.history['auc'][-1])+')' )
plt.plot(history.history['val_accuracy'], label = 'val_accuracy')
plt.xlabel('Epoch')
plt.ylabel('Value')
plt.ylim([0.5, 1])
plt.legend(loc='lower right') 
plt.savefig((OUT_PATH + FILE_ID_RUN +'train-plot1.png'))
display(Image.open((OUT_PATH + FILE_ID_RUN +'train-plot1.png')))

summarize_diagnostics(history, (OUT_PATH + FILE_ID_RUN +'train-plot2.png') )
 
# Save the model 
print("  - Saving model...") 
modelNameSaved = OUT_PATH + FILE_ID_RUN +'MODEL.h5'
model.save(modelNameSaved) 
print("      - Model architecture saved in:", modelNameSaved, '\n')

dot_img_file = OUT_PATH + FILE_ID_RUN +'MODEL-architecture.png'
tf.keras.utils.plot_model(model, to_file=dot_img_file, show_shapes=True)
print("      - Image architecture saved in:", dot_img_file, '\n')

* Conducting the training and generating the model...
  - Starting
Epoch 1/100

C:\Users\Pichau\AppData\Roaming\Python\Python38\site-packages\tensorflow\python\keras\engine\training.py:1844: UserWarning: `Model.fit_generator` is deprecated and will be removed in a future version. Please use `Model.fit`, which supports generators.
  warnings.warn('`Model.fit_generator` is deprecated and '

4460/4460 [==============================] - 1959s 439ms/step - loss: 0.5951 - accuracy: 0.6387 - precision: 0.6357 - recall: 0.6300 - mean_squared_error: 0.2074 - auc: 0.7056 - val_loss: 0.2236 - val_accuracy: 0.9267 - val_precision: 0.9033 - val_recall: 0.9557 - val_mean_squared_error: 0.0587 - val_auc: 0.9717
-----------------
Memory Ram: 15.75 (Free: 75.36%)
-----------------
Epoch 2/100
4460/4460 [==============================] - 115s 26ms/step - loss: 0.1969 - accuracy: 0.9424 - precision: 0.9379 - recall: 0.9481 - mean_squared_error: 0.0484 - auc: 0.9747 - val_loss: 0.1579 - val_accuracy: 0.9516 - val_precision: 0.9481 - val_recall: 0.9554 - val_mean_squared_error: 0.0393 - val_auc: 0.9838
-----------------
Memory Ram: 15.75 (Free: 75.37%)
-----------------
Epoch 3/100
4460/4460 [==============================] - 116s 26ms/step - loss: 0.1532 - accuracy: 0.9557 - precision: 0.9572 - recall: 0.9542 - mean_squared_error: 0.0371 - auc: 0.9828 - val_loss: 0.1336 - val_accuracy: 0.9607 - val_precision: 0.9637 - val_recall: 0.9574 - val_mean_squared_error: 0.0330 - val_auc: 0.9866
-----------------
Memory Ram: 15.9 (Free: 75.14%)
-----------------
Epoch 4/100
4460/4460 [==============================] - 123s 28ms/step - loss: 0.1328 - accuracy: 0.9610 - precision: 0.9632 - recall: 0.9584 - mean_squared_error: 0.0324 - auc: 0.9862 - val_loss: 0.1193 - val_accuracy: 0.9648 - val_precision: 0.9656 - val_recall: 0.9639 - val_mean_squared_error: 0.0296 - val_auc: 0.9894
-----------------
Memory Ram: 15.9 (Free: 75.13%)
-----------------
Epoch 5/100
4460/4460 [==============================] - 122s 27ms/step - loss: 0.1264 - accuracy: 0.9628 - precision: 0.9658 - recall: 0.9593 - mean_squared_error: 0.0308 - auc: 0.9872 - val_loss: 0.1133 - val_accuracy: 0.9676 - val_precision: 0.9764 - val_recall: 0.9583 - val_mean_squared_error: 0.0278 - val_auc: 0.9903
-----------------
Memory Ram: 15.87 (Free: 75.18%)
-----------------
Epoch 6/100
4460/4460 [==============================] - 122s 27ms/step - loss: 0.1181 - accuracy: 0.9649 - precision: 0.9676 - recall: 0.9620 - mean_squared_error: 0.0289 - auc: 0.9885 - val_loss: 0.1073 - val_accuracy: 0.9673 - val_precision: 0.9680 - val_recall: 0.9666 - val_mean_squared_error: 0.0275 - val_auc: 0.9909
-----------------
Memory Ram: 15.83 (Free: 75.25%)
-----------------
Epoch 7/100
4460/4460 [==============================] - 117s 26ms/step - loss: 0.1125 - accuracy: 0.9664 - precision: 0.9698 - recall: 0.9629 - mean_squared_error: 0.0276 - auc: 0.9893 - val_loss: 0.1021 - val_accuracy: 0.9662 - val_precision: 0.9649 - val_recall: 0.9677 - val_mean_squared_error: 0.0266 - val_auc: 0.9917
-----------------
Memory Ram: 15.78 (Free: 75.32%)
-----------------
Epoch 8/100
4460/4460 [==============================] - 114s 26ms/step - loss: 0.1068 - accuracy: 0.9672 - precision: 0.9705 - recall: 0.9636 - mean_squared_error: 0.0265 - auc: 0.9903 - val_loss: 0.1023 - val_accuracy: 0.9669 - val_precision: 0.9612 - val_recall: 0.9732 - val_mean_squared_error: 0.0273 - val_auc: 0.9923
-----------------
Memory Ram: 15.79 (Free: 75.3%)
-----------------
Epoch 9/100
4460/4460 [==============================] - 110s 25ms/step - loss: 0.1011 - accuracy: 0.9687 - precision: 0.9720 - recall: 0.9654 - mean_squared_error: 0.0253 - auc: 0.9912 - val_loss: 0.0980 - val_accuracy: 0.9685 - val_precision: 0.9840 - val_recall: 0.9525 - val_mean_squared_error: 0.0244 - val_auc: 0.9930
-----------------
Memory Ram: 15.8 (Free: 75.3%)
-----------------
Epoch 10/100
4460/4460 [==============================] - 111s 25ms/step - loss: 0.0987 - accuracy: 0.9685 - precision: 0.9714 - recall: 0.9655 - mean_squared_error: 0.0248 - auc: 0.9915 - val_loss: 0.0919 - val_accuracy: 0.9709 - val_precision: 0.9753 - val_recall: 0.9662 - val_mean_squared_error: 0.0233 - val_auc: 0.9929
-----------------
Memory Ram: 15.89 (Free: 75.16%)
-----------------
Epoch 11/100
4460/4460 [==============================] - 117s 26ms/step - loss: 0.0952 - accuracy: 0.9706 - precision: 0.9737 - recall: 0.9674 - mean_squared_error: 0.0237 - auc: 0.9919 - val_loss: 0.0902 - val_accuracy: 0.9706 - val_precision: 0.9707 - val_recall: 0.9704 - val_mean_squared_error: 0.0235 - val_auc: 0.9933
-----------------
Memory Ram: 16.18 (Free: 74.7%)
-----------------
Epoch 12/100
4460/4460 [==============================] - 121s 27ms/step - loss: 0.0924 - accuracy: 0.9715 - precision: 0.9746 - recall: 0.9681 - mean_squared_error: 0.0230 - auc: 0.9922 - val_loss: 0.0860 - val_accuracy: 0.9750 - val_precision: 0.9778 - val_recall: 0.9721 - val_mean_squared_error: 0.0218 - val_auc: 0.9935
-----------------
Memory Ram: 16.12 (Free: 74.78%)
-----------------
Epoch 13/100
4460/4460 [==============================] - 118s 26ms/step - loss: 0.0894 - accuracy: 0.9719 - precision: 0.9749 - recall: 0.9687 - mean_squared_error: 0.0224 - auc: 0.9927 - val_loss: 0.0866 - val_accuracy: 0.9740 - val_precision: 0.9849 - val_recall: 0.9627 - val_mean_squared_error: 0.0214 - val_auc: 0.9937
-----------------
Memory Ram: 16.28 (Free: 74.55%)
-----------------
Epoch 14/100
4460/4460 [==============================] - 120s 27ms/step - loss: 0.0899 - accuracy: 0.9717 - precision: 0.9740 - recall: 0.9692 - mean_squared_error: 0.0225 - auc: 0.9926 - val_loss: 0.0821 - val_accuracy: 0.9750 - val_precision: 0.9834 - val_recall: 0.9662 - val_mean_squared_error: 0.0205 - val_auc: 0.9944
-----------------
Memory Ram: 16.36 (Free: 74.42%)
-----------------
Epoch 15/100
4460/4460 [==============================] - 119s 27ms/step - loss: 0.0833 - accuracy: 0.9742 - precision: 0.9772 - recall: 0.9710 - mean_squared_error: 0.0207 - auc: 0.9933 - val_loss: 0.0861 - val_accuracy: 0.9766 - val_precision: 0.9868 - val_recall: 0.9661 - val_mean_squared_error: 0.0206 - val_auc: 0.9940
-----------------
Memory Ram: 16.32 (Free: 74.47%)
-----------------
Epoch 16/100
4460/4460 [==============================] - 118s 26ms/step - loss: 0.0844 - accuracy: 0.9742 - precision: 0.9761 - recall: 0.9722 - mean_squared_error: 0.0210 - auc: 0.9933 - val_loss: 0.0858 - val_accuracy: 0.9744 - val_precision: 0.9892 - val_recall: 0.9594 - val_mean_squared_error: 0.0211 - val_auc: 0.9944
-----------------
Memory Ram: 16.71 (Free: 73.86%)
-----------------
Epoch 17/100
4460/4460 [==============================] - 122s 27ms/step - loss: 0.0815 - accuracy: 0.9741 - precision: 0.9768 - recall: 0.9716 - mean_squared_error: 0.0203 - auc: 0.9938 - val_loss: 0.0800 - val_accuracy: 0.9741 - val_precision: 0.9727 - val_recall: 0.9756 - val_mean_squared_error: 0.0207 - val_auc: 0.9943
-----------------
Memory Ram: 17.04 (Free: 73.35%)
-----------------
Epoch 18/100
4460/4460 [==============================] - 117s 26ms/step - loss: 0.0777 - accuracy: 0.9757 - precision: 0.9780 - recall: 0.9732 - mean_squared_error: 0.0192 - auc: 0.9942 - val_loss: 0.0860 - val_accuracy: 0.9714 - val_precision: 0.9634 - val_recall: 0.9800 - val_mean_squared_error: 0.0228 - val_auc: 0.9943
-----------------
Memory Ram: 16.93 (Free: 73.53%)
-----------------
Epoch 19/100
4460/4460 [==============================] - 119s 27ms/step - loss: 0.0754 - accuracy: 0.9774 - precision: 0.9795 - recall: 0.9753 - mean_squared_error: 0.0184 - auc: 0.9941 - val_loss: 0.0748 - val_accuracy: 0.9762 - val_precision: 0.9778 - val_recall: 0.9745 - val_mean_squared_error: 0.0190 - val_auc: 0.9949
-----------------
Memory Ram: 16.9 (Free: 73.57%)
-----------------
Epoch 20/100
4460/4460 [==============================] - 117s 26ms/step - loss: 0.0759 - accuracy: 0.9764 - precision: 0.9787 - recall: 0.9737 - mean_squared_error: 0.0189 - auc: 0.9946 - val_loss: 0.0799 - val_accuracy: 0.9742 - val_precision: 0.9838 - val_recall: 0.9644 - val_mean_squared_error: 0.0200 - val_auc: 0.9943
-----------------
Memory Ram: 16.96 (Free: 73.48%)
-----------------
Epoch 21/100
4460/4460 [==============================] - 124s 28ms/step - loss: 0.0747 - accuracy: 0.9769 - precision: 0.9794 - recall: 0.9744 - mean_squared_error: 0.0186 - auc: 0.9946 - val_loss: 0.0763 - val_accuracy: 0.9764 - val_precision: 0.9757 - val_recall: 0.9771 - val_mean_squared_error: 0.0193 - val_auc: 0.9946
-----------------
Memory Ram: 16.91 (Free: 73.56%)
-----------------
Epoch 22/100
4460/4460 [==============================] - 116s 26ms/step - loss: 0.0731 - accuracy: 0.9772 - precision: 0.9795 - recall: 0.9750 - mean_squared_error: 0.0184 - auc: 0.9949 - val_loss: 0.0790 - val_accuracy: 0.9765 - val_precision: 0.9888 - val_recall: 0.9640 - val_mean_squared_error: 0.0187 - val_auc: 0.9944
-----------------
Memory Ram: 16.84 (Free: 73.67%)
-----------------
Epoch 23/100
4460/4460 [==============================] - 116s 26ms/step - loss: 0.0713 - accuracy: 0.9778 - precision: 0.9800 - recall: 0.9754 - mean_squared_error: 0.0178 - auc: 0.9951 - val_loss: 0.0733 - val_accuracy: 0.9767 - val_precision: 0.9839 - val_recall: 0.9692 - val_mean_squared_error: 0.0182 - val_auc: 0.9950
-----------------
Memory Ram: 16.72 (Free: 73.85%)
-----------------
Epoch 24/100
4460/4460 [==============================] - 117s 26ms/step - loss: 0.0711 - accuracy: 0.9782 - precision: 0.9799 - recall: 0.9762 - mean_squared_error: 0.0176 - auc: 0.9950 - val_loss: 0.0752 - val_accuracy: 0.9771 - val_precision: 0.9843 - val_recall: 0.9698 - val_mean_squared_error: 0.0185 - val_auc: 0.9946
-----------------
Memory Ram: 16.75 (Free: 73.8%)
-----------------
Epoch 25/100
4460/4460 [==============================] - 118s 26ms/step - loss: 0.0688 - accuracy: 0.9786 - precision: 0.9803 - recall: 0.9769 - mean_squared_error: 0.0171 - auc: 0.9952 - val_loss: 0.0786 - val_accuracy: 0.9763 - val_precision: 0.9879 - val_recall: 0.9644 - val_mean_squared_error: 0.0186 - val_auc: 0.9942
-----------------
Memory Ram: 16.7 (Free: 73.88%)
-----------------
Epoch 26/100
4460/4460 [==============================] - 118s 27ms/step - loss: 0.0685 - accuracy: 0.9787 - precision: 0.9806 - recall: 0.9766 - mean_squared_error: 0.0169 - auc: 0.9953 - val_loss: 0.0719 - val_accuracy: 0.9796 - val_precision: 0.9856 - val_recall: 0.9734 - val_mean_squared_error: 0.0170 - val_auc: 0.9945
-----------------
Memory Ram: 16.71 (Free: 73.87%)
-----------------
Epoch 27/100
4460/4460 [==============================] - 117s 26ms/step - loss: 0.0707 - accuracy: 0.9783 - precision: 0.9801 - recall: 0.9763 - mean_squared_error: 0.0175 - auc: 0.9950 - val_loss: 0.0828 - val_accuracy: 0.9762 - val_precision: 0.9863 - val_recall: 0.9658 - val_mean_squared_error: 0.0193 - val_auc: 0.9941
-----------------
Memory Ram: 16.86 (Free: 73.63%)
-----------------
Epoch 28/100
4460/4460 [==============================] - 120s 27ms/step - loss: 0.0653 - accuracy: 0.9797 - precision: 0.9821 - recall: 0.9771 - mean_squared_error: 0.0162 - auc: 0.9957 - val_loss: 0.0726 - val_accuracy: 0.9776 - val_precision: 0.9809 - val_recall: 0.9741 - val_mean_squared_error: 0.0181 - val_auc: 0.9953
-----------------
Memory Ram: 16.89 (Free: 73.59%)
-----------------
Epoch 29/100
4460/4460 [==============================] - 118s 26ms/step - loss: 0.0655 - accuracy: 0.9798 - precision: 0.9816 - recall: 0.9777 - mean_squared_error: 0.0161 - auc: 0.9955 - val_loss: 0.0731 - val_accuracy: 0.9788 - val_precision: 0.9838 - val_recall: 0.9737 - val_mean_squared_error: 0.0177 - val_auc: 0.9946
-----------------
Memory Ram: 16.92 (Free: 73.55%)
-----------------
Epoch 30/100
4460/4460 [==============================] - 119s 27ms/step - loss: 0.0634 - accuracy: 0.9807 - precision: 0.9819 - recall: 0.9795 - mean_squared_error: 0.0155 - auc: 0.9957 - val_loss: 0.0733 - val_accuracy: 0.9791 - val_precision: 0.9833 - val_recall: 0.9748 - val_mean_squared_error: 0.0173 - val_auc: 0.9947
-----------------
Memory Ram: 16.85 (Free: 73.65%)
-----------------
Epoch 31/100
4460/4460 [==============================] - 119s 27ms/step - loss: 0.0631 - accuracy: 0.9804 - precision: 0.9821 - recall: 0.9786 - mean_squared_error: 0.0156 - auc: 0.9959 - val_loss: 0.0698 - val_accuracy: 0.9804 - val_precision: 0.9860 - val_recall: 0.9746 - val_mean_squared_error: 0.0163 - val_auc: 0.9949
-----------------
Memory Ram: 16.8 (Free: 73.73%)
-----------------
Epoch 32/100
4460/4460 [==============================] - 120s 27ms/step - loss: 0.0630 - accuracy: 0.9808 - precision: 0.9825 - recall: 0.9791 - mean_squared_error: 0.0155 - auc: 0.9958 - val_loss: 0.0707 - val_accuracy: 0.9798 - val_precision: 0.9828 - val_recall: 0.9766 - val_mean_squared_error: 0.0168 - val_auc: 0.9947
-----------------
Memory Ram: 17.34 (Free: 72.88%)
-----------------
Epoch 33/100
4460/4460 [==============================] - 118s 27ms/step - loss: 0.0617 - accuracy: 0.9809 - precision: 0.9823 - recall: 0.9794 - mean_squared_error: 0.0153 - auc: 0.9960 - val_loss: 0.0709 - val_accuracy: 0.9787 - val_precision: 0.9787 - val_recall: 0.9787 - val_mean_squared_error: 0.0175 - val_auc: 0.9948
-----------------
Memory Ram: 17.38 (Free: 72.81%)
-----------------
Epoch 34/100
4460/4460 [==============================] - 117s 26ms/step - loss: 0.0607 - accuracy: 0.9817 - precision: 0.9832 - recall: 0.9803 - mean_squared_error: 0.0148 - auc: 0.9960 - val_loss: 0.0697 - val_accuracy: 0.9794 - val_precision: 0.9791 - val_recall: 0.9796 - val_mean_squared_error: 0.0168 - val_auc: 0.9950
-----------------
Memory Ram: 17.42 (Free: 72.76%)
-----------------
Epoch 35/100
4460/4460 [==============================] - 116s 26ms/step - loss: 0.0580 - accuracy: 0.9824 - precision: 0.9836 - recall: 0.9813 - mean_squared_error: 0.0140 - auc: 0.9961 - val_loss: 0.0721 - val_accuracy: 0.9795 - val_precision: 0.9800 - val_recall: 0.9790 - val_mean_squared_error: 0.0179 - val_auc: 0.9954
-----------------
Memory Ram: 17.37 (Free: 72.84%)
-----------------
Epoch 36/100
4460/4460 [==============================] - 114s 26ms/step - loss: 0.0604 - accuracy: 0.9820 - precision: 0.9832 - recall: 0.9807 - mean_squared_error: 0.0147 - auc: 0.9960 - val_loss: 0.0723 - val_accuracy: 0.9790 - val_precision: 0.9815 - val_recall: 0.9765 - val_mean_squared_error: 0.0173 - val_auc: 0.9947
-----------------
Memory Ram: 17.44 (Free: 72.72%)
-----------------
Epoch 37/100
4460/4460 [==============================] - 117s 26ms/step - loss: 0.0572 - accuracy: 0.9831 - precision: 0.9840 - recall: 0.9821 - mean_squared_error: 0.0138 - auc: 0.9962 - val_loss: 0.0708 - val_accuracy: 0.9795 - val_precision: 0.9824 - val_recall: 0.9765 - val_mean_squared_error: 0.0170 - val_auc: 0.9949
-----------------
Memory Ram: 17.42 (Free: 72.75%)
-----------------
Epoch 38/100
4460/4460 [==============================] - 113s 25ms/step - loss: 0.0571 - accuracy: 0.9824 - precision: 0.9832 - recall: 0.9815 - mean_squared_error: 0.0142 - auc: 0.9964 - val_loss: 0.0698 - val_accuracy: 0.9807 - val_precision: 0.9858 - val_recall: 0.9756 - val_mean_squared_error: 0.0163 - val_auc: 0.9948
-----------------
Memory Ram: 17.34 (Free: 72.89%)
-----------------
Epoch 39/100
4460/4460 [==============================] - 114s 26ms/step - loss: 0.0571 - accuracy: 0.9830 - precision: 0.9842 - recall: 0.9818 - mean_squared_error: 0.0138 - auc: 0.9963 - val_loss: 0.0745 - val_accuracy: 0.9798 - val_precision: 0.9826 - val_recall: 0.9770 - val_mean_squared_error: 0.0169 - val_auc: 0.9940
-----------------
Memory Ram: 17.41 (Free: 72.77%)
-----------------
Epoch 40/100
4460/4460 [==============================] - 116s 26ms/step - loss: 0.0559 - accuracy: 0.9833 - precision: 0.9845 - recall: 0.9820 - mean_squared_error: 0.0136 - auc: 0.9965 - val_loss: 0.0720 - val_accuracy: 0.9795 - val_precision: 0.9839 - val_recall: 0.9749 - val_mean_squared_error: 0.0167 - val_auc: 0.9946
-----------------
Memory Ram: 17.35 (Free: 72.87%)
-----------------
Epoch 41/100
4460/4460 [==============================] - 116s 26ms/step - loss: 0.0562 - accuracy: 0.9834 - precision: 0.9849 - recall: 0.9818 - mean_squared_error: 0.0137 - auc: 0.9965 - val_loss: 0.0716 - val_accuracy: 0.9823 - val_precision: 0.9896 - val_recall: 0.9748 - val_mean_squared_error: 0.0158 - val_auc: 0.9946
-----------------
Memory Ram: 17.39 (Free: 72.81%)
-----------------
Epoch 42/100
4460/4460 [==============================] - 114s 26ms/step - loss: 0.0553 - accuracy: 0.9833 - precision: 0.9840 - recall: 0.9827 - mean_squared_error: 0.0134 - auc: 0.9965 - val_loss: 0.0731 - val_accuracy: 0.9803 - val_precision: 0.9871 - val_recall: 0.9734 - val_mean_squared_error: 0.0163 - val_auc: 0.9942
-----------------
Memory Ram: 17.33 (Free: 72.9%)
-----------------
Epoch 43/100
4460/4460 [==============================] - 117s 26ms/step - loss: 0.0521 - accuracy: 0.9848 - precision: 0.9855 - recall: 0.9841 - mean_squared_error: 0.0126 - auc: 0.9968 - val_loss: 0.0750 - val_accuracy: 0.9771 - val_precision: 0.9723 - val_recall: 0.9823 - val_mean_squared_error: 0.0182 - val_auc: 0.9944
-----------------
Memory Ram: 17.34 (Free: 72.88%)
-----------------
Epoch 44/100
4460/4460 [==============================] - 116s 26ms/step - loss: 0.0532 - accuracy: 0.9839 - precision: 0.9845 - recall: 0.9833 - mean_squared_error: 0.0129 - auc: 0.9967 - val_loss: 0.0733 - val_accuracy: 0.9800 - val_precision: 0.9824 - val_recall: 0.9775 - val_mean_squared_error: 0.0165 - val_auc: 0.9942
-----------------
Memory Ram: 17.38 (Free: 72.82%)
-----------------
Epoch 45/100
4460/4460 [==============================] - 117s 26ms/step - loss: 0.0534 - accuracy: 0.9836 - precision: 0.9847 - recall: 0.9825 - mean_squared_error: 0.0131 - auc: 0.9967 - val_loss: 0.0710 - val_accuracy: 0.9792 - val_precision: 0.9775 - val_recall: 0.9809 - val_mean_squared_error: 0.0171 - val_auc: 0.9948
-----------------
Memory Ram: 17.34 (Free: 72.88%)
-----------------
Epoch 46/100
4460/4460 [==============================] - 114s 26ms/step - loss: 0.0518 - accuracy: 0.9840 - precision: 0.9849 - recall: 0.9830 - mean_squared_error: 0.0128 - auc: 0.9970 - val_loss: 0.0725 - val_accuracy: 0.9806 - val_precision: 0.9809 - val_recall: 0.9803 - val_mean_squared_error: 0.0166 - val_auc: 0.9946
-----------------
Memory Ram: 17.36 (Free: 72.85%)
-----------------
Epoch 47/100
4460/4460 [==============================] - 117s 26ms/step - loss: 0.0514 - accuracy: 0.9846 - precision: 0.9849 - recall: 0.9842 - mean_squared_error: 0.0125 - auc: 0.9971 - val_loss: 0.0707 - val_accuracy: 0.9806 - val_precision: 0.9827 - val_recall: 0.9784 - val_mean_squared_error: 0.0163 - val_auc: 0.9949
-----------------
Memory Ram: 17.39 (Free: 72.81%)
-----------------
Epoch 48/100
4460/4460 [==============================] - 123s 28ms/step - loss: 0.0510 - accuracy: 0.9849 - precision: 0.9857 - recall: 0.9841 - mean_squared_error: 0.0123 - auc: 0.9970 - val_loss: 0.0713 - val_accuracy: 0.9811 - val_precision: 0.9896 - val_recall: 0.9725 - val_mean_squared_error: 0.0160 - val_auc: 0.9946
-----------------
Memory Ram: 17.36 (Free: 72.85%)
-----------------
Epoch 49/100
4460/4460 [==============================] - 118s 26ms/step - loss: 0.0486 - accuracy: 0.9850 - precision: 0.9859 - recall: 0.9843 - mean_squared_error: 0.0120 - auc: 0.9973 - val_loss: 0.0759 - val_accuracy: 0.9761 - val_precision: 0.9699 - val_recall: 0.9828 - val_mean_squared_error: 0.0193 - val_auc: 0.9947
-----------------
Memory Ram: 17.39 (Free: 72.8%)
-----------------
Epoch 50/100
4460/4460 [==============================] - 117s 26ms/step - loss: 0.0494 - accuracy: 0.9851 - precision: 0.9850 - recall: 0.9854 - mean_squared_error: 0.0121 - auc: 0.9973 - val_loss: 0.0726 - val_accuracy: 0.9793 - val_precision: 0.9804 - val_recall: 0.9782 - val_mean_squared_error: 0.0170 - val_auc: 0.9944
-----------------
Memory Ram: 17.41 (Free: 72.77%)
-----------------
Epoch 51/100
4460/4460 [==============================] - 121s 27ms/step - loss: 0.0486 - accuracy: 0.9855 - precision: 0.9858 - recall: 0.9854 - mean_squared_error: 0.0118 - auc: 0.9972 - val_loss: 0.0775 - val_accuracy: 0.9804 - val_precision: 0.9844 - val_recall: 0.9763 - val_mean_squared_error: 0.0167 - val_auc: 0.9935
-----------------
Memory Ram: 17.58 (Free: 72.5%)
-----------------
Epoch 52/100
4460/4460 [==============================] - 120s 27ms/step - loss: 0.0471 - accuracy: 0.9859 - precision: 0.9863 - recall: 0.9856 - mean_squared_error: 0.0114 - auc: 0.9973 - val_loss: 0.0738 - val_accuracy: 0.9794 - val_precision: 0.9852 - val_recall: 0.9734 - val_mean_squared_error: 0.0166 - val_auc: 0.9943
-----------------
Memory Ram: 17.42 (Free: 72.76%)
-----------------
Epoch 53/100
4460/4460 [==============================] - 111s 25ms/step - loss: 0.0471 - accuracy: 0.9860 - precision: 0.9866 - recall: 0.9854 - mean_squared_error: 0.0114 - auc: 0.9973 - val_loss: 0.0704 - val_accuracy: 0.9811 - val_precision: 0.9862 - val_recall: 0.9758 - val_mean_squared_error: 0.0158 - val_auc: 0.9940
-----------------
Memory Ram: 17.41 (Free: 72.77%)
-----------------
Epoch 54/100
4460/4460 [==============================] - 111s 25ms/step - loss: 0.0452 - accuracy: 0.9865 - precision: 0.9867 - recall: 0.9863 - mean_squared_error: 0.0109 - auc: 0.9976 - val_loss: 0.0815 - val_accuracy: 0.9796 - val_precision: 0.9906 - val_recall: 0.9683 - val_mean_squared_error: 0.0171 - val_auc: 0.9935
-----------------
Memory Ram: 17.39 (Free: 72.8%)
-----------------
Epoch 55/100
4460/4460 [==============================] - 113s 25ms/step - loss: 0.0445 - accuracy: 0.9867 - precision: 0.9876 - recall: 0.9858 - mean_squared_error: 0.0109 - auc: 0.9976 - val_loss: 0.0757 - val_accuracy: 0.9793 - val_precision: 0.9813 - val_recall: 0.9773 - val_mean_squared_error: 0.0170 - val_auc: 0.9939
-----------------
Memory Ram: 17.39 (Free: 72.8%)
-----------------
Epoch 56/100
4460/4460 [==============================] - 115s 26ms/step - loss: 0.0439 - accuracy: 0.9868 - precision: 0.9872 - recall: 0.9862 - mean_squared_error: 0.0107 - auc: 0.9977 - val_loss: 0.0816 - val_accuracy: 0.9738 - val_precision: 0.9637 - val_recall: 0.9848 - val_mean_squared_error: 0.0205 - val_auc: 0.9943
-----------------
Memory Ram: 17.61 (Free: 72.46%)
-----------------
Epoch 57/100
4460/4460 [==============================] - 114s 26ms/step - loss: 0.0430 - accuracy: 0.9872 - precision: 0.9868 - recall: 0.9876 - mean_squared_error: 0.0105 - auc: 0.9977 - val_loss: 0.0799 - val_accuracy: 0.9793 - val_precision: 0.9828 - val_recall: 0.9757 - val_mean_squared_error: 0.0176 - val_auc: 0.9938
-----------------
Memory Ram: 17.56 (Free: 72.54%)
-----------------
Epoch 58/100
4460/4460 [==============================] - 114s 25ms/step - loss: 0.0432 - accuracy: 0.9870 - precision: 0.9868 - recall: 0.9870 - mean_squared_error: 0.0105 - auc: 0.9978 - val_loss: 0.0762 - val_accuracy: 0.9809 - val_precision: 0.9889 - val_recall: 0.9728 - val_mean_squared_error: 0.0162 - val_auc: 0.9940
-----------------
Memory Ram: 17.51 (Free: 72.62%)
-----------------
Epoch 59/100
4460/4460 [==============================] - 113s 25ms/step - loss: 0.0432 - accuracy: 0.9871 - precision: 0.9870 - recall: 0.9873 - mean_squared_error: 0.0104 - auc: 0.9976 - val_loss: 0.0731 - val_accuracy: 0.9798 - val_precision: 0.9847 - val_recall: 0.9748 - val_mean_squared_error: 0.0164 - val_auc: 0.9941
-----------------
Memory Ram: 17.52 (Free: 72.61%)
-----------------
Epoch 60/100
4460/4460 [==============================] - 115s 26ms/step - loss: 0.0396 - accuracy: 0.9887 - precision: 0.9888 - recall: 0.9885 - mean_squared_error: 0.0094 - auc: 0.9979 - val_loss: 0.0764 - val_accuracy: 0.9794 - val_precision: 0.9839 - val_recall: 0.9748 - val_mean_squared_error: 0.0170 - val_auc: 0.9940
-----------------
Memory Ram: 17.54 (Free: 72.57%)
-----------------
Epoch 61/100
4460/4460 [==============================] - 172s 39ms/step - loss: 0.0408 - accuracy: 0.9879 - precision: 0.9878 - recall: 0.9880 - mean_squared_error: 0.0098 - auc: 0.9979 - val_loss: 0.0748 - val_accuracy: 0.9808 - val_precision: 0.9830 - val_recall: 0.9786 - val_mean_squared_error: 0.0163 - val_auc: 0.9936
-----------------
Memory Ram: 17.73 (Free: 72.27%)
-----------------
Epoch 62/100
4460/4460 [==============================] - 117s 26ms/step - loss: 0.0395 - accuracy: 0.9882 - precision: 0.9882 - recall: 0.9881 - mean_squared_error: 0.0096 - auc: 0.9981 - val_loss: 0.0770 - val_accuracy: 0.9800 - val_precision: 0.9855 - val_recall: 0.9742 - val_mean_squared_error: 0.0166 - val_auc: 0.9938
-----------------
Memory Ram: 17.89 (Free: 72.02%)
-----------------
Epoch 63/100
4460/4460 [==============================] - 111s 25ms/step - loss: 0.0401 - accuracy: 0.9879 - precision: 0.9880 - recall: 0.9877 - mean_squared_error: 0.0098 - auc: 0.9980 - val_loss: 0.0850 - val_accuracy: 0.9720 - val_precision: 0.9610 - val_recall: 0.9840 - val_mean_squared_error: 0.0220 - val_auc: 0.9943
-----------------
Memory Ram: 17.88 (Free: 72.03%)
-----------------
Epoch 64/100
4460/4460 [==============================] - 112s 25ms/step - loss: 0.0403 - accuracy: 0.9883 - precision: 0.9878 - recall: 0.9887 - mean_squared_error: 0.0096 - auc: 0.9978 - val_loss: 0.0821 - val_accuracy: 0.9754 - val_precision: 0.9685 - val_recall: 0.9828 - val_mean_squared_error: 0.0194 - val_auc: 0.9935
-----------------
Memory Ram: 17.91 (Free: 71.99%)
-----------------
Epoch 65/100
4460/4460 [==============================] - 112s 25ms/step - loss: 0.0378 - accuracy: 0.9884 - precision: 0.9886 - recall: 0.9884 - mean_squared_error: 0.0092 - auc: 0.9982 - val_loss: 0.0716 - val_accuracy: 0.9797 - val_precision: 0.9793 - val_recall: 0.9802 - val_mean_squared_error: 0.0164 - val_auc: 0.9941
-----------------
Memory Ram: 17.68 (Free: 72.35%)
-----------------
Epoch 66/100
4460/4460 [==============================] - 112s 25ms/step - loss: 0.0383 - accuracy: 0.9885 - precision: 0.9880 - recall: 0.9889 - mean_squared_error: 0.0092 - auc: 0.9981 - val_loss: 0.0748 - val_accuracy: 0.9800 - val_precision: 0.9849 - val_recall: 0.9749 - val_mean_squared_error: 0.0167 - val_auc: 0.9943
-----------------
Memory Ram: 17.68 (Free: 72.35%)
-----------------
Epoch 67/100
4460/4460 [==============================] - 111s 25ms/step - loss: 0.0372 - accuracy: 0.9894 - precision: 0.9890 - recall: 0.9897 - mean_squared_error: 0.0088 - auc: 0.9980 - val_loss: 0.0812 - val_accuracy: 0.9747 - val_precision: 0.9673 - val_recall: 0.9826 - val_mean_squared_error: 0.0203 - val_auc: 0.9946
-----------------
Memory Ram: 17.78 (Free: 72.19%)
-----------------
Epoch 68/100
4460/4460 [==============================] - 123s 28ms/step - loss: 0.0365 - accuracy: 0.9889 - precision: 0.9890 - recall: 0.9889 - mean_squared_error: 0.0089 - auc: 0.9983 - val_loss: 0.0818 - val_accuracy: 0.9777 - val_precision: 0.9733 - val_recall: 0.9824 - val_mean_squared_error: 0.0183 - val_auc: 0.9933
-----------------
Memory Ram: 17.8 (Free: 72.16%)
-----------------
Epoch 69/100
4460/4460 [==============================] - 124s 28ms/step - loss: 0.0370 - accuracy: 0.9889 - precision: 0.9882 - recall: 0.9895 - mean_squared_error: 0.0089 - auc: 0.9982 - val_loss: 0.0778 - val_accuracy: 0.9807 - val_precision: 0.9832 - val_recall: 0.9780 - val_mean_squared_error: 0.0165 - val_auc: 0.9936
-----------------
Memory Ram: 17.91 (Free: 71.99%)
-----------------
Epoch 70/100
4460/4460 [==============================] - 121s 27ms/step - loss: 0.0355 - accuracy: 0.9895 - precision: 0.9885 - recall: 0.9904 - mean_squared_error: 0.0085 - auc: 0.9982 - val_loss: 0.0930 - val_accuracy: 0.9702 - val_precision: 0.9577 - val_recall: 0.9838 - val_mean_squared_error: 0.0234 - val_auc: 0.9934
-----------------
Memory Ram: 17.95 (Free: 71.93%)
-----------------
Epoch 71/100
4460/4460 [==============================] - 119s 27ms/step - loss: 0.0351 - accuracy: 0.9902 - precision: 0.9895 - recall: 0.9910 - mean_squared_error: 0.0083 - auc: 0.9983 - val_loss: 0.0822 - val_accuracy: 0.9802 - val_precision: 0.9828 - val_recall: 0.9775 - val_mean_squared_error: 0.0170 - val_auc: 0.9929
-----------------
Memory Ram: 17.9 (Free: 72.01%)
-----------------
Epoch 72/100
4460/4460 [==============================] - 116s 26ms/step - loss: 0.0346 - accuracy: 0.9903 - precision: 0.9894 - recall: 0.9914 - mean_squared_error: 0.0081 - auc: 0.9983 - val_loss: 0.0794 - val_accuracy: 0.9798 - val_precision: 0.9827 - val_recall: 0.9767 - val_mean_squared_error: 0.0170 - val_auc: 0.9935
-----------------
Memory Ram: 18.16 (Free: 71.6%)
-----------------
Epoch 73/100
4460/4460 [==============================] - 111s 25ms/step - loss: 0.0320 - accuracy: 0.9908 - precision: 0.9906 - recall: 0.9910 - mean_squared_error: 0.0076 - auc: 0.9985 - val_loss: 0.0816 - val_accuracy: 0.9782 - val_precision: 0.9754 - val_recall: 0.9811 - val_mean_squared_error: 0.0185 - val_auc: 0.9929
-----------------
Memory Ram: 18.13 (Free: 71.65%)
-----------------
Epoch 74/100
4460/4460 [==============================] - 112s 25ms/step - loss: 0.0342 - accuracy: 0.9902 - precision: 0.9894 - recall: 0.9908 - mean_squared_error: 0.0081 - auc: 0.9983 - val_loss: 0.0861 - val_accuracy: 0.9795 - val_precision: 0.9790 - val_recall: 0.9800 - val_mean_squared_error: 0.0173 - val_auc: 0.9925
-----------------
Memory Ram: 18.24 (Free: 71.47%)
-----------------
Epoch 75/100
4460/4460 [==============================] - 124s 28ms/step - loss: 0.0324 - accuracy: 0.9910 - precision: 0.9905 - recall: 0.9914 - mean_squared_error: 0.0076 - auc: 0.9985 - val_loss: 0.0840 - val_accuracy: 0.9796 - val_precision: 0.9762 - val_recall: 0.9830 - val_mean_squared_error: 0.0176 - val_auc: 0.9930
-----------------
Memory Ram: 18.29 (Free: 71.4%)
-----------------
Epoch 76/100
4460/4460 [==============================] - 124s 28ms/step - loss: 0.0319 - accuracy: 0.9908 - precision: 0.9901 - recall: 0.9915 - mean_squared_error: 0.0076 - auc: 0.9984 - val_loss: 0.0859 - val_accuracy: 0.9793 - val_precision: 0.9818 - val_recall: 0.9767 - val_mean_squared_error: 0.0173 - val_auc: 0.9923
-----------------
Memory Ram: 18.19 (Free: 71.56%)
-----------------
Epoch 77/100
4460/4460 [==============================] - 125s 28ms/step - loss: 0.0319 - accuracy: 0.9909 - precision: 0.9902 - recall: 0.9916 - mean_squared_error: 0.0075 - auc: 0.9984 - val_loss: 0.0920 - val_accuracy: 0.9792 - val_precision: 0.9827 - val_recall: 0.9756 - val_mean_squared_error: 0.0174 - val_auc: 0.9926
-----------------
Memory Ram: 18.36 (Free: 71.29%)
-----------------
Epoch 78/100
4460/4460 [==============================] - 119s 27ms/step - loss: 0.0319 - accuracy: 0.9910 - precision: 0.9906 - recall: 0.9915 - mean_squared_error: 0.0076 - auc: 0.9985 - val_loss: 0.0808 - val_accuracy: 0.9786 - val_precision: 0.9763 - val_recall: 0.9811 - val_mean_squared_error: 0.0173 - val_auc: 0.9934
-----------------
Memory Ram: 16.2 (Free: 74.67%)
-----------------
Epoch 79/100
4460/4460 [==============================] - 120s 27ms/step - loss: 0.0313 - accuracy: 0.9911 - precision: 0.9902 - recall: 0.9918 - mean_squared_error: 0.0075 - auc: 0.9985 - val_loss: 0.0878 - val_accuracy: 0.9773 - val_precision: 0.9769 - val_recall: 0.9778 - val_mean_squared_error: 0.0189 - val_auc: 0.9929
-----------------
Memory Ram: 16.42 (Free: 74.32%)
-----------------
Epoch 80/100
4460/4460 [==============================] - 122s 27ms/step - loss: 0.0308 - accuracy: 0.9907 - precision: 0.9898 - recall: 0.9916 - mean_squared_error: 0.0075 - auc: 0.9987 - val_loss: 0.0916 - val_accuracy: 0.9756 - val_precision: 0.9680 - val_recall: 0.9836 - val_mean_squared_error: 0.0201 - val_auc: 0.9926
-----------------
Memory Ram: 16.26 (Free: 74.57%)
-----------------
Epoch 81/100
4460/4460 [==============================] - 121s 27ms/step - loss: 0.0298 - accuracy: 0.9917 - precision: 0.9907 - recall: 0.9927 - mean_squared_error: 0.0070 - auc: 0.9987 - val_loss: 0.0833 - val_accuracy: 0.9794 - val_precision: 0.9794 - val_recall: 0.9795 - val_mean_squared_error: 0.0173 - val_auc: 0.9931
-----------------
Memory Ram: 16.32 (Free: 74.48%)
-----------------
Epoch 82/100
4460/4460 [==============================] - 120s 27ms/step - loss: 0.0295 - accuracy: 0.9920 - precision: 0.9914 - recall: 0.9927 - mean_squared_error: 0.0068 - auc: 0.9985 - val_loss: 0.0991 - val_accuracy: 0.9794 - val_precision: 0.9894 - val_recall: 0.9692 - val_mean_squared_error: 0.0180 - val_auc: 0.9926
-----------------
Memory Ram: 16.47 (Free: 74.24%)
-----------------
Epoch 83/100
4460/4460 [==============================] - 129s 29ms/step - loss: 0.0289 - accuracy: 0.9919 - precision: 0.9909 - recall: 0.9928 - mean_squared_error: 0.0068 - auc: 0.9988 - val_loss: 0.0891 - val_accuracy: 0.9790 - val_precision: 0.9771 - val_recall: 0.9811 - val_mean_squared_error: 0.0178 - val_auc: 0.9926
-----------------
Memory Ram: 16.88 (Free: 73.6%)
-----------------
Epoch 84/100
4460/4460 [==============================] - 130s 29ms/step - loss: 0.0277 - accuracy: 0.9920 - precision: 0.9912 - recall: 0.9928 - mean_squared_error: 0.0066 - auc: 0.9988 - val_loss: 0.0909 - val_accuracy: 0.9771 - val_precision: 0.9720 - val_recall: 0.9824 - val_mean_squared_error: 0.0189 - val_auc: 0.9927
-----------------
Memory Ram: 16.91 (Free: 73.55%)
-----------------
Epoch 85/100
4460/4460 [==============================] - 129s 29ms/step - loss: 0.0294 - accuracy: 0.9915 - precision: 0.9901 - recall: 0.9930 - mean_squared_error: 0.0070 - auc: 0.9987 - val_loss: 0.0865 - val_accuracy: 0.9784 - val_precision: 0.9817 - val_recall: 0.9749 - val_mean_squared_error: 0.0176 - val_auc: 0.9927
-----------------
Memory Ram: 16.88 (Free: 73.61%)
-----------------
Epoch 86/100
4460/4460 [==============================] - 128s 29ms/step - loss: 0.0270 - accuracy: 0.9927 - precision: 0.9915 - recall: 0.9937 - mean_squared_error: 0.0063 - auc: 0.9988 - val_loss: 0.0906 - val_accuracy: 0.9788 - val_precision: 0.9783 - val_recall: 0.9792 - val_mean_squared_error: 0.0178 - val_auc: 0.9929
-----------------
Memory Ram: 16.65 (Free: 73.96%)
-----------------
Epoch 87/100
4460/4460 [==============================] - 130s 29ms/step - loss: 0.0268 - accuracy: 0.9926 - precision: 0.9915 - recall: 0.9936 - mean_squared_error: 0.0063 - auc: 0.9989 - val_loss: 0.0918 - val_accuracy: 0.9793 - val_precision: 0.9775 - val_recall: 0.9812 - val_mean_squared_error: 0.0176 - val_auc: 0.9922
-----------------
Memory Ram: 16.75 (Free: 73.81%)
-----------------
Epoch 88/100
4460/4460 [==============================] - 128s 29ms/step - loss: 0.0256 - accuracy: 0.9927 - precision: 0.9915 - recall: 0.9940 - mean_squared_error: 0.0061 - auc: 0.9990 - val_loss: 0.0886 - val_accuracy: 0.9798 - val_precision: 0.9825 - val_recall: 0.9769 - val_mean_squared_error: 0.0172 - val_auc: 0.9922
-----------------
Memory Ram: 16.95 (Free: 73.5%)
-----------------
Epoch 89/100
4460/4460 [==============================] - 129s 29ms/step - loss: 0.0261 - accuracy: 0.9928 - precision: 0.9915 - recall: 0.9940 - mean_squared_error: 0.0061 - auc: 0.9989 - val_loss: 0.0990 - val_accuracy: 0.9802 - val_precision: 0.9862 - val_recall: 0.9741 - val_mean_squared_error: 0.0172 - val_auc: 0.9918
-----------------
Memory Ram: 17.11 (Free: 73.24%)
-----------------
Epoch 90/100
4460/4460 [==============================] - 126s 28ms/step - loss: 0.0267 - accuracy: 0.9925 - precision: 0.9914 - recall: 0.9936 - mean_squared_error: 0.0063 - auc: 0.9989 - val_loss: 0.0919 - val_accuracy: 0.9751 - val_precision: 0.9690 - val_recall: 0.9816 - val_mean_squared_error: 0.0199 - val_auc: 0.9924
-----------------
Memory Ram: 16.88 (Free: 73.6%)
-----------------
Epoch 91/100
4460/4460 [==============================] - 126s 28ms/step - loss: 0.0245 - accuracy: 0.9933 - precision: 0.9921 - recall: 0.9945 - mean_squared_error: 0.0057 - auc: 0.9990 - val_loss: 0.0946 - val_accuracy: 0.9784 - val_precision: 0.9808 - val_recall: 0.9759 - val_mean_squared_error: 0.0180 - val_auc: 0.9922
-----------------
Memory Ram: 16.95 (Free: 73.5%)
-----------------
Epoch 92/100
4460/4460 [==============================] - 132s 30ms/step - loss: 0.0266 - accuracy: 0.9928 - precision: 0.9916 - recall: 0.9941 - mean_squared_error: 0.0062 - auc: 0.9988 - val_loss: 0.0924 - val_accuracy: 0.9780 - val_precision: 0.9787 - val_recall: 0.9774 - val_mean_squared_error: 0.0184 - val_auc: 0.9925
-----------------
Memory Ram: 16.96 (Free: 73.48%)
-----------------
Epoch 93/100
4460/4460 [==============================] - 133s 30ms/step - loss: 0.0259 - accuracy: 0.9927 - precision: 0.9913 - recall: 0.9941 - mean_squared_error: 0.0060 - auc: 0.9988 - val_loss: 0.0903 - val_accuracy: 0.9798 - val_precision: 0.9811 - val_recall: 0.9783 - val_mean_squared_error: 0.0171 - val_auc: 0.9926
-----------------
Memory Ram: 17.02 (Free: 73.38%)
-----------------
Epoch 94/100
4460/4460 [==============================] - 129s 29ms/step - loss: 0.0238 - accuracy: 0.9938 - precision: 0.9924 - recall: 0.9952 - mean_squared_error: 0.0054 - auc: 0.9989 - val_loss: 0.1034 - val_accuracy: 0.9727 - val_precision: 0.9635 - val_recall: 0.9825 - val_mean_squared_error: 0.0220 - val_auc: 0.9923
-----------------
Memory Ram: 16.85 (Free: 73.65%)
-----------------
Epoch 95/100
4460/4460 [==============================] - 129s 29ms/step - loss: 0.0238 - accuracy: 0.9933 - precision: 0.9915 - recall: 0.9951 - mean_squared_error: 0.0056 - auc: 0.9991 - val_loss: 0.1061 - val_accuracy: 0.9806 - val_precision: 0.9881 - val_recall: 0.9729 - val_mean_squared_error: 0.0168 - val_auc: 0.9915
-----------------
Memory Ram: 17.19 (Free: 73.12%)
-----------------
Epoch 96/100
4460/4460 [==============================] - 126s 28ms/step - loss: 0.0225 - accuracy: 0.9939 - precision: 0.9927 - recall: 0.9950 - mean_squared_error: 0.0051 - auc: 0.9990 - val_loss: 0.1073 - val_accuracy: 0.9770 - val_precision: 0.9738 - val_recall: 0.9804 - val_mean_squared_error: 0.0194 - val_auc: 0.9912
-----------------
Memory Ram: 17.3 (Free: 72.95%)
-----------------
Epoch 97/100
4460/4460 [==============================] - 127s 28ms/step - loss: 0.0227 - accuracy: 0.9939 - precision: 0.9926 - recall: 0.9952 - mean_squared_error: 0.0053 - auc: 0.9992 - val_loss: 0.0945 - val_accuracy: 0.9784 - val_precision: 0.9779 - val_recall: 0.9788 - val_mean_squared_error: 0.0182 - val_auc: 0.9925
-----------------
Memory Ram: 17.31 (Free: 72.92%)
-----------------
Epoch 98/100
4460/4460 [==============================] - 126s 28ms/step - loss: 0.0244 - accuracy: 0.9931 - precision: 0.9918 - recall: 0.9944 - mean_squared_error: 0.0058 - auc: 0.9990 - val_loss: 0.0991 - val_accuracy: 0.9790 - val_precision: 0.9799 - val_recall: 0.9782 - val_mean_squared_error: 0.0177 - val_auc: 0.9923
-----------------
Memory Ram: 17.4 (Free: 72.8%)
-----------------
Epoch 99/100
4460/4460 [==============================] - 129s 29ms/step - loss: 0.0208 - accuracy: 0.9942 - precision: 0.9930 - recall: 0.9955 - mean_squared_error: 0.0049 - auc: 0.9993 - val_loss: 0.1206 - val_accuracy: 0.9799 - val_precision: 0.9877 - val_recall: 0.9719 - val_mean_squared_error: 0.0176 - val_auc: 0.9906
-----------------
Memory Ram: 17.22 (Free: 73.07%)
-----------------
Epoch 100/100
4460/4460 [==============================] - 128s 29ms/step - loss: 0.0231 - accuracy: 0.9937 - precision: 0.9925 - recall: 0.9949 - mean_squared_error: 0.0054 - auc: 0.9991 - val_loss: 0.0988 - val_accuracy: 0.9780 - val_precision: 0.9791 - val_recall: 0.9769 - val_mean_squared_error: 0.0181 - val_auc: 0.9921
-----------------
Memory Ram: 18.21 (Free: 71.52%)
-----------------
  - Finished
  - Time:  13839.99s  (230.66649999999998m)(3.8444416666666665h)

  - Saving model...
      - Model architecture saved in: D:/data_tb/temps/result-execution__25-10-2022_165320__MODEL.h5 

      - Image architecture saved in: D:/data_tb/temps/result-execution__25-10-2022_165320__MODEL-architecture.png 

# Evaluate model in validation data
print('  - Evaluate model in validation data')
out_loss_, out_acc, out_precision, out_recall, out_mse, out_auc = model.evaluate_generator(validation_datagen, steps=len(validation_datagen), verbose=1)
print('  - validation Accuracy: %.3f' % (out_acc * 100))

# ================================== 
# Save train/validation results and settings
print("  - Saving train/validation results and settings...") 
modelConfigNameSaved = OUT_PATH + FILE_ID_RUN +'_result-training-validation.txt'
f = open(modelConfigNameSaved, "a")   
f.write("---------------------------------------------------------------------------\n")
f.write("Evaluate model in validation data\n")
f.write("* Accuracy train/validation  : "+"{:.4f}".format(out_acc)+"\n")  
f.write("* Precision train/validation : "+"{:.4f}".format(out_precision)+"\n")  
f.write("* Recall train/validation    : "+"{:.4f}".format(out_recall)+"\n")  
f.write("* MSE train/validation       : "+"{:.4f}".format(out_mse)+"\n")  
f.write("* AUC train/validation       : "+"{:.4f}".format(out_auc)+"\n")  
f.write("\n\n")
f.write("---------------------------------------------------------------------------\n")
f.write("* Time training: "+time_model_train+"\n")
f.write("* Path_dataset : "+PATH_DATASET+"\n")
f.write("* Batch_size   : "+str(BATCH_SIZE)+"\n")
f.write("* Epochs       : "+str(EPOCHS_SIZE)+"\n")
f.write("* Run in       : "+type_exec+"\n") 
f.write("\n\n")
f.write("---------------------------------------------------------------------------\n")
f.write("* Model architecture:\n" )
f.write( json.dumps(model.get_config(), indent=3, sort_keys=True) )

tmp_smry = StringIO()
model.summary(print_fn=lambda x: tmp_smry.write(x + '\n'))
f.write("\nSummary:\n")
f.write(tmp_smry.getvalue()+"\n")

f.close()
  
print("Saved file content:")   
a_file = open(modelConfigNameSaved)
file_contents = a_file.read()
print(file_contents)
a_file.close() 
# ==============================================================================================================
# ==============================================================================================================

  - Evaluate model in validation data
  3/609 [..............................] - ETA: 19s - loss: 0.0308 - accuracy: 0.9867 - precision: 1.0000 - recall: 0.9750 - mean_squared_error: 0.0095 - auc: 0.9993

C:\Users\Pichau\AppData\Roaming\Python\Python38\site-packages\tensorflow\python\keras\engine\training.py:1877: UserWarning: `Model.evaluate_generator` is deprecated and will be removed in a future version. Please use `Model.evaluate`, which supports generators.
  warnings.warn('`Model.evaluate_generator` is deprecated and '

609/609 [==============================] - 12s 20ms/step - loss: 0.0988 - accuracy: 0.9780 - precision: 0.9791 - recall: 0.9769 - mean_squared_error: 0.0181 - auc: 0.9921
  - validation Accuracy: 97.798
  - Saving train/validation results and settings...
Saved file content:
---------------------------------------------------------------------------
Evaluate model in validation data
* Accuracy train/validation  : 0.9780
* Precision train/validation : 0.9791
* Recall train/validation    : 0.9769
* MSE train/validation       : 0.0181
* AUC train/validation       : 0.9921


---------------------------------------------------------------------------
* Time training: 13839.99s  (230.66649999999998m)(3.8444416666666665h)
* Path_dataset : D:/tuberculosis-phonecamera/dataset_1/data/
* Batch_size   : 25
* Epochs       : 100
* Run in       : GPU


---------------------------------------------------------------------------
* Model architecture:
{
   "layers": [
      {
         "class_name": "InputLayer",
         "config": {
            "batch_input_shape": [
               null,
               20,
               20,
               3
            ],
            "dtype": "float32",
            "name": "conv2d_input",
            "ragged": false,
            "sparse": false
         }
      },
      {
         "class_name": "Conv2D",
         "config": {
            "activation": "relu",
            "activity_regularizer": null,
            "batch_input_shape": [
               null,
               20,
               20,
               3
            ],
            "bias_constraint": null,
            "bias_initializer": {
               "class_name": "Zeros",
               "config": {}
            },
            "bias_regularizer": null,
            "data_format": "channels_last",
            "dilation_rate": [
               1,
               1
            ],
            "dtype": "float32",
            "filters": 7,
            "groups": 1,
            "kernel_constraint": null,
            "kernel_initializer": {
               "class_name": "GlorotUniform",
               "config": {
                  "seed": null
               }
            },
            "kernel_regularizer": null,
            "kernel_size": [
               3,
               3
            ],
            "name": "conv2d",
            "padding": "valid",
            "strides": [
               1,
               1
            ],
            "trainable": true,
            "use_bias": true
         }
      },
      {
         "class_name": "MaxPooling2D",
         "config": {
            "data_format": "channels_last",
            "dtype": "float32",
            "name": "max_pooling2d",
            "padding": "valid",
            "pool_size": [
               2,
               2
            ],
            "strides": [
               2,
               2
            ],
            "trainable": true
         }
      },
      {
         "class_name": "Conv2D",
         "config": {
            "activation": "relu",
            "activity_regularizer": null,
            "bias_constraint": null,
            "bias_initializer": {
               "class_name": "Zeros",
               "config": {}
            },
            "bias_regularizer": null,
            "data_format": "channels_last",
            "dilation_rate": [
               1,
               1
            ],
            "dtype": "float32",
            "filters": 12,
            "groups": 1,
            "kernel_constraint": null,
            "kernel_initializer": {
               "class_name": "HeUniform",
               "config": {
                  "seed": null
               }
            },
            "kernel_regularizer": null,
            "kernel_size": [
               3,
               3
            ],
            "name": "conv2d_1",
            "padding": "same",
            "strides": [
               1,
               1
            ],
            "trainable": true,
            "use_bias": true
         }
      },
      {
         "class_name": "Flatten",
         "config": {
            "data_format": "channels_last",
            "dtype": "float32",
            "name": "flatten",
            "trainable": true
         }
      },
      {
         "class_name": "Dense",
         "config": {
            "activation": "relu",
            "activity_regularizer": null,
            "bias_constraint": null,
            "bias_initializer": {
               "class_name": "Zeros",
               "config": {}
            },
            "bias_regularizer": null,
            "dtype": "float32",
            "kernel_constraint": null,
            "kernel_initializer": {
               "class_name": "GlorotUniform",
               "config": {
                  "seed": null
               }
            },
            "kernel_regularizer": null,
            "name": "dense",
            "trainable": true,
            "units": 500,
            "use_bias": true
         }
      },
      {
         "class_name": "Dense",
         "config": {
            "activation": "sigmoid",
            "activity_regularizer": null,
            "bias_constraint": null,
            "bias_initializer": {
               "class_name": "Zeros",
               "config": {}
            },
            "bias_regularizer": null,
            "dtype": "float32",
            "kernel_constraint": null,
            "kernel_initializer": {
               "class_name": "GlorotUniform",
               "config": {
                  "seed": null
               }
            },
            "kernel_regularizer": null,
            "name": "dense_1",
            "trainable": true,
            "units": 1,
            "use_bias": true
         }
      }
   ],
   "name": "sequential"
}
Summary:
Model: "sequential"
_________________________________________________________________
Layer (type)                 Output Shape              Param #   
=================================================================
conv2d (Conv2D)              (None, 18, 18, 7)         196       
_________________________________________________________________
max_pooling2d (MaxPooling2D) (None, 9, 9, 7)           0         
_________________________________________________________________
conv2d_1 (Conv2D)            (None, 9, 9, 12)          768       
_________________________________________________________________
flatten (Flatten)            (None, 972)               0         
_________________________________________________________________
dense (Dense)                (None, 500)               486500    
_________________________________________________________________
dense_1 (Dense)              (None, 1)                 501       
=================================================================
Total params: 487,965
Trainable params: 487,965
Non-trainable params: 0
_________________________________________________________________

5. Evaluate the model on the test dataset in detail

# ==============================================================================================================
# ==============================================================================================================
# Evaluate the model on the test dataset in detail            
# ==============================================================================================================
# ==============================================================================================================
# Copy datagen
data_run_datagen        = test_datagen 
data_run_datagen_labels = data_run_datagen.classes
data_run_datagen.reset()  
    
modelTestFileSaved = OUT_PATH + FILE_ID_RUN +'test-result.txt'
f = open(modelTestFileSaved, "a")  
f.write("Result in test set\n\n") 

 
print('\n* Evaluating test set...') 

data_run_datagen.reset()   
Y_pred = model.predict(data_run_datagen, steps=len(data_run_datagen), verbose=1)
data_run_datagen.reset()  
 
Y_pred = np.around(Y_pred)
Y_pred = Y_pred.astype(int)
y_pred = np.squeeze(np.asarray(Y_pred))
print('  - Ok') 
 

# ----------------------
# Confusion matrix
confusion = confusion_matrix(data_run_datagen_labels, y_pred)
f.write('Confusion Matrix\n')
f.write(str(confusion))
f.write('\n\n')

np.set_printoptions(precision=2)

# Plot non-normalized confusion matrix
ax = plot_confusion_matrix(data_run_datagen_labels, y_pred, classes=np.array(['bacillus', 'no-bacillus']),
                      title='Confusion matrix, without normalization (test dataset)') 
pyplot.savefig(  OUT_PATH + FILE_ID_RUN +'CM-test.png' )

 
# Plot normalized confusion matrix
plot_confusion_matrix(data_run_datagen_labels, y_pred, classes= np.array(['bacillus', 'no-bacillus']), normalize=True,
                      title='Normalized confusion matrix (test dataset)')
pyplot.savefig(  OUT_PATH + FILE_ID_RUN +'CM-test-normalized.png' )

  
target_names = ['bacillus', 'no-bacillus']
f.write( classification_report(data_run_datagen_labels, y_pred, target_names=target_names) +"\n") 
# ----------------------


# ----------------------
# AUC 
fpr_keras, tpr_keras, thresholds_keras = roc_curve(data_run_datagen_labels, y_pred)
auc_keras = auc(fpr_keras, tpr_keras)
# ----------------------


# ----------------------
# Details 
report_tp          = confusion[1][1]     
report_tn          = confusion[0][0]    
report_fp          = confusion[0][1]      
report_fn          = confusion[1][0]   
report_accuracy    = metrics.accuracy_score(data_run_datagen_labels, y_pred) 
report_recall      = metrics.recall_score(data_run_datagen_labels, y_pred)
report_specificity = (report_tn / float(report_tn + report_fp))
report_precision = metrics.precision_score(data_run_datagen_labels, y_pred) 
report_f1        = metrics.f1_score(data_run_datagen_labels, y_pred) 
report_mse = metrics.mean_squared_error(data_run_datagen_labels, y_pred) 
report_auc = auc_keras
 
f.write("TP         : "+'{:7.0f}'.format(report_tp) +" # true positive (TP) "+"\n") 
f.write("TN         : "+'{:7.0f}'.format(report_tn) +" # true negative (TN)"+"\n") 
f.write("FP         : "+'{:7.0f}'.format(report_fp) +" # false positive (FP)"+"\n") 
f.write("FN         : "+'{:7.0f}'.format(report_fn) +" # false negative (FN)"+"\n")  
f.write("Accuracy   : "+'{:7.4f}'.format(report_accuracy)  +" # accuracy"+"\n") 
f.write("Recall     : "+'{:7.4f}'.format(report_recall)    +" # sensitivity, recall, hit rate, or true positive rate (TPR)"+"\n") 
f.write("Specificity: "+'{:7.4f}'.format(report_specificity) +" # specificity, selectivity or true negative rate (TNR) "+"\n") 
f.write("Precision  : "+'{:7.4f}'.format(report_precision) +" # precision or positive predictive value (PPV)"+"\n") 
f.write("F1 score   : "+'{:7.4f}'.format(report_f1) +" # F1 score"+"\n") 
f.write("MSE        : "+'{:7.4f}'.format(report_mse) +" # mean squared error"+"\n") 
f.write("AUC        : "+'{:7.4f}'.format(report_auc) +" # Area Under the Receiver Operating Characteristic Curve (ROC AUC)"+"\n") 
# ----------------------

f.close()

print("  - Saved file content:\n")   
a_file = open(modelTestFileSaved)
file_contents = a_file.read()
print(file_contents)
a_file.close()
# ==============================================================================================================
# ==============================================================================================================

* Evaluating test set...
1252/1252 [==============================] - 37s 29ms/step
  - Ok
  - Saved file content:

Result in test set

Confusion Matrix
[[15332   308]
 [  314 15326]]

              precision    recall  f1-score   support

    bacillus       0.98      0.98      0.98     15640
 no-bacillus       0.98      0.98      0.98     15640

    accuracy                           0.98     31280
   macro avg       0.98      0.98      0.98     31280
weighted avg       0.98      0.98      0.98     31280

TP         :   15326 # true positive (TP) 
TN         :   15332 # true negative (TN)
FP         :     308 # false positive (FP)
FN         :     314 # false negative (FN)
Accuracy   :  0.9801 # accuracy
Recall     :  0.9799 # sensitivity, recall, hit rate, or true positive rate (TPR)
Specificity:  0.9803 # specificity, selectivity or true negative rate (TNR) 
Precision  :  0.9803 # precision or positive predictive value (PPV)
F1 score   :  0.9801 # F1 score
MSE        :  0.0199 # mean squared error
AUC        :  0.9801 # Area Under the Receiver Operating Characteristic Curve (ROC AUC)

Website references and acknowledgments

https://www.tensorflow.org/tutorials/images/cnn

https://medium.com/ibm-data-ai/memory-hygiene-with-tensorflow-during-model-training-and-deployment-for-inference-45cf49a15688

https://machinelearningmastery.com/visualize-deep-learning-neural-network-model-keras/

https://github.com/tensorflow/tensorflow/issues/31312

https://www.dlology.com/blog/simple-guide-on-how-to-generate-roc-plot-for-keras-classifier/

https://stackoverflow.com/questions/71958700/valueerror-input-0-of-layer-lstm-14-is-incompatible-with-the-layer-expected-nd

And many other websites...