Tuple index out of range when trying to fit the CNN model - python-3.x

Question

The dataset that I am using is the standard chest Xray dataset https://www.kaggle.com/datasets/paultimothymooney/chest-xray-pneumonia. Have been getting this error (tuple index out of range) while fitting the CNN model. Is there a way to circumvent this issue? I suppose argument "validation_data" needs to be appended in some way.
import os
import glob
import cv2
import numpy as np
import pandas as pd
from PIL import Image
import tensorflow as tf
import random
#from pathlib import path
import pathlib2 as pathlib
from pathlib2 import Path
#from keras.models import sequential, Model, load_model
from tensorflow.keras.models import Sequential, Model, load_model
from tensorflow.keras.applications.vgg16 import VGG16, preprocess_input
from tensorflow.keras.preprocessing.image import ImageDataGenerator, load_img, img_to_array
from tensorflow.keras.layers import Conv2D, MaxPooling2D, Dense, Dropout, Input, Flatten, Activation
from tensorflow.keras.optimizers import Adam, SGD, RMSprop
from tensorflow.keras.callbacks import Callback, EarlyStopping
from tensorflow.keras.utils import to_categorical
from sklearn.metrics import confusion_matrix
from tensorflow.keras import backend as K
import tensorflow as tf
import matplotlib.pyplot as plt
from tensorflow.keras.preprocessing import image
%matplotlib inline
import shutup; shutup.please()
# DATA PATH #
print (os.listdir("C:/Users/Syd_R/OneDrive/Desktop/Peeumonia_data/archive/chest_xray/chest_xray/"))
data_dir = Path("C:/Users/Syd_R/OneDrive/Desktop/Peeumonia_data/archive/chest_xray/chest_xray/")
train_dir = data_dir/'train'
val_dir = data_dir/'val'
test_dir = data_dir/'test'
# LOAD TRAINING DATA TO DATAFRAME #
def load_train():
normal_cases_dir =train_dir/'NORMAL'
pneumonia_cases_dir = train_dir/ 'PNEUMONIA'
# list of all images
normal_cases = normal_cases_dir.glob('*.jpeg')
pneumonia_cases = pneumonia_cases_dir.glob('*.jpeg')
train_data=[]
train_label=[]
for img in normal_cases:
train_data.append(img)
train_label.append('NORMAL')
for img in pneumonia_cases:
train_data.append(img)
train_label.append('PNEUMONIA')
df=pd.DataFrame(train_data)
df.columns = ['images']
df['labels'] = train_label
df=df.sample(frac=1).reset_index(drop=True)
return df
train_data = load_train()
train_data.shape
# VIZUALIZE THE AMOUNT OF TRAINING DATA WITH LABELS #
plt.bar(train_data['labels'].value_counts().index,train_data['labels'].value_counts().values)
plt.show()
# VIZUALIZE THE TRAINING IMAGE DATA BY RANDOM SAMPLING#
plt.figure(figsize=(10,5))
for i in range(10):
ax = plt.subplot(2,5,i+1)
num= random.randint(0, 5000+i)
im=train_data.loc[num].at['images']
im1=train_data.loc[num].at['labels']
img = cv2.imread(str(im))
img = cv2.resize(img, (224,224))
plt.imshow(img)
plt.title(im1)
plt.axis("off")
print(num)
# DATA PRE-PROCESSING #
def prepare_and_load(isval=True):
if isval==True:
normal_dir=val_dir/'NORMAL'
pneumonia_dir=val_dir/'PNEUMONIA'
else:
normal_dir=test_dir/'NORMAL'
pneumonia_dir=test_dir/'PNEUMONIA'
normal_cases = normal_dir.glob('*.jpeg')
pneumonia_cases = pneumonia_dir.glob('*.jpeg')
data,labels=([] for x in range (2))
def prepare(case):
for img in case:
img = cv2.imread(str(img))
img = cv2.resize(img, (224,224))
if img.shape[2] ==1:
img = np.dstack([img, img, img])
img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)
img = img.astype(np.float32)/255
if case==normal_cases:
label = to_categorical(0, num_classes=2)
else:
label = to_categorical(1, num_classes=2)
data.append(img)
labels.append(label)
return data,labels
prepare(normal_cases)
d,l=prepare(pneumonia_cases)
d=np.array(d)
l=np.array(1)
return d,l
val_data,val_labels = prepare_and_load(isval=True)
test_data,test_labels = prepare_and_load(isval=False)
print('Number of test images -->', len(test_data))
print('Number of validation images -->', len(val_data))
# DEFINE A FUNCTION TO GENERATE BATCHES FROM TRAINING IMAGES #
def data_gen(data, batch_size):
# Get tiotal number of samples in the data
n= len(data)
steps = n//batch_size
# Define two numpy arrays for containing batch data and labels
batch_data = np.zeros((batch_size, 224, 224, 3), dtype=np.float32)
batch_labels = np.zeros((batch_size,2), dtype=np.float32)
# Get a numpy array of all the indices of the input data
indices = np.arange(n)
# Initalize a counter
i=0
while True:
np.random.shuffle(indices)
# Get the next batch
count = 0
next_batch =indices [(i*batch_size): (i+1)*batch_size]
for j,idx in enumerate(next_batch):
img_name = data.iloc[idx]['images']
label = data.iloc[idx]['images']
if label=='NORMAL':
label=0
else:
label=1
# one hot encoding
encoded_label = to_categorical(label, num_classes=2)
# read the image and resize
img = cv2.imread(str(img_name))
img = cv2.resize(img,(224,224))
# check if it's grayscale
if img.shape[2]==1:
img = np.dstack([img, img, img])
# cv2 reads in BGR mode by default
orig_imag = cv2.cvtColor(img, cv2. COLOR_BGR2RGB)
# normalize the image pixels
orig_img = img.astype(np.float32)/255
batch_data[count]= orig_img
batch_labels[count] = encoded_label
count+=1
if count==batch_size-1:
break
i+=1
yield batch_data, batch_labels
if i>=steps:
i=0
# DEFINE THE CNN MODEL #
model = Sequential()
model.add(Conv2D(32, (3,3), input_shape=(224, 224, 3)))
model.add(Activation('relu'))
model.add(MaxPooling2D(pool_size=(2,2)))
model.add(Conv2D(32, (3,3)))
model.add(Activation('relu'))
model.add(MaxPooling2D(pool_size=(2,2)))
model.add(Conv2D(64, (3,3)))
model.add(Activation('relu'))
model.add(MaxPooling2D(pool_size=(2,2)))
model.add(Flatten()) # this converts our 3D feature maps to 1D feature vectors
model.add(Dense(64))
model.add(Activation('relu'))
model.add(Dense(2))
model.add(Activation('softmax'))
# DEFINE PARAMETERS FOR THE CNN MODEL #
batch_size = 64
nb_epochs = 3
# Get a train data generator
train_data_gen = data_gen(data= train_data, batch_size=batch_size)
# DEFINE THE NUMBER OF TRAINING STEPS #
nb_train_steps = train_data.shape[0]//batch_size
print("Number of training and validation steps: {} and {}".format(nb_train_steps, len(val_data)))
model.compile(loss='binary_crossentropy',
optimizer='rmsprop',
metrics=['accuracy'])
# FIT THE MODEL #
history = model.fit_generator(train_data_gen,
epochs=nb_epochs,
steps_per_epoch=nb_train_steps,
validation_data=(val_data, val_labels))