I am building a keras model. The features are coming from pandas.DataFrame. I build the tf.Dataset through from_generator API. I followed this page to process the categorical string features.
output_sig= ...
features = [...]
def iter_to_gen(it):
def f():
for x in it:
# x is a list, with the last element being the label
key_to_feature = {key: x[i] for i, key in enumerate(features)}
yield key_to_feature, x[-1]
return f
train_ds = tf.data.Dataset.from_generator( iter_to_gen(map(tuple, train_data.values)), output_signature=output_sig, name='train').batch(batch_size)
def get_category_encoding_layer(name, dataset, dtype, max_tokens=None):
# Create a layer that turns strings into integer indices.
if dtype == 'string':
index = layers.StringLookup(max_tokens=max_tokens)
# Otherwise, create a layer that turns integer values into integer indices.
else:
index = layers.IntegerLookup(max_tokens=max_tokens)
# Prepare a `tf.data.Dataset` that only yields the feature.
feature_ds = dataset.map(lambda x, y : x[name])
# Learn the set of possible values and assign them a fixed integer index.
index.adapt(feature_ds)
# Encode the integer indices.
encoder = layers.CategoryEncoding(num_tokens=index.vocabulary_size())
# Apply multi-hot encoding to the indices. The lambda function captures the
# layer, so you can use them, or include them in the Keras Functional model later.
return lambda feature: encoder(index(feature))
all_inputs = []
encoded_features = []
categorical_cols = ['feature_A']
for header in categorical_cols:
if header == 'feature_A':
categorical_col = tf.keras.Input(shape=(None,), name=header, dtype='string')
else:
categorical_col = tf.keras.Input(shape=(1,), name=header, dtype='string')
encoding_layer = get_category_encoding_layer(name=header,
dataset=train_ds,
dtype='string',
max_tokens=50) # tune the max tokens
encoded_categorical_col = encoding_layer(categorical_col)
all_inputs.append(categorical_col)
encoded_features.append(encoded_categorical_col)
all_features = tf.keras.layers.concatenate(encoded_features)
print(all_features.shape)
x = tf.keras.layers.Dense(32, activation="relu")(all_features)
# x = tf.keras.layers.Dropout(0.5)(x)
output = tf.keras.layers.Dense(num_class)(x)
model = tf.keras.Model(all_inputs, output)
model.compile(optimizer='SGD',
loss=tf.keras.losses.CategoricalCrossentropy(from_logits=True),
metrics=["accuracy"],
run_eagerly=True)
model.fit(train_ds, epochs=10, verbose=2) <------ ValueError: Unexpected result of #`train_function` (Empty logs). Please use `Model.compile(..., run_eagerly=True)`, or #`tf.config.run_functions_eagerly(True)` for more information of where went wrong, or file a #issue/bug to `tf.keras`.
And then if I reproduce the train_ds and skip directly to run model.fit, it would run only 2 epochs and end. I am wondering why is it.
Epoch 1/10
4984/4984 - 71s - loss: 2.5564 - accuracy: 0.4191 - 71s/epoch - 14ms/step
Epoch 2/10
4984/4984 - 0s - loss: 0.0000e+00 - accuracy: 0.0000e+00 - 12ms/epoch - 2us/step
<keras.callbacks.History at 0x....>
I found the first error was raised because model.fit got a empty dataset. I also verified the size of dataset by dataset.as_numpy_array() and it is empty. I am wondering why.
Thanks.
Related
I've been looking through samples but am unable to understand how to integrate the precision, recall and f1 metrics for my model. My code is as follows:
for epoch in range(num_epochs):
#Calculate Accuracy (stack tutorial no n_total)
n_correct = 0
n_total = 0
for i, (words, labels) in enumerate(train_loader):
words = words.to(device)
labels = labels.to(dtype=torch.long).to(device)
# Forward pass
outputs = model(words)
loss = criterion(outputs, labels)
# Backward and optimize
optimizer.zero_grad()
loss.backward()
optimizer.step()
#feedforward tutorial solution
_, predicted = torch.max(outputs, 1)
n_correct += (predicted == labels).sum().item()
n_total += labels.shape[0]
accuracy = 100 * n_correct/n_total
#Push to matplotlib
train_losses.append(loss.item())
train_epochs.append(epoch)
train_acc.append(accuracy)
#Loss and Accuracy
if (epoch+1) % 10 == 0:
print(f'Epoch [{epoch+1}/{num_epochs}], Loss: {loss.item():.2f}, Acc: {accuracy:.2f}')
Since you have the predicted and the labels variables, you can aggregate them during the epoch loop and convert them to numpy arrays to calculate the required metrics.
At the beginning of the epoch, initialize two empty lists; one for true labels and one for ground truth labels.
for epoch in range(num_epochs):
predicted_labels, ground_truth_labels = [], []
...
Then, keep appending the respective entries to each list during the epoch:
...
_, predicted = torch.max(outputs, 1)
n_correct += (predicted == labels).sum().item()
# appending
predicted_labels.append(predicted.cpu().detach().numpy())
ground_truth_labels.append(labels.cpu().detach().numpy())
...
Then, at the epoch end, you could use precision_recall_fscore_support with predicted_labels and ground_truth_labels as inputs.
Notes:
You'll probably have to refer something like this to flatten the above two lists.
Read about torch.no_grad() to apply it as a good practice during the calculations of metrics.
I wrote a Python 3.5 script for doing street segmentation. Since I'm new in Image Segementation, I did not use predefined dataloaders from pytorch, instead I wrote them by my self (for better understanding). Until now I only use a batch size of 1. Now I want to generalize this for arbitrary batch sizes.
This is a snippet of my Dataloader:
def augment_data(batch_size):
# [...] defining some paths and data transformation (including ToTensor() function)
# The images are named by numbers (Frame numbers), this allows me to find the correct label image for a given input image.
all_input_image_paths = {int(elem.split('\\')[-1].split('.')[0]) : elem for idx, elem in enumerate(glob.glob(input_dir + "*"))}
all_label_image_paths = {int(elem.split('\\')[-1].split('.')[0]) : elem for idx, elem in enumerate(glob.glob(label_dir + "*"))}
dataloader = {"train":[], "val":[]}
all_samples = []
img_counter = 0
for key, value in all_input_image_paths.items():
input_img = Image.open(all_input_image_paths[key])
label_img = Image.open(all_label_image_paths[key])
# Here I use my own augmentation function which crops the input and label on the same position and do other things.
# We get a list of new augmented data
augmented_images = generate_augmented_images(input_img, label_img)
for elem in augmented_images:
input_as_tensor = data_transforms['norm'](elem[0])
label_as_tensor = data_transforms['val'](elem[1])
input_as_tensor.unsqueeze_(0)
label_as_tensor.unsqueeze_(0)
is_training_data = random.uniform(0.0, 1.0)
if is_training_data <= 0.7:
dataloader["train"].append([input_as_tensor, label_as_tensor])
else:
dataloader["val"].append([input_as_tensor, label_as_tensor])
img_counter += 1
shuffle(dataloader["train"])
shuffle(dataloader["val"])
dataloader_batched = {"train":[], "val":[]}
# Here I group my data to a given batch size
for elem in dataloader["train"]:
batch = []
for i in range(batch_size):
batch.append(elem)
dataloader_batched["train"].append(batch)
for elem in dataloader["val"]:
batch = []
for i in range(batch_size):
batch.append(elem)
dataloader_batched["val"].append(batch)
return dataloader_batched
This is a snippet of my training method with batch size 1:
while epoch <= num_epochs:
# Each epoch has a training and validation phase
for phase in ['train', 'val']:
if phase == 'train':
scheduler.step(3)
model.train() # Set model to training mode
else:
model.eval() # Set model to evaluate mode
running_loss = 0.0
counter = 0
# Iterate over data.
for inputs, labels in dataloaders[phase]:
counter += 1
max_num = len(dataloaders[phase])
inputs = inputs.to(device)
labels = labels.to(device)
# zero the parameter gradients
optimizer.zero_grad()
# forward
# track history if only in train
with torch.set_grad_enabled(phase == 'train'):
outputs = model(inputs)
loss = criterion(outputs, labels)
# backward + optimize only if in training phase
if phase == 'train':
loss.backward()
optimizer.step()
# statistics
running_loss += loss.item() * inputs.size(0)
epoch_loss = running_loss / dataset_sizes[phase]
If I execute this, I get of course the error:
for inputs, labels in dataloaders[phase]:
ValueError: not enough values to unpack (expected 2, got 1)
I understand why, because now I have a list of images and not only an input and label image as before. So guessed I need a second for loop which iterates over these batches. So I tried this:
# Iterate over data.
for elem in dataloaders[phase]:
for inputs, labels in elem:
counter += 1
max_num = len(dataloaders[phase])
inputs = inputs.to(device)
labels = labels.to(device)
# zero the parameter gradients
optimizer.zero_grad()
# forward
# track history if only in train
with torch.set_grad_enabled(phase == 'train'):
outputs = model(inputs)
# _, preds = torch.max(outputs, 1)
loss = criterion(outputs, labels)
# backward + optimize only if in training phase
if phase == 'train':
loss.backward()
optimizer.step()
But for me it looks like the optimization step (back-prop) is only applied on the last image of the batch. Is that true? And if so, how can I fix this? I guess if I indent the with-Block, then I get again a batch size 1 optimization.
Thanks in advance
But for me it looks like the optimization step (back-prop) is only applied on the last image of the batch.
It should not apply only based on the last image. It should apply based on the batch size.
If you set bs=2 and it should apply to the batch of two images.
Optimization step actually will update the params of your network. Backprop is a fancy name for PyTorch autograd system that computes the first order gradients.
I am running a neural network with logging of training accuracy,Validation accuracy and validation loss. here is my code snippet.
def show_progress(epoch, feed_dict_train, feed_dict_validate, val_loss):
acc = session.run(accuracy, feed_dict=feed_dict_train)
val_acc = session.run(accuracy, feed_dict=feed_dict_validate)
msg = "Training Epoch {0} --- Training Accuracy: {1:>6.1%}, Validation Accuracy: {2:>6.1%}, Validation Loss: {3:.3f}"
print(msg.format(epoch + 1, acc, val_acc, val_loss))
return acc,val_acc
total_iterations = 0
#writer=tf.summary.FileWriter(options.tensorboard,session)
saver = tf.train.Saver()
def train(num_iteration):
global total_iterations
writer=tf.summary.FileWriter(options.tensorboard,session.graph)
#global writer
for i in range(total_iterations,
total_iterations + num_iteration):
x_batch, y_true_batch, _, cls_batch = data.train.next_batch(batch_size)
x_valid_batch, y_valid_batch, _, valid_cls_batch = data.valid.next_batch(batch_size)
feed_dict_tr = {x: x_batch,
y_true: y_true_batch}
feed_dict_val = {x: x_valid_batch,
y_true: y_valid_batch}
session.run(optimizer, feed_dict=feed_dict_tr)
if i % 10 == 0:
val_loss = session.run(cost, feed_dict=feed_dict_val)
epoch = int(i /10)
accu,valid_accu=show_progress(epoch, feed_dict_tr, feed_dict_val, val_loss)
#getting values for visualising inside the tensorboard
tf.summary.scalar("training_accuracy",accu)
tf.summary.scalar("Validation_accuracy",valid_accu)
tf.summary.scalar("Validation_loss",val_loss)
#tf.summary.scalar("epoch",epoch)
#merging all the values (serializing)
merged=tf.summary.merge_all()
summary=session.run(merged)
#adding them to the events directory
writer.add_summary(summary,epoch)
saver.save(session, options.save)
total_iterations += num_iteration
train(num_iteration=10)
Now I am getting a tensor board output, as for each epoch the accuracy,validation accuracy and validation loss as separate plots with single points.
For each epoch I am getting these three plots again with another point.
I want to get a continuous points for these three plots so as it forms a line graph.
Each of your call to tf.summary.scalar() will create a node in the computation graph. Specifically, in your code, the calls are inside the training loop and therefore metrics at different epochs get written to different plots.
tf.summary.scalar("training_accuracy", accu)
tf.summary.scalar("Validation_accuracy", valid_accu)
tf.summary.scalar("Validation_loss", val_loss)
What you can do is to define the summary ops before the loop with placeholders. Then, in the eval loop, you can feed these tensor with real values.
# Define a placeholder and wire it to the summary op.
accu_tensor = tf.placeholder(tf.float32)
tf.summary.scalar("training_accuracy", accu_tensor)
summary_op = tf.summary.merge_all()
# Create a session after defining ops.
sess = tf.Session()
writer = tf.summary.FileWriter(<some-directory>, sess.graph)
for i in range(total_iterations,
total_iterations + num_iteration):
# run training ops to get values for accu
# ...
# run the summary op with a feed_dict to feed the value.
summaries = sess.run(summary_op, feed_dict={accu_tensor: accu})
writer.add_summary(summaries, epoch)
Apologies if this is the wrong place to raise my issue (please help me out with where best to raise it if that's the case). I'm a novice with Keras and Python so hope responses have that in mind.
I'm trying to train a CNN steering model that takes images as input. It's a fairly large dataset, so I created a data generator to work with fit_generator(). It's not clear to me how to make this method trains on batches, so I assumed that the generator has to return batches to fit_generator(). The generator looks like this:
def gen(file_name, batchsz = 64):
csvfile = open(file_name)
reader = csv.reader(csvfile)
batchCount = 0
while True:
for line in reader:
inputs = []
targets = []
temp_image = cv2.imread(line[1]) # line[1] is path to image
measurement = line[3] # steering angle
inputs.append(temp_image)
targets.append(measurement)
batchCount += 1
if batchCount >= batchsz:
batchCount = 0
X = np.array(inputs)
y = np.array(targets)
yield X, y
csvfile.seek(0)
It reads a csv file containing telemetry data (steering angle etc) and paths to image samples, and returns arrays of size: batchsz
The call to fit_generator() looks like this:
tgen = gen('h:/Datasets/dataset14-no.zero.speed.trn.csv', batchsz = 128) # Train data generator
vgen = gen('h:/Datasets/dataset14-no.zero.speed.val.csv', batchsz = 128) # Validation data generator
try:
model.fit_generator(
tgen,
samples_per_epoch=113526,
nb_epoch=6,
validation_data=vgen,
nb_val_samples=20001
)
The dataset contains 113526 sample points yet the model training update output reads like this (for example):
1020/113526 [..............................] - ETA: 27737s - loss: 0.0080
1021/113526 [..............................] - ETA: 27723s - loss: 0.0080
1022/113526 [..............................] - ETA: 27709s - loss: 0.0080
1023/113526 [..............................] - ETA: 27696s - loss: 0.0080
Which appears to be training sample by sample (stochastically?).
The resultant model is useless. I previously trained on a much smaller dataset using .fit() with the whole dataset loaded into memory, and that produced a model that at least works even if poorly. Clearly something is wrong with my fit_generator() approach. Will be very grateful for some help with this.
This:
for line in reader:
inputs = []
targets = []
... is resetting your batch for every line in the csv files. You're not training with your entire data, but with just a single sample in 128.
Suggestion:
for line in reader:
if batchCount == 0:
inputs = []
targets = []
....
....
As someone commented, the in fit generator, samples_per_epoch should be equal to total_samples / batchsz
Even though, I think your loss should be going down anyway. If it isn't, there might still be another problem in the code, perhaps in the way you load the data, or in the model's initialization or structure.
Try to plot your images and print the data in the generator:
for X,y in tgen: #careful, this is an infinite loop, make it stop
print(X.shape[0]) # is this really the number of batches you expect?
for image in X:
...some method to plot X so you can see it, or just print
print(y)
Check if the yielded values are ok with what you expect them to be.
So far I have come up with this hacky code here, this code runs and outputs
Epoch 10/10
1/3000 [..............................] - ETA: 27s - loss: 0.3075 - acc: 0.7270
6/3000 [..............................] - ETA: 54s - loss: 0.3075 - acc: 0.7355
.....
2996/3000 [============================>.] - ETA: 0s - loss: 0.3076 - acc: 0.7337
2998/3000 [============================>.] - ETA: 0s - loss: 0.3076 - acc: 0.7337
3000/3000 [==============================] - 59s - loss: 0.3076 - acc: 0.7337
Traceback (most recent call last):
File "C:/Users/Def/PycharmProjects/KerasUkExpenditure/TweetParsing.py", line 140, in <module>
(loss, acc) = model.fit_generator(generator(tokenizer=t, startIndex=startIndex,batchSize=amountOfData),
TypeError: 'History' object is not iterable
Process finished with exit code 1
I'm confused by "'History' object is not iterable", what does this mean?
This is the first time I've tried to do batch training and testing and I'm not sure i've implemented it correctly as most the examples I've seen online are for images. Here is the code
from keras.models import Sequential
from keras.layers import Dense, Dropout
from keras.preprocessing.text import Tokenizer
import numpy as np
import pandas as pd
import pickle
import matplotlib.pyplot as plt
import re
"""
amount of samples out to the 1 million to use, my 960m 2GB can only handel
about 30,000ish at the moment depending on the amount of neurons in the
deep layer and the amount fo layers.
"""
maxSamples = 3000
#Load the CSV and get the correct columns
data = pd.read_csv("C:\\Users\\Def\\Desktop\\Sentiment Analysis Dataset1.csv")
dataX = pd.DataFrame()
dataY = pd.DataFrame()
dataY[['Sentiment']] = data[['Sentiment']]
dataX[['SentimentText']] = data[['SentimentText']]
dataY = dataY.iloc[0:maxSamples]
dataX = dataX.iloc[0:maxSamples]
testY = dataY.iloc[-1: -maxSamples]
testX = dataX.iloc[-1: -maxSamples]
"""
here I filter the data and clean it up bu remove # tags and hyper links and
also any characters that are not alpha numeric, I then add it to the vec list
"""
def removeTagsAndLinks(dataframe):
vec = []
for x in dataframe.iterrows():
#Removes Hyperlinks
zero = re.sub("(http|ftp|https)://([\w_-]+(?:(?:\.[\w_-]+)+))([\w.,#?^=%&:/~+#-]*[\w#?^=%&/~+#-])?", "", x[1].values[0])
#Removes # tags
one = re.sub("#\\w+", '', zero)
#keeps only alpha-numeric chars
two = re.sub("\W+", ' ', one)
vec.append(two)
return vec
vec = removeTagsAndLinks(dataX)
xTest = removeTagsAndLinks(testX)
yTest = removeTagsAndLinks(testY)
"""
This loop looks for any Tweets with characters shorter than 2 and once found write the
index of that Tweet to an array so I can remove from the Dataframe of sentiment and the
list of Tweets later
"""
indexOfBlankStrings = []
for index, string in enumerate(vec):
if len(string) < 2:
del vec[index]
indexOfBlankStrings.append(index)
for row in indexOfBlankStrings:
dataY.drop(row, axis=0, inplace=True)
"""
This makes a BOW model out of all the tweets then creates a
vector for each of the tweets containing all the words from
the BOW model, each vector is the same size becuase the
network expects it
"""
def vectorise(tokenizer, list):
tokenizer.fit_on_texts(list)
return tokenizer.texts_to_matrix(list)
#Make BOW model and vectorise it
t = Tokenizer(lower=False, num_words=1000)
dim = vectorise(t, vec)
xTest = vectorise(t, xTest)
"""
Here im experimenting with multiple layers of the total
amount of words in the syllabus divided by ^2 - This
has given me quite accurate results compared to random guess's
of amount of neron's and amounts of layers.
"""
l1 = int(len(dim[0]) / 4) #To big for my GPU
l2 = int(len(dim[0]) / 8) #To big for my GPU
l3 = int(len(dim[0]) / 16)
l4 = int(len(dim[0]) / 32)
l5 = int(len(dim[0]) / 64)
l6 = int(len(dim[0]) / 128)
#Make the model
model = Sequential()
model.add(Dense(l1, input_dim=dim.shape[1]))
model.add(Dropout(0.15))
model.add(Dense(l2))
model.add(Dense(l1))
model.add(Dense(l3))
model.add(Dropout(0.2))
model.add(Dense(l4))
model.add(Dense(1, activation='relu'))
#Compile the model
model.compile(optimizer='RMSProp', loss='binary_crossentropy', metrics=['acc'])
"""
This here will use multiple batches to train the model.
startIndex:
This is the starting index of the array for which you want to
start training the network from.
dataRange:
The number of elements use to train the network in each batch so
since dataRange = 1000 this mean it goes from
startIndex...dataRange OR 0...1000
amountOfEpochs:
This is kinda self explanitory, the more Epochs the more it
is supposed to learn AKA updates the optimisation algo numbers
"""
amountOfEpochs = 10
dataRange = 1000
startIndex = 0
def generator(tokenizer, batchSize, totalSize=maxSamples, startIndex=0):
f = tokenizer.texts_to_sequences(vec[startIndex:totalSize])
l = np.asarray(dataY.iloc[startIndex:totalSize])
while True:
for i in range(1000, totalSize, batchSize):
batch_features = tokenizer.sequences_to_matrix(f[startIndex: batchSize])
batch_labels = l[startIndex: batchSize]
yield batch_features, batch_labels
##This runs the model for batch AKA load a little them process then load a little more
for amountOfData in range(1000, maxSamples, 1000):
#(loss, acc) = model.train_on_batch(x=dim[startIndex:amountOfData], y=np.asarray(dataY.iloc[startIndex:amountOfData]))
(loss, acc) = model.fit_generator(generator(tokenizer=t, startIndex=startIndex,batchSize=amountOfData),
steps_per_epoch=maxSamples, epochs=amountOfEpochs,
validation_data=(np.array(xTest), np.array(yTest)))
startIndex += 1000
The part towards the bottom is where I've tried to implement the fit_generator() and make my own generator, I wanted to load say 75,000 maxSamples then train the network 1000 samples at a time until it reaches the maxSample var which is why I've setup range to do the (0, maxSample, 1000) which I use in the generator() was this the correct use?
I ask because my network is not using the validation data and it seems to fit to the data extremely quickly which suggests overfitting or just using a very small dataset. am I iterating over all the maxSamples int he correct way? or am I just looping over the first iterations several times?
Thanks
The problem lies in this line:
(loss, acc) = model.fit_generator(...)
as fit_generator returns a single object of keras.callbacks.history class. That's why you have this error as singe object is not iterable. In order to get loss lists you need to retrieve them from history field in this callback which is a dictionary of recorded losses:
history = model.fit_generator(...)
loss = history.history["loss"]