train_data_path = '/content/gdrive/MyDrive/dataset/train'
train_data = datasets.ImageFolder(root=train_data_path, transform=data_transforms)
train_data_loader = torch.utils.data.DataLoader(train_data, batch_size=8, shuffle=True)
for xs, ts in train_data_loader:
print(xs)
print(ts)
I'm currently trying to shuffle and transform images in a folder to a dataset of tensors. One issue I'm facing is that this results in a set callable error, and I'm very confused as to why this is happening. The path specified contains a subfolder of images.
Related
TensorFlow version = 2.11.0
Laptop OS = Ubuntu Linux _86_64
Python version = 3.10.6
Coding on Visual Studio Code on a venv
I am trying to resume the training of my Keras model by loading it the following way:
# Loading pretrained model from .h5 file
h5_path = 'pretrained.h5'
model = tf.keras.models.load_model(h5_path, compile=False)
# Recover desired LR
LR = model.optimizer.learning_rate
pretrain_optimizer = tf.keras.optimizers.Nadam(learning_rate=LR)
# Compile model with desired qualities
model.compile(optimizer=pretrain_optimizer, loss = GxGyLoss_keras, metrics=[tf.keras.metrics.MeanAbsoluteError(),tf.keras.metrics.MeanAbsolutePercentageError()])
This is how I compiled and saved the 'pretrained.h5' model before
EPOCHS = 50
BATCH_SIZE = 30
OUT_CHANNELS = 1
IN_CHANNELS = 1
LR = 1e-3
IMG_SIZE = (256,256)
# Build model from 0
model = get_model(img_size=IMG_SIZE, in_channels=IN_CHANNELS, out_channels=OUT_CHANNELS)
# Compile model from 0
model.compile(optimizer=optimizers.Nadam(learning_rate=LR), loss=GxGyLoss_keras, metrics=[tf.keras.metrics.MeanAbsoluteError(),tf.keras.metrics.MeanAbsolutePercentageError()])
# Save best model weights
callbacks = [
# Save model every epoch regarding the best val_loss
keras.callbacks.ModelCheckpoint(
filepath = 'pretrained.h5',
save_best_only = True,
save_weights_only = False,
mode = 'min',
monitor = 'val_loss',),
# Save history data on the same csv file so that resuming training doesn't affect the plots
keras.callbacks.CSVLogger(
filename = f'training_hist_log',
separator = ',',
append = True)
]
# Train the model, doing validation at the end of each epoch.
history = model.fit(x = X_train,
y = y_train,
epochs = EPOCHS,
batch_size = BATCH_SIZE,
validation_data = (X_val, y_val),
callbacks = callbacks)
It is important to declare that this is how I defined the custom loss function:
from keras import backend as K
def GxGyLoss_keras(y_pred,y_true):
intImgSize = 256
summatory = K.sum(K.square(y_pred-y_true))
eval = K.sqrt(summatory) / intImgSize**2
return eval
I was trying to do the workaround of setting compile = False, since that seems to work on other users, but my problem is that I need to load the optimizer state to resume the training, and using that I get this apparently obvious error:
Traceback (most recent call last):
File "/home/victus-linux/Escritorio/MasterThesis_CODE/to_share/main_copy.py", line 114, in
LR = model.optimizer.learning_rate
AttributeError: 'NoneType' object has no attribute 'learning_rate'
I understand that it pops because I am not compiling the previous model neither the optimizer state, I would like to have a solution for resume the training of my model (with it's custom loss) and keeping at the same time the optimizer state (keep gradients, loss state...etc)
If I use
tensorflow.keras.load_model(h5_path, compile=True)
the problem stands in the compilation of the custom loss function. I also used:
tensorflow.keras.load_model(h5_path, compile=True,
custom_objects=GxGyLoss_keras)
but the exact same error was popping up:
File
"/home/victus-linux/Escritorio/MasterThesis_CODE/to_share/venv_master/lib/python3.10/site-packages/keras/saving/legacy/serialization.py",
line 557, in deserialize_keras_object
raise ValueError( ValueError: Unknown loss function: 'GxGyLoss_keras'. Please ensure you are using a
keras.utils.custom_object_scope and that this object is included in
the scope. See
https://www.tensorflow.org/guide/keras/save_and_serialize#registering_the_custom_object
for details.
[DISCLAIMER: This is my first time asking around here, I am sorry in advance for the possible mistakes over the format of my question]
I am creating my custom layers tf.keras model using mobile net pretrained layer. Model training is running fine but when saving the best picked model it is giving an error. Below is the snippet of the code that I used
pretrained_model = tf.keras.applications.MobileNetV2(
weights='imagenet',
include_top=False,
input_shape=[*IMAGE_SIZE, IMG_CHANNELS])
pretrained_model.trainable = True #fine tuning
model = tf.keras.Sequential([
tf.keras.layers.Lambda(# Convert image from int[0, 255] to the format expect by this model
lambda data:tf.keras.applications.mobilenet.preprocess_input(
tf.cast(data, tf.float32)), input_shape=[*IMAGE_SIZE, 3]),
pretrained_model,
tf.keras.layers.GlobalAveragePooling2D()])
model.add(tf.keras.layers.Dense(64, name='object_dense',kernel_regularizer=tf.keras.regularizers.l2(l2=0.001)))
model.add(tf.keras.layers.BatchNormalization(scale=False, center = False))
model.add(tf.keras.layers.Activation('relu', name='relu_dense_64'))
model.add(tf.keras.layers.Dropout(rate=0.2, name='dropout_dense_64'))
model.add(tf.keras.layers.Dense(32, name='object_dense_2',kernel_regularizer=tf.keras.regularizers.l2(l2=0.01)))
model.add(tf.keras.layers.BatchNormalization(scale=False, center = False))
model.add(tf.keras.layers.Activation('relu', name='relu_dense_32'))
model.add(tf.keras.layers.Dropout(rate=0.2, name='dropout_dense_32'))
model.add(tf.keras.layers.Dense(16, name='object_dense_16', kernel_regularizer=tf.keras.regularizers.l2(l2=0.01)))
model.add(tf.keras.layers.Dense(len(CLASS_NAMES), activation='softmax', name='object_prob'))
m1 = tf.keras.metrics.CategoricalAccuracy()
m2 = tf.keras.metrics.Recall()
m3 = tf.keras.metrics.Precision()
optimizers = [
tfa.optimizers.AdamW(learning_rate=lr * .001 , weight_decay=wd),
tfa.optimizers.AdamW(learning_rate=lr, weight_decay=wd)
]
optimizers_and_layers = [(optimizers[0], model.layers[0]), (optimizers[1], model.layers[1:])]
optimizer = tfa.optimizers.MultiOptimizer(optimizers_and_layers)
model.compile(
optimizer= optimizer,
loss = 'categorical_crossentropy',
metrics=[m1, m2, m3],
)
checkpoint_path = os.getcwd() + os.sep + 'keras_model'
checkpoint_cb = tf.keras.callbacks.ModelCheckpoint(filepath=os.path.join(checkpoint_path),
monitor = 'categorical_accuracy',
save_best_only=True,
save_weights_only=True)
history = model.fit(train_data, validation_data=test_data, epochs=N_EPOCHS, callbacks=[checkpoint_cb])
At tf.keras.callbacks.ModelCheckpoint is giving me an error
TypeError: Unable to serialize 1.0000000656873453e-05 to JSON. Unrecognized type <class 'tensorflow.python.framework.ops.EagerTensor'>.
Below is the link to the Google Colab notebook in case you want to replicate the issue
https://colab.research.google.com/drive/1wQbUFfhtDaB5Xta574UkAXJtthui7Bt9?usp=sharing
This seems to be a bug in Tensorflow or Keras. The tensor that's being serialized to JSON is from your optimizer definition.
model.optimizer.optimizer_specs[0]["optimizer"].get_config()["weight_decay"]
<tf.Tensor: shape=(), dtype=float32, numpy=1.0000001e-05>
From the implementation of tfa.optimizers.AdamW, the weight_decay is serialized using tf.keras.optimizers.Adam._serialize_hyperparameter. This function assumes that if you pass in a callable for the hyperparameter, it returns a non-tensor value when called, but in your notebook, it was implemented as
wd = lambda: 1e-02 * schedule(step)
where schedule() returns a Tensor. I tried some various ways to try to convert the tensor to a scalar value, but I couldn't get them to work. As a workaround, I implemented wd as a LearningRateSchedule so it'll serialize properly, though the code was clunkier. Replacing the definitions of wd and lr with this code allowed model training to complete for me without any issues.
class MyExponentialDecay(tf.keras.optimizers.schedules.ExponentialDecay):
def __call__(self, step):
return 1e-2 * super().__call__(step)
wd = MyExponentialDecay(
initial_learning_rate,
decay_steps=14,
decay_rate=0.8,
staircase=True)
lr = 1e2 * schedule(step)
After training completes, the model.save() call will fail. I believe this is the same issue which was reported here in the Tensorflow Addons Github. The summary of this issue is that the get_config() function for the optimizers will include a "gv" key in the config which stores Tensor objects, which aren't JSON serializable.
At the time of writing, this issue has not been resolved yet. If you don't need the optimizer state for the final saved model, you can pass in the include_optimizer=False argument to model.save() which worked for me. Otherwise, you may need to patch the library or the specific optimizer class implementation to get rid of the "gw" key in the config like the OP did in that thread.
I'm trying to fine tuning Bert for document classification.
I started by tokenizing the documents to generate the input_ids, attention_mask and token_type_ids lists to feed my TFBertModel:
def tokenize_sequences(tokenizer, max_length, corpus):
input_ids = []
token_type_ids = []
attention_masks = []
for i in tqdm(range(len(corpus))):
encoded = tokenizer.encode_plus(
corpus[i],
max_length=max_length,
add_special_tokens=True,
padding='max_length',
truncation=True,
return_token_type_ids=True,
return_attention_mask=True, # add attention mask to not focus on pad tokens)
)
input_ids.append(encoded["input_ids"])
attention_masks.append(encoded["attention_mask"])
token_type_ids.append(encoded["token_type_ids"])
input_ids = tf.convert_to_tensor(input_ids)
attention_masks = tf.convert_to_tensor(attention_masks)
token_type_ids = tf.convert_to_tensor(token_type_ids)
#print(input_ids.shape, attention_masks.shape, token_type_ids.shape)
return [input_ids, attention_masks, token_type_ids]
Then, I tried to fit my model:
x_train = tokenize_sequences(tokenizer, MAXLEN, corpus_train)
model = loadBertModel()
model.fit(
x_train, y_bin_train,
epochs=N_EPOCHS,
verbose=1,
batch_size=4,
)
And I get this error:
InvalidArgumentError: indices[3] = [1,5] is out of order. Many sparse ops require sorted indices.
Use tf.sparse.reorder to create a correctly ordered copy.
I tried to solve the issue following this suggestion. I did this by modifying input_ids, attention_masks, token_type_ids tensors returned by tokenize_sequences.
input_ids = tf.sparse.reorder(input_ids)
attention_masks = tf.sparse.reorder(attention_masks)
token_type_ids = tf.sparse.reorder(token_type_ids)
But then another error occurred:
TypeError: Input must be a SparseTensor.
PS: When I checked the type of my tensors, I noticed that they were <class 'tensorflow.python.framework.ops.EagerTensor'>.
Any ideas on how to solve this?
I don't have enough points to comment, so I'm trying to comment by answering..
It seems this question is same as yours:
Multiclass text classification TypeError: Input must be a SparseTensor
In my case, I solved a similar issue by simply converting the inputs using .toarray() instead of trying to reorder them.
input_ids = input_ids.toarray()
For a project on neural networks, I am using Pytorch and am working with the EMNIST dataset.
The code that is already given loads in the dataset:
train_dataset = dsets.MNIST(root='./data',
train=True,
transform=transforms.ToTensor(),
download=True)
And prepares it:
train_loader = torch.utils.data.DataLoader(dataset=train_dataset,
batch_size=batch_size,
shuffle=True)
Then, when all the configurations of the network are defined, there is a for loop to train the model per epoch:
for i, (images, labels) in enumerate(train_loader):
In the example code this works fine.
For my task, I am given a dataset that I load as follows:
emnist = scipy.io.loadmat("DIRECTORY/emnist-letters.mat")
data = emnist ['dataset']
X_train = data ['train'][0, 0]['images'][0, 0]
y_train = data ['train'][0, 0]['labels'][0, 0]
Then, I create the train_dataset as follows:
train_dataset = np.concatenate((X_train, y_train), axis = 1)
train_dataset = torch.from_numpy(train_dataset)
And use the same step to prepare it:
train_loader = torch.utils.data.DataLoader(dataset=train_dataset,
batch_size=batch_size,
shuffle=True)
However, when I try to use the same loop as before:
for i, (images, labels) in enumerate(train_loader):
I get the following error:
ValueError: too many values to unpack (expected 2)
Who knows what I can do so that I can train my dataset with this loop?
The dataset you created from the EMNIST data is a single tensor, and therefore, the data loader will also produce a single tensor, where the first dimension is the batch dimensions. This results in trying to unpack that tensor across the batch dimension, which doesn't work because your batch size is greater than two, but is also not what you want to happen.
You can use torch.utils.data.TensorDataset to easily create a dataset, which produces a tuple of images and their respective labels, just like the MNIST dataset does.
train_dataset = torch.utils.data.TensorDataset(torch.from_numpy(X_train), torch.from_numpy(y_train))
I try to build a generator for a Keras model which will be trained on a large hdf store.
To speed up the training, I pre-calculated all features incl. one-hot encoding already in the hdfstore. So the call from that should be straight forward.
To feed chunks of my data into the network, I try to use fit_generator, but struggle to get it up and running.
The generator:
def myGenerator(myStore, generateFrom,generateTo):
# Create empty arrays to contain batch of features and labels#
while True:
X = pd.read_hdf(myStore,'X',start=generateFrom,stop=generateTo)
y = pd.read_hdf(myStore,'y',start=generateFrom,stop=generateTo)
yield X,y
Network and fitting:
def get_model(shape):
'''Create a keras model.'''
inputlayer = Input(shape=shape)
model = BatchNormalization()(inputlayer)
model = Dense(1024, activation='relu')(model)
model = Dropout(0.25)(model)
model = BatchNormalization()(inputlayer)
model = Dense(512, activation='relu')(model)
model = Dropout(0.25)(model)
model = BatchNormalization()(inputlayer)
model = Dense(256, activation='relu')(model)
model = Dropout(0.25)(model)
model = BatchNormalization()(inputlayer)
model = Dense(128, activation='relu')(model)
model = Dropout(0.25)(model)
# 11 because background noise has been taken out
model = Dense(2, activation='tanh')(model)
model = Model(inputs=inputlayer, outputs=model)
return model
shape = (6603,10000)
model = get_model(shape)
model.compile(loss='mean_squared_error', optimizer=Adam(), metrics=['accuracy'])
#X = generator(myStore)
#Xt = generator(myStore)
labelbinarizer = LabelBinarizer()
y = labelbinarizer.fit_transform(y)
#yt = labelbinarizer.fit_transform(yt)
generateFrom = 0
for i in range(10):
generateTo=generateFrom+10000
model.fit_generator(
generator=myGenerator(myStore,generateFrom,generateTo),
epochs=1,
steps_per_epoch=X[0].shape[0] // 1000)
generateFrom=generateTo
I have tried both, to have the fit_generator within a loop and plug in the range (as shown above), but also to handle the range inside the generator. Both does not work. Currently running into
TypeError: 'generator' object is not subscriptable
Likely I have some misunderstanding how fit_generator() is supposed to be used in this context. Most examples out there are around generating tensors from pictures.
Any hint is appreciated.
Thanks
The function read_hdf returns a panda object, you need to convert it to numpy array.