I would like to remove all the batch normalization layers from a Keras model that includes short-skip connections. For example, let's consider EfficientNetB0 as follow:
import tensorflow as tf
model = tf.keras.applications.EfficientNetB0(weights=None, include_top=True)
I was using the 3D version of efficientnet, which I don't think is important for the question ,but I'm going to show it anyway:
import keras
from keras import layers
from keras.models import Model
input_shape = (32,32,32,3)
import efficientnet_3D.keras as efn
model = efn.EfficientNetB0(input_shape=input_shape, weights='imagenet')
X = model.layers[-1].output
X = layers.Flatten()(X)
X = layers.Dense(16)(X)
X = layers.Dense(16)(X)
X = layers.Dense(1)(X)
model = Model(inputs=model.inputs, outputs=X)
model.compile(loss='mse',
optimizer='adam',
metrics=['mean_absolute_error']
)
model.summary()
I tried to develop my own way of removing, and it seems to be totally wrong. Because the output model is pretty messy in terms of all the shortcut connections.
import keras
from keras import layers
from keras.models import Model
ind = [i for i, l in enumerate(model.layers) if 'bn' in l.name]
X = model.layers[0].output
for i in range(1, len(model.layers)+1):
# Skipping Batch Normalization layers
if i in ind:
# model.layers[i]._inbound_nodes = []
# model.layers[i]._outbound_nodes = []
continue
# If there is a short skip
if isinstance(model.layers[i].input, list):
input_names = [j.name for j in model.layers[i].input]
assert len(input_names) == 2
input_names.remove(X.name)
input_names = input_names[0].split('/')[0]
# X = [model.get_layer(input_names).output, X]
X = [model.layers[6].output, X]
if isinstance(X, list):
print(i)
X = model.layers[i](X)
new_model = Model(inputs=model.inputs, outputs=X)
I think there should be a better way that I'm not aware of. I tried a similar question for removing a layer, but I think because my model includes skip-connection, those methods don't work. Any help is appreciated.
Related
I am trying to train a model using transfer learning with data augmentation.
My image data is 32 x 32 x 3 and I want to import EfficientNet07, but every time I run
from tensorflow.keras.applications import EfficientNetB0
model = EfficientNetB0(weights='imagenet')
I get error message:
ImportError: cannot import name 'EfficientNetB7' from 'tensorflow.keras.applications' (C:\Users\…..
I unwillingly reverted to Resnet50, that works fine. But I really wish to work with efficientnets.
Secondly, I was hoping to use keras.expermental.preprocessing layers to add data augmentation while the model runs, but I get error
from tensorflow.keras.layers.experimental import preprocessing
from tensorflow.keras.models import Sequential
from tensorflow.keras import layers
img_augmentation = Sequential(
[
preprocessing.RandomRotation(factor=0.15),
preprocessing.RandomTranslation(height_factor=0.1, width_factor=0.1),
preprocessing.RandomFlip(),
preprocessing.RandomContrast(factor=0.1),
],
name="img_augmentation",
)
AttributeError: module 'tensorflow.keras.layers.experimental.preprocessing' has no attribute 'RandomRotation'
I took all this code from https://keras.io/examples/vision/image_classification_efficientnet_fine_tuning/
and have read it top to bottom thrice. I went on to use ImageDataGenerator and wrote this code
def choosen_args(args):
#this function is to randomly choose the augmentation techniques from the list available in keras
key_list = []
arg_list = list(args.keys())
for i in range(len(arg_list)):
if np.random.randint(2) == 0:
pass
else:
key_list.append(arg_list[i])
arg_dict = dict((k, args[k]) for k in key_list)
return arg_dict
#this is the list of all augmentation I am using - but only few will get selected for final transformation
args = dict(
rotation_range = 120,
width_shift_range=0.4,
height_shift_range=0.4,
shear_range=0.2,
zoom_range=0.2,
fill_mode ='nearest',
zca_whitening=True,
zca_epsilon=1e-06,
brightness_range=(9.0, 46.0),
channel_shift_range=60.0,
horizontal_flip=True,
vertical_flip=True,
rescale=random.randint(1,11))
data_augmentation_train = ImageDataGenerator( choosen_args(args) )
data_augmentation_validation = ImageDataGenerator( choosen_args(args) )
data_augmentation_train.fit(x_train, augment = True)
data_augmentation_validation.fit(x_test, augment = True)
Kindly help with:
How to get efficientnet07 and use its weights for transfer learninig
The data augmentation code I have used - how do I visualize what it does - that is, transforms my images
Is there an easier way to do data augmentation - how do I fix the Import and attribute errors
After I fit my data in generator - how do I use it with rest of my transfer learning code.
base_model = tf.keras.applications.ResNet50(weights='imagenet',
include_top = False, input_shape = input_size)
> x_train = tf.keras.applications.resnet50.preprocess_input(x_train)
> x_test = tf.keras.applications.resnet50.preprocess_input(x_test)
> base_model.trainable = False
inputs = keras.Input(shape=(32, 32, 3))
model = base_model_ResNet50(inputs, training=False)
model = keras.layers.GlobalAveragePooling2D()(model )
model= keras.layers.Flatten()(model)
model = keras.layers.Dropout(0.2)(model)
output = layers.Dense(100, activation="softmax")(model)
history = model.fit(data_augmentation_train.flow (x_train, labels_train, batch_size = 64), validation_data =
data_augmentation_validation.flow(x_test,labels_test))
Thank you
try importing as :
from keras_efficientnets import EfficientNetB0
model = EfficientNetB0(weights='imagenet')
First install efficientnet module:
!pip install -U efficientnet
Then import it as:
import efficientnet.keras as effnet
Create the model:
model = effnet.EfficientNetB0(weights = 'imagenet')
model = tf.keras.applications.EfficientNetB0()
Options to replace EfficientNetB0 are listed here: https://keras.io/api/applications/#usage-examples-for-image-classification-models
E.g. model = tf.keras.applications.EfficientNetB7() or model = tf.keras.applications.EfficientNetV2B3()
(FYI you may have the change size of the image input)
Hope this helps
I'm using Keras to fit a function, and I'm new to Keras.
With a very simple network, the Keras can fit my function very well, I just want to know what the function is and try to understand why it works very well. But the "predict" function hide the details.
Here is the code I create the network:
import numpy as np
import tensorflow as tf
from tensorflow import keras
LABEL_COLUMN = "shat"
BATCH_SIZE = 16
EPOCHS = 20
trainfilePath = "F:\\PyworkingFolder\\WWSHat\\_Data\\alpha0train.csv"
testfilePath = "F:\\PyworkingFolder\\WWSHat\\_Data\\alpha0test.csv"
with open(trainfilePath, encoding='utf-8') as txtContent:
trainArray = np.loadtxt(txtContent, delimiter=",")
with open(testfilePath, encoding='utf-8') as txtContent:
testArray = np.loadtxt(txtContent, delimiter=",")
trainSample = trainArray[:, 0:14]
trainLable = trainArray[:, 14]
testSample = testArray[:, 0:14]
testLable = testArray[:, 14]
model = keras.Sequential([
keras.layers.Dense(14, activation='relu', input_shape=[14]),
keras.layers.Dense(15, activation='relu'),
keras.layers.Dense(1)
])
optimizer = tf.keras.optimizers.RMSprop(0.001)
# optimizer = keras.optimizers.Adadelta(lr=1.0, rho=0.95, epsilon=None, decay=0.0)
model.compile(loss='mse',
optimizer=optimizer,
metrics=['mae', 'mse'])
model.summary()
history = model.fit(trainSample, trainLable, epochs=EPOCHS, batch_size=BATCH_SIZE)
model.evaluate(testSample, testLable, verbose=1)
model.save("F:\\PyworkingFolder\\WWSHat\\_Data\\alpha0.h5")
What I understand is:
the layers are weight matrices and basis matrices, it works as
out=max(0, weight * input + basis)
After some search, I find I can read the .h5 file using
import h5py
import numpy as np
FILENAME = "F:\\PyworkingFolder\\WWSHat\\_Data\\alpha0.h5"
with h5py.File(FILENAME, 'r') as f:
dense_1 = f['/model_weights/dense_1/dense_1']
dense_1_bias = dense_1['bias:0'][:]
dense_1_kernel = dense_1['kernel:0'][:]
dense_2 = f['/model_weights/dense_2/dense_2']
dense_2_bias = dense_2['bias:0'][:]
dense_2_kernel = dense_2['kernel:0'][:]
# print("Weight matrix 1:\n")
# print(dense_1_kernel)
# print("Basis matrix 1:\n")
# print(dense_1_bias)
# print("Weight matrix 2:\n")
# print(dense_2_kernel)
# print("Basis matrix 2:\n")
# print(dense_2_bias)
def layer_output(v, kernel, bias):
return np.dot(v, kernel) + bias
reluFunction = np.vectorize(lambda x: x if x >= 0.0 else 0.0)
testV = np.array([[-0.004090321213057993,
0.009615388501909157,
-0.24223693596921558,
0.015504079563927319,
-0.02659541428995062,
0.018512968977547152,
0.00836788544720289,
-0.10874776132746002,
-0.045863474556415526,
-0.010195799916571194,
0.09474219315939948,
0.03606698737846194,
-0.004560110004741025,
0.028042417959738858]])
output_1 = layer_output(testV, dense_1_kernel, dense_1_bias)
output_2 = reluFunction(output_1)
output_3 = layer_output(output_2, dense_2_kernel, dense_2_bias)
output_4 = reluFunction(output_3)
however, the result of output_4 is very different from what I get using
loaded_model = keras.models.load_model("F:\\PyworkingFolder\\WWSHat\\_Data\\alpha0.h5")
predicted = loaded_model(testV)
The "predicted" is very close to the ground truth while "output_4" is far away from the ground truth.
I get stuck here and don't know why and failed to find information about how to extract the function I want from the Keras model, I need your help!
Thanks!
model = keras.Sequential([
keras.layers.Dense(14, activation='relu', input_shape=[14]),
keras.layers.Dense(15, activation='relu'),
keras.layers.Dense(1)
])
In your model, there are 3 layers, the last dense layer has weight and biases too, you didn't consider them in your calculation.
I am trying to create a forecasting model which takes the lag features along with embeddings to predict next 10 days cumulative value. Embedding layer is trained by using the order basket with gensim.
Below is my network:
from keras.layers import Embedding, Flatten, Input, Dense, Dropout, Flatten, Activation
inp = Input(shape=(1, )) #ucode length will be 1
x = Embedding(len(model.wv.vocab), WV_DIM,
weights=[model.wv.vectors],
trainable=False)(inp)
x = Flatten()(x)
x = Dense(32, activation='relu', name='Embedding_out')(x)
features_input = Input(shape=(122,)) ##lag Features
concat = concatenate([features_input, x],name="ConcatenatedwFeatures")
output = Dense(256, activation="relu",name="L1_Relu")(concat)
output = Dense(128, activation="relu",name="L2_Relu")(output)
output = Dense(1)(output)
EmbeddingModel = Model(inputs=[inp,features_input], outputs=output)
EmbeddingModel.summary()
adam = optimizers.adam(clipvalue=1.,lr=3e-4)
EmbeddingModel.compile(loss='mse',
optimizer=adam,
metrics = ['mae', 'mse'])
hist = EmbeddingModel.fit([ucode_array[20:25],X_train[20:25]], [y_train[20:25]], validation_split=0.05,
epochs=10, batch_size=32)
Error:
ValueError: could not convert string to float: 'I33946'
Input Values:
ucode_array=sales_train_grid['ucode']
ucode_array[20:25]
15683 I33946
15685 I33946
15687 I33946
15688 126310
15689 126310
Name: ucode, dtype: object
Testing if value is present in embedding layer:
test1=model.wv.most_similar(positive=['I00731'], topn=10)
display(test1)
[x[0] for x in test1]
Returns 10 similar objects. Returns none if i had pasted any random values.
Following things tried:
1. ucode_array[20:25].values
2. ucode_array[20:25].values.tolist()
gensim version: 3.4.0
TensorFlow version: 1.12.0
Usually, we have to feed numerical values to the training process.
Making sure converting all the object and strings to embeddings will solve the problem.
putting down basic preprocessing operation which before we actually, for others to find this helpful.
sample code.
tokenizer = Tokenizer()
tokenizer.fit_on_texts(list(model.wv.vocab.keys())
encoded_ucode = tokenizer.texts_to_sequences(ucode_array)
Check this,
use float to convert a string into a float. I guess this will solve your problem.
I'm attempting to train multiple keras models with different parameter values using multiple threads (and the tensorflow backend). I've seen a few examples of using the same model within multiple threads, but in this particular case, I run into various errors regarding conflicting graphs, etc. Here's a simple example of what I'd like to be able to do:
from concurrent.futures import ThreadPoolExecutor
import numpy as np
import tensorflow as tf
from keras import backend as K
from keras.layers import Dense
from keras.models import Sequential
sess = tf.Session()
def example_model(size):
model = Sequential()
model.add(Dense(size, input_shape=(5,)))
model.add(Dense(1))
model.compile(optimizer='sgd', loss='mse')
return model
if __name__ == '__main__':
K.set_session(sess)
X = np.random.random((10, 5))
y = np.random.random((10, 1))
models = [example_model(i) for i in range(5, 10)]
e = ThreadPoolExecutor(4)
res_list = [e.submit(model.fit, X, y) for model in models]
for res in res_list:
print(res.result())
The resulting error is ValueError: Tensor("Variable:0", shape=(5, 5), dtype=float32_ref) must be from the same graph as Tensor("Variable_2/read:0", shape=(), dtype=float32).. I've also tried initializing the models within the threads which gives a similar failure.
Any thoughts on the best way to go about this? I'm not at all attached to this exact structure, but I'd prefer to be able to use multiple threads rather than processes so all the models are trained within the same GPU memory allocation.
Tensorflow Graphs are not threadsafe (see https://www.tensorflow.org/api_docs/python/tf/Graph) and when you create a new Tensorflow Session, it by default uses the default graph.
You can get around this by creating a new session with a new graph in your parallelized function and constructing your keras model there.
Here is some code that creates and fits a model on each available gpu in parallel:
import concurrent.futures
import numpy as np
import keras.backend as K
from keras.layers import Dense
from keras.models import Sequential
import tensorflow as tf
from tensorflow.python.client import device_lib
def get_available_gpus():
local_device_protos = device_lib.list_local_devices()
return [x.name for x in local_device_protos if x.device_type == 'GPU']
xdata = np.random.randn(100, 8)
ytrue = np.random.randint(0, 2, 100)
def fit(gpu):
with tf.Session(graph=tf.Graph()) as sess:
K.set_session(sess)
with tf.device(gpu):
model = Sequential()
model.add(Dense(12, input_dim=8, activation='relu'))
model.add(Dense(8, activation='relu'))
model.add(Dense(1, activation='sigmoid'))
model.compile(loss='binary_crossentropy', optimizer='adam')
model.fit(xdata, ytrue, verbose=0)
return model.evaluate(xdata, ytrue, verbose=0)
gpus = get_available_gpus()
with concurrent.futures.ThreadPoolExecutor(len(gpus)) as executor:
results = [x for x in executor.map(fit, gpus)]
print('results: ', results)
Im trying to understand how to use LSTM to classify a certain dataset that i have.
I researched and found this example of keras and imdb :
https://github.com/fchollet/keras/blob/master/examples/imdb_lstm.py
However, im confused about how the data set must be processed to input.
I know keras has pre-processing text methods, but im not sure which to use.
The x contain n lines with texts and the y classify the text by happiness/sadness. Basically, 1.0 means 100% happy and 0.0 means totally sad. the numbers may vary, for example 0.25~~ and so on.
So my question is, How i input x and y properly? Do i have to use bag of words?
Any tip is appreciated!
I coded this below but i keep getting the same error:
#('Bad input argument to theano function with name ... at index 1(0-based)',
'could not convert string to float: negative')
import keras.preprocessing.text
import numpy as np
np.random.seed(1337) # for reproducibility
from keras.preprocessing import sequence
from keras.models import Sequential
from keras.layers.core import Dense, Activation
from keras.layers.embeddings import Embedding
from keras.layers.recurrent import LSTM
print('Loading data...')
import pandas
thedata = pandas.read_csv("dataset/text.csv", sep=', ', delimiter=',', header='infer', names=None)
x = thedata['text']
y = thedata['sentiment']
x = x.iloc[:].values
y = y.iloc[:].values
###################################
tk = keras.preprocessing.text.Tokenizer(nb_words=2000, filters=keras.preprocessing.text.base_filter(), lower=True, split=" ")
tk.fit_on_texts(x)
x = tk.texts_to_sequences(x)
###################################
max_len = 80
print "max_len ", max_len
print('Pad sequences (samples x time)')
x = sequence.pad_sequences(x, maxlen=max_len)
#########################
max_features = 20000
model = Sequential()
print('Build model...')
model = Sequential()
model.add(Embedding(max_features, 128, input_length=max_len, dropout=0.2))
model.add(LSTM(128, dropout_W=0.2, dropout_U=0.2))
model.add(Dense(1))
model.add(Activation('sigmoid'))
model.compile(loss='binary_crossentropy', optimizer='rmsprop')
model.fit(x, y=y, batch_size=200, nb_epoch=1, verbose=1, validation_split=0.2, show_accuracy=True, shuffle=True)
# at index 1(0-based)', 'could not convert string to float: negative')
Review how you are using your CSV parser to read the text in. Ensure that the fields are in the format Text, Sentiment if you want to to make use of the parser as you've written it in your code.