Related
MSE loss function is convex by definition but when I am generating a high-dimensional dataset then the problem is no more convex.
rng = np.random.RandomState(0)
X, y, coefficients = make_regression(n_samples = 1000, coef = True , n_features = 100, effective_rank = 3,noise = 0.0, n_informative = 100, random_state=rng)
Now splitting it into train, validation and test dataset.
from sklearn.model_selection import train_test_split
def ttv_split(dataX, dataY, train_ratio, validation_ratio, test_ratio):
x_train, x_test, y_train, y_test = train_test_split( dataX, dataY, test_size = round(1 - train_ratio, 2), random_state = 42 )
x_val, x_test, y_val, y_test = train_test_split(x_test, y_test, test_size = test_ratio/(test_ratio + validation_ratio), random_state = 42 )
return x_train, y_train, x_val, y_val, x_test, y_test
x_train, y_train, x_val, y_val, x_test, y_test = ttv_split(X, y, .05, .025, (1-0.075) )
print("TrainingX Shape = ", x_train.shape, "TrainY shape = ", y_train.shape)
print("ValidationX Shape = ", x_val.shape, "ValY shape = ", y_val.shape)
print("TestX Shape = ", x_test.shape, "TestY shape = ", y_test.shape)
#Output :
TrainingX Shape = (50, 100) TrainY shape = (50,)
ValidationX Shape = (25, 100) ValY shape = (25,)
TestX Shape = (925, 100) TestY shape = (925,)
Now, converting into QP and getting eigenvalues :
def quadratic_form(X,Y):
P = np.matmul(X.T,X)
Q = -2*np.array(np.matmul(Y.T,X))
YtY = np.matmul(Y.T,Y)
return P, Q, YtY
# Training, val and Test QP
P_tr, Q_tr, YtY_tr = quadratic_form( x_train , y_train )
P_val, Q_val, YtY_val = quadratic_form( x_val , y_val )
P_test, Q_test, YtY_test = quadratic_form( x_test , y_test )
np.linalg.eigvals(P_tr)
#Output :
array([ 4.95125590e-02+0.00000000e+00j, 3.59197391e-02+0.00000000e+00j,
3.37542388e-02+0.00000000e+00j, 2.42792525e-02+0.00000000e+00j,
1.59022501e-02+0.00000000e+00j, 1.45537102e-02+0.00000000e+00j,
1.07802480e-02+0.00000000e+00j, 9.44592832e-03+0.00000000e+00j,
8.84433990e-03+0.00000000e+00j, 9.23145161e-03+0.00000000e+00j,
7.92091792e-03+0.00000000e+00j, 7.58909877e-03+0.00000000e+00j,
5.83030469e-03+0.00000000e+00j, 5.03155542e-03+0.00000000e+00j,
4.47966869e-03+0.00000000e+00j, 3.74600673e-03+0.00000000e+00j,
3.43228263e-03+0.00000000e+00j, 3.27013758e-03+0.00000000e+00j,
2.82173367e-03+0.00000000e+00j, 2.72670221e-03+0.00000000e+00j,
2.38166848e-03+0.00000000e+00j, 2.31223051e-03+0.00000000e+00j,
1.93671892e-03+0.00000000e+00j, 1.78860097e-03+0.00000000e+00j,
1.68008668e-03+0.00000000e+00j, 1.54299547e-03+0.00000000e+00j,
1.40873741e-03+0.00000000e+00j, 1.09740264e-03+0.00000000e+00j,
1.19998089e-03+0.00000000e+00j, 9.80403331e-04+0.00000000e+00j,
9.03025762e-04+0.00000000e+00j, 8.31922845e-04+0.00000000e+00j,
7.53863100e-04+0.00000000e+00j, 7.11801927e-04+0.00000000e+00j,
6.05375236e-04+0.00000000e+00j, 5.28839466e-04+0.00000000e+00j,
4.93423676e-04+0.00000000e+00j, 4.60982147e-04+0.00000000e+00j,
4.25653579e-04+0.00000000e+00j, 2.04024243e-04+0.00000000e+00j,
2.52868938e-04+0.00000000e+00j, 2.85681188e-04+0.00000000e+00j,
3.54971061e-04+0.00000000e+00j, 3.48320578e-04+0.00000000e+00j,
1.67819656e-04+0.00000000e+00j, 1.36960992e-04+0.00000000e+00j,
7.17807793e-05+0.00000000e+00j, 8.51775301e-05+0.00000000e+00j,
1.02035484e-04+0.00000000e+00j, 1.05242480e-04+0.00000000e+00j,
3.13944338e-18+0.00000000e+00j, -2.55435320e-18+7.36995996e-19j,
-2.55435320e-18-7.36995996e-19j, -2.74984812e-18+0.00000000e+00j,
-2.18137804e-18+1.12544689e-19j, -2.18137804e-18-1.12544689e-19j,
2.32659249e-18+7.44570928e-20j, 2.32659249e-18-7.44570928e-20j,
1.94702401e-18+4.43765321e-19j, 1.94702401e-18-4.43765321e-19j,
-1.77691688e-18+0.00000000e+00j, 1.72290651e-18+0.00000000e+00j,
-1.58792504e-18+0.00000000e+00j, -1.36379048e-18+5.13259206e-19j,
-1.36379048e-18-5.13259206e-19j, -1.31631049e-18+0.00000000e+00j,
1.61107269e-18+1.89156345e-19j, 1.61107269e-18-1.89156345e-19j,
1.13646471e-18+8.04003199e-19j, 1.13646471e-18-8.04003199e-19j,
-1.16234738e-18+3.42680578e-19j, -1.16234738e-18-3.42680578e-19j,
-1.14431437e-18+1.74642682e-19j, -1.14431437e-18-1.74642682e-19j,
1.43019099e-18+2.69165895e-19j, 1.43019099e-18-2.69165895e-19j,
-5.25583651e-19+6.24253440e-19j, -5.25583651e-19-6.24253440e-19j,
-1.33992327e-19+7.06675184e-19j, -1.33992327e-19-7.06675184e-19j,
-8.22843670e-19+1.30454635e-20j, -8.22843670e-19-1.30454635e-20j,
1.04544115e-18+0.00000000e+00j, -6.73238048e-19+0.00000000e+00j,
9.03262709e-19+3.30466756e-19j, 9.03262709e-19-3.30466756e-19j,
5.04446680e-19+5.35740963e-19j, 5.04446680e-19-5.35740963e-19j,
-1.66403419e-19+2.50921451e-19j, -1.66403419e-19-2.50921451e-19j,
-3.72846642e-19+1.15075692e-19j, -3.72846642e-19-1.15075692e-19j,
7.63884776e-19+3.00700070e-19j, 7.63884776e-19-3.00700070e-19j,
6.09980347e-19+0.00000000e+00j, 3.61128263e-19+2.85977381e-19j,
3.61128263e-19-2.85977381e-19j, 1.58873504e-19+0.00000000e+00j,
5.01289831e-19+1.26078990e-19j, 5.01289831e-19-1.26078990e-19j])
I don't know what is it that I am missing here, Some help please :)
I'm currently training a tensorflow model with GPU.
I tested the code below and GPU until rose up to 60%, which means my GPU settings are just fine.
mnist = tf.keras.datasets.mnist
(x_train, y_train), (x_test, y_test) = mnist.load_data()
x_train, x_test = x_train / 255.0, x_test / 255.0
model = tf.keras.models.Sequential([
tf.keras.layers.Flatten(input_shape=(28, 28)),
tf.keras.layers.Dense(1024, activation='relu'),
tf.keras.layers.Dense(1024, activation='relu'),
tf.keras.layers.Dense(1024, activation='relu'),
tf.keras.layers.Dropout(0.2),
tf.keras.layers.Dense(10, activation='softmax')
])
model.compile(optimizer='adam',
loss='sparse_categorical_crossentropy',
metrics=['accuracy'])
model.fit(x_train, y_train, epochs=50)
Training my custom model, I found that GPU is not utilized at all (0% when nvidia-smi) only when it is training.
After training, calculating validation accuracy(in the same epoch with training), GPU until gets high enough.
At the second epoch, training gets started, GPU until goes down to 0 again.
What is going on here?
####################### MODEL #######################
class FM_layer(tf.keras.layers.Layer):
def __init__(self, field_dims, embedding_size):
super(FM_layer, self).__init__()
self.w0 = tf.Variable(tf.random.normal(shape=[1],
mean=0.0,
stddev=0.1),
name='w0')
self.wixi_embeddings = tf.keras.layers.Embedding(input_dim = sum(field_dims),
output_dim=1,
name='wixi',
embeddings_initializer=tf.keras.initializers.GlorotNormal()
)
def call(self, inputs, latent_inputs):
# linear_terms
sum_wixi = tf.math.reduce_sum(self.wixi_embeddings(inputs=inputs), axis=1)
linear_terms = tf.math.add(sum_wixi, self.w0)
# interarction _terms
square_of_sum = tf.square(tf.reduce_sum(latent_inputs, [1], keepdims=True) )
sum_of_square = tf.reduce_sum(tf.square(latent_inputs), [1], keepdims=True)
interaction_terms = 0.5 * tf.reduce_sum(square_of_sum - sum_of_square, [2])
linear_terms = tf.reshape(linear_terms, [-1, 1])
interaction_terms = tf.reshape(interaction_terms, [-1, 1])
y_fm = tf.math.add(linear_terms , interaction_terms) # batch_size, 1
return y_fm
class Deep_layer(tf.keras.layers.Layer):
def __init__(self, field_dims, embedding_size):
super(Deep_layer, self).__init__()
self.field_dims = field_dims
self.embedding_size = embedding_size
self.layers1 = tf.keras.layers.Dense(units=512, activation='relu', kernel_initializer=tf.keras.initializers.GlorotNormal())
self.dropout1 = tf.keras.layers.Dropout(rate=0.5)
self.layers2 = tf.keras.layers.Dense(units=512, activation='relu', kernel_initializer=tf.keras.initializers.GlorotNormal())
self.dropout2 = tf.keras.layers.Dropout(rate=0.5)
self.layers3 = tf.keras.layers.Dense(units=512, activation='relu', kernel_initializer=tf.keras.initializers.GlorotNormal())
self.dropout3 = tf.keras.layers.Dropout(rate=0.5)
self.layers4 = tf.keras.layers.Dense(units=512, activation='relu', kernel_initializer=tf.keras.initializers.GlorotNormal())
self.dropout4 = tf.keras.layers.Dropout(rate=0.5)
self.batch1 = tf.keras.layers.BatchNormalization()
self.batch2 = tf.keras.layers.BatchNormalization()
self.batch3 = tf.keras.layers.BatchNormalization()
self.batch4 = tf.keras.layers.BatchNormalization()
self.final = tf.keras.layers.Dense(units=1, activation='tanh')#tf.nn.leaky_relu ) # relu
def call(self, latent_inputs):
latent_inputs = tf.reshape(latent_inputs, [-1, len(self.field_dims)*self.embedding_size]) #flatten
y_deep = self.layers1(latent_inputs)
y_deep = self.batch1(y_deep)
y_deep = self.dropout1(y_deep)
y_deep = self.layers2(y_deep)
y_deep = self.batch2(y_deep)
y_deep = self.dropout2(y_deep)
y_deep = self.layers3(y_deep)
y_deep = self.batch3(y_deep)
y_deep = self.dropout3(y_deep)
y_deep = self.layers4(y_deep)
y_deep = self.batch4(y_deep)
y_deep = self.dropout4(y_deep)
y_deep = self.final(y_deep)
return y_deep
class DeepFM(tf.keras.Model):
def __init__(self, field_dims, offset, embedding_size):
super(DeepFM, self).__init__()
self.field_dims = field_dims
self.offset = offset
self.embedding_size = embedding_size
self.V_embeddings = tf.keras.layers.Embedding(input_dim= sum(field_dims),
output_dim=embedding_size,
name='V',
embeddings_initializer=tf.keras.initializers.GlorotNormal()
)
self.fm_layer = FM_layer(field_dims, embedding_size)
self.deep_layer = Deep_layer(field_dims, embedding_size)
def call(self, inputs):
inputs = tf.math.add(inputs, self.offset)
latent_inputs = self.V_embeddings(inputs=inputs)
# 1) fm Component
y_fm = self.fm_layer(inputs, latent_inputs)
# 2) Deep Component
y_deep = self.deep_layer(latent_inputs)
# Concatenation
y_fm = tf.reshape(y_fm, [-1, 1])
y_deep = tf.reshape(y_deep, [-1, 1])
y_pred = tf.math.sigmoid(tf.math.add(y_fm, y_deep))
y_pred = tf.reshape(y_pred, [-1, 1])
return y_pred
deepfm_model = DeepFM(
field_dims=FIELD_DIMS,
offset = OFFSET,
embedding_size=EMBEDDING_SIZE
)
deepfm_model.compile(
loss=BinaryCrossentropy(),
optimizer = 'Adam',
metrics=['binary_accuracy', 'AUC']
)
history = deepfm_model.fit(
X_train,
Y_train,
epochs= 30,
validation_data=(X_val[:], Y_val[:]),
batch_size = 1024
)
Is my custom model not able to use GPU fundamentally?
I am implementing artificial bee colony optimization in ANN using [this][1] api. but i am getting this error. This is my code:
def ANN(optimizer = "adam", neurons = 32, batch_size = 32, epochs = 50, activation = "relu", patience =5, loss = 'mse'):
model = Sequential()
model.add(Dense(neurons, input_dim=look_back, activation= activation))
model.add(Dense(neurons, activation= activation))
model.add(Dense(1))
model.compile(optimizer = optimizer, loss = loss)
early_stopping = EarlyStopping(monitor = "loss", patience = patience)
history = model.fit(x_train, y_train, batch_size = batch_size, epochs = epochs, callbacks = [early_stopping], verbose = 0)
return model
boundaries = [(0,2), (0,2), (0,2), (0,2), (10,100), (20,50), (3,20)]
def performance(x_train, y_train, x_test, y_test, optimizer = None, activation = None, loss = None, batch_size = None, neurons = None, epochs = None, patience=None):
model = ANN(optimizer=optimizer, activation= activation, loss=loss, batch_size=batch_size, neurons= neurons, epochs = epochs, patience=patience)
trainScore = model.evaluate(x_train, y_train, verbose=0)
print('Train Score: %.2f MSE (%.2f RMSE)' % (trainScore, math.sqrt(trainScore)))
testScore = model.evaluate(x_test, y_test, verbose=0)
print('Test Score: %.2f MSE (%.2f RMSE)' % (testScore, math.sqrt(testScore)))
trainPredict = model.predict(x_train)
testPredict = model.predict(x_test)
#calculate mean absolute percent error
trainMAPE = mean_absolute_error(y_train, trainPredict)
testMAPE = mean_absolute_error(y_test, testPredict)
return print('testMAPE: %.2f MAPE' % trainMAPE), print('testMAPE: %.2f MAPE' % testMAPE)
writer = pd.ExcelWriter('/content/Scores.xlsx')
for sheetNum in range(1,5):
dataframe = pd.read_excel('Fri.xlsx',sheet_name='Sheet'+str(sheetNum))
# load the dataset
dataset = dataframe.values
dataset = dataset.astype('float32')
train_size = int(len(dataset) * 0.48)
test_size = len(dataset) - train_size
train, test = dataset[0:train_size, :], dataset[train_size:len(dataset), :]
# reshape into X=t and Y=t+1
look_back = 10
x_train, y_train = create_dataset(train, look_back)
x_test, y_test = create_dataset(test, look_back)
# normalize the dataset
scaler = MinMaxScaler(feature_range=(0, 1))
x_train = scaler.fit_transform(x_train)
x_test = scaler.fit_transform(x_test)
abc_obj = abc(performance(x_train, y_train, x_test, y_test), boundaries)
abc_obj.fit()
#Get solution obtained after fit() execution:
solution = abc_obj.get_solution()
This is my error:
TypeError Traceback (most recent call last)
<ipython-input-38-f9098d8d18fc> in <module>()
23 x_train = scaler.fit_transform(x_train)
24 x_test = scaler.fit_transform(x_test)
---> 25 abc_obj = abc(performance(x_train, y_train, x_test, y_test), boundaries)
26 abc_obj.fit()
27
2 frames
/usr/local/lib/python3.7/dist-packages/keras/layers/core.py in __init__(self, units, activation, use_bias, kernel_initializer, bias_initializer, kernel_regularizer, bias_regularizer, activity_regularizer, kernel_constraint, bias_constraint, **kwargs)
1144 activity_regularizer=activity_regularizer, **kwargs)
1145
-> 1146 self.units = int(units) if not isinstance(units, int) else units
1147 self.activation = activations.get(activation)
1148 self.use_bias = use_bias
TypeError: int() argument must be a string, a bytes-like object or a number, not 'NoneType'
can you help me with this, please? I think i am not defining the function "performance" correctly. but I don't understand how can I make it better.
[1]: https://pypi.org/project/beecolpy/
I'm trying to apply logistic regression on my dataset but its giving accuracy of 1
df = pd.read_csv("train.csv", header=0)
df = df[["PassengerId", "Survived", "Sex", "Age", "Embarked"]]
df.dropna(inplace=True)
X = df[["Sex", "Age"]]
X_train = np.array(X)
Y = df["Survived"]
Y_train = np.array(Y)
clf = LogisticRegression()
clf.fit(X_train, Y_train)
df1 = pd.read_csv("test.csv", header=0)
df1 = df1[["PassengerId", "Survived", "Sex", "Age", "Embarked"]]
df1.dropna(inplace=True)
X = df1[["Sex", "Age"]]
X_test = np.array(X)
Y = df1["Survived"]
Y_test = np.array(Y)
X_test = X_test.astype(float)
Y_test = Y_test.astype(float)
#to convert string data to float
accuracy = clf.score(X_test, Y_test)
print("Accuracy = ", accuracy)
I expect the output between 0 and 1, but always getting 1.0
Amateur problem but i cannot solve this issue on my own.
I was trying to make a neural network for churn modelling dataset on bank data
Every time i run this network i get an accuracy of 1.0 so i think there is something wrong and its not working.
Can anyone help me figure out what is wrong?
Also please explain how i can avoid problems like these in the future
The code is :
import pandas as pd
import numpy as np
data = pd.read_csv('D:\Churn_Modelling.csv')
X = data.iloc[:, 3:13].values
Y = data.iloc[:, 13].values
from sklearn.preprocessing import LabelEncoder, OneHotEncoder
label_encoder_x_1 = LabelEncoder()
X[:, 1] = label_encoder_x_1.fit_transform(X[:, 1])
label_encoder_x_2 = LabelEncoder()
X[:, 2] = label_encoder_x_2.fit_transform(X[:, 2])
one_hot_encoder = OneHotEncoder(categorical_features = [1])
X = one_hot_encoder.fit_transform(X).toarray()
X = X[:, 1:]
from sklearn.model_selection import train_test_split
X_train, X_test, Y_train, Y_test =
train_test_split(X, Y, test_size = 0.2)
from sklearn.preprocessing import StandardScaler
sc = StandardScaler()
X_train = sc.fit_transform(X_train)
X_test = sc.fit_transform(X_test)
import tensorflow as tf
epochs = 20
batch_size = 50
learning_rate = 0.003
n_output = 1
n_input = X_train.shape[1]
X_placeholder = tf.placeholder("float32", [None, n_input], name = "X")
Y_placeholder = tf.placeholder("float32", [None, 1], name = "y")
n_neurons_1 = 64
n_neurons_2 = 32
n_neurons_3 = 16
layer_1 = {'weights': tf.Variable
(tf.random_normal([n_input, n_neurons_1])),
'biases': tf.Variable(tf.random_normal([n_neurons_1]))
}
layer_2 = {'weights': tf.Variable
(tf.random_normal([n_neurons_1, n_neurons_2])),
'biases': tf.Variable(tf.random_normal([n_neurons_2]))
}
layer_3 = {'weights': tf.Variable
(tf.random_normal([n_neurons_2, n_neurons_3])),
'biases': tf.Variable(tf.random_normal([n_neurons_3]))
}
output_layer = {'weights': tf.Variable(
tf.random_normal([n_neurons_3, n_output])),
'biases': tf.Variable(tf.random_normal([n_output]))
}
l1 = tf.add(tf.matmul(X_placeholder,
layer_1['weights']), layer_1['biases'])
l1 = tf.nn.relu(l1)
l2 = tf.add(tf.matmul(l1, layer_2['weights']),
layer_2['biases'])
l2 = tf.nn.relu(l2)
l3 = tf.add(tf.matmul(l2, layer_3['weights']),
layer_3['biases'])
l3 = tf.nn.relu(l3)
output_layer = tf.matmul(l3,
output_layer['weights']) + output_layer['biases']
output_layer = tf.nn.sigmoid(output_layer)
cost = tf.reduce_mean(tf.reduce_sum(
tf.square(Y_placeholder - output_layer), reduction_indices = [1]))
optimizer = tf.train.AdamOptimizer().minimize(cost)
correct_prediction = tf.equal(tf.argmax(
Y_placeholder, 1), tf.argmax(output_layer, 1))
accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32))
def next_batch(size, x, y):
idx = np.arange(0, len(x))
np.random.shuffle(idx)
idx = idx[:size]
x_shuffle = [x[ i] for i in idx]
y_shuffle = [y[ i] for i in idx]
return np.asarray(x_shuffle), np.asarray(y_shuffle)
with tf.Session() as sess:
sess.run(tf.global_variables_initializer())
total_batches = int(len(X_train) / batch_size)
for epoch in range(epochs):
avg_cost = 0
print('epoch: ', epoch)
for batch in range(total_batches):
x_batch_data, y_batch_data =
next_batch(batch_size, X_train, Y_train)
y_batch_data = y_batch_data.reshape((50, 1))
_, c = sess.run([optimizer, cost],
feed_dict = {X_placeholder: x_batch_data,
Y_placeholder: y_batch_data})
avg_cost += c / total_batches
print("Epoch:", (epoch + 1), "cost =", "{:.3f}".format(avg_cost))
Y_test_temp = Y_test.reshape((2000, 1))
print('accuracy: ', sess.run(accuracy,
feed_dict = {X_placeholder: X_test, Y_placeholder: Y_test_temp}))