Python: Cost keeps increasing in a neural network that uses TensorFlow - python-3.x

Question

I am trying to create a neural network using TensorFlow but my cost keeps increasing.
This is my code so far:
class AI_core:
def __init__(self, nodes_in_each_layer):
self.data_in_placeholder = tf.placeholder("float", [None, nodes_in_each_layer[0]])
self.data_out_placeholder = tf.placeholder("float")
self.init_neural_network(nodes_in_each_layer)
def init_neural_network(self, n_nodes_h):
#n_nodes_h contains the number of nodes for each layer
#n_nodes_h[0] = number of inputs
#n_nodes_h[-1] = number of outputs
self.layers = [None for i in range(len(n_nodes_h)-1)]
for i in range(1, len(n_nodes_h)):
self.layers[i-1] = {"weights":tf.Variable(tf.random_normal([n_nodes_h[i-1], n_nodes_h[i]])),
"biases":tf.Variable(tf.random_normal([n_nodes_h[i]]))}
def neural_network_model(self, data):
for i in range(len(self.layers)):
data = tf.matmul(data, self.layers[i]["weights"]) + self.layers[i]["biases"]
if i != len(self.layers):
data = tf.nn.relu(data)
return data
def train_neural_network(self, data):
prediction = self.neural_network_model(self.data_in_placeholder)
cost = tf.reduce_mean(tf.square(self.data_out_placeholder-prediction))
optimiser = tf.train.GradientDescentOptimizer(learning_rate=0.0001).minimize(cost)
with tf.Session() as sess:
sess.run(tf.global_variables_initializer())
epoch_loss = 0
for _ in range(int(data.length)):
epoch_x, epoch_y = data.next_batch()
c = sess.run(cost, feed_dict={self.data_in_placeholder: epoch_x, self.data_out_placeholder: epoch_y})
_ = sess.run(optimiser, feed_dict={self.data_in_placeholder: epoch_x, self.data_out_placeholder: epoch_y})
epoch_loss += np.sum(c)
print("loss =", epoch_loss)
For now I am trying to get the network to approximate the math.sin function.
I have set the nodes_in_each_layer = [1, 5, 5, 5, 1] and batch_size = 3. This is the output:
loss = 0.8417138457298279
loss = 1.190976768732071
loss = 1.8150676786899567
loss = 2.433938592672348
loss = 3.092040628194809
loss = 3.478498786687851
loss = 3.7894928753376007
loss = 4.598285228013992
loss = 5.418278068304062
loss = 5.555390268564224

Answer 1

It looks like you keep adding the value of loss with those in previous iterations.
with tf.Session() as sess:
sess.run(tf.global_variables_initializer())
epoch_loss = 0
for _ in range(int(data.length)):
epoch_x, epoch_y = data.next_batch()
c = sess.run(cost, feed_dict={self.data_in_placeholder: epoch_x, self.data_out_placeholder: epoch_y})
_ = sess.run(optimiser, feed_dict={self.data_in_placeholder: epoch_x, self.data_out_placeholder: epoch_y})
epoch_loss += np.sum(c)
print("loss =", epoch_loss)