Deep Q learning-Training issue - python-3.x

Question

A snake game was created using Pygame and I tried to solve it using an AI. Initially I didn't increase the body length to check if the snake head moves towards the food. Grid size is 5*5. DDQN network was used. Most of time the head moves towards the wall or gets struck in a continuous loop.The maximum score attained was 4 even if I train it for 5000 episodes.
State: It is an array of size 16. The first 8 values has the distance between the head and wall at 8 directions(left , left top, top, right top, right , right bottom, bottom, left bottom). Next 8 values has the distance between head and food at 8 directions. All the values are in the range 0 to 1. 1 means the object is near and 0 means it is very far.
Action : There are 3 actions 0,1,2. 0- Head moves in same direction. 1- Head turns left. 2- Head turns right.
Reward: Reward of +50 if it collects the food and reward of -200 if it touches the wall.
I am not able to understand why my neural network learns in the wrong way. Please do help me solve this issue. I have attached the code here.
Code:
import pygame
pygame.font.init()
import time
import random
import numpy as np
from math import hypot
from collections import deque
from keras.models import Sequential
from keras.layers import Dense
from keras.optimizers import Adam
import tensorflow as tf
import os
seeds = 1001
os.environ['PYTHONHASHSEED']=str(seeds)
np.random.seed(seeds)
random.seed(seeds)
tf.random.set_seed(seeds)
batch = 16
class Food:
def __init__(self,width,n):
self.width = width
self.n = n
self.size = width //n
self.blocks = self.blocks_total()
def blocks_total(self):# Total available positions
x = y = 0
s=[]
for i in range(self.n):
for j in range(self.n):
s.append([x,y])
y += self.size
x += self.size
y = 0
return s
def food_pos(self,s):# Random food position
food_blocks = self.blocks_total()
try:
for i in self.blocks:
for j in s:
if i[0] == j[0] and i[1] == j[1]:
food_blocks.remove([j[0],j[1]])
break
a = random.choice(food_blocks)
return a
except:
return 0.1,0.1
class Agent:
def __init__(self,width,n,state_size,action_size=3,gamma = 0.98):
self.width = width
self.n = n
self.size = width //n
self.state_size = state_size
self.action_size = action_size
self.gamma = gamma
self.epsilon = 1
self.epsilon_min = 0
self.epsilon_decay = 0.99
self.memory = deque(maxlen=5000)
self.model = self.build_model()
self.train_model = self.build_model()
def reshape(self,state):# Reshaping state for input in nueral network
return np.reshape(state,[1,state.shape[0]])
def build_model(self):# Nueral network
model = Sequential()
model.add(Dense(16,input_shape=(self.state_size,),activation='relu'))
model.add(Dense(12,activation='relu'))
model.add(Dense(12,activation='relu'))
model.add(Dense(3,activation='softmax'))
model.compile(loss='mse',optimizer=Adam(0.0001))
return model
def get_action(self,state):
if np.random.rand() <= self.epsilon:
return random.randint(0,2)
a = self.reshape(state)
p = self.model.predict(a)
return np.argmax(p[0])
def remember(self,state,action,reward,new_state,done): # Saving in memory
self.memory.append((state,action,reward,new_state,done))
def replay(self): # Training of nueral network
minibatch = random.sample(self.memory,batch)
for state,action,reward,new_state,done in minibatch:
target = reward
state = self.reshape(state)
new_state = self.reshape(new_state)
if not done:
target = reward +(self.gamma*(np.max(self.train_model.predict(new_state)[0])))
target_f = self.model.predict(state)
target_f[0][action] = target
self.model.fit(state,target_f,epochs =1, verbose = 0)
if self.epsilon > self.epsilon_min:
self.epsilon *= self.epsilon_decay
def save_model(self):
self.model.save_weights('nn1.h5')
def load_model(self):
self.train_model.load_weights('nn1.h5')
class Game:
def __init__(self,width=500,n=5):
self.width = width
self.n = n
self.size = width//n
self.display = pygame.display.set_mode((width,width))
pygame.display.set_caption('A')
self.food = Food(width,n)
self.agent = Agent(width, n, state_size=16)
self.game_over = False
self.dirc = 'r'# starting direction of snake
self.snake_list = []
self.snake_length = 1
self.wall_touch = False
def display_player(self,disp,s_list,dirc):# displaying the snake head and its eyes
l = len(s_list)
if l > self.snake_length:
del s_list[0]
l -=1
for idx,i in enumerate(s_list):
if idx == l-1:
pygame.draw.rect(disp,(255,255,255),(i[0],i[1],self.size-1,self.size-1))
else:
pygame.draw.rect(disp,(255,165,0),(i[0],i[1],self.size-1,self.size-1))
a = s_list.copy()
x,y = a.pop()
rad = 10
if dirc == 'u':
pygame.draw.circle(disp,(0,0,0),(x+50,y+30),rad)
elif dirc == 'd':
pygame.draw.circle(disp,(0,0,0),(x+50,y+70),rad)
elif dirc == 'r':
pygame.draw.circle(disp,(0,0,0),(x+70,y+49),rad)
elif dirc == 'l':
pygame.draw.circle(disp,(0,0,0),(x+30,y+49),rad)
def move(self,dirc,s_list):# Constantly moving the snake on that particular direction
x,y = s_list.pop()
if dirc == 'l':
x -= self.size
if x <0:
x = 0
self.wall_touch = True
elif dirc == 'r':
x += self.size
if x > self.width - self.size:
x = self.width - self.size
self.wall_touch = True
elif dirc == 'u':
y -= self.size
if y <0:
y = 0
self.wall_touch = True
elif dirc == 'd':
y += self.size
if y > self.width - self.size:
y = self.width - self.size
self.wall_touch = True
self.snake_list.append([x,y])
def check_food_collect(self,fx,fy,s_list):# Check if head position and food position are same
x,y =s_list.pop()
if x == fx and y == fy:
return True
return False
def display_msg(self,msg,font='freesansbold.ttf',size=15,color=(255,255,255),loc=(410,15)):
mymsg = pygame.font.Font(font,size).render(msg,True,color)
self.display.blit(mymsg,loc)
def get_direction(self,change):# Changing the direction of snake based on the current direction
if self.dirc == 'r':
if change == 'r':
self.dirc= 'd'
elif change == 'l':
self.dirc= 'u'
elif self.dirc == 'l':
if change == 'r':
self.dirc= 'u'
elif change == 'l':
self.dirc= 'd'
elif self.dirc == 'd' :
if change == 'r':
self.dirc= 'l'
elif change == 'l':
self.dirc= 'r'
elif self.dirc == 'u':
if change == 'r':
self.dirc= 'r'
elif change == 'l':
self.dirc= 'l'
def near_wall(self,n,s_list): # Distance of nearby wall
a,b = n
x,y = s_list.pop()
i =0
while True:
xx = x +(self.size * i * a)
yy = y +(self.size * i * b)
dis = abs(xx-x)/self.size,abs(yy-y)/self.size
i +=1
if xx <0 or yy <0 or xx > self.width - self.size or yy > self.width - self.size:
return 1/ hypot(dis[0],dis[1])
def get_wall_dis(self,s_list): # Wall distance at 8 direction
j = [[-1,0],[-1,-1],[0,-1],[1,-1],[1,0],[1,1],[0,1],[-1,1]]
s = []
for i in j:
s.append(self.near_wall(i, s_list.copy()))
s = np.asarray(s)
return s
def near_food(self,fx,fy,n,s_list):# Head looks at 8 direction for the food
a,b = n
x,y = s_list.pop()
i =0
while True:
xx = x +(self.size * i * a)
yy = y +(self.size * i * b)
dis = abs(x-fx)/self.size,abs(fy-y)/self.size
i +=1
if xx <0 or yy <0 or xx > self.width - self.size or yy > self.width - self.size:
return 0
else:
if xx == fx and yy == fy:
if dis[0] ==0 and dis[1] ==0:
return 0
return 1/hypot(dis[0],dis[1])
def get_state(self,fx,fy,s_list,w): # Array of size 16
j = [[-1,0],[-1,-1],[0,-1],[1,-1],[1,0],[1,1],[0,1],[-1,1]]
s = []
for i in j:
s.append(self.near_food(fx,fy,i, s_list.copy()))
s = np.asarray(s)
a = np.append(w,s)
return a
def reset(self): # Initialising the value when a new game is started
self.agent.load_model()
self.game_over = False
self.wall_touch = False
self.dirc = 'r'
self.snake_length = 1
self.snake_list = []
def startgame(self,e):
sx,sy = 0,0 # Starting position of snake
self.snake_list.append([sx,sy])
fx,fy = 200,200 # Initial position of food
step = 0
action = 0
score = 0
change = None
wall = self.get_wall_dis(self.snake_list.copy())
state = self.get_state(fx, fy, self.snake_list.copy(), wall)
reward = 0
t = 0
save = False
j = 0
while not self.game_over:
j +=1
if j >500: # If snake struck in continuous loop
print('Ended')
break
action = 0
action = self.agent.get_action(state)
if action ==1:
change = 'l'
step = 1
elif action ==2:
change = 'r'
step = 1
else:
action = 0
change = None
if step == 0:
step = 1
else:
t +=1
save = True
self.get_direction(change)
change = None
self.move(self.dirc,self.snake_list.copy())
wall = self.get_wall_dis(self.snake_list.copy())
new_state = self.get_state(fx, fy, self.snake_list.copy(), wall)
if self.wall_touch:
reward = -200
print('Walled')
self.agent.remember(state, action, reward, new_state, True)
break
food_collect = self.check_food_collect(fx, fy, self.snake_list.copy())
if food_collect:
reward += 50
self.agent.remember(state, action, reward, new_state, False)
save = False
score +=1
step = 0
t = 0
reward = 0
fx,fy = self.food.food_pos(self.snake_list.copy())
if fx == 0.1 and fy ==0.1:
print('COMPLETED')
pygame.quit()
if save:
save = False
self.agent.remember(state, action, reward, new_state, False)
self.display.fill((0,0,0))
self.display_msg('Score :'+str(score))
pygame.draw.rect(self.display,(0,255,0),(fx,fy,self.size-1,self.size-1))
self.display_player(self.display, self.snake_list,self.dirc)
pygame.display.update()
#time.sleep(1)
state = new_state
if e > 2500:
time.sleep(0.1)
print('E : {} , Epsilon :{:.2} , Score : {}'.format(e,np.float32(self.agent.epsilon),score))
if e%10 ==0:
self.agent.save_model()
if len(self.agent.memory) > batch:
self.agent.replay()
game = Game()
for e in range(10000):
game.startgame(e)
game.reset()
pygame.quit()

Answer 1

A few things you could try:
The way you have defined your state looks a little complicated. Won't top and bottom give the same information, one being negative of the other? Also if your snake head is at (1,1) and your fruit at (3,4), well then the fruit won't show up in the state at all. There will be very limited times when the snake agent can actually see the fruit. Maybe you can try defining the state another way?
In RL, things go south very frequently so it often makes sense to start with basic agents and basic games and move up the ladder. Try using the same agent for a simple openai gym environment like mountaincar, to check if the agent class works as intended.