DDPG not converging for VRep maze environment - pytorch

Question

I'm sorry for the lengthy post, just wanted to provide with implementation details in advance.
Also, sorry for the code, I know it's a mess.
I've been using a combination of PyTorch and VRep to setup environment for the differential drive robot to learn maneuvering the maze. The robot only has 6 IC proximity sensors along its rim that measure distances from 10-80cm. As a first simple version, I've created VRep environment containing just a narrow corridor (width of the corridor is ~2*robot_width). Starting position for the robot is somewhere in the middle of the corridor, with the goal point being somewhere at the end of the corridor, with goal velocities being ~0 m/s. Idea is that my learned agent handles both navigation and low level robot control at the same time.
Now, I've been using preexisting DDPG implementation which works on Gym environments, but it does not seem to converge for my case. I thought that using only 6 proximity sensors is the part of the problem, so I've introduced to the state vector position and velocity readings, together with errors for the desired state (as some papers suggest).
I would appreciate any help.
ddpg_agent
class Agent():
"""Interacts with and learns from the environment."""
def __init__(self, state_size, action_size, random_seed):
"""Initialize an Agent object.
Params
======
state_size (int): dimension of each state
action_size (int): dimension of each action
random_seed (int): random seed
"""
self.state_size = state_size
self.action_size = action_size
self.seed = random.seed(random_seed)
# Actor Network (w/ Target Network)
self.actor_local = Actor(state_size, action_size, random_seed).to(device)
self.actor_target = Actor(state_size, action_size, random_seed).to(device)
self.actor_optimizer = optim.Adam(self.actor_local.parameters(), lr=LR_ACTOR)
# Critic Network (w/ Target Network)
self.critic_local = Critic(state_size, action_size, random_seed).to(device)
self.critic_target = Critic(state_size, action_size, random_seed).to(device)
self.critic_optimizer = optim.Adam(self.critic_local.parameters(), lr=LR_CRITIC, weight_decay=WEIGHT_DECAY)
# Noise process
self.noise = OUNoise(action_size, random_seed)
# Replay memory
self.memory = ReplayBuffer(action_size, BUFFER_SIZE, BATCH_SIZE, random_seed)
def step(self, state, action, reward, next_state, done):
"""Save experience in replay memory, and use random sample from buffer to learn."""
# Save experience / reward
self.memory.add(state, action, reward, next_state, done)
def act(self, state, add_noise=True):
"""Returns actions for given state as per current policy."""
state = torch.from_numpy(state).float().to(device)
self.actor_local.eval()
with torch.no_grad():
action = self.actor_local(state).cpu().data.numpy()
self.actor_local.train()
if add_noise:
action += self.noise.sample()
return np.clip(action, -2, 2)
def reset(self):
self.noise.reset()
def start_learn(self):
if len(self.memory) > MIN_BUFFER_SIZE:
experiences = self.memory.sample()
self.learn(experiences, GAMMA)
def learn(self, experiences, gamma):
"""Update policy and value parameters using given batch of experience tuples.
Q_targets = r + γ * critic_target(next_state, actor_target(next_state))
where:
actor_target(state) -> action
critic_target(state, action) -> Q-value
Params
======
experiences (Tuple[torch.Tensor]): tuple of (s, a, r, s', done) tuples
gamma (float): discount factor
"""
states, actions, rewards, next_states, dones = experiences
# ---------------------------- update critic ---------------------------- #
# Get predicted next-state actions and Q values from target models
actions_next = self.actor_target(next_states)
Q_targets_next = self.critic_target(next_states, actions_next)
# Compute Q targets for current states (y_i)
Q_targets = rewards + (gamma * Q_targets_next * (1 - dones)).detach()
# Compute critic loss
Q_expected = self.critic_local(states, actions)
critic_loss = F.mse_loss(Q_expected, Q_targets)
# Minimize the loss
self.critic_optimizer.zero_grad()
critic_loss.backward()
# torch.nn.utils.clip_grad_norm_(self.critic_local.parameters(), 1)
self.critic_optimizer.step()
# ---------------------------- update actor ---------------------------- #
# Compute actor loss
actions_pred = self.actor_local(states)
actor_loss = -self.critic_local(states, actions_pred).mean()
# Minimize the loss
self.actor_optimizer.zero_grad()
actor_loss.backward()
self.actor_optimizer.step()
# ----------------------- update target networks ----------------------- #
self.soft_update(self.critic_local, self.critic_target, TAU)
self.soft_update(self.actor_local, self.actor_target, TAU)
def soft_update(self, local_model, target_model, tau):
"""Soft update model parameters.
θ_target = τ*θ_local + (1 - τ)*θ_target
Params
======
local_model: PyTorch model (weights will be copied from)
target_model: PyTorch model (weights will be copied to)
tau (float): interpolation parameter
"""
for target_param, local_param in zip(target_model.parameters(), local_model.parameters()):
target_param.data.copy_(tau*local_param.data + (1.0-tau)*target_param.data)
def save_samples(self):
fileSamples = open('samples.obj', 'w')
pickle.dump(self.memory, fileSamples)
def load_samples(self):
fileSamples = open('samples.obj', 'r')
return pickle.load(fileSamples)
class OUNoise:
"""Ornstein-Uhlenbeck process."""
def __init__(self, size, seed, mu=0., theta=0.15, sigma=0.2):
"""Initialize parameters and noise process."""
self.mu = mu * np.ones(size)
self.theta = theta
self.sigma = sigma
self.seed = random.seed(seed)
self.reset()
def reset(self):
"""Reset the internal state (= noise) to mean (mu)."""
self.state = copy.copy(self.mu)
def sample(self):
"""Update internal state and return it as a noise sample."""
x = self.state
dx = self.theta * (self.mu - x) + self.sigma * np.array([np.random.randn() for i in range(len(x))])
self.state = x + dx
return self.state
class ReplayBuffer:
"""Fixed-size buffer to store experience tuples."""
def __init__(self, action_size, buffer_size, batch_size, seed):
"""Initialize a ReplayBuffer object.
Params
======
buffer_size (int): maximum size of buffer
batch_size (int): size of each training batch
"""
self.action_size = action_size
self.memory = deque(maxlen=buffer_size) # internal memory (deque)
self.batch_size = batch_size
self.experience = namedtuple("Experience", field_names=["state", "action", "reward", "next_state", "done"])
self.seed = random.seed(seed)
def add(self, state, action, reward, next_state, done):
"""Add a new experience to memory."""
e = self.experience(state, action, reward, next_state, done)
self.memory.append(e)
def sample(self):
"""Randomly sample a batch of experiences from memory."""
experiences = random.sample(self.memory, k=self.batch_size)
states = torch.from_numpy(np.vstack([e.state for e in experiences if e is not None])).float().to(device)
actions = torch.from_numpy(np.vstack([e.action for e in experiences if e is not None])).float().to(device)
rewards = torch.from_numpy(np.vstack([e.reward for e in experiences if e is not None])).float().to(device)
next_states = torch.from_numpy(np.vstack([e.next_state for e in experiences if e is not None])).float().to(device)
dones = torch.from_numpy(np.vstack([e.done for e in experiences if e is not None]).astype(np.uint8)).float().to(device)
return (states, actions, rewards, next_states, dones)
def __len__(self):
"""Return the current size of internal memory."""
return len(self.memory)
Model:
class Actor(nn.Module):
"""Actor (Policy) Model."""
def __init__(self, state_size, action_size, seed, fc1_units=600, fc2_units=400, fc3_units=300):
"""Initialize parameters and build model.
Params
======
state_size (int): Dimension of each state
action_size (int): Dimension of each action
seed (int): Random seed
fc1_units (int): Number of nodes in first hidden layer
fc2_units (int): Number of nodes in second hidden layer
"""
super(Actor, self).__init__()
self.seed = torch.manual_seed(seed)
self.fc1 = nn.Linear(state_size, fc1_units)
self.bn1 = nn.BatchNorm1d(fc1_units)
self.fc2 = nn.Linear(fc1_units, fc2_units)
self.fc3 = nn.Linear(fc2_units, fc3_units)
self.fc4 = nn.Linear(fc3_units, action_size)
self.reset_parameters()
def reset_parameters(self):
self.fc1.weight.data.uniform_(*hidden_init(self.fc1))
self.fc2.weight.data.uniform_(*hidden_init(self.fc2))
self.fc3.weight.data.uniform_(*hidden_init(self.fc3))
self.fc4.weight.data.uniform_(-3e-3, 3e-3)
def forward(self, state):
"""Build an actor (policy) network that maps states -> actions."""
# x = F.relu(self.bn1(self.fc1(state.unsqueeze(0))))
x = F.relu(self.fc1(state))
x = F.relu(self.fc2(x))
x = F.relu(self.fc3(x))
return torch.tanh(self.fc4(x))
class Critic(nn.Module):
"""Critic (Value) Model."""
def __init__(self, state_size, action_size, seed, fcs1_units=600, fc2_units=400, fc3_units=300):
"""Initialize parameters and build model.
Params
======
state_size (int): Dimension of each state
action_size (int): Dimension of each action
seed (int): Random seed
fcs1_units (int): Number of nodes in the first hidden layer
fc2_units (int): Number of nodes in the second hidden layer
"""
super(Critic, self).__init__()
self.seed = torch.manual_seed(seed)
self.fcs1 = nn.Linear(state_size, fcs1_units)
self.bn1 = nn.BatchNorm1d(fcs1_units)
self.fc2 = nn.Linear(fcs1_units+action_size, fc2_units)
self.fc3 = nn.Linear(fc2_units, fc3_units)
self.fc4 = nn.Linear(fc3_units, 1)
self.reset_parameters()
def reset_parameters(self):
self.fcs1.weight.data.uniform_(*hidden_init(self.fcs1))
self.fc2.weight.data.uniform_(*hidden_init(self.fc2))
self.fc3.weight.data.uniform_(*hidden_init(self.fc3))
self.fc4.weight.data.uniform_(-3e-3, 3e-3)
def forward(self, state, action):
"""Build a critic (value) network that maps (state, action) pairs -> Q-values."""
# xs = F.relu(self.bn1(self.fcs1(state.unsqueeze(0))))
xs = F.relu(self.fcs1(state))
x = torch.cat((xs, action), dim=1)
x = F.relu(self.fc2(x))
x = F.relu(self.fc3(x))
return self.fc4(x)
Main:
def main():
vrepHeadlessMode = True
state_dim = 18 # x, y, yaw, vx, vy, v_yaw, e_x, e_y, e_yaw, e_vx, e_vy, e_v_yaw, prox 0 ... prox5
action_dim = 2
action_space = np.array([[-2, 2], [-2, 2]])
action_lim = [-2.0, 2.0] # 2 o/sec is the max angular speed of each motor, max. linear velocity is 0.5 m/s
learn_every = 1 # number of steps after which the network update occurs [20]
num_learn = 1 # number of network updates done in a row [10]
episodes = 10000
steps = 500
desiredState = [-1.4, 0.3, -np.pi, 0.0, 0.0, 0.0] # x, y, yawAngle, vx, vy, yawVelocity
mobRob = MobRob(['MobRob'],
['leftMotor', 'rightMotor'],
['proximitySensor0', 'proximitySensor1', 'proximitySensor2', 'proximitySensor3', 'proximitySensor4',
'proximitySensor5'])
env = LabEnv(mobRob, vrepHeadlessMode)
random_seed = 7
mobRob = Agent(state_dim, action_dim, random_seed)
total_num_of_steps = 0
actions = np.zeros((episodes, steps+1, action_dim), dtype=np.float)
total_rewards = []
save_rewards = []
durations = []
for episode in range(episodes):
cur_state = env.restart(desiredState)
mobRob.reset()
start_time = time.time()
reason = ''
episode_rewards = []
for step in range(steps+1):
total_num_of_steps += 1
action = mobRob.act(cur_state)
actions[episode][step] = action
# print(action)
new_state, reward, done = env.step(action, desiredState)
mobRob.step(cur_state, action, reward, new_state, done)
cur_state = new_state
episode_rewards.append(reward)
if step % learn_every == 0:
for _ in range(num_learn):
mobRob.start_learn()
if step < steps and done and ~env.collision:
reason = 'COMPLETED'
break
if step == steps: # time budget for episode was overstepped
reason = 'TIMEOUT '
break
if env.collision:
reason = 'COLLISION'
break
mean_score = np.mean(episode_rewards)
min_score = np.min(episode_rewards)
max_score = np.max(episode_rewards)
total_rewards.append(mean_score)
duration = time.time() - start_time
durations.append(duration)
save_rewards.append([total_rewards[episode], episode])
eta = np.mean(durations)*(episodes-episode) / 60 / 60
if eta < 1.0:
etaString = str(np.round(eta * 60, 2)) + " min"
else:
etaString = str(np.round(eta, 2)) + " h"
print(
'\rEpisode {}\t{}\tMean episode reward: {:.2f}\tMin: {:.2f}\tMax: {:.2f}\tDuration: {:.2f}\tETA: {}'
.format(episode, reason, mean_score, min_score, max_score, duration, etaString))
gc.collect()
torch.save(mobRob.actor_local.state_dict(), './actor.pth')
torch.save(mobRob.critic_local.state_dict(), './critic.pth')
np.save('mean_episode_rewards', save_rewards)
if __name__ == "__main__":
main()
Typical output
Episode 3179 TIMEOUT Mean episode reward: 0.36 Min: 0.00 Max: 0.69 Duration: 36.86 ETA: 134.16 h
Episode 3180 TIMEOUT Mean episode reward: 0.22 Min: 0.00 Max: 0.72 Duration: 37.39 ETA: 134.12 h
Episode 3181 COLLISION Mean episode reward: 0.26 Min: -49.50 Max: 0.54 Duration: 29.11 ETA: 134.08 h
Episode 3182 COLLISION Mean episode reward: -0.39 Min: -50.00 Max: 0.21 Duration: 9.50 ETA: 134.02 h
Episode 3183 COLLISION Mean episode reward: -0.06 Min: -50.00 Max: 0.32 Duration: 27.10 ETA: 133.98 h
Episode 3184 COLLISION Mean episode reward: 0.38 Min: -49.32 Max: 0.69 Duration: 37.90 ETA: 133.94 h
Episode 3185 COLLISION Mean episode reward: -0.52 Min: -50.00 Max: 0.21 Duration: 7.28 ETA: 133.88 h
Position reached!
Speed reached!
Episode 3186 COMPLETED Mean episode reward: 0.39 Min: 0.00 Max: 80.72 Duration: 37.73 ETA: 133.84 h
Episode 3187 COLLISION Mean episode reward: 0.36 Min: -49.36 Max: 0.68 Duration: 34.68 ETA: 133.8 h
Episode 3188 COLLISION Mean episode reward: 0.32 Min: -49.43 Max: 0.62 Duration: 35.35 ETA: 133.76 h
Episode 3189 COLLISION Mean episode reward: -0.23 Min: -50.00 Max: 0.21 Duration: 16.59 ETA: 133.71 h
Episode 3190 TIMEOUT Mean episode reward: 0.39 Min: 0.00 Max: 0.65 Duration: 38.15 ETA: 133.67 h
Episode 3191 TIMEOUT Mean episode reward: 0.35 Min: 0.00 Max: 0.67 Duration: 37.76 ETA: 133.63 h
Position reached!
Episode 3192 COMPLETED Mean episode reward: 0.41 Min: 0.00 Max: 30.71 Duration: 36.36 ETA: 133.59 h
Episode 3193 TIMEOUT Mean episode reward: 0.35 Min: 0.00 Max: 0.72 Duration: 36.98 ETA: 133.55 h
Even after a high number of episodes, I cannot see any apparent rise in rewards.