Solving MountainCar-v0 by predicting future velocity? Why does this work? - openai-gym

Question

I solved the MountainCar-v0 OpenAI gym challenge by predicting future velocity based on current action. This system uses the absolute value of velocity as the reward function.
My question is why does this work?
My hypothesis is that this is working similar to q-learning, where we are creating a map/table of all actions and their results, then the action picker just picks the highest velocity. This works since velocity is linear, so picking the highest future velocity will very likely help you reach the next highest velocity.
Here is the code. It should take less than 3 minutes to reach it's goal.
import numpy as np
import gym
from collections import deque, namedtuple
import random
import torch
import torch.nn as nn
import torch.optim as optim
EPSILON_DECAY = 0.99975
MIN_EPSILON = 0.001
env = gym.make("MountainCar-v0")
class Model(nn.Module):
def __init__(self):
super(Model, self).__init__()
self.layer1 = nn.Linear(2, 128)
self.relu1 = nn.ReLU()
self.layer2 = nn.Linear(128, 3)
def forward(self, x):
l1 = self.relu1(self.layer1(x))
output = self.layer2(l1)
return output
model = Model()
target = Model()
target.load_state_dict(model.state_dict())
mse = nn.MSELoss()
optimizer = optim.Adam(model.parameters(), lr = 0.0001)
actionPoint = namedtuple("actionPoint", ["currentState", "action", "reward", "observation", "done"])
actionMap = deque()
epsilon = 1
for epoch in range(100000):
observation, info = env.reset(return_info=True)
currentState = observation
totalReward = 0
maxSpeed = 0
done = False
while not done:
if np.random.random() > epsilon:
action = model(torch.from_numpy(currentState))
action = torch.argmax(action).item()
else:
action = env.action_space.sample()
observation, reward, done, info = env.step(action)
totalReward += reward
maxSpeed = max(maxSpeed, abs(observation[1]))
if done and totalReward != -200:
print("reached goal", totalReward)
actionMap.append(actionPoint(currentState, action, reward, observation, done))
currentState = observation
if len(actionMap) > 128:
samples = random.sample(actionMap, 128)
sampleReward = torch.FloatTensor([abs(s.observation[1]) for s in samples])
sampleCurrentState = torch.from_numpy(np.array([s.currentState for s in samples]))
futurePrediction = target(sampleCurrentState)
for i, s in enumerate(samples):
futurePrediction[i][s.action] = sampleReward[i]
currentPrediction = model(sampleCurrentState)
optimizer.zero_grad()
loss = mse(currentPrediction, futurePrediction.detach())
loss.backward()
optimizer.step()
print(maxSpeed)
target.load_state_dict(model.state_dict())
if epsilon > MIN_EPSILON:
epsilon *= EPSILON_DECAY
epsilon = max(epsilon, MIN_EPSILON)
env.close()