Trjtdtk

I'm trying to build a model for the Mountain Car game, following this Actor-Critic code: https://github.com/nikhilbarhate99/Actor-Critic
(However, in this case, it's discrete action space, while it's continuous for my problem. Also, it's not the MountainCar game in this github code.

So, I want to use the actor critic model in order to makes a player of the famous Mountain Car game. All the environment code is here: https://github.com/nbrosson/Actor-critic-MountainCar/ Everything about the environment works fine. The only file that I have to worry about is agent.py

import numpy as np
import torch
import torch.nn as nn
import torch.optim as optim
import torch.nn.functional as F
from torch.distributions import Normal

"""
Contains the definition of the agent that will run in an
environment.
"""

class ActorCritic(nn.Module):
 def __init__(self):
 super(ActorCritic, self).__init__()
 self.affine = nn.Linear(2, 32)

 self.action_layer = nn.Linear(32, 2)
 self.value_layer = nn.Linear(32, 1)

 self.logprobs = []
 self.state_values = []
 self.rewards = []
 self.actions = []


 def forward(self, observation):
 # Convert tuple into tensor
 observation_as_list = []
 observation_as_list.append(observation[0])
 observation_as_list.append(observation[1])
 observation_as_list = np.asarray(observation_as_list)
 observation_as_list = observation_as_list.reshape(1,2)
 observation = observation_as_list

 state = torch.from_numpy(observation).float()
 state = F.relu(self.affine(state))

 state_value = self.value_layer(state)
 action_parameters = F.tanh(self.action_layer(state))
 action_distribution = Normal(action_parameters[0][0], action_parameters[0][1])

 action = action_distribution.sample() # Torch.tensor; action

 self.logprobs.append(action_distribution.log_prob(action)+ 1e-6)
 self.state_values.append(state_value)
 return action.item() # Float element



 def calculateLoss(self, gamma=0.99):

 # calculating discounted rewards:
 rewards = []
 dis_reward = 0
 for reward in self.rewards[::-1]:
 dis_reward = reward + gamma * dis_reward
 rewards.insert(0, dis_reward)

 # normalizing the rewards:
 rewards = torch.tensor(rewards)
 rewards = (rewards - rewards.mean()) / (rewards.std())

 loss = 0
 for logprob, value, reward in zip(self.logprobs, self.state_values, rewards):
 advantage = reward - value.item()
 action_loss = -logprob * advantage
 value_loss = F.smooth_l1_loss(value, reward)
 loss += (action_loss + value_loss) 

 return loss

 def clearMemory(self):
 del self.logprobs[:]
 del self.state_values[:]
 del self.rewards[:]




class RandomAgent():
 def __init__(self):
 """Init a new agent.
 """
 #self.theta = np.zeros((3, 2))
 #self.state = RandomAgent.reset(self,[-20,20])

 self.count_episodes = -1
 self.max_position = -0.4
 self.epsilon = 0.9
 self.gamma = 0.99
 self.running_rewards = 0
 self.policy = ActorCritic()
 self.optimizer = optim.Adam(self.policy.parameters(), lr=0.01, betas=(0.9, 0.999))
 self.check_new_episode = 1
 self.count_iter = 0

 def reset(self, x_range):
 """Reset the state of the agent for the start of new game.

 Parameters of the environment do not change, but your initial
 location is randomized.

 x_range = [xmin, xmax] contains the range of possible values for x

 range for vx is always [-20, 20]
 """
 self.epsilon = (self.epsilon * 0.99)
 self.count_episodes += 1
 return (np.random.uniform(x_range[0],x_range[1]), np.random.uniform(-20,20))

 def act(self, observation):
 """Acts given an observation of the environment.

 Takes as argument an observation of the current state, and
 returns the chosen action.

 observation = (x, vx)
 """

# observation_as_list = []
# observation_as_list.append(observation[0])
# observation_as_list.append(observation[1])
# observation_as_list = np.asarray(observation_as_list)
# observation_as_list = observation_as_list.reshape(1,2)
# observation = observation_as_list


 if np.random.rand(1) < self.epsilon:
 return np.random.uniform(-1,1)
 else:
 action = self.policy(observation)
 return action

 def reward(self, observation, action, reward):
 """Receive a reward for performing given action on
 given observation.

 This is where your agent can learn.
 """
 self.count_iter +=1
 self.policy.rewards.append(reward)
 self.running_rewards += reward
 if self.count_iter == 100:
 # We want first to update the critic agent:
 self.optimizer.zero_grad()
 self.loss = self.policy.calculateLoss(self.gamma)
 self.loss.backward()
 self.optimizer.step() 
 self.policy.clearMemory()

 self.count_iter = 0


Agent = RandomAgent

However, my model does not provide good results. It doesn't even improve with 200 episodes.

Any ideas what is wrong on my code?? Any suggestions??

Thanks a lot !!