Tensorflow Acrobot和CartPole的强化学习

我想使用相同的代理解决cartpole和acrobot。我在一本书的帮助下编写了这段代码，但我注意到它不适用于Acrobot

我使用了神经网络：

keras.backend.clear_session()
tf.random.set_seed(42)

np.random.seed(42)

env = gym.make("Acrobot-v1")
env = gym.make("CartPole-v1")
input_shape = env.observation_space.shape
n_outputs  = env.action_space.n
max_episode_steps = 500

model = keras.models.Sequential([
    keras.layers.Dense(32, activation="elu", input_shape=input_shape),
    keras.layers.Dense(32, activation="elu"),
    keras.layers.Dense(n_outputs)
])

这项贪婪的政策：

#we pick the action with the largest predicted QValue.
#To ensure that the agent explores the environment, we will use an ε-greedy policy 

def epsilon_greedy_policy(state, epsilon=0):
    if np.random.rand() < epsilon:
        return np.random.randint(env.action_space.n)
        #return env.action_space.sample()
    else:
        Q_values = model.predict(state[np.newaxis])
        return np.argmax(Q_values[0])

我确信损失函数有一个错误，因为我在这个图中只得到了正的回报：

你有什么建议吗

#we will store all experiences in a replay buffer (or replay memory),
from collections import deque
replay_memory = deque(maxlen=1000)

#five elements: a state, the action the agent took,
#the resulting reward, the next state it reached, and finally a Boolean indicating
#whether the episode ended at that point (done).
def sample_experiences(batch_size):
    indices = np.random.randint(len(replay_memory), size=batch_size)
    batch = [replay_memory[index] for index in indices]
    states, actions, rewards, next_states, dones = [
        np.array([experience[field_index] for experience in batch])
        for field_index in range(5)]
    return states, actions, rewards, next_states, dones

#create a function that will play a single step using the ε-greedy policy, then
#store the resulting experience in the replay buffer (IN[5]):

def play_one_step(env, state, epsilon):
    action = epsilon_greedy_policy(state, epsilon)
    next_state, reward, done, info = env.step(action)
    replay_memory.append((state, action, reward, next_state, done))
    return next_state, reward, done, info

#define some hyperparameters and create the optimizer and the loss function.
batch_size = 32
discount_rate = 0.95
optimizer = keras.optimizers.Adam(lr=1e-3)
loss_fn = keras.losses.mean_squared_error

def training_step(batch_size):
    experiences = sample_experiences(batch_size)
    states, actions, rewards, next_states, dones = experiences
    next_Q_values = model.predict(next_states)
    max_next_Q_values = np.max(next_Q_values, axis=1)
    target_Q_values = (rewards + (1 - dones) * discount_rate * max_next_Q_values)
    target_Q_values = target_Q_values.reshape(-1, 1) #reshape() convert target_Q_values in column vector
    mask = tf.one_hot(actions, n_outputs)
    with tf.GradientTape() as tape:
        all_Q_values = model(states)
        Q_values = tf.reduce_sum(all_Q_values * mask, axis=1, keepdims=True)
        loss = tf.reduce_mean(loss_fn(target_Q_values, Q_values))
    grads = tape.gradient(loss, model.trainable_variables)
    optimizer.apply_gradients(zip(grads, model.trainable_variables))