Python 在线PPO:TensorFlow会话返回NaN

我正在尝试使用tensorflow训练一个在线策略优化模型，但过了一会儿，tensorflow会话开始返回NAN。这导致我的代理使用这些NAN执行

步骤

，最终整个事情变得一团糟

self.train_opt = tf.train.AdamOptimizer(CR_LR)
self.train_opt = self.train_opt.apply_gradients(zip(gradients, variables))

控制台中的短片段：

self.train_opt = tf.train.AdamOptimizer(CR_LR)
self.train_opt = self.train_opt.apply_gradients(zip(gradients, variables))

采取的行动[2.1.3305835 0.9937418]
观察[0.69689728-0.46114012-11.39961704-0.05004346-0.05004346
0.74720544   3.49857114   3.05071477  -1.10276782  -9.71530186]
获得的奖励-0.023699851569145534
采取的行动[2.0.62562937 1.0081608]
观察[0.71591491-0.47488649 11.84026042-0.05004346-0.05004346 0.75886336
3.49857114  3.07180685 -1.12458586 -9.84382414]
获得的奖励-0.015462812448075767
采取的行动[南]
观测值[nan-0.05004346-0.05004346 nan
[南南]
获得的奖励
采取的行动[南]
观测值[nan-0.05004346-0.05004346 nan
[南南]
获得的奖励

我的代码[更新]：

self.train_opt = tf.train.AdamOptimizer(CR_LR)
self.train_opt = self.train_opt.apply_gradients(zip(gradients, variables))

导入健身房
将numpy作为np导入
导入tensorflow作为tf
进口火箭着陆器健身房
EP_LEN=200
伽马=0.9
SL_LR=1e-4
CR_LR=1e-4
批次=5
参与者更新步骤=20
评论家\u更新\u步骤=20
状态尺寸，动作尺寸=10,3
方法=[
dict（name='kl_罚金'，kl_目标=0.01，lam=0.5），
dict（name='clip'，ε=0.2），
][1]
打印调试消息=True
PPO类：
定义初始化（自）：
self.tfsess=tf.Session（）
self.tf_state=tf.placeholder（tf.float32，[None，state_DIM]，'state'）
#评论家（价值网络）
使用tf.variable_scope（'critic'）：
#层
l1=tf.layers.density（self.tf_状态，100，tf.nn.relu）
#价值观
self.value=tf.layers.density（l1，1）
#折扣奖励：未来的奖励
self.tf\u dreward=tf.placeholder（tf.float32，[None，1]，“折扣奖励”）
#优势：确定行动的质量
self.advantage=self.tf\u dreward-self.value
#损失函数：随着时间的推移，将优势最小化
#损失函数是均方误差
自我损失=tf.减少平均值（tf.平方（自我优势））
#使用Adam优化器的梯度下降
self.train_opt=tf.train.AdamOptimizer（CR_LR）
梯度，变量=zip（*self.train\u opt.compute\u梯度（self.loss））
渐变，u=tf。按全局标准剪裁（渐变，1.0）
self.train_opt=self.train_opt.apply_渐变（zip（渐变，变量））
#参与者（政策网络）
pi，pi_params=self.tinynn（'pi'，trainable=True）
old_pi，old_pi参数=self.tinynn（'old_pi'，trainable=False）
#来自旧策略网络和新策略网络的示例操作
使用tf.variable_scope（'sample_action'）：
#从分发列表中选择一个操作
self.sample_操作=tf.squence（pi.sample（1），轴=0）
使用tf.variable_scope（'update_old_pi'）：
#从分发列表中选择一个操作
self.update_old_pi_operation=[old_pi.assign（p）for p，old_pi in zip（pi参数，old_pi参数）]
#行动和优势的占位符
self.tf\u action=tf.placeholder（tf.float32，[None，ACT\u DIM]，“action”）
self.tf_advantage=tf.placeholder（tf.float32，[None，1]，“advantage”）
#计算损失函数
使用tf.variable_范围（“损失”）：
使用tf.variable_scope（“代理”）：
比率=pi.prob（自身tf优势）/旧pi.prob（自身tf优势）
代理=比率*self.tf\u优势
#吉隆坡罚则
如果方法['name']=='kl_惩罚'：
#兰姆达
self.tf_lambda=tf.placeholder（tf.float32，None，'lambda'）
#计算新旧保单之间的KL差异
kl=tf.contrib.distributions.kl_散度（旧的π，π）
#刻薄
self.kl_均值=tf.reduce_均值（kl）
#使用代理计算损失
self.aloss=-（tf.reduce_平均值（代理项-self.tf_lambda*kl））
其他：
self.aloss=-tf.reduce_mean（tf.minimum（代理，tf.clip_乘以_值（比率，1.-方法['epsilon'，1.+方法['epsilon']）*self.tf_优势））
#使用梯度下降法最小化损失
使用tf.variable_scope（'atrain'）：
self.atrain_operation=tf.train.AdamOptimizer（SL_LR）
梯度，变量=zip（*self.atrain_operation.compute_梯度（self.aloss））
渐变，u=tf。按全局标准剪裁（渐变，1.0）
self.atrain_operation=self.atrain_operation.apply_梯度（zip（梯度，变量））
#写入磁盘
FileWriter（“log/”，self.tfsess.graph）
#运行会话
self.tfsess.run（tf.global\u variables\u initializer（））
def更新（自我、状态、行动、奖励）：
self.tfsess.run（self.update\u old\u pi\u操作）
advantage=self.tfsess.run（self.advantage，{self.tf_state:state，self.tf_dreward:raward}）
#更新参与者（策略）
如果方法['name']=='kl_惩罚'：
对于范围内的（参与者更新步骤）：
_，kl=self.tfsess.run（[self.atrain_operation，self.kl_mean]，{self.tf_state:state，self.tf_action:action，tf_advantage:advantage，self.tf_lambda:METHOD['lam']}）
如果kl>4*方法['kl_目标]：
打破
如果kl<方法['kl_目标]]/1.5：
#自适应λ
方法['lam']/=2
elif kl>方法['kl_目标]*1.5：
方法['lam']*=2
#Lambda可能会爆炸，我们需要把它剪掉
方法['lam']=
self.atrain_operation = self.atrain_operation.apply_gradients(zip(gradients, variables))

self.atrain_op = self.atrain_operation.apply_gradients(zip(gradients, variables))

#[self.tfsess.run(self.atrain_operation, {self.tf_state: state, self.tf_action: action, self.tf_advantage: advantage}) for _ in range(ACTOR_UPDATE_STEPS)]
[self.tfsess.run(self.atrain_op, {self.tf_state: state, self.tf_action: action, self.tf_advantage: advantage}) for _ in range(ACTOR_UPDATE_STEPS)]

#[self.tfsess.run(self.train_opt, {self.tf_state: state, self.tf_dreward: reward}) for _ in range(CRITIC_UPDATE_STEPS)]
[self.tfsess.run(self.ctrain_op, {self.tf_state: state, self.tf_dreward: reward}) for _ in range(CRITIC_UPDATE_STEPS)]

mu = 2 * tf.layers.dense(l1, ACT_DIM, tf.nn.tanh, trainable=trainable)

mu = self.env.action_space.high * tf.layers.dense(l1, ACT_DIM, tf.nn.tanh, name='mu', trainable=trainable)

return np.clip(a, self.env.action_space.low, self.env.action_space.high)

"""
if __name__ == '__main__':
    ppo = PPO()
    #env = gym.make('RocketLander-v0')
    env = gym.make('Pendulum-v0')
    reward = ppo.train(env, ppo, 100)
    print(reward)
"""    
if __name__ == '__main__':
    #env = gym.make('RocketLander-v0')
    env = gym.make('Pendulum-v0')
    ppo = PPO(env)
    reward = ppo.train(env, ppo, 100)
    print(reward)

[-1076.4211985938728, -1089.7948555704293, -1115.6341917789869, -1147.7961139172062, -1162.9589624975872, -1193.6444573268725, -1214.9662239699737, -1219.295151702447, -1228.3773779343328, -1211.7559065793157, -1239.1770034164979, -1256.5497739717612, -1248.942050034072, -1251.5809026533057, -1246.350714892043, -1223.1414157442061, -1231.5288547710811, -1223.5475405502032, -1217.095971096193, -1215.639878904649, -1182.084416025169, -1174.3085216226718, -1176.5976104186886, -1188.5439312195451, -1160.6565487872776, -1132.5758139546506, -1148.7299082836548, -1149.1097155137375, -1124.4154423538491, -1100.4411098048593, -1081.2445587548245, -1035.7597376533809, -1039.5657416397464, -1046.8627585876952, -1007.554202371864, -997.4072232047926, -924.0742105089892, -872.5268280283873, -889.6594740458157, -929.8577808816676, -957.1616193294444, -887.3960001717214, -811.6005555799227, -769.4648914456843, -692.6909819129986, -623.7238271047137, -656.6829518032941, -629.9657550649539, -651.9125731231816, -678.5172027274579, -683.0097144683796, -640.7089935328387, -589.4306203212271, -556.3242756529115, -526.881331084439, -539.3604006694065, -511.27673189202727, -526.1856726355412, -512.7768642430646, -514.7892695498354, -527.2777710366902, -516.3731318862425, -504.3876365547384, -466.66983741261095, -446.0724507306932, -414.25670263412803, -449.7266236253488, -471.7990471628901, -492.56922815695845, -455.6665136249609, -436.67493361178475, -393.1425637497276, -445.3335873259794, -440.30325932671377, -437.07634044015583, -406.7068409952513, -379.062809279313, -444.46652386541916, -439.60389029825603, -422.0043960746679, -424.80904663279813, -486.0321568909586, -476.00519893661306, -493.3553901668465, -457.4723683354885, -450.83268159600254, -458.6995892890558, -514.3951245072926, -519.3061062950538, -507.1919061966863, -469.59914342990675, -422.66056322913045, -439.53868966691357, -395.9325190449425, -369.7488471733708, -398.1944563259144, -397.3649275140671, -401.18423175784426, -400.9083352836444, -374.0640183220304]

import gym
import numpy as np
import tensorflow as tf
#import rocket_lander_gym

EP_LEN = 200
GAMMA = 0.9
SL_LR = 1e-4
CR_LR = 1e-4
BATCH = 5
ACTOR_UPDATE_STEPS = 20
CRITIC_UPDATE_STEPS = 20
#STATE_DIM, ACT_DIM = 10, 3
STATE_DIM, ACT_DIM = 3, 1


METHOD = [
    dict(name='kl_penalty', kl_target=0.01, lam=0.5),   
    dict(name='clip', epsilon=0.2),
][1]

PRINT_DEBUG_MSG = False

class PPO:
    def __init__(self, env):
        self.env = env
        self.tfsess = tf.Session()
        self.tf_state = tf.placeholder(tf.float32, [None, STATE_DIM], 'state')

        # Critic (value network)
        with tf.variable_scope('critic'):
            # Layers
            l1 = tf.layers.dense(self.tf_state, 100, tf.nn.relu)
            # Value
            self.value = tf.layers.dense(l1, 1)
            # Discounted reward: reward in the furture
            self.tf_dreward = tf.placeholder(tf.float32, [None, 1], 'discounted_reward')
            # Advantage: determine quality of action
            self.advantage = self.tf_dreward - self.value
            # Loss function: minimize the advantage over time
            # The loss function is a mean squared error
            self.loss = tf.reduce_mean(tf.square(self.advantage))

            # Gradient descent using Adam optimizer
            self.train_opt = tf.train.AdamOptimizer(CR_LR)
            gradients, variables = zip(*self.train_opt.compute_gradients(self.loss))
            gradients, _ = tf.clip_by_global_norm(gradients, 1.0)

            #self.train_opt = self.train_opt.apply_gradients(zip(gradients, variables))
            self.ctrain_op = self.train_opt.apply_gradients(zip(gradients, variables))

        # Actor (policy network)
        pi, pi_params = self.tinynn('pi', trainable=True)
        old_pi, old_pi_params = self.tinynn('old_pi', trainable=False)

        # Sample actions from both the old and the new policy networks
        with tf.variable_scope('sample_action'):
            # Choose an action from the distribution learnt
            self.sample_operation = tf.squeeze(pi.sample(1), axis=0)
        with tf.variable_scope('update_old_pi'):
            # Choose an action from the distribution learnt
            self.update_old_pi_operation = [old_pi.assign(p) for p, old_pi in zip(pi_params, old_pi_params)]

        # Placeholder for the action and the advantage
        self.tf_action = tf.placeholder(tf.float32, [None, ACT_DIM], 'action')
        self.tf_advantage = tf.placeholder(tf.float32, [None, 1], 'advantage')

        # Compute loss function
        with tf.variable_scope('loss'):
            with tf.variable_scope('surrogate'):
                #ratio = pi.prob(self.tf_advantage) / old_pi.prob(self.tf_advantage)

                ratio = pi.prob(self.tf_action) / old_pi.prob(self.tf_action)

                surrogate = ratio * self.tf_advantage

            # KL penalty
            if METHOD['name'] == 'kl_penalty':
                # Lambda
                self.tf_lambda = tf.placeholder(tf.float32, None, 'lambda')
                # Compute KL divergence between old and new policy
                kl = tf.contrib.distributions.kl_divergence(old_pi, pi)
                # Get mean
                self.kl_mean = tf.reduce_mean(kl)
                # Compute loss using surrogate
                self.aloss = -(tf.reduce_mean(surrogate - self.tf_lambda * kl))
            else:
                self.aloss = -tf.reduce_mean(tf.minimum(surrogate, tf.clip_by_value(ratio, 1.-METHOD['epsilon'],  1.+METHOD['epsilon']) * self.tf_advantage))

        # Minimize the loss using gradient descent
        with tf.variable_scope('atrain'):
            self.atrain_operation = tf.train.AdamOptimizer(SL_LR)
            gradients, variables = zip(*self.atrain_operation.compute_gradients(self.aloss))
            gradients, _ = tf.clip_by_global_norm(gradients, 1.0)
            #self.atrain_operation = self.atrain_operation.apply_gradients(zip(gradients, variables))
            self.atrain_op = self.atrain_operation.apply_gradients(zip(gradients, variables))

        # Write to disk
        tf.summary.FileWriter("log/", self.tfsess.graph)

        # Run the session
        self.tfsess.run(tf.global_variables_initializer())


    def update(self, state, action, reward):
        self.tfsess.run(self.update_old_pi_operation)

        advantage = self.tfsess.run(self.advantage, {self.tf_state: state, self.tf_dreward: reward})

        # Update actor (policy)
        if METHOD['name'] == 'kl_penalty':
            for _ in range(ACTOR_UPDATE_STEPS):
                _, kl = self.tfsess.run([self.atrain_operation, self.kl_mean], {self.tf_state: state, self.tf_action: action, tf_advantage: advantage, self.tf_lambda: METHOD['lam']})
                if kl > 4*METHOD['kl_target']:
                    break
            if kl < METHOD['kl_target'] / 1.5:
                # Adaptive lambda
                METHOD['lam'] /= 2
            elif kl > METHOD['kl_target'] * 1.5:
                METHOD['lam'] *= 2
            # Lambda might explode, we need to clip it
            METHOD['lam'] = np.clip(METHOD['lam'], 1e-4, 10)
        else:
            #[self.tfsess.run(self.atrain_operation, {self.tf_state: state, self.tf_action: action, self.tf_advantage: advantage}) for _ in range(ACTOR_UPDATE_STEPS)]
            [self.tfsess.run(self.atrain_op, {self.tf_state: state, self.tf_action: action, self.tf_advantage: advantage}) for _ in range(ACTOR_UPDATE_STEPS)]

        # Update critic (value)
        #[self.tfsess.run(self.train_opt, {self.tf_state: state, self.tf_dreward: reward}) for _ in range(CRITIC_UPDATE_STEPS)]
        [self.tfsess.run(self.ctrain_op, {self.tf_state: state, self.tf_dreward: reward}) for _ in range(CRITIC_UPDATE_STEPS)]

    def tinynn(self, name, trainable):
        with tf.variable_scope(name):
            l1 = tf.layers.dense(self.tf_state, 100, tf.nn.relu, trainable=trainable)
            #mu = 2 * tf.layers.dense(l1, ACT_DIM, tf.nn.tanh, trainable=trainable)
            mu = self.env.action_space.high * tf.layers.dense(l1, ACT_DIM, tf.nn.tanh, name='mu', trainable=trainable)
            sigma = tf.layers.dense(l1, ACT_DIM, tf.nn.softplus, trainable=trainable)
            norm_dist = tf.distributions.Normal(loc=mu, scale=sigma)
        params = tf.get_collection(tf.GraphKeys.GLOBAL_VARIABLES, scope=name)
        return norm_dist, params


    def choose_action(self, state):
        state = state[np.newaxis, :]
        action = self.tfsess.run(self.sample_operation, {self.tf_state: state})[0]
        #return np.clip(action, -1, 1)
        return np.clip(action, self.env.action_space.low, self.env.action_space.high)

    def get_value(self, state):
        if state.ndim < 2: state = state[np.newaxis, :]
        return self.tfsess.run(self.value, {self.tf_state: state})[0, 0]


    def train(self, env, ppo, epochs, render=False):
        # Rewards
        all_ep_r = []
        # Training loop
        for ep in range(epochs):
            # Initial state
            s = env.reset()
            # States, actions and rewards
            buffer_s, buffer_a, buffer_r = [], [], []
            # Initial reward
            ep_r = 0
            # For a single episode
            for t in range(EP_LEN):
                if render:
                    # Render the environment
                    env.render()
                # Choose best action
                a = ppo.choose_action(s)
                # State,reward,done,info
                s_, r, done, _ = env.step(a)
                if PRINT_DEBUG_MSG:
                    print("Action Taken  ",a)
                    print("Observation   ",s_)
                    print("Reward Gained ",r, end='\n\n')
                # Add to buffers
                buffer_s.append(s)
                buffer_a.append(a)
                buffer_r.append((r+8)/8)    # normalize reward, find to be useful
                s = s_
                # Total reward
                ep_r += r

                # Update PPO
                if (t+1) % BATCH == 0 or t == EP_LEN - 1:
                    # Get value
                    v_s_ = ppo.get_value(s_)

                    # Discounted reward
                    discounted_r = []
                    # Update rewards
                    for r in buffer_r[::-1]:
                        v_s_ = r + GAMMA * v_s_
                        discounted_r.append(v_s_)
                    discounted_r.reverse()

                    # Buffer states actions rewards
                    bs, ba, br = np.vstack(buffer_s), np.vstack(buffer_a), np.array(discounted_r)[:, np.newaxis]
                    buffer_s, buffer_a, buffer_r = [], [], []
                    ppo.update(bs, ba, br)

                # Check if done
                if done:
                    #print("Simulation done.")
                    break
            # Append episode rewards
            if ep == 0: all_ep_r.append(ep_r)
            else: all_ep_r.append(all_ep_r[-1]*0.9 + ep_r*0.1)
            # Close the environment
            env.close()
        # Return all episode rewards
        return all_ep_r

"""
if __name__ == '__main__':
    ppo = PPO()
    #env = gym.make('RocketLander-v0')
    env = gym.make('Pendulum-v0')
    reward = ppo.train(env, ppo, 100)
    print(reward)
"""    
if __name__ == '__main__':
    #env = gym.make('RocketLander-v0')
    env = gym.make('Pendulum-v0')
    ppo = PPO(env)
    reward = ppo.train(env, ppo, 100)
    print(reward)