在强化学习实验中，梯度L2范数理论上是收敛的，在许多论文里面这个指标也是收敛的。但是我在复现一个论文实验MD-PGT的时候将模型梯度l2范数绘制出来的时候却是不收敛，且在剧烈波动。我又从网上找了一些策略梯度的强化实验复现，但是画出来的l2范数也是不收敛的，图片如下：

此项目是我从李宏毅深度学习教程LeeDL-Tutorial所找的一个策略梯度强化学习实验，仅修改了迭代次数与增加模型梯度L2范数计算与绘制部分代码，可以看出奖励是呈收敛趋势的，但是画出来的梯度范数图却急剧波动，与我所复现的论文实验图绘制出来的梯度图趋势差不多。运行的代码如下：

# 加载需要的包

from pyvirtualdisplay import Display

import matplotlib.pyplot as plt

from IPython import display

import numpy as np

import pandas as pd

import torch

from torch import nn

from torch import optim

from torch.nn import functional as F

import typing as typ

from torch.distributions import Categorical

# from tqdm.auto import tqdm

from tqdm import tqdm # 下载notebook 仍能显示

import gym

import random

import os

from rich.console import Console

import warnings

warnings.filterwarnings('ignore')

cs = Console()

v_dispaly = Display(visible=0, size=(1400, 900))

v_dispaly.start()

print("gym.version=", gym.version)

def all_seed(seed=6666, env=None):

if env is not None:

    env.seed(seed)

    env.action_space.seed(seed)

np.random.seed(seed)

random.seed(seed)

# CPU

torch.manual_seed(seed)

# GPU

if torch.cuda.is_available():

    torch.cuda.manual_seed_all(seed)

    torch.cuda.manual_seed(seed)

# python全局

os.environ['PYTHONHASHSEED'] = str(seed)

# cudnn

torch.backends.cudnn.deterministic = True

torch.backends.cudnn.benchmark = False

torch.backends.cudnn.enabled = False

print(f'Set env random_seed = {seed}')

def gym_env_desc(env_name):

"""

对环境的简单描述

"""

env = gym.make(env_name)

state_shape = env.observation_space.shape

cs.print("observation_space:\\n\\t", env.observation_space)

cs.print("action_space:\\n\\t", env.action_space)

try:

    action_shape = env.action_space.n

    action_type = '离散'

    extra=''

except Exception as e:

    action_shape = env.action_space.shape

    low_ = env.action_space.low[0]  # 连续动作的最小值

    up_ = env.action_space.high[0]  # 连续动作的最大值

    extra=f'<{low_} -> {up_}>'

    action_type = '连续'

print(f'[ {env_name} ](state: {state_shape},action: {action_shape}({action_type} {extra}))')

return

env_name= 'LunarLander-v2'

gym_env_desc(env_name)

env = gym.make(env_name)

class PolicyGradientNet(nn.Module):

def __init__(self, state_dim: int, action_dim: int):

    super(PolicyGradientNet, self).__init__()

    self.q_net = nn.ModuleList([

        nn.ModuleDict({

            'linear': nn.Linear(state_dim, 32),

            'linear_activation': nn.Tanh()

        }),

        nn.ModuleDict({

            'linear': nn.Linear(32, 32),

            'linear_activation': nn.Tanh()

        }),

        nn.ModuleDict({

            'linear': nn.Linear(32, action_dim),

            'linear_activation': nn.Softmax(dim=-1)

        })

    ])

    

def forward(self, x):

    for layer in self.q_net:

        x = layer['linear_activation'](layer['linear'](x))

    return x

from torch.optim.lr_scheduler import StepLR

from typing import List, Dict, AnyStr

from functools import reduce

class REINFORCE():

def __init__(self, state_dim: int, action_dim: int, lr: float=0.001, gamma: float=0.9, 

             stepLR_step_size:int = 200,

             stepLR_gamma:float = 0.1,

             normalize_reward:bool=False):

    self.device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')

    self.policy_net = PolicyGradientNet(state_dim, action_dim)

    self.policy_net.to(self.device)

    self.opt = optim.Adam(self.policy_net.parameters(), lr=lr)

    self.stepLR_step_size = stepLR_step_size

    self.stepLR_gamma = stepLR_gamma

    self.sche = StepLR(self.opt, step_size=self.stepLR_step_size, gamma=self.stepLR_gamma)

    self.gamma = gamma

    self.normalize_reward = normalize_reward

    self.training = True



def train(self):

    self.training = True

    self.policy_net.train()

def eval(self):

    self.training = False

    self.policy_net.eval()

    

@torch.no_grad()

def policy(self, state):

    """

    sample action

    """

    action_prob = self.policy_net(torch.FloatTensor(state).to(self.device))

    action_dist = Categorical(action_prob)

    action = action_dist.sample()

    return action.detach().cpu().item()

def batch_update(self, batch_episode: List[Dict[AnyStr, List]]):

    l2_norm_squared_list = []

    for transition_dict in batch_episode:

        l2_norm_squared = self.update(transition_dict)

        l2_norm_squared_list.append(l2_norm_squared)

    self.sche.step()

    return l2_norm_squared_list

    

def update(self, transition_dict):

    reward_list = transition_dict['rewards']

    state_list = transition_dict['states']

    action_list = transition_dict['actions']

    # 分数 normalize

    if self.normalize_reward:

        reward_list = (np.array(reward_list) - np.mean(reward_list)) / (np.std(reward_list) + 1e-9)

    Rt = 0

    self.opt.zero_grad()

    for i in reversed(range(len(reward_list))):  # 从最后一步算起

        reward = reward_list[i]

        state = torch.tensor([state_list[i]], dtype=torch.float).to(self.device)

        action = torch.tensor([action_list[i]]).unsqueeze(0).to(self.device)

        log_prob = torch.log(self.policy_net(state).gather(1, action.long()))

        # Rt = \\sum_{i=t}^T \\gamma ^ {i-t} r_i

        Rt = self.gamma \* Rt + reward

        loss = -log_prob \* Rt

        loss.backward()

    l2_norm_squared = sum((p.grad \*\* 2).sum().item() for p in self.policy_net.parameters())

    self.opt.step() 

    return l2_norm_squared

    

def save_model(self, model_dir):

    if not os.path.exists(model_dir):

        os.makedirs(model_dir)

    file_path = os.path.join(model_dir, 'policy_net.ckpt')

    torch.save(self.policy_net.state_dict(), file_path)

def load_model(self, model_dir):

    file_path = os.path.join(model_dir, 'policy_net.ckpt')

    self.policy_net.load_state_dict(torch.load(file_path))

def train_on_policy(

agent, 

env, 

num_batch=450,

random_batch=2,

episode_per_batch=3,

episode_max_step=300,

save_mdoel_dir='./check_point'

"""

on policy 强化学习算法学习简单函数

params:

    agent: 智能体

    env: 环境

    random_batch: 前N个batch用random Agent收集数据

    num_batch: 训练多少个batch

    episode_per_batch： 一个batch下多少个episode

    episode_max_step: 每个episode最大步数

    save_mdoel_dir: 模型保存的文件夹

"""

EPISODE_PER_BATCH = episode_per_batch

NUM_BATCH = num_batch

RANDOM_BATCH = random_batch

MAX_STEP = episode_max_step

avg_total_rewards, avg_final_rewards, avg_total_steps = [], [], []

l2_norm_squared_list = []

agent.train()

tq_bar = tqdm(range(RANDOM_BATCH + NUM_BATCH))

recent_best = -np.inf

batch_best = -np.inf

all_rewards = []

for batch in tq_bar:

    tq_bar.set_description(f"[ {batch+1}/{NUM_BATCH} ]")

    batch_recordes = []

    total_rewards = []

    total_steps = []

    final_rewards = []

    for ep in range(EPISODE_PER_BATCH):

        rec_dict = {'states': [], 'actions': [], 'next_states': [], 'rewards': [], 'dones': []}

        state = env.reset()

        total_reward, total_step = 0, 0

        while True:

            a = agent.policy(state)

            if batch < RANDOM_BATCH:

                a = env.action_space.sample()

            n_state, reward, done, _ = env.step(a)

            # 收集每一步的信息

            rec_dict['states'].append(state)

            rec_dict['actions'].append(a)

            rec_dict['next_states'].append(n_state)

            rec_dict['rewards'].append(reward)

            rec_dict['dones'].append(done)

            state = n_state

            total_reward += reward

            total_step += 1

            if done or total_step > MAX_STEP:

                # 一个episode结束后 收集相关信息

                final_rewards.append(reward)

                total_steps.append(total_step)

                total_rewards.append(total_reward)

                all_rewards.append(total_reward)

                batch_recordes.append(rec_dict)

                break

    avg_total_reward = sum(total_rewards) / len(total_rewards)

    avg_final_reward = sum(final_rewards) / len(final_rewards)

    avg_total_step = sum(total_steps) / len(total_steps)

    recent_batch_best = np.mean(all_rewards[-10:])

    avg_total_rewards.append(avg_total_reward)

    avg_final_rewards.append(avg_final_reward)

    avg_total_steps.append(avg_total_step)

    # 在进度条后面显示关注的信息

    tq_bar.set_postfix({

        "Total": f"{avg_total_reward: 4.1f}", 

        "Recent": f"{recent_batch_best: 4.1f}", 

        "RecentBest": f"{recent_best: 4.1f}", 

        "Final": f"{avg_final_reward: 4.1f}", 

        "Steps": f"{avg_total_step: 4.1f}"})

    # agent.batch_update(batch_recordes)

    l2_norm_squared_batch = agent.batch_update(batch_recordes)

    l2_norm_squared_list.append(np.mean(l2_norm_squared_batch))

    

    if avg_total_reward > batch_best and (batch > 4 + RANDOM_BATCH):

        batch_best = avg_total_reward

        agent.save_model(save_mdoel_dir + "_batchBest")

    if recent_batch_best > recent_best and (batch > 4 + RANDOM_BATCH):

        recent_best = recent_batch_best

        agent.save_model(save_mdoel_dir)

    

return avg_total_rewards, avg_final_rewards, avg_total_steps, l2_norm_squared_list

# test sample env

env_name= 'CartPole-v1'

gym_env_desc(env_name)

env = gym.make(env_name)

all_seed(seed=NOTEBOOK_SEED, env=env)

agent = REINFORCE(

state_dim=env.observation_space.shape[0], 

action_dim=env.action_space.n, 

lr=0.01,

gamma=0.8,

stepLR_step_size=80

)

avg_total_rewards, avg_final_rewards, avg_total_steps, l2_norm_squared_list = train_on_policy(

agent, env, 

num_batch=1600,

random_batch=2,

episode_per_batch=5,

episode_max_step=300,

save_mdoel_dir='./check_point'

)

fig, axes = plt.subplots(1, 3, figsize=(24, 6))

axes[0].plot(avg_total_rewards, label='Total Rewards')

axes[0].plot(avg_final_rewards, label='Final Rewards')

axes[0].set_title(f'{env_name} - Rewards Curve')

axes[0].legend()

axes[1].plot(avg_total_steps, label='Steps')

axes[1].set_title(f'{env_name} - Steps Curve')

axes[1].legend()

axes[2].plot(l2_norm_squared_list, label='L2 Norm Squared of Gradients')

axes[2].set_title(f'{env_name} - L2 Norm Squared Curve')

axes[2].legend()

plt.show()

此外，我在飞浆平台上也找了一个简单的策略梯度模型实现基于paddle的策略梯度模型实现，也是添加了绘制梯度L2范数/平方的部分，绘制出来的图奖励是收敛的，但是梯度L2范数剧烈波动，如下：

运行的代码如下：

# 简单模型多轮次

import os

from itertools import count

import paddle

import gym

import numpy as np

from tqdm import tqdm

import paddle.nn.functional as F

from paddle.distribution import Categorical

import matplotlib.pyplot as plt

class PolicyAgent(paddle.nn.Layer):    def init(self, obs_space, action_space):        #初始化        super(PolicyAgent, self).init()        #设样本批量数为batch，输入形状大小为[batch, obs_space]，输出形状大小为[batch, 16]        self.linear1 = paddle.nn.Linear(obs_space, 16)        #输入形状大小为[batch, 16]，输出形状大小为[batch, action_space]        self.linear2 = paddle.nn.Linear(16, action_space)        #log_probs列表用来保存历次行为的概率        self.log_probs = []        #rewards列表用来保存智能体每次与环境交互所产生的回报        self.rewards = []
    def forward(self, x):        #前向传播过程        #把输入x转化为tensor格式数据，并转成浮点数。        x = paddle.to_tensor(x, dtype="float32")        #输入x经过linear1网络层运算，再接上relu激活函数        x = F.relu(self.linear1(x))        #再经过linear2网络层运算，然后在最后一个维度上接softmax函数        action_probs = F.softmax(self.linear2(x), axis=-1)        #把上述的行动概率，转成分类分布        action_distribution = Categorical(action_probs)        #从分类分布中进行抽样        action = action_distribution.sample([1])        #根据上述抽样的结果，求解该抽样结果的概率，对概率进行log运算        #每次得到的结果，追加到log_probs列表之中        self.log_probs.append(action_distribution.log_prob(action))        #返回行动值        return action.numpy().item()        def loss_calculate(self, gamma):        #损失函数        rewards = []        dis_reward = 0        log_probs = self.log_probs        for reward in self.rewards[::-1]: #把rewards倒序排列            #计算折扣回报率            dis_reward = reward + gamma * dis_reward            #每次从0位置插入，后进来的数排在前面。            rewards.insert(0, dis_reward)        #把rewards转化为tensor，并对rewards进行标准化        rewards = paddle.to_tensor(rewards)        rewards = (rewards - rewards.mean()) / (rewards.std())        loss = 0        for logprob, reward in zip(log_probs, rewards):            #参考算法推导部分            action_loss = -logprob * reward            loss += action_loss        return loss
    def clear_memory(self):        #清除智能体记忆的函数        del self.rewards[:]        del self.log_probs[:]
def train():    #创建平衡杆cartpole环境    env = gym.make('CartPole-v1')    env = env.unwrapped    env.seed(1)    paddle.seed(1)    obs_space = env.observation_space.shape[0] #状态空间，数值为4    action_space = env.action_space.n #行动空间，数值为2    #设置超参数    gamma = 0.9    episodes = 300    learning_rate=0.02    #模型初始化    model = PolicyAgent(obs_space, action_space)    #优化器初始化    optimizer = paddle.optimizer.Adam(learning_rate=learning_rate, parameters=model.parameters())    #开始训练    duration = [] #记录每轮eposide所持续的时间    grad_norms = []    max_norm = float(5.0)    for i in tqdm(range(episodes), desc="Training Episodes"):        state = env.reset() #state的类型是ndarray        for t in count():#一直不断地循环            action = model(state)#输入状态，输出智能体选择的行动            #环境会根据智能体的动作，输出下一个状态、回报、完成等信息            state, reward, done, info = env.step(action)            model.rewards.append(reward)            if done or t >= 100: #episode终止或玩游戏持续次数超过100                break #跳出本局循环        duration.append(t) #把本局的持续时间t,追加到持久期里面        if i % 250 == 0:#每过250次保存模型            paddle.save(model.state_dict(), './lup/'+str(i)+'.pdparams')        #清零梯度        optimizer.clear_grad()        #生成损失函数        loss = model.loss_calculate(gamma)        #对损失函数求梯度，反向传播        loss.backward()
        # 计算最后一层（self.linear2）的梯度范数        total_norm = []        for p in model.linear2.parameters():            if p.grad is not None:                param_norm = np.linalg.norm(p.grad)                total_norm += param_norm**2        grad_norms.append(param_norm)
        #优化器用反向传播计算出来的梯度，对参数进行更新        optimizer.step()        #清空智能体记忆        model.clear_memory()             return duration,grad_norms
#训练智能体if name == 'main':    duration, grad_norms = train()
    fig, axes = plt.subplots(1, 2, figsize=(18, 6))    axes[0].plot(duration, label='Duration')    axes[0].set_title(f'Duration')    axes[0].legend()    axes[1].plot(grad_norms, label='Gradient Norm over Epochs')    axes[1].set_title(f'Gradient Norm over Epochs')    axes[1].legend()    plt.show()

是我绘制梯度L2范数/平方的代码错误了吗还是别的原因，现在无法绘制出一个收敛的梯度范数图。