添加 A2C/QAC 算法实现及训练结果

- 新增 RL_Algothrithms 模块，包含 A2C、QAC 智能体
- 添加 SAC 章节笔记和 C10 笔记
- 上传训练结果图片
- 完善 README 与 .gitignore

RL_Algothrithms/agents/a2c.py

@@ -0,0 +1,59 @@
+import torch
+import torch.optim as optim
+import torch.nn.functional as F
+import torch.distributions as distributions
+from networks import Actor, VCritic
+
+class A2CAgent:
+    def __init__(self, state_dim, action_dim, device, actor_lr=0.001, critic_lr=0.002, gamma=0.99):
+        self.device = device
+        self.gamma = gamma
+        
+        # A2C 使用 VCritic
+        self.actor = Actor(state_dim, action_dim).to(self.device)
+        self.critic = VCritic(state_dim).to(self.device)
+        
+        self.actor_optimizer = optim.Adam(self.actor.parameters(), lr=actor_lr)
+        self.critic_optimizer = optim.Adam(self.critic.parameters(), lr=critic_lr)
+
+    def select_action(self, state):
+        with torch.no_grad():
+            state_tensor = torch.FloatTensor(state).unsqueeze(0).to(self.device)
+            action_probs = self.actor(state_tensor)
+            action = distributions.Categorical(action_probs).sample().item()
+        return action
+
+    # 注意：A2C 的更新不需要 next_action，但为了与 QAC 的接口统一，此处用 *args 吸收多余参数
+    def update(self, state, action, reward, next_state, next_action, done):
+        state = torch.FloatTensor(state).unsqueeze(0).to(self.device)
+        next_state = torch.FloatTensor(next_state).unsqueeze(0).to(self.device)
+        reward = torch.FloatTensor([reward]).unsqueeze(0).to(self.device)
+        
+        # --- Critic 更新 ---
+        v_value = self.critic(state)
+        next_v_value = self.critic(next_state).detach()
+        
+        td_target = reward + self.gamma * next_v_value * (1 - int(done))
+        # 计算优势函数 (Advantage)
+        advantage = td_target - v_value
+        
+        critic_loss = F.mse_loss(v_value, td_target)
+        
+        self.critic_optimizer.zero_grad()
+        critic_loss.backward()
+        self.critic_optimizer.step()
+        
+        # --- Actor 更新 ---
+        action_probs = self.actor(state)
+        dist = distributions.Categorical(action_probs)
+        log_prob = dist.log_prob(torch.tensor([action]).to(self.device))
+
+        # 计算策略的熵，鼓励探索
+        entropy = dist.entropy()
+        
+        # Actor 梯度上升目标：ln(pi) * Advantage，方差更小
+        actor_loss = -(log_prob * advantage.detach()).mean()- 0.01 * entropy.mean()
+        
+        self.actor_optimizer.zero_grad()
+        actor_loss.backward()
+        self.actor_optimizer.step()

RL_Algothrithms/agents/qac.py

@@ -0,0 +1,59 @@
+import torch
+import torch.optim as optim
+import torch.nn.functional as F
+import torch.distributions as distributions
+from networks import Actor, QCritic
+
+class QACAgent:
+    def __init__(self, state_dim, action_dim, device, actor_lr=0.001, critic_lr=0.002, gamma=0.99):
+        # 接收设备参数，确保网络挂载在 GPU 或 CPU 上
+        self.device = device
+        self.gamma = gamma
+        
+        # 实例化网络并移动到指定设备
+        self.actor = Actor(state_dim, action_dim).to(self.device)
+        self.critic = QCritic(state_dim, action_dim).to(self.device)
+        
+        self.actor_optimizer = optim.Adam(self.actor.parameters(), lr=actor_lr)
+        self.critic_optimizer = optim.Adam(self.critic.parameters(), lr=critic_lr)
+
+    def select_action(self, state):
+        # 推理时禁用梯度图计算，加快速度
+        with torch.no_grad():
+            state_tensor = torch.FloatTensor(state).unsqueeze(0).to(self.device)
+            action_probs = self.actor(state_tensor)
+            action = distributions.Categorical(action_probs).sample().item()
+        return action
+
+    def update(self, state, action, reward, next_state, next_action, done):
+        # 将数据转换为张量并送入 GPU
+        state = torch.FloatTensor(state).unsqueeze(0).to(self.device)
+        next_state = torch.FloatTensor(next_state).unsqueeze(0).to(self.device)
+        reward = torch.FloatTensor([reward]).unsqueeze(0).to(self.device)
+        
+        # --- Critic 更新 ---
+        q_values = self.critic(state)
+        current_q = q_values[0, action]
+        
+        next_q_values = self.critic(next_state).detach()
+        next_q = next_q_values[0, next_action]
+        
+        # 计算 TD 目标
+        td_target = reward + self.gamma * next_q * (1 - int(done))
+        critic_loss = F.mse_loss(current_q, td_target.squeeze())
+        
+        self.critic_optimizer.zero_grad()
+        critic_loss.backward()
+        self.critic_optimizer.step()
+        
+        # --- Actor 更新 ---
+        action_probs = self.actor(state)
+        dist = distributions.Categorical(action_probs)
+        log_prob = dist.log_prob(torch.tensor([action]).to(self.device))
+        
+        # Actor 梯度上升目标：ln(pi) * Q(s, a)
+        actor_loss = -(log_prob * current_q.detach())
+        
+        self.actor_optimizer.zero_grad()
+        actor_loss.backward()
+        self.actor_optimizer.step()

RL_Algothrithms/main.py

@@ -0,0 +1,71 @@
+import gymnasium as gym
+import torch
+from agents.qac import QACAgent
+from agents.a2c import A2CAgent
+from utils import plot_comparison
+
+def train_agent(env_name, agent_class, device, num_episodes=500):
+    """
+    通用的训练循环函数
+    """
+    env = gym.make(env_name)
+    state_dim = env.observation_space.shape[0] # type: ignore
+    action_dim = int(env.action_space.n)       # type: ignore
+    
+    # 实例化传入的算法代理
+    agent = agent_class(state_dim, action_dim, device)
+    
+    rewards_history = []
+    
+    for episode in range(num_episodes):
+        state, _ = env.reset()
+        episode_reward = 0
+        
+        # QAC 属于 Sarsa 类，需要提前采样第一个动作
+        action = agent.select_action(state)
+        
+        while True:
+            next_state, reward, terminated, truncated, _ = env.step(action)
+            done = terminated or truncated
+            
+            # 预采样下一个动作
+            next_action = agent.select_action(next_state)
+            
+            # 统一的接口调用更新
+            agent.update(state, action, reward, next_state, next_action, done)
+            
+            state = next_state
+            action = next_action
+            episode_reward += reward # type: ignore
+            
+            if done:
+                break
+                
+        rewards_history.append(episode_reward)
+        if (episode + 1) % 100 == 0:
+            print(f"[{agent_class.__name__}] 回合 {episode+1}/{num_episodes}, 近100回合均分: {sum(rewards_history[-100:])/100:.2f}")
+            
+    env.close()
+    return rewards_history
+
+if __name__ == "__main__":
+    # 检测 GPU
+    device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
+    print(f"正在使用的计算设备: {device}")
+    
+    ENV_NAME = 'CartPole-v1'
+    EPISODES = 600
+    
+    print("\n--- 开始训练 QAC ---")
+    qac_rewards = train_agent(ENV_NAME, QACAgent, device, EPISODES)
+    
+    print("\n--- 开始训练 A2C ---")
+    a2c_rewards = train_agent(ENV_NAME, A2CAgent, device, EPISODES)
+    
+    # 收集结果并绘图对比
+    results = {
+        'QAC (High Variance)': qac_rewards,
+        'A2C (Low Variance)': a2c_rewards
+    }
+    
+    plot_comparison(results, window=50, save_path='qac_vs_a2c_gpu.png')

RL_Algothrithms/networks.py

@@ -0,0 +1,43 @@
+import torch
+import torch.nn as nn
+import torch.nn.functional as F
+
+# 策略网络 (离散动作 Actor)
+class Actor(nn.Module):
+    def __init__(self, state_dim, action_dim):
+        super(Actor, self).__init__()
+        # 定义两层隐藏层
+        self.fc1 = nn.Linear(state_dim, 128)
+        self.fc2 = nn.Linear(128, action_dim)
+        
+    def forward(self, state):
+        x = F.relu(self.fc1(state))
+        # 输出动作的概率分布
+        action_probs = F.softmax(self.fc2(x), dim=-1)
+        return action_probs
+
+# Q值网络 (用于 QAC)
+class QCritic(nn.Module):
+    def __init__(self, state_dim, action_dim):
+        super(QCritic, self).__init__()
+        self.fc1 = nn.Linear(state_dim, 128)
+        self.fc2 = nn.Linear(128, action_dim)
+        
+    def forward(self, state):
+        x = F.relu(self.fc1(state))
+        # 输出各个动作的具体价值
+        q_values = self.fc2(x)
+        return q_values
+
+# V值网络 (用于 A2C，由于加入了基线，只需输出状态价值标量)
+class VCritic(nn.Module):
+    def __init__(self, state_dim):
+        super(VCritic, self).__init__()
+        self.fc1 = nn.Linear(state_dim, 128)
+        self.fc2 = nn.Linear(128, 1)
+        
+    def forward(self, state):
+        x = F.relu(self.fc1(state))
+        # 输出当前状态的价值评估
+        v_value = self.fc2(x)
+        return v_value

RL_Algothrithms/utils.py

@@ -0,0 +1,34 @@
+import matplotlib
+# 关键设置：针对 Linux 服务器无 GUI 环境，强制使用 Agg 后端进行纯文件渲染
+matplotlib.use('Agg') 
+import matplotlib.pyplot as plt
+import numpy as np
+
+def plot_comparison(results_dict, window=50, save_path='comparison_result.png'):
+    """
+    绘制并保存算法对比曲线
+    :param results_dict: 字典格式 {'QAC': [奖励列表], 'A2C': [奖励列表]}
+    :param window: 移动平均的窗口大小
+    :param save_path: 图片保存路径
+    """
+    plt.figure(figsize=(10, 6))
+    
+    for algo_name, rewards in results_dict.items():
+        # 绘制原始透明度较低的曲线
+        plt.plot(rewards, alpha=0.3, label=f'{algo_name} (Raw)')
+        
+        # 计算并绘制移动平均曲线，使趋势更平滑
+        if len(rewards) >= window:
+            moving_avg = np.convolve(rewards, np.ones(window)/window, mode='valid')
+            plt.plot(np.arange(window-1, len(rewards)), moving_avg, linewidth=2, label=f'{algo_name} (Avg {window})')
+            
+    plt.xlabel('Episode')
+    plt.ylabel('Total Reward')
+    plt.title('Algorithm Comparison: QAC vs A2C')
+    plt.legend()
+    plt.grid(True, alpha=0.3)
+    
+    # 将图像保存到服务器硬盘
+    plt.savefig(save_path, dpi=300)
+    plt.close()
+    print(f"对比图像已成功保存至: {save_path}")