{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 多智能体玩具实验：攀爬博弈（Climbing Game, Claus & Boutilier 1998）。\n",
    "\n",
    "两个智能体各选一个动作，得到相同的团队奖励。比较“独立 Q 学习”与“联合动作学习”。\n",
    "\n",
    "配套教程：[多智能体强化学习](https://dqn.cn/advanced/multi-agent)（DQN.CN 强化学习中文教程）\n",
    "\n",
    "源码：`examples/advanced/marl_climbing.py`，本 notebook 由 `scripts/build-notebooks.mjs` 自动生成，代码与 .py 文件一致。代码默认在 CPU 上运行，固定了随机种子；部分脚本支持设置环境变量 `DQN_QUICK=1` 只跑很短的版本（见下一个单元）。"
   ],
   "id": "advanced-marl_climbing-py-0"
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 环境准备：在 Colab 等缺少依赖的环境中自动安装（本地已装好依赖时什么也不做）\n",
    "import importlib.util, os, subprocess, sys\n",
    "sys.argv = sys.argv[:1]   # notebook 内核会带上 -f 等参数，清掉以免脚本把它们当成命令行参数\n",
    "# os.environ[\"DQN_QUICK\"] = \"1\"   # 取消注释：支持该开关的脚本只跑很短的版本，用于快速检查"
   ],
   "id": "advanced-marl_climbing-py-1"
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "\n",
    "# 行 = 智能体 1 的动作，列 = 智能体 2 的动作；最优联合动作是 (0, 0)，奖励 11\n",
    "PAYOFF = np.array([[11, -30, 0],\n",
    "                   [-30, 7, 6],\n",
    "                   [0, 0, 5]], dtype=float)"
   ],
   "id": "advanced-marl_climbing-py-2"
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def independent_q(episodes=5000, alpha=0.1, seed=0):\n",
    "    rng = np.random.default_rng(seed)\n",
    "    q1, q2 = np.zeros(3), np.zeros(3)          # 每个智能体只看自己的动作价值\n",
    "    for t in range(episodes):\n",
    "        eps = max(0.05, 1.0 - t / (0.8 * episodes))\n",
    "        a1 = rng.integers(3) if rng.random() < eps else int(np.argmax(q1))\n",
    "        a2 = rng.integers(3) if rng.random() < eps else int(np.argmax(q2))\n",
    "        r = PAYOFF[a1, a2]\n",
    "        q1[a1] += alpha * (r - q1[a1])          # 单步博弈：目标就是即时奖励\n",
    "        q2[a2] += alpha * (r - q2[a2])\n",
    "    return int(np.argmax(q1)), int(np.argmax(q2))"
   ],
   "id": "advanced-marl_climbing-py-3"
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def joint_q(episodes=5000, alpha=0.1, seed=0):\n",
    "    rng = np.random.default_rng(seed)\n",
    "    q = np.zeros((3, 3))                       # 集中式：学习联合动作价值 Q(a1, a2)\n",
    "    for t in range(episodes):\n",
    "        eps = max(0.05, 1.0 - t / (0.8 * episodes))\n",
    "        if rng.random() < eps:\n",
    "            a1, a2 = rng.integers(3), rng.integers(3)\n",
    "        else:\n",
    "            a1, a2 = np.unravel_index(np.argmax(q), q.shape)\n",
    "        q[a1, a2] += alpha * (PAYOFF[a1, a2] - q[a1, a2])\n",
    "    return tuple(int(x) for x in np.unravel_index(np.argmax(q), q.shape))"
   ],
   "id": "advanced-marl_climbing-py-4"
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "for name, fn in [(\"独立 Q 学习\", independent_q), (\"联合动作 Q 学习\", joint_q)]:\n",
    "    outcomes = [fn(seed=s) for s in range(100)]\n",
    "    rewards = [PAYOFF[a] for a in outcomes]\n",
    "    hit = np.mean([o == (0, 0) for o in outcomes])\n",
    "    print(f\"{name:10s} | 收敛到最优 (0,0) 的比例 {hit:.2f} | 最终联合动作平均奖励 {np.mean(rewards):5.2f}\")"
   ],
   "id": "advanced-marl_climbing-py-5"
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "name": "python"
  },
  "colab": {
   "provenance": []
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}