{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 蒙特卡洛控制（首次访问、ε-贪心、同策略）：Blackjack-v1。\n",
    "\n",
    "\n",
    "\n",
    "配套教程：[蒙特卡洛方法](https://dqn.cn/basics/monte-carlo)（DQN.CN 强化学习中文教程）\n",
    "\n",
    "源码：`examples/basics/mc_blackjack.py`，本 notebook 由 `scripts/build-notebooks.mjs` 自动生成，代码与 .py 文件一致。代码默认在 CPU 上运行，固定了随机种子；部分脚本支持设置环境变量 `DQN_QUICK=1` 只跑很短的版本（见下一个单元）。"
   ],
   "id": "basics-mc_blackjack-py-0"
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 环境准备：在 Colab 等缺少依赖的环境中自动安装（本地已装好依赖时什么也不做）\n",
    "import importlib.util, os, subprocess, sys\n",
    "_need = [m for m in [\"gymnasium\"] if importlib.util.find_spec(m) is None]\n",
    "if _need or \"google.colab\" in sys.modules:\n",
    "    subprocess.run([sys.executable, \"-m\", \"pip\", \"install\", \"-q\", \"gymnasium>=1.0\"], check=True)\n",
    "sys.argv = sys.argv[:1]   # notebook 内核会带上 -f 等参数，清掉以免脚本把它们当成命令行参数\n",
    "# os.environ[\"DQN_QUICK\"] = \"1\"   # 取消注释：支持该开关的脚本只跑很短的版本，用于快速检查"
   ],
   "id": "basics-mc_blackjack-py-1"
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from collections import defaultdict\n",
    "\n",
    "import gymnasium as gym\n",
    "import numpy as np\n",
    "\n",
    "env = gym.make(\"Blackjack-v1\", sab=True)   # sab=True：与 Sutton & Barto 书中规则一致\n",
    "nA = env.action_space.n                     # 0 = 停牌(stick)，1 = 要牌(hit)\n",
    "Q = defaultdict(lambda: np.zeros(nA))\n",
    "N = defaultdict(lambda: np.zeros(nA))       # 访问次数，用于增量式平均\n",
    "gamma, epsilon = 1.0, 0.1\n",
    "rng = np.random.default_rng(0)"
   ],
   "id": "basics-mc_blackjack-py-2"
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def act(state, eps):\n",
    "    if rng.random() < eps:\n",
    "        return int(rng.integers(nA))\n",
    "    return int(np.argmax(Q[state]))"
   ],
   "id": "basics-mc_blackjack-py-3"
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "for ep in range(500_000):\n",
    "    # 1) 用当前 ε-贪心策略生成一个完整回合\n",
    "    episode = []\n",
    "    s, _ = env.reset(seed=ep)\n",
    "    done = False\n",
    "    while not done:\n",
    "        a = act(s, epsilon)\n",
    "        s2, r, term, trunc, _ = env.step(a)\n",
    "        episode.append((s, a, r))\n",
    "        s, done = s2, term or trunc\n",
    "\n",
    "    # 2) 从后往前计算回报，首次访问时更新 Q\n",
    "    G = 0.0\n",
    "    first_visit = {}\n",
    "    for t, (s, a, _) in enumerate(episode):\n",
    "        first_visit.setdefault((s, a), t)\n",
    "    for t in reversed(range(len(episode))):\n",
    "        s, a, r = episode[t]\n",
    "        G = gamma * G + r\n",
    "        if first_visit[(s, a)] == t:\n",
    "            N[s][a] += 1\n",
    "            Q[s][a] += (G - Q[s][a]) / N[s][a]     # 增量式均值"
   ],
   "id": "basics-mc_blackjack-py-4"
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def evaluate(policy_fn, episodes=100_000):\n",
    "    total = 0.0\n",
    "    for ep in range(episodes):\n",
    "        s, _ = env.reset(seed=10_000_000 + ep)\n",
    "        done = False\n",
    "        while not done:\n",
    "            s, r, term, trunc, _ = env.step(policy_fn(s))\n",
    "            done = term or trunc\n",
    "        total += r\n",
    "    return total / episodes"
   ],
   "id": "basics-mc_blackjack-py-5"
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "greedy = lambda s: int(np.argmax(Q[s]))\n",
    "naive = lambda s: 0 if s[0] >= 20 else 1     # 书中的“点数≥20 才停牌”策略\n",
    "print(f\"学到的贪心策略平均回报: {evaluate(greedy):+.4f}\")\n",
    "print(f\"“≥20 停牌”策略平均回报: {evaluate(naive):+.4f}\")"
   ],
   "id": "basics-mc_blackjack-py-6"
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "name": "python"
  },
  "colab": {
   "provenance": []
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}