Add Chapter 10: Function Approximation (code, tests, workflow)

Author: srikanthbaride

.github/workflows/ch10.yml

@@ -0,0 +1,38 @@
+name: ch10 — Function Approximation
+
+on:
+  push:
+    paths:
+      - 'ch10_function_approx/**'
+      - '.github/workflows/ch10.yml'
+  pull_request:
+    paths:
+      - 'ch10_function_approx/**'
+      - '.github/workflows/ch10.yml'
+
+jobs:
+  test:
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        python-version: ['3.9', '3.10', '3.11']
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: ${{ matrix.python-version }}
+          cache: 'pip'
+
+      - name: Install dependencies
+        run: |
+          python -m pip install -U pip
+          pip install -r ch10_function_approx/requirements.txt
+
+      - name: Run Chapter 10 tests
+        env:
+          PYTHONPATH: .
+        run: |
+          pytest -q ch10_function_approx/tests

ch10_function_approx/README_ch10.md

@@ -0,0 +1,8 @@
+# Chapter 10 — Function Approximation Basics
+
+Usage:
+```bash
+pip install -r ch10_function_approx/requirements.txt
+pytest ch10_function_approx/tests
+python -m ch10_function_approx.scripts.run_mc_demo
+```

ch10_function_approx/__init__.py

@@ -0,0 +1,15 @@
+# Chapter 10 — Function Approximation Basics
+from .features.tile_coding import TileCoder, ActionBlockTileCoder
+from .agents.linear_sarsa import LinearSarsaAgent
+from .agents.linear_td0 import LinearTD0
+from .envs.mountain_car import MountainCar
+from .utils.policies import epsilon_greedy
+
+__all__ = [
+    "TileCoder",
+    "ActionBlockTileCoder",
+    "LinearSarsaAgent",
+    "LinearTD0",
+    "MountainCar",
+    "epsilon_greedy",
+]

ch10_function_approx/agents/__init__.py

@@ -0,0 +1 @@
+# package

ch10_function_approx/agents/linear_sarsa.py

@@ -0,0 +1,35 @@
+from __future__ import annotations
+import numpy as np
+from dataclasses import dataclass
+from ..utils.policies import epsilon_greedy
+
+@dataclass
+class LinearSarsaAgent:
+    d: int
+    nA: int
+    gamma: float = 1.0
+    alpha: float = 0.5
+    eps: float = 0.05
+    seed: int | None = None
+
+    def __post_init__(self):
+        self.w = np.zeros(self.d, dtype=float)
+        self.rng = np.random.default_rng(self.seed)
+
+    def q_row(self, phi_fn, s_vec) -> np.ndarray:
+        vals = np.zeros(self.nA, dtype=float)
+        for a in range(self.nA):
+            vals[a] = self.w @ phi_fn(s_vec, a)
+        return vals
+
+    def act(self, phi_fn, s_vec) -> int:
+        q = self.q_row(phi_fn, s_vec)
+        return epsilon_greedy(q, self.eps, self.rng)
+
+    def step(self, phi_fn, s_vec, a, r, ns_vec, na):
+        phi_sa = phi_fn(s_vec, a)
+        phi_ns_na = phi_fn(ns_vec, na)
+        td_target = r + self.gamma * (self.w @ phi_ns_na)
+        td_err = td_target - (self.w @ phi_sa)
+        self.w += self.alpha * td_err * phi_sa
+        return td_err

ch10_function_approx/agents/linear_td0.py

@@ -0,0 +1,22 @@
+from __future__ import annotations
+import numpy as np
+from dataclasses import dataclass
+
+@dataclass
+class LinearTD0:
+    d: int
+    gamma: float = 0.99
+    alpha: float = 0.1
+    seed: int | None = None
+
+    def __post_init__(self):
+        self.w = np.zeros(self.d, dtype=float)
+        self.rng = np.random.default_rng(self.seed)
+
+    def predict(self, phi_s: np.ndarray) -> float:
+        return float(self.w @ phi_s)
+
+    def update(self, phi_s: np.ndarray, r: float, phi_ns: np.ndarray):
+        delta = r + self.gamma * (self.w @ phi_ns) - (self.w @ phi_s)
+        self.w += self.alpha * delta * phi_s
+        return delta

ch10_function_approx/envs/__init__.py

@@ -0,0 +1 @@
+# package

ch10_function_approx/envs/mountain_car.py

@@ -0,0 +1,50 @@
+from __future__ import annotations
+from dataclasses import dataclass
+import numpy as np
+
+@dataclass
+class MountainCarConfig:
+    x_min: float = -1.2
+    x_max: float = 0.6
+    v_min: float = -0.07
+    v_max: float = 0.07
+    goal_x: float = 0.5
+    gamma: float = 1.0
+    max_steps: int = 2000
+
+class MountainCar:
+    LEFT, NEUTRAL, RIGHT = 0, 1, 2
+
+    def __init__(self, cfg: MountainCarConfig = MountainCarConfig()):
+        self.cfg = cfg
+        self.rng = np.random.default_rng()
+        self.reset()
+
+    @property
+    def nA(self): return 3
+
+    def reset(self, seed: int | None = None):
+        if seed is not None:
+            self.rng = np.random.default_rng(seed)
+        self.x = self.rng.uniform(-0.6, -0.4)
+        self.v = 0.0
+        self.t = 0
+        return np.array([self.x, self.v], dtype=float)
+
+    def step(self, a: int):
+        assert 0 <= a < self.nA
+        force = {self.LEFT: -1.0, self.NEUTRAL: 0.0, self.RIGHT: +1.0}[a]
+        v = self.v + 0.001 * force - 0.0025 * np.cos(3 * self.x)
+        v = np.clip(v, self.cfg.v_min, self.cfg.v_max)
+        x = self.x + v
+        if x < self.cfg.x_min:
+            x = self.cfg.x_min
+            v = 0.0
+        self.x, self.v = x, v
+        self.t += 1
+        done = (self.x >= self.cfg.goal_x) or (self.t >= self.cfg.max_steps)
+        reward = 0.0 if (self.x >= self.cfg.goal_x) else -1.0
+        return np.array([self.x, self.v], dtype=float), reward, done, {}
+
+    def state(self):
+        return np.array([self.x, self.v], dtype=float)

ch10_function_approx/examples/__init__.py

@@ -0,0 +1 @@
+# package

ch10_function_approx/examples/mountain_car_linear.py

@@ -0,0 +1,40 @@
+from __future__ import annotations
+import numpy as np
+from ..envs.mountain_car import MountainCar, MountainCarConfig
+from ..features.tile_coding import TileCoder, ActionBlockTileCoder
+from ..agents.linear_sarsa import LinearSarsaAgent
+
+def make_tilecoder(n_tilings=8, bins=(8,8)):
+    lows = np.array([-1.2, -0.07], dtype=float)
+    highs = np.array([0.6, 0.07], dtype=float)
+    offsets = []
+    rng = np.random.default_rng(0)
+    for t in range(n_tilings):
+        offsets.append(rng.random(2) * 0.999)
+    tc = TileCoder(lows=lows, highs=highs, bins_per_dim=bins, n_tilings=n_tilings, offsets=offsets)
+    return tc
+
+def run(episodes=50, seed=0, n_tilings=8):
+    env = MountainCar(MountainCarConfig())
+    tc = make_tilecoder(n_tilings=n_tilings, bins=(8,8))
+    atc = ActionBlockTileCoder(tc, n_actions=env.nA)
+    agent = LinearSarsaAgent(d=atc.d, nA=env.nA, gamma=1.0, alpha=0.5/n_tilings, eps=0.05, seed=seed)
+
+    steps_per_ep = []
+    for ep in range(episodes):
+        s = env.reset(seed + ep)
+        a = agent.act(atc.phi, s)
+        steps = 0
+        while True:
+            ns, r, done, _ = env.step(a)
+            na = agent.act(atc.phi, ns)
+            agent.step(atc.phi, s, a, r, ns, na)
+            s, a = ns, na
+            steps += 1
+            if done: break
+        steps_per_ep.append(steps)
+    return np.array(steps_per_ep), agent.w
+
+if __name__ == "__main__":
+    steps, w = run(episodes=20, seed=123, n_tilings=8)
+    print("Steps per episode:", steps)