import gymnasium as gym
from stable_baselines3 import PPO

env = gym.make("CartPole-v1")
model = PPO("MlpPolicy", env, verbose=1, n_steps=1024, batch_size=64)
model.learn(total_timesteps=50_000)

scores = []
for _ in range(10):
    obs, _ = env.reset(); total = 0; done = False
    while not done:
        action, _ = model.predict(obs, deterministic=True)
        obs, r, term, trunc, _ = env.step(action); total += r; done = term or trunc
    scores.append(total)
print(f"Trained agent: {sum(scores)/len(scores):.0f}/500 avg score")

import gymnasium as gym
import numpy as np

env = gym.make("FrozenLake-v1", map_name="4x4", is_slippery=False)
q = np.zeros((16, 4)); lr, gamma, eps = 0.8, 0.95, 1.0

for ep in range(20_000):
    s, _ = env.reset(); done = False
    while not done:
        a = env.action_space.sample() if np.random.random() < eps else np.argmax(q[s])
        s2, r, t, tr, _ = env.step(a)
        q[s, a] += lr * (r + gamma * np.max(q[s2]) - q[s, a])
        s, done = s2, t or tr
    eps = max(0.05, eps * 0.999)

arrows = ["←", "↓", "→", "↑"]; holes = {5,7,11,12}
for row in range(4):
    line = ""
    for c in range(4):
        idx = row*4+c
        if idx in holes: line += " H "
        elif idx == 15: line += " G "
        else: line += f" {['L','D','R','U'][np.argmax(q[idx])]} "
    print(line)

wins = 0
for _ in range(1000):
    s, _ = env.reset()
    for _ in range(100):
        s, r, t, tr, _ = env.step(np.argmax(q[s]))
        if t or tr: break
    wins += r
print(f"Win rate: {wins/10:.1f}%")

import random, numpy as np, chess, torch, torch.nn as nn

def board_to_tensor(b):
    p = np.zeros((12,8,8), dtype=np.float32)
    for sq, piece in b.piece_map().items():
        r,c = divmod(sq,8)
        p[(piece.piece_type-1)+(6 if piece.color==chess.BLACK else 0)][r][c]=1.
    return torch.tensor(p)

class Net(nn.Module):
    def __init__(self):
        super().__init__()
        self.net = nn.Sequential(nn.Conv2d(12,32,3,padding=1),nn.ReLU(),
            nn.Conv2d(32,32,3,padding=1),nn.ReLU(),nn.Flatten(),
            nn.Linear(32*64,256),nn.ReLU(),nn.Linear(256,4096))
    def forward(self,x): return self.net(x)

data = []
for _ in range(500):
    b = chess.Board()
    for _ in range(40):
        if b.is_game_over(): break
        m = random.choice(list(b.legal_moves)); data.append((b.copy(),m)); b.push(m)
print(f"{len(data)} positions")

model = Net(); opt = torch.optim.Adam(model.parameters(), lr=1e-3)
for epoch in range(5):
    random.shuffle(data); correct = total = 0
    for i in range(0,len(data)-64,64):
        batch = data[i:i+64]
        X = torch.stack([board_to_tensor(b) for b,_ in batch])
        Y = torch.tensor([m.from_square*64+m.to_square for _,m in batch])
        opt.zero_grad(); loss = nn.CrossEntropyLoss()(model(X),Y)
        loss.backward(); opt.step()
        correct += (model(X).argmax(1)==Y).sum().item(); total += len(Y)
    print(f"Epoch {epoch+1}: {correct/total*100:.1f}% accuracy")

import random

G, skill = 8, 0.3
prompts = [f"{a}+{b}" for a in range(1,20) for b in range(1,20)]
random.shuffle(prompts)

print("Simulating GRPO on simple addition...")
print(f"Starting accuracy: {skill:.0%}")
print()

for epoch in range(10):
    rewards = []
    for p in prompts[:50]:
        correct_answer = eval(p)
        group_rewards = []
        for _ in range(G):
            if random.random() < skill:
                group_rewards.append(1.0)  # correct
            else:
                group_rewards.append(0.0)  # wrong
        rewards.extend(group_rewards)
    skill = min(0.99, skill + (1-skill) * 0.15)
    avg_r = sum(rewards) / len(rewards)
    print(f"Epoch {epoch+1:2d} | avg reward: {avg_r:.3f} | model accuracy: {skill:.1%}")

print(f"\nFinal accuracy: {skill:.1%} (started at 30%)")

import numpy as np
np.random.seed(42)

true_rewards = np.random.normal([1.0, 1.5, 0.8, 2.0, 1.2], 0.3)
Q = np.zeros(5)  # estimated value of each arm
N = np.zeros(5)  # times each arm was pulled

for t in range(1, 2001):
    if np.random.random() < 0.1:
        arm = np.random.randint(5)  # explore: random arm
    else:
        arm = np.argmax(Q)          # exploit: best known arm

    reward = np.random.normal(true_rewards[arm], 0.5)
    N[arm] += 1
    Q[arm] += (reward - Q[arm]) / N[arm]  # incremental mean update

    if t in [100, 500, 1000, 2000]:
        print(f"t={t:4d} | Q={np.round(Q,2)} | pulls={N.astype(int)}")

print(f"\nBest arm: {np.argmax(Q)} (true best: {np.argmax(true_rewards)})")

import gymnasium as gym
import numpy as np

env = gym.make("Taxi-v3")
q = np.zeros((env.observation_space.n, env.action_space.n))
lr, gamma, eps = 0.1, 0.99, 1.0

for _ in range(10_000):
    s, info = env.reset(); done = False
    while not done:
        mask = info.get("action_mask", np.ones(env.action_space.n))
        if np.random.random() < eps:
            a = np.random.choice(np.where(mask)[0])  # explore legal actions
        else:
            a = np.argmax(np.where(mask, q[s], -1e9))  # exploit best legal
        s2, r, t, tr, info = env.step(a)
        q[s, a] += lr * (r + gamma * np.max(q[s2]) - q[s, a])
        s, done = s2, t or tr
    eps = max(0.01, eps * 0.9995)

results = []
for _ in range(100):
    s, info = env.reset(); tot = 0
    for _ in range(200):
        mask = info.get("action_mask", np.ones(6))
        s, r, t, tr, info = env.step(np.argmax(np.where(mask, q[s], -1e9)))
        tot += r
        if t or tr: break
    results.append(tot)
print(f"Avg reward: {np.mean(results):.1f} (optimal ~8)")

import gymnasium as gym
import numpy as np

env = gym.make("MountainCar-v0")

def train(env, reward_fn, episodes=3000):
    bins = [20, 20]
    lo, hi = env.observation_space.low, env.observation_space.high
    sz = (hi - lo) / bins
    q = np.zeros(bins + [env.action_space.n])
    def discretize(obs):
        return tuple(np.clip(((obs - lo) / sz).astype(int), 0, np.array(bins) - 1))
    lr, gamma, ep = 0.1, 0.99, 1.0
    for _ in range(episodes):
        obs, _ = env.reset(); s = discretize(obs)
        for _ in range(200):
            a = env.action_space.sample() if np.random.random() < ep else np.argmax(q[s])
            obs2, r, t, tr, _ = env.step(a)
            shaped_r = reward_fn(obs, obs2, r, t)
            s2 = discretize(obs2)
            q[s][a] += lr * (shaped_r + gamma * np.max(q[s2]) - q[s][a])
            s, obs = s2, obs2
            if t or tr: break
        ep = max(0.01, ep * 0.999)
    return q, discretize

for name, fn in [
    ("Default reward", lambda o, o2, r, t: r),
    ("Shaped reward",  lambda o, o2, r, t: r + (o2[0] - o[0]) * 10 + abs(o2[1]) * 5),
]:
    q, disc = train(env, fn)
    wins = 0
    for _ in range(100):
        obs, _ = env.reset()
        for _ in range(200):
            obs, r, t, tr, _ = env.step(np.argmax(q[disc(obs)]))
            if t: wins += 1; break
            if tr: break
    print(f"{name}: {wins}/100 reached the goal")

print("\nSame algorithm, same environment.")
print("Different reward function = completely different behavior.")