\documentclass[12pt,a4paper]{article}
\usepackage[margin=0.5in]{geometry}
\usepackage{setspace}
\usepackage{amsmath,amssymb}
\usepackage{enumitem}
\usepackage{parskip}
\usepackage{titlesec}

% Compact section titles
\titleformat{\section}{\large\bfseries}{}{0em}{}

\begin{document}

\begin{center}
    {\Large \textbf{Bellman Equation via Banach Fixed-Point}} \vspace{1cm}
\end{center}
\begin{center}    
    \textbf{Name:} \rule{6cm}{0.4pt} \hfill \textbf{Date:} \rule{3cm}{0.4pt}
\end{center}

\vspace{1em}

\section*{Q1}
\textbf{Question:} Under a fixed policy $\pi$ with natural filtration $\mathcal{F}_t = \sigma(S_0, A_0, \dots, S_t)$, define
\[
\delta_t = R_t + \beta\,V_\pi(S_{t+1}) - V_\pi(S_t).
\]
\begin{enumerate}[label=(\alph*)]
    \item Show that $\mathbb{E}[\delta_t \mid \mathcal{F}_t] = 0$.
    \item Let $\displaystyle M_t := \sum_{k=0}^{t-1} \delta_k$. Deduce that $(M_t)_{t\ge 1}$ is a martingale with respect to $(\mathcal{F}_t)_{t\ge 1}$.
\end{enumerate}

% \vspace{2.5cm}
% \vspace{2.5cm}
% \vspace{2.5cm}
% \vspace
\newpage

\section*{Q2}
\textbf{Question:} Is it possible for the expected discounted return
\[
\mathbb{E}\!\left[\sum_{t=0}^{\infty} \beta^t R_t\right]
\]
to diverge when $0 \le \beta < 1$ and the rewards are non-negative and bounded? 
Explain precisely why or why not. 
\vspace{3cm}
\section*{Q3}
\textbf{Question:} In your python notebook, the+ MDP has 3 states and 2 actions, with discount factor $\beta = 0.85$ and initial values $V_0 = (0, 0, 0)$. 
For state $s = 0$, compute one complete \emph{value-iteration update} step:
\[
V_1(0) = \max_a \sum_{s'} P(s' \mid 0,a)\,[R(0,a,s') + 0.85\,V_0(s')].
\]
Show all intermediate quantities (each action’s $Q_a$) and give your final number for $V_1(0)$.


\vspace{3cm}
\vspace{3cm}

\end{document}
