\documentclass[11pt]{exam}
\usepackage{amsmath}
\usepackage{graphicx}
\usepackage{xstring}

\title{\vspace{-5em}Reinforcement Learning---HW 7 \ifprintanswers Solution \fi}
\date{\vspace{-2em}March 23, 2020}

% If filename contains 'solution', then print answers
\IfSubStr*{\jobname}{solution}{\printanswers}{}

\begin{document}
\maketitle
\thispagestyle{empty} % no page numbers


\begin{questions}

\question 
Your TD algorithm generates the following episode using policy $\pi$ when interacting with its environment.
This is the first episode that has been generated.

\begin{tabular}{r|l|l|l}
\textbf{Timestep}&\textbf{Reward} & \textbf{State} & \textbf{Action} \\ \hline
0 &                    & S1             & A1              \\ \hline
1 & 16               & S1             & A2              \\ \hline
2 & 12               & S1             & A1              \\ \hline
3 & 24               & S1             & A1              \\ \hline
4 & 16               &   T             &                 \\ 
\end{tabular}

Assume the discount factor, $\gamma$, is $\frac{1}{2}$, the step size, $\alpha$, is 0.1, and that all $q_\pi$ values are currently 0.

What are the estimates of: $q_\pi(S1, A1)$ and $q_\pi(S1, A2)$ assuming the use of 2-step Sarsa? Show your work.

  
\end{questions}



\end{document}
