def episode(sim, agent, params):
  """Builds an episode graph using a Simulator and an Agent

  Args:
    sim: a Simulator object
    agent: an Agent object
    params: application level constants

  Returns:
    (a_ts, q_ts)
    a_ts: (batch_size, num_actions)[time_steps], tf.int32, the chosen actions
    q_ts: (batch_size, num_actions)[time_steps], tf.float32, expected future return for each action
  """
  a_ts = []
  q_ts = []
  
  for t in range(params.time_steps):
    o_ts = sim.get_observation()
    a_t, q_t = agent.next_action(o_ts)

    a_ts.append(a_t)
    q_ts.append(q_t)

    sim.do_step(a_t)

  return (a_ts, q_ts)