'''
The Q-learning algorithm is used to learn a function approximator
for the state-action values of Tic-Tac-Toe positions.
'''
from capstone.game.games import TicTacToe
from capstone.game.players import RandPlayer
from capstone.rl import Environment, GameMDP, FixedGameMDP
from capstone.rl.learners import ApproximateQLearningSelfPlay, ApproximateQLearning
from capstone.rl.policies import EGreedy
from capstone.rl.utils import EpisodicWLDPlotter, Callback, LinearAnnealing
from capstone.rl.value_functions import QNetwork

mapping = { int(x): int(x) - 1 for x in range(1, 10)}

game = TicTacToe()
# mdp = GameMDP(game)
mdp = FixedGameMDP(game, RandPlayer(), 1)
env = Environment(mdp)
qnetwork = QNetwork(n_input_units=9, n_hidden_layers=1, n_output_units=9, n_hidden_units=100)
egreedy = EGreedy(env.actions, qnetwork, 1.0)
qlearning = ApproximateQLearning(
    env=env,
    qfunction=qnetwork,
    policy=egreedy,
    discount_factor=1.0,
    n_episodes=4000,
    experience_replay=True,
    batch_size=32
)
qlearning.train(
    callbacks=[
        EpisodicWLDPlotter(
            game=game,
            opp_player=RandPlayer(),
            n_matches=1000,
            period=250,
            filepath='figures/tic_deep_ql.pdf'
        ),
        LinearAnnealing(egreedy, 'epsilon', init=1.0, final=0.1, n_episodes=2000)
    ]
)

@@ -303,38 +303,8 @@
 port
- ApproximateQLearningSelfPlay,
  App
@@ -317,24 +317,24 @@
 teQLearning%0A
+
 from capston
@@ -669,16 +669,25 @@
 Network(
+mapping, 
 n_input_
@@ -918,29 +918,8 @@
 .0,%0A
-    n_episodes=4000,%0A
     
@@ -962,16 +962,16 @@
 ze=32%0A)%0A
-
 qlearnin
@@ -979,16 +979,37 @@
 .train(%0A
+    n_episodes=4000,%0A
     call

Update experiment