Fixed awarding and penalty at the end of each game

a_player.py

@@ -1,6 +1,12 @@
+import enum
+
 import numpy as np
 from numpy.random import uniform
 
+class Reason(enum.Enum):
+    Won = 1
+    Lost = 2
+    Undecided = 3
 
 class APlayer(object):
     def __init__(self, mark):
@@ -15,6 +21,9 @@ class APlayer(object):
     def move(self, state: np.array):
         return state, False
 
+    def reward(self, value):
+        pass
+
     def new_game(self):
         pass
 

tic_tac_toe.py

@@ -1,6 +1,6 @@
 import numpy as np
 from numpy.random import uniform
-from a_player import APlayer
+from a_player import APlayer, Reason
 from helper import sample, create_empty_state, to_state_string
 
 float_formatter = "{:.3f}".format
@@ -21,6 +21,12 @@ class MachinePlayer(APlayer):
     def init_values(self):
         self.values = {}
 
+    def print_state_table(self):
+        count = 0
+        for key in self.values:
+            print(f"{self.mark}: {count:05d}: {key} = {self.values[key]:0.3f}")
+            count += 1
+
     @staticmethod
     def to_key(state: np.array):
         key = ''
@@ -29,14 +35,11 @@ class MachinePlayer(APlayer):
         return key
 
     def get_value(self, state: np.array):
-        if state is None:
-            return 0
-
         key = self.to_key(state)
         try:
             result = self.values[key]
         except KeyError:
-            result = 0
+            result = 0.5
 
         return result
 
@@ -44,7 +47,7 @@ class MachinePlayer(APlayer):
         self.values[self.to_key(state)] = value
 
     def reward(self, value):
-        self.set_value(self.state_last, value)
+        self.set_value(self.state, value)
 
     @staticmethod
     def get_potential_moves(state: np.array) -> np.array:
@@ -57,35 +60,44 @@ class MachinePlayer(APlayer):
         self.state_last = None
 
     def move(self, state: np.array):
+        do_sample = False
         values = np.array([])
         # get possible move
         moves = self.get_potential_moves(state)
         can_move = moves.size > 0
+        best_move = None
+        best_value = -1
         if can_move:
             for move in moves:
                 # create hypothetical next state
                 state_next = self.state_from_move(state.copy(), move)
                 # evaluate value
                 value = self.get_value(state_next)
+                if best_value < value:
+                    best_value = value
+                    best_move = move
                 values = np.append(values, value)
 
+            next_move = best_move
             is_exp = False
             if uniform() < self.p_exp:
                 is_exp = True
                 index = np.random.randint(len(moves))
-            else:
+                next_move = moves[index]
+            elif do_sample:
                 index = sample(values)
+                next_move = moves[index]
 
-            move = moves[index]
             if self.with_debug:
                 print(f"{self.mark}: Values    = {values}")
                 print(f"{self.mark}: Moves     = {moves+1}")
-                print(f"{self.mark}: Move = {move+1}, is_exp={is_exp}")
+                print(f"{self.mark}: Best move = {best_move+1}")
+                print(f"{self.mark}: Next move = {next_move+1}, is_exp={is_exp}")
 
             if self.state is not None:
                 self.state_last = self.state.copy()
 
-            self.state = self.state_from_move(state.copy(), move)
+            self.state = self.state_from_move(state.copy(), next_move)
 
             # Learn
             if not is_exp and self.state_last is not None:
@@ -106,31 +118,28 @@ class MachinePlayer(APlayer):
 
 class PlayerProvider:
     def __init__(self, p1: APlayer, p2: APlayer):
-        self.p1 = p1
+        self.players = [p1, p2]
-        self.p2 = p2
 
-
+    def choose(self) -> list[APlayer]:
-def choose_player(player_provider: PlayerProvider) -> list[APlayer]:
         # Wer fängt an?
-    p1 = player_provider.p1
+        players = self.players
-    p2 = player_provider.p2
         if uniform() < 0.5:
-        result = [p1, p2]
+            players.reverse()
-    else:
-        result = [p2, p1]
 
-    p1.new_game()
+        result = players
-    p2.new_game()
+        players[0].new_game()
+        players[1].new_game()
 
         return result
 
 
 def play(player_provider: PlayerProvider, k_max=10000, with_print=False):
     for k in range(0, k_max):
-        players = choose_player(player_provider)
+        players = player_provider.choose()
         state = create_empty_state()
         move = 1
         run = True
+        other_player = players[-1]
         while run:
             for player in players:
                 if with_print:
@@ -141,32 +150,39 @@ def play(player_provider: PlayerProvider, k_max=10000, with_print=False):
                 state, has_moved = player.move(state)
                 if with_print:
                     print(to_state_string(last_state, state))
-
                 if not has_moved:
                     if with_print:
                         print(f"{player.mark}: No more moves")
+                    if isinstance(player, MachinePlayer):
+                        player.reward(0.0)
+                    if isinstance(other_player, MachinePlayer):
+                        other_player.reward(0.0)
                     run = False
                 if player.has_won(state):
                     if with_print:
                         print(f"{player.mark}: Has won the game")
                     if isinstance(player, MachinePlayer):
                         player.reward(1.0)
+                    if isinstance(other_player, MachinePlayer):
+                        other_player.reward(0.0)
                     run = False
 
+                other_player = player
                 if not run:
                     break
 
                 move += 1
 
+
 px = MachinePlayer(mark='X', with_debug=False)
 po = MachinePlayer(mark='O', with_debug=False)
 
-
+do_training = 1
-do_training = 0
 if do_training:
     players = PlayerProvider(px, po)
-    play(players, 4000, False)
+    play(players, 40000, False)
 
 players = PlayerProvider(MachinePlayer(mark='X', with_debug=True, values=px.values), MachinePlayer(mark='O', with_debug=True, values=po.values))
-play(players, 4000, True)
+play(players, 1000, True)
 
+px.print_state_table()