- control debug output

- fixed last state
- refactored

tic_tac_toe.py

@@ -11,12 +11,13 @@ def softmax(values: np.array) -> np.array:
     return result
 
 
-def sample(values: np.array):
+def sample(values: np.array, with_debug=False):
     # Normalize and sort
     probs = softmax(values + 1.0E-6)
     z = uniform()
-    print(f"Probs={probs}")
+    if with_debug:
-    print(f"Z={z:.3f}")
+        print(f"Probs={probs}")
+        print(f"Z={z:.3f}")
     index = None
     p_sum = 0
     for idx, p in enumerate(probs):
@@ -70,66 +71,14 @@ def to_state_string(state, state_nex=None):
     return state_str
 
 
-class Player(object):
+class APlayer(object):
-    def __init__(self, mark='X'):
+    def __init__(self, mark):
-        self.values = {}
         self.mark = mark
+        self.state = None
         self.state_last = None
 
-    def get_value(self, state):
-        try:
-            result = self.values[state]
-        except KeyError:
-            result = 0
-
-        return result
-
-    def set_value(self, value, state=None):
-        if state is None:
-            if self.state_last is not None:
-                self.values[self.state_last] = value
-        else:
-            self.values[state] = value
-
-    @staticmethod
-    def get_potential_moves(state) -> np.array:
-        indices = [idx for idx, s in enumerate(state) if '-' in s]
-        return np.array(indices)
-
     def move(self, state):
-        values = np.array([])
+        return state, False
-        # get possible move
-        moves = self.get_potential_moves(state)
-        can_move = moves.size > 0
-        state_next = state
-
-        if can_move:
-            for field in moves:
-                # crate hypothetical next state
-                state_next = self.state_from_move(state, field)
-                # evaluate value
-                value = self.get_value(state_next)
-                values = np.append(values, value)
-
-            index, is_exp = sample(values)
-            field = moves[index]
-            print(f"{player.mark}: Chose {index}, is_exp={is_exp}")
-            state_next = self.state_from_move(state, field)
-
-            # Learn
-            if not is_exp and self.state_last is not None:
-                v0 = self.get_value(self.state_last)
-                v1 = self.get_value(state_next)
-                d = max(0, v1-v0)
-                if d > 0:
-                    self.set_value(0.1*d)
-                print(f"{player.mark}: Learned {d:0.3f}")
-
-            self.state_last = state_next
-        return state_next, can_move
-
-    def state_from_move(self, state, field):
-        return state[:field] + self.mark + state[field + 1:]
 
     def has_won(self, state):
         ref = self.mark + self.mark + self.mark
@@ -169,38 +118,109 @@ class Player(object):
         return False
 
 
-p_x = Player(mark='X')
+class MachinePlayer(APlayer):
-p_o = Player(mark='O')
+    def __init__(self, mark='X', with_debug=False):
+        APlayer.__init__(self, mark)
+        self.with_debug = with_debug
+        self.values = {}
+
+    def get_value(self, state):
+        try:
+            result = self.values[state]
+        except KeyError:
+            result = 0
+
+        return result
+
+    def set_value(self, value):
+        if self.state_last is not None:
+            self.values[self.state_last] = value
+
+    @staticmethod
+    def get_potential_moves(state) -> np.array:
+        indices = [idx for idx, s in enumerate(state) if '-' in s]
+        return np.array(indices)
+
+    def move(self, state):
+        values = np.array([])
+        # get possible move
+        moves = self.get_potential_moves(state)
+        can_move = moves.size > 0
+
+        if can_move:
+            for field in moves:
+                # crate hypothetical next state
+                state_next = self.state_from_move(state, field)
+                # evaluate value
+                value = self.get_value(state_next)
+                values = np.append(values, value)
+
+            index, is_exp = sample(values, self.with_debug)
+            field = moves[index]
+            if self.with_debug:
+                print(f"{self.mark}: Chose {index}, is_exp={is_exp}")
+            self.state_last = self.state
+            self.state = self.state_from_move(state, field)
+
+            # Learn
+            if not is_exp and self.state_last is not None:
+                v0 = self.get_value(self.state_last)
+                v1 = self.get_value(self.state)
+                d = max(0, v1-v0)
+                if d > 0:
+                    self.set_value(0.1*d)
+                    if self.with_debug:
+                        print(f"{self.mark}: Learned {d:0.3f}")
+
+        return self.state, can_move
+
+    def state_from_move(self, state, field):
+        return state[:field] + self.mark + state[field + 1:]
 
 
-for k in range(0, 1000):
+def play(players: list[APlayer], k_max=10000, with_print=False):
+    for k in range(0, k_max):
+        state = "---------"
+        move = 1
+        run = True
+        while run:
+            for player in players:
+                if with_print:
+                    print(f"---------------------------------------------------")
+                    print(f"- Game {k:06d}, Move {move} -----------------------------")
+                    print(f"---------------------------------------------------")
+                last_state = state
+                state, has_moved = player.move(state)
+                if with_print:
+                    print(to_state_string(last_state, state))
+
+                if not has_moved:
+                    if with_print:
+                        print(f"{player.mark}: No more moves")
+                    run = False
+                if player.has_won(state):
+                    if with_print:
+                        print(f"{player.mark}: Has won the game")
+                    if isinstance(player, MachinePlayer):
+                        player.set_value(1.0)
+                    run = False
+
+                if not run:
+                    break
+
+                move += 1
+
+
+def choose_player(p1: APlayer, p2: APlayer) -> list[APlayer]:
     # Wer fängt an?
     if uniform() < 0.5:
-        players = [p_x, p_o]
+        result = [p1, p2]
     else:
-        players = [p_o, p_x]
+        result = [p2, p1]
-    state = "---------"
-    move = 1
-    run = True
-    while run:
-        for player in players:
-            print(f"---------------------------------------------------")
-            print(f"- Game {k:06d}, Move {move} -----------------------------")
-            print(f"---------------------------------------------------")
-            last_state = state
-            state, has_moved = player.move(state)
-            print(to_state_string(last_state, state))
 
-            if not has_moved:
+    return result
-                print(f"{player.mark}: No more moves")
-                run = False
-            if player.has_won(state):
-                print(f"{player.mark}: Has won the game")
-                player.set_value(1.0)
-                run = False
 
-            if not run:
-                break
 
-            move += 1
+opponents = choose_player(MachinePlayer(mark='X', with_debug=True), MachinePlayer(mark='O', with_debug=True))
+play(opponents, 1000, True)