From 7cad1442df2b51e8b0ccd74f8c363bf0a74a1924 Mon Sep 17 00:00:00 2001
From: Jens Ahrensfeld <jens@jayfield.org>
Date: Sun, 9 Jun 2024 17:21:35 +0200
Subject: [PATCH] Save values to file

---
 .gitignore        |  1 +
 machine_player.py | 81 ++++++++++++++++++++++++-----------------------
 tic_tac_toe.py    | 19 +++++++++--
 3 files changed, 58 insertions(+), 43 deletions(-)

diff --git a/.gitignore b/.gitignore
index c18dd8d..5bf2b8c 100644
--- a/.gitignore
+++ b/.gitignore
@@ -1 +1,2 @@
 __pycache__/
+values_*.json
diff --git a/machine_player.py b/machine_player.py
index f5a3796..3336395 100644
--- a/machine_player.py
+++ b/machine_player.py
@@ -61,52 +61,53 @@ class MachinePlayer(APlayer):
         values = np.array([])
         # get possible move
         moves = self.get_potential_moves(state)
-        can_move = moves.size > 0
+        if moves.size == 0:
+            return state, False
+
         best_move = None
         best_value = -1
-        if can_move:
-            for move in moves:
-                # create hypothetical next state
-                state_next = self.state_from_move(state.copy(), move)
-                # evaluate value
-                value = self.get_value(state_next)
-                if best_value < value:
-                    best_value = value
-                    best_move = move
-                values = np.append(values, value)
+        for move in moves:
+            # create hypothetical next state
+            state_next = self.state_from_move(state.copy(), move)
+            # evaluate value
+            value = self.get_value(state_next)
+            if best_value < value:
+                best_value = value
+                best_move = move
+            values = np.append(values, value)
 
-            next_move = best_move
-            is_exp = False
-            if uniform() < self.p_exp:
-                is_exp = True
-                index = np.random.randint(len(moves))
-                next_move = moves[index]
-            elif do_sample:
-                index = sample(values)
-                next_move = moves[index]
+        next_move = best_move
+        is_exp = False
+        if uniform() < self.p_exp:
+            is_exp = True
+            index = np.random.randint(len(moves))
+            next_move = moves[index]
+        elif do_sample:
+            index = sample(values)
+            next_move = moves[index]
 
+        if self.with_debug:
+            print(f"{self.mark}: Values    = {values}")
+            print(f"{self.mark}: Moves     = {moves+1}")
+            print(f"{self.mark}: Best move = {best_move+1}")
+            print(f"{self.mark}: Next move = {next_move+1}, is_exp={is_exp}")
+
+        if self.state is not None:
+            self.state_last = self.state.copy()
+
+        self.state = self.state_from_move(state.copy(), next_move)
+
+        # Learn
+        if not is_exp and self.state_last is not None:
+            v0 = self.get_value(self.state_last)
+            v1 = self.get_value(self.state)
+            d = v0 + self.alpha*(v1-v0)
+            if d > 0:
+                self.set_value(self.state_last, d)
             if self.with_debug:
-                print(f"{self.mark}: Values    = {values}")
-                print(f"{self.mark}: Moves     = {moves+1}")
-                print(f"{self.mark}: Best move = {best_move+1}")
-                print(f"{self.mark}: Next move = {next_move+1}, is_exp={is_exp}")
+                print(f"{self.mark}: Learned {d:0.3f}")
 
-            if self.state is not None:
-                self.state_last = self.state.copy()
-
-            self.state = self.state_from_move(state.copy(), next_move)
-
-            # Learn
-            if not is_exp and self.state_last is not None:
-                v0 = self.get_value(self.state_last)
-                v1 = self.get_value(self.state)
-                d = v0 + self.alpha*(v1-v0)
-                if d > 0:
-                    self.set_value(self.state_last, d)
-                if self.with_debug:
-                    print(f"{self.mark}: Learned {d:0.3f}")
-
-        return self.state, can_move
+        return self.state, True
 
     def state_from_move(self, state: np.array, field):
         state.reshape(state.size)[field] = self.mark
diff --git a/tic_tac_toe.py b/tic_tac_toe.py
index 23064e7..36f1629 100644
--- a/tic_tac_toe.py
+++ b/tic_tac_toe.py
@@ -2,11 +2,13 @@ import numpy as np
 from helper import create_empty_state, to_state_string
 from machine_player import MachinePlayer
 from player_provider import PlayerProvider
+import json
 
 float_formatter = "{:.3f}".format
 np.set_printoptions(formatter={'float_kind': float_formatter})
 
 
+
 def play(player_provider: PlayerProvider, k_max=10000, with_print=False):
     for k in range(0, k_max):
         players = player_provider.choose()
@@ -48,8 +50,14 @@ def play(player_provider: PlayerProvider, k_max=10000, with_print=False):
                 move += 1
 
 
-px = MachinePlayer(mark='X', with_debug=False)
-po = MachinePlayer(mark='O', with_debug=False)
+with open("values_x.json", "r") as fp:
+    x_values = json.load(fp)
+
+with open("values_o.json", "r") as fp:
+    o_values = json.load(fp)
+
+px = MachinePlayer(mark='X', with_debug=False, values=x_values)
+po = MachinePlayer(mark='O', with_debug=False, values=o_values)
 
 do_training = 1
 if do_training:
@@ -59,4 +67,9 @@ if do_training:
 players = PlayerProvider(MachinePlayer(mark='X', with_debug=True, values=px.values), MachinePlayer(mark='O', with_debug=True, values=po.values))
 play(players, 1000, True)
 
-px.print_state_table()
+# Convert and write JSON object to file
+with open("values_x.json", "w") as fp:
+    json.dump(px.values, fp, indent=0)
+
+with open("values_o.json", "w") as fp:
+    json.dump(po.values, fp, indent=0)