Fixed sample

tic_tac_toe.py

@@ -2,41 +2,41 @@ import numpy as np
 from numpy.random import uniform
 
 float_formatter = "{:.3f}".format
-np.set_printoptions(formatter={'float_kind':float_formatter})
+np.set_printoptions(formatter={'float_kind': float_formatter})
 
 
-def softmax(values: np.array) -> np.array:
-    result = values / sum(values)
-    result = np.sort(result)
+def softmax(values: np.array, eps=1.0E-6) -> np.array:
+    result = (values + eps) / (sum(values) + eps)
     return result
 
 
 def sample(values: np.array, with_debug=False):
     # Normalize and sort
-    probs = softmax(values + 1.0E-6)
+    probs = softmax(values)
+    sorted_indices = np.argsort(probs)
+    sorted_probs = probs[sorted_indices]
     z = uniform()
     if with_debug:
         print(f"Probs={probs}")
         print(f"Z={z:.3f}")
     index = None
     p_sum = 0
-    for idx, p in enumerate(probs):
+    for idx, p in enumerate(sorted_probs):
         p_sum += p
         if z <= p_sum:
-            index = idx
+            index = sorted_indices[idx]
             break
 
-    probs_max = np.max(probs)
-    is_exploration = probs[index] < probs_max
+    is_exploration = probs[index] < np.max(probs)
     return index, is_exploration
 
 
-def test_sample():
-    p = np.array([0.1, 0.1, 0.3, 0.5])
-    c = np.array([0, 0, 0, 0])
+def test_sample(data):
+    p = np.array(data)
+    c = np.array([0]*len(data))
 
     for i in range(0, 1000):
-        index = sample(p)
+        index, _ = sample(p, with_debug=False)
         c[index] += 1
 
     print(c)
@@ -127,7 +127,6 @@ class MachinePlayer(APlayer):
         else:
             self.values = values
 
-
     def get_value(self, state):
         try:
             result = self.values[state]
@@ -152,19 +151,22 @@ class MachinePlayer(APlayer):
         can_move = moves.size > 0
 
         if can_move:
-            for field in moves:
-                # crate hypothetical next state
-                state_next = self.state_from_move(state, field)
+            for move in moves:
+                # create hypothetical next state
+                state_next = self.state_from_move(state, move)
                 # evaluate value
                 value = self.get_value(state_next)
                 values = np.append(values, value)
 
-            index, is_exp = sample(values, self.with_debug)
-            field = moves[index]
+            index, is_exp = sample(values)
+            move = moves[index]
             if self.with_debug:
-                print(f"{self.mark}: Chose {index}, is_exp={is_exp}")
+                print(f"{self.mark}: Values = {values}")
+                print(f"{self.mark}: Moves = {moves+1}")
+                print(f"{self.mark}: Move = {move+1}, is_exp={is_exp}")
+
             self.state_last = self.state
-            self.state = self.state_from_move(state, field)
+            self.state = self.state_from_move(state, move)
 
             # Learn
             if not is_exp and self.state_last is not None:
@@ -172,7 +174,7 @@ class MachinePlayer(APlayer):
                 v1 = self.get_value(self.state)
                 d = max(0, v1-v0)
                 if d > 0:
-                    self.set_value(0.1*d)
+                    self.set_value(0.01*d)
                     if self.with_debug:
                         print(f"{self.mark}: Learned {d:0.3f}")
 
@@ -234,13 +236,22 @@ def play(player_provider: PlayerProvider, k_max=10000, with_print=False):
                 move += 1
 
 
-px = MachinePlayer(mark='X')
-po = MachinePlayer(mark='O')
+px = MachinePlayer(mark='X', with_debug=True)
+po = MachinePlayer(mark='O', with_debug=True)
 
-do_training = 0
+do_training = 1
 if do_training:
-    players = PlayerProvider(px, po)
-    play(players, 10000, False)
+    players = PlayerProvider(px, po, )
+    play(players, 2000, True)
 
-players = PlayerProvider(MachinePlayer(mark='X', with_debug=True, values=px.values), MachinePlayer(mark='O', with_debug=True, values=po.values))
-play(players, 100, True)
+#players = PlayerProvider(MachinePlayer(mark='X', with_debug=True, values=px.values), MachinePlayer(mark='O', with_debug=True, values=po.values))
+#play(players, 100, True)
+
+print("Testing")
+test_sample([0.7, 0.1, 0.1, 0.1])
+test_sample([0.1, 0.1, 0.2, 0.1])
+test_sample([0.2, 0.8])
+test_sample([0.1, 0.1])
+test_sample([0.3, 0.0, 0.2, 0.0])
+test_sample([0.0, 0.0, 0.0, 0.0])
+test_sample([0.5, 0.0, 0.2, 0.3])