Added probability paramter for exploration

tic_tac_toe.py

@@ -27,8 +27,7 @@ def sample(values: np.array, with_debug=False):
             index = sorted_indices[idx]
             break
 
-    is_exploration = probs[index] < np.max(probs)
-    return index, is_exploration
+    return index
 
 
 def test_sample(data):
@@ -36,7 +35,7 @@ def test_sample(data):
     c = np.array([0]*len(data))
 
     for i in range(0, 1000):
-        index, _ = sample(p, with_debug=False)
+        index = sample(p, with_debug=False)
         c[index] += 1
 
     print(c)
@@ -75,6 +74,7 @@ class APlayer(object):
     def __init__(self, mark):
         self.mark = mark
         self.state = None
+        self.state_opp = None
         self.state_last = None
 
     def move(self, state):
@@ -121,6 +121,8 @@ class APlayer(object):
 class MachinePlayer(APlayer):
     def __init__(self, mark='X', with_debug=False, values=None):
         APlayer.__init__(self, mark)
+        self.p_exp = 0.2
+        self.alpha = 0.2
         self.with_debug = with_debug
         if values is None:
             self.values = {}
@@ -135,9 +137,11 @@ class MachinePlayer(APlayer):
 
         return result
 
-    def set_value(self, value):
-        if self.state_last is not None:
-            self.values[self.state_last] = value
+    def set_value(self, state, value):
+        self.values[state] = value
+
+    def reward(self, value):
+        self.set_value(self.state_last, value)
 
     @staticmethod
     def get_potential_moves(state) -> np.array:
@@ -149,7 +153,7 @@ class MachinePlayer(APlayer):
         # get possible move
         moves = self.get_potential_moves(state)
         can_move = moves.size > 0
-
+        self.state_opp = state
         if can_move:
             for move in moves:
                 # create hypothetical next state
@@ -158,7 +162,13 @@ class MachinePlayer(APlayer):
                 value = self.get_value(state_next)
                 values = np.append(values, value)
 
-            index, is_exp = sample(values)
+            is_exp = False
+            if uniform() < self.p_exp:
+                is_exp = True
+                index = np.random.randint(len(moves))
+            else:
+                index = sample(values)
+
             move = moves[index]
             if self.with_debug:
                 print(f"{self.mark}: Values = {values}")
@@ -174,7 +184,7 @@ class MachinePlayer(APlayer):
                 v1 = self.get_value(self.state)
                 d = max(0, v1-v0)
                 if d > 0:
-                    self.set_value(0.2*d)
+                    self.set_value(self.state_last, self.alpha*d)
                     if self.with_debug:
                         print(f"{self.mark}: Learned {d:0.3f}")
 
@@ -227,7 +237,7 @@ def play(player_provider: PlayerProvider, k_max=10000, with_print=False):
                     if with_print:
                         print(f"{player.mark}: Has won the game")
                     if isinstance(player, MachinePlayer):
-                        player.set_value(1.0)
+                        player.reward(1.0)
                     run = False
 
                 if not run:
@@ -241,8 +251,8 @@ po = MachinePlayer(mark='O', with_debug=True)
 
 do_training = 1
 if do_training:
-    players = PlayerProvider(px, po, )
-    play(players, 2000, True)
+    players = PlayerProvider(px, po)
+    play(players, 10000, True)
 
 #players = PlayerProvider(MachinePlayer(mark='X', with_debug=True, values=px.values), MachinePlayer(mark='O', with_debug=True, values=po.values))
 #play(players, 100, True)
@@ -255,3 +265,4 @@ test_sample([0.1, 0.1])
 test_sample([0.3, 0.0, 0.2, 0.0])
 test_sample([0.0, 0.0, 0.0, 0.0])
 test_sample([0.5, 0.0, 0.2, 0.3])
+test_sample([0.3, 0.1, 0.4, 0.2])