diff --git a/tic_tac_toe.py b/tic_tac_toe.py index d24d0c4..6b053da 100644 --- a/tic_tac_toe.py +++ b/tic_tac_toe.py @@ -2,41 +2,41 @@ import numpy as np from numpy.random import uniform float_formatter = "{:.3f}".format -np.set_printoptions(formatter={'float_kind':float_formatter}) +np.set_printoptions(formatter={'float_kind': float_formatter}) -def softmax(values: np.array) -> np.array: - result = values / sum(values) - result = np.sort(result) +def softmax(values: np.array, eps=1.0E-6) -> np.array: + result = (values + eps) / (sum(values) + eps) return result def sample(values: np.array, with_debug=False): # Normalize and sort - probs = softmax(values + 1.0E-6) + probs = softmax(values) + sorted_indices = np.argsort(probs) + sorted_probs = probs[sorted_indices] z = uniform() if with_debug: print(f"Probs={probs}") print(f"Z={z:.3f}") index = None p_sum = 0 - for idx, p in enumerate(probs): + for idx, p in enumerate(sorted_probs): p_sum += p if z <= p_sum: - index = idx + index = sorted_indices[idx] break - probs_max = np.max(probs) - is_exploration = probs[index] < probs_max + is_exploration = probs[index] < np.max(probs) return index, is_exploration -def test_sample(): - p = np.array([0.1, 0.1, 0.3, 0.5]) - c = np.array([0, 0, 0, 0]) +def test_sample(data): + p = np.array(data) + c = np.array([0]*len(data)) for i in range(0, 1000): - index = sample(p) + index, _ = sample(p, with_debug=False) c[index] += 1 print(c) @@ -127,7 +127,6 @@ class MachinePlayer(APlayer): else: self.values = values - def get_value(self, state): try: result = self.values[state] @@ -152,19 +151,22 @@ class MachinePlayer(APlayer): can_move = moves.size > 0 if can_move: - for field in moves: - # crate hypothetical next state - state_next = self.state_from_move(state, field) + for move in moves: + # create hypothetical next state + state_next = self.state_from_move(state, move) # evaluate value value = self.get_value(state_next) values = np.append(values, value) - index, is_exp = sample(values, self.with_debug) - field = moves[index] + index, is_exp = sample(values) + move = moves[index] if self.with_debug: - print(f"{self.mark}: Chose {index}, is_exp={is_exp}") + print(f"{self.mark}: Values = {values}") + print(f"{self.mark}: Moves = {moves+1}") + print(f"{self.mark}: Move = {move+1}, is_exp={is_exp}") + self.state_last = self.state - self.state = self.state_from_move(state, field) + self.state = self.state_from_move(state, move) # Learn if not is_exp and self.state_last is not None: @@ -172,7 +174,7 @@ class MachinePlayer(APlayer): v1 = self.get_value(self.state) d = max(0, v1-v0) if d > 0: - self.set_value(0.1*d) + self.set_value(0.01*d) if self.with_debug: print(f"{self.mark}: Learned {d:0.3f}") @@ -234,13 +236,22 @@ def play(player_provider: PlayerProvider, k_max=10000, with_print=False): move += 1 -px = MachinePlayer(mark='X') -po = MachinePlayer(mark='O') +px = MachinePlayer(mark='X', with_debug=True) +po = MachinePlayer(mark='O', with_debug=True) -do_training = 0 +do_training = 1 if do_training: - players = PlayerProvider(px, po) - play(players, 10000, False) + players = PlayerProvider(px, po, ) + play(players, 2000, True) -players = PlayerProvider(MachinePlayer(mark='X', with_debug=True, values=px.values), MachinePlayer(mark='O', with_debug=True, values=po.values)) -play(players, 100, True) +#players = PlayerProvider(MachinePlayer(mark='X', with_debug=True, values=px.values), MachinePlayer(mark='O', with_debug=True, values=po.values)) +#play(players, 100, True) + +print("Testing") +test_sample([0.7, 0.1, 0.1, 0.1]) +test_sample([0.1, 0.1, 0.2, 0.1]) +test_sample([0.2, 0.8]) +test_sample([0.1, 0.1]) +test_sample([0.3, 0.0, 0.2, 0.0]) +test_sample([0.0, 0.0, 0.0, 0.0]) +test_sample([0.5, 0.0, 0.2, 0.3])