From 5b7430f208a03193724e7259405743149eeace5b Mon Sep 17 00:00:00 2001 From: Jens Ahrensfeld Date: Sat, 8 Jun 2024 20:24:38 +0200 Subject: [PATCH] - control debug output - fixed last state - refactored --- tic_tac_toe.py | 190 +++++++++++++++++++++++++++---------------------- 1 file changed, 105 insertions(+), 85 deletions(-) diff --git a/tic_tac_toe.py b/tic_tac_toe.py index 61b95ac..98a19e3 100644 --- a/tic_tac_toe.py +++ b/tic_tac_toe.py @@ -11,12 +11,13 @@ def softmax(values: np.array) -> np.array: return result -def sample(values: np.array): +def sample(values: np.array, with_debug=False): # Normalize and sort probs = softmax(values + 1.0E-6) z = uniform() - print(f"Probs={probs}") - print(f"Z={z:.3f}") + if with_debug: + print(f"Probs={probs}") + print(f"Z={z:.3f}") index = None p_sum = 0 for idx, p in enumerate(probs): @@ -70,66 +71,14 @@ def to_state_string(state, state_nex=None): return state_str -class Player(object): - def __init__(self, mark='X'): - self.values = {} +class APlayer(object): + def __init__(self, mark): self.mark = mark + self.state = None self.state_last = None - def get_value(self, state): - try: - result = self.values[state] - except KeyError: - result = 0 - - return result - - def set_value(self, value, state=None): - if state is None: - if self.state_last is not None: - self.values[self.state_last] = value - else: - self.values[state] = value - - @staticmethod - def get_potential_moves(state) -> np.array: - indices = [idx for idx, s in enumerate(state) if '-' in s] - return np.array(indices) - def move(self, state): - values = np.array([]) - # get possible move - moves = self.get_potential_moves(state) - can_move = moves.size > 0 - state_next = state - - if can_move: - for field in moves: - # crate hypothetical next state - state_next = self.state_from_move(state, field) - # evaluate value - value = self.get_value(state_next) - values = np.append(values, value) - - index, is_exp = sample(values) - field = moves[index] - print(f"{player.mark}: Chose {index}, is_exp={is_exp}") - state_next = self.state_from_move(state, field) - - # Learn - if not is_exp and self.state_last is not None: - v0 = self.get_value(self.state_last) - v1 = self.get_value(state_next) - d = max(0, v1-v0) - if d > 0: - self.set_value(0.1*d) - print(f"{player.mark}: Learned {d:0.3f}") - - self.state_last = state_next - return state_next, can_move - - def state_from_move(self, state, field): - return state[:field] + self.mark + state[field + 1:] + return state, False def has_won(self, state): ref = self.mark + self.mark + self.mark @@ -169,38 +118,109 @@ class Player(object): return False -p_x = Player(mark='X') -p_o = Player(mark='O') +class MachinePlayer(APlayer): + def __init__(self, mark='X', with_debug=False): + APlayer.__init__(self, mark) + self.with_debug = with_debug + self.values = {} + + def get_value(self, state): + try: + result = self.values[state] + except KeyError: + result = 0 + + return result + + def set_value(self, value): + if self.state_last is not None: + self.values[self.state_last] = value + + @staticmethod + def get_potential_moves(state) -> np.array: + indices = [idx for idx, s in enumerate(state) if '-' in s] + return np.array(indices) + + def move(self, state): + values = np.array([]) + # get possible move + moves = self.get_potential_moves(state) + can_move = moves.size > 0 + + if can_move: + for field in moves: + # crate hypothetical next state + state_next = self.state_from_move(state, field) + # evaluate value + value = self.get_value(state_next) + values = np.append(values, value) + + index, is_exp = sample(values, self.with_debug) + field = moves[index] + if self.with_debug: + print(f"{self.mark}: Chose {index}, is_exp={is_exp}") + self.state_last = self.state + self.state = self.state_from_move(state, field) + + # Learn + if not is_exp and self.state_last is not None: + v0 = self.get_value(self.state_last) + v1 = self.get_value(self.state) + d = max(0, v1-v0) + if d > 0: + self.set_value(0.1*d) + if self.with_debug: + print(f"{self.mark}: Learned {d:0.3f}") + + return self.state, can_move + + def state_from_move(self, state, field): + return state[:field] + self.mark + state[field + 1:] -for k in range(0, 1000): +def play(players: list[APlayer], k_max=10000, with_print=False): + for k in range(0, k_max): + state = "---------" + move = 1 + run = True + while run: + for player in players: + if with_print: + print(f"---------------------------------------------------") + print(f"- Game {k:06d}, Move {move} -----------------------------") + print(f"---------------------------------------------------") + last_state = state + state, has_moved = player.move(state) + if with_print: + print(to_state_string(last_state, state)) + + if not has_moved: + if with_print: + print(f"{player.mark}: No more moves") + run = False + if player.has_won(state): + if with_print: + print(f"{player.mark}: Has won the game") + if isinstance(player, MachinePlayer): + player.set_value(1.0) + run = False + + if not run: + break + + move += 1 + + +def choose_player(p1: APlayer, p2: APlayer) -> list[APlayer]: # Wer fängt an? if uniform() < 0.5: - players = [p_x, p_o] + result = [p1, p2] else: - players = [p_o, p_x] - state = "---------" - move = 1 - run = True - while run: - for player in players: - print(f"---------------------------------------------------") - print(f"- Game {k:06d}, Move {move} -----------------------------") - print(f"---------------------------------------------------") - last_state = state - state, has_moved = player.move(state) - print(to_state_string(last_state, state)) + result = [p2, p1] - if not has_moved: - print(f"{player.mark}: No more moves") - run = False - if player.has_won(state): - print(f"{player.mark}: Has won the game") - player.set_value(1.0) - run = False + return result - if not run: - break - move += 1 +opponents = choose_player(MachinePlayer(mark='X', with_debug=True), MachinePlayer(mark='O', with_debug=True)) +play(opponents, 1000, True)