diff --git a/a_player.py b/a_player.py index 1f81372..8be8a1d 100644 --- a/a_player.py +++ b/a_player.py @@ -1,6 +1,12 @@ +import enum + import numpy as np from numpy.random import uniform +class Reason(enum.Enum): + Won = 1 + Lost = 2 + Undecided = 3 class APlayer(object): def __init__(self, mark): @@ -15,6 +21,9 @@ class APlayer(object): def move(self, state: np.array): return state, False + def reward(self, value): + pass + def new_game(self): pass diff --git a/tic_tac_toe.py b/tic_tac_toe.py index 1e767c1..de1dc20 100644 --- a/tic_tac_toe.py +++ b/tic_tac_toe.py @@ -1,6 +1,6 @@ import numpy as np from numpy.random import uniform -from a_player import APlayer +from a_player import APlayer, Reason from helper import sample, create_empty_state, to_state_string float_formatter = "{:.3f}".format @@ -21,6 +21,12 @@ class MachinePlayer(APlayer): def init_values(self): self.values = {} + def print_state_table(self): + count = 0 + for key in self.values: + print(f"{self.mark}: {count:05d}: {key} = {self.values[key]:0.3f}") + count += 1 + @staticmethod def to_key(state: np.array): key = '' @@ -29,14 +35,11 @@ class MachinePlayer(APlayer): return key def get_value(self, state: np.array): - if state is None: - return 0 - key = self.to_key(state) try: result = self.values[key] except KeyError: - result = 0 + result = 0.5 return result @@ -44,7 +47,7 @@ class MachinePlayer(APlayer): self.values[self.to_key(state)] = value def reward(self, value): - self.set_value(self.state_last, value) + self.set_value(self.state, value) @staticmethod def get_potential_moves(state: np.array) -> np.array: @@ -57,35 +60,44 @@ class MachinePlayer(APlayer): self.state_last = None def move(self, state: np.array): + do_sample = False values = np.array([]) # get possible move moves = self.get_potential_moves(state) can_move = moves.size > 0 + best_move = None + best_value = -1 if can_move: for move in moves: # create hypothetical next state state_next = self.state_from_move(state.copy(), move) # evaluate value value = self.get_value(state_next) + if best_value < value: + best_value = value + best_move = move values = np.append(values, value) + next_move = best_move is_exp = False if uniform() < self.p_exp: is_exp = True index = np.random.randint(len(moves)) - else: + next_move = moves[index] + elif do_sample: index = sample(values) + next_move = moves[index] - move = moves[index] if self.with_debug: - print(f"{self.mark}: Values = {values}") - print(f"{self.mark}: Moves = {moves+1}") - print(f"{self.mark}: Move = {move+1}, is_exp={is_exp}") + print(f"{self.mark}: Values = {values}") + print(f"{self.mark}: Moves = {moves+1}") + print(f"{self.mark}: Best move = {best_move+1}") + print(f"{self.mark}: Next move = {next_move+1}, is_exp={is_exp}") if self.state is not None: self.state_last = self.state.copy() - self.state = self.state_from_move(state.copy(), move) + self.state = self.state_from_move(state.copy(), next_move) # Learn if not is_exp and self.state_last is not None: @@ -106,31 +118,28 @@ class MachinePlayer(APlayer): class PlayerProvider: def __init__(self, p1: APlayer, p2: APlayer): - self.p1 = p1 - self.p2 = p2 + self.players = [p1, p2] + def choose(self) -> list[APlayer]: + # Wer fängt an? + players = self.players + if uniform() < 0.5: + players.reverse() -def choose_player(player_provider: PlayerProvider) -> list[APlayer]: - # Wer fängt an? - p1 = player_provider.p1 - p2 = player_provider.p2 - if uniform() < 0.5: - result = [p1, p2] - else: - result = [p2, p1] + result = players + players[0].new_game() + players[1].new_game() - p1.new_game() - p2.new_game() - - return result + return result def play(player_provider: PlayerProvider, k_max=10000, with_print=False): for k in range(0, k_max): - players = choose_player(player_provider) + players = player_provider.choose() state = create_empty_state() move = 1 run = True + other_player = players[-1] while run: for player in players: if with_print: @@ -141,32 +150,39 @@ def play(player_provider: PlayerProvider, k_max=10000, with_print=False): state, has_moved = player.move(state) if with_print: print(to_state_string(last_state, state)) - if not has_moved: if with_print: print(f"{player.mark}: No more moves") + if isinstance(player, MachinePlayer): + player.reward(0.0) + if isinstance(other_player, MachinePlayer): + other_player.reward(0.0) run = False if player.has_won(state): if with_print: print(f"{player.mark}: Has won the game") if isinstance(player, MachinePlayer): player.reward(1.0) + if isinstance(other_player, MachinePlayer): + other_player.reward(0.0) run = False + other_player = player if not run: break move += 1 + px = MachinePlayer(mark='X', with_debug=False) po = MachinePlayer(mark='O', with_debug=False) - -do_training = 0 +do_training = 1 if do_training: players = PlayerProvider(px, po) - play(players, 4000, False) + play(players, 40000, False) players = PlayerProvider(MachinePlayer(mark='X', with_debug=True, values=px.values), MachinePlayer(mark='O', with_debug=True, values=po.values)) -play(players, 4000, True) +play(players, 1000, True) +px.print_state_table()