From cf7379bf5082c279cbe3f063dfdf863c01086c86 Mon Sep 17 00:00:00 2001 From: Jens Ahrensfeld Date: Tue, 11 Jun 2024 19:55:51 +0200 Subject: [PATCH] add learning after episode --- a_player.py | 4 ++-- machine_player.py | 28 +++++++++++++++------------- player_provider.py | 8 +++++--- tic_tac_toe.py | 35 ++++++++++++++++++++++------------- 4 files changed, 44 insertions(+), 31 deletions(-) diff --git a/a_player.py b/a_player.py index 4ddeab6..d53e552 100644 --- a/a_player.py +++ b/a_player.py @@ -21,10 +21,10 @@ class APlayer(object): state.reshape(state.size)[move] = self.mark return state - def new_game(self): + def new_game(self, state: np.array): pass - def end_game(self, reward): + def end_game(self, state: np.array, value): pass def has_won(self, state: np.array): diff --git a/machine_player.py b/machine_player.py index b386328..e5e6771 100644 --- a/machine_player.py +++ b/machine_player.py @@ -6,9 +6,10 @@ from state import get_potential_moves class MachinePlayer(APlayer): class Params: - def __init__(self, p_explore, alpha): + def __init__(self, p_explore, alpha, do_learn_from_history=True): self.p_exp = p_explore self.alpha = alpha + self.do_learn_from_history = do_learn_from_history def __init__(self, mark, params: Params, values): APlayer.__init__(self, mark) @@ -46,16 +47,21 @@ class MachinePlayer(APlayer): def set_value(self, state: np.array, value): self.values[self.to_key(state)] = value - def end_game(self, value): + def end_game(self, state: np.array, value): self.set_value(self.state, value) - print(self.episode_history) + if self.params.do_learn_from_history: + self.learn_from_history() - def new_game(self): - self.state = None + def new_game(self, state: np.array): + self.state = state self.episode_history = [] def learn_from_history(self): - pass + rev_hist = list(reversed(self.episode_history)) + for hist in rev_hist: + d = self.calc_value(hist['state'], hist['next_state']) + if hist['is_exp']: + self.set_value(hist['state'], d) def calc_value(self, state: np.array, next_state: np.array): v0 = self.get_value(state) @@ -99,10 +105,9 @@ class MachinePlayer(APlayer): # Finally create next state next_state = self.to_state(state.copy(), next_move) - next_value = self.get_value(next_state) # Maintain history - self.episode_history.append((self.to_key(next_state), next_value, next_move, is_exp)) + self.episode_history.append({'state': self.state, 'next_state': next_state, "is_exp": is_exp}) if self.with_debug: print(f"{self.mark}: Values = {values}") @@ -111,12 +116,9 @@ class MachinePlayer(APlayer): print(f"{self.mark}: Next move = {next_move+1}, is_exp={is_exp}") # Learn - if not is_exp and self.state is not None: + if not (self.params.do_learn_from_history or is_exp): d = self.calc_value(self.state, next_state) - if d > 0: - self.set_value(self.state, d) - if self.with_debug: - print(f"{self.mark}: Learned {d:0.3f}") + self.set_value(self.state, d) self.state = next_state diff --git a/player_provider.py b/player_provider.py index 8dfd06f..791796f 100644 --- a/player_provider.py +++ b/player_provider.py @@ -1,3 +1,5 @@ +import numpy as np + from a_player import APlayer from numpy.random import uniform @@ -10,15 +12,15 @@ class PlayerProvider: for p in self.players: p.set_debug(with_debug) - def choose(self) -> list[APlayer]: + def choose(self, state: np.array) -> list[APlayer]: # Wer fängt an? players = self.players if uniform() < 0.5: players.reverse() result = players - players[0].new_game() - players[1].new_game() + players[0].new_game(state) + players[1].new_game(state) return result diff --git a/tic_tac_toe.py b/tic_tac_toe.py index 5bc4ec6..3260288 100644 --- a/tic_tac_toe.py +++ b/tic_tac_toe.py @@ -25,8 +25,8 @@ def play(player_provider: PlayerProvider, k_max=10000, with_debug=False): player_provider.set_debug(with_debug) for k in range(0, k_max): report(k) - players = player_provider.choose() state = create_empty_state() + players = player_provider.choose(state) move = 1 run = True other_player = players[-1] @@ -43,14 +43,14 @@ def play(player_provider: PlayerProvider, k_max=10000, with_debug=False): if not has_moved: if with_debug: print(f"{player.mark}: No more moves") - player.end_game(0.0) - other_player.end_game(0.0) + player.end_game(state, 0.0) + other_player.end_game(state, 0.0) run = False if player.has_won(state): if with_debug: print(f"{player.mark}: Has won the game") - player.end_game(1.0) - other_player.end_game(0.0) + player.end_game(state, 1.0) + other_player.end_game(state, 0.0) run = False other_player = player @@ -60,21 +60,30 @@ def play(player_provider: PlayerProvider, k_max=10000, with_debug=False): move += 1 -with open("values_x.json", "r") as fp: - x_values = json.load(fp) - -with open("values_o.json", "r") as fp: - o_values = json.load(fp) - do_training = 1 -do_human_player = 1 +do_human_player = 0 +do_load_values = 1 + +if do_load_values: + try: + with open("values_x.json", "r") as fp: + x_values = json.load(fp) + + with open("values_o.json", "r") as fp: + o_values = json.load(fp) + except Exception: + x_values = {} + o_values = {} +else: + x_values = {} + o_values = {} p_hx = HumanPlayer(mark='X', name="Jens") p_mx = MachinePlayer(mark='X', params=MachinePlayer.Params(p_explore=0.2, alpha=0.1), values=x_values) p_mo = MachinePlayer(mark='O', params=MachinePlayer.Params(p_explore=0.2, alpha=0.1), values=o_values) if do_training: - play(PlayerProvider(p_mx, p_mo), 10000, False) + play(PlayerProvider(p_mx, p_mo), 5000, True) # Values after training # Convert and write JSON object to file