diff --git a/.idea/codeStyles/codeStyleConfig.xml b/.idea/codeStyles/codeStyleConfig.xml new file mode 100644 index 0000000..a55e7a1 --- /dev/null +++ b/.idea/codeStyles/codeStyleConfig.xml @@ -0,0 +1,5 @@ + + + + \ No newline at end of file diff --git a/a_player.py b/a_player.py index 9532fbb..dedbacf 100644 --- a/a_player.py +++ b/a_player.py @@ -2,72 +2,72 @@ import numpy as np class APlayer(object): - def __init__(self, mark): - self.mark = mark - self.other_mark = 'O' - if 'O' in mark: - self.other_mark = 'X' + def __init__(self, mark): + self.mark = mark + self.other_mark = 'O' + if 'O' in mark: + self.other_mark = 'X' - self.state = None - self.state_last = None + self.state = None + self.state_last = None - def set_debug(self, with_debug): - pass + def set_debug(self, with_debug): + pass - @staticmethod - def get_potential_moves(state: np.array) -> np.array: - st = state.reshape(state.size) - indices = [idx for idx, s in enumerate(st) if '-' in s] - return np.array(indices) + @staticmethod + def get_potential_moves(state: np.array) -> np.array: + st = state.reshape(state.size) + indices = [idx for idx, s in enumerate(st) if '-' in s] + return np.array(indices) - def move(self, state: np.array): - return state, False + def move(self, state: np.array): + return state, False - def state_from_move(self, state: np.array, field): - state.reshape(state.size)[field] = self.mark - return state + def state_from_move(self, state: np.array, field): + state.reshape(state.size)[field] = self.mark + return state - def reward(self, value): - pass + def reward(self, value): + pass - def new_game(self): - pass + def new_game(self): + pass - @staticmethod - def f_state_slices(): - nr = 3 - nc = 3 - result = [] - # Create row finishing states - d1 = () - d2_r = () - for r in range(0, nr): - d1 += (r,) - for c in range(0, nc): - d2 = (c,) * nc - result.append((d1, d2)) + @staticmethod + def f_state_slices(): + nr = 3 + nc = 3 + result = [] + # Create row finishing states + d1 = () + d2_r = () + for r in range(0, nr): + d1 += (r,) + for c in range(0, nc): + d2 = (c,) * nc + result.append((d1, d2)) - # Create column finishing states - for c in range(0, nc): - d2 = (c,) * nc - result.append((d2, d1)) - for c in range(0, nc): - d2_r += (nc - c - 1,) + # Create column finishing states + for c in range(0, nc): + d2 = (c,) * nc + result.append((d2, d1)) + for c in range(0, nc): + d2_r += (nc - c - 1,) - # Create diagonal finishing states #1 - result.append((d1, d1)) + # Create diagonal finishing states #1 + result.append((d1, d1)) - # Create diagonal finishing states #2 - result.append((d1, d2_r)) + # Create diagonal finishing states #2 + result.append((d1, d2_r)) - return result + return result - def has_won(self, state: np.array): - result = False - ref = [self.mark, self.mark, self.mark] - for f_state_slice in APlayer.f_state_slices(): - if np.all(state[f_state_slice] == ref): - result = True - break + def has_won(self, state: np.array): + result = False + ref = [self.mark, self.mark, self.mark] + for f_state_slice in APlayer.f_state_slices(): + if np.all(state[f_state_slice] == ref): + result = True + break - return result + return result diff --git a/helper.py b/helper.py index 62cbac3..e59f98e 100644 --- a/helper.py +++ b/helper.py @@ -3,86 +3,86 @@ from numpy.random import uniform def create_test_state() -> np.array: - return np.array([['1', '2', '3'], ['4', '5', '6'], ['7', '8', '9']]) + return np.array([['1', '2', '3'], ['4', '5', '6'], ['7', '8', '9']]) def create_empty_state() -> np.array: - return np.array([['-', '-', '-'], ['-', '-', '-'], ['-', '-', '-']]) + return np.array([['-', '-', '-'], ['-', '-', '-'], ['-', '-', '-']]) def softmax(values: np.array, eps=1.0E-6) -> np.array: - result = (values + eps) / (sum(values + eps)) - return result + result = (values + eps) / (sum(values + eps)) + return result def sample(values: np.array, with_debug=False): - # Normalize and sort - probs = softmax(values) - sorted_indices = np.argsort(probs) - sorted_probs = probs[sorted_indices] - z = uniform() - if with_debug: - print(f"Probs={probs}") - print(f"Z={z:.3f}") - index = None - p_sum = 0 - for idx, p in enumerate(sorted_probs): - p_sum += p - if z <= p_sum: - index = sorted_indices[idx] - break + # Normalize and sort + probs = softmax(values) + sorted_indices = np.argsort(probs) + sorted_probs = probs[sorted_indices] + z = uniform() + if with_debug: + print(f"Probs={probs}") + print(f"Z={z:.3f}") + index = None + p_sum = 0 + for idx, p in enumerate(sorted_probs): + p_sum += p + if z <= p_sum: + index = sorted_indices[idx] + break - return index + return index def test_sample(data): - p = np.array(data) - c = np.array([0]*len(data)) + p = np.array(data) + c = np.array([0]*len(data)) - for i in range(0, 1000): - index = sample(p, with_debug=False) - c[index] += 1 + for i in range(0, 1000): + index = sample(p, with_debug=False) + c[index] += 1 - print(c) + print(c) def to_state_string(state, state_nex=None): - sp = ' ' - sp_arrow = ' => ' - sp_ = [sp, sp_arrow, sp] + sp = ' ' + sp_arrow = ' => ' + sp_ = [sp, sp_arrow, sp] - def col(str_in, state): - str_out = str_in - for c in range(0, 3): - char = state[r][c] - if char == '-': - char = ' ' - str_out += "|" + char - str_out += '|' - return str_out + def col(str_in, state): + str_out = str_in + for c in range(0, 3): + char = state[r][c] + if char == '-': + char = ' ' + str_out += "|" + char + str_out += '|' + return str_out - state_str = '' - for r in range(0, 3): - if state is not None: - state_str = col(state_str, state) + state_str = '' + for r in range(0, 3): + if state is not None: + state_str = col(state_str, state) - if state_nex is not None: - state_str += sp_[r] - state_str = col(state_str, state_nex) + if state_nex is not None: + state_str += sp_[r] + state_str = col(state_str, state_nex) - if r != 2: - state_str += '\x0A' + if r != 2: + state_str += '\x0A' - return state_str + return state_str if __name__ == '__main__': - print("Testing sample()") - test_sample([0.7, 0.1, 0.1, 0.1]) - test_sample([0.1, 0.1, 0.2, 0.1]) - test_sample([0.2, 0.8]) - test_sample([0.1, 0.1]) - test_sample([0.3, 0.0, 0.2, 0.0]) - test_sample([0.0, 0.0, 0.0, 0.0]) - test_sample([0.5, 0.0, 0.2, 0.3]) - test_sample([0.3, 0.1, 0.4, 0.2]) + print("Testing sample()") + test_sample([0.7, 0.1, 0.1, 0.1]) + test_sample([0.1, 0.1, 0.2, 0.1]) + test_sample([0.2, 0.8]) + test_sample([0.1, 0.1]) + test_sample([0.3, 0.0, 0.2, 0.0]) + test_sample([0.0, 0.0, 0.0, 0.0]) + test_sample([0.5, 0.0, 0.2, 0.3]) + test_sample([0.3, 0.1, 0.4, 0.2]) diff --git a/human_player.py b/human_player.py index 3a9e542..fb2b853 100644 --- a/human_player.py +++ b/human_player.py @@ -5,55 +5,55 @@ from helper import to_state_string, create_empty_state, create_test_state class HumanPlayer(APlayer): - def __init__(self, mark, name: str): - APlayer.__init__(self, mark) - self.name = name + def __init__(self, mark, name: str): + APlayer.__init__(self, mark) + self.name = name - def move(self, state: np.array): - move = 0 - has_moved = False - while True: - print(f"{self.name}, Du bist dran. Das Spielfeld sieht so aus:") - print(to_state_string(state, create_test_state())) + def move(self, state: np.array): + move = 0 + has_moved = False + while True: + print(f"{self.name}, Du bist dran. Das Spielfeld sieht so aus:") + print(to_state_string(state, create_test_state())) - print(f"{self.name}, in welches Feld willst Du Dein \"{self.mark}\" setzen?") - choice = input("Feldnummer: ") + print(f"{self.name}, in welches Feld willst Du Dein \"{self.mark}\" setzen?") + choice = input("Feldnummer: ") - if choice in "Qq": - print("Abbruch") - break + if choice in "Qq": + print("Abbruch") + break - if not choice in "123456789" or len(choice) != 1: - print("Ungültiges Feld!") - print("Versuche es nochmal") - continue + if not choice in "123456789" or len(choice) != 1: + print("Ungültiges Feld!") + print("Versuche es nochmal") + continue - move = int(choice) - 1 - if not move in self.get_potential_moves(state): - print("Feld is bereits belegt!") - print("Versuche es nochmal") - continue + move = int(choice) - 1 + if not move in self.get_potential_moves(state): + print("Feld is bereits belegt!") + print("Versuche es nochmal") + continue - print("So soll es geschehen") - has_moved = True - break + print("So soll es geschehen") + has_moved = True + break - state_next = self.state_from_move(state.copy(), move) - return state_next, has_moved + state_next = self.state_from_move(state.copy(), move) + return state_next, has_moved - def reward(self, value): - if value > 0: - print(f"{self.name}, Du hast gewonnen, super!") - else: - print(f"{self.name}, Du hast leider verloren!") + def reward(self, value): + if value > 0: + print(f"{self.name}, Du hast gewonnen, super!") + else: + print(f"{self.name}, Du hast leider verloren!") if __name__ == '__main__': - p = HumanPlayer("X", "Jens") - state = create_empty_state() - while True: - state, has_moved = p.move(state) - if not has_moved: - break + p = HumanPlayer("X", "Jens") + state = create_empty_state() + while True: + state, has_moved = p.move(state) + if not has_moved: + break - p.reward(1.0) + p.reward(1.0) diff --git a/machine_player.py b/machine_player.py index 340dee3..583a158 100644 --- a/machine_player.py +++ b/machine_player.py @@ -5,103 +5,103 @@ from numpy.random import uniform class MachinePlayer(APlayer): - class Params: - def __init__(self, p_exp, alpha): - self.p_exp = p_exp - self.alpha = alpha + class Params: + def __init__(self, p_exp, alpha): + self.p_exp = p_exp + self.alpha = alpha - def __init__(self, mark, params: Params, values): - APlayer.__init__(self, mark) - self.params = params - self.values = {} - self.with_debug = False - self.values = values + def __init__(self, mark, params: Params, values): + APlayer.__init__(self, mark) + self.params = params + self.values = {} + self.with_debug = False + self.values = values - def set_debug(self, with_debug): - self.with_debug = with_debug + def set_debug(self, with_debug): + self.with_debug = with_debug - def print_state_table(self): - count = 0 - for key in self.values: - print(f"{self.mark}: {count:05d}: {key} = {self.values[key]:0.3f}") - count += 1 + def print_state_table(self): + count = 0 + for key in self.values: + print(f"{self.mark}: {count:05d}: {key} = {self.values[key]:0.3f}") + count += 1 - @staticmethod - def to_key(state: np.array): - key = '' - for st in state.reshape(state.size): - key += st - return key + @staticmethod + def to_key(state: np.array): + key = '' + for st in state.reshape(state.size): + key += st + return key - def get_value(self, state: np.array): - key = self.to_key(state) - try: - result = self.values[key] - except KeyError: - result = 0.5 + def get_value(self, state: np.array): + key = self.to_key(state) + try: + result = self.values[key] + except KeyError: + result = 0.5 - return result + return result - def set_value(self, state: np.array, value): - self.values[self.to_key(state)] = value + def set_value(self, state: np.array, value): + self.values[self.to_key(state)] = value - def reward(self, value): - self.set_value(self.state, value) + def reward(self, value): + self.set_value(self.state, value) - def new_game(self): - self.state = None - self.state_last = None + def new_game(self): + self.state = None + self.state_last = None - def move(self, state: np.array): - do_sample = False - values = np.array([]) - # get possible move - moves = self.get_potential_moves(state) - if moves.size == 0: - return state, False + def move(self, state: np.array): + do_sample = False + values = np.array([]) + # get possible move + moves = self.get_potential_moves(state) + if moves.size == 0: + return state, False - best_move = None - best_value = -1 - for move in moves: - # create hypothetical next state - state_next = self.state_from_move(state.copy(), move) - # evaluate value - value = self.get_value(state_next) - if best_value < value: - best_value = value - best_move = move - values = np.append(values, value) + best_move = None + best_value = -1 + for move in moves: + # create hypothetical next state + state_next = self.state_from_move(state.copy(), move) + # evaluate value + value = self.get_value(state_next) + if best_value < value: + best_value = value + best_move = move + values = np.append(values, value) - next_move = best_move - is_exp = False - if uniform() < self.params.p_exp: - index = np.random.randint(len(moves)) - next_move = moves[index] - is_exp = best_move != next_move - elif do_sample: - index = sample(values) - next_move = moves[index] + next_move = best_move + is_exp = False + if uniform() < self.params.p_exp: + index = np.random.randint(len(moves)) + next_move = moves[index] + is_exp = best_move != next_move + elif do_sample: + index = sample(values) + next_move = moves[index] - if self.with_debug: - print(f"{self.mark}: Values = {values}") - print(f"{self.mark}: Moves = {moves+1}") - print(f"{self.mark}: Best move = {best_move+1}") - print(f"{self.mark}: Next move = {next_move+1}, is_exp={is_exp}") + if self.with_debug: + print(f"{self.mark}: Values = {values}") + print(f"{self.mark}: Moves = {moves+1}") + print(f"{self.mark}: Best move = {best_move+1}") + print(f"{self.mark}: Next move = {next_move+1}, is_exp={is_exp}") - if self.state is not None: - self.state_last = self.state.copy() + if self.state is not None: + self.state_last = self.state.copy() - self.state = self.state_from_move(state.copy(), next_move) + self.state = self.state_from_move(state.copy(), next_move) - # Learn - if not is_exp and self.state_last is not None: - v0 = self.get_value(self.state_last) - v1 = self.get_value(self.state) - d = v0 + self.params.alpha*(v1-v0) - if d > 0: - self.set_value(self.state_last, d) - if self.with_debug: - print(f"{self.mark}: Learned {d:0.3f}") + # Learn + if not is_exp and self.state_last is not None: + v0 = self.get_value(self.state_last) + v1 = self.get_value(self.state) + d = v0 + self.params.alpha*(v1-v0) + if d > 0: + self.set_value(self.state_last, d) + if self.with_debug: + print(f"{self.mark}: Learned {d:0.3f}") - return self.state, True + return self.state, True diff --git a/player_provider.py b/player_provider.py index 4be15fa..8dfd06f 100644 --- a/player_provider.py +++ b/player_provider.py @@ -3,22 +3,22 @@ from numpy.random import uniform class PlayerProvider: - def __init__(self, p1: APlayer, p2: APlayer): - self.players = [p1, p2] + def __init__(self, p1: APlayer, p2: APlayer): + self.players = [p1, p2] - def set_debug(self, with_debug): - for p in self.players: - p.set_debug(with_debug) + def set_debug(self, with_debug): + for p in self.players: + p.set_debug(with_debug) - def choose(self) -> list[APlayer]: - # Wer fängt an? - players = self.players - if uniform() < 0.5: - players.reverse() + def choose(self) -> list[APlayer]: + # Wer fängt an? + players = self.players + if uniform() < 0.5: + players.reverse() - result = players - players[0].new_game() - players[1].new_game() + result = players + players[0].new_game() + players[1].new_game() - return result + return result diff --git a/tic_tac_toe.py b/tic_tac_toe.py index 6630814..7dfcfb7 100644 --- a/tic_tac_toe.py +++ b/tic_tac_toe.py @@ -13,58 +13,58 @@ report_counter = REPORT_INTERVAL def report(k): - global report_counter - if report_counter > 0: - report_counter -= 1 - else: - report_counter = REPORT_INTERVAL-1 - print(f"Iteration: {k:8d}") + global report_counter + if report_counter > 0: + report_counter -= 1 + else: + report_counter = REPORT_INTERVAL-1 + print(f"Iteration: {k:8d}") def play(player_provider: PlayerProvider, k_max=10000, with_debug=False): - player_provider.set_debug(with_debug) - for k in range(0, k_max): - report(k) - players = player_provider.choose() - state = create_empty_state() - move = 1 - run = True - other_player = players[-1] - while run: - for player in players: - if with_debug: - print(f"---------------------------------------------------") - print(f"- Game {k:06d}, Move {move} -----------------------------") - print(f"---------------------------------------------------") - last_state = state - state, has_moved = player.move(state) - if with_debug: - print(to_state_string(last_state, state)) - if not has_moved: - if with_debug: - print(f"{player.mark}: No more moves") - player.reward(0.0) - other_player.reward(0.0) - run = False - if player.has_won(state): - if with_debug: - print(f"{player.mark}: Has won the game") - player.reward(1.0) - other_player.reward(0.0) - run = False + player_provider.set_debug(with_debug) + for k in range(0, k_max): + report(k) + players = player_provider.choose() + state = create_empty_state() + move = 1 + run = True + other_player = players[-1] + while run: + for player in players: + if with_debug: + print(f"---------------------------------------------------") + print(f"- Game {k:06d}, Move {move} -----------------------------") + print(f"---------------------------------------------------") + last_state = state + state, has_moved = player.move(state) + if with_debug: + print(to_state_string(last_state, state)) + if not has_moved: + if with_debug: + print(f"{player.mark}: No more moves") + player.reward(0.0) + other_player.reward(0.0) + run = False + if player.has_won(state): + if with_debug: + print(f"{player.mark}: Has won the game") + player.reward(1.0) + other_player.reward(0.0) + run = False - other_player = player - if not run: - break + other_player = player + if not run: + break - move += 1 + move += 1 with open("values_x.json", "r") as fp: - x_values = json.load(fp) + x_values = json.load(fp) with open("values_o.json", "r") as fp: - o_values = json.load(fp) + o_values = json.load(fp) #px = MachinePlayer(mark='X', params=MachinePlayer.Params(p_exp=0.2, alpha=0.1), values=x_values) px = HumanPlayer(mark='X', name="Jens") @@ -74,13 +74,13 @@ player_provider = PlayerProvider(px, po) do_training = 1 if do_training: - play(player_provider, 10000, True) + play(player_provider, 10000, True) play(player_provider, 10, True) # Convert and write JSON object to file with open("values_x.json", "w") as fp: - json.dump(x_values, fp, indent=0) + json.dump(x_values, fp, indent=0) with open("values_o.json", "w") as fp: - json.dump(o_values, fp, indent=0) + json.dump(o_values, fp, indent=0)