jens/tic_tac_toe
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Re-indent with tabs

Browse Source
This commit is contained in:
jens
2024-06-10 20:37:26 +02:00
parent d5b86b057e
commit 285549568d
7 changed files with 298 additions and 293 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.idea/codeStyles/codeStyleConfig.xml
Generated
+5
View File
@@ -0,0 +1,5 @@
<component name="ProjectCodeStyleConfiguration">
<state>
<option name="PREFERRED_PROJECT_CODE_STYLE" value="Default" />
</state>
</component>
a_player.py
+55 -55
View File
@@ -2,72 +2,72 @@ import numpy as np
class APlayer(object): class APlayer(object):
def __init__(self, mark): def __init__(self, mark):
self.mark = mark self.mark = mark
self.other_mark = 'O' self.other_mark = 'O'
if 'O' in mark: if 'O' in mark:
self.other_mark = 'X' self.other_mark = 'X'
self.state = None self.state = None
self.state_last = None self.state_last = None
def set_debug(self, with_debug): def set_debug(self, with_debug):
pass pass
@staticmethod @staticmethod
def get_potential_moves(state: np.array) -> np.array: def get_potential_moves(state: np.array) -> np.array:
st = state.reshape(state.size) st = state.reshape(state.size)
indices = [idx for idx, s in enumerate(st) if '-' in s] indices = [idx for idx, s in enumerate(st) if '-' in s]
return np.array(indices) return np.array(indices)
def move(self, state: np.array): def move(self, state: np.array):
return state, False return state, False
def state_from_move(self, state: np.array, field): def state_from_move(self, state: np.array, field):
state.reshape(state.size)[field] = self.mark state.reshape(state.size)[field] = self.mark
return state return state
def reward(self, value): def reward(self, value):
pass pass
def new_game(self): def new_game(self):
pass pass
@staticmethod @staticmethod
def f_state_slices(): def f_state_slices():
nr = 3 nr = 3
nc = 3 nc = 3
result = [] result = []
# Create row finishing states # Create row finishing states
d1 = () d1 = ()
d2_r = () d2_r = ()
for r in range(0, nr): for r in range(0, nr):
d1 += (r,) d1 += (r,)
for c in range(0, nc): for c in range(0, nc):
d2 = (c,) * nc d2 = (c,) * nc
result.append((d1, d2)) result.append((d1, d2))
# Create column finishing states # Create column finishing states
for c in range(0, nc): for c in range(0, nc):
d2 = (c,) * nc d2 = (c,) * nc
result.append((d2, d1)) result.append((d2, d1))
for c in range(0, nc): for c in range(0, nc):
d2_r += (nc - c - 1,) d2_r += (nc - c - 1,)
# Create diagonal finishing states #1 # Create diagonal finishing states #1
result.append((d1, d1)) result.append((d1, d1))
# Create diagonal finishing states #2 # Create diagonal finishing states #2
result.append((d1, d2_r)) result.append((d1, d2_r))
return result return result
def has_won(self, state: np.array): def has_won(self, state: np.array):
result = False result = False
ref = [self.mark, self.mark, self.mark] ref = [self.mark, self.mark, self.mark]
for f_state_slice in APlayer.f_state_slices(): for f_state_slice in APlayer.f_state_slices():
if np.all(state[f_state_slice] == ref): if np.all(state[f_state_slice] == ref):
result = True result = True
break break
return result return result
helper.py
+57 -57
View File
@@ -3,86 +3,86 @@ from numpy.random import uniform
def create_test_state() -> np.array: def create_test_state() -> np.array:
return np.array([['1', '2', '3'], ['4', '5', '6'], ['7', '8', '9']]) return np.array([['1', '2', '3'], ['4', '5', '6'], ['7', '8', '9']])
def create_empty_state() -> np.array: def create_empty_state() -> np.array:
return np.array([['-', '-', '-'], ['-', '-', '-'], ['-', '-', '-']]) return np.array([['-', '-', '-'], ['-', '-', '-'], ['-', '-', '-']])
def softmax(values: np.array, eps=1.0E-6) -> np.array: def softmax(values: np.array, eps=1.0E-6) -> np.array:
result = (values + eps) / (sum(values + eps)) result = (values + eps) / (sum(values + eps))
return result return result
def sample(values: np.array, with_debug=False): def sample(values: np.array, with_debug=False):
# Normalize and sort # Normalize and sort
probs = softmax(values) probs = softmax(values)
sorted_indices = np.argsort(probs) sorted_indices = np.argsort(probs)
sorted_probs = probs[sorted_indices] sorted_probs = probs[sorted_indices]
z = uniform() z = uniform()
if with_debug: if with_debug:
print(f"Probs={probs}") print(f"Probs={probs}")
print(f"Z={z:.3f}") print(f"Z={z:.3f}")
index = None index = None
p_sum = 0 p_sum = 0
for idx, p in enumerate(sorted_probs): for idx, p in enumerate(sorted_probs):
p_sum += p p_sum += p
if z <= p_sum: if z <= p_sum:
index = sorted_indices[idx] index = sorted_indices[idx]
break break
return index return index
def test_sample(data): def test_sample(data):
p = np.array(data) p = np.array(data)
c = np.array([0]*len(data)) c = np.array([0]*len(data))
for i in range(0, 1000): for i in range(0, 1000):
index = sample(p, with_debug=False) index = sample(p, with_debug=False)
c[index] += 1 c[index] += 1
print(c) print(c)
def to_state_string(state, state_nex=None): def to_state_string(state, state_nex=None):
sp = ' ' sp = ' '
sp_arrow = ' => ' sp_arrow = ' => '
sp_ = [sp, sp_arrow, sp] sp_ = [sp, sp_arrow, sp]
def col(str_in, state): def col(str_in, state):
str_out = str_in str_out = str_in
for c in range(0, 3): for c in range(0, 3):
char = state[r][c] char = state[r][c]
if char == '-': if char == '-':
char = ' ' char = ' '
str_out += "|" + char str_out += "|" + char
str_out += '|' str_out += '|'
return str_out return str_out
state_str = '' state_str = ''
for r in range(0, 3): for r in range(0, 3):
if state is not None: if state is not None:
state_str = col(state_str, state) state_str = col(state_str, state)
if state_nex is not None: if state_nex is not None:
state_str += sp_[r] state_str += sp_[r]
state_str = col(state_str, state_nex) state_str = col(state_str, state_nex)
if r != 2: if r != 2:
state_str += '\x0A' state_str += '\x0A'
return state_str return state_str
if __name__ == '__main__': if __name__ == '__main__':
print("Testing sample()") print("Testing sample()")
test_sample([0.7, 0.1, 0.1, 0.1]) test_sample([0.7, 0.1, 0.1, 0.1])
test_sample([0.1, 0.1, 0.2, 0.1]) test_sample([0.1, 0.1, 0.2, 0.1])
test_sample([0.2, 0.8]) test_sample([0.2, 0.8])
test_sample([0.1, 0.1]) test_sample([0.1, 0.1])
test_sample([0.3, 0.0, 0.2, 0.0]) test_sample([0.3, 0.0, 0.2, 0.0])
test_sample([0.0, 0.0, 0.0, 0.0]) test_sample([0.0, 0.0, 0.0, 0.0])
test_sample([0.5, 0.0, 0.2, 0.3]) test_sample([0.5, 0.0, 0.2, 0.3])
test_sample([0.3, 0.1, 0.4, 0.2]) test_sample([0.3, 0.1, 0.4, 0.2])
human_player.py
+40 -40
View File
@@ -5,55 +5,55 @@ from helper import to_state_string, create_empty_state, create_test_state
class HumanPlayer(APlayer): class HumanPlayer(APlayer):
def __init__(self, mark, name: str): def __init__(self, mark, name: str):
APlayer.__init__(self, mark) APlayer.__init__(self, mark)
self.name = name self.name = name
def move(self, state: np.array): def move(self, state: np.array):
move = 0 move = 0
has_moved = False has_moved = False
while True: while True:
print(f"{self.name}, Du bist dran. Das Spielfeld sieht so aus:") print(f"{self.name}, Du bist dran. Das Spielfeld sieht so aus:")
print(to_state_string(state, create_test_state())) print(to_state_string(state, create_test_state()))
print(f"{self.name}, in welches Feld willst Du Dein \"{self.mark}\" setzen?") print(f"{self.name}, in welches Feld willst Du Dein \"{self.mark}\" setzen?")
choice = input("Feldnummer: ") choice = input("Feldnummer: ")
if choice in "Qq": if choice in "Qq":
print("Abbruch") print("Abbruch")
break break
if not choice in "123456789" or len(choice) != 1: if not choice in "123456789" or len(choice) != 1:
print("Ungültiges Feld!") print("Ungültiges Feld!")
print("Versuche es nochmal") print("Versuche es nochmal")
continue continue
move = int(choice) - 1 move = int(choice) - 1
if not move in self.get_potential_moves(state): if not move in self.get_potential_moves(state):
print("Feld is bereits belegt!") print("Feld is bereits belegt!")
print("Versuche es nochmal") print("Versuche es nochmal")
continue continue
print("So soll es geschehen") print("So soll es geschehen")
has_moved = True has_moved = True
break break
state_next = self.state_from_move(state.copy(), move) state_next = self.state_from_move(state.copy(), move)
return state_next, has_moved return state_next, has_moved
def reward(self, value): def reward(self, value):
if value > 0: if value > 0:
print(f"{self.name}, Du hast gewonnen, super!") print(f"{self.name}, Du hast gewonnen, super!")
else: else:
print(f"{self.name}, Du hast leider verloren!") print(f"{self.name}, Du hast leider verloren!")
if __name__ == '__main__': if __name__ == '__main__':
p = HumanPlayer("X", "Jens") p = HumanPlayer("X", "Jens")
state = create_empty_state() state = create_empty_state()
while True: while True:
state, has_moved = p.move(state) state, has_moved = p.move(state)
if not has_moved: if not has_moved:
break break
p.reward(1.0) p.reward(1.0)
machine_player.py
+82 -82
View File
@@ -5,103 +5,103 @@ from numpy.random import uniform
class MachinePlayer(APlayer): class MachinePlayer(APlayer):
class Params: class Params:
def __init__(self, p_exp, alpha): def __init__(self, p_exp, alpha):
self.p_exp = p_exp self.p_exp = p_exp
self.alpha = alpha self.alpha = alpha
def __init__(self, mark, params: Params, values): def __init__(self, mark, params: Params, values):
APlayer.__init__(self, mark) APlayer.__init__(self, mark)
self.params = params self.params = params
self.values = {} self.values = {}
self.with_debug = False self.with_debug = False
self.values = values self.values = values
def set_debug(self, with_debug): def set_debug(self, with_debug):
self.with_debug = with_debug self.with_debug = with_debug
def print_state_table(self): def print_state_table(self):
count = 0 count = 0
for key in self.values: for key in self.values:
print(f"{self.mark}: {count:05d}: {key} = {self.values[key]:0.3f}") print(f"{self.mark}: {count:05d}: {key} = {self.values[key]:0.3f}")
count += 1 count += 1
@staticmethod @staticmethod
def to_key(state: np.array): def to_key(state: np.array):
key = '' key = ''
for st in state.reshape(state.size): for st in state.reshape(state.size):
key += st key += st
return key return key
def get_value(self, state: np.array): def get_value(self, state: np.array):
key = self.to_key(state) key = self.to_key(state)
try: try:
result = self.values[key] result = self.values[key]
except KeyError: except KeyError:
result = 0.5 result = 0.5
return result return result
def set_value(self, state: np.array, value): def set_value(self, state: np.array, value):
self.values[self.to_key(state)] = value self.values[self.to_key(state)] = value
def reward(self, value): def reward(self, value):
self.set_value(self.state, value) self.set_value(self.state, value)
def new_game(self): def new_game(self):
self.state = None self.state = None
self.state_last = None self.state_last = None
def move(self, state: np.array): def move(self, state: np.array):
do_sample = False do_sample = False
values = np.array([]) values = np.array([])
# get possible move # get possible move
moves = self.get_potential_moves(state) moves = self.get_potential_moves(state)
if moves.size == 0: if moves.size == 0:
return state, False return state, False
best_move = None best_move = None
best_value = -1 best_value = -1
for move in moves: for move in moves:
# create hypothetical next state # create hypothetical next state
state_next = self.state_from_move(state.copy(), move) state_next = self.state_from_move(state.copy(), move)
# evaluate value # evaluate value
value = self.get_value(state_next) value = self.get_value(state_next)
if best_value < value: if best_value < value:
best_value = value best_value = value
best_move = move best_move = move
values = np.append(values, value) values = np.append(values, value)
next_move = best_move next_move = best_move
is_exp = False is_exp = False
if uniform() < self.params.p_exp: if uniform() < self.params.p_exp:
index = np.random.randint(len(moves)) index = np.random.randint(len(moves))
next_move = moves[index] next_move = moves[index]
is_exp = best_move != next_move is_exp = best_move != next_move
elif do_sample: elif do_sample:
index = sample(values) index = sample(values)
next_move = moves[index] next_move = moves[index]
if self.with_debug: if self.with_debug:
print(f"{self.mark}: Values = {values}") print(f"{self.mark}: Values = {values}")
print(f"{self.mark}: Moves = {moves+1}") print(f"{self.mark}: Moves = {moves+1}")
print(f"{self.mark}: Best move = {best_move+1}") print(f"{self.mark}: Best move = {best_move+1}")
print(f"{self.mark}: Next move = {next_move+1}, is_exp={is_exp}") print(f"{self.mark}: Next move = {next_move+1}, is_exp={is_exp}")
if self.state is not None: if self.state is not None:
self.state_last = self.state.copy() self.state_last = self.state.copy()
self.state = self.state_from_move(state.copy(), next_move) self.state = self.state_from_move(state.copy(), next_move)
# Learn # Learn
if not is_exp and self.state_last is not None: if not is_exp and self.state_last is not None:
v0 = self.get_value(self.state_last) v0 = self.get_value(self.state_last)
v1 = self.get_value(self.state) v1 = self.get_value(self.state)
d = v0 + self.params.alpha*(v1-v0) d = v0 + self.params.alpha*(v1-v0)
if d > 0: if d > 0:
self.set_value(self.state_last, d) self.set_value(self.state_last, d)
if self.with_debug: if self.with_debug:
print(f"{self.mark}: Learned {d:0.3f}") print(f"{self.mark}: Learned {d:0.3f}")
return self.state, True return self.state, True
player_provider.py
+14 -14
View File
@@ -3,22 +3,22 @@ from numpy.random import uniform
class PlayerProvider: class PlayerProvider:
def __init__(self, p1: APlayer, p2: APlayer): def __init__(self, p1: APlayer, p2: APlayer):
self.players = [p1, p2] self.players = [p1, p2]
def set_debug(self, with_debug): def set_debug(self, with_debug):
for p in self.players: for p in self.players:
p.set_debug(with_debug) p.set_debug(with_debug)
def choose(self) -> list[APlayer]: def choose(self) -> list[APlayer]:
# Wer fängt an? # Wer fängt an?
players = self.players players = self.players
if uniform() < 0.5: if uniform() < 0.5:
players.reverse() players.reverse()
result = players result = players
players[0].new_game() players[0].new_game()
players[1].new_game() players[1].new_game()
return result return result
tic_tac_toe.py
+45 -45
View File
@@ -13,58 +13,58 @@ report_counter = REPORT_INTERVAL
def report(k): def report(k):
global report_counter global report_counter
if report_counter > 0: if report_counter > 0:
report_counter -= 1 report_counter -= 1
else: else:
report_counter = REPORT_INTERVAL-1 report_counter = REPORT_INTERVAL-1
print(f"Iteration: {k:8d}") print(f"Iteration: {k:8d}")
def play(player_provider: PlayerProvider, k_max=10000, with_debug=False): def play(player_provider: PlayerProvider, k_max=10000, with_debug=False):
player_provider.set_debug(with_debug) player_provider.set_debug(with_debug)
for k in range(0, k_max): for k in range(0, k_max):
report(k) report(k)
players = player_provider.choose() players = player_provider.choose()
state = create_empty_state() state = create_empty_state()
move = 1 move = 1
run = True run = True
other_player = players[-1] other_player = players[-1]
while run: while run:
for player in players: for player in players:
if with_debug: if with_debug:
print(f"---------------------------------------------------") print(f"---------------------------------------------------")
print(f"- Game {k:06d}, Move {move} -----------------------------") print(f"- Game {k:06d}, Move {move} -----------------------------")
print(f"---------------------------------------------------") print(f"---------------------------------------------------")
last_state = state last_state = state
state, has_moved = player.move(state) state, has_moved = player.move(state)
if with_debug: if with_debug:
print(to_state_string(last_state, state)) print(to_state_string(last_state, state))
if not has_moved: if not has_moved:
if with_debug: if with_debug:
print(f"{player.mark}: No more moves") print(f"{player.mark}: No more moves")
player.reward(0.0) player.reward(0.0)
other_player.reward(0.0) other_player.reward(0.0)
run = False run = False
if player.has_won(state): if player.has_won(state):
if with_debug: if with_debug:
print(f"{player.mark}: Has won the game") print(f"{player.mark}: Has won the game")
player.reward(1.0) player.reward(1.0)
other_player.reward(0.0) other_player.reward(0.0)
run = False run = False
other_player = player other_player = player
if not run: if not run:
break break
move += 1 move += 1
with open("values_x.json", "r") as fp: with open("values_x.json", "r") as fp:
x_values = json.load(fp) x_values = json.load(fp)
with open("values_o.json", "r") as fp: with open("values_o.json", "r") as fp:
o_values = json.load(fp) o_values = json.load(fp)
#px = MachinePlayer(mark='X', params=MachinePlayer.Params(p_exp=0.2, alpha=0.1), values=x_values) #px = MachinePlayer(mark='X', params=MachinePlayer.Params(p_exp=0.2, alpha=0.1), values=x_values)
px = HumanPlayer(mark='X', name="Jens") px = HumanPlayer(mark='X', name="Jens")
@@ -74,13 +74,13 @@ player_provider = PlayerProvider(px, po)
do_training = 1 do_training = 1
if do_training: if do_training:
play(player_provider, 10000, True) play(player_provider, 10000, True)
play(player_provider, 10, True) play(player_provider, 10, True)
# Convert and write JSON object to file # Convert and write JSON object to file
with open("values_x.json", "w") as fp: with open("values_x.json", "w") as fp:
json.dump(x_values, fp, indent=0) json.dump(x_values, fp, indent=0)
with open("values_o.json", "w") as fp: with open("values_o.json", "w") as fp:
json.dump(o_values, fp, indent=0) json.dump(o_values, fp, indent=0)