- control debug output
- fixed last state - refactored
This commit is contained in:
+105
-85
@@ -11,12 +11,13 @@ def softmax(values: np.array) -> np.array:
|
||||
return result
|
||||
|
||||
|
||||
def sample(values: np.array):
|
||||
def sample(values: np.array, with_debug=False):
|
||||
# Normalize and sort
|
||||
probs = softmax(values + 1.0E-6)
|
||||
z = uniform()
|
||||
print(f"Probs={probs}")
|
||||
print(f"Z={z:.3f}")
|
||||
if with_debug:
|
||||
print(f"Probs={probs}")
|
||||
print(f"Z={z:.3f}")
|
||||
index = None
|
||||
p_sum = 0
|
||||
for idx, p in enumerate(probs):
|
||||
@@ -70,66 +71,14 @@ def to_state_string(state, state_nex=None):
|
||||
return state_str
|
||||
|
||||
|
||||
class Player(object):
|
||||
def __init__(self, mark='X'):
|
||||
self.values = {}
|
||||
class APlayer(object):
|
||||
def __init__(self, mark):
|
||||
self.mark = mark
|
||||
self.state = None
|
||||
self.state_last = None
|
||||
|
||||
def get_value(self, state):
|
||||
try:
|
||||
result = self.values[state]
|
||||
except KeyError:
|
||||
result = 0
|
||||
|
||||
return result
|
||||
|
||||
def set_value(self, value, state=None):
|
||||
if state is None:
|
||||
if self.state_last is not None:
|
||||
self.values[self.state_last] = value
|
||||
else:
|
||||
self.values[state] = value
|
||||
|
||||
@staticmethod
|
||||
def get_potential_moves(state) -> np.array:
|
||||
indices = [idx for idx, s in enumerate(state) if '-' in s]
|
||||
return np.array(indices)
|
||||
|
||||
def move(self, state):
|
||||
values = np.array([])
|
||||
# get possible move
|
||||
moves = self.get_potential_moves(state)
|
||||
can_move = moves.size > 0
|
||||
state_next = state
|
||||
|
||||
if can_move:
|
||||
for field in moves:
|
||||
# crate hypothetical next state
|
||||
state_next = self.state_from_move(state, field)
|
||||
# evaluate value
|
||||
value = self.get_value(state_next)
|
||||
values = np.append(values, value)
|
||||
|
||||
index, is_exp = sample(values)
|
||||
field = moves[index]
|
||||
print(f"{player.mark}: Chose {index}, is_exp={is_exp}")
|
||||
state_next = self.state_from_move(state, field)
|
||||
|
||||
# Learn
|
||||
if not is_exp and self.state_last is not None:
|
||||
v0 = self.get_value(self.state_last)
|
||||
v1 = self.get_value(state_next)
|
||||
d = max(0, v1-v0)
|
||||
if d > 0:
|
||||
self.set_value(0.1*d)
|
||||
print(f"{player.mark}: Learned {d:0.3f}")
|
||||
|
||||
self.state_last = state_next
|
||||
return state_next, can_move
|
||||
|
||||
def state_from_move(self, state, field):
|
||||
return state[:field] + self.mark + state[field + 1:]
|
||||
return state, False
|
||||
|
||||
def has_won(self, state):
|
||||
ref = self.mark + self.mark + self.mark
|
||||
@@ -169,38 +118,109 @@ class Player(object):
|
||||
return False
|
||||
|
||||
|
||||
p_x = Player(mark='X')
|
||||
p_o = Player(mark='O')
|
||||
class MachinePlayer(APlayer):
|
||||
def __init__(self, mark='X', with_debug=False):
|
||||
APlayer.__init__(self, mark)
|
||||
self.with_debug = with_debug
|
||||
self.values = {}
|
||||
|
||||
def get_value(self, state):
|
||||
try:
|
||||
result = self.values[state]
|
||||
except KeyError:
|
||||
result = 0
|
||||
|
||||
return result
|
||||
|
||||
def set_value(self, value):
|
||||
if self.state_last is not None:
|
||||
self.values[self.state_last] = value
|
||||
|
||||
@staticmethod
|
||||
def get_potential_moves(state) -> np.array:
|
||||
indices = [idx for idx, s in enumerate(state) if '-' in s]
|
||||
return np.array(indices)
|
||||
|
||||
def move(self, state):
|
||||
values = np.array([])
|
||||
# get possible move
|
||||
moves = self.get_potential_moves(state)
|
||||
can_move = moves.size > 0
|
||||
|
||||
if can_move:
|
||||
for field in moves:
|
||||
# crate hypothetical next state
|
||||
state_next = self.state_from_move(state, field)
|
||||
# evaluate value
|
||||
value = self.get_value(state_next)
|
||||
values = np.append(values, value)
|
||||
|
||||
index, is_exp = sample(values, self.with_debug)
|
||||
field = moves[index]
|
||||
if self.with_debug:
|
||||
print(f"{self.mark}: Chose {index}, is_exp={is_exp}")
|
||||
self.state_last = self.state
|
||||
self.state = self.state_from_move(state, field)
|
||||
|
||||
# Learn
|
||||
if not is_exp and self.state_last is not None:
|
||||
v0 = self.get_value(self.state_last)
|
||||
v1 = self.get_value(self.state)
|
||||
d = max(0, v1-v0)
|
||||
if d > 0:
|
||||
self.set_value(0.1*d)
|
||||
if self.with_debug:
|
||||
print(f"{self.mark}: Learned {d:0.3f}")
|
||||
|
||||
return self.state, can_move
|
||||
|
||||
def state_from_move(self, state, field):
|
||||
return state[:field] + self.mark + state[field + 1:]
|
||||
|
||||
|
||||
for k in range(0, 1000):
|
||||
def play(players: list[APlayer], k_max=10000, with_print=False):
|
||||
for k in range(0, k_max):
|
||||
state = "---------"
|
||||
move = 1
|
||||
run = True
|
||||
while run:
|
||||
for player in players:
|
||||
if with_print:
|
||||
print(f"---------------------------------------------------")
|
||||
print(f"- Game {k:06d}, Move {move} -----------------------------")
|
||||
print(f"---------------------------------------------------")
|
||||
last_state = state
|
||||
state, has_moved = player.move(state)
|
||||
if with_print:
|
||||
print(to_state_string(last_state, state))
|
||||
|
||||
if not has_moved:
|
||||
if with_print:
|
||||
print(f"{player.mark}: No more moves")
|
||||
run = False
|
||||
if player.has_won(state):
|
||||
if with_print:
|
||||
print(f"{player.mark}: Has won the game")
|
||||
if isinstance(player, MachinePlayer):
|
||||
player.set_value(1.0)
|
||||
run = False
|
||||
|
||||
if not run:
|
||||
break
|
||||
|
||||
move += 1
|
||||
|
||||
|
||||
def choose_player(p1: APlayer, p2: APlayer) -> list[APlayer]:
|
||||
# Wer fängt an?
|
||||
if uniform() < 0.5:
|
||||
players = [p_x, p_o]
|
||||
result = [p1, p2]
|
||||
else:
|
||||
players = [p_o, p_x]
|
||||
state = "---------"
|
||||
move = 1
|
||||
run = True
|
||||
while run:
|
||||
for player in players:
|
||||
print(f"---------------------------------------------------")
|
||||
print(f"- Game {k:06d}, Move {move} -----------------------------")
|
||||
print(f"---------------------------------------------------")
|
||||
last_state = state
|
||||
state, has_moved = player.move(state)
|
||||
print(to_state_string(last_state, state))
|
||||
result = [p2, p1]
|
||||
|
||||
if not has_moved:
|
||||
print(f"{player.mark}: No more moves")
|
||||
run = False
|
||||
if player.has_won(state):
|
||||
print(f"{player.mark}: Has won the game")
|
||||
player.set_value(1.0)
|
||||
run = False
|
||||
return result
|
||||
|
||||
if not run:
|
||||
break
|
||||
|
||||
move += 1
|
||||
opponents = choose_player(MachinePlayer(mark='X', with_debug=True), MachinePlayer(mark='O', with_debug=True))
|
||||
play(opponents, 1000, True)
|
||||
|
||||
|
||||
Reference in New Issue
Block a user