jens/tic_tac_toe
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Fixed sample

Browse Source
This commit is contained in:
jens
2024-06-08 23:31:37 +02:00
parent b9f83c26a1
commit cdc71b3eab
1 changed files with 40 additions and 29 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tic_tac_toe.py
+40 -29
View File
@@ -2,41 +2,41 @@ import numpy as np
from numpy.random import uniform from numpy.random import uniform
float_formatter = "{:.3f}".format float_formatter = "{:.3f}".format
np.set_printoptions(formatter={'float_kind':float_formatter}) np.set_printoptions(formatter={'float_kind': float_formatter})
def softmax(values: np.array) -> np.array: def softmax(values: np.array, eps=1.0E-6) -> np.array:
result = values / sum(values) result = (values + eps) / (sum(values) + eps)
result = np.sort(result)
return result return result
def sample(values: np.array, with_debug=False): def sample(values: np.array, with_debug=False):
# Normalize and sort # Normalize and sort
probs = softmax(values + 1.0E-6) probs = softmax(values)
sorted_indices = np.argsort(probs)
sorted_probs = probs[sorted_indices]
z = uniform() z = uniform()
if with_debug: if with_debug:
print(f"Probs={probs}") print(f"Probs={probs}")
print(f"Z={z:.3f}") print(f"Z={z:.3f}")
index = None index = None
p_sum = 0 p_sum = 0
for idx, p in enumerate(probs): for idx, p in enumerate(sorted_probs):
p_sum += p p_sum += p
if z <= p_sum: if z <= p_sum:
index = idx index = sorted_indices[idx]
break break
probs_max = np.max(probs) is_exploration = probs[index] < np.max(probs)
is_exploration = probs[index] < probs_max
return index, is_exploration return index, is_exploration
def test_sample(): def test_sample(data):
p = np.array([0.1, 0.1, 0.3, 0.5]) p = np.array(data)
c = np.array([0, 0, 0, 0]) c = np.array([0]*len(data))
for i in range(0, 1000): for i in range(0, 1000):
index = sample(p) index, _ = sample(p, with_debug=False)
c[index] += 1 c[index] += 1
print(c) print(c)
@@ -127,7 +127,6 @@ class MachinePlayer(APlayer):
else: else:
self.values = values self.values = values
def get_value(self, state): def get_value(self, state):
try: try:
result = self.values[state] result = self.values[state]
@@ -152,19 +151,22 @@ class MachinePlayer(APlayer):
can_move = moves.size > 0 can_move = moves.size > 0
if can_move: if can_move:
for field in moves: for move in moves:
# crate hypothetical next state # create hypothetical next state
state_next = self.state_from_move(state, field) state_next = self.state_from_move(state, move)
# evaluate value # evaluate value
value = self.get_value(state_next) value = self.get_value(state_next)
values = np.append(values, value) values = np.append(values, value)
index, is_exp = sample(values, self.with_debug) index, is_exp = sample(values)
field = moves[index] move = moves[index]
if self.with_debug: if self.with_debug:
print(f"{self.mark}: Chose {index}, is_exp={is_exp}") print(f"{self.mark}: Values = {values}")
print(f"{self.mark}: Moves = {moves+1}")
print(f"{self.mark}: Move = {move+1}, is_exp={is_exp}")
self.state_last = self.state self.state_last = self.state
self.state = self.state_from_move(state, field) self.state = self.state_from_move(state, move)
# Learn # Learn
if not is_exp and self.state_last is not None: if not is_exp and self.state_last is not None:
@@ -172,7 +174,7 @@ class MachinePlayer(APlayer):
v1 = self.get_value(self.state) v1 = self.get_value(self.state)
d = max(0, v1-v0) d = max(0, v1-v0)
if d > 0: if d > 0:
self.set_value(0.1*d) self.set_value(0.01*d)
if self.with_debug: if self.with_debug:
print(f"{self.mark}: Learned {d:0.3f}") print(f"{self.mark}: Learned {d:0.3f}")
@@ -234,13 +236,22 @@ def play(player_provider: PlayerProvider, k_max=10000, with_print=False):
move += 1 move += 1
px = MachinePlayer(mark='X') px = MachinePlayer(mark='X', with_debug=True)
po = MachinePlayer(mark='O') po = MachinePlayer(mark='O', with_debug=True)
do_training = 0 do_training = 1
if do_training: if do_training:
players = PlayerProvider(px, po) players = PlayerProvider(px, po, )
play(players, 10000, False) play(players, 2000, True)
players = PlayerProvider(MachinePlayer(mark='X', with_debug=True, values=px.values), MachinePlayer(mark='O', with_debug=True, values=po.values)) #players = PlayerProvider(MachinePlayer(mark='X', with_debug=True, values=px.values), MachinePlayer(mark='O', with_debug=True, values=po.values))
play(players, 100, True) #play(players, 100, True)
print("Testing")
test_sample([0.7, 0.1, 0.1, 0.1])
test_sample([0.1, 0.1, 0.2, 0.1])
test_sample([0.2, 0.8])
test_sample([0.1, 0.1])
test_sample([0.3, 0.0, 0.2, 0.0])
test_sample([0.0, 0.0, 0.0, 0.0])
test_sample([0.5, 0.0, 0.2, 0.3])