diff --git a/.idea/.gitignore b/.idea/.gitignore
new file mode 100644
index 0000000..26d3352
--- /dev/null
+++ b/.idea/.gitignore
@@ -0,0 +1,3 @@
+# Default ignored files
+/shelf/
+/workspace.xml
diff --git a/.idea/inspectionProfiles/profiles_settings.xml b/.idea/inspectionProfiles/profiles_settings.xml
new file mode 100644
index 0000000..105ce2d
--- /dev/null
+++ b/.idea/inspectionProfiles/profiles_settings.xml
@@ -0,0 +1,6 @@
+
+
+
+
+
+
\ No newline at end of file
diff --git a/.idea/misc.xml b/.idea/misc.xml
new file mode 100644
index 0000000..72aeef8
--- /dev/null
+++ b/.idea/misc.xml
@@ -0,0 +1,7 @@
+
+
+
+
+
+
+
\ No newline at end of file
diff --git a/.idea/modules.xml b/.idea/modules.xml
new file mode 100644
index 0000000..219fec2
--- /dev/null
+++ b/.idea/modules.xml
@@ -0,0 +1,8 @@
+
+
+
+
+
+
+
+
\ No newline at end of file
diff --git a/.idea/tic_tac_toe.iml b/.idea/tic_tac_toe.iml
new file mode 100644
index 0000000..2c80e12
--- /dev/null
+++ b/.idea/tic_tac_toe.iml
@@ -0,0 +1,10 @@
+
+
+
+
+
+
+
+
+
+
\ No newline at end of file
diff --git a/.idea/vcs.xml b/.idea/vcs.xml
new file mode 100644
index 0000000..94a25f7
--- /dev/null
+++ b/.idea/vcs.xml
@@ -0,0 +1,6 @@
+
+
+
+
+
+
\ No newline at end of file
diff --git a/tic_tac_toe.py b/tic_tac_toe.py
index 4a9131e..61b95ac 100644
--- a/tic_tac_toe.py
+++ b/tic_tac_toe.py
@@ -1,6 +1,9 @@
import numpy as np
from numpy.random import uniform
+float_formatter = "{:.3f}".format
+np.set_printoptions(formatter={'float_kind':float_formatter})
+
def softmax(values: np.array) -> np.array:
result = values / sum(values)
@@ -13,17 +16,18 @@ def sample(values: np.array):
probs = softmax(values + 1.0E-6)
z = uniform()
print(f"Probs={probs}")
- print(f"Z={z}")
- result = None
+ print(f"Z={z:.3f}")
+ index = None
p_sum = 0
for idx, p in enumerate(probs):
p_sum += p
if z <= p_sum:
- result = idx
+ index = idx
break
- is_exploration = result != (len(probs) - 1)
- return result, is_exploration
+ probs_max = np.max(probs)
+ is_exploration = probs[index] < probs_max
+ return index, is_exploration
def test_sample():
@@ -37,15 +41,31 @@ def test_sample():
print(c)
-def to_state_string(state):
- state_str = ''
- for i in range(0, 3):
- for j in range(0, 3):
- char = state[3*i+j]
+def to_state_string(state, state_nex=None):
+ sp = ' '
+ sp_arrow = ' => '
+ sp_ = [sp, sp_arrow, sp]
+
+ def col(str_in, state):
+ str_out = str_in
+ for c in range(0, 3):
+ char = state[3*r+c]
if char == '-':
char = ' '
- state_str += "|"+char
- state_str += '|\x0A'
+ str_out += "|" + char
+ str_out += '|'
+ return str_out
+
+ state_str = ''
+ for r in range(0, 3):
+ state_str = col(state_str, state)
+
+ if state_nex is not None:
+ state_str += sp_[r]
+ state_str = col(state_str, state_nex)
+
+ if r != 2:
+ state_str += '\x0A'
return state_str
@@ -93,7 +113,7 @@ class Player(object):
index, is_exp = sample(values)
field = moves[index]
- print(f"{player.mark}: Chose {index}")
+ print(f"{player.mark}: Chose {index}, is_exp={is_exp}")
state_next = self.state_from_move(state, field)
# Learn
@@ -103,7 +123,7 @@ class Player(object):
d = max(0, v1-v0)
if d > 0:
self.set_value(0.1*d)
- print(f"{player.mark}: Learned {d}")
+ print(f"{player.mark}: Learned {d:0.3f}")
self.state_last = state_next
return state_next, can_move
@@ -153,7 +173,7 @@ p_x = Player(mark='X')
p_o = Player(mark='O')
-for k in range(0, 10000):
+for k in range(0, 1000):
# Wer fängt an?
if uniform() < 0.5:
players = [p_x, p_o]
@@ -162,12 +182,14 @@ for k in range(0, 10000):
state = "---------"
move = 1
run = True
- last_state = None
while run:
- last_state = state
for player in players:
- print(to_state_string(state))
+ print(f"---------------------------------------------------")
+ print(f"- Game {k:06d}, Move {move} -----------------------------")
+ print(f"---------------------------------------------------")
+ last_state = state
state, has_moved = player.move(state)
+ print(to_state_string(last_state, state))
if not has_moved:
print(f"{player.mark}: No more moves")
@@ -175,12 +197,10 @@ for k in range(0, 10000):
if player.has_won(state):
print(f"{player.mark}: Has won the game")
player.set_value(1.0)
- print(to_state_string(state))
run = False
if not run:
break
move += 1
- for player in players:
- print(f"{player.mark}: Values: {player.values}")
+