add learning after episode

a_player.py

@@ -21,10 +21,10 @@ class APlayer(object):
 		state.reshape(state.size)[move] = self.mark
 		return state
 
-	def new_game(self):
+	def new_game(self, state: np.array):
 		pass
 
-	def end_game(self, reward):
+	def end_game(self, state: np.array, value):
 		pass
 
 	def has_won(self, state: np.array):

machine_player.py

@@ -6,9 +6,10 @@ from state import get_potential_moves
 
 class MachinePlayer(APlayer):
 	class Params:
-		def __init__(self, p_explore, alpha):
+		def __init__(self, p_explore, alpha, do_learn_from_history=True):
 			self.p_exp = p_explore
 			self.alpha = alpha
+			self.do_learn_from_history = do_learn_from_history
 
 	def __init__(self, mark, params: Params, values):
 		APlayer.__init__(self, mark)
@@ -46,16 +47,21 @@ class MachinePlayer(APlayer):
 	def set_value(self, state: np.array, value):
 		self.values[self.to_key(state)] = value
 
-	def end_game(self, value):
+	def end_game(self, state: np.array, value):
 		self.set_value(self.state, value)
-		print(self.episode_history)
+		if self.params.do_learn_from_history:
+			self.learn_from_history()
 
-	def new_game(self):
+	def new_game(self, state: np.array):
-		self.state = None
+		self.state = state
 		self.episode_history = []
 
 	def learn_from_history(self):
-		pass
+		rev_hist = list(reversed(self.episode_history))
+		for hist in rev_hist:
+				d = self.calc_value(hist['state'], hist['next_state'])
+				if hist['is_exp']:
+					self.set_value(hist['state'], d)
 
 	def calc_value(self, state: np.array, next_state: np.array):
 		v0 = self.get_value(state)
@@ -99,10 +105,9 @@ class MachinePlayer(APlayer):
 
 		# Finally create next state
 		next_state = self.to_state(state.copy(), next_move)
-		next_value = self.get_value(next_state)
 
 		# Maintain history
-		self.episode_history.append((self.to_key(next_state), next_value, next_move, is_exp))
+		self.episode_history.append({'state': self.state, 'next_state': next_state, "is_exp": is_exp})
 
 		if self.with_debug:
 			print(f"{self.mark}: Values    = {values}")
@@ -111,12 +116,9 @@ class MachinePlayer(APlayer):
 			print(f"{self.mark}: Next move = {next_move+1}, is_exp={is_exp}")
 
 		# Learn
-		if not is_exp and self.state is not None:
+		if not (self.params.do_learn_from_history or is_exp):
 			d = self.calc_value(self.state, next_state)
-			if d > 0:
+			self.set_value(self.state, d)
-				self.set_value(self.state, d)
-			if self.with_debug:
-				print(f"{self.mark}: Learned {d:0.3f}")
 
 		self.state = next_state
 

player_provider.py

@@ -1,3 +1,5 @@
+import numpy as np
+
 from a_player import APlayer
 from numpy.random import uniform
 
@@ -10,15 +12,15 @@ class PlayerProvider:
 		for p in self.players:
 			p.set_debug(with_debug)
 
-	def choose(self) -> list[APlayer]:
+	def choose(self, state: np.array) -> list[APlayer]:
 		# Wer fängt an?
 		players = self.players
 		if uniform() < 0.5:
 			players.reverse()
 
 		result = players
-		players[0].new_game()
+		players[0].new_game(state)
-		players[1].new_game()
+		players[1].new_game(state)
 
 		return result
 

tic_tac_toe.py

@@ -25,8 +25,8 @@ def play(player_provider: PlayerProvider, k_max=10000, with_debug=False):
 	player_provider.set_debug(with_debug)
 	for k in range(0, k_max):
 		report(k)
-		players = player_provider.choose()
 		state = create_empty_state()
+		players = player_provider.choose(state)
 		move = 1
 		run = True
 		other_player = players[-1]
@@ -43,14 +43,14 @@ def play(player_provider: PlayerProvider, k_max=10000, with_debug=False):
 				if not has_moved:
 					if with_debug:
 						print(f"{player.mark}: No more moves")
-						player.end_game(0.0)
+						player.end_game(state, 0.0)
-						other_player.end_game(0.0)
+						other_player.end_game(state, 0.0)
 					run = False
 				if player.has_won(state):
 					if with_debug:
 						print(f"{player.mark}: Has won the game")
-						player.end_game(1.0)
+						player.end_game(state, 1.0)
-						other_player.end_game(0.0)
+						other_player.end_game(state, 0.0)
 					run = False
 
 				other_player = player
@@ -60,21 +60,30 @@ def play(player_provider: PlayerProvider, k_max=10000, with_debug=False):
 				move += 1
 
 
-with open("values_x.json", "r") as fp:
-	x_values = json.load(fp)
-
-with open("values_o.json", "r") as fp:
-	o_values = json.load(fp)
-
 do_training = 1
-do_human_player = 1
+do_human_player = 0
+do_load_values = 1
+
+if do_load_values:
+	try:
+		with open("values_x.json", "r") as fp:
+			x_values = json.load(fp)
+
+		with open("values_o.json", "r") as fp:
+			o_values = json.load(fp)
+	except Exception:
+		x_values = {}
+		o_values = {}
+else:
+	x_values = {}
+	o_values = {}
 
 p_hx = HumanPlayer(mark='X', name="Jens")
 p_mx = MachinePlayer(mark='X', params=MachinePlayer.Params(p_explore=0.2, alpha=0.1), values=x_values)
 p_mo = MachinePlayer(mark='O', params=MachinePlayer.Params(p_explore=0.2, alpha=0.1), values=o_values)
 
 if do_training:
-	play(PlayerProvider(p_mx, p_mo), 10000, False)
+	play(PlayerProvider(p_mx, p_mo), 5000, True)
 
 	# Values after training
 	# Convert and write JSON object to file