add learning after episode

machine_player.py

@@ -6,9 +6,10 @@ from state import get_potential_moves
 
 class MachinePlayer(APlayer):
 	class Params:
-		def __init__(self, p_explore, alpha):
+		def __init__(self, p_explore, alpha, do_learn_from_history=True):
 			self.p_exp = p_explore
 			self.alpha = alpha
+			self.do_learn_from_history = do_learn_from_history
 
 	def __init__(self, mark, params: Params, values):
 		APlayer.__init__(self, mark)
@@ -46,16 +47,21 @@ class MachinePlayer(APlayer):
 	def set_value(self, state: np.array, value):
 		self.values[self.to_key(state)] = value
 
-	def end_game(self, value):
+	def end_game(self, state: np.array, value):
 		self.set_value(self.state, value)
-		print(self.episode_history)
+		if self.params.do_learn_from_history:
+			self.learn_from_history()
 
-	def new_game(self):
-		self.state = None
+	def new_game(self, state: np.array):
+		self.state = state
 		self.episode_history = []
 
 	def learn_from_history(self):
-		pass
+		rev_hist = list(reversed(self.episode_history))
+		for hist in rev_hist:
+				d = self.calc_value(hist['state'], hist['next_state'])
+				if hist['is_exp']:
+					self.set_value(hist['state'], d)
 
 	def calc_value(self, state: np.array, next_state: np.array):
 		v0 = self.get_value(state)
@@ -99,10 +105,9 @@ class MachinePlayer(APlayer):
 
 		# Finally create next state
 		next_state = self.to_state(state.copy(), next_move)
-		next_value = self.get_value(next_state)
 
 		# Maintain history
-		self.episode_history.append((self.to_key(next_state), next_value, next_move, is_exp))
+		self.episode_history.append({'state': self.state, 'next_state': next_state, "is_exp": is_exp})
 
 		if self.with_debug:
 			print(f"{self.mark}: Values    = {values}")
@@ -111,12 +116,9 @@ class MachinePlayer(APlayer):
 			print(f"{self.mark}: Next move = {next_move+1}, is_exp={is_exp}")
 
 		# Learn
-		if not is_exp and self.state is not None:
+		if not (self.params.do_learn_from_history or is_exp):
 			d = self.calc_value(self.state, next_state)
-			if d > 0:
-				self.set_value(self.state, d)
-			if self.with_debug:
-				print(f"{self.mark}: Learned {d:0.3f}")
+			self.set_value(self.state, d)
 
 		self.state = next_state