- refactored learning function

- remove state_last, use state instead
- added episode history

machine_player.py

@@ -16,6 +16,7 @@ class MachinePlayer(APlayer):
 		self.values = {}
 		self.with_debug = False
 		self.values = values
+		self.episode_history = None
 
 	def set_debug(self, with_debug):
 		self.with_debug = with_debug
@@ -45,22 +46,30 @@ class MachinePlayer(APlayer):
 	def set_value(self, state: np.array, value):
 		self.values[self.to_key(state)] = value
 
-	def reward(self, value):
+	def end_game(self, value):
 		self.set_value(self.state, value)
+		print(self.episode_history)
 
 	def new_game(self):
 		self.state = None
-		self.state_last = None
+		self.episode_history = []
 
-	def move(self, state: np.array):
-		values = np.array([])
-		# get possible move
-		moves = get_potential_moves(state)
-		if moves.size == 0:
-			return state, False
+	def learn_from_history(self):
+		pass
 
+	def calc_value(self, state: np.array, next_state: np.array):
+		v0 = self.get_value(state)
+		v1 = self.get_value(next_state)
+		d = v0 + self.params.alpha*(v1-v0)
+		if self.with_debug:
+			print(f"{self.mark}: Learned {d:0.3f}")
+
+		return d
+
+	def get_best_move(self, state: np.array, moves: np.array):
 		best_move = None
 		best_value = -1
+		values = np.array([])
 		for move in moves:
 			# create hypothetical next state
 			state_next = self.to_state(state.copy(), move)
@@ -71,6 +80,15 @@ class MachinePlayer(APlayer):
 				best_move = move
 			values = np.append(values, value)
 
+		return best_move, best_value, values
+
+	def move(self, state: np.array):
+		# get possible move
+		moves = get_potential_moves(state)
+		if moves.size == 0:
+			return state, False
+
+		best_move, best_value, values = self.get_best_move(state, moves)
 		next_move = best_move
 		is_exp = False
 		# Randomly perform exploratory move
@@ -79,26 +97,28 @@ class MachinePlayer(APlayer):
 			next_move = moves[index]
 			is_exp = best_move != next_move
 
+		# Finally create next state
+		next_state = self.to_state(state.copy(), next_move)
+		next_value = self.get_value(next_state)
+
+		# Maintain history
+		self.episode_history.append((self.to_key(next_state), next_value, next_move, is_exp))
+
 		if self.with_debug:
 			print(f"{self.mark}: Values    = {values}")
 			print(f"{self.mark}: Moves     = {moves+1}")
 			print(f"{self.mark}: Best move = {best_move+1}")
 			print(f"{self.mark}: Next move = {next_move+1}, is_exp={is_exp}")
 
-		if self.state is not None:
-			self.state_last = self.state.copy()
-
-		self.state = self.to_state(state.copy(), next_move)
-
 		# Learn
-		if not is_exp and self.state_last is not None:
-			v0 = self.get_value(self.state_last)
-			v1 = self.get_value(self.state)
-			d = v0 + self.params.alpha*(v1-v0)
+		if not is_exp and self.state is not None:
+			d = self.calc_value(self.state, next_state)
 			if d > 0:
-				self.set_value(self.state_last, d)
+				self.set_value(self.state, d)
 			if self.with_debug:
 				print(f"{self.mark}: Learned {d:0.3f}")
 
-		return self.state, True
+		self.state = next_state
+
+		return next_state, True