- refactored learning function

- remove state_last, use state instead
- added episode history

a_player.py

@@ -10,7 +10,6 @@ class APlayer(object):
 			self.other_mark = 'X'
 
 		self.state = None
-		self.state_last = None
 
 	def set_debug(self, with_debug):
 		pass
@@ -22,10 +21,10 @@ class APlayer(object):
 		state.reshape(state.size)[move] = self.mark
 		return state
 
-	def reward(self, value):
+	def new_game(self):
 		pass
 
-	def new_game(self):
+	def end_game(self, reward):
 		pass
 
 	def has_won(self, state: np.array):

human_player.py

@@ -42,7 +42,7 @@ class HumanPlayer(APlayer):
 		state_next = self.to_state(state.copy(), move)
 		return state_next, has_moved
 
-	def reward(self, value):
+	def end_game(self, value):
 		if value > 0:
 			print(f"{self.name}, Du hast gewonnen, super!")
 		else:
@@ -57,4 +57,4 @@ if __name__ == '__main__':
 		if not has_moved:
 			break
 
-	p.reward(1.0)
+	p.end_game(1.0)

machine_player.py

@@ -16,6 +16,7 @@ class MachinePlayer(APlayer):
 		self.values = {}
 		self.with_debug = False
 		self.values = values
+		self.episode_history = None
 
 	def set_debug(self, with_debug):
 		self.with_debug = with_debug
@@ -45,22 +46,30 @@ class MachinePlayer(APlayer):
 	def set_value(self, state: np.array, value):
 		self.values[self.to_key(state)] = value
 
-	def reward(self, value):
+	def end_game(self, value):
 		self.set_value(self.state, value)
+		print(self.episode_history)
 
 	def new_game(self):
 		self.state = None
-		self.state_last = None
+		self.episode_history = []
 
-	def move(self, state: np.array):
+	def learn_from_history(self):
-		values = np.array([])
+		pass
-		# get possible move
-		moves = get_potential_moves(state)
-		if moves.size == 0:
-			return state, False
 
+	def calc_value(self, state: np.array, next_state: np.array):
+		v0 = self.get_value(state)
+		v1 = self.get_value(next_state)
+		d = v0 + self.params.alpha*(v1-v0)
+		if self.with_debug:
+			print(f"{self.mark}: Learned {d:0.3f}")
+
+		return d
+
+	def get_best_move(self, state: np.array, moves: np.array):
 		best_move = None
 		best_value = -1
+		values = np.array([])
 		for move in moves:
 			# create hypothetical next state
 			state_next = self.to_state(state.copy(), move)
@@ -71,6 +80,15 @@ class MachinePlayer(APlayer):
 				best_move = move
 			values = np.append(values, value)
 
+		return best_move, best_value, values
+
+	def move(self, state: np.array):
+		# get possible move
+		moves = get_potential_moves(state)
+		if moves.size == 0:
+			return state, False
+
+		best_move, best_value, values = self.get_best_move(state, moves)
 		next_move = best_move
 		is_exp = False
 		# Randomly perform exploratory move
@@ -79,26 +97,28 @@ class MachinePlayer(APlayer):
 			next_move = moves[index]
 			is_exp = best_move != next_move
 
+		# Finally create next state
+		next_state = self.to_state(state.copy(), next_move)
+		next_value = self.get_value(next_state)
+
+		# Maintain history
+		self.episode_history.append((self.to_key(next_state), next_value, next_move, is_exp))
+
 		if self.with_debug:
 			print(f"{self.mark}: Values    = {values}")
 			print(f"{self.mark}: Moves     = {moves+1}")
 			print(f"{self.mark}: Best move = {best_move+1}")
 			print(f"{self.mark}: Next move = {next_move+1}, is_exp={is_exp}")
 
-		if self.state is not None:
-			self.state_last = self.state.copy()
-
-		self.state = self.to_state(state.copy(), next_move)
-
 		# Learn
-		if not is_exp and self.state_last is not None:
+		if not is_exp and self.state is not None:
-			v0 = self.get_value(self.state_last)
+			d = self.calc_value(self.state, next_state)
-			v1 = self.get_value(self.state)
-			d = v0 + self.params.alpha*(v1-v0)
 			if d > 0:
-				self.set_value(self.state_last, d)
+				self.set_value(self.state, d)
 			if self.with_debug:
 				print(f"{self.mark}: Learned {d:0.3f}")
 
-		return self.state, True
+		self.state = next_state
+
+		return next_state, True
 

tic_tac_toe.py

@@ -43,14 +43,14 @@ def play(player_provider: PlayerProvider, k_max=10000, with_debug=False):
 				if not has_moved:
 					if with_debug:
 						print(f"{player.mark}: No more moves")
-						player.reward(0.0)
+						player.end_game(0.0)
-						other_player.reward(0.0)
+						other_player.end_game(0.0)
 					run = False
 				if player.has_won(state):
 					if with_debug:
 						print(f"{player.mark}: Has won the game")
-						player.reward(1.0)
+						player.end_game(1.0)
-						other_player.reward(0.0)
+						other_player.end_game(0.0)
 					run = False
 
 				other_player = player