From f70cf1fc1ea9e9b8b305459acaff8ff091f57cb6 Mon Sep 17 00:00:00 2001
From: Jens Ahrensfeld <jens@jayfield.org>
Date: Thu, 13 Jun 2024 08:43:08 +0200
Subject: [PATCH] - added reward sum and reward mean

---
 machine_player.py | 27 ++++++++++++++++++++-------
 tic_tac_toe.py    | 36 +++++++++++++++++++++---------------
 2 files changed, 41 insertions(+), 22 deletions(-)

diff --git a/machine_player.py b/machine_player.py
index 492c99d..48135a4 100644
--- a/machine_player.py
+++ b/machine_player.py
@@ -18,6 +18,7 @@ class MachinePlayer(APlayer):
 		self.with_debug = False
 		self.values = values
 		self.episode_history = None
+		self.reward_sum = 0
 
 	def set_debug(self, with_debug):
 		self.with_debug = with_debug
@@ -44,31 +45,43 @@ class MachinePlayer(APlayer):
 
 		return result
 
-	def set_value(self, state: np.array, value):
-		self.values[self.to_key(state)] = value
+	def set_value(self, state: np.array, reward):
+		self.values[self.to_key(state)] = reward
+		self.reward_sum += reward
+		if self.with_debug:
+			print(f"{self.mark}: Rewarded {reward:0.3f}")
 
 	def end_game(self, state: np.array, value):
 		self.set_value(self.state, value)
 		if self.params.do_learn_from_history:
 			self.learn_from_history()
 
+		r_sum = self.reward_sum
+		r_mean = self.reward_sum/self.count
+
+		if self.with_debug:
+			print(f"Reward sum  : {r_sum}")
+			print(f"Reward mean : {r_mean}")
+
+		return r_sum, r_mean
+
 	def new_game(self, state: np.array):
 		self.state = state
 		self.episode_history = []
+		self.reward_sum = 0
+		self.count = 0
 
 	def learn_from_history(self):
 		rev_hist = list(reversed(self.episode_history))
 		for hist in rev_hist:
+			if not hist['is_exp']:
 				d = self.calc_value(hist['state'], hist['next_state'])
-				if not hist['is_exp']:
-					self.set_value(hist['state'], d)
+				self.set_value(hist['state'], d)
 
 	def calc_value(self, state: np.array, next_state: np.array):
 		v0 = self.get_value(state)
 		v1 = self.get_value(next_state)
 		d = v0 + self.params.alpha*(v1-v0)
-		if self.with_debug:
-			print(f"{self.mark}: Learned {d:0.3f}")
 
 		return d
 
@@ -121,6 +134,6 @@ class MachinePlayer(APlayer):
 			self.set_value(self.state, d)
 
 		self.state = next_state
-
+		self.count += 1
 		return next_state, True
 
diff --git a/tic_tac_toe.py b/tic_tac_toe.py
index 3054243..244c33c 100644
--- a/tic_tac_toe.py
+++ b/tic_tac_toe.py
@@ -21,36 +21,42 @@ def play(player_provider: PlayerProvider, k_max=10000, with_debug=False):
 		run = True
 		other_player = players[-1]
 		while run:
-			for player in players:
+			do_stop = False
+			this_reward = 0
+			other_reward = 0
+			for this_player in players:
 				if with_debug:
 					print(f"---------------------------------------------------")
 					print(f"- Game {k:06d}, Move {move} -----------------------------")
 					print(f"---------------------------------------------------")
 				last_state = state
-				state, has_moved = player.move(state)
+				state, has_moved = this_player.move(state)
 				if with_debug:
 					print(to_state_string(last_state, state))
 				if not has_moved:
-					player.end_game(state, 0.0)
-					other_player.end_game(state, 0.0)
-					run = False
+					this_reward = 0
+					other_reward = 0
+					do_stop = True
 					if with_debug:
-						print(f"{player.mark}: No more moves")
-				if player.has_won(state):
-					player.end_game(state, 1.0)
-					other_player.end_game(state, 0.0)
-					run = False
+						print(f"{this_player.mark}: No more moves")
+				if this_player.has_won(state):
+					this_reward = 1
+					other_reward = 0
+					do_stop = True
 					if with_debug:
-						print(f"{player.mark}: Has won the game")
+						print(f"{this_player.mark}: Has won the game")
 
-				other_player = player
-				if not run:
+				if do_stop:
+					this_player.end_game(state, this_reward)
+					other_player.end_game(state, other_reward)
+					run = False
 					break
 
+				other_player = this_player
 				move += 1
 
 
-do_training = 1
+do_training = 0
 do_human_player = 0
 do_load_values = 1
 
@@ -72,7 +78,7 @@ p_hx = HumanPlayer(mark='X', name="Jens")
 p_mx = MachinePlayer(mark='X', params=MachinePlayer.Params(p_explore=0.1, alpha=0.1), values=x_values)
 p_mo = MachinePlayer(mark='O', params=MachinePlayer.Params(p_explore=0.1, alpha=0.1), values=o_values)
 
-K = 50
+K = 5
 N = 1000
 if do_training:
 	for k in range(0, K):