From 7836541a4927044424f42986676ba28767182ac7 Mon Sep 17 00:00:00 2001
From: Jens Ahrensfeld <jens@jayfield.org>
Date: Sat, 22 Jun 2024 08:58:35 +0200
Subject: [PATCH] - refactored - fixed evaluation of mean reward

---
 k-bandits.py | 75 +++++++++++++++++++++++++++++++---------------------
 1 file changed, 45 insertions(+), 30 deletions(-)

diff --git a/k-bandits.py b/k-bandits.py
index e960317..ee2b296 100644
--- a/k-bandits.py
+++ b/k-bandits.py
@@ -1,5 +1,6 @@
 import numpy as np
 import matplotlib.pyplot as pl
+from typing import Callable
 
 float_formatter = "{:.3f}".format
 np.set_printoptions(formatter={'float_kind': float_formatter})
@@ -7,50 +8,64 @@ np.set_printoptions(formatter={'float_kind': float_formatter})
 K = 10
 
 
-def bandit(bias):
-	zn = np.random.uniform() + bias
+def bandit(a: int, bias: np.array):
+	zn = np.random.uniform() + bias[a]
 	return zn
 
 
+def test(episode_len: int, epsilon: float, qu: np.array, nu: np.array, bandit: Callable) -> np.array:
+	r_sum = 0
+	r_mean = []
+	n = 0
+	for j in range(0, episode_len):
+		# choose action
+		z = np.random.uniform()
+		if z <= epsilon:
+			# Choose random action
+			a = np.random.randint(low=0, high=K)
+		else:
+			# Choose best action
+			a = np.argmax(qu)
+
+		# get reward from bandit
+		r = bandit(a)
+		nu[a] = nu[a] + 1
+		qu[a] = qu[a] + (r - qu[a]) / nu[a]
+
+		# Reduce tendency to explore with number of steps (or with age for humans)
+		epsilon = epsilon * (1 - rho)
+
+		# Statistics
+		r_sum += r
+		n += 1
+		r_mean.append(r_sum/n)
+
+	return np.array(r_mean)
+
+
 if __name__ == '__main__':
 
-	epsilon = 0.0		# Anti-greediness
-	rho = 0.01			# reduce epsilon with age
+	epsilon = 0.1		# Anti-greediness
+	rho = 0.01		# reduce epsilon with age
 
-	qu = np.zeros(K)
-	nu = np.zeros(K)
-	r_vec = []
 	# init bandits with different biases for shifting reward probability
 	# -> expected reward q*(a)
-	ql_star = [-0.4, -0.3, -0.2, -0.1, 0.0, +0.1, +0.2, +0.3, +0.4, +0.5]
+	ql_star = np.array([-0.4, -0.3, -0.2, -0.1, 0.0, +0.1, +0.2, +0.3, +0.4, +0.5])
 
-	N = 1000
-	n_ages = 1000
-	for age in range(0, n_ages):
-		r_sum = 0
-		for j in range(0, N):
-			# choose action
-			z = np.random.uniform()
-			if z <= epsilon:
-				# Choose random action
-				a = np.random.randint(low=0, high=K)
-			else:
-				# Choose best action
-				a = np.argmax(qu)
+	num_realisations = 10
+	episode_len = 1000
+	r_mean = np.zeros(episode_len)
+	for age in range(0, num_realisations):
+		# Init Q and N
+		qu = np.zeros(K)
+		nu = np.zeros(K)
 
-			# get reward from bandit
-			r = bandit(ql_star[a])
-			nu[a] = nu[a] + 1
-			qu[a] = qu[a] + (r - qu[a])/nu[a]
-			r_sum += r
-		r_vec.append(r_sum/N)
-		# Reduce tendency to explore with number of steps (or with age for humans)
-		epsilon = epsilon*(1-rho)
+		r_mean += test(episode_len, epsilon, qu, nu, bandit=lambda a: bandit(a, ql_star))
 
 	print(f"ql_star  = {ql_star}")
 	print(f"qu       = {qu}")
 	print(f"nu       = {nu}")
 
-	pl.plot(np.array(r_vec))
+	pl.plot(r_mean/num_realisations)
 	pl.grid()
 	pl.show()