- run with different param (epsilon, rho)

- plot run per plot

k-bandits.py

@@ -7,23 +7,26 @@ np.set_printoptions(formatter={'float_kind': float_formatter})
 
 K = 10
 
-
 def bandit(a: int, bias: np.array):
 	zn = np.random.normal() + bias[a]
 	return zn
 
 
-def test(_episode_len: int, _qu: np.array, _nu: np.array, _bandit: Callable) -> np.array:
-	_epsilon = 0.5		# Anti-greediness (ability to explore)
-	_rho = 0.005		# reduce epsilon with age
+def test(_episode_len: int, _qu: np.array, _nu: np.array, _bandit: Callable, _param: tuple[float, float]) -> np.array:
+	_epsilon = _param[0]		# Anti-greediness (ability to explore)
+	_rho = _param[1]		# reduce epsilon with age
 	_r_mean = []
+
+	# Calc z in advance
+	z = np.random.uniform(size=_episode_len)
+
+	# Calc a_expl in advance
+	a_expl = np.random.randint(low=0, high=K, size=_episode_len)
+
 	for j in range(0, _episode_len):
+		a = a_expl[j]
 		# choose action
-		z = np.random.uniform()
-		if z <= _epsilon:
-			# Choose random action
-			a = np.random.randint(low=0, high=K)
-		else:
+		if z[j] > _epsilon:
 			# Choose best action
 			a = np.argmax(_qu)
 
@@ -42,27 +45,34 @@ def test(_episode_len: int, _qu: np.array, _nu: np.array, _bandit: Callable) ->
 
 
 if __name__ == '__main__':
+
 	# Init parameters
 	num_realisations = 2000
 	episode_len = 1000
 
-	# Init r_mean
-	r_mean = np.zeros(episode_len)
-	for realization in range(0, num_realisations):
-		# init bandits with different biases for shifting reward probability
-		# -> expected reward q*(a)
-		ql_star = np.random.uniform(-1.5, +1.5, K)
+	params = [(0.0, 0.0), (0.01, 0.002), (0.1, 0.002)]
+	legend = []
+	for param in params:
+		# Init r_mean
+		r_mean = np.zeros(episode_len)
+		for realization in range(0, num_realisations):
+			# init bandits with different biases for shifting reward probability
+			# -> expected reward q*(a)
+			ql_star = np.random.uniform(-1.5, +1.5, K)
 
-		# Init Q and N
-		qu = np.zeros(K)
-		nu = np.zeros(K)
-		r_mean += test(episode_len, qu, nu, _bandit=lambda a: bandit(a, ql_star))
+			# Init Q and N
+			qu = np.zeros(K)
+			nu = np.zeros(K)
+			r_mean += test(episode_len, qu, nu, _bandit=lambda a: bandit(a, ql_star), _param=param)
+
+		pl.plot(r_mean/num_realisations)
+		legend.append(f"Param {param}")
 
-	pl.plot(r_mean/num_realisations)
 	pl.grid()
 	pl.title(f"Norm. E(R), num. realizations: Z={num_realisations}, num. bandits: K={K}")
 	pl.xlabel("Episode")
 	pl.ylabel("E(R)/Z")
+	pl.legend(legend)
 	ax = pl.gca()
 	ax.set_xlim([-episode_len/10, episode_len])
 	ax.set_ylim([0, 1.5])