fixed whole k-bandit test bed

k-bandits.py

@@ -5,20 +5,18 @@ from typing import Callable
 float_formatter = "{:.3f}".format
 np.set_printoptions(formatter={'float_kind': float_formatter})
 
-K = 18
+K = 10
 
 
 def bandit(a: int, bias: np.array):
-	zn = np.random.uniform() + bias[a]
+	zn = np.random.normal() + bias[a]
 	return zn
 
 
 def test(_episode_len: int, _qu: np.array, _nu: np.array, _bandit: Callable) -> np.array:
-	_epsilon = 0.5		# Anti-greediness
+	_epsilon = 0.5		# Anti-greediness (ability to explore)
 	_rho = 0.005		# reduce epsilon with age
-	_r_sum = 0
 	_r_mean = []
-	_n = 0
 	for j in range(0, _episode_len):
 		# choose action
 		z = np.random.uniform()
@@ -38,41 +36,28 @@ def test(_episode_len: int, _qu: np.array, _nu: np.array, _bandit: Callable) ->
 		_epsilon = _epsilon * (1 - _rho)
 
 		# Statistics
-		_r_sum += r
-		_n += 1
-		_r_mean.append(_r_sum/_n)
+		_r_mean.append(r)
 
 	return np.array(_r_mean)
 
 
 if __name__ == '__main__':
-	# init bandits with different biases for shifting reward probability
-	# -> expected reward q*(a)
-	ql_star = np.array(np.linspace(-0.5, +0.5, K))
-
 	# Init parameters
-	num_realisations = 10
-	episode_len = 10000
-
-	# Init Q and N
-	qu = np.zeros(K)
-	nu = np.zeros(K)
+	num_realisations = 2000
+	episode_len = 1000
 
 	# Init r_mean
 	r_mean = np.zeros(episode_len)
 	for realization in range(0, num_realisations):
-		# Re-Init Q and N
+		# init bandits with different biases for shifting reward probability
+		# -> expected reward q*(a)
+		ql_star = np.random.uniform(-1.5, +1.5, K)
+
+		# Init Q and N
 		qu = np.zeros(K)
 		nu = np.zeros(K)
-#		qu = np.random.normal(qu)
-#		nu = np.random.normal(nu)
 		r_mean += test(episode_len, qu, nu, _bandit=lambda a: bandit(a, ql_star))
 
-		print(f"qu       = {qu}")
-		print(f"nu       = {nu}")
-
-	print(f"ql_star  = {ql_star}")
-
 	pl.plot(r_mean/num_realisations)
 	pl.grid()
 	pl.title(f"Norm. E(R), num. realizations: Z={num_realisations}, num. bandits: K={K}")
@@ -80,6 +65,6 @@ if __name__ == '__main__':
 	pl.ylabel("E(R)/Z")
 	ax = pl.gca()
 	ax.set_xlim([-episode_len/10, episode_len])
-	ax.set_ylim([0, 1])
+	ax.set_ylim([0, 1.5])
 
 	pl.show()