refactored

k-bandits.py

@@ -5,7 +5,7 @@ from typing import Callable
 float_formatter = "{:.3f}".format
 np.set_printoptions(formatter={'float_kind': float_formatter})
 
-K = 10
+K = 18
 
 
 def bandit(a: int, bias: np.array):
@@ -13,58 +13,65 @@ def bandit(a: int, bias: np.array):
 	return zn
 
 
-def test(episode_len: int, epsilon: float, qu: np.array, nu: np.array, bandit: Callable) -> np.array:
-	r_sum = 0
-	r_mean = []
-	n = 0
-	for j in range(0, episode_len):
+def test(_episode_len: int, _qu: np.array, _nu: np.array, _bandit: Callable) -> np.array:
+	_epsilon = 0.5		# Anti-greediness
+	_rho = 0.005		# reduce epsilon with age
+	_r_sum = 0
+	_r_mean = []
+	_n = 0
+	for j in range(0, _episode_len):
 		# choose action
 		z = np.random.uniform()
-		if z <= epsilon:
+		if z <= _epsilon:
 			# Choose random action
 			a = np.random.randint(low=0, high=K)
 		else:
 			# Choose best action
-			a = np.argmax(qu)
+			a = np.argmax(_qu)
 
 		# get reward from bandit
-		r = bandit(a)
-		nu[a] = nu[a] + 1
-		qu[a] = qu[a] + (r - qu[a]) / nu[a]
+		r = _bandit(a)
+		_nu[a] = _nu[a] + 1
+		_qu[a] = _qu[a] + (r - _qu[a]) / _nu[a]
 
 		# Reduce tendency to explore with number of steps (or with age for humans)
-		epsilon = epsilon * (1 - rho)
+		_epsilon = _epsilon * (1 - _rho)
 
 		# Statistics
-		r_sum += r
-		n += 1
-		r_mean.append(r_sum/n)
+		_r_sum += r
+		_n += 1
+		_r_mean.append(_r_sum/_n)
 
-	return np.array(r_mean)
+	return np.array(_r_mean)
 
 
 if __name__ == '__main__':
-
-	epsilon = 0.5		# Anti-greediness
-	rho = 0.005		# reduce epsilon with age
-
 	# init bandits with different biases for shifting reward probability
 	# -> expected reward q*(a)
 	ql_star = np.array(np.linspace(-0.5, +0.5, K))
 
+	# Init parameters
 	num_realisations = 10
 	episode_len = 10000
+
+	# Init Q and N
+	qu = np.zeros(K)
+	nu = np.zeros(K)
+
+	# Init r_mean
 	r_mean = np.zeros(episode_len)
-	for age in range(0, num_realisations):
-		# Init Q and N
+	for realization in range(0, num_realisations):
+		# Re-Init Q and N
 		qu = np.zeros(K)
 		nu = np.zeros(K)
+#		qu = np.random.normal(qu)
+#		nu = np.random.normal(nu)
+		r_mean += test(episode_len, qu, nu, _bandit=lambda a: bandit(a, ql_star))
 
-		r_mean += test(episode_len, epsilon, qu, nu, bandit=lambda a: bandit(a, ql_star))
+		print(f"qu       = {qu}")
+		print(f"nu       = {nu}")
 
 	print(f"ql_star  = {ql_star}")
-	print(f"qu       = {qu}")
-	print(f"nu       = {nu}")
 
 	pl.plot(r_mean/num_realisations)
 	pl.grid()