diff --git a/k-bandits.py b/k-bandits.py index 43e7d67..4012535 100644 --- a/k-bandits.py +++ b/k-bandits.py @@ -5,20 +5,18 @@ from typing import Callable float_formatter = "{:.3f}".format np.set_printoptions(formatter={'float_kind': float_formatter}) -K = 18 +K = 10 def bandit(a: int, bias: np.array): - zn = np.random.uniform() + bias[a] + zn = np.random.normal() + bias[a] return zn def test(_episode_len: int, _qu: np.array, _nu: np.array, _bandit: Callable) -> np.array: - _epsilon = 0.5 # Anti-greediness + _epsilon = 0.5 # Anti-greediness (ability to explore) _rho = 0.005 # reduce epsilon with age - _r_sum = 0 _r_mean = [] - _n = 0 for j in range(0, _episode_len): # choose action z = np.random.uniform() @@ -38,41 +36,28 @@ def test(_episode_len: int, _qu: np.array, _nu: np.array, _bandit: Callable) -> _epsilon = _epsilon * (1 - _rho) # Statistics - _r_sum += r - _n += 1 - _r_mean.append(_r_sum/_n) + _r_mean.append(r) return np.array(_r_mean) if __name__ == '__main__': - # init bandits with different biases for shifting reward probability - # -> expected reward q*(a) - ql_star = np.array(np.linspace(-0.5, +0.5, K)) - # Init parameters - num_realisations = 10 - episode_len = 10000 - - # Init Q and N - qu = np.zeros(K) - nu = np.zeros(K) + num_realisations = 2000 + episode_len = 1000 # Init r_mean r_mean = np.zeros(episode_len) for realization in range(0, num_realisations): - # Re-Init Q and N + # init bandits with different biases for shifting reward probability + # -> expected reward q*(a) + ql_star = np.random.uniform(-1.5, +1.5, K) + + # Init Q and N qu = np.zeros(K) nu = np.zeros(K) -# qu = np.random.normal(qu) -# nu = np.random.normal(nu) r_mean += test(episode_len, qu, nu, _bandit=lambda a: bandit(a, ql_star)) - print(f"qu = {qu}") - print(f"nu = {nu}") - - print(f"ql_star = {ql_star}") - pl.plot(r_mean/num_realisations) pl.grid() pl.title(f"Norm. E(R), num. realizations: Z={num_realisations}, num. bandits: K={K}") @@ -80,6 +65,6 @@ if __name__ == '__main__': pl.ylabel("E(R)/Z") ax = pl.gca() ax.set_xlim([-episode_len/10, episode_len]) - ax.set_ylim([0, 1]) + ax.set_ylim([0, 1.5]) pl.show()