diff --git a/k-bandits.py b/k-bandits.py index 4012535..8a751ce 100644 --- a/k-bandits.py +++ b/k-bandits.py @@ -7,23 +7,26 @@ np.set_printoptions(formatter={'float_kind': float_formatter}) K = 10 - def bandit(a: int, bias: np.array): zn = np.random.normal() + bias[a] return zn -def test(_episode_len: int, _qu: np.array, _nu: np.array, _bandit: Callable) -> np.array: - _epsilon = 0.5 # Anti-greediness (ability to explore) - _rho = 0.005 # reduce epsilon with age +def test(_episode_len: int, _qu: np.array, _nu: np.array, _bandit: Callable, _param: tuple[float, float]) -> np.array: + _epsilon = _param[0] # Anti-greediness (ability to explore) + _rho = _param[1] # reduce epsilon with age _r_mean = [] + + # Calc z in advance + z = np.random.uniform(size=_episode_len) + + # Calc a_expl in advance + a_expl = np.random.randint(low=0, high=K, size=_episode_len) + for j in range(0, _episode_len): + a = a_expl[j] # choose action - z = np.random.uniform() - if z <= _epsilon: - # Choose random action - a = np.random.randint(low=0, high=K) - else: + if z[j] > _epsilon: # Choose best action a = np.argmax(_qu) @@ -42,27 +45,34 @@ def test(_episode_len: int, _qu: np.array, _nu: np.array, _bandit: Callable) -> if __name__ == '__main__': + # Init parameters num_realisations = 2000 episode_len = 1000 - # Init r_mean - r_mean = np.zeros(episode_len) - for realization in range(0, num_realisations): - # init bandits with different biases for shifting reward probability - # -> expected reward q*(a) - ql_star = np.random.uniform(-1.5, +1.5, K) + params = [(0.0, 0.0), (0.01, 0.002), (0.1, 0.002)] + legend = [] + for param in params: + # Init r_mean + r_mean = np.zeros(episode_len) + for realization in range(0, num_realisations): + # init bandits with different biases for shifting reward probability + # -> expected reward q*(a) + ql_star = np.random.uniform(-1.5, +1.5, K) - # Init Q and N - qu = np.zeros(K) - nu = np.zeros(K) - r_mean += test(episode_len, qu, nu, _bandit=lambda a: bandit(a, ql_star)) + # Init Q and N + qu = np.zeros(K) + nu = np.zeros(K) + r_mean += test(episode_len, qu, nu, _bandit=lambda a: bandit(a, ql_star), _param=param) + + pl.plot(r_mean/num_realisations) + legend.append(f"Param {param}") - pl.plot(r_mean/num_realisations) pl.grid() pl.title(f"Norm. E(R), num. realizations: Z={num_realisations}, num. bandits: K={K}") pl.xlabel("Episode") pl.ylabel("E(R)/Z") + pl.legend(legend) ax = pl.gca() ax.set_xlim([-episode_len/10, episode_len]) ax.set_ylim([0, 1.5])