81 lines
1.9 KiB
Python
81 lines
1.9 KiB
Python
import numpy as np
|
|
import matplotlib.pyplot as pl
|
|
from typing import Callable
|
|
|
|
float_formatter = "{:.3f}".format
|
|
np.set_printoptions(formatter={'float_kind': float_formatter})
|
|
|
|
K = 10
|
|
|
|
def bandit(a: int, bias: np.array):
|
|
zn = np.random.normal() + bias[a]
|
|
return zn
|
|
|
|
|
|
def test(_episode_len: int, _qu: np.array, _nu: np.array, _bandit: Callable, _param: tuple[float, float]) -> np.array:
|
|
_epsilon = _param[0] # Anti-greediness (ability to explore)
|
|
_rho = _param[1] # reduce epsilon with age
|
|
_r_mean = []
|
|
|
|
# Calc z in advance
|
|
z = np.random.uniform(size=_episode_len)
|
|
|
|
# Calc a_expl in advance
|
|
a_expl = np.random.randint(low=0, high=K, size=_episode_len)
|
|
|
|
for j in range(0, _episode_len):
|
|
a = a_expl[j]
|
|
# choose action
|
|
if z[j] > _epsilon:
|
|
# Choose best action
|
|
a = np.argmax(_qu)
|
|
|
|
# get reward from bandit
|
|
r = _bandit(a)
|
|
_nu[a] = _nu[a] + 1
|
|
_qu[a] = _qu[a] + (r - _qu[a]) / _nu[a]
|
|
|
|
# Reduce tendency to explore with number of steps (or with age for humans)
|
|
_epsilon = _epsilon * (1 - _rho)
|
|
|
|
# Statistics
|
|
_r_mean.append(r)
|
|
|
|
return np.array(_r_mean)
|
|
|
|
|
|
if __name__ == '__main__':
|
|
|
|
# Init parameters
|
|
num_realisations = 2000
|
|
episode_len = 1000
|
|
|
|
params = [(0.0, 0.0), (0.01, 0.002), (0.1, 0.002)]
|
|
legend = []
|
|
for param in params:
|
|
# Init r_mean
|
|
r_mean = np.zeros(episode_len)
|
|
for realization in range(0, num_realisations):
|
|
# init bandits with different biases for shifting reward probability
|
|
# -> expected reward q*(a)
|
|
ql_star = np.random.uniform(-1.5, +1.5, K)
|
|
|
|
# Init Q and N
|
|
qu = np.zeros(K)
|
|
nu = np.zeros(K)
|
|
r_mean += test(episode_len, qu, nu, _bandit=lambda a: bandit(a, ql_star), _param=param)
|
|
|
|
pl.plot(r_mean/num_realisations)
|
|
legend.append(f"Param {param}")
|
|
|
|
pl.grid()
|
|
pl.title(f"Norm. E(R), num. realizations: Z={num_realisations}, num. bandits: K={K}")
|
|
pl.xlabel("Episode")
|
|
pl.ylabel("E(R)/Z")
|
|
pl.legend(legend)
|
|
ax = pl.gca()
|
|
ax.set_xlim([-episode_len/10, episode_len])
|
|
ax.set_ylim([0, 1.5])
|
|
|
|
pl.show()
|