129 lines
3.3 KiB
Python
129 lines
3.3 KiB
Python
import numpy as np
|
|
import matplotlib.pyplot as pl
|
|
from tqdm import tqdm
|
|
|
|
# For alternate implementation, see:
|
|
# https://www.kaggle.com/code/parsasam/reinforcement-learning-notes-multi-armed-bandits
|
|
|
|
float_formatter = "{:.3f}".format
|
|
np.set_printoptions(formatter={'float_kind': float_formatter})
|
|
|
|
# Init parameters
|
|
k_arms = 10
|
|
num_realisations = 2000
|
|
episode_len = 1000
|
|
|
|
REWARD_VARIANCE = 1.0
|
|
|
|
def simple_max(Q, N, t, _tie_break):
|
|
am = np.argmax(Q + _tie_break[t])
|
|
return am
|
|
# fm = Q == Q.max()
|
|
# ffm = np.flatnonzero(fm)
|
|
# return np.random.choice(ffm) # breaking ties randomly
|
|
|
|
def test(_k_arms: int, _episode_len: int, _param: tuple[float, float], ql_star, _tie_break) -> np.array:
|
|
_epsilon = _param[0] # Anti-greediness (ability to explore)
|
|
_rho = _param[1] # reduce epsilon with age
|
|
|
|
rewards = np.zeros(_episode_len)
|
|
actions = np.zeros(_episode_len)
|
|
|
|
# Init Q and N
|
|
_qu = np.zeros(_k_arms)
|
|
_nu = np.zeros(_k_arms)
|
|
|
|
# Calc z in advance
|
|
z = np.random.uniform(size=_episode_len)
|
|
|
|
# Calc a in advance
|
|
_a_expl = np.random.randint(_k_arms, size=_episode_len)
|
|
|
|
# Calc rewards in advance
|
|
_reward_z = REWARD_VARIANCE * np.random.normal(size=_episode_len)
|
|
|
|
best_action = np.argmax(ql_star)
|
|
for j in range(0, _episode_len):
|
|
# choose action
|
|
if z[j] < _epsilon:
|
|
_a = _a_expl[j]
|
|
else:
|
|
_a = simple_max(_qu, _nu, j, _tie_break)
|
|
|
|
# get reward from bandit
|
|
_reward = _reward_z[j] + ql_star[_a]
|
|
|
|
# calc
|
|
_nu[_a] = _nu[_a] + 1
|
|
_qu[_a] = _qu[_a] + (_reward - _qu[_a]) / _nu[_a]
|
|
|
|
# Reduce tendency to explore with number of steps (or with age for humans)
|
|
_epsilon = _epsilon * (1 - _rho)
|
|
|
|
# Statistics
|
|
rewards[j] += _reward
|
|
|
|
if _a == best_action:
|
|
actions[j] += 1
|
|
|
|
return rewards, actions
|
|
|
|
|
|
if __name__ == '__main__':
|
|
q_star = np.random.normal(0, 1, (num_realisations, k_arms))
|
|
arms = np.zeros((num_realisations, k_arms))
|
|
for i in range(k_arms):
|
|
arms[:,i] = np.random.normal(q_star[0, i], 1, num_realisations) # first problem as a sample
|
|
|
|
pl.figure(figsize=(12, 8))
|
|
pl.ylabel('Rewards distribution')
|
|
pl.xlabel('Actions')
|
|
pl.xticks(range(1, k_arms+1))
|
|
pl.yticks(np.arange(-5, 5, 0.5))
|
|
pl.violinplot(arms, positions=range(1, k_arms+1), showmedians=True)
|
|
|
|
pl.figure(figsize=(12, 8))
|
|
params = [(0.0, 0.0), (0.01, 0.0), (0.1, 0.0)]
|
|
legend = []
|
|
for param in params:
|
|
# Init stats
|
|
r_mean = np.zeros(episode_len)
|
|
a_mean = np.zeros(episode_len)
|
|
for k in tqdm(range(0, num_realisations)):
|
|
# init bandits with different biases for shifting reward probability
|
|
# -> expected reward q*(a)
|
|
tie_break = 0.05 * np.random.normal(size=(episode_len, k_arms))
|
|
r, a = test(_k_arms=k_arms, _episode_len=episode_len, _param=param, ql_star=q_star[k], _tie_break=tie_break)
|
|
r_mean += r
|
|
a_mean += a
|
|
|
|
legend.append(f"Param {param}")
|
|
|
|
pl.subplot(2, 1, 1)
|
|
pl.plot(r_mean/num_realisations)
|
|
|
|
pl.subplot(2, 1, 2)
|
|
pl.plot(100*a_mean/num_realisations)
|
|
|
|
pl.subplot(2, 1, 1)
|
|
pl.title(f"E(R) over {num_realisations} realizations, num. bandit arms: K={k_arms}")
|
|
pl.legend(legend)
|
|
pl.grid()
|
|
ax = pl.gca()
|
|
ax.set_xlim([-episode_len / 10, episode_len])
|
|
ax.set_ylim([0, 1.6])
|
|
pl.ylabel("E(R)")
|
|
|
|
pl.subplot(2, 1, 2)
|
|
pl.legend(legend)
|
|
pl.grid()
|
|
ax = pl.gca()
|
|
ax.set_xlim([-episode_len / 10, episode_len])
|
|
ax.set_ylim([0, 100])
|
|
pl.ylabel("Percentage of optimal actions")
|
|
pl.ylabel("%")
|
|
|
|
pl.xlabel("Episode")
|
|
|
|
pl.show()
|