commit e0503564e0f4b6e7793e9d36ccf05a7677435d5b
Author: Jens Ahrensfeld <jens@jayfield.org>
Date:   Sat Jun 15 12:15:25 2024 +0200

    Initial commit

diff --git a/.idea/.gitignore b/.idea/.gitignore
new file mode 100644
index 0000000..26d3352
--- /dev/null
+++ b/.idea/.gitignore
@@ -0,0 +1,3 @@
+# Default ignored files
+/shelf/
+/workspace.xml
diff --git a/.idea/inspectionProfiles/profiles_settings.xml b/.idea/inspectionProfiles/profiles_settings.xml
new file mode 100644
index 0000000..105ce2d
--- /dev/null
+++ b/.idea/inspectionProfiles/profiles_settings.xml
@@ -0,0 +1,6 @@
+<component name="InspectionProjectProfileManager">
+  <settings>
+    <option name="USE_PROJECT_PROFILE" value="false" />
+    <version value="1.0" />
+  </settings>
+</component>
\ No newline at end of file
diff --git a/.idea/k-bandits.iml b/.idea/k-bandits.iml
new file mode 100644
index 0000000..2c80e12
--- /dev/null
+++ b/.idea/k-bandits.iml
@@ -0,0 +1,10 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<module type="PYTHON_MODULE" version="4">
+  <component name="NewModuleRootManager">
+    <content url="file://$MODULE_DIR$">
+      <excludeFolder url="file://$MODULE_DIR$/.venv" />
+    </content>
+    <orderEntry type="inheritedJdk" />
+    <orderEntry type="sourceFolder" forTests="false" />
+  </component>
+</module>
\ No newline at end of file
diff --git a/.idea/misc.xml b/.idea/misc.xml
new file mode 100644
index 0000000..3a900f8
--- /dev/null
+++ b/.idea/misc.xml
@@ -0,0 +1,7 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project version="4">
+  <component name="Black">
+    <option name="sdkName" value="Python 3.12 (k-bandits)" />
+  </component>
+  <component name="ProjectRootManager" version="2" project-jdk-name="Python 3.12 (k-bandits)" project-jdk-type="Python SDK" />
+</project>
\ No newline at end of file
diff --git a/.idea/modules.xml b/.idea/modules.xml
new file mode 100644
index 0000000..54fa38e
--- /dev/null
+++ b/.idea/modules.xml
@@ -0,0 +1,8 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project version="4">
+  <component name="ProjectModuleManager">
+    <modules>
+      <module fileurl="file://$PROJECT_DIR$/.idea/k-bandits.iml" filepath="$PROJECT_DIR$/.idea/k-bandits.iml" />
+    </modules>
+  </component>
+</project>
\ No newline at end of file
diff --git a/README.md b/README.md
new file mode 100644
index 0000000..e69de29
diff --git a/k-bandits.py b/k-bandits.py
new file mode 100644
index 0000000..cac169f
--- /dev/null
+++ b/k-bandits.py
@@ -0,0 +1,58 @@
+import numpy as np
+import matplotlib.pyplot as pl
+
+float_formatter = "{:.3f}".format
+np.set_printoptions(formatter={'float_kind': float_formatter})
+
+K = 10
+
+
+def bandit(bias):
+	zn = np.random.uniform() + bias
+	return zn
+
+
+if __name__ == '__main__':
+
+	kv = 1.0
+	epsilon = 0.1
+	rho = 0.01			# reduce epsilon with age
+	alpha = 0.1
+
+	qu = np.zeros(K)
+	nu = np.zeros(K)
+	r_vec = []
+	# init bandits with different biases for shifting reward probability
+	# -> expected reward q*(a)
+	ql_star = [-0.4, -0.3, -0.2, -0.1, 0.0, +0.1, +0.2, +0.3, +0.4, +0.5]
+
+	N = 1000
+	n_ages = 1000
+	for age in range(0, n_ages):
+		r_sum = 0
+		for j in range(0, N):
+			# choose action
+			z = np.random.uniform()
+			if z <= epsilon:
+				# Choose random action
+				a = np.random.randint(low=0, high=K)
+			else:
+				# Choose best action
+				a = np.argmax(qu)
+
+			# get reward from bandit
+			r = bandit(ql_star[a])
+			nu[a] = nu[a] + 1
+			qu[a] = qu[a] + alpha*(r - qu[a])/nu[a]
+			r_sum += r
+		r_vec.append(r_sum/N)
+		# Reduce tendency to explore with number of steps (or with age for humans)
+		epsilon = epsilon*(1-rho)
+
+	print(f"ql_star  = {ql_star}")
+	print(f"qu       = {qu}")
+	print(f"nu       = {nu}")
+
+	pl.plot(np.array(r_vec))
+	pl.grid()
+	pl.show()