From efcb5a65a1a76d907d9ea250927daff072461d2b Mon Sep 17 00:00:00 2001
From: Jens Ahrensfeld <jens@jayfield.org>
Date: Sun, 29 Dec 2019 16:11:46 +0000
Subject: [PATCH] Initial import

git-svn-id: http://moon:8086/svn/projects/RL-lab@339 fda53097-d464-4ada-af97-ba876c37ca34
---
 .idea/RL-lab.iml    |  11 +++
 .idea/misc.xml      |   4 +
 .idea/modules.xml   |   8 ++
 .idea/workspace.xml | 196 ++++++++++++++++++++++++++++++++++++++++++++
 rl_lab.py           |  89 ++++++++++++++++++++
 5 files changed, 308 insertions(+)
 create mode 100644 .idea/RL-lab.iml
 create mode 100644 .idea/misc.xml
 create mode 100644 .idea/modules.xml
 create mode 100644 .idea/workspace.xml
 create mode 100644 rl_lab.py

diff --git a/.idea/RL-lab.iml b/.idea/RL-lab.iml
new file mode 100644
index 0000000..6711606
--- /dev/null
+++ b/.idea/RL-lab.iml
@@ -0,0 +1,11 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<module type="PYTHON_MODULE" version="4">
+  <component name="NewModuleRootManager">
+    <content url="file://$MODULE_DIR$" />
+    <orderEntry type="inheritedJdk" />
+    <orderEntry type="sourceFolder" forTests="false" />
+  </component>
+  <component name="TestRunnerService">
+    <option name="PROJECT_TEST_RUNNER" value="Unittests" />
+  </component>
+</module>
\ No newline at end of file
diff --git a/.idea/misc.xml b/.idea/misc.xml
new file mode 100644
index 0000000..a2e120d
--- /dev/null
+++ b/.idea/misc.xml
@@ -0,0 +1,4 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project version="4">
+  <component name="ProjectRootManager" version="2" project-jdk-name="Python 3.7" project-jdk-type="Python SDK" />
+</project>
\ No newline at end of file
diff --git a/.idea/modules.xml b/.idea/modules.xml
new file mode 100644
index 0000000..45d7b12
--- /dev/null
+++ b/.idea/modules.xml
@@ -0,0 +1,8 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project version="4">
+  <component name="ProjectModuleManager">
+    <modules>
+      <module fileurl="file://$PROJECT_DIR$/.idea/RL-lab.iml" filepath="$PROJECT_DIR$/.idea/RL-lab.iml" />
+    </modules>
+  </component>
+</project>
\ No newline at end of file
diff --git a/.idea/workspace.xml b/.idea/workspace.xml
new file mode 100644
index 0000000..66d4cb5
--- /dev/null
+++ b/.idea/workspace.xml
@@ -0,0 +1,196 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project version="4">
+  <component name="ChangeListManager">
+    <list default="true" id="b130113f-81e3-4dcd-82b8-c1ebcf414021" name="Default Changelist" comment="" />
+    <option name="EXCLUDED_CONVERTED_TO_IGNORED" value="true" />
+    <option name="SHOW_DIALOG" value="false" />
+    <option name="HIGHLIGHT_CONFLICTS" value="true" />
+    <option name="HIGHLIGHT_NON_ACTIVE_CHANGELIST" value="false" />
+    <option name="LAST_RESOLUTION" value="IGNORE" />
+  </component>
+  <component name="FileEditorManager">
+    <leaf>
+      <file pinned="false" current-in-tab="true">
+        <entry file="file://$PROJECT_DIR$/rl_lab.py">
+          <provider selected="true" editor-type-id="text-editor">
+            <state relative-caret-position="684">
+              <caret line="72" column="35" selection-start-line="72" selection-start-column="35" selection-end-line="72" selection-end-column="35" />
+            </state>
+          </provider>
+        </entry>
+      </file>
+    </leaf>
+  </component>
+  <component name="FileTemplateManagerImpl">
+    <option name="RECENT_TEMPLATES">
+      <list>
+        <option value="Python Script" />
+      </list>
+    </option>
+  </component>
+  <component name="FindInProjectRecents">
+    <findStrings>
+      <find>P_door</find>
+      <find>pos</find>
+    </findStrings>
+  </component>
+  <component name="IdeDocumentHistory">
+    <option name="CHANGED_PATHS">
+      <list>
+        <option value="$PROJECT_DIR$/rl_lab.py" />
+      </list>
+    </option>
+  </component>
+  <component name="ProjectFrameBounds" extendedState="6">
+    <option name="x" value="148" />
+    <option name="y" value="59" />
+    <option name="width" value="1370" />
+    <option name="height" value="1021" />
+  </component>
+  <component name="ProjectView">
+    <navigator proportions="" version="1">
+      <foldersAlwaysOnTop value="true" />
+    </navigator>
+    <panes>
+      <pane id="ProjectPane">
+        <subPane>
+          <expand>
+            <path>
+              <item name="RL-lab" type="b2602c69:ProjectViewProjectNode" />
+              <item name="RL-lab" type="462c0819:PsiDirectoryNode" />
+            </path>
+          </expand>
+          <select />
+        </subPane>
+      </pane>
+      <pane id="Scope" />
+    </panes>
+  </component>
+  <component name="RunDashboard">
+    <option name="ruleStates">
+      <list>
+        <RuleState>
+          <option name="name" value="ConfigurationTypeDashboardGroupingRule" />
+        </RuleState>
+        <RuleState>
+          <option name="name" value="StatusDashboardGroupingRule" />
+        </RuleState>
+      </list>
+    </option>
+  </component>
+  <component name="RunManager">
+    <configuration name="rl_lab" type="PythonConfigurationType" factoryName="Python" temporary="true">
+      <module name="RL-lab" />
+      <option name="INTERPRETER_OPTIONS" value="" />
+      <option name="PARENT_ENVS" value="true" />
+      <envs>
+        <env name="PYTHONUNBUFFERED" value="1" />
+      </envs>
+      <option name="SDK_HOME" value="" />
+      <option name="WORKING_DIRECTORY" value="$PROJECT_DIR$" />
+      <option name="IS_MODULE_SDK" value="true" />
+      <option name="ADD_CONTENT_ROOTS" value="true" />
+      <option name="ADD_SOURCE_ROOTS" value="true" />
+      <option name="SCRIPT_NAME" value="$PROJECT_DIR$/rl_lab.py" />
+      <option name="PARAMETERS" value="" />
+      <option name="SHOW_COMMAND_LINE" value="false" />
+      <option name="EMULATE_TERMINAL" value="false" />
+      <option name="MODULE_MODE" value="false" />
+      <option name="REDIRECT_INPUT" value="false" />
+      <option name="INPUT_FILE" value="" />
+      <method v="2" />
+    </configuration>
+    <recent_temporary>
+      <list>
+        <item itemvalue="Python.rl_lab" />
+      </list>
+    </recent_temporary>
+  </component>
+  <component name="SvnConfiguration">
+    <configuration />
+  </component>
+  <component name="TaskManager">
+    <task active="true" id="Default" summary="Default task">
+      <changelist id="b130113f-81e3-4dcd-82b8-c1ebcf414021" name="Default Changelist" comment="" />
+      <created>1577548737198</created>
+      <option name="number" value="Default" />
+      <option name="presentableId" value="Default" />
+      <updated>1577548737198</updated>
+    </task>
+    <servers />
+  </component>
+  <component name="ToolWindowManager">
+    <frame x="70" y="27" width="1850" height="1053" extended-state="6" />
+    <editor active="true" />
+    <layout>
+      <window_info id="Favorites" side_tool="true" />
+      <window_info content_ui="combo" id="Project" order="0" visible="true" weight="0.24972618" />
+      <window_info id="Structure" order="1" side_tool="true" weight="0.25" />
+      <window_info anchor="bottom" id="Version Control" />
+      <window_info anchor="bottom" id="Python Console" weight="0.32897604" />
+      <window_info anchor="bottom" id="Terminal" weight="0.32897604" />
+      <window_info anchor="bottom" id="Event Log" side_tool="true" />
+      <window_info anchor="bottom" id="Message" order="0" />
+      <window_info anchor="bottom" id="Find" order="1" />
+      <window_info anchor="bottom" id="Run" order="2" weight="0.32897604" />
+      <window_info anchor="bottom" id="Debug" order="3" weight="0.39978212" />
+      <window_info anchor="bottom" id="Cvs" order="4" weight="0.25" />
+      <window_info anchor="bottom" id="Inspection" order="5" weight="0.4" />
+      <window_info anchor="bottom" id="TODO" order="6" />
+      <window_info anchor="right" id="Commander" internal_type="SLIDING" order="0" type="SLIDING" weight="0.4" />
+      <window_info anchor="right" id="Ant Build" order="1" weight="0.25" />
+      <window_info anchor="right" content_ui="combo" id="Hierarchy" order="2" weight="0.25" />
+    </layout>
+  </component>
+  <component name="editorHistoryManager">
+    <entry file="file:///usr/lib/python3/dist-packages/numpy/core/fromnumeric.py">
+      <provider selected="true" editor-type-id="text-editor">
+        <state relative-caret-position="188">
+          <caret line="3373" column="4" selection-start-line="3373" selection-start-column="4" selection-end-line="3373" selection-end-column="4" />
+        </state>
+      </provider>
+    </entry>
+    <entry file="file://$USER_HOME$/.PyCharmCE2019.1/system/python_stubs/-1247971762/numpy/random/mtrand.py">
+      <provider selected="true" editor-type-id="text-editor">
+        <state relative-caret-position="-54">
+          <caret line="2776" column="4" selection-start-line="2776" selection-start-column="4" selection-end-line="2776" selection-end-column="4" />
+        </state>
+      </provider>
+    </entry>
+    <entry file="file://$USER_HOME$/.PyCharmCE2019.1/system/python_stubs/-1247971762/numpy/core/_multiarray_umath.py">
+      <provider selected="true" editor-type-id="text-editor">
+        <state relative-caret-position="188">
+          <caret line="10668" column="8" selection-start-line="10668" selection-start-column="8" selection-end-line="10668" selection-end-column="8" />
+        </state>
+      </provider>
+    </entry>
+    <entry file="file://$APPLICATION_HOME_DIR$/helpers/pydev/_pydev_imps/_pydev_execfile.py">
+      <provider selected="true" editor-type-id="text-editor">
+        <state relative-caret-position="323">
+          <caret line="17" selection-start-line="17" selection-end-line="17" />
+        </state>
+      </provider>
+    </entry>
+    <entry file="file://$APPLICATION_HOME_DIR$/helpers/pydev/pydevd.py">
+      <provider selected="true" editor-type-id="text-editor">
+        <state relative-caret-position="352">
+          <caret line="1740" selection-start-line="1740" selection-end-line="1740" />
+        </state>
+      </provider>
+    </entry>
+    <entry file="file:///usr/lib/python3/dist-packages/apport_python_hook.py">
+      <provider selected="true" editor-type-id="text-editor">
+        <state relative-caret-position="185">
+          <caret line="151" selection-start-line="151" selection-end-line="151" />
+        </state>
+      </provider>
+    </entry>
+    <entry file="file://$PROJECT_DIR$/rl_lab.py">
+      <provider selected="true" editor-type-id="text-editor">
+        <state relative-caret-position="684">
+          <caret line="72" column="35" selection-start-line="72" selection-start-column="35" selection-end-line="72" selection-end-column="35" />
+        </state>
+      </provider>
+    </entry>
+  </component>
+</project>
\ No newline at end of file
diff --git a/rl_lab.py b/rl_lab.py
new file mode 100644
index 0000000..16cf899
--- /dev/null
+++ b/rl_lab.py
@@ -0,0 +1,89 @@
+import numpy as np
+
+
+lab_size_x = 5
+lab_size_y = 5
+num_doors_per_room = 4
+
+
+P_door = np.ndarray((lab_size_x, lab_size_y, num_doors_per_room))
+P_door = np.random.uniform(0,1,P_door.shape)
+
+print('P_door.shape = ', P_door.shape)
+lab = np.rint(np.random.uniform(0, num_doors_per_room-1, size=(lab_size_x, lab_size_y)))
+
+
+#    1
+#    |
+# 4 -0- 2
+#    |
+#    3
+
+move = np.array([(0,0), (-1,0), (0,1), (1,0), (0,-1)])
+lab = np.array([
+	[3, 0, 0, 0, -1],
+	[3, 0, 0, 2, 1],
+	[2, 2, 3, 1, 4],
+	[3, 4, 4, 2, 1],
+	[2, 2, 2, 1, 0]
+])
+
+def choose_door(Pn):
+	Pnorm = np.cumsum(Pn)
+	size = len(Pn)
+	Z = np.random.uniform(0,1)
+
+	result = None
+	for n in range(0, size):
+		if Z <= Pnorm[n]:
+			result = n
+			break
+
+	return result
+
+def run(pos, N_trials, learning_rate):
+	lab_visited = np.zeros(P_door.shape)
+	moves_needed = 0
+	for n in range(0, N_trials):
+
+#		print ("Pos={}".format(pos))
+		if lab[pos] == -1:
+			break
+
+		P = P_door[pos]
+		Pn = P/np.sum(P)
+
+		while True:
+			door = choose_door(Pn)
+			pos_new = tuple(pos + move[door + 1])
+
+			if pos_new[0] < 0 or pos_new[1] < 0:
+				continue
+
+			if pos_new[0] >= lab_size_x or pos_new[1] >= lab_size_y:
+				continue
+
+			if lab[pos_new] == 0:
+				continue
+
+			break
+
+		moves_needed += 1
+		k = lab_visited[pos][door]
+		P[door] = max(0, P[door] - learning_rate)
+		P_door[pos] = P
+
+		pos = pos_new
+		lab_visited[pos][door] += 1
+
+	return moves_needed, lab_visited
+
+for i in range(0, 100):
+	moves_needed, lab_visited = run(pos=(0,0), N_trials=1000, learning_rate=0.001)
+	print ('Visited map after {} trials: {}'.format(moves_needed, lab_visited))
+	print ('P_door after {} trials: {}'.format(moves_needed, P_door))
+
+moves_needed, lab_visited = run(pos=(0,0), N_trials=1000, learning_rate=0.0)
+print ('Finished after {} trials'.format(moves_needed))
+
+#	P_door[pos, door] += learning_rate