127 lines
3.3 KiB
Python
127 lines
3.3 KiB
Python
import numpy as np
|
|
from a_player import APlayer
|
|
from numpy.random import uniform
|
|
from state import get_potential_moves
|
|
|
|
|
|
class MachinePlayer(APlayer):
|
|
class Params:
|
|
def __init__(self, p_explore, alpha, do_learn_from_history=True):
|
|
self.p_exp = p_explore
|
|
self.alpha = alpha
|
|
self.do_learn_from_history = do_learn_from_history
|
|
|
|
def __init__(self, mark, params: Params, values):
|
|
APlayer.__init__(self, mark)
|
|
self.params = params
|
|
self.values = {}
|
|
self.with_debug = False
|
|
self.values = values
|
|
self.episode_history = None
|
|
|
|
def set_debug(self, with_debug):
|
|
self.with_debug = with_debug
|
|
|
|
def print_state_table(self):
|
|
count = 0
|
|
for key in self.values:
|
|
print(f"{self.mark}: {count:05d}: {key} = {self.values[key]:0.3f}")
|
|
count += 1
|
|
|
|
@staticmethod
|
|
def to_key(state: np.array):
|
|
key = ''
|
|
for st in state.reshape(state.size):
|
|
key += st
|
|
return key
|
|
|
|
def get_value(self, state: np.array):
|
|
key = self.to_key(state)
|
|
try:
|
|
result = self.values[key]
|
|
except KeyError:
|
|
result = 0.5
|
|
|
|
return result
|
|
|
|
def set_value(self, state: np.array, value):
|
|
self.values[self.to_key(state)] = value
|
|
|
|
def end_game(self, state: np.array, value):
|
|
self.set_value(self.state, value)
|
|
if self.params.do_learn_from_history:
|
|
self.learn_from_history()
|
|
|
|
def new_game(self, state: np.array):
|
|
self.state = state
|
|
self.episode_history = []
|
|
|
|
def learn_from_history(self):
|
|
rev_hist = list(reversed(self.episode_history))
|
|
for hist in rev_hist:
|
|
d = self.calc_value(hist['state'], hist['next_state'])
|
|
if not hist['is_exp']:
|
|
self.set_value(hist['state'], d)
|
|
|
|
def calc_value(self, state: np.array, next_state: np.array):
|
|
v0 = self.get_value(state)
|
|
v1 = self.get_value(next_state)
|
|
d = v0 + self.params.alpha*(v1-v0)
|
|
if self.with_debug:
|
|
print(f"{self.mark}: Learned {d:0.3f}")
|
|
|
|
return d
|
|
|
|
def get_best_move(self, state: np.array, moves: np.array):
|
|
best_move = None
|
|
best_value = -1
|
|
values = np.array([])
|
|
for move in moves:
|
|
# create hypothetical next state
|
|
state_next = self.to_state(state.copy(), move)
|
|
# evaluate value
|
|
value = self.get_value(state_next)
|
|
if best_value < value:
|
|
best_value = value
|
|
best_move = move
|
|
values = np.append(values, value)
|
|
|
|
return best_move, best_value, values
|
|
|
|
def move(self, state: np.array):
|
|
# get possible move
|
|
moves = get_potential_moves(state)
|
|
if moves.size == 0:
|
|
return state, False
|
|
|
|
best_move, best_value, values = self.get_best_move(state, moves)
|
|
next_move = best_move
|
|
is_exp = False
|
|
# Randomly perform exploratory move
|
|
if uniform() < self.params.p_exp:
|
|
index = np.random.randint(len(moves))
|
|
next_move = moves[index]
|
|
is_exp = best_move != next_move
|
|
|
|
# Finally create next state
|
|
next_state = self.to_state(state.copy(), next_move)
|
|
|
|
# Maintain history
|
|
self.episode_history.append({'state': self.state, 'next_state': next_state, "is_exp": is_exp})
|
|
|
|
if self.with_debug:
|
|
print(f"{self.mark}: Values = {values}")
|
|
print(f"{self.mark}: Moves = {moves+1}")
|
|
print(f"{self.mark}: Best move = {best_move+1}")
|
|
print(f"{self.mark}: Next move = {next_move+1}, is_exp={is_exp}")
|
|
|
|
# Learn
|
|
if not (self.params.do_learn_from_history or is_exp):
|
|
d = self.calc_value(self.state, next_state)
|
|
self.set_value(self.state, d)
|
|
|
|
self.state = next_state
|
|
|
|
return next_state, True
|
|
|