- improved

- avoid division by zero
- added buy-only  game

git-svn-id: http://moon:8086/svn/projects/Stock@325 fda53097-d464-4ada-af97-ba876c37ca34

pandas_eval.py

@@ -1,11 +1,13 @@
 import pandas as pd
 import pandas_datareader.data as web
+from pandas_datareader._utils import RemoteDataError
 import numpy as np
 import math
 import matplotlib.pyplot as plt
 from scipy.interpolate import UnivariateSpline
 import datetime as dt
 import os
+import time
 import matplotlib as mpl
 
 # Stock Investors Financial Math
@@ -22,13 +24,14 @@ import matplotlib as mpl
 # https://ntguardian.wordpress.com/2016/09/19/introduction-stock-market-data-python-1
 
 key = '0UO7Z2MVZ2YSQSVE'
-show_range_days = 40
+show_range_days = 5
 show_symbols = ['OHB.DE', 'ITMPF', 'PLUG', 'MOR.DE', 'CSCO', 'ERCA.DE', 'AVGO', 'DIS', 'UBSFF', 'DHER.DE', 'AIR']
 show_symbols = ['CSCO']
 #show_symbols = ['OHB.DE']
 #show_symbols = ['UBSFF']
-#show_symbols = ['DHER.DE']
+show_symbols = ['DHER.DE']
-show_symbols = ['WDI.DE']
+#show_symbols = ['WDI.DE']
+#show_symbols = ['EVT.DE']
 show_symbols = []
 k_euro = 1 / 1.11
 ema_alpha = 0.75
@@ -39,6 +42,16 @@ fetch_on_outdated = True
 mpl.rc('figure', max_open_warning = 0)
 
 symbols = {
+
+	'RIG' : {'name' : 'Transocean Ltd.', 'currency' : '$'},
+	'BBIO' : {'name' : 'BridgeBio Pharma, Inc.', 'currency' : '$'},
+	'APA' : {'name' : 'Apache Corporation', 'currency' : '$'},
+	'CBB-PB' : {'name' : 'Cincinnati Bell Inc.', 'currency' : '$'},
+	'NMHLY' : {'name' : 'NMC Health Plc', 'currency' : '$'},
+	'PINS' : {'name' : 'Pinterest', 'currency' : '$'},
+	'TSLA' : {'name' : 'Tesla Inc.', 'currency' : '$'},
+	'ACB' : {'name' : 'Aurora Cannabis Inc.', 'currency' : '$'},
+	'ITCI' : {'name' : 'Intra-Cellular Therapies', 'currency' : '$'},
 	'WDI.DE' : {'name' : 'WireCard', 'currency' : '€'},
 	'AAPL' : {'name' : 'Apple', 'currency' : '$'},
 	'XLNX' : {'name' : 'Xilinx', 'currency' : '$'},
@@ -57,11 +70,11 @@ symbols = {
 	'HD' : {'name' : 'Home Depot', 'currency' : '$'},
 	'AMZN' : {'name' : 'Amazon', 'currency' : '$'},
 	'GOOGL' : {'name' : 'Google', 'currency' : '$'},
-	'FB2A.DE' : {'currency' : '€'},
+	'FB2A.DE' : {'name': 'Facebook Inc.', 'currency' : '€'},
-	'ZIL2.DE' : {'currency' : '€'},
+	'ZIL2.DE' : {'name': 'ElringKlinger', 'currency' : '€'},
-	'SIS.DE' : {'currency' : '€'},
+	'SIS.DE' : {'name': 'First Sensor', 'currency' : '€'},
 	'SIE.DE' : {'name': 'Siemens', 'currency' : '€'},
-	'GFT.DE' : {'currency' : '€'},
+	'GFT.DE' : {'name': 'GFT Technologies', 'currency' : '€'},
 	'AMD.DE' : {'name': 'Advanced Micro Devices', 'currency' : '€'},
 	'CAP.DE' : {'name': 'Encavis', 'currency' : '€'},
 	'ADBE' : {'name': 'Adobe', 'currency' : '$'},
@@ -76,17 +89,17 @@ symbols = {
 	'DAI.DE' : {'name': 'Daimler', 'currency' : '€'},
 	'SHA.DE' : {'name': 'Schaeffler', 'currency' : '€'},
 	'CON.DE' : {'name': 'Continental', 'currency' : '€'},
-	'DHER.DE' : {'name' : 'Delivery Hero', 'currency' : '€'},
+	'DHER.DE' : {'name': 'Delivery Hero', 'currency' : '€'},
 	'BSL.DE' : {'name': 'Basler', 'currency' : '€'},
-	'D6H.DE' : {'currency' : '€'},
+	'D6H.DE' : {'name': 'DATAGROUP', 'currency' : '€'},
-	'TC1.DE' : {'currency' : '€'},
+	'TC1.DE' : {'name': 'Tele Columbus', 'currency' : '€'},
-	'VODI.DE' : {'currency' : '€'},
+	'VODI.DE' : {'name': 'Vodaphone Group', 'currency' : '€'},
-	'ATVI' : {'currency' : '$'},
+	'ATVI' : {'name': 'Activision Blizzard', 'currency' : '$'},
-	'GME' : {'currency' : '$'},
+	'GME' : {'name': 'GameStop', 'currency' : '$'},
 	'NTO.F' : {'name': 'Nintendo', 'currency' : '$'},
 	'UBSFF' : {'name': 'UBI Soft', 'currency' : '$'},
 	'NXPRF' : {'name': 'Nexans', 'currency' : '$'},
-	'IMPUF' : {'name': 'Impala Platinum Holdings', 'currency' : '$'},
+#	'IMPUF' : {'name': 'Impala Platinum Holdings', 'currency' : '$'},
 	'NMPNF' : {'name': 'Northam Platinum', 'currency' : '$'},
 	'ALSMY' : {'name': 'Alstom', 'currency' : '$'},
 	'ARRD.F' : {'name': 'Arcelor Mittal', 'currency' : '$'},
@@ -103,18 +116,18 @@ symbols = {
 	'ITMPF' : {'name': 'ITM Power', 'currency' : '$'},
 	'PLUG' : {'name': 'PlugPower', 'currency' : '$'},
 	'PCELF' : {'name': 'PowerCell', 'currency' : '$'},
-	'SU.PA' : {'currency' : '$'},
+	'SU.PA' : {'name': 'Schneider Electric', 'currency' : '$'},
-#	'SON1.DE' : {'currency' : '€'},
+#	'SON1.DE' : {'name': '', 'currency' : '€'},
 	'STM.DE' : {'name': 'STM Micro', 'currency' : '€'},
 	'BC8.DE' : {'name': 'Bechtle', 'currency' : '€'},
 	'MOR.DE' : {'name' : 'Morphosys', 'currency' : '€'},
-	'BAYN.DE' : {'currency' : '€'},
+	'BAYN.DE' : {'name' : 'Bayer', 'currency' : '€'},
-	'BEI.DE' : {'currency' : '€'},
+	'BEI.DE' : {'name' : 'Beiersdorf', 'currency' : '€'},
-	'EUZ.DE' : {'currency' : '€'},
+	'EUZ.DE' : {'name': 'Eckert & Ziegler', 'currency' : '€'},
 	'SLAB' : {'name': 'Silicon Laboratories', 'currency' : '$'},
 	'SPLK' : {'name' : 'Splunk', 'currency' : '$'},
 	'IAG' : {'name' : 'IAG', 'currency' : '$'},
-	'INTC' : {'currency' : '$'},
+	'INTC' : {'name' : 'Intel Corporation', 'currency' : '$'},
 	'I' : {'name': 'IntelSat', 'currency' : '$'}}
 
 def exponential_moving_average(data, key, alpha=0.5, ic=None):
@@ -207,14 +220,20 @@ def moving_min(data, key, window_days, ic=1e9):
 
 def normalize(data, key, ic=None):
 	result = np.zeros_like(data[key])
+	cumsum = 0
 	if ic is None:
 		prev = data[key][0]
 	else:
 		prev = ic
 	n = 0
 	for v in data[key]:
-		v_n = (v - prev)/prev
+		if prev == 0:
-		result[n] = 100*v_n
+			print (prev)
+		cumsum += 100*(v - prev)/(prev)
+		result[n] = cumsum
+
+		if v > 0:
+			prev = v
 		n += 1
 
 	return result
@@ -244,30 +263,41 @@ def bollinger(data, window_days, f=2):
 	result = {'index' : data['index'], 'upper' : upper, 'mid' : mid, 'lower' : lower}
 	return result
 
-def agent_buy(name, data, cand_window, thresh_max=-10, thresh_min=1):
+def agent_buy(name, data, range_days, cand_window, marker_key='close_n', thresh_max=-10, thresh_min=1, buy_callback=None):
 	N = len(data['index'])
-	buy_list = np.array([None]*N)
+	buy_list = {'index' : np.array([None]*N), 'Buy_dip' : np.array([None]*N)}
 	cand = None
-	for n in range(0, len(data['index'])):
+	do_buy = False
+	for n in range(N-range_days, N):
 		vmax = data['Qmax'][n]
 		vmin = data['Qmin'][n]
-		trend = data['macd_fd'][n]
+		trend = data['macd_fdd'][n]
+		index = data['index'][n]
+		value = data[marker_key][n]
 		if vmin <= thresh_min:
 			if vmax <= thresh_max:
 				cand = n
 				if trend >= 0:
-					print ("{}: Buy on {} at {:0.2f}%".format(name, data['index'][cand], data['close_n'][cand]))
+					do_buy = True
-					y = data['close_n'][n]
+					print ("{}: Buy on {} at {:0.2f}".format(name, index, value))
-					buy_list[n] = y
 					cand = None
 
 		if cand is not None:
 			if n - cand <= cand_window:
 				if trend >= 0:
-					print("{}: Delayed buy on {}".format(name, data['index'][n]))
+					do_buy = True
-					y = data['close_n'][n]
+					former = data[marker_key][cand]
-					buy_list[n] = y
+					print("{}: Delayed buy on {} at {:0.2f} ({:0.2f})".format(name, index, value, former-value))
 					cand = None
+			else:
+				cand = None
+
+		if do_buy:
+			do_buy = False
+			buy_list['index'][n] = index
+			buy_list['Buy_dip'][n] = value
+			if buy_callback is not None:
+				buy_callback({'name': name, 'item': {'value': value, 'date': index}})
 
 	return buy_list
 
@@ -294,6 +324,9 @@ class Title(object):
 		filename = self.symbol.replace('.', '_') + '.h5'
 		fetch = False
 		today = dt.date.today()
+		start = dt.datetime(today.year, 1, 1)
+		end = today
+
 		try:
 			hdf = pd.HDFStore(filename, 'r')
 			hdf.close()
@@ -308,25 +341,36 @@ class Title(object):
 			fetch = True
 
 		if fetch:
-			print ("Fetching \"{}\"".format(self.symbol))
 			# Get stock price via data reader
-			start = dt.datetime(today.year,1,1)
+			while(True):
-			end = dt.date.today()
+				try:
-			data = web.DataReader(self.symbol, "av-daily", start, end, api_key=key)
+					print("Fetching \"{}\"".format(self.symbol))
+					data = web.DataReader(self.symbol, "av-daily", start, end, api_key=key)
+					break
+				except RemoteDataError:
+					for timeout in reversed(range(0, 60)):
+						print ("Try again in {} s".format(timeout))
+						time.sleep(1)
+
 			hdf = pd.HDFStore(filename)
 			hdf[self.symbol] = data
 		else:
 			hdf = pd.HDFStore(filename, 'r')
 
 		data = hdf[self.symbol] * self.currency_corr
-		self.data['index'] = np.array(data.index)
+		N = len(data.index)
-		self.data['close'] = np.array(data['close'])
+		self.data['index'] = np.array(data.index[0:N])
-		self.data['high'] = np.array(data['high'])
+		self.data['close'] = np.array(data['close'][0:N])
-		self.data['low'] = np.array(data['low'])
+		self.data['high'] = np.array(data['high'][0:N])
+		self.data['low'] = np.array(data['low'][0:N])
 
 		hdf.close()
 
-	def analyze(self):
+	def get_latest(self, key):
+		N = len(self.data['index'])
+		return self.data[key][N-1]
+
+	def statistics(self):
 		N = len(self.data['index'])
 		self.data['ema'] = exponential_moving_average(self.data, key='close', alpha=ema_alpha)
 		self.data['sma'] = moving_average(self.data, key='close', window_days=sma_days)
@@ -343,13 +387,23 @@ class Title(object):
 		y_r = self.data['macd']
 		spl = UnivariateSpline(x_r, y_r)
 		spl.set_smoothing_factor(0.25)
+		spl_d = spl.derivative()
+		spl_dd = spl_d.derivative()
 		yf = spl(x_r)
-		yf_d = spl.derivative()(x_r)
+		yf_d = spl_d(x_r)
+		yf_dd = spl_dd(x_r)
 
 		self.data['macd_f'] = np.transpose(yf)
 		self.data['macd_fd'] = np.transpose(yf_d)
+		self.data['macd_fdd'] = np.transpose(yf_dd)
 
-		self.data['Buy'] = agent_buy(self.symbol, self.data, cand_window=q_days)
+	def analyze(self, buy_callback, range_days):
+		return agent_buy(self.symbol, self.data, marker_key='close_n', cand_window=5, range_days=range_days, buy_callback=buy_callback)
+
+	@staticmethod
+	def has_candidate(data, key):
+		result = np.count_nonzero(data[key] != None)
+		return result > 0
 
 	@staticmethod
 	def slice(data, start, stop):
@@ -388,57 +442,80 @@ class Title(object):
 		for key in keys:
 			plt.plot(xr, sliced[key], 'go', label=key)
 
-	def has_candidate(self, key, range_days):
+	def show(self, indicators, figNum=1):
-		N = len(self.data['index'])
+		num_subplots = 4
-		start = max(0, N - range_days)
-		stop = N
-		sliced = Title.slice(self.data, start, stop)
-		result = np.count_nonzero(sliced[key] != None)
-		return result > 0
-
-	def show(self, figNum=1):
-
 		self.fig = plt.figure(figNum)
-		self.ax = plt.subplot(311)
+		self.ax = plt.subplot(100*num_subplots + 10 + 1)
 		self.__plot(self.data, ['min', 'max', 'close_n'])
-		self.__ind(self.data, ['Buy'])
+		self.__ind(indicators, ['Buy_dip'])
 		plt.title('{} ({})'.format(self.name, self.symbol))
 		plt.legend()
 		plt.grid()
 
-		plt.subplot(312)
+		self.ax = plt.subplot(100*num_subplots + 10 + 2)
 #		self.__plot(self.boll, ['lower', 'mid', 'upper'])
 		self.__plot(self.data, ['Qmin', 'Qmax'])
 		plt.legend()
 		plt.grid()
 
 
-		plt.subplot(313)
+		self.ax = plt.subplot(100*num_subplots + 10 + 3)
-		self.__plot(self.data, ['macd', 'macd_f', 'macd_fd'])
+		self.__plot(self.data, ['macd', 'macd_f'])
+		plt.legend()
+		plt.grid()
+
+		self.ax = plt.subplot(100*num_subplots + 10 + 4)
+		self.__plot(self.data, ['macd_fd', 'macd_fdd'])
 		plt.legend()
 		plt.grid()
 
 
+buy_list = {}
+def buy_callback(data):
+	name = data['name']
+	item = data['item']
+	if name not in buy_list:
+		buy_list[name] = {'items': [item]}
+	else:
+		buy_list[name]['items'].append(item)
+
+	print(data)
 
 figNum = 1
-
 if len(show_symbols) > 0:
 	for symbol in show_symbols:
 		if symbol in symbols:
 			title = Title(symbol, symbols[symbol])
 			title.fetch()
-			title.analyze()
+			title.statistics()
-			title.show(figNum=figNum)
+			ind = title.analyze(buy_callback, range_days=show_range_days)
+			title.show(ind, figNum=figNum)
 			figNum += 1
 else:
 	for symbol in symbols:
-		print('-----------------------------------------------')
 		title = Title(symbol, symbols[symbol])
 		title.fetch()
-		title.analyze()
+		title.statistics()
-		if title.has_candidate(key='Buy', range_days=show_range_days):
+		ind = title.analyze(buy_callback, range_days=show_range_days)
-			title.show(figNum=figNum)
+		if Title.has_candidate(ind, 'Buy_dip'):
+			print('-----------------------------------------------')
+			title.show(ind, figNum=figNum)
 			figNum += 1
 
+gain_accum = 0
+num_stocks = 0
+for symbol in buy_list:
+	items = buy_list[symbol]['items']
+	title = Title(symbol, symbols[symbol])
+	title.fetch()
+	title.statistics()
+	gain = title.get_latest('close_n') - items[0]['value']
+	gain_accum += gain
+	num_stocks += 1
+	print ('{}: Gain = {} %'.format(symbol, gain))
+
+if num_stocks > 0:
+	print ('Gain total = {} %'.format(gain_accum/num_stocks))
+
 plt.show()