#encoding:gbk
import pandas as pd
import numpy as np
import time
import datetime
import calendar
from scipy import stats
import tensorflow as tf
import warnings
warnings.filterwarnings("ignore")
def get_holdings(accountid,datatype):
holdinglist={}
resultlist=get_trade_detail_data(accountid,datatype,"POSITION")
for obj in resultlist:
holdinglist[obj.m_strInstrumentID+"."+obj.m_strExchangeID]=obj.m_nVolume
return holdinglist
def get_portfolio(accountid,datatype):
result=0
resultlist=get_trade_detail_data(accountid,datatype,"ACCOUNT")
for obj in resultlist:
result=obj.m_dAvailable
return result
def timetag_to_date(timetag, format):
timetag = timetag/1000
#time_local = time.localtime(timetag)
return time.strftime(format,format)
def init(ContextInfo):
ContextInfo.index = '000016.SH'
#ContextInfo.stocks = ContextInfo.get_sector('000300.SH')
#ContextInfo.set_universe(ContextInfo.stocks)
ContextInfo.stype = 'index'
ContextInfo.accountID='110000035020'
ContextInfo.isFirst = True
ContextInfo.factors = ['Beta','btop',
'etop','cetop','etp5','earning_yield',
'agro']
#,'rstr','lncap',
#'净利率', '净资产收益率', '毛利率','资产收益率',
#'资产负债率','市盈率','流通市值']
'''
ContextInfo.factors = ['Beta.beta','btop.btop',
'EARNINGS_YIELD.etop','EARNINGS_YIELD.cetop','EARNINGS_YIELD.etp5','EARNINGS_YIELD.earning_yield',
'GROWTH.agro',
'MOMENTUM.rstr','NLSIZE.lncap',
'净利率.考虑披露期延迟净利率', '净资产收益率.考虑披露期延迟净资产收益率', '毛利率.考虑披露期延迟毛利率','资产收益率.考虑披露期延迟资产收益率',
'资产负债率.考虑披露期延迟资产负债率','市盈率.考虑披露期延迟市盈率','流通市值.流通市值']
'''
ContextInfo.financials = ['PERSHAREINDEX.du_return_on_equity', 'PERSHAREINDEX.sales_gross_profit', 'PERSHAREINDEX.inc_revenue_rate',
'ERSHAREINDEX.du_profit_rate', 'PERSHAREINDEX.inc_net_profit_rate','PERSHAREINDEX.adjusted_net_profit_rate',
'PERSHAREINDEX.inc_total_revenue_annual', 'PERSHAREINDEX.inc_net_profit_to_shareholders_annual', 'PERSHAREINDEX.adjusted_profit_to_profit_annual',
'PERSHAREINDEX.equity_roe', 'PERSHAREINDEX.net_roe', 'PERSHAREINDEX.total_roe',
'PERSHAREINDEX.gross_profit', 'PERSHAREINDEX.net_profit', 'PERSHAREINDEX.actual_tax_rate',
'PERSHAREINDEX.gear_ratio', 'PERSHAREINDEX.inventory_turnover']
ContextInfo.shift = 12 # 回溯时间步
ContextInfo.training_iters = 1000 # 单周期训练次数
ContextInfo.stop = 20 # 停下来查看loss的时间步
ContextInfo.lr = 0.05 # 学习率
ContextInfo.n_hidden_units = 100 # neurons in hidden layer
ContextInfo.lstm_size = ContextInfo.n_hidden_units # 单层 lstm 的 hidden_units 数量
ContextInfo.n_layers = 2 # lstm 层数
def handlebar(ContextInfo):
index = ContextInfo.index
shift = ContextInfo.shift
barpos = ContextInfo.barpos
realtime = ContextInfo.get_bar_timetag(barpos)
ContextInfo.current_dt = timetag_to_datetime(realtime,'%Y%m%d')
today = ContextInfo.current_dt
bkstart = ContextInfo.start
bkend = ContextInfo.end
#bkstartday = substr(bkstart)
#bkday = substr(bkend)
#preDate1 = datetime.datetime.strptime(today, "%Y%m%d") + datetime.timedelta(days = -shift)
#preDate = preDate1.strftime('%Y%m%d')
#preDate2 = datetime.datetime.strptime(today, "%Y%m%d") + datetime.timedelta(days = -1)
#ContextInfo.previous_date = preDate2.strftime('%Y%m%d')
#print(preDate, type(preDate))
index = ContextInfo.index
#获取股票列表
if ContextInfo.stype == 'index':
ContextInfo.stocks = ContextInfo.get_sector(index)#, realtime)
elif ContextInfo.stype == 'industry':
ContextInfo.stocks = ContextInfo.get_industry(index)
elif ContextInfo.stype == 'sector':
ContextInfo.stocks = ContextInfo.get_stock_list_in_sector(index, realtime)
else:
ContextInfo.stocks = index
ContextInfo.set_universe(ContextInfo.stocks)
stockList = ContextInfo.stocks
factorslist = ContextInfo.factors
set_slip_fee(ContextInfo)
print('时间戳获取中……')
tradingday = ContextInfo.get_trading_dates('000001.SH', '', today, shift+1, '1d')
shift_index_pred = list(tradingday).index(today) # 获取today对应的绝对日期排序数
pred_days = tradingday[shift_index_pred - shift+1 : shift_index_pred+1] # 获取回溯需要的日期array
start_date_pred = pred_days[0]
end_date_pred = pred_days[-1]
ContextInfo.previous_date =pred_days[-1]
print('pred:', start_date_pred, end_date_pred)
# 训练时间list
shift_index_train = list(tradingday).index(today) -1 # 获取yesterday对应的绝对日期排序数
train_days = tradingday[shift_index_train - shift+1 : shift_index_train+1 ]
start_date_train = train_days[0]
end_date_train = train_days[-1]
#ContextInfo.previous_date =train_days[-1]
print('train:',start_date_train, end_date_train, ContextInfo.previous_date)
'''
#剔除ST股
st_data=get_extras('is_st',stockList, count = 1,end_date = date)
stockList = [stock for stock in stockList if not st_data[stock][0]]
#剔除停牌、新股及退市股票
stockList=delect_stop(stockList,date,date)
# stockList = stockList[:9]
'''
print('训练数据获取中……')
x_tech_train = get_train_x(ContextInfo, stockList,factorslist,start_date_train, end_date_train)
#print(x_tech_train)
#x_fund_train = get_fund(ContextInfo, stockList, start_date_train, end_date_train)
#print(x_fund_train)
#x_train = pd.concat([x_tech_train, np.transpose(x_fund_train, (1,0,2))], axis =2)
#x_train = pd.concat([x_tech_train, x_fund_train], axis=1)
#print(x_train)
x_train = x_tech_train
y_train = get_train_y(ContextInfo, stockList, start_date_train, end_date_train)
print('预测预备数据获取中……')
x_tech_pred = get_train_x(ContextInfo, stockList,factorslist,start_date_pred, end_date_pred)
#x_tech_pred = x_tech_train
# x_fund_pred = get_fund(stockList, pred_days,industry_old_code,industry_new_code)
x_pred = x_tech_pred
print('单周期数据准备完成！！')
# 参数
training_iters = ContextInfo.training_iters # 单周期训练次数
stop = ContextInfo.stop # 停下来查看loss的时间步
lr = ContextInfo.lr # 学习率
n_hidden_units = ContextInfo.n_hidden_units # neurons in hidden layer
lstm_size = n_hidden_units # 单层 lstm 的 hidden_units 数量
n_layers = ContextInfo.n_layers # lstm层数
print('获取预测标的中……')
#print(x_train)
x_sub_train = x_train.get_values().astype(np.float32)#array(x_train.astype(np.float32))
y_sub_train = y_train.get_values().astype(np.float32)#array(y_train.astype(np.float32))
x_test = x_pred.get_values().astype(np.float32) #array(x_pred.astype(np.float32))
# 参数
n_inputs = (x_sub_train.shape[2]) # 输入参数维度
n_steps = (x_sub_train.shape[1]) # time steps
n_classes = y_sub_train.shape[2] # 分类元素
n_layers = ContextInfo.n_layers # lstm层数
lr = ContextInfo.lr # 学习率
training_iters = ContextInfo.training_iters # 训练次数
stop = ContextInfo.stop # 停止步数
print('预测中……')
pred = lstmtrain(x_sub_train, y_sub_train, x_pred, n_hidden_units,n_inputs, n_classes,n_steps,n_layers,lr, training_iters, stop)
#print('pred:', pred)
print('获取买入卖出池子中……')
buy,sell,df = get_buy_sell(pred, y_train, stockList)
buy_position = get_buy_position(df,buy)
print('预测结果：',df)
print('买入配资', buy_position)
print('买入池子：',buy)
print('卖出池子：',sell)
ContextInfo.buy = list(buy_position.index)
ContextInfo.sell = sell
ContextInfo.buy_position = buy_position
sell = list(sell)
# 止盈止损池子
today = ContextInfo.current_dt
yesterday = ContextInfo.previous_date
yesterday = str(yesterday)
holdinglist = get_holdings(ContextInfo.accountID,'STOCK')
#print('holdinglist', holdinglist)
for stock in holdinglist:
floating_return = get_floating_return(ContextInfo, stock,yesterday,today)
if floating_return < -0.03 or floating_return > 0.1:
list(sell).append(stock)
ContextInfo.sell = list(sell)
market_open(ContextInfo)
# 【根据涨幅t1/t0倍数加权建仓】
def market_open(ContextInfo):
print('【开盘】(market_open):'+str(ContextInfo.current_dt))
df_buy = ContextInfo.buy_position
buy = ContextInfo.buy
sell = ContextInfo.sell
ContextInfo.holdings = get_holdings(ContextInfo.accountID,"STOCK")
print(ContextInfo.holdings)
# 获取每个标的的配资权重
weight_for_stocks = {}
for stock in df_buy.index:
weight_for_stocks[stock] = list(df_buy.ix[stock])[-1]
cash = get_portfolio(ContextInfo.accountID,'STOCK')
# 买入股票
for stock in ContextInfo.buy:
if stock not in ContextInfo.holdings:
order_value(stock, weight_for_stocks[stock]*cash, ContextInfo, ContextInfo.accountID)
print('买入',stock)
for stock in ContextInfo.sell:
if stock in ContextInfo.holdings:
order_target_percent(stock, 0, 'COMPETE', ContextInfo, ContextInfo.accountID)
print('卖出',stock)
# 设置滑点手续费
def set_slip_fee(ContextInfo):
# 将滑点设置为0
ContextInfo.set_slippage(1, 0.0)
# 根据不同的时间段设置手续费
dt = ContextInfo.current_dt
if dt > '20130101': #datetime.datetime(2013,1, 1):
commissionList = [0.0003,0.0013,0.0003, 0.0003, 0, 5]
elif dt > '20110101': #datetime.datetime(2011,1, 1):
commissionList = [0.001,0.002,0.0003, 0.0003, 0, 5]
elif dt > '20090101':#datetime.datetime(2009,1, 1):
commissionList = [0.002,0.003,0.0003, 0.0003, 0, 5]
else:
commissionList = [0.003,0.004,0.0003, 0.0003, 0, 5]
ContextInfo.set_commission(0, commissionList)
def get_data_from_date(ContextInfo, start_date,end_date,stocks):
'''
获取时间轴数据
'''
trade_date = ContextInfo.get_trading_dates(stocks, start_date,end_date, 1, '1d')
print(trade_date[0])
df = get_factors_one_stock(stocks,trade_date[0])
for date in trade_date[1:]:
df1 = get_factors_one_stock(stocks,date)
df = pd.concat([df,df1])
return df
def get_fund(ContextInfo, stockList, start_date_pre, end_date_pre):
fieldList = ['PERSHAREINDEX.du_return_on_equity', 'PERSHAREINDEX.sales_gross_profit', 'PERSHAREINDEX.inc_revenue_rate',
'ERSHAREINDEX.du_profit_rate', 'PERSHAREINDEX.inc_net_profit_rate','PERSHAREINDEX.adjusted_net_profit_rate',
'PERSHAREINDEX.inc_total_revenue_annual', 'PERSHAREINDEX.inc_net_profit_to_shareholders_annual', 'PERSHAREINDEX.adjusted_profit_to_profit_annual',
'PERSHAREINDEX.equity_roe', 'PERSHAREINDEX.net_roe', 'PERSHAREINDEX.total_roe',
'PERSHAREINDEX.gross_profit', 'PERSHAREINDEX.net_profit', 'PERSHAREINDEX.actual_tax_rate',
'PERSHAREINDEX.gear_ratio', 'PERSHAREINDEX.inventory_turnover']
x_fund = ContextInfo.get_financial_data(fieldList, stockList,start_date_pre, end_date_pre, report_type = 'announce_time')
return x_fund
# 训练数据获取
# 单周期训练数据【x】
def get_train_x(ContextInfo, stockList, factorslist, start_date_pre, end_date_pre):
x_train = pd.DataFrame(columns=ContextInfo.factors)
timeArray = time.strptime(start_date_pre, "%Y%m%d")
start = time.strftime("%Y-%m-%d %H:%M:%S", timeArray)
timeArrayend = time.strptime(end_date_pre, "%Y%m%d")
end = time.strftime("%Y-%m-%d %H:%M:%S", timeArrayend)
data = ext_data_range(factorslist[0],'000001.SH' ,start,end, ContextInfo)
dfdate = pd.DataFrame.from_dict(data, orient='index', columns=['Beta'])
dfdate.index.name = 'dates'
del dfdate['Beta']
#dfdate.index.map(substr)
dfdate2 = dfdate.rename(index = substr)
#print('*******:',dfdate2)
# 训练数据【x】
input_data1 = {}
for stock in stockList:
ddd = dfdate2
for factor in factorslist:
factor_data= ext_data_range(factor,stock ,start,end,ContextInfo)
dffactor = pd.DataFrame.from_dict(factor_data, orient='index', columns=[factor])
#dffactor.index.name = 'dates'
#dffactor.index.map(substr)
dffactor2 = dffactor.rename(index = substr)
#print(dffactor2)
ddd = ddd.join(dffactor2, on='dates')
#print(ddd)
dffund = ContextInfo.get_financial_data(ContextInfo.financials, [stock],start_date_pre, end_date_pre, report_type = 'announce_time')
dfalldata = pd.concat([ddd, dffund], axis=1)
#dfalldata = ddd
#print(dfalldata)
# 有些因子没有的，用别的近似代替
#dfalldata.fillna(0, inplace = True)
#input_data1[stock] = dfalldata
# 去inf
a = np.array(dfalldata)
where_are_inf = np.isinf(a)
a[where_are_inf] = 'nan'
dfdata =pd.DataFrame(a, index=dfalldata.index, columns = dfalldata.columns)
#print(dfalldata.columns)
dfdata.fillna(0, inplace = True)
data_pro = winsorize_and_standarlize(dfdata)
#print('##############', data_pro)
input_data1[stock] = data_pro
#print('##############', type(input_data1), input_data1)
#print('****************', input_data1)
x_train = pd.Panel(input_data1)
#print(type(x_train), x_train)
return x_train
# 单周期训练数据【y】
def get_train_y(ContextInfo, stockList, start_date_last, end_date_last):
# 训练数据【y】
input_data = {}
for i in stockList:
data = ContextInfo.get_market_data(['close'], stock_code = [i],
start_time = start_date_last, end_time = end_date_last, skip_paused = True,
period = ContextInfo.period, dividend_type = 'none', count = ContextInfo.shift)
#print(data[data.columns[0]])
data = data[data.columns[0]]
input_data[i] = pd.DataFrame(data)
y_train = pd.Panel(input_data) # 训练因子数据
return y_train
# 【取每一时间步的输出为拿来训练的预测值】多层LSTM， 输出结果就是prediction
def multi_layer_lstm_for_each_step(X, n_lstm_layer, n_hidden_units, n_inputs, n_classes,n_steps,n_layers):
# n_lstm_layer: lstm的层数
# lstm_size 为 单层lstm的hidden_units的个数
# 权重
W = {
# (n_inputs, hidden_units)
'in': tf.Variable(tf.truncated_normal([n_inputs, n_hidden_units]), name ='in'),
# (hidden_units, n_classes)
'out': tf.Variable(tf.truncated_normal([n_hidden_units, n_classes]), name ='out')
}
b = {
# (n_hidden_units)
'in': tf.Variable(tf.constant(0.1, shape = [n_hidden_units]), name ='in'),
# (n_classes)
'out': tf.Variable(tf.constant(0.1, shape = [n_classes]), name ='out'),
}
# 放入 lstm 前的 hidden_layer
_x = tf.transpose(X, [1,0,2]) # permute n_steps and batch_size
# new shape (n_steps, batch_size, n_inputs)
# reshape to prepare input to hidden activation
_x = tf.reshape(_x, [-1, n_inputs])
# new shape: (n_steps * batch_size, n_input )
# ReLu activation
_x = tf.nn.relu(tf.matmul(_x, W['in']) + b['in'])
# Split data because rnn_cell needs a list of inputs for the RNN inner loop
_x = tf.split(_x, n_steps, 0)
# new shape: n_steps * (batch_size, n_hidden)
_x = tf.transpose(_x, [1,0,2])
# new shape: (batch_size, n_steps, hidden_units)
# 定义多层lstm
layers = [tf.contrib.rnn.BasicLSTMCell(num_units=n_hidden_units,
activation=tf.nn.relu)
for layer in range(n_layers)]
multi_layer_cell = tf.contrib.rnn.MultiRNNCell(layers)
# 运算RNN
output,final_states = tf.nn.dynamic_rnn(multi_layer_cell,_x,dtype=tf.float32)
# output 为每一个时间步隐藏层的输出值，shape:(batch_size, n_step, hidden_units)
# 放入 lstm 前的 hidden_layer
_output = tf.transpose(output, [1,0,2]) # permute n_steps and batch_size
# new shape (n_steps, batch_size, hidden_units)
_output = tf.reshape(_output, [-1, n_hidden_units])
# # new shape: (n_steps * batch_size, n_input )
# ReLu activation
_output = tf.nn.relu(tf.matmul(_output, W['out']) + b['out'])
# Split data because rnn_cell needs a list of inputs for the RNN inner loop
_output = tf.split(_output, n_steps, 0)
# new shape: n_steps * (batch_size, n_hidden)
_output = tf.transpose(_output, [1,0,2])
# new shape: (batch_size, n_steps, hidden_units)
pred = _output
return pred
# 单周期多层lstm预测
def lstmtrain(x_sub_train, y_sub_train, x_pred, n_hidden_units,n_inputs, n_classes,n_steps,n_layers,lr, training_iters, stop):
# 单周期多层LSTM（加上回溯前面周期）训练 +预测
warnings.filterwarnings("ignore")
tf.reset_default_graph()
n_inputs = (x_sub_train.shape[2]) # 输入参数维度
n_steps = (x_sub_train.shape[1]) # time steps
n_classes = y_sub_train.shape[2] # 分类元素
# 单层lstm
# prediction = single_layer_lstm_for_last_step(x, lstm_size)
# prediction = single_layer_lstm_for_each_step(x, lstm_size)
# # 多层lstm
#n_layers = 2
#prediction = multi_layer_lstm_for_last_step(x, n_layers, n_hidden_units)
prediction = multi_layer_lstm_for_each_step(x_sub_train, n_layers, n_hidden_units, n_inputs, n_classes,n_steps,n_layers)
#损失函数,均方差
# y_cost = tf.transpose(y,[1,0,2]) # 转换格式 (时间步，标的数量，分类结果)
# y_cost_for_train = y_cost[-1] # 提取样本内最后一个时间步（周期）的分类结果, shape=(标的数量,分类结果【一维数据，1 or 0】)
# MSE损失函数
loss = tf.reduce_mean(tf.square(y_sub_train - prediction))
# loss = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits_v2(logits=prediction, labels=y))
#梯度下降: AdamOptimizer 收敛速度快，但是过拟合严重
train_step = tf.train.AdamOptimizer(learning_rate=lr).minimize(loss) #learning_rate可以调整
# # 预测
# correct_pred = tf.equal(tf.argmax(prediction,1), tf.argmax(y,1))
# accuracy = tf.reduce_mean(tf.cast(correct_pred, tf.float32))
with tf.Session() as sess:
sess.run(tf.global_variables_initializer()) # 初始化
train_loss_list = []
for epoach in range(training_iters):
sess.run(train_step)
if epoach % stop == 0:
# 训练误差
loss_train = sess.run(loss)
train_loss_list.append(loss_train)
# 预测
pred = sess.run(prediction)
print ('Iter' + str(epoach) + ' Training Loss:' + str(loss_train) )
return pred
# 获取买入卖出池子
def get_buy_sell(pred,y_train,stockList):
# 整合结果
# 上一个交易日的收盘价
df_y_t0 = dict(np.transpose((y_train),(1,0,2)))[list(dict(np.transpose((y_train),(1,0,2))).keys())[-1]]
# 本交易日预测价格
p = np.transpose(pred, (1,0,2))
df_pred = pd.DataFrame(p[-1])
df_pred.index = df_y_t0.index
# 整合
df = pd.concat([df_y_t0, df_pred], axis =1)
df.columns = ['t0','pre_t1']
# 获取预测标的池子
diff1 = (df['pre_t1']-df['t0'])
# 表格整理
df1 = df.copy()
df1['buy_decision'] = 0
for i in range(len(df)):
if list(diff1)[i] > 0:
df1.iloc[i,2] = True
else :
df1.iloc[i,2] = False
# 买入池子
buy = []
for i in range(len(diff1)):
if list(diff1)[i] > 0 :
buy.append(diff1.index[i])
# 卖出池子
sell = set(stockList) - set(buy)
return buy,sell,df1
# 获取标的日间浮动收益率
def get_floating_return(ContextInfo, stock,yesterday,today):
#print( [stock], str(yesterday), str(today), ContextInfo.period)
a = ContextInfo.get_market_data(['close'], [stock], yesterday,today, True,ContextInfo.period, 'none', 2)
past_p = list(a['close'])[0]
current_p = list(a['close'])[-1]
floating_return = current_p/past_p -1
return floating_return
# 【资金加权】
def get_buy_position(df,buy):
# 涨跌幅获取
df['increase_pct'] = df['pre_t1']/df['t0']
# 提取预测涨的股票
df_buy = df.ix[buy]
# 降序
df_buy = df_buy.sort_values(by = ['increase_pct'], ascending = False)
# 配资比率
df_buy['buy%'] = df_buy['increase_pct']/sum(df['increase_pct'])
return df_buy
def substr(dtstr):
data = dtstr[0:4]+dtstr[5:7]+dtstr[8:10]
return data
def winsorize_and_standarlize(data,qrange=[0.05,0.95],axis=0):
'''
input:
data:Dataframe or series,输入数据
qrange:list,list[0]下分位数，list[1]，上分位数，极值用分位数代替
'''
if isinstance(data,pd.DataFrame):
if axis == 0:
q_down = data.quantile(qrange[0])
q_up = data.quantile(qrange[1])
index = data.index
col = data.columns
for n in col:
data[n][data[n] > q_up[n]] = q_up[n]
data[n][data[n] < q_down[n]] = q_down[n]
data = (data - data.mean())/data.std()
data = data.fillna(0)
else:
data = data.stack()
data = data.unstack(0)
q = data.quantile(qrange)
index = data.index
col = data.columns
for n in col:
data[n][data[n] > q[n]] = q[n]
data = (data - data.mean())/data.std()
data = data.stack().unstack(0)
data = data.fillna(0)
elif isinstance(data,pd.Series):
name = data.name
q = data.quantile(qrange)
data[data>q] = q
data = (data - data.mean())/data.std()
return data
|
