增加 alpha 检测工具
This commit is contained in:
@@ -0,0 +1,760 @@
|
||||
import requests
|
||||
from os import environ
|
||||
from time import sleep
|
||||
import time
|
||||
import json
|
||||
import pandas as pd
|
||||
import random
|
||||
import pickle
|
||||
from urllib.parse import urljoin
|
||||
from itertools import product
|
||||
from itertools import combinations
|
||||
from collections import defaultdict
|
||||
import pickle
|
||||
|
||||
|
||||
|
||||
basic_ops = ["reverse", "inverse", "rank", "zscore", "quantile", "normalize"]
|
||||
|
||||
ts_ops = ["ts_rank", "ts_zscore", "ts_delta", "ts_sum", "ts_delay",
|
||||
"ts_std_dev", "ts_mean", "ts_arg_min", "ts_arg_max","ts_scale", "ts_quantile"]
|
||||
|
||||
ops_set = basic_ops + ts_ops
|
||||
|
||||
def login():
|
||||
|
||||
username = ""
|
||||
password = ""
|
||||
|
||||
# Create a session to persistently store the headers
|
||||
s = requests.Session()
|
||||
|
||||
# Save credentials into session
|
||||
s.auth = (username, password)
|
||||
|
||||
# Send a POST request to the /authentication API
|
||||
response = s.post('https://api.worldquantbrain.com/authentication')
|
||||
print(response.content)
|
||||
return s
|
||||
|
||||
|
||||
def get_datasets(
|
||||
s,
|
||||
instrument_type: str = 'EQUITY',
|
||||
region: str = 'USA',
|
||||
delay: int = 1,
|
||||
universe: str = 'TOP3000'
|
||||
):
|
||||
url = "https://api.worldquantbrain.com/data-sets?" +\
|
||||
f"instrumentType={instrument_type}®ion={region}&delay={str(delay)}&universe={universe}"
|
||||
result = s.get(url)
|
||||
datasets_df = pd.DataFrame(result.json()['results'])
|
||||
return datasets_df
|
||||
|
||||
|
||||
def get_datafields(
|
||||
s,
|
||||
instrument_type: str = 'EQUITY',
|
||||
region: str = 'USA',
|
||||
delay: int = 1,
|
||||
universe: str = 'TOP3000',
|
||||
dataset_id: str = '',
|
||||
search: str = ''
|
||||
):
|
||||
if len(search) == 0:
|
||||
url_template = "https://api.worldquantbrain.com/data-fields?" +\
|
||||
f"&instrumentType={instrument_type}" +\
|
||||
f"®ion={region}&delay={str(delay)}&universe={universe}&dataset.id={dataset_id}&limit=50" +\
|
||||
"&offset={x}"
|
||||
count = s.get(url_template.format(x=0)).json()['count']
|
||||
|
||||
else:
|
||||
url_template = "https://api.worldquantbrain.com/data-fields?" +\
|
||||
f"&instrumentType={instrument_type}" +\
|
||||
f"®ion={region}&delay={str(delay)}&universe={universe}&limit=50" +\
|
||||
f"&search={search}" +\
|
||||
"&offset={x}"
|
||||
count = 100
|
||||
|
||||
datafields_list = []
|
||||
for x in range(0, count, 50):
|
||||
datafields = s.get(url_template.format(x=x))
|
||||
datafields_list.append(datafields.json()['results'])
|
||||
|
||||
datafields_list_flat = [item for sublist in datafields_list for item in sublist]
|
||||
|
||||
datafields_df = pd.DataFrame(datafields_list_flat)
|
||||
return datafields_df
|
||||
|
||||
def get_vec_fields(fields):
|
||||
|
||||
# 请在此处添加获得权限的Vector操作符
|
||||
vec_ops = ["vec_avg", "vec_sum"]
|
||||
vec_fields = []
|
||||
|
||||
for field in fields:
|
||||
for vec_op in vec_ops:
|
||||
if vec_op == "vec_choose":
|
||||
vec_fields.append("%s(%s, nth=-1)"%(vec_op, field))
|
||||
vec_fields.append("%s(%s, nth=0)"%(vec_op, field))
|
||||
else:
|
||||
vec_fields.append("%s(%s)"%(vec_op, field))
|
||||
|
||||
return(vec_fields)
|
||||
|
||||
def process_datafields(df):
|
||||
|
||||
datafields = []
|
||||
datafields += df[df['type'] == "MATRIX"]["id"].tolist()
|
||||
datafields += get_vec_fields(df[df['type'] == "VECTOR"]["id"].tolist())
|
||||
return ["winsorize(ts_backfill(%s, 120), std=4)"%field for field in datafields]
|
||||
|
||||
def ts_factory(op, field):
|
||||
output = []
|
||||
#days = [3, 5, 10, 20, 60, 120, 240]
|
||||
days = [5, 22, 66, 120, 240]
|
||||
|
||||
for day in days:
|
||||
|
||||
alpha = "%s(%s, %d)"%(op, field, day)
|
||||
output.append(alpha)
|
||||
|
||||
return output
|
||||
|
||||
def first_order_factory(fields, ops_set):
|
||||
alpha_set = []
|
||||
#for field in fields:
|
||||
for field in fields:
|
||||
#reverse op does the work
|
||||
alpha_set.append(field)
|
||||
#alpha_set.append("-%s"%field)
|
||||
for op in ops_set:
|
||||
|
||||
if op == "ts_percentage":
|
||||
|
||||
alpha_set += ts_comp_factory(op, field, "percentage", [0.5])
|
||||
|
||||
elif op == "ts_decay_exp_window":
|
||||
|
||||
alpha_set += ts_comp_factory(op, field, "factor", [0.5])
|
||||
|
||||
elif op == "ts_moment":
|
||||
|
||||
alpha_set += ts_comp_factory(op, field, "k", [2, 3, 4])
|
||||
|
||||
elif op == "ts_entropy":
|
||||
|
||||
alpha_set += ts_comp_factory(op, field, "buckets", [10])
|
||||
|
||||
elif op.startswith("ts_") or op == "inst_tvr":
|
||||
|
||||
alpha_set += ts_factory(op, field)
|
||||
|
||||
elif op.startswith("vector"):
|
||||
|
||||
alpha_set += vector_factory(op, field)
|
||||
|
||||
elif op == "signed_power":
|
||||
|
||||
alpha = "%s(%s, 2)"%(op, field)
|
||||
alpha_set.append(alpha)
|
||||
|
||||
else:
|
||||
alpha = "%s(%s)"%(op, field)
|
||||
alpha_set.append(alpha)
|
||||
|
||||
return alpha_set
|
||||
|
||||
|
||||
def load_task_pool(alpha_list, limit_of_children_simulations, limit_of_multi_simulations):
|
||||
'''
|
||||
Input:
|
||||
alpha_list : list of (alpha, decay) tuples
|
||||
limit_of_multi_simulations : number of children simulation in a multi-simulation
|
||||
limit_of_multi_simulations : number of simultaneous multi-simulations
|
||||
Output:
|
||||
task : [10 * (alpha, decay)] for a multi-simulation
|
||||
pool : [10 * [10 * (alpha, decay)]] for simultaneous multi-simulations
|
||||
pools : [[10 * [10 * (alpha, decay)]]]
|
||||
|
||||
'''
|
||||
tasks = [alpha_list[i:i + limit_of_children_simulations] for i in range(0, len(alpha_list), limit_of_children_simulations)]
|
||||
pools = [tasks[i:i + limit_of_multi_simulations] for i in range(0, len(tasks), limit_of_multi_simulations)]
|
||||
return pools
|
||||
|
||||
def multi_simulate(alpha_pools, neut, region, universe, start):
|
||||
|
||||
s = login()
|
||||
|
||||
brain_api_url = 'https://api.worldquantbrain.com'
|
||||
|
||||
for x, pool in enumerate(alpha_pools):
|
||||
if x < start: continue
|
||||
progress_urls = []
|
||||
for y, task in enumerate(pool):
|
||||
# 10 tasks, 10 alpha in each task
|
||||
sim_data_list = generate_sim_data(task, region, universe, neut)
|
||||
try:
|
||||
simulation_response = s.post('https://api.worldquantbrain.com/simulations', json=sim_data_list)
|
||||
simulation_progress_url = simulation_response.headers['Location']
|
||||
progress_urls.append(simulation_progress_url)
|
||||
except:
|
||||
print("location key error: %s"%simulation_response.content)
|
||||
sleep(600)
|
||||
s = login()
|
||||
|
||||
print("pool %d task %d post done"%(x,y))
|
||||
|
||||
for j, progress in enumerate(progress_urls):
|
||||
try:
|
||||
while True:
|
||||
simulation_progress = s.get(progress)
|
||||
if simulation_progress.headers.get("Retry-After", 0) == 0:
|
||||
break
|
||||
#print("Sleeping for " + simulation_progress.headers["Retry-After"] + " seconds")
|
||||
sleep(float(simulation_progress.headers["Retry-After"]))
|
||||
|
||||
status = simulation_progress.json().get("status", 0)
|
||||
if status != "COMPLETE":
|
||||
print("Not complete : %s"%(progress))
|
||||
|
||||
"""
|
||||
#alpha_id = simulation_progress.json()["alpha"]
|
||||
children = simulation_progress.json().get("children", 0)
|
||||
children_list = []
|
||||
for child in children:
|
||||
child_progress = s.get(brain_api_url + "/simulations/" + child)
|
||||
alpha_id = child_progress.json()["alpha"]
|
||||
|
||||
set_alpha_properties(s,
|
||||
alpha_id,
|
||||
name = "%s"%name,
|
||||
color = None,)
|
||||
"""
|
||||
except KeyError:
|
||||
print("look into: %s"%progress)
|
||||
except Exception as e:
|
||||
print(f"other error: {e}")
|
||||
|
||||
|
||||
print("pool %d task %d simulate done"%(x, j))
|
||||
|
||||
print("Simulate done")
|
||||
|
||||
def generate_sim_data(alpha_list, region, uni, neut):
|
||||
sim_data_list = []
|
||||
for alpha, decay in alpha_list:
|
||||
simulation_data = {
|
||||
'type': 'REGULAR',
|
||||
'settings': {
|
||||
'instrumentType': 'EQUITY',
|
||||
'region': region,
|
||||
'universe': uni,
|
||||
'delay': 1,
|
||||
'decay': decay,
|
||||
'neutralization': neut,
|
||||
'truncation': 0.08,
|
||||
'pasteurization': 'ON',
|
||||
'testPeriod': 'P0Y',
|
||||
'unitHandling': 'VERIFY',
|
||||
'nanHandling': 'ON',
|
||||
'language': 'FASTEXPR',
|
||||
'visualization': False,
|
||||
},
|
||||
'regular': alpha}
|
||||
|
||||
sim_data_list.append(simulation_data)
|
||||
return sim_data_list
|
||||
|
||||
def set_alpha_properties(
|
||||
s,
|
||||
alpha_id,
|
||||
name: str = None,
|
||||
color: str = None,
|
||||
selection_desc: str = "None",
|
||||
combo_desc: str = "None",
|
||||
tags: str = ["ace_tag"],
|
||||
):
|
||||
"""
|
||||
Function changes alpha's description parameters
|
||||
"""
|
||||
|
||||
params = {
|
||||
"color": color,
|
||||
"name": name,
|
||||
"tags": tags,
|
||||
"category": None,
|
||||
"regular": {"description": None},
|
||||
"combo": {"description": combo_desc},
|
||||
"selection": {"description": selection_desc},
|
||||
}
|
||||
response = s.patch(
|
||||
"https://api.worldquantbrain.com/alphas/" + alpha_id, json=params
|
||||
)
|
||||
|
||||
def get_alphas(start_date, end_date, sharpe_th, fitness_th, region, alpha_num, usage):
|
||||
s = login()
|
||||
output = []
|
||||
# 3E large 3C less
|
||||
count = 0
|
||||
for i in range(0, alpha_num, 100):
|
||||
print(i)
|
||||
url_e = "https://api.worldquantbrain.com/users/self/alphas?limit=100&offset=%d"%(i) \
|
||||
+ "&status=UNSUBMITTED%1FIS_FAIL&dateCreated%3E=2025-" + start_date \
|
||||
+ "T00:00:00-04:00&dateCreated%3C2025-" + end_date \
|
||||
+ "T00:00:00-04:00&is.fitness%3E" + str(fitness_th) + "&is.sharpe%3E" \
|
||||
+ str(sharpe_th) + "&settings.region=" + region + "&order=-is.sharpe&hidden=false&type!=SUPER"
|
||||
url_c = "https://api.worldquantbrain.com/users/self/alphas?limit=100&offset=%d"%(i) \
|
||||
+ "&status=UNSUBMITTED%1FIS_FAIL&dateCreated%3E=2025-" + start_date \
|
||||
+ "T00:00:00-04:00&dateCreated%3C2025-" + end_date \
|
||||
+ "T00:00:00-04:00&is.fitness%3C-" + str(fitness_th) + "&is.sharpe%3C-" \
|
||||
+ str(sharpe_th) + "&settings.region=" + region + "&order=is.sharpe&hidden=false&type!=SUPER"
|
||||
urls = [url_e]
|
||||
if usage != "submit":
|
||||
urls.append(url_c)
|
||||
for url in urls:
|
||||
response = s.get(url)
|
||||
#print(response.json())
|
||||
try:
|
||||
alpha_list = response.json()["results"]
|
||||
#print(response.json())
|
||||
for j in range(len(alpha_list)):
|
||||
alpha_id = alpha_list[j]["id"]
|
||||
name = alpha_list[j]["name"]
|
||||
dateCreated = alpha_list[j]["dateCreated"]
|
||||
sharpe = alpha_list[j]["is"]["sharpe"]
|
||||
fitness = alpha_list[j]["is"]["fitness"]
|
||||
turnover = alpha_list[j]["is"]["turnover"]
|
||||
margin = alpha_list[j]["is"]["margin"]
|
||||
longCount = alpha_list[j]["is"]["longCount"]
|
||||
shortCount = alpha_list[j]["is"]["shortCount"]
|
||||
decay = alpha_list[j]["settings"]["decay"]
|
||||
exp = alpha_list[j]['regular']['code']
|
||||
count += 1
|
||||
#if (sharpe > 1.2 and sharpe < 1.6) or (sharpe < -1.2 and sharpe > -1.6):
|
||||
if (longCount + shortCount) > 100:
|
||||
if sharpe < -sharpe_th:
|
||||
exp = "-%s"%exp
|
||||
rec = [alpha_id, exp, sharpe, turnover, fitness, margin, dateCreated, decay]
|
||||
print(rec)
|
||||
if turnover > 0.7:
|
||||
rec.append(decay*4)
|
||||
elif turnover > 0.6:
|
||||
rec.append(decay*3+3)
|
||||
elif turnover > 0.5:
|
||||
rec.append(decay*3)
|
||||
elif turnover > 0.4:
|
||||
rec.append(decay*2)
|
||||
elif turnover > 0.35:
|
||||
rec.append(decay+4)
|
||||
elif turnover > 0.3:
|
||||
rec.append(decay+2)
|
||||
output.append(rec)
|
||||
except:
|
||||
print("%d finished re-login"%i)
|
||||
s = login()
|
||||
|
||||
print("count: %d"%count)
|
||||
return output, s # 新增返回会话s
|
||||
|
||||
def prune(next_alpha_recs, prefix, keep_num):
|
||||
# prefix is the datafield prefix, fnd6, mdl175 ...
|
||||
# keep_num is the num of top sharpe same-datafield alpha
|
||||
output = []
|
||||
num_dict = defaultdict(int)
|
||||
for rec in next_alpha_recs:
|
||||
exp = rec[1]
|
||||
field = exp.split(prefix)[-1].split(",")[0]
|
||||
sharpe = rec[2]
|
||||
if sharpe < 0:
|
||||
field = "-%s"%field
|
||||
if num_dict[field] < keep_num:
|
||||
num_dict[field] += 1
|
||||
decay = rec[-1]
|
||||
exp = rec[1]
|
||||
output.append([exp,decay])
|
||||
return output
|
||||
|
||||
def get_group_second_order_factory(first_order, group_ops, region):
|
||||
second_order = []
|
||||
for fo in first_order:
|
||||
for group_op in group_ops:
|
||||
second_order += group_factory(group_op, fo, region)
|
||||
return second_order
|
||||
|
||||
|
||||
def group_factory(op, field, region):
|
||||
output = []
|
||||
vectors = ["cap"]
|
||||
|
||||
chn_group_13 = ['pv13_h_min2_sector', 'pv13_di_6l', 'pv13_rcsed_6l', 'pv13_di_5l', 'pv13_di_4l',
|
||||
'pv13_di_3l', 'pv13_di_2l', 'pv13_di_1l', 'pv13_parent', 'pv13_level']
|
||||
|
||||
|
||||
chn_group_1 = ['sta1_top3000c30','sta1_top3000c20','sta1_top3000c10','sta1_top3000c2','sta1_top3000c5']
|
||||
|
||||
chn_group_2 = ['sta2_top3000_fact4_c10','sta2_top2000_fact4_c50','sta2_top3000_fact3_c20']
|
||||
|
||||
hkg_group_13 = ['pv13_10_f3_g2_minvol_1m_sector', 'pv13_10_minvol_1m_sector', 'pv13_20_minvol_1m_sector',
|
||||
'pv13_2_minvol_1m_sector', 'pv13_5_minvol_1m_sector', 'pv13_1l_scibr', 'pv13_3l_scibr',
|
||||
'pv13_2l_scibr', 'pv13_4l_scibr', 'pv13_5l_scibr']
|
||||
|
||||
hkg_group_1 = ['sta1_allc50','sta1_allc5','sta1_allxjp_513_c20','sta1_top2000xjp_513_c5']
|
||||
|
||||
hkg_group_2 = ['sta2_all_xjp_513_all_fact4_c10','sta2_top2000_xjp_513_top2000_fact3_c10',
|
||||
'sta2_allfactor_xjp_513_13','sta2_top2000_xjp_513_top2000_fact3_c20']
|
||||
|
||||
twn_group_13 = ['pv13_2_minvol_1m_sector','pv13_20_minvol_1m_sector','pv13_10_minvol_1m_sector',
|
||||
'pv13_5_minvol_1m_sector','pv13_10_f3_g2_minvol_1m_sector','pv13_5_f3_g2_minvol_1m_sector',
|
||||
'pv13_2_f4_g3_minvol_1m_sector']
|
||||
|
||||
twn_group_1 = ['sta1_allc50','sta1_allxjp_513_c50','sta1_allxjp_513_c20','sta1_allxjp_513_c2',
|
||||
'sta1_allc20','sta1_allxjp_513_c5','sta1_allxjp_513_c10','sta1_allc2','sta1_allc5']
|
||||
|
||||
twn_group_2 = ['sta2_allfactor_xjp_513_0','sta2_all_xjp_513_all_fact3_c20',
|
||||
'sta2_all_xjp_513_all_fact4_c20','sta2_all_xjp_513_all_fact4_c50']
|
||||
|
||||
usa_group_13 = ['pv13_h_min2_3000_sector','pv13_r2_min20_3000_sector','pv13_r2_min2_3000_sector',
|
||||
'pv13_r2_min2_3000_sector', 'pv13_h_min2_focused_pureplay_3000_sector']
|
||||
|
||||
usa_group_1 = ['sta1_top3000c50','sta1_allc20','sta1_allc10','sta1_top3000c20','sta1_allc5']
|
||||
|
||||
usa_group_2 = ['sta2_top3000_fact3_c50','sta2_top3000_fact4_c20','sta2_top3000_fact4_c10']
|
||||
|
||||
usa_group_6 = ['mdl10_group_name']
|
||||
|
||||
asi_group_13 = ['pv13_20_minvol_1m_sector', 'pv13_5_f3_g2_minvol_1m_sector', 'pv13_10_f3_g2_minvol_1m_sector',
|
||||
'pv13_2_f4_g3_minvol_1m_sector', 'pv13_10_minvol_1m_sector', 'pv13_5_minvol_1m_sector']
|
||||
|
||||
asi_group_1 = ['sta1_allc50', 'sta1_allc10', 'sta1_minvol1mc50','sta1_minvol1mc20',
|
||||
'sta1_minvol1m_normc20', 'sta1_minvol1m_normc50']
|
||||
|
||||
jpn_group_1 = ['sta1_alljpn_513_c5', 'sta1_alljpn_513_c50', 'sta1_alljpn_513_c2', 'sta1_alljpn_513_c20']
|
||||
|
||||
jpn_group_2 = ['sta2_top2000_jpn_513_top2000_fact3_c20', 'sta2_all_jpn_513_all_fact1_c5',
|
||||
'sta2_allfactor_jpn_513_9', 'sta2_all_jpn_513_all_fact1_c10']
|
||||
|
||||
jpn_group_13 = ['pv13_2_minvol_1m_sector', 'pv13_2_f4_g3_minvol_1m_sector', 'pv13_10_minvol_1m_sector',
|
||||
'pv13_10_f3_g2_minvol_1m_sector', 'pv13_all_delay_1_parent', 'pv13_all_delay_1_level']
|
||||
|
||||
kor_group_13 = ['pv13_10_f3_g2_minvol_1m_sector', 'pv13_5_minvol_1m_sector', 'pv13_5_f3_g2_minvol_1m_sector',
|
||||
'pv13_2_minvol_1m_sector', 'pv13_20_minvol_1m_sector', 'pv13_2_f4_g3_minvol_1m_sector']
|
||||
|
||||
kor_group_1 = ['sta1_allc20','sta1_allc50','sta1_allc2','sta1_allc10','sta1_minvol1mc50',
|
||||
'sta1_allxjp_513_c10', 'sta1_top2000xjp_513_c50']
|
||||
|
||||
kor_group_2 =['sta2_all_xjp_513_all_fact1_c50','sta2_top2000_xjp_513_top2000_fact2_c50',
|
||||
'sta2_all_xjp_513_all_fact4_c50','sta2_all_xjp_513_all_fact4_c5']
|
||||
|
||||
eur_group_13 = ['pv13_5_sector', 'pv13_2_sector', 'pv13_v3_3l_scibr', 'pv13_v3_2l_scibr', 'pv13_2l_scibr',
|
||||
'pv13_52_sector', 'pv13_v3_6l_scibr', 'pv13_v3_4l_scibr', 'pv13_v3_1l_scibr']
|
||||
|
||||
eur_group_1 = ['sta1_allc10', 'sta1_allc2', 'sta1_top1200c2', 'sta1_allc20', 'sta1_top1200c10']
|
||||
|
||||
eur_group_2 = ['sta2_top1200_fact3_c50','sta2_top1200_fact3_c20','sta2_top1200_fact4_c50']
|
||||
|
||||
glb_group_13 = ["pv13_10_f2_g3_sector", "pv13_2_f3_g2_sector", "pv13_2_sector", "pv13_52_all_delay_1_sector"]
|
||||
|
||||
glb_group_1 = ['sta1_allc20', 'sta1_allc10', 'sta1_allc50', 'sta1_allc5']
|
||||
|
||||
glb_group_2 = ['sta2_all_fact4_c50', 'sta2_all_fact4_c20', 'sta2_all_fact3_c20', 'sta2_all_fact4_c10']
|
||||
|
||||
glb_group_13 = ['pv13_2_sector', 'pv13_10_sector', 'pv13_3l_scibr', 'pv13_2l_scibr', 'pv13_1l_scibr',
|
||||
'pv13_52_minvol_1m_all_delay_1_sector','pv13_52_minvol_1m_sector','pv13_52_minvol_1m_sector']
|
||||
|
||||
amr_group_13 = ['pv13_4l_scibr', 'pv13_1l_scibr', 'pv13_hierarchy_min51_f1_sector',
|
||||
'pv13_hierarchy_min2_600_sector', 'pv13_r2_min2_sector', 'pv13_h_min20_600_sector']
|
||||
|
||||
#bps_group = "bucket(rank(fnd28_value_05480), range='0.1, 1, 0.1')"
|
||||
#pb_group = "bucket(rank(close/fnd28_value_05480), range='0.1, 1, 0.1')"
|
||||
cap_group = "bucket(rank(cap), range='0.1, 1, 0.1')"
|
||||
asset_group = "bucket(rank(assets),range='0.1, 1, 0.1')"
|
||||
sector_cap_group = "bucket(group_rank(cap, sector),range='0.1, 1, 0.1')"
|
||||
sector_asset_group = "bucket(group_rank(assets, sector),range='0.1, 1, 0.1')"
|
||||
|
||||
vol_group = "bucket(rank(ts_std_dev(returns,20)),range = '0.1, 1, 0.1')"
|
||||
|
||||
liquidity_group = "bucket(rank(close*volume),range = '0.1, 1, 0.1')"
|
||||
|
||||
groups = ["market","sector", "industry", "subindustry",
|
||||
cap_group, asset_group, sector_cap_group, sector_asset_group, vol_group, liquidity_group]
|
||||
|
||||
if region == "CHN":
|
||||
groups += chn_group_13 + chn_group_1 + chn_group_2
|
||||
if region == "TWN":
|
||||
groups += twn_group_13 + twn_group_1 + twn_group_2
|
||||
if region == "ASI":
|
||||
groups += asi_group_13 + asi_group_1
|
||||
if region == "USA":
|
||||
groups += usa_group_13 + usa_group_1 + usa_group_2
|
||||
if region == "HKG":
|
||||
groups += hkg_group_13 + hkg_group_1 + hkg_group_2
|
||||
if region == "KOR":
|
||||
groups += kor_group_13 + kor_group_1 + kor_group_2
|
||||
if region == "EUR":
|
||||
groups += eur_group_13 + eur_group_1 + eur_group_2
|
||||
if region == "GLB":
|
||||
groups += glb_group_13 + glb_group_1 + glb_group_2
|
||||
if region == "AMR":
|
||||
groups += amr_group_13
|
||||
if region == "JPN":
|
||||
groups += jpn_group_1 + jpn_group_2 + jpn_group_13
|
||||
|
||||
for group in groups:
|
||||
if op.startswith("group_vector"):
|
||||
for vector in vectors:
|
||||
alpha = "%s(%s,%s,densify(%s))"%(op, field, vector, group)
|
||||
output.append(alpha)
|
||||
elif op.startswith("group_percentage"):
|
||||
alpha = "%s(%s,densify(%s),percentage=0.5)"%(op, field, group)
|
||||
output.append(alpha)
|
||||
else:
|
||||
alpha = "%s(%s,densify(%s))"%(op, field, group)
|
||||
output.append(alpha)
|
||||
|
||||
return output
|
||||
|
||||
|
||||
def trade_when_factory(op,field,region):
|
||||
output = []
|
||||
open_events = ["ts_arg_max(volume, 5) == 0", "ts_corr(close, volume, 20) < 0",
|
||||
"ts_corr(close, volume, 5) < 0", "ts_mean(volume,10)>ts_mean(volume,60)",
|
||||
"group_rank(ts_std_dev(returns,60), sector) > 0.7", "ts_zscore(returns,60) > 2",
|
||||
"ts_arg_min(volume, 5) > 3",
|
||||
"ts_std_dev(returns, 5) > ts_std_dev(returns, 20)",
|
||||
"ts_arg_max(close, 5) == 0", "ts_arg_max(close, 20) == 0",
|
||||
"ts_corr(close, volume, 5) > 0", "ts_corr(close, volume, 5) > 0.3", "ts_corr(close, volume, 5) > 0.5",
|
||||
"ts_corr(close, volume, 20) > 0", "ts_corr(close, volume, 20) > 0.3", "ts_corr(close, volume, 20) > 0.5",
|
||||
"ts_regression(returns, %s, 5, lag = 0, rettype = 2) > 0"%field,
|
||||
"ts_regression(returns, %s, 20, lag = 0, rettype = 2) > 0"%field,
|
||||
"ts_regression(returns, ts_step(20), 20, lag = 0, rettype = 2) > 0",
|
||||
"ts_regression(returns, ts_step(5), 5, lag = 0, rettype = 2) > 0"]
|
||||
|
||||
exit_events = ["abs(returns) > 0.1", "-1"]
|
||||
|
||||
usa_events = ["rank(rp_css_business) > 0.8", "ts_rank(rp_css_business, 22) > 0.8", "rank(vec_avg(mws82_sentiment)) > 0.8",
|
||||
"ts_rank(vec_avg(mws82_sentiment),22) > 0.8", "rank(vec_avg(nws48_ssc)) > 0.8",
|
||||
"ts_rank(vec_avg(nws48_ssc),22) > 0.8", "rank(vec_avg(mws50_ssc)) > 0.8", "ts_rank(vec_avg(mws50_ssc),22) > 0.8",
|
||||
"ts_rank(vec_sum(scl12_alltype_buzzvec),22) > 0.9", "pcr_oi_270 < 1", "pcr_oi_270 > 1",]
|
||||
|
||||
asi_events = ["rank(vec_avg(mws38_score)) > 0.8", "ts_rank(vec_avg(mws38_score),22) > 0.8"]
|
||||
|
||||
eur_events = ["rank(rp_css_business) > 0.8", "ts_rank(rp_css_business, 22) > 0.8",
|
||||
"rank(vec_avg(oth429_research_reports_fundamental_keywords_4_method_2_pos)) > 0.8",
|
||||
"ts_rank(vec_avg(oth429_research_reports_fundamental_keywords_4_method_2_pos),22) > 0.8",
|
||||
"rank(vec_avg(mws84_sentiment)) > 0.8", "ts_rank(vec_avg(mws84_sentiment),22) > 0.8",
|
||||
"rank(vec_avg(mws85_sentiment)) > 0.8", "ts_rank(vec_avg(mws85_sentiment),22) > 0.8",
|
||||
"rank(mdl110_analyst_sentiment) > 0.8", "ts_rank(mdl110_analyst_sentiment, 22) > 0.8",
|
||||
"rank(vec_avg(nws3_scores_posnormscr)) > 0.8",
|
||||
"ts_rank(vec_avg(nws3_scores_posnormscr),22) > 0.8",
|
||||
"rank(vec_avg(mws36_sentiment_words_positive)) > 0.8",
|
||||
"ts_rank(vec_avg(mws36_sentiment_words_positive),22) > 0.8"]
|
||||
|
||||
glb_events = ["rank(vec_avg(mdl109_news_sent_1m)) > 0.8",
|
||||
"ts_rank(vec_avg(mdl109_news_sent_1m),22) > 0.8",
|
||||
"rank(vec_avg(nws20_ssc)) > 0.8",
|
||||
"ts_rank(vec_avg(nws20_ssc),22) > 0.8",
|
||||
"vec_avg(nws20_ssc) > 0",
|
||||
"rank(vec_avg(nws20_bee)) > 0.8",
|
||||
"ts_rank(vec_avg(nws20_bee),22) > 0.8",
|
||||
"rank(vec_avg(nws20_qmb)) > 0.8",
|
||||
"ts_rank(vec_avg(nws20_qmb),22) > 0.8"]
|
||||
|
||||
chn_events = ["rank(vec_avg(oth111_xueqiunaturaldaybasicdivisionstat_senti_conform)) > 0.8",
|
||||
"ts_rank(vec_avg(oth111_xueqiunaturaldaybasicdivisionstat_senti_conform),22) > 0.8",
|
||||
"rank(vec_avg(oth111_gubanaturaldaydevicedivisionstat_senti_conform)) > 0.8",
|
||||
"ts_rank(vec_avg(oth111_gubanaturaldaydevicedivisionstat_senti_conform),22) > 0.8",
|
||||
"rank(vec_avg(oth111_baragedivisionstat_regi_senti_conform)) > 0.8",
|
||||
"ts_rank(vec_avg(oth111_baragedivisionstat_regi_senti_conform),22) > 0.8"]
|
||||
|
||||
kor_events = ["rank(vec_avg(mdl110_analyst_sentiment)) > 0.8",
|
||||
"ts_rank(vec_avg(mdl110_analyst_sentiment),22) > 0.8",
|
||||
"rank(vec_avg(mws38_score)) > 0.8",
|
||||
"ts_rank(vec_avg(mws38_score),22) > 0.8"]
|
||||
|
||||
twn_events = ["rank(vec_avg(mdl109_news_sent_1m)) > 0.8",
|
||||
"ts_rank(vec_avg(mdl109_news_sent_1m),22) > 0.8",
|
||||
"rank(rp_ess_business) > 0.8",
|
||||
"ts_rank(rp_ess_business,22) > 0.8"]
|
||||
|
||||
for oe in open_events:
|
||||
for ee in exit_events:
|
||||
alpha = "%s(%s, %s, %s)"%(op, oe, field, ee)
|
||||
output.append(alpha)
|
||||
return output
|
||||
|
||||
|
||||
def check_submission(alpha_bag, gold_bag, start):
|
||||
depot = []
|
||||
s = login()
|
||||
for idx, g in enumerate(alpha_bag):
|
||||
if idx < start:
|
||||
continue
|
||||
if idx % 5 == 0:
|
||||
print(idx)
|
||||
if idx % 200 == 0:
|
||||
s = login()
|
||||
#print(idx)
|
||||
pc = get_check_submission(s, g)
|
||||
if pc == "sleep":
|
||||
sleep(100)
|
||||
s = login()
|
||||
alpha_bag.append(g)
|
||||
elif pc != pc:
|
||||
# pc is nan
|
||||
print("check self-corrlation error")
|
||||
sleep(100)
|
||||
alpha_bag.append(g)
|
||||
elif pc == "fail":
|
||||
continue
|
||||
elif pc == "error":
|
||||
depot.append(g)
|
||||
else:
|
||||
print(g)
|
||||
gold_bag.append((g, pc))
|
||||
print(depot)
|
||||
return gold_bag
|
||||
|
||||
def get_check_submission(s, alpha_id):
|
||||
while True:
|
||||
result = s.get("https://api.worldquantbrain.com/alphas/" + alpha_id + "/check")
|
||||
if "retry-after" in result.headers:
|
||||
time.sleep(float(result.headers["Retry-After"]))
|
||||
else:
|
||||
break
|
||||
try:
|
||||
if result.json().get("is", 0) == 0:
|
||||
print("logged out")
|
||||
return "sleep"
|
||||
checks_df = pd.DataFrame(
|
||||
result.json()["is"]["checks"]
|
||||
)
|
||||
pc = checks_df[checks_df.name == "PROD_CORRELATION"]["value"].values[0]
|
||||
if not any(checks_df["result"] == "FAIL"):
|
||||
return pc
|
||||
else:
|
||||
return "fail"
|
||||
except:
|
||||
print("catch: %s"%(alpha_id))
|
||||
return "error"
|
||||
|
||||
|
||||
def view_alphas(gold_bag):
|
||||
s = login()
|
||||
sharp_list = []
|
||||
for gold, pc in gold_bag:
|
||||
|
||||
triple = locate_alpha(s, gold)
|
||||
info = [triple[0], triple[2], triple[3], triple[4], triple[5], triple[6], triple[1]]
|
||||
info.append(pc)
|
||||
sharp_list.append(info)
|
||||
|
||||
sharp_list.sort(reverse=True, key = lambda x : x[1])
|
||||
for i in sharp_list:
|
||||
print(i)
|
||||
|
||||
def locate_alpha(s, alpha_id):
|
||||
while True:
|
||||
alpha = s.get("https://api.worldquantbrain.com/alphas/" + alpha_id)
|
||||
if "retry-after" in alpha.headers:
|
||||
time.sleep(float(alpha.headers["Retry-After"]))
|
||||
else:
|
||||
break
|
||||
string = alpha.content.decode('utf-8')
|
||||
metrics = json.loads(string)
|
||||
#print(metrics["regular"]["code"])
|
||||
|
||||
dateCreated = metrics["dateCreated"]
|
||||
sharpe = metrics["is"]["sharpe"]
|
||||
fitness = metrics["is"]["fitness"]
|
||||
turnover = metrics["is"]["turnover"]
|
||||
margin = metrics["is"]["margin"]
|
||||
decay = metrics["settings"]["decay"]
|
||||
exp = metrics['regular']['code']
|
||||
|
||||
triple = [alpha_id, exp, sharpe, turnover, fitness, margin, dateCreated, decay]
|
||||
return triple
|
||||
|
||||
|
||||
|
||||
# some factory for other operators
|
||||
def vector_factory(op, field):
|
||||
output = []
|
||||
vectors = ["cap"]
|
||||
|
||||
for vector in vectors:
|
||||
|
||||
alpha = "%s(%s, %s)"%(op, field, vector)
|
||||
output.append(alpha)
|
||||
|
||||
return output
|
||||
|
||||
|
||||
def ts_comp_factory(op, field, factor, paras):
|
||||
output = []
|
||||
#l1, l2 = [3, 5, 10, 20, 60, 120, 240], paras
|
||||
l1, l2 = [5, 22, 66, 240], paras
|
||||
comb = list(product(l1, l2))
|
||||
|
||||
for day,para in comb:
|
||||
|
||||
if type(para) == float:
|
||||
alpha = "%s(%s, %d, %s=%.1f)"%(op, field, day, factor, para)
|
||||
elif type(para) == int:
|
||||
alpha = "%s(%s, %d, %s=%d)"%(op, field, day, factor, para)
|
||||
|
||||
output.append(alpha)
|
||||
|
||||
return output
|
||||
|
||||
def twin_field_factory(op, field, fields):
|
||||
|
||||
output = []
|
||||
#days = [3, 5, 10, 20, 60, 120, 240]
|
||||
days = [5, 22, 66, 240]
|
||||
outset = list(set(fields) - set([field]))
|
||||
|
||||
for day in days:
|
||||
for counterpart in outset:
|
||||
alpha = "%s(%s, %s, %d)"%(op, field, counterpart, day)
|
||||
output.append(alpha)
|
||||
|
||||
return output
|
||||
|
||||
def login_hk():
|
||||
|
||||
username = ""
|
||||
password = ""
|
||||
|
||||
# Create a session to persistently store the headers
|
||||
s = requests.Session()
|
||||
|
||||
# Save credentials into session
|
||||
s.auth = (username, password)
|
||||
|
||||
# Send a POST request to the /authentication API
|
||||
response = s.post('https://api.worldquantbrain.com/authentication')
|
||||
|
||||
if response.status_code == requests.codes.unauthorized:
|
||||
# Check if biometrics is required
|
||||
if response.headers.get("WWW-Authenticate") == "persona":
|
||||
print(
|
||||
"Complete biometrics authentication by scanning your face. Follow the link: \n"
|
||||
+ urljoin(response.url, response.headers["Location"]) + "\n"
|
||||
)
|
||||
input("Press any key after you complete the biometrics authentication.")
|
||||
|
||||
# Retry the authentication after biometrics
|
||||
biometrics_response = s.post(urljoin(response.url, response.headers["Location"]))
|
||||
|
||||
while biometrics_response.status_code != 201:
|
||||
input("Biometrics authentication is not complete. Please try again and press any key when completed.")
|
||||
biometrics_response = s.post(urljoin(response.url, response.headers["Location"]))
|
||||
|
||||
print("Biometrics authentication completed.")
|
||||
else:
|
||||
print("\nIncorrect username or password. Please check your credentials.\n")
|
||||
else:
|
||||
print("Logged in successfully.")
|
||||
|
||||
return s
|
||||
@@ -0,0 +1,477 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"encoding/base64"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"math"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/valyala/fasthttp"
|
||||
)
|
||||
|
||||
const (
|
||||
baseURL = "https://api.worldquantbrain.com"
|
||||
zeroStreakThreshold = 5 * 252
|
||||
requiredDays = 2920
|
||||
)
|
||||
|
||||
type Client struct {
|
||||
client *fasthttp.Client
|
||||
username string
|
||||
password string
|
||||
}
|
||||
|
||||
type AlphaRecord struct {
|
||||
ID string `json:"id"`
|
||||
Name string `json:"name"`
|
||||
DateCreated string `json:"dateCreated"`
|
||||
Sharpe float64 `json:"sharpe"`
|
||||
Fitness float64 `json:"fitness"`
|
||||
Turnover float64 `json:"turnover"`
|
||||
Margin float64 `json:"margin"`
|
||||
LongCount float64 `json:"longCount"`
|
||||
ShortCount float64 `json:"shortCount"`
|
||||
Decay int `json:"decay"`
|
||||
Code string `json:"code"`
|
||||
}
|
||||
|
||||
type AlphaResponse struct {
|
||||
Results []struct {
|
||||
ID string `json:"id"`
|
||||
Name string `json:"name"`
|
||||
DateCreated string `json:"dateCreated"`
|
||||
Is map[string]interface{} `json:"is"`
|
||||
Settings struct {
|
||||
Decay int `json:"decay"`
|
||||
} `json:"settings"`
|
||||
Regular struct {
|
||||
Code string `json:"code"`
|
||||
} `json:"regular"`
|
||||
} `json:"results"`
|
||||
}
|
||||
|
||||
type PnlResponse struct {
|
||||
Records [][]interface{} `json:"records"`
|
||||
}
|
||||
|
||||
func NewClient(username, password string) *Client {
|
||||
return &Client{
|
||||
client: &fasthttp.Client{},
|
||||
username: username,
|
||||
password: password,
|
||||
}
|
||||
}
|
||||
|
||||
func (c *Client) getAuthHeader() string {
|
||||
auth := base64.StdEncoding.EncodeToString([]byte(c.username + ":" + c.password))
|
||||
return "Basic " + auth
|
||||
}
|
||||
|
||||
func (c *Client) Login() error {
|
||||
req := fasthttp.AcquireRequest()
|
||||
resp := fasthttp.AcquireResponse()
|
||||
defer fasthttp.ReleaseRequest(req)
|
||||
defer fasthttp.ReleaseResponse(resp)
|
||||
|
||||
req.SetRequestURI(baseURL + "/authentication")
|
||||
req.Header.SetMethod("POST")
|
||||
req.Header.Set("Authorization", c.getAuthHeader())
|
||||
|
||||
err := c.client.Do(req, resp)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
fmt.Println(string(resp.Body()))
|
||||
return nil
|
||||
}
|
||||
|
||||
func (c *Client) WaitGet(url string, maxRetries int) (*fasthttp.Response, error) {
|
||||
retries := 0
|
||||
for retries < maxRetries {
|
||||
for {
|
||||
req := fasthttp.AcquireRequest()
|
||||
resp := fasthttp.AcquireResponse()
|
||||
req.SetRequestURI(url)
|
||||
req.Header.SetMethod("GET")
|
||||
req.Header.Set("Authorization", c.getAuthHeader())
|
||||
|
||||
err := c.client.Do(req, resp)
|
||||
if err != nil {
|
||||
fasthttp.ReleaseRequest(req)
|
||||
fasthttp.ReleaseResponse(resp)
|
||||
return nil, err
|
||||
}
|
||||
|
||||
retryAfter := resp.Header.Peek("Retry-After")
|
||||
if len(retryAfter) == 0 {
|
||||
fasthttp.ReleaseRequest(req)
|
||||
if resp.StatusCode() < 400 {
|
||||
return resp, nil
|
||||
}
|
||||
fasthttp.ReleaseResponse(resp)
|
||||
break
|
||||
}
|
||||
|
||||
sleepSec, _ := strconv.ParseFloat(string(retryAfter), 64)
|
||||
time.Sleep(time.Duration(sleepSec) * time.Second)
|
||||
fasthttp.ReleaseRequest(req)
|
||||
fasthttp.ReleaseResponse(resp)
|
||||
}
|
||||
|
||||
time.Sleep(time.Duration(math.Pow(2, float64(retries))) * time.Second)
|
||||
retries++
|
||||
}
|
||||
|
||||
return nil, fmt.Errorf("max retries exceeded")
|
||||
}
|
||||
|
||||
func (c *Client) Get(url string) (*fasthttp.Response, error) {
|
||||
req := fasthttp.AcquireRequest()
|
||||
resp := fasthttp.AcquireResponse()
|
||||
req.SetRequestURI(url)
|
||||
req.Header.SetMethod("GET")
|
||||
req.Header.Set("Authorization", c.getAuthHeader())
|
||||
|
||||
err := c.client.Do(req, resp)
|
||||
if err != nil {
|
||||
fasthttp.ReleaseRequest(req)
|
||||
fasthttp.ReleaseResponse(resp)
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return resp, nil
|
||||
}
|
||||
|
||||
func (c *Client) Patch(url string, data map[string]interface{}) (*fasthttp.Response, error) {
|
||||
req := fasthttp.AcquireRequest()
|
||||
resp := fasthttp.AcquireResponse()
|
||||
defer fasthttp.ReleaseRequest(req)
|
||||
|
||||
req.SetRequestURI(url)
|
||||
req.Header.SetMethod("PATCH")
|
||||
req.Header.Set("Authorization", c.getAuthHeader())
|
||||
req.Header.SetContentType("application/json")
|
||||
|
||||
jsonData, _ := json.Marshal(data)
|
||||
req.SetBody(jsonData)
|
||||
|
||||
err := c.client.Do(req, resp)
|
||||
if err != nil {
|
||||
fasthttp.ReleaseResponse(resp)
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return resp, nil
|
||||
}
|
||||
|
||||
func GetAlphas(c *Client, startDate, endDate string, sharpeTh, fitnessTh float64, region string, alphaNum int, usage string) ([][]interface{}, *Client, error) {
|
||||
output := make([][]interface{}, 0)
|
||||
count := 0
|
||||
|
||||
for i := 0; i < alphaNum; i += 100 {
|
||||
fmt.Println(i)
|
||||
|
||||
urlE := fmt.Sprintf("%s/users/self/alphas?limit=100&offset=%d&status=UNSUBMITTED%%1FIS_FAIL&dateCreated%%3E=2025-%sT00:00:00-04:00&dateCreated%%3C2025-%sT00:00:00-04:00&is.fitness%%3E%f&is.sharpe%%3E%f&settings.region=%s&order=-is.sharpe&hidden=false&type!=SUPER",
|
||||
baseURL, i, startDate, endDate, fitnessTh, sharpeTh, region)
|
||||
|
||||
urlC := fmt.Sprintf("%s/users/self/alphas?limit=100&offset=%d&status=UNSUBMITTED%%1FIS_FAIL&dateCreated%%3E=2025-%sT00:00:00-04:00&dateCreated%%3C2025-%sT00:00:00-04:00&is.fitness%%3C-%f&is.sharpe%%3C-%f&settings.region=%s&order=is.sharpe&hidden=false&type!=SUPER",
|
||||
baseURL, i, startDate, endDate, fitnessTh, sharpeTh, region)
|
||||
|
||||
urls := []string{urlE}
|
||||
if usage != "submit" {
|
||||
urls = append(urls, urlC)
|
||||
}
|
||||
|
||||
for _, url := range urls {
|
||||
resp, err := c.Get(url)
|
||||
if err != nil {
|
||||
fmt.Printf("%d finished re-login\n", i)
|
||||
c.Login()
|
||||
continue
|
||||
}
|
||||
|
||||
var alphaResp AlphaResponse
|
||||
if err := json.Unmarshal(resp.Body(), &alphaResp); err != nil {
|
||||
fasthttp.ReleaseResponse(resp)
|
||||
fmt.Printf("%d finished re-login\n", i)
|
||||
c.Login()
|
||||
continue
|
||||
}
|
||||
fasthttp.ReleaseResponse(resp)
|
||||
|
||||
for _, item := range alphaResp.Results {
|
||||
alphaID := item.ID
|
||||
name := item.Name
|
||||
dateCreated := item.DateCreated
|
||||
sharpe := getFloat(item.Is, "sharpe")
|
||||
fitness := getFloat(item.Is, "fitness")
|
||||
turnover := getFloat(item.Is, "turnover")
|
||||
margin := getFloat(item.Is, "margin")
|
||||
longCount := getFloat(item.Is, "longCount")
|
||||
shortCount := getFloat(item.Is, "shortCount")
|
||||
decay := item.Settings.Decay
|
||||
exp := item.Regular.Code
|
||||
|
||||
count++
|
||||
|
||||
if (longCount + shortCount) > 100 {
|
||||
if sharpe < -sharpeTh {
|
||||
exp = "-" + exp
|
||||
}
|
||||
|
||||
rec := []interface{}{alphaID, exp, sharpe, turnover, fitness, margin, dateCreated, decay}
|
||||
fmt.Println(rec)
|
||||
|
||||
if turnover > 0.7 {
|
||||
rec = append(rec, float64(decay)*4)
|
||||
} else if turnover > 0.6 {
|
||||
rec = append(rec, float64(decay)*3+3)
|
||||
} else if turnover > 0.5 {
|
||||
rec = append(rec, float64(decay)*3)
|
||||
} else if turnover > 0.4 {
|
||||
rec = append(rec, float64(decay)*2)
|
||||
} else if turnover > 0.35 {
|
||||
rec = append(rec, float64(decay)+4)
|
||||
} else if turnover > 0.3 {
|
||||
rec = append(rec, float64(decay)+2)
|
||||
}
|
||||
|
||||
output = append(output, rec)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fmt.Printf("count: %d\n", count)
|
||||
return output, c, nil
|
||||
}
|
||||
|
||||
func getFloat(m map[string]interface{}, key string) float64 {
|
||||
if v, ok := m[key]; ok {
|
||||
switch val := v.(type) {
|
||||
case float64:
|
||||
return val
|
||||
case int:
|
||||
return float64(val)
|
||||
}
|
||||
}
|
||||
return 0
|
||||
}
|
||||
|
||||
func CheckConsecutiveNonZeroValues(alphaID string, data [][]interface{}, requiredStreak int) bool {
|
||||
if len(data) < requiredStreak {
|
||||
return true
|
||||
}
|
||||
|
||||
checkColumn := func(columnData []float64) bool {
|
||||
if len(columnData) < requiredStreak {
|
||||
return true
|
||||
}
|
||||
|
||||
currentStreakCount := 0
|
||||
var currentStreakValue interface{}
|
||||
|
||||
for _, value := range columnData {
|
||||
if value != 0 {
|
||||
if currentStreakValue != nil && value == currentStreakValue {
|
||||
currentStreakCount++
|
||||
} else {
|
||||
currentStreakValue = value
|
||||
currentStreakCount = 1
|
||||
}
|
||||
} else {
|
||||
currentStreakValue = nil
|
||||
currentStreakCount = 0
|
||||
}
|
||||
|
||||
if currentStreakCount >= requiredStreak {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
var column1Values, column2Values []float64
|
||||
for _, row := range data {
|
||||
if len(row) >= 3 {
|
||||
if v, ok := row[1].(float64); ok {
|
||||
column1Values = append(column1Values, v)
|
||||
}
|
||||
if v, ok := row[2].(float64); ok {
|
||||
column2Values = append(column2Values, v)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if len(column1Values) > 0 && len(column2Values) > 0 {
|
||||
isCol1AllZeros := allZeros(column1Values)
|
||||
isCol2AllZeros := allZeros(column2Values)
|
||||
if isCol1AllZeros || isCol2AllZeros {
|
||||
fmt.Println(alphaID, "不合法")
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
if !checkColumn(column1Values) {
|
||||
fmt.Println(alphaID, "不合法")
|
||||
return false
|
||||
}
|
||||
|
||||
if !checkColumn(column2Values) {
|
||||
fmt.Println(alphaID, "不合法")
|
||||
return false
|
||||
}
|
||||
|
||||
return true
|
||||
}
|
||||
|
||||
func allZeros(arr []float64) bool {
|
||||
for _, v := range arr {
|
||||
if v != 0 {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
func GetAlphaPnlLegal(c *Client, alphaID string) bool {
|
||||
notLegalID := make([]string, 0)
|
||||
|
||||
url := baseURL + "/alphas/" + alphaID + "/recordsets/pnl"
|
||||
resp, err := c.WaitGet(url, 10)
|
||||
if err != nil {
|
||||
return false
|
||||
}
|
||||
defer fasthttp.ReleaseResponse(resp)
|
||||
|
||||
var pnlResp PnlResponse
|
||||
if err := json.Unmarshal(resp.Body(), &pnlResp); err != nil {
|
||||
return false
|
||||
}
|
||||
|
||||
records := pnlResp.Records
|
||||
if len(records) == 0 {
|
||||
return false
|
||||
}
|
||||
|
||||
var dateList []time.Time
|
||||
for _, record := range records {
|
||||
if len(record) == 0 {
|
||||
continue
|
||||
}
|
||||
dateStr, ok := record[0].(string)
|
||||
if !ok {
|
||||
return false
|
||||
}
|
||||
dateObj, err := time.Parse("2006-01-02", dateStr)
|
||||
if err != nil {
|
||||
return false
|
||||
}
|
||||
dateList = append(dateList, dateObj)
|
||||
}
|
||||
|
||||
if len(dateList) == 0 {
|
||||
return false
|
||||
}
|
||||
|
||||
minDate := dateList[0]
|
||||
maxDate := dateList[0]
|
||||
for _, d := range dateList {
|
||||
if d.Before(minDate) {
|
||||
minDate = d
|
||||
}
|
||||
if d.After(maxDate) {
|
||||
maxDate = d
|
||||
}
|
||||
}
|
||||
|
||||
totalDays := int(maxDate.Sub(minDate).Hours() / 24)
|
||||
if totalDays < requiredDays {
|
||||
return false
|
||||
}
|
||||
|
||||
col1Zeros := make([]bool, 0)
|
||||
for _, record := range records {
|
||||
if len(record) >= 2 {
|
||||
if v, ok := record[1].(float64); ok {
|
||||
col1Zeros = append(col1Zeros, v == 0)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
col1MaxZeroStreak := maxConsecutiveZeros(col1Zeros)
|
||||
|
||||
if col1MaxZeroStreak >= zeroStreakThreshold {
|
||||
fmt.Printf("%s 不合法：存在连续%d年零值\n", alphaID, zeroStreakThreshold/252)
|
||||
notLegalID = append(notLegalID, alphaID)
|
||||
return false
|
||||
}
|
||||
|
||||
if !CheckConsecutiveNonZeroValues(alphaID, records, 200) {
|
||||
return false
|
||||
}
|
||||
|
||||
_ = notLegalID
|
||||
return true
|
||||
}
|
||||
|
||||
func maxConsecutiveZeros(arr []bool) int {
|
||||
maxStreak := 0
|
||||
currentStreak := 0
|
||||
for _, val := range arr {
|
||||
if val {
|
||||
currentStreak++
|
||||
if currentStreak > maxStreak {
|
||||
maxStreak = currentStreak
|
||||
}
|
||||
} else {
|
||||
currentStreak = 0
|
||||
}
|
||||
}
|
||||
return maxStreak
|
||||
}
|
||||
|
||||
func Mute(c *Client, alphaID string) {
|
||||
url := baseURL + "/alphas/" + alphaID
|
||||
data := map[string]interface{}{
|
||||
"hidden": true,
|
||||
}
|
||||
c.Patch(url, data)
|
||||
}
|
||||
|
||||
func main() {
|
||||
client := NewClient("", "")
|
||||
client.Login()
|
||||
|
||||
foTracker, c, err := GetAlphas(client, "12-01", "12-31", 1, 0.5, "USA", 1000, "submit")
|
||||
if err != nil {
|
||||
fmt.Println("Error:", err)
|
||||
return
|
||||
}
|
||||
|
||||
fNum := len(foTracker)
|
||||
fmt.Printf("%d 个alpha 进行pnl合法检测，请耐心等待\n", fNum)
|
||||
fmt.Println(len(foTracker))
|
||||
|
||||
count := 0
|
||||
for i := len(foTracker) - 1; i >= 0; i-- {
|
||||
if count%25 == 0 {
|
||||
fmt.Printf("=========== %d ===========\n", count)
|
||||
}
|
||||
count++
|
||||
|
||||
alphaID, ok := foTracker[i][0].(string)
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
|
||||
if !GetAlphaPnlLegal(c, alphaID) {
|
||||
fmt.Println(alphaID, "已经隐藏")
|
||||
Mute(c, alphaID)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,223 @@
|
||||
import time
|
||||
import httpx
|
||||
import datetime
|
||||
|
||||
|
||||
def login():
|
||||
username = "jack0210_@hotmail.com"
|
||||
password = "!QAZ2wsx+0913"
|
||||
|
||||
timeout = httpx.Timeout(60.0, connect=10.0)
|
||||
limits = httpx.Limits(max_keepalive_connections=20, max_connections=100)
|
||||
transport = httpx.HTTPTransport(retries=3)
|
||||
|
||||
s = httpx.Client(
|
||||
auth=(username, password),
|
||||
timeout=timeout,
|
||||
limits=limits,
|
||||
transport=transport
|
||||
)
|
||||
response = s.post('https://api.worldquantbrain.com/authentication')
|
||||
print(response.content)
|
||||
return s
|
||||
|
||||
|
||||
def wait_get(s, url: str, max_retries: int = 10):
|
||||
retries = 0
|
||||
while retries < max_retries:
|
||||
while True:
|
||||
simulation_progress = s.get(url)
|
||||
if simulation_progress.headers.get("Retry-After", 0) == 0:
|
||||
break
|
||||
time.sleep(float(simulation_progress.headers["Retry-After"]))
|
||||
if simulation_progress.status_code < 400:
|
||||
break
|
||||
else:
|
||||
time.sleep(2 ** retries)
|
||||
retries += 1
|
||||
return simulation_progress
|
||||
|
||||
|
||||
def get_alphas(start_date, end_date, sharpe_th, fitness_th, region, alpha_num, usage):
|
||||
s = login()
|
||||
output = []
|
||||
count = 0
|
||||
for i in range(0, alpha_num, 100):
|
||||
print(i)
|
||||
url_e = "https://api.worldquantbrain.com/users/self/alphas?limit=100&offset=%d" % (i) \
|
||||
+ "&status=UNSUBMITTED%1FIS_FAIL&dateCreated%3E=2025-" + start_date \
|
||||
+ "T00:00:00-04:00&dateCreated%3C2025-" + end_date \
|
||||
+ "T00:00:00-04:00&is.fitness%3E" + str(fitness_th) + "&is.sharpe%3E" \
|
||||
+ str(sharpe_th) + "&settings.region=" + region + "&order=-is.sharpe&hidden=false&type!=SUPER"
|
||||
url_c = "https://api.worldquantbrain.com/users/self/alphas?limit=100&offset=%d" % (i) \
|
||||
+ "&status=UNSUBMITTED%1FIS_FAIL&dateCreated%3E=2025-" + start_date \
|
||||
+ "T00:00:00-04:00&dateCreated%3C2025-" + end_date \
|
||||
+ "T00:00:00-04:00&is.fitness%3C-" + str(fitness_th) + "&is.sharpe%3C-" \
|
||||
+ str(sharpe_th) + "&settings.region=" + region + "&order=is.sharpe&hidden=false&type!=SUPER"
|
||||
urls = [url_e]
|
||||
if usage != "submit":
|
||||
urls.append(url_c)
|
||||
for url in urls:
|
||||
response = s.get(url)
|
||||
try:
|
||||
alpha_list = response.json()["results"]
|
||||
for j in range(len(alpha_list)):
|
||||
alpha_id = alpha_list[j]["id"]
|
||||
name = alpha_list[j]["name"]
|
||||
dateCreated = alpha_list[j]["dateCreated"]
|
||||
sharpe = alpha_list[j]["is"]["sharpe"]
|
||||
fitness = alpha_list[j]["is"]["fitness"]
|
||||
turnover = alpha_list[j]["is"]["turnover"]
|
||||
margin = alpha_list[j]["is"]["margin"]
|
||||
longCount = alpha_list[j]["is"]["longCount"]
|
||||
shortCount = alpha_list[j]["is"]["shortCount"]
|
||||
decay = alpha_list[j]["settings"]["decay"]
|
||||
exp = alpha_list[j]['regular']['code']
|
||||
count += 1
|
||||
if (longCount + shortCount) > 100:
|
||||
if sharpe < -sharpe_th:
|
||||
exp = "-%s" % exp
|
||||
rec = [alpha_id, exp, sharpe, turnover, fitness, margin, dateCreated, decay]
|
||||
print(rec)
|
||||
if turnover > 0.7:
|
||||
rec.append(decay * 4)
|
||||
elif turnover > 0.6:
|
||||
rec.append(decay * 3 + 3)
|
||||
elif turnover > 0.5:
|
||||
rec.append(decay * 3)
|
||||
elif turnover > 0.4:
|
||||
rec.append(decay * 2)
|
||||
elif turnover > 0.35:
|
||||
rec.append(decay + 4)
|
||||
elif turnover > 0.3:
|
||||
rec.append(decay + 2)
|
||||
output.append(rec)
|
||||
except:
|
||||
print("%d finished re-login" % i)
|
||||
s = login()
|
||||
|
||||
print("count: %d" % count)
|
||||
return output, s
|
||||
|
||||
|
||||
def check_consecutive_non_zero_values(alpha_id, data, required_streak=200):
|
||||
if not data or len(data) < required_streak:
|
||||
return True
|
||||
|
||||
def check_column(column_data):
|
||||
if len(column_data) < required_streak:
|
||||
return True
|
||||
|
||||
current_streak_count = 0
|
||||
current_streak_value = None
|
||||
|
||||
for value in column_data:
|
||||
if value != 0:
|
||||
if value == current_streak_value:
|
||||
current_streak_count += 1
|
||||
else:
|
||||
current_streak_value = value
|
||||
current_streak_count = 1
|
||||
else:
|
||||
current_streak_value = None
|
||||
current_streak_count = 0
|
||||
|
||||
if current_streak_count >= required_streak:
|
||||
return False
|
||||
return True
|
||||
|
||||
column1_values = []
|
||||
column2_values = []
|
||||
for row in data:
|
||||
if len(row) >= 3:
|
||||
column1_values.append(row[1])
|
||||
column2_values.append(row[2])
|
||||
|
||||
if column1_values and column2_values:
|
||||
is_col1_all_zeros = all(v == 0 for v in column1_values)
|
||||
is_col2_all_zeros = all(v == 0 for v in column2_values)
|
||||
if is_col1_all_zeros or is_col2_all_zeros:
|
||||
print(alpha_id, "不合法")
|
||||
return False
|
||||
|
||||
if not check_column(column1_values):
|
||||
print(alpha_id, "不合法")
|
||||
return False
|
||||
|
||||
if not check_column(column2_values):
|
||||
print(alpha_id, "不合法")
|
||||
return False
|
||||
|
||||
return True
|
||||
|
||||
|
||||
def get_alpha_pnl_legal(s, alpha_id: str) -> bool:
|
||||
not_legal_id = []
|
||||
pnl = wait_get(s, "https://api.worldquantbrain.com/alphas/" + alpha_id + "/recordsets/pnl").json()
|
||||
records = pnl["records"]
|
||||
|
||||
if not records:
|
||||
return False
|
||||
|
||||
date_list = []
|
||||
for record in records:
|
||||
try:
|
||||
date_obj = datetime.datetime.strptime(record[0], '%Y-%m-%d').date()
|
||||
date_list.append(date_obj)
|
||||
except Exception:
|
||||
return False
|
||||
|
||||
min_date = min(date_list)
|
||||
max_date = max(date_list)
|
||||
total_days = (max_date - min_date).days
|
||||
if total_days < 2920:
|
||||
return False
|
||||
|
||||
zero_streak_threshold = 5 * 252
|
||||
col1_zeros = [record[1] == 0 for record in records]
|
||||
|
||||
def max_consecutive_zeros(arr):
|
||||
max_streak = current_streak = 0
|
||||
for val in arr:
|
||||
current_streak = current_streak + 1 if val else 0
|
||||
max_streak = max(max_streak, current_streak)
|
||||
return max_streak
|
||||
|
||||
col1_max_zero_streak = max_consecutive_zeros(col1_zeros)
|
||||
|
||||
if col1_max_zero_streak >= zero_streak_threshold:
|
||||
print(f"{alpha_id} 不合法：存在连续{zero_streak_threshold // 252}年零值")
|
||||
not_legal_id.append(str(alpha_id))
|
||||
return False
|
||||
|
||||
if not check_consecutive_non_zero_values(alpha_id, records):
|
||||
return False
|
||||
|
||||
return True
|
||||
|
||||
|
||||
def mute(s, alpha_id):
|
||||
url = "https://api.worldquantbrain.com/alphas/" + alpha_id
|
||||
data = {
|
||||
"hidden": True
|
||||
}
|
||||
response = s.patch(url, json=data)
|
||||
|
||||
|
||||
def main():
|
||||
fo_tracker, s = get_alphas('12-01', '12-31', 1, 0.5, 'USA', 1000, 'submit')
|
||||
f_num = len(fo_tracker)
|
||||
print(f_num, "个alpha 进行pnl合法检测，请耐心等待")
|
||||
count = 0
|
||||
print(len(fo_tracker))
|
||||
for i in fo_tracker[::-1][0:]:
|
||||
if count % 25 == 0:
|
||||
print('===========', count, '===========')
|
||||
count += 1
|
||||
if get_alpha_pnl_legal(s, i[0]) == False:
|
||||
print(i[0], '已经隐藏')
|
||||
mute(s, i[0])
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Reference in New Issue
Block a user