++
This commit is contained in:
jack
@@ -0,0 +1,169 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
import os
|
||||
import json
|
||||
import openai
|
||||
import httpx
|
||||
from datetime import datetime
|
||||
|
||||
|
||||
def load_config(config_file="ai_config.json"):
|
||||
try:
|
||||
with open(config_file, 'r', encoding='utf-8') as f:
|
||||
return json.load(f)
|
||||
except FileNotFoundError:
|
||||
print(f"创建配置文件: {config_file}")
|
||||
with open(config_file, 'w', encoding='utf-8') as f:
|
||||
json.dump({}, f, indent=4)
|
||||
print("请在config.json中配置siliconflow的api信息")
|
||||
return None
|
||||
|
||||
|
||||
def create_prompt_file():
|
||||
if not os.path.exists("alpha_prompt.txt"):
|
||||
with open("alpha_prompt.txt", 'w', encoding='utf-8') as f:
|
||||
f.write("")
|
||||
print("已创建alpha_prompt.txt，请填入提示词后重新运行")
|
||||
return False
|
||||
return True
|
||||
|
||||
|
||||
def read_prompt():
|
||||
with open("alpha_prompt.txt", 'r', encoding='utf-8') as f:
|
||||
prompt = f.read().strip()
|
||||
if not prompt:
|
||||
print("alpha_prompt.txt是空的，请填入提示词")
|
||||
return None
|
||||
return prompt
|
||||
|
||||
def read_operator():
|
||||
with open("wqb_operator.txt", 'r', encoding='utf-8') as f:
|
||||
operator = f.read().strip()
|
||||
if not operator:
|
||||
print("operator.txt是空的，请填入操作符")
|
||||
return None
|
||||
return operator
|
||||
|
||||
|
||||
def create_result_folder():
|
||||
# 修改这里：创建 generated_alpha 文件夹
|
||||
folder_name = "generated_alpha"
|
||||
if not os.path.exists(folder_name):
|
||||
os.makedirs(folder_name)
|
||||
return folder_name
|
||||
|
||||
|
||||
def call_siliconflow(api_key, prompt, model, base_url):
|
||||
try:
|
||||
client = openai.OpenAI(
|
||||
api_key=api_key,
|
||||
base_url=base_url
|
||||
)
|
||||
|
||||
response = client.chat.completions.create(
|
||||
model=model,
|
||||
messages=[{"role": "user", "content": prompt}]
|
||||
)
|
||||
|
||||
return response.choices[0].message.content
|
||||
|
||||
except openai.AuthenticationError:
|
||||
print("API密钥错误")
|
||||
except openai.RateLimitError:
|
||||
print("调用频率限制")
|
||||
except openai.APIError as e:
|
||||
print(f"API错误: {e}")
|
||||
except Exception as e:
|
||||
print(f"其他错误: {e}")
|
||||
return None
|
||||
|
||||
|
||||
def save_result(result, folder):
|
||||
timestamp = datetime.now().strftime("%Y%m%d%H%M%S")
|
||||
filename = f"{timestamp}.txt"
|
||||
filepath = os.path.join(folder, filename)
|
||||
|
||||
with open(filepath, 'w', encoding='utf-8') as f:
|
||||
f.write(result)
|
||||
print(f"结果保存到: {filepath}")
|
||||
|
||||
|
||||
def get_ai_config(config):
|
||||
if not config:
|
||||
return None
|
||||
|
||||
service_name = list(config.keys())[0]
|
||||
service_config = config[service_name]
|
||||
|
||||
required_fields = ["api_keys", "model", "base_url"]
|
||||
for field in required_fields:
|
||||
if field not in service_config:
|
||||
print(f"缺少配置字段: {field}")
|
||||
return None
|
||||
|
||||
return service_name, service_config
|
||||
|
||||
def get_user_info():
|
||||
# 获取用户信息
|
||||
with open('ai_config.json', 'r') as f:
|
||||
config = json.load(f)
|
||||
siliconflow = config['siliconflow']
|
||||
token = siliconflow['api_keys']
|
||||
|
||||
headers = {"Authorization": f"Bearer {token}"}
|
||||
|
||||
url = "https://api.siliconflow.cn/v1/user/info"
|
||||
|
||||
response = httpx.get(url, headers=headers)
|
||||
|
||||
data = response.json()['data']
|
||||
balance = data['balance']
|
||||
print(f"余额: {balance}")
|
||||
|
||||
|
||||
def main():
|
||||
get_user_info()
|
||||
|
||||
config = load_config()
|
||||
if not config:
|
||||
return
|
||||
|
||||
if not create_prompt_file():
|
||||
return
|
||||
|
||||
prompt = read_prompt()
|
||||
if not prompt:
|
||||
return
|
||||
|
||||
config_result = get_ai_config(config)
|
||||
if not config_result:
|
||||
return
|
||||
|
||||
operator = read_operator()
|
||||
|
||||
if operator:
|
||||
prompt = prompt + "\n\n以下是我的账号有权限使用的操作符以及操作符的使用方法, 请严格按照操作符, 进行生成,组合因子\n\n" + operator
|
||||
|
||||
service_name, service_config = config_result
|
||||
|
||||
print(f"使用服务: {service_name}")
|
||||
print(f"使用模型: {service_config['model']}")
|
||||
|
||||
folder = create_result_folder()
|
||||
|
||||
print("正在调用AI...")
|
||||
result = call_siliconflow(
|
||||
service_config["api_keys"],
|
||||
prompt,
|
||||
service_config["model"],
|
||||
service_config["base_url"]
|
||||
)
|
||||
|
||||
if result:
|
||||
print(f"AI回复: {result[:200]}...")
|
||||
save_result(result, folder)
|
||||
else:
|
||||
print("AI调用失败")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,396 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
import os
|
||||
import time
|
||||
import json
|
||||
import httpx
|
||||
import psycopg2
|
||||
from httpx import BasicAuth
|
||||
|
||||
|
||||
class DatabaseManager:
|
||||
def __init__(self):
|
||||
self.connection = None
|
||||
self.database_name = "alpha"
|
||||
self.init_database()
|
||||
|
||||
def create_database(self):
|
||||
"""创建数据库（如果不存在）"""
|
||||
try:
|
||||
# 先连接到默认的postgres数据库来创建alpha数据库
|
||||
conn = psycopg2.connect(
|
||||
host="localhost",
|
||||
port="5432",
|
||||
database="postgres",
|
||||
user="jack",
|
||||
password="aaaAAA111"
|
||||
)
|
||||
conn.autocommit = True
|
||||
cursor = conn.cursor()
|
||||
|
||||
# 检查数据库是否存在
|
||||
cursor.execute("SELECT 1 FROM pg_catalog.pg_database WHERE datname = %s", (self.database_name,))
|
||||
exists = cursor.fetchone()
|
||||
|
||||
if not exists:
|
||||
cursor.execute(f"CREATE DATABASE {self.database_name}")
|
||||
print(f"数据库 {self.database_name} 创建成功")
|
||||
else:
|
||||
print(f"数据库 {self.database_name} 已存在")
|
||||
|
||||
cursor.close()
|
||||
conn.close()
|
||||
|
||||
except Exception as e:
|
||||
print(f"创建数据库时出错: {e}")
|
||||
raise
|
||||
|
||||
def get_connection(self):
|
||||
"""获取数据库连接"""
|
||||
if self.connection is None or self.connection.closed:
|
||||
self.connection = psycopg2.connect(
|
||||
host="localhost",
|
||||
port="5432",
|
||||
database=self.database_name,
|
||||
user="jack",
|
||||
password="aaaAAA111"
|
||||
)
|
||||
return self.connection
|
||||
|
||||
def init_database(self):
|
||||
"""初始化数据库和表结构"""
|
||||
# 先创建数据库
|
||||
self.create_database()
|
||||
|
||||
# 然后连接到此数据库创建表
|
||||
conn = self.get_connection()
|
||||
cursor = conn.cursor()
|
||||
|
||||
# 创建 alpha_prepare 表
|
||||
cursor.execute('''
|
||||
CREATE TABLE IF NOT EXISTS alpha_prepare
|
||||
(
|
||||
id
|
||||
SERIAL
|
||||
PRIMARY
|
||||
KEY,
|
||||
alpha
|
||||
TEXT
|
||||
NOT
|
||||
NULL
|
||||
UNIQUE,
|
||||
unused
|
||||
BOOLEAN
|
||||
NOT
|
||||
NULL
|
||||
DEFAULT
|
||||
TRUE,
|
||||
created_time
|
||||
TIMESTAMP
|
||||
DEFAULT
|
||||
CURRENT_TIMESTAMP
|
||||
)
|
||||
''')
|
||||
|
||||
# 创建 simulation 表
|
||||
cursor.execute('''
|
||||
CREATE TABLE IF NOT EXISTS simulation
|
||||
(
|
||||
id
|
||||
SERIAL
|
||||
PRIMARY
|
||||
KEY,
|
||||
expression
|
||||
TEXT
|
||||
NOT
|
||||
NULL,
|
||||
time_consuming
|
||||
REAL
|
||||
NOT
|
||||
NULL,
|
||||
status
|
||||
TEXT
|
||||
NOT
|
||||
NULL,
|
||||
timestamp
|
||||
TEXT
|
||||
NOT
|
||||
NULL,
|
||||
alpha_id
|
||||
TEXT,
|
||||
message
|
||||
TEXT,
|
||||
created_time
|
||||
TIMESTAMP
|
||||
DEFAULT
|
||||
CURRENT_TIMESTAMP
|
||||
)
|
||||
''')
|
||||
|
||||
conn.commit()
|
||||
print(f"数据库 {self.database_name} 表结构初始化完成")
|
||||
|
||||
def get_unused_alpha(self):
|
||||
"""获取所有未使用的alpha表达式"""
|
||||
conn = self.get_connection()
|
||||
cursor = conn.cursor()
|
||||
|
||||
cursor.execute("SELECT alpha FROM alpha_prepare WHERE unused = TRUE")
|
||||
results = cursor.fetchall()
|
||||
|
||||
alpha_list = [result[0] for result in results]
|
||||
return alpha_list
|
||||
|
||||
def mark_alpha_used(self, alpha):
|
||||
"""将alpha标记为已使用"""
|
||||
conn = self.get_connection()
|
||||
cursor = conn.cursor()
|
||||
|
||||
cursor.execute("UPDATE alpha_prepare SET unused = FALSE WHERE alpha = %s", (alpha,))
|
||||
conn.commit()
|
||||
|
||||
def insert_simulation_result(self, result_data):
|
||||
"""插入模拟结果到simulation表"""
|
||||
conn = self.get_connection()
|
||||
cursor = conn.cursor()
|
||||
|
||||
cursor.execute('''
|
||||
INSERT INTO simulation
|
||||
(expression, time_consuming, status, timestamp, alpha_id, message)
|
||||
VALUES (%s, %s, %s, %s, %s, %s)
|
||||
''', (
|
||||
result_data["expression"],
|
||||
result_data["time_consuming"],
|
||||
result_data["status"],
|
||||
result_data["timestamp"],
|
||||
result_data.get("alpha_id"),
|
||||
result_data.get("message", "")
|
||||
))
|
||||
|
||||
conn.commit()
|
||||
|
||||
def close_connection(self):
|
||||
"""关闭数据库连接"""
|
||||
if self.connection and not self.connection.closed:
|
||||
self.connection.close()
|
||||
|
||||
|
||||
class AlphaSimulator:
|
||||
def __init__(self, credentials_file='account.txt'):
|
||||
self.credentials_file = credentials_file
|
||||
self.client = None
|
||||
self.brain_api_url = 'https://api.worldquantbrain.com'
|
||||
self.db_manager = DatabaseManager()
|
||||
|
||||
def __del__(self):
|
||||
"""析构函数，确保数据库连接被关闭"""
|
||||
if hasattr(self, 'db_manager'):
|
||||
self.db_manager.close_connection()
|
||||
|
||||
def load_credentials(self):
|
||||
if not os.path.exists(self.credentials_file):
|
||||
print("未找到 account.txt 文件")
|
||||
with open(self.credentials_file, 'w') as f:
|
||||
f.write("")
|
||||
print("account.txt 文件已创建，请填写账号密码, 格式: ['username', 'password]")
|
||||
exit(1)
|
||||
|
||||
with open(self.credentials_file) as f:
|
||||
credentials = eval(f.read())
|
||||
return credentials[0], credentials[1]
|
||||
|
||||
def login(self):
|
||||
username, password = self.load_credentials()
|
||||
self.client = httpx.Client(auth=BasicAuth(username, password))
|
||||
|
||||
response = self.client.post(f'{self.brain_api_url}/authentication')
|
||||
print(f"登录状态: {response.status_code}")
|
||||
|
||||
if response.status_code == 201:
|
||||
print(f"登录成功!:{response.json()}")
|
||||
return response.json()
|
||||
else:
|
||||
print(f"登录失败: {response.json()}")
|
||||
return {}
|
||||
|
||||
def load_alpha_list(self):
|
||||
"""从数据库加载未使用的alpha表达式"""
|
||||
alpha_list = self.db_manager.get_unused_alpha()
|
||||
return alpha_list
|
||||
|
||||
def simulate_alpha(self, expression):
|
||||
if self.client is None:
|
||||
raise Exception("请先登录")
|
||||
|
||||
settings = {
|
||||
'instrumentType': 'EQUITY',
|
||||
'region': 'USA',
|
||||
'universe': 'TOP3000',
|
||||
'delay': 1,
|
||||
'decay': 0,
|
||||
'neutralization': 'INDUSTRY',
|
||||
'truncation': 0.08,
|
||||
'pasteurization': 'ON',
|
||||
'unitHandling': 'VERIFY',
|
||||
'nanHandling': 'OFF',
|
||||
'language': 'FASTEXPR',
|
||||
'visualization': False,
|
||||
}
|
||||
|
||||
simulation_data = {
|
||||
'type': 'REGULAR',
|
||||
'settings': settings,
|
||||
'regular': expression
|
||||
}
|
||||
|
||||
sim_resp = self.client.post(f'{self.brain_api_url}/simulations', json=simulation_data)
|
||||
print(f"模拟提交状态: {sim_resp.status_code}")
|
||||
|
||||
sim_progress_url = sim_resp.headers['location']
|
||||
|
||||
while True:
|
||||
sim_progress_resp = self.client.get(sim_progress_url)
|
||||
retry_after_sec = float(sim_progress_resp.headers.get("Retry-After", 0))
|
||||
|
||||
if retry_after_sec == 0:
|
||||
break
|
||||
|
||||
if sim_progress_resp.json():
|
||||
result = sim_progress_resp.json()
|
||||
progress = result.get('progress', 0)
|
||||
print(f"模拟进度: {float(progress) * 100}%")
|
||||
|
||||
print(f"等待 {retry_after_sec} 秒...")
|
||||
time.sleep(retry_after_sec)
|
||||
|
||||
if sim_progress_resp.json().get("status") == "ERROR":
|
||||
result = sim_progress_resp.json().get("message", "未知错误")
|
||||
print(f"因子模拟失败: {result}")
|
||||
return {"status": "err", "message": result}
|
||||
|
||||
alpha_id = sim_progress_resp.json().get("alpha")
|
||||
print(f"生成的Alpha ID: {alpha_id}")
|
||||
|
||||
return {"status": "ok", "alpha_id": alpha_id}
|
||||
|
||||
def run_batch_simulation(self, alpha_list, batch_size=3):
|
||||
success_count = 0
|
||||
fail_count = 0
|
||||
|
||||
for i in range(0, len(alpha_list), batch_size):
|
||||
batch = alpha_list[i:i + batch_size]
|
||||
print(f"\n开始处理第 {i // batch_size + 1} 批因子，共 {len(batch)} 个")
|
||||
|
||||
for expression in batch:
|
||||
print(f"\n模拟因子: {expression}")
|
||||
start_time = time.time()
|
||||
|
||||
try:
|
||||
result = self.simulate_alpha(expression)
|
||||
end_time = time.time()
|
||||
time_consuming = round(end_time - start_time, 2)
|
||||
|
||||
result_data = {
|
||||
"expression": expression,
|
||||
"time_consuming": time_consuming,
|
||||
"status": result["status"],
|
||||
"timestamp": time.strftime("%Y-%m-%d %H:%M:%S")
|
||||
}
|
||||
|
||||
if result["status"] == "ok":
|
||||
result_data["alpha_id"] = result["alpha_id"]
|
||||
success_count += 1
|
||||
print(f"✅ 模拟成功 - Alpha ID: {result['alpha_id']} - 耗时: {time_consuming}秒")
|
||||
else:
|
||||
result_data["message"] = result.get("message", "")
|
||||
fail_count += 1
|
||||
print(f"❌ 模拟失败 - {result.get('message', '未知错误')}")
|
||||
|
||||
# 无论成功失败，都标记为已使用并保存结果到数据库
|
||||
self.db_manager.mark_alpha_used(expression)
|
||||
self.db_manager.insert_simulation_result(result_data)
|
||||
|
||||
except Exception as e:
|
||||
end_time = time.time()
|
||||
time_consuming = round(end_time - start_time, 2)
|
||||
error_result = {
|
||||
"expression": expression,
|
||||
"time_consuming": time_consuming,
|
||||
"status": "err",
|
||||
"message": str(e),
|
||||
"timestamp": time.strftime("%Y-%m-%d %H:%M:%S")
|
||||
}
|
||||
fail_count += 1
|
||||
|
||||
# 异常情况也标记为已使用并保存结果到数据库
|
||||
self.db_manager.mark_alpha_used(expression)
|
||||
self.db_manager.insert_simulation_result(error_result)
|
||||
|
||||
print(f"❌ 模拟异常 - {str(e)}")
|
||||
|
||||
print(f"第 {i // batch_size + 1} 批处理完成")
|
||||
|
||||
self._print_summary(success_count, fail_count)
|
||||
return success_count, fail_count
|
||||
|
||||
def _print_summary(self, success_count, fail_count):
|
||||
"""打印总结信息"""
|
||||
now = datetime.now().strftime('%Y-%m-%d %H:%M:%S')
|
||||
print_result = f"\n总计: 成功 {success_count} 个, 失败 {fail_count} 个\n\n 完成时间: {now}"
|
||||
print(print_result)
|
||||
print(f"所有结果已保存到 PostgreSQL 数据库 {self.db_manager.database_name} 的 simulation 表中")
|
||||
_send_to_gotify(print_result, now)
|
||||
|
||||
def _send_to_gotify(self, text, now):
|
||||
"""将结果推送到 gotify """
|
||||
title = f"alpha模拟结果 时间: {now}"
|
||||
try:
|
||||
resp = httpx.post("https://gotify.erhe.top/message?token=AvKJCJwQKU6yLP8", json={'title': title, 'message': text})
|
||||
except Exception as e:
|
||||
print(e)
|
||||
|
||||
def main():
|
||||
simulator = AlphaSimulator()
|
||||
|
||||
def refresh_login():
|
||||
"""刷新登录token"""
|
||||
login_result = simulator.login()
|
||||
if not login_result:
|
||||
print("登录失败，程序退出")
|
||||
exit(1)
|
||||
return login_result
|
||||
|
||||
# 初始登录
|
||||
login_result = refresh_login()
|
||||
|
||||
while True:
|
||||
try:
|
||||
expiry = int(login_result['token']['expiry'])
|
||||
if expiry < 1800:
|
||||
print(f"Token剩余 {expiry}秒，重新登录...")
|
||||
login_result = refresh_login()
|
||||
|
||||
alpha_list = simulator.load_alpha_list()
|
||||
if not alpha_list:
|
||||
print("暂无待处理的alpha表达式，10分钟后重新检查...")
|
||||
time.sleep(600)
|
||||
continue
|
||||
|
||||
print(f"共加载 {len(alpha_list)} 个需要模拟的因子表达式")
|
||||
|
||||
success_count, fail_count = simulator.run_batch_simulation(alpha_list, batch_size=3)
|
||||
print(f"本轮处理完成: 成功 {success_count} 个, 失败 {fail_count} 个")
|
||||
|
||||
print("等待5分钟后继续检查...")
|
||||
time.sleep(300)
|
||||
|
||||
except KeyboardInterrupt:
|
||||
print("\n程序被用户中断")
|
||||
break
|
||||
except Exception as e:
|
||||
print(f"程序执行异常: {e}")
|
||||
print("5秒后重试...")
|
||||
time.sleep(5)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,195 @@
|
||||
import os
|
||||
import json
|
||||
import random
|
||||
import time
|
||||
import httpx
|
||||
from httpx import BasicAuth
|
||||
|
||||
|
||||
class BrainLogin:
|
||||
def __init__(self, credentials_file='account.txt'):
|
||||
self.credentials_file = credentials_file
|
||||
self.client = None
|
||||
self.brain_api_url = 'https://api.worldquantbrain.com'
|
||||
|
||||
def load_credentials(self):
|
||||
if not os.path.exists(self.credentials_file):
|
||||
print("未找到 account.txt 文件")
|
||||
with open(self.credentials_file, 'w') as f:
|
||||
f.write("")
|
||||
print("account.txt 文件已创建，请填写账号密码, 格式: ['username', 'password']")
|
||||
exit(1)
|
||||
|
||||
with open(self.credentials_file) as f:
|
||||
credentials = eval(f.read())
|
||||
return credentials[0], credentials[1]
|
||||
|
||||
def login(self):
|
||||
try:
|
||||
username, password = self.load_credentials()
|
||||
self.client = httpx.Client(auth=BasicAuth(username, password))
|
||||
|
||||
response = self.client.post(f'{self.brain_api_url}/authentication')
|
||||
print(f"登录状态: {response.status_code}")
|
||||
|
||||
if response.status_code in [200, 201]:
|
||||
print("登录成功!")
|
||||
print(f"账户信息: {response.json()}")
|
||||
return self.client
|
||||
else:
|
||||
print(f"登录失败: {response.json()}")
|
||||
return None
|
||||
|
||||
except Exception as e:
|
||||
print(f"登录过程中出现错误: {e}")
|
||||
return None
|
||||
|
||||
def get_client(self):
|
||||
return self.client
|
||||
|
||||
|
||||
class DataSetDownloader:
|
||||
def __init__(self, client):
|
||||
self.client = client
|
||||
self.base_api_url = 'https://api.worldquantbrain.com'
|
||||
|
||||
def debug_detailed_response(self, endpoint, data_set_id, offset, limit=20):
|
||||
print(f"\n=== 调试请求: {endpoint} ===")
|
||||
url = f"{self.base_api_url}/{endpoint}"
|
||||
params = {
|
||||
'dataset.id': data_set_id,
|
||||
'delay': 1,
|
||||
'instrumentType': 'EQUITY',
|
||||
'limit': limit,
|
||||
'offset': offset,
|
||||
'region': 'USA',
|
||||
'universe': 'TOP3000'
|
||||
}
|
||||
|
||||
response = self.client.get(url, params=params)
|
||||
|
||||
if response.status_code == 200:
|
||||
data = response.json()
|
||||
print(f"count: {data.get('count')}")
|
||||
print(f"results 长度: {len(data.get('results', []))}")
|
||||
print(f"响应键: {list(data.keys())}")
|
||||
|
||||
def process_data(self, raw_data):
|
||||
processed_data = []
|
||||
|
||||
for item in raw_data:
|
||||
processed_item = {
|
||||
'id': item.get('id', ''),
|
||||
'description': item.get('description', ''),
|
||||
'dataset_id': item.get('dataset', {}).get('id', ''),
|
||||
'dataset_name': item.get('dataset', {}).get('name', ''),
|
||||
'category_id': item.get('category', {}).get('id', ''),
|
||||
'category_name': item.get('category', {}).get('name', ''),
|
||||
'region': item.get('region', ''),
|
||||
'delay': item.get('delay', ''),
|
||||
'universe': item.get('universe', ''),
|
||||
'type': item.get('type', '')
|
||||
}
|
||||
processed_data.append(processed_item)
|
||||
|
||||
return processed_data
|
||||
|
||||
def download_data_set(self, endpoint, data_set_id):
|
||||
output_dir = 'reference_fields'
|
||||
if not os.path.exists(output_dir):
|
||||
os.makedirs(output_dir)
|
||||
|
||||
self.debug_detailed_response(endpoint, data_set_id, offset=0, limit=20)
|
||||
|
||||
url = f"{self.base_api_url}/{endpoint}"
|
||||
params = {
|
||||
'dataset.id': data_set_id,
|
||||
'delay': 1,
|
||||
'instrumentType': 'EQUITY',
|
||||
'limit': 1,
|
||||
'offset': 0,
|
||||
'region': 'USA',
|
||||
'universe': 'TOP3000'
|
||||
}
|
||||
|
||||
response = self.client.get(url, params=params)
|
||||
data = response.json()
|
||||
total_count = data.get('count', 0)
|
||||
|
||||
print(f"📊 数据集总数: {total_count}")
|
||||
|
||||
if total_count == 0:
|
||||
print("❌ 没有找到数据")
|
||||
return
|
||||
|
||||
limit = 50
|
||||
all_data = []
|
||||
|
||||
print("🚀 开始下载数据...")
|
||||
for offset in range(0, total_count, limit):
|
||||
sleep_time = random.uniform(1.0, 1.5)
|
||||
time.sleep(sleep_time)
|
||||
|
||||
params = {
|
||||
'dataset.id': data_set_id,
|
||||
'delay': 1,
|
||||
'instrumentType': 'EQUITY',
|
||||
'limit': limit,
|
||||
'offset': offset,
|
||||
'region': 'USA',
|
||||
'universe': 'TOP3000'
|
||||
}
|
||||
|
||||
print(f"📥 下载进度: {offset}/{total_count} ({offset / total_count * 100:.1f}%)")
|
||||
|
||||
try:
|
||||
response = self.client.get(url, params=params)
|
||||
|
||||
if response.status_code == 200:
|
||||
data = response.json()
|
||||
results = data.get('results', [])
|
||||
|
||||
print(f"✅ 本页获取到 {len(results)} 条记录")
|
||||
all_data.extend(results)
|
||||
print(f"✅ 累计获取 {len(all_data)} 条记录")
|
||||
|
||||
if len(results) < limit:
|
||||
print("🎯 到达数据末尾")
|
||||
break
|
||||
else:
|
||||
print(f"❌ 请求失败: {response.status_code}")
|
||||
break
|
||||
|
||||
except Exception as e:
|
||||
print(f"❌ 下载过程中出错: {e}")
|
||||
break
|
||||
|
||||
print("🔄 处理数据中...")
|
||||
processed_data = self.process_data(all_data)
|
||||
|
||||
output_file = os.path.join(output_dir, f"{data_set_id}_{endpoint}.json")
|
||||
with open(output_file, 'w', encoding='utf-8') as f:
|
||||
json.dump(processed_data, f, ensure_ascii=False, indent=2)
|
||||
|
||||
print(f"💾 处理后的数据已保存到: {output_file}")
|
||||
print(f"🎉 总共处理了 {len(processed_data)} 条记录")
|
||||
|
||||
if processed_data:
|
||||
print("\n📋 处理后数据示例:")
|
||||
print(json.dumps(processed_data[0], indent=2, ensure_ascii=False))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
brain_login = BrainLogin()
|
||||
client = brain_login.login()
|
||||
|
||||
if client:
|
||||
downloader = DataSetDownloader(client)
|
||||
|
||||
endpoint_list = ['data-sets', 'data-fields']
|
||||
endpoint = endpoint_list[0]
|
||||
data_set_id = 'analyst4'
|
||||
|
||||
downloader.download_data_set(endpoint, data_set_id)
|
||||
else:
|
||||
print("❌ 登录失败，无法下载数据")
|
||||
@@ -0,0 +1,197 @@
|
||||
Operator: abs(x)
|
||||
Description: Absolute value of x
|
||||
|
||||
Operator: add(x, y, filter = false), x + y
|
||||
Description: Add all inputs (at least 2 inputs required). If filter = true, filter all input NaN to 0 before adding
|
||||
|
||||
Operator: densify(x)
|
||||
Description: Converts a grouping field of many buckets into lesser number of only available buckets so as to make working with grouping fields computationally efficient
|
||||
|
||||
Operator: divide(x, y), x / y
|
||||
Description: x / y
|
||||
|
||||
Operator: inverse(x)
|
||||
Description: 1 / x
|
||||
|
||||
Operator: log(x)
|
||||
Description: Natural logarithm. For example: Log(high/low) uses natural logarithm of high/low ratio as stock weights.
|
||||
|
||||
Operator: max(x, y, ..)
|
||||
Description: Maximum value of all inputs. At least 2 inputs are required
|
||||
|
||||
Operator: min(x, y ..)
|
||||
Description: Minimum value of all inputs. At least 2 inputs are required
|
||||
|
||||
Operator: multiply(x ,y, ... , filter=false), x * y
|
||||
Description: Multiply all inputs. At least 2 inputs are required. Filter sets the NaN values to 1
|
||||
|
||||
Operator: power(x, y)
|
||||
Description: x ^ y
|
||||
|
||||
Operator: reverse(x)
|
||||
Description: - x
|
||||
|
||||
Operator: sign(x)
|
||||
Description: if input > 0, return 1; if input < 0, return -1; if input = 0, return 0; if input = NaN, return NaN;
|
||||
|
||||
Operator: signed_power(x, y)
|
||||
Description: x raised to the power of y such that final result preserves sign of x
|
||||
|
||||
Operator: sqrt(x)
|
||||
Description: Square root of x
|
||||
|
||||
Operator: subtract(x, y, filter=false), x - y
|
||||
Description: x-y. If filter = true, filter all input NaN to 0 before subtracting
|
||||
|
||||
Operator: and(input1, input2)
|
||||
Description: Logical AND operator, returns true if both operands are true and returns false otherwise
|
||||
|
||||
Operator: if_else(input1, input2, input 3)
|
||||
Description: If input1 is true then return input2 else return input3.
|
||||
|
||||
Operator: input1 < input2
|
||||
Description: If input1 < input2 return true, else return false
|
||||
|
||||
Operator: input1 <= input2
|
||||
Description: Returns true if input1 <= input2, return false otherwise
|
||||
|
||||
Operator: input1 == input2
|
||||
Description: Returns true if both inputs are same and returns false otherwise
|
||||
|
||||
Operator: input1 > input2
|
||||
Description: Logic comparison operators to compares two inputs
|
||||
|
||||
Operator: input1 >= input2
|
||||
Description: Returns true if input1 >= input2, return false otherwise
|
||||
|
||||
Operator: input1!= input2
|
||||
Description: Returns true if both inputs are NOT the same and returns false otherwise
|
||||
|
||||
Operator: is_nan(input)
|
||||
Description: If (input == NaN) return 1 else return 0
|
||||
|
||||
Operator: not(x)
|
||||
Description: Returns the logical negation of x. If x is true (1), it returns false (0), and if input is false (0), it returns true (1).
|
||||
|
||||
Operator: or(input1, input2)
|
||||
Description: Logical OR operator returns true if either or both inputs are true and returns false otherwise
|
||||
|
||||
Operator: days_from_last_change(x)
|
||||
Description: Amount of days since last change of x
|
||||
|
||||
Operator: hump(x, hump = 0.01)
|
||||
Description: Limits amount and magnitude of changes in input (thus reducing turnover)
|
||||
|
||||
Operator: kth_element(x, d, k)
|
||||
Description: Returns K-th value of input by looking through lookback days. This operator can be used to backfill missing data if k=1
|
||||
|
||||
Operator: last_diff_value(x, d)
|
||||
Description: Returns last x value not equal to current x value from last d days
|
||||
|
||||
Operator: ts_arg_max(x, d)
|
||||
Description: Returns the relative index of the max value in the time series for the past d days. If the current day has the max value for the past d days, it returns 0. If previous day has the max value for the past d days, it returns 1
|
||||
|
||||
Operator: ts_arg_min(x, d)
|
||||
Description: Returns the relative index of the min value in the time series for the past d days; If the current day has the min value for the past d days, it returns 0; If previous day has the min value for the past d days, it returns 1.
|
||||
|
||||
Operator: ts_av_diff(x, d)
|
||||
Description: Returns x - tsmean(x, d), but deals with NaNs carefully. That is NaNs are ignored during mean computation
|
||||
|
||||
Operator: ts_backfill(x,lookback = d, k=1, ignore="NAN")
|
||||
Description: Backfill is the process of replacing the NAN or 0 values by a meaningful value (i.e., a first non-NaN value)
|
||||
|
||||
Operator: ts_corr(x, y, d)
|
||||
Description: Returns correlation of x and y for the past d days
|
||||
|
||||
Operator: ts_count_nans(x ,d)
|
||||
Description: Returns the number of NaN values in x for the past d days
|
||||
|
||||
Operator: ts_covariance(y, x, d)
|
||||
Description: Returns covariance of y and x for the past d days
|
||||
|
||||
Operator: ts_decay_linear(x, d, dense = false)
|
||||
Description: Returns the linear decay on x for the past d days. Dense parameter=false means operator works in sparse mode and we treat NaN as 0. In dense mode we do not.
|
||||
|
||||
Operator: ts_delay(x, d)
|
||||
Description: Returns x value d days ago
|
||||
|
||||
Operator: ts_delta(x, d)
|
||||
Description: Returns x - ts_delay(x, d)
|
||||
|
||||
Operator: ts_mean(x, d)
|
||||
Description: Returns average value of x for the past d days.
|
||||
|
||||
Operator: ts_product(x, d)
|
||||
Description: Returns product of x for the past d days
|
||||
|
||||
Operator: ts_quantile(x,d, driver="gaussian" )
|
||||
Description: It calculates ts_rank and apply to its value an inverse cumulative density function from driver distribution. Possible values of driver (optional ) are "gaussian", "uniform", "cauchy" distribution where "gaussian" is the default.
|
||||
|
||||
Operator: ts_rank(x, d, constant = 0)
|
||||
Description: Rank the values of x for each instrument over the past d days, then return the rank of the current value + constant. If not specified, by default, constant = 0.
|
||||
|
||||
Operator: ts_regression(y, x, d, lag = 0, rettype = 0)
|
||||
Description: Returns various parameters related to regression function
|
||||
|
||||
Operator: ts_scale(x, d, constant = 0)
|
||||
Description: Returns (x - ts_min(x, d)) / (ts_max(x, d) - ts_min(x, d)) + constant. This operator is similar to scale down operator but acts in time series space
|
||||
|
||||
Operator: ts_std_dev(x, d)
|
||||
Description: Returns standard deviation of x for the past d days
|
||||
|
||||
Operator: ts_step(1)
|
||||
Description: Returns days' counter
|
||||
|
||||
Operator: ts_sum(x, d)
|
||||
Description: Sum values of x for the past d days.
|
||||
|
||||
Operator: ts_zscore(x, d)
|
||||
Description: Z-score is a numerical measurement that describes a value's relationship to the mean of a group of values. Z-score is measured in terms of standard deviations from the mean: (x - tsmean(x,d)) / tsstddev(x,d). This operator may help reduce outliers and drawdown.
|
||||
|
||||
Operator: normalize(x, useStd = false, limit = 0.0)
|
||||
Description: Calculates the mean value of all valid alpha values for a certain date, then subtracts that mean from each element
|
||||
|
||||
Operator: quantile(x, driver = gaussian, sigma = 1.0)
|
||||
Description: Rank the raw vector, shift the ranked Alpha vector, apply distribution (gaussian, cauchy, uniform). If driver is uniform, it simply subtract each Alpha value with the mean of all Alpha values in the Alpha vector
|
||||
|
||||
Operator: rank(x, rate=2)
|
||||
Description: Ranks the input among all the instruments and returns an equally distributed number between 0.0 and 1.0. For precise sort, use the rate as 0
|
||||
|
||||
Operator: scale(x, scale=1, longscale=1, shortscale=1)
|
||||
Description: Scales input to booksize. We can also scale the long positions and short positions to separate scales by mentioning additional parameters to the operator
|
||||
|
||||
Operator: winsorize(x, std=4)
|
||||
Description: Winsorizes x to make sure that all values in x are between the lower and upper limits, which are specified as multiple of std.
|
||||
|
||||
Operator: zscore(x)
|
||||
Description: Z-score is a numerical measurement that describes a value's relationship to the mean of a group of values. Z-score is measured in terms of standard deviations from the mean
|
||||
|
||||
Operator: vec_avg(x)
|
||||
Description: Taking mean of the vector field x
|
||||
|
||||
Operator: vec_sum(x)
|
||||
Description: Sum of vector field x
|
||||
|
||||
Operator: bucket(rank(x), range="0, 1, 0.1" or buckets = "2,5,6,7,10")
|
||||
Description: Convert float values into indexes for user-specified buckets. Bucket is useful for creating group values, which can be passed to GROUP as input
|
||||
|
||||
Operator: trade_when(x, y, z)
|
||||
Description: Used in order to change Alpha values only under a specified condition and to hold Alpha values in other cases. It also allows to close Alpha positions (assign NaN values) under a specified condition
|
||||
|
||||
Operator: group_backfill(x, group, d, std = 4.0)
|
||||
Description: If a certain value for a certain date and instrument is NaN, from the set of same group instruments, calculate winsorized mean of all non-NaN values over last d days
|
||||
|
||||
Operator: group_mean(x, weight, group)
|
||||
Description: All elements in group equals to the mean
|
||||
|
||||
Operator: group_neutralize(x, group)
|
||||
Description: Neutralizes Alpha against groups. These groups can be subindustry, industry, sector, country or a constant
|
||||
|
||||
Operator: group_rank(x, group)
|
||||
Description: Each elements in a group is assigned the corresponding rank in this group
|
||||
|
||||
Operator: group_scale(x, group)
|
||||
Description: Normalizes the values in a group to be between 0 and 1. (x - groupmin) / (groupmax - groupmin)
|
||||
|
||||
Operator: group_zscore(x, group)
|
||||
Description: Calculates group Z-score - numerical measurement that describes a value's relationship to the mean of a group of values. Z-score is measured in terms of standard deviations from the mean. zscore = (data - mean) / stddev of x for each instrument within its group.
|
||||
Reference in New Issue
Block a user