++
This commit is contained in:
jack
@@ -0,0 +1,264 @@
|
||||
import httpx
|
||||
from httpx import BasicAuth, Timeout
|
||||
import pandas as pd
|
||||
from tqdm import tqdm
|
||||
from pathlib import Path
|
||||
import logging
|
||||
import time
|
||||
import random
|
||||
|
||||
|
||||
class SimpleAlphaFetcher:
|
||||
def __init__(self, base_path=None):
|
||||
"""
|
||||
初始化 Alpha 获取器
|
||||
|
||||
Args:
|
||||
base_path: 输出文件保存路径
|
||||
"""
|
||||
self.client = None
|
||||
self.base_path = Path(base_path) if base_path else Path.cwd()
|
||||
self.logger = self._setup_logger()
|
||||
|
||||
# 登录
|
||||
self.login()
|
||||
|
||||
def _setup_logger(self):
|
||||
"""设置日志记录器"""
|
||||
logger = logging.getLogger(__name__)
|
||||
logger.setLevel(logging.INFO)
|
||||
|
||||
if not logger.handlers:
|
||||
handler = logging.StreamHandler()
|
||||
formatter = logging.Formatter('%(asctime)s - %(levelname)s - %(message)s')
|
||||
handler.setFormatter(formatter)
|
||||
logger.addHandler(handler)
|
||||
|
||||
return logger
|
||||
|
||||
def login(self):
|
||||
"""登录 WorldQuant Brain API"""
|
||||
try:
|
||||
# 从 nacos 获取账号密码
|
||||
with httpx.Client(timeout=10.0) as temp_client:
|
||||
nacos_resp = temp_client.get(
|
||||
'http://192.168.31.41:30848/nacos/v1/cs/configs?dataId=wq_account&group=quantify'
|
||||
)
|
||||
|
||||
if nacos_resp.status_code != 200:
|
||||
self.logger.error('获取账号密码失败')
|
||||
return False
|
||||
|
||||
config = nacos_resp.json()
|
||||
username = config.get('user_name')
|
||||
password = config.get('password')
|
||||
|
||||
if not username or not password:
|
||||
self.logger.error('账号密码不完整')
|
||||
return False
|
||||
|
||||
self.logger.info(f"正在登录账户: {username}")
|
||||
|
||||
# 创建客户端并设置超时
|
||||
timeout = Timeout(connect=30.0, read=60.0, write=30.0, pool=30.0)
|
||||
self.client = httpx.Client(
|
||||
auth=BasicAuth(username, password),
|
||||
timeout=timeout
|
||||
)
|
||||
|
||||
# 发送登录请求
|
||||
response = self.client.post('https://api.worldquantbrain.com/authentication')
|
||||
|
||||
if response.status_code == 201:
|
||||
self.logger.info("登录成功!")
|
||||
return True
|
||||
else:
|
||||
self.logger.error(f"登录失败: {response.status_code} - {response.text}")
|
||||
self.client.close()
|
||||
self.client = None
|
||||
return False
|
||||
|
||||
except Exception as e:
|
||||
self.logger.error(f"登录异常: {e}")
|
||||
return False
|
||||
|
||||
def fetch_alphas(
|
||||
self,
|
||||
max_pages: int = 100,
|
||||
limit: int = 100,
|
||||
delay: int = 1,
|
||||
region: str = "USA",
|
||||
universe: str = "TOP3000",
|
||||
hidden: str = "false",
|
||||
output_file_name: str = "alpha_list.csv",
|
||||
mode: str = "w",
|
||||
max_retries: int = 3,
|
||||
):
|
||||
"""
|
||||
获取 Alpha 列表
|
||||
|
||||
Args:
|
||||
max_pages: 最大爬取页数
|
||||
limit: 每页数量
|
||||
delay: API 请求的延迟设置
|
||||
region: 市场区域
|
||||
universe: 股票池
|
||||
hidden: 是否搜索已隐藏的 Alpha
|
||||
output_file_name: 输出的 CSV 文件名
|
||||
mode: 写入模式："w"(覆盖写入) 或 "a"(追加写入)
|
||||
max_retries: 请求失败时的最大重试次数
|
||||
|
||||
Returns:
|
||||
list: Alpha 列表
|
||||
"""
|
||||
if not self.client:
|
||||
self.logger.error("客户端未登录，无法执行获取")
|
||||
return []
|
||||
|
||||
fetched_alphas = []
|
||||
offset = 0
|
||||
total_accessed = 0
|
||||
pages_fetched = 0
|
||||
|
||||
# 先获取总数
|
||||
count_url = (
|
||||
f"https://api.worldquantbrain.com/users/self/alphas?stage=IS&hidden={hidden}"
|
||||
f"&limit=1&settings.delay={delay}&settings.region={region}&status=UNSUBMITTED%1FIS_FAIL&settings.universe={universe}"
|
||||
)
|
||||
|
||||
total_available = 0
|
||||
for attempt in range(max_retries):
|
||||
try:
|
||||
count_response = self.client.get(count_url)
|
||||
total_available = count_response.json()["count"]
|
||||
break
|
||||
except Exception as e:
|
||||
self.logger.warning(f"获取 Alpha 总数失败 (尝试 {attempt+1}/{max_retries}): {e}")
|
||||
if attempt < max_retries - 1:
|
||||
time.sleep(random.uniform(3, 5))
|
||||
else:
|
||||
self.logger.error("获取 Alpha 总数最终失败")
|
||||
return []
|
||||
|
||||
if total_available == 0:
|
||||
self.logger.warning("未找到任何 Alpha")
|
||||
return []
|
||||
|
||||
# 计算实际最大页数
|
||||
actual_max_pages = min(max_pages, (total_available + limit - 1) // limit)
|
||||
self.logger.info(f"共找到 {total_available} 个 Alpha，计划爬取 {actual_max_pages} 页...")
|
||||
|
||||
pbar = tqdm(total=actual_max_pages, desc="爬取 Alpha 页面", unit="页")
|
||||
|
||||
while pages_fetched < actual_max_pages:
|
||||
# 构建请求 URL
|
||||
url = (
|
||||
f"https://api.worldquantbrain.com/users/self/alphas?stage=IS&limit={limit}"
|
||||
f"&offset={offset}&settings.delay={delay}&settings.region={region}&hidden={hidden}&status=UNSUBMITTED%1FIS_FAIL&settings.universe={universe}"
|
||||
)
|
||||
|
||||
try:
|
||||
# 使用重试机制
|
||||
response = None
|
||||
for attempt in range(max_retries):
|
||||
try:
|
||||
response = self.client.get(url)
|
||||
break
|
||||
except Exception as e:
|
||||
self.logger.warning(f"请求失败 (尝试 {attempt+1}/{max_retries}): {e}")
|
||||
if attempt < max_retries - 1:
|
||||
time.sleep(random.uniform(3, 5))
|
||||
else:
|
||||
raise
|
||||
|
||||
if response.status_code == 400:
|
||||
self.logger.warning(f"遇到 API 限制 (offset={offset})，停止获取")
|
||||
break
|
||||
|
||||
response_data = response.json()
|
||||
|
||||
if not isinstance(response_data, dict) or "results" not in response_data:
|
||||
self.logger.error(f"API 返回了意外的数据: {response_data}")
|
||||
break
|
||||
|
||||
alphas = response_data["results"]
|
||||
|
||||
if not alphas:
|
||||
break
|
||||
|
||||
fetched_alphas.extend(alphas)
|
||||
total_accessed += len(alphas)
|
||||
pages_fetched += 1
|
||||
|
||||
# 更新进度条
|
||||
pbar.update(1)
|
||||
pbar.set_postfix({
|
||||
"Region": region,
|
||||
"已获取": total_accessed,
|
||||
"本页": len(alphas),
|
||||
})
|
||||
|
||||
if len(alphas) < limit:
|
||||
self.logger.info(f"本页数据不足 {limit} 条，已到达末尾")
|
||||
break
|
||||
|
||||
offset += limit
|
||||
|
||||
# 添加延迟避免请求过快
|
||||
time.sleep(0.5)
|
||||
|
||||
except Exception as e:
|
||||
self.logger.error(f"请求失败: {e}")
|
||||
break
|
||||
|
||||
pbar.close()
|
||||
|
||||
if not fetched_alphas:
|
||||
self.logger.warning("未获取到任何 Alpha！")
|
||||
return []
|
||||
|
||||
# 按 fitness 排序
|
||||
df = pd.DataFrame(fetched_alphas)
|
||||
df["temp_fitness"] = df.apply(
|
||||
lambda row: row["is"].get("fitness", 0) if isinstance(row.get("is"), dict) else 0,
|
||||
axis=1
|
||||
)
|
||||
df_sorted = df.sort_values(by="temp_fitness", ascending=False)
|
||||
df_sorted = df_sorted.drop("temp_fitness", axis=1)
|
||||
|
||||
output_path = self.base_path / output_file_name
|
||||
|
||||
if mode == "w":
|
||||
df_sorted.to_csv(output_path, index=False)
|
||||
elif mode == "a":
|
||||
df_sorted.to_csv(output_path, mode="a", index=False, header=False)
|
||||
|
||||
self.logger.info(f"Alpha 列表已保存！共 {len(fetched_alphas)} 条记录！\n文件路径: {output_path}")
|
||||
|
||||
return fetched_alphas
|
||||
|
||||
def close(self):
|
||||
"""关闭客户端"""
|
||||
if self.client:
|
||||
self.client.close()
|
||||
|
||||
|
||||
# 使用示例
|
||||
if __name__ == "__main__":
|
||||
# 创建获取器实例（会自动登录）
|
||||
fetcher = SimpleAlphaFetcher()
|
||||
|
||||
if fetcher.client:
|
||||
# 获取 Alpha 列表
|
||||
results = fetcher.fetch_alphas(
|
||||
max_pages=100, # 最多爬取 100 页
|
||||
limit=100, # 每页 100 条
|
||||
region="USA", # 美股市场
|
||||
universe="TOP3000", # TOP3000 股票池
|
||||
output_file_name="alpha_list.csv" # 输出文件名
|
||||
)
|
||||
|
||||
print(f"成功获取 {len(results) if results else 0} 个 Alpha")
|
||||
|
||||
# 关闭客户端
|
||||
fetcher.close()
|
||||
@@ -0,0 +1,231 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Alpha 特征分析数据库 - 长表方案
|
||||
适用场景：95,003 条表达式，6,488 个字段，84 个算子
|
||||
"""
|
||||
|
||||
import sqlite3
|
||||
import os
|
||||
import re
|
||||
from pathlib import Path
|
||||
|
||||
DB_PATH = Path(__file__).parent / "alpha_analysis.db"
|
||||
|
||||
# 84 个算子完整列表
|
||||
OPERATORS = [
|
||||
# Arithmetic (16)
|
||||
'add', 'abs', 'log', 'subtract', 'signed_power', 'sign', 'reverse', 'power',
|
||||
'multiply', 'min', 'max', 'inverse', 'sqrt', 's_log_1p', 'densify', 'divide',
|
||||
# Logical (12)
|
||||
'not', 'and', 'less', 'equal', 'or', 'not_equal', 'greater', 'greater_equal',
|
||||
'less_equal', 'is_nan', 'if_else',
|
||||
# Time Series (29)
|
||||
'ts_sum', 'ts_scale', 'ts_mean', 'ts_zscore', 'ts_std_dev', 'kth_element',
|
||||
'inst_tvr', 'ts_corr', 'ts_count_nans', 'ts_target_tvr_decay', 'ts_median',
|
||||
'ts_covariance', 'ts_decay_linear', 'ts_product', 'ts_regression', 'ts_delta_limit',
|
||||
'ts_step', 'ts_decay_exp_window', 'ts_quantile', 'days_from_last_change', 'hump',
|
||||
'last_diff_value', 'ts_arg_max', 'ts_arg_min', 'ts_av_diff', 'ts_backfill',
|
||||
'ts_rank', 'ts_delay', 'ts_delta',
|
||||
# Cross Sectional (8)
|
||||
'winsorize', 'truncate', 'regression_neut', 'scale', 'rank', 'quantile',
|
||||
'normalize', 'zscore',
|
||||
# Vector (7)
|
||||
'vec_min', 'vec_count', 'vec_stddev', 'vec_range', 'vec_avg', 'vec_sum', 'vec_max',
|
||||
# Transformational (4)
|
||||
'left_tail', 'trade_when', 'right_tail', 'bucket',
|
||||
# Group (8)
|
||||
'group_rank', 'group_cartesian_product', 'group_backfill', 'group_mean',
|
||||
'group_neutralize', 'group_normalize', 'group_median', 'group_scale', 'group_zscore'
|
||||
]
|
||||
|
||||
# 常用窗口参数
|
||||
WINDOWS = [1, 2, 5, 10, 20, 30, 60, 90, 252]
|
||||
|
||||
# 中性化方式
|
||||
NEUTRALIZATIONS = ['FAST', 'SUBINDUSTRY', 'INDUSTRY', 'NONE']
|
||||
|
||||
|
||||
def get_connection():
|
||||
"""获取数据库连接"""
|
||||
conn = sqlite3.connect(str(DB_PATH))
|
||||
conn.row_factory = sqlite3.Row
|
||||
return conn
|
||||
|
||||
|
||||
def create_tables(conn):
|
||||
"""创建所有表"""
|
||||
cursor = conn.cursor()
|
||||
|
||||
# =========================================================
|
||||
# 表1: alpha_success（成功样本 - 主表）
|
||||
# =========================================================
|
||||
cursor.execute("""
|
||||
CREATE TABLE IF NOT EXISTS alpha_success (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
alpha_id VARCHAR(64) UNIQUE,
|
||||
expression TEXT NOT NULL,
|
||||
sharpe REAL,
|
||||
fitness REAL,
|
||||
returns REAL,
|
||||
drawdown REAL,
|
||||
turnover REAL,
|
||||
long_count INTEGER,
|
||||
short_count INTEGER,
|
||||
neutralization VARCHAR(32),
|
||||
universe VARCHAR(32),
|
||||
dataset VARCHAR(32),
|
||||
score REAL,
|
||||
created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP
|
||||
)
|
||||
""")
|
||||
|
||||
# 索引
|
||||
cursor.execute("CREATE INDEX IF NOT EXISTS idx_success_score ON alpha_success(score)")
|
||||
cursor.execute("CREATE INDEX IF NOT EXISTS idx_success_sharpe ON alpha_success(sharpe)")
|
||||
cursor.execute("CREATE INDEX IF NOT EXISTS idx_success_neutralization ON alpha_success(neutralization)")
|
||||
|
||||
# =========================================================
|
||||
# 表2: alpha_feature_long（特征长表 - 核心）
|
||||
# 每个特征一行：算子、字段、窗口、配置
|
||||
# =========================================================
|
||||
cursor.execute("""
|
||||
CREATE TABLE IF NOT EXISTS alpha_feature_long (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
alpha_id INTEGER NOT NULL,
|
||||
feature_type VARCHAR(32) NOT NULL, -- operator, field, window, neutralization
|
||||
feature_name VARCHAR(128) NOT NULL,
|
||||
feature_value INTEGER DEFAULT 1, -- 1=存在，窗口值存具体数字
|
||||
created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
|
||||
FOREIGN KEY (alpha_id) REFERENCES alpha_success(id) ON DELETE CASCADE
|
||||
)
|
||||
""")
|
||||
|
||||
# 索引（加速统计查询）
|
||||
cursor.execute("CREATE INDEX IF NOT EXISTS idx_feature_type_name ON alpha_feature_long(feature_type, feature_name)")
|
||||
cursor.execute("CREATE INDEX IF NOT EXISTS idx_feature_alpha_id ON alpha_feature_long(alpha_id)")
|
||||
|
||||
# =========================================================
|
||||
# 表3: feature_statistics（物化统计表 - 反馈核心）
|
||||
# 存储每个特征的增益值，供生成器使用
|
||||
# =========================================================
|
||||
cursor.execute("""
|
||||
CREATE TABLE IF NOT EXISTS feature_statistics (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
feature_type VARCHAR(32) NOT NULL,
|
||||
feature_name VARCHAR(128) NOT NULL,
|
||||
gain_value REAL, -- 增益值 = 特征平均分 - 全局平均分
|
||||
avg_score REAL, -- 包含该特征的表达式平均分
|
||||
global_avg_score REAL, -- 统计时的全局平均分
|
||||
sample_count INTEGER, -- 包含该特征的表达式数量
|
||||
positive_effect INTEGER DEFAULT 0, -- 1=正向特征(gain>0.05), -1=负向特征(gain<-0.05)
|
||||
is_active INTEGER DEFAULT 1, -- 是否启用反馈
|
||||
calculated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
|
||||
UNIQUE(feature_type, feature_name)
|
||||
)
|
||||
""")
|
||||
|
||||
cursor.execute("CREATE INDEX IF NOT EXISTS idx_stats_positive ON feature_statistics(positive_effect)")
|
||||
cursor.execute("CREATE INDEX IF NOT EXISTS idx_stats_gain ON feature_statistics(gain_value DESC)")
|
||||
|
||||
# =========================================================
|
||||
# 表4: alpha_failure（失败样本 - 不兼容组合）
|
||||
# =========================================================
|
||||
cursor.execute("""
|
||||
CREATE TABLE IF NOT EXISTS alpha_failure (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
operator VARCHAR(64) NOT NULL,
|
||||
field VARCHAR(64) NOT NULL,
|
||||
expression TEXT,
|
||||
error_type VARCHAR(32),
|
||||
created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP
|
||||
)
|
||||
""")
|
||||
|
||||
cursor.execute("CREATE INDEX IF NOT EXISTS idx_failure_combo ON alpha_failure(operator, field)")
|
||||
|
||||
# =========================================================
|
||||
# 表5: operator_field_blacklist（黑名单 - 从不兼容统计生成）
|
||||
# =========================================================
|
||||
cursor.execute("""
|
||||
CREATE TABLE IF NOT EXISTS operator_field_blacklist (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
operator VARCHAR(64) NOT NULL,
|
||||
field VARCHAR(64) NOT NULL,
|
||||
fail_count INTEGER DEFAULT 0,
|
||||
total_attempts INTEGER DEFAULT 0,
|
||||
fail_rate REAL DEFAULT 0.0,
|
||||
is_active INTEGER DEFAULT 1,
|
||||
created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
|
||||
updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
|
||||
UNIQUE(operator, field)
|
||||
)
|
||||
""")
|
||||
|
||||
cursor.execute("CREATE INDEX IF NOT EXISTS idx_blacklist_active ON operator_field_blacklist(is_active)")
|
||||
|
||||
# =========================================================
|
||||
# 表6: generation_bias（生成偏向配置 - 反馈到生成器）
|
||||
# =========================================================
|
||||
cursor.execute("""
|
||||
CREATE TABLE IF NOT EXISTS generation_bias (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
feature_type VARCHAR(32) NOT NULL,
|
||||
feature_name VARCHAR(128) NOT NULL,
|
||||
bias_weight REAL DEFAULT 1.0, -- >1 正向，<1 负向，1 中性
|
||||
gain_value REAL,
|
||||
sample_count INTEGER,
|
||||
is_active INTEGER DEFAULT 1,
|
||||
created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
|
||||
updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
|
||||
UNIQUE(feature_type, feature_name)
|
||||
)
|
||||
""")
|
||||
|
||||
cursor.execute("CREATE INDEX IF NOT EXISTS idx_bias_weight ON generation_bias(bias_weight DESC)")
|
||||
|
||||
conn.commit()
|
||||
print("✅ 所有表创建成功")
|
||||
print(f"📁 数据库路径: {DB_PATH}")
|
||||
print(f"\n📊 表结构:")
|
||||
print(f" - alpha_success: 主表，存储表达式和回测结果")
|
||||
print(f" - alpha_feature_long: 长表，每个特征一行")
|
||||
print(f" - feature_statistics: 物化统计，特征增益值")
|
||||
print(f" - alpha_failure: 失败样本记录")
|
||||
print(f" - operator_field_blacklist: 不兼容黑名单")
|
||||
print(f" - generation_bias: 生成偏向配置")
|
||||
|
||||
|
||||
def show_table_info(conn):
|
||||
"""显示各表记录数"""
|
||||
cursor = conn.cursor()
|
||||
tables = ['alpha_success', 'alpha_feature_long', 'feature_statistics',
|
||||
'alpha_failure', 'operator_field_blacklist', 'generation_bias']
|
||||
|
||||
print("\n📋 当前表记录数:")
|
||||
for table in tables:
|
||||
cursor.execute(f"SELECT COUNT(*) FROM {table}")
|
||||
count = cursor.fetchone()[0]
|
||||
print(f" - {table}: {count:,}")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
# 检查数据库是否存在
|
||||
if DB_PATH.exists():
|
||||
print(f"⚠️ 数据库已存在: {DB_PATH}")
|
||||
response = input("是否删除并重建？(y/N): ").strip().lower()
|
||||
if response == 'y':
|
||||
os.remove(DB_PATH)
|
||||
print("🗑️ 已删除旧数据库")
|
||||
else:
|
||||
print("❌ 已取消操作")
|
||||
exit(0)
|
||||
|
||||
# 创建数据库和表
|
||||
conn = get_connection()
|
||||
try:
|
||||
create_tables(conn)
|
||||
show_table_info(conn)
|
||||
finally:
|
||||
conn.close()
|
||||
|
||||
print("\n✅ 初始化完成！下一步：导入数据并提取特征")
|
||||
@@ -0,0 +1,232 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
从 CSV 导入 Alpha 数据到 SQLite（修复去重版）
|
||||
"""
|
||||
|
||||
import sqlite3
|
||||
import pandas as pd
|
||||
import ast
|
||||
import re
|
||||
from pathlib import Path
|
||||
|
||||
# 路径配置
|
||||
CSV_PATH = Path(__file__).parent / "alpha_list.csv"
|
||||
SQLITE_PATH = Path(__file__).parent / "alpha_analysis.db"
|
||||
|
||||
# 84 个算子集合（同上，省略重复部分，你复制完整的）
|
||||
OPERATORS_SET = {
|
||||
'add', 'abs', 'log', 'subtract', 'signed_power', 'sign', 'reverse', 'power',
|
||||
'multiply', 'min', 'max', 'inverse', 'sqrt', 's_log_1p', 'densify', 'divide',
|
||||
'not', 'and', 'less', 'equal', 'or', 'not_equal', 'greater', 'greater_equal',
|
||||
'less_equal', 'is_nan', 'if_else', 'ts_sum', 'ts_scale', 'ts_mean', 'ts_zscore',
|
||||
'ts_std_dev', 'kth_element', 'inst_tvr', 'ts_corr', 'ts_count_nans',
|
||||
'ts_target_tvr_decay', 'ts_median', 'ts_covariance', 'ts_decay_linear',
|
||||
'ts_product', 'ts_regression', 'ts_delta_limit', 'ts_step', 'ts_decay_exp_window',
|
||||
'ts_quantile', 'days_from_last_change', 'hump', 'last_diff_value', 'ts_arg_max',
|
||||
'ts_arg_min', 'ts_av_diff', 'ts_backfill', 'ts_rank', 'ts_delay', 'ts_delta',
|
||||
'winsorize', 'truncate', 'regression_neut', 'scale', 'rank', 'quantile',
|
||||
'normalize', 'zscore', 'vec_min', 'vec_count', 'vec_stddev', 'vec_range',
|
||||
'vec_avg', 'vec_sum', 'vec_max', 'left_tail', 'trade_when', 'right_tail',
|
||||
'bucket', 'group_rank', 'group_cartesian_product', 'group_backfill', 'group_mean',
|
||||
'group_neutralize', 'group_normalize', 'group_median', 'group_scale', 'group_zscore'
|
||||
}
|
||||
|
||||
WINDOWS = {1, 2, 5, 10, 20, 30, 60, 90, 252}
|
||||
|
||||
|
||||
def safe_parse_dict(s):
|
||||
"""安全解析 Python 字典字符串"""
|
||||
if pd.isna(s) or s == '':
|
||||
return {}
|
||||
try:
|
||||
return ast.literal_eval(s)
|
||||
except:
|
||||
return {}
|
||||
|
||||
|
||||
def extract_features(expression):
|
||||
"""从表达式提取特征"""
|
||||
if not expression:
|
||||
return [], [], []
|
||||
|
||||
operators = set()
|
||||
for match in re.finditer(r'\b([a-z_][a-z0-9_]*)\s*\(', expression):
|
||||
op = match.group(1)
|
||||
if op in OPERATORS_SET:
|
||||
operators.add(op)
|
||||
|
||||
candidates = set(re.findall(r'\b([a-z][a-z0-9_]*)\b', expression))
|
||||
fields = set()
|
||||
for c in candidates:
|
||||
if (c not in OPERATORS_SET and
|
||||
not c.isdigit() and
|
||||
len(c) > 2 and
|
||||
c not in ['true', 'false', 'nan', 'null', 'constant', 'filter']):
|
||||
fields.add(c)
|
||||
|
||||
windows = set()
|
||||
for match in re.finditer(r'\b(\d+)\b', expression):
|
||||
num = int(match.group(1))
|
||||
if num in WINDOWS:
|
||||
windows.add(num)
|
||||
|
||||
return list(operators), list(fields), list(windows)
|
||||
|
||||
|
||||
def main():
|
||||
print("=" * 60)
|
||||
print("开始导入 CSV 到 SQLite")
|
||||
print("=" * 60)
|
||||
|
||||
# 1. 读取 CSV
|
||||
print("\n📖 读取 CSV...")
|
||||
df = pd.read_csv(CSV_PATH)
|
||||
print(f" 共 {len(df):,} 条记录")
|
||||
|
||||
# 2. 解析字段
|
||||
print("\n🔍 解析字段...")
|
||||
|
||||
df['regular_dict'] = df['regular'].apply(safe_parse_dict)
|
||||
df['expression'] = df['regular_dict'].apply(lambda x: x.get('code', ''))
|
||||
|
||||
df['settings_dict'] = df['settings'].apply(safe_parse_dict)
|
||||
df['neutralization'] = df['settings_dict'].apply(lambda x: x.get('neutralization'))
|
||||
df['universe'] = df['settings_dict'].apply(lambda x: x.get('universe'))
|
||||
|
||||
df['is_dict'] = df['is'].apply(safe_parse_dict)
|
||||
df['sharpe'] = df['is_dict'].apply(lambda x: x.get('sharpe'))
|
||||
df['fitness'] = df['is_dict'].apply(lambda x: x.get('fitness'))
|
||||
df['drawdown'] = df['is_dict'].apply(lambda x: x.get('drawdown'))
|
||||
df['turnover'] = df['is_dict'].apply(lambda x: x.get('turnover'))
|
||||
|
||||
# 过滤有效数据
|
||||
df_valid = df[df['expression'].notna() & (df['expression'] != '') & df['fitness'].notna()].copy()
|
||||
print(f" 有效记录: {len(df_valid):,} 条")
|
||||
|
||||
# 3. 去重：同一个 alpha_id 保留 fitness 最高的
|
||||
print("\n🔄 去重（按 alpha_id 保留最高 fitness）...")
|
||||
df_unique = df_valid.sort_values('fitness', ascending=False).drop_duplicates(subset=['id'], keep='first')
|
||||
print(f" 去重后: {len(df_unique):,} 条")
|
||||
|
||||
# 4. 连接 SQLite
|
||||
conn = sqlite3.connect(str(SQLITE_PATH))
|
||||
cursor = conn.cursor()
|
||||
|
||||
# 清空旧数据
|
||||
print("\n🗑️ 清空旧数据...")
|
||||
cursor.execute("DELETE FROM alpha_feature_long")
|
||||
cursor.execute("DELETE FROM alpha_success")
|
||||
conn.commit()
|
||||
|
||||
# 5. 写入 alpha_success
|
||||
print("\n💾 写入 alpha_success 表...")
|
||||
|
||||
for idx, row in df_unique.iterrows():
|
||||
try:
|
||||
cursor.execute("""
|
||||
INSERT INTO alpha_success
|
||||
(alpha_id, expression, sharpe, fitness, drawdown, turnover, neutralization, universe)
|
||||
VALUES (?, ?, ?, ?, ?, ?, ?, ?)
|
||||
""", (
|
||||
row['id'],
|
||||
row['expression'],
|
||||
row['sharpe'],
|
||||
row['fitness'],
|
||||
row['drawdown'],
|
||||
row['turnover'],
|
||||
row['neutralization'],
|
||||
row['universe']
|
||||
))
|
||||
except sqlite3.IntegrityError:
|
||||
# 如果还有重复，跳过（理论上不应该）
|
||||
print(f" 跳过重复: {row['id']}")
|
||||
continue
|
||||
|
||||
conn.commit()
|
||||
|
||||
# 获取插入后的 id 映射
|
||||
cursor.execute("SELECT alpha_id, rowid FROM alpha_success")
|
||||
id_map = {row[0]: row[1] for row in cursor.fetchall()}
|
||||
print(f" 已写入 {len(id_map):,} 条")
|
||||
|
||||
# 6. 提取特征并写入 alpha_feature_long
|
||||
print("\n🔧 提取特征并写入 alpha_feature_long...")
|
||||
|
||||
feature_count = 0
|
||||
processed = 0
|
||||
|
||||
for idx, row in df_unique.iterrows():
|
||||
alpha_id = row['id']
|
||||
if alpha_id not in id_map:
|
||||
continue
|
||||
|
||||
sqlite_id = id_map[alpha_id]
|
||||
expression = row['expression']
|
||||
|
||||
operators, fields, windows = extract_features(expression)
|
||||
|
||||
# 插入算子
|
||||
for op in operators:
|
||||
cursor.execute(
|
||||
"INSERT INTO alpha_feature_long (alpha_id, feature_type, feature_name) VALUES (?, ?, ?)",
|
||||
(sqlite_id, 'operator', op)
|
||||
)
|
||||
feature_count += 1
|
||||
|
||||
# 插入字段
|
||||
for field in fields:
|
||||
cursor.execute(
|
||||
"INSERT INTO alpha_feature_long (alpha_id, feature_type, feature_name) VALUES (?, ?, ?)",
|
||||
(sqlite_id, 'field', field)
|
||||
)
|
||||
feature_count += 1
|
||||
|
||||
# 插入窗口
|
||||
for w in windows:
|
||||
cursor.execute(
|
||||
"INSERT INTO alpha_feature_long (alpha_id, feature_type, feature_name, feature_value) VALUES (?, ?, ?, ?)",
|
||||
(sqlite_id, 'window', str(w), w)
|
||||
)
|
||||
feature_count += 1
|
||||
|
||||
# 插入中性化方式
|
||||
if pd.notna(row['neutralization']):
|
||||
cursor.execute(
|
||||
"INSERT INTO alpha_feature_long (alpha_id, feature_type, feature_name) VALUES (?, ?, ?)",
|
||||
(sqlite_id, 'neutralization', row['neutralization'])
|
||||
)
|
||||
feature_count += 1
|
||||
|
||||
processed += 1
|
||||
if processed % 1000 == 0:
|
||||
print(f" 已处理 {processed:,} 条，特征数 {feature_count:,}")
|
||||
conn.commit() # 每 1000 条提交一次
|
||||
|
||||
conn.commit()
|
||||
|
||||
# 7. 最终统计
|
||||
cursor.execute("SELECT COUNT(*) FROM alpha_success")
|
||||
success_count = cursor.fetchone()[0]
|
||||
cursor.execute("SELECT COUNT(*) FROM alpha_feature_long")
|
||||
final_feature_count = cursor.fetchone()[0]
|
||||
|
||||
print(f"\n✅ 导入完成!")
|
||||
print(f" alpha_success: {success_count:,} 条")
|
||||
print(f" alpha_feature_long: {final_feature_count:,} 条特征")
|
||||
|
||||
# 8. 验证一条数据
|
||||
print("\n🔍 验证前 10 条特征:")
|
||||
cursor.execute("""
|
||||
SELECT a.alpha_id, f.feature_type, f.feature_name
|
||||
FROM alpha_success a
|
||||
JOIN alpha_feature_long f ON a.rowid = f.alpha_id
|
||||
LIMIT 10
|
||||
""")
|
||||
for row in cursor.fetchall():
|
||||
print(f" {row[0]} | {row[1]}: {row[2]}")
|
||||
|
||||
conn.close()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,189 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
计算特征增益值（修复版 v2）
|
||||
"""
|
||||
|
||||
import sqlite3
|
||||
import pandas as pd
|
||||
from pathlib import Path
|
||||
|
||||
SQLITE_PATH = Path(__file__).parent / "alpha_analysis.db"
|
||||
|
||||
def main():
|
||||
print("=" * 60)
|
||||
print("开始计算特征增益值")
|
||||
print("=" * 60)
|
||||
|
||||
conn = sqlite3.connect(str(SQLITE_PATH))
|
||||
|
||||
# 1. 获取所有 Alpha 的 fitness（使用 id 作为主键）
|
||||
print("\n📊 获取 Alpha 数据...")
|
||||
df_alpha = pd.read_sql_query("""
|
||||
SELECT id, alpha_id, fitness
|
||||
FROM alpha_success
|
||||
WHERE fitness IS NOT NULL
|
||||
""", conn)
|
||||
|
||||
print(f" 列名: {list(df_alpha.columns)}")
|
||||
|
||||
global_avg = df_alpha['fitness'].mean()
|
||||
print(f" 全局平均 fitness: {global_avg:.4f}")
|
||||
print(f" 样本数: {len(df_alpha):,}")
|
||||
|
||||
# 2. 获取所有特征
|
||||
print("\n🔧 获取特征数据...")
|
||||
df_features = pd.read_sql_query("""
|
||||
SELECT alpha_id, feature_type, feature_name
|
||||
FROM alpha_feature_long
|
||||
""", conn)
|
||||
print(f" 总特征数: {len(df_features):,}")
|
||||
|
||||
# 3. 合并 Alpha 的 fitness 到特征
|
||||
print("\n📈 计算每个特征的平均分...")
|
||||
|
||||
# 建立 fitness 映射（使用 id）
|
||||
fitness_map = dict(zip(df_alpha['id'], df_alpha['fitness']))
|
||||
df_features['fitness'] = df_features['alpha_id'].map(fitness_map)
|
||||
|
||||
# 过滤掉没有 fitness 的特征
|
||||
df_features = df_features[df_features['fitness'].notna()]
|
||||
print(f" 有效特征数: {len(df_features):,}")
|
||||
|
||||
# 按特征分组统计
|
||||
stats = df_features.groupby(['feature_type', 'feature_name']).agg(
|
||||
avg_score=('fitness', 'mean'),
|
||||
sample_count=('fitness', 'count'),
|
||||
total_fitness=('fitness', 'sum')
|
||||
).reset_index()
|
||||
|
||||
# 计算增益值
|
||||
stats['gain'] = stats['avg_score'] - global_avg
|
||||
|
||||
# 按样本数过滤（至少出现 10 次）
|
||||
stats_filtered = stats[stats['sample_count'] >= 10].copy()
|
||||
print(f" 过滤后特征数（样本≥10）: {len(stats_filtered):,}")
|
||||
|
||||
# 4. 按增益值排序
|
||||
stats_positive = stats_filtered[stats_filtered['gain'] > 0.05].sort_values('gain', ascending=False)
|
||||
stats_negative = stats_filtered[stats_filtered['gain'] < -0.05].sort_values('gain', ascending=True)
|
||||
|
||||
print(f"\n✅ 统计完成:")
|
||||
print(f" 总特征类型数: {len(stats_filtered):,}")
|
||||
print(f" 正向特征 (gain > 0.05): {len(stats_positive):,}")
|
||||
print(f" 负向特征 (gain < -0.05): {len(stats_negative):,}")
|
||||
|
||||
# 5. 输出正向特征 Top 30
|
||||
print("\n" + "=" * 60)
|
||||
print("📈 正向特征 Top 30 (增益值 > 0.05)")
|
||||
print("=" * 60)
|
||||
for _, row in stats_positive.head(30).iterrows():
|
||||
print(f" {row['feature_type']:15} {row['feature_name']:35} 增益: {row['gain']:+.4f} (样本: {row['sample_count']:,})")
|
||||
|
||||
# 6. 输出负向特征 Bottom 30
|
||||
if len(stats_negative) > 0:
|
||||
print("\n" + "=" * 60)
|
||||
print("📉 负向特征 Bottom 30 (增益值 < -0.05)")
|
||||
print("=" * 60)
|
||||
for _, row in stats_negative.head(30).iterrows():
|
||||
print(f" {row['feature_type']:15} {row['feature_name']:35} 增益: {row['gain']:+.4f} (样本: {row['sample_count']:,})")
|
||||
|
||||
# 7. 保存结果到 feature_statistics 表
|
||||
print("\n💾 保存到 feature_statistics 表...")
|
||||
|
||||
# 清空旧数据
|
||||
conn.execute("DELETE FROM feature_statistics")
|
||||
|
||||
# 插入新数据
|
||||
for _, row in stats_filtered.iterrows():
|
||||
positive_effect = 0
|
||||
if row['gain'] > 0.05:
|
||||
positive_effect = 1
|
||||
elif row['gain'] < -0.05:
|
||||
positive_effect = -1
|
||||
|
||||
conn.execute("""
|
||||
INSERT INTO feature_statistics
|
||||
(feature_type, feature_name, gain_value, avg_score, global_avg_score, sample_count, positive_effect)
|
||||
VALUES (?, ?, ?, ?, ?, ?, ?)
|
||||
""", (
|
||||
row['feature_type'],
|
||||
row['feature_name'],
|
||||
row['gain'],
|
||||
row['avg_score'],
|
||||
global_avg,
|
||||
row['sample_count'],
|
||||
positive_effect
|
||||
))
|
||||
|
||||
conn.commit()
|
||||
|
||||
# 8. 更新 generation_bias 表
|
||||
print("\n💾 更新 generation_bias 表...")
|
||||
conn.execute("DELETE FROM generation_bias")
|
||||
|
||||
# 只取增益绝对值 > 0.05 且样本数 >= 10 的特征
|
||||
bias_features = stats_filtered[abs(stats_filtered['gain']) > 0.05]
|
||||
|
||||
for _, row in bias_features.iterrows():
|
||||
bias_weight = 1.0
|
||||
if row['gain'] > 0.05:
|
||||
bias_weight = 1.0 + min(row['gain'] * 2, 2.0)
|
||||
elif row['gain'] < -0.05:
|
||||
bias_weight = max(0.1, 1.0 - abs(row['gain']) * 2)
|
||||
|
||||
conn.execute("""
|
||||
INSERT INTO generation_bias
|
||||
(feature_type, feature_name, bias_weight, gain_value, sample_count, is_active)
|
||||
VALUES (?, ?, ?, ?, ?, 1)
|
||||
""", (
|
||||
row['feature_type'],
|
||||
row['feature_name'],
|
||||
bias_weight,
|
||||
row['gain'],
|
||||
row['sample_count']
|
||||
))
|
||||
|
||||
conn.commit()
|
||||
|
||||
# 9. 统计信息
|
||||
cursor = conn.cursor()
|
||||
cursor.execute("SELECT COUNT(*) FROM feature_statistics")
|
||||
stats_count = cursor.fetchone()[0]
|
||||
cursor.execute("SELECT COUNT(*) FROM generation_bias")
|
||||
bias_count = cursor.fetchone()[0]
|
||||
|
||||
print(f"\n✅ 完成!")
|
||||
print(f" feature_statistics: {stats_count:,} 条")
|
||||
print(f" generation_bias: {bias_count:,} 条")
|
||||
|
||||
# 10. 显示一些关键发现
|
||||
print("\n" + "=" * 60)
|
||||
print("💡 关键发现")
|
||||
print("=" * 60)
|
||||
|
||||
# 最佳算子
|
||||
best_ops = stats_positive[stats_positive['feature_type'] == 'operator'].head(5)
|
||||
if len(best_ops) > 0:
|
||||
print("\n🏆 最佳算子 (增益最高):")
|
||||
for _, row in best_ops.iterrows():
|
||||
print(f" {row['feature_name']}: 增益 {row['gain']:+.4f}")
|
||||
|
||||
# 最佳字段
|
||||
best_fields = stats_positive[stats_positive['feature_type'] == 'field'].head(5)
|
||||
if len(best_fields) > 0:
|
||||
print("\n🏆 最佳字段 (增益最高):")
|
||||
for _, row in best_fields.iterrows():
|
||||
print(f" {row['feature_name']}: 增益 {row['gain']:+.4f}")
|
||||
|
||||
# 最差算子
|
||||
worst_ops = stats_negative[stats_negative['feature_type'] == 'operator'].tail(5)
|
||||
if len(worst_ops) > 0:
|
||||
print("\n⚠️ 最差算子 (增益最低):")
|
||||
for _, row in worst_ops.iterrows():
|
||||
print(f" {row['feature_name']}: 增益 {row['gain']:+.4f}")
|
||||
|
||||
conn.close()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,62 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
import re
|
||||
import pandas as pd
|
||||
import json
|
||||
import ast # Python 的 ast.literal_eval 可以安全解析 Python 字典字符串
|
||||
|
||||
def test_code_001():
|
||||
df = pd.read_csv("alpha_list.csv", nrows=5)
|
||||
|
||||
# 方法1：用 ast.literal_eval（推荐，能解析 Python 字面量）
|
||||
for i, row in df.iterrows():
|
||||
regular_str = row['regular']
|
||||
if pd.notna(regular_str):
|
||||
try:
|
||||
regular_dict = ast.literal_eval(regular_str)
|
||||
code = regular_dict.get('code', '')
|
||||
print(f"Row {i}: {code[:80]}...")
|
||||
except Exception as e:
|
||||
print(f"Row {i}: 解析失败 - {e}")
|
||||
|
||||
# 方法2：看看原始字符串前200个字符
|
||||
print("\n原始字符串示例:")
|
||||
print(df.iloc[0]['regular'][:200])
|
||||
|
||||
def test_code_002():
|
||||
expression = "last_diff_value(ts_sum(subtract(implied_volatility_call_120, implied_volatility_put_90), 20), 5)"
|
||||
|
||||
# 提取算子
|
||||
operators = set()
|
||||
for match in re.finditer(r'\b([a-z_][a-z0-9_]*)\s*\(', expression):
|
||||
op = match.group(1)
|
||||
operators.add(op)
|
||||
print(f"匹配到算子: {op}")
|
||||
|
||||
print(f"\n所有算子: {operators}")
|
||||
|
||||
# 提取字段
|
||||
candidates = set(re.findall(r'\b([a-z][a-z0-9_]*)\b', expression))
|
||||
print(f"\n所有候选词: {candidates}")
|
||||
|
||||
# 过滤掉算子
|
||||
fields = [c for c in candidates if c not in operators and not c.isdigit() and len(c) > 2]
|
||||
print(f"\n字段: {fields}")
|
||||
|
||||
def test_code_003():
|
||||
df = pd.read_csv("alpha_list.csv")
|
||||
|
||||
# 检查 id 列的类型
|
||||
print(f"id 列类型: {df['id'].dtype}")
|
||||
|
||||
# 检查是否有重复
|
||||
duplicates = df[df.duplicated(subset=['id'], keep=False)]
|
||||
print(f"\n重复的 id 数量: {len(duplicates)}")
|
||||
|
||||
if len(duplicates) > 0:
|
||||
print("\n重复的 id 示例:")
|
||||
for aid in duplicates['id'].unique()[:5]:
|
||||
rows = df[df['id'] == aid]
|
||||
print(f" {aid}: 出现 {len(rows)} 次")
|
||||
# 打印这些行的 fitness 看是否相同
|
||||
for _, row in rows.iterrows():
|
||||
print(f" fitness: {row['is'][:100] if isinstance(row['is'], str) else row['is']}...")
|
||||
Reference in New Issue
Block a user