|
|
#!/usr/bin/env python3
|
|
|
"""
|
|
|
Alpha 特征分析数据库 - 长表方案
|
|
|
适用场景：95,003 条表达式，6,488 个字段，84 个算子
|
|
|
"""
|
|
|
|
|
|
import sqlite3
|
|
|
import os
|
|
|
import re
|
|
|
from pathlib import Path
|
|
|
|
|
|
DB_PATH = Path(__file__).parent / "alpha_analysis.db"
|
|
|
|
|
|
# 84 个算子完整列表
|
|
|
OPERATORS = [
|
|
|
# Arithmetic (16)
|
|
|
'add', 'abs', 'log', 'subtract', 'signed_power', 'sign', 'reverse', 'power',
|
|
|
'multiply', 'min', 'max', 'inverse', 'sqrt', 's_log_1p', 'densify', 'divide',
|
|
|
# Logical (12)
|
|
|
'not', 'and', 'less', 'equal', 'or', 'not_equal', 'greater', 'greater_equal',
|
|
|
'less_equal', 'is_nan', 'if_else',
|
|
|
# Time Series (29)
|
|
|
'ts_sum', 'ts_scale', 'ts_mean', 'ts_zscore', 'ts_std_dev', 'kth_element',
|
|
|
'inst_tvr', 'ts_corr', 'ts_count_nans', 'ts_target_tvr_decay', 'ts_median',
|
|
|
'ts_covariance', 'ts_decay_linear', 'ts_product', 'ts_regression', 'ts_delta_limit',
|
|
|
'ts_step', 'ts_decay_exp_window', 'ts_quantile', 'days_from_last_change', 'hump',
|
|
|
'last_diff_value', 'ts_arg_max', 'ts_arg_min', 'ts_av_diff', 'ts_backfill',
|
|
|
'ts_rank', 'ts_delay', 'ts_delta',
|
|
|
# Cross Sectional (8)
|
|
|
'winsorize', 'truncate', 'regression_neut', 'scale', 'rank', 'quantile',
|
|
|
'normalize', 'zscore',
|
|
|
# Vector (7)
|
|
|
'vec_min', 'vec_count', 'vec_stddev', 'vec_range', 'vec_avg', 'vec_sum', 'vec_max',
|
|
|
# Transformational (4)
|
|
|
'left_tail', 'trade_when', 'right_tail', 'bucket',
|
|
|
# Group (8)
|
|
|
'group_rank', 'group_cartesian_product', 'group_backfill', 'group_mean',
|
|
|
'group_neutralize', 'group_normalize', 'group_median', 'group_scale', 'group_zscore'
|
|
|
]
|
|
|
|
|
|
# 常用窗口参数
|
|
|
WINDOWS = [1, 2, 5, 10, 20, 30, 60, 90, 252]
|
|
|
|
|
|
# 中性化方式
|
|
|
NEUTRALIZATIONS = ['FAST', 'SUBINDUSTRY', 'INDUSTRY', 'NONE']
|
|
|
|
|
|
|
|
|
def get_connection():
|
|
|
"""获取数据库连接"""
|
|
|
conn = sqlite3.connect(str(DB_PATH))
|
|
|
conn.row_factory = sqlite3.Row
|
|
|
return conn
|
|
|
|
|
|
|
|
|
def create_tables(conn):
|
|
|
"""创建所有表"""
|
|
|
cursor = conn.cursor()
|
|
|
|
|
|
# =========================================================
|
|
|
# 表1: alpha_success（成功样本 - 主表）
|
|
|
# =========================================================
|
|
|
cursor.execute("""
|
|
|
CREATE TABLE IF NOT EXISTS alpha_success (
|
|
|
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
|
alpha_id VARCHAR(64) UNIQUE,
|
|
|
expression TEXT NOT NULL,
|
|
|
sharpe REAL,
|
|
|
fitness REAL,
|
|
|
returns REAL,
|
|
|
drawdown REAL,
|
|
|
turnover REAL,
|
|
|
long_count INTEGER,
|
|
|
short_count INTEGER,
|
|
|
neutralization VARCHAR(32),
|
|
|
universe VARCHAR(32),
|
|
|
dataset VARCHAR(32),
|
|
|
score REAL,
|
|
|
created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP
|
|
|
)
|
|
|
""")
|
|
|
|
|
|
# 索引
|
|
|
cursor.execute("CREATE INDEX IF NOT EXISTS idx_success_score ON alpha_success(score)")
|
|
|
cursor.execute("CREATE INDEX IF NOT EXISTS idx_success_sharpe ON alpha_success(sharpe)")
|
|
|
cursor.execute("CREATE INDEX IF NOT EXISTS idx_success_neutralization ON alpha_success(neutralization)")
|
|
|
|
|
|
# =========================================================
|
|
|
# 表2: alpha_feature_long（特征长表 - 核心）
|
|
|
# 每个特征一行：算子、字段、窗口、配置
|
|
|
# =========================================================
|
|
|
cursor.execute("""
|
|
|
CREATE TABLE IF NOT EXISTS alpha_feature_long (
|
|
|
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
|
alpha_id INTEGER NOT NULL,
|
|
|
feature_type VARCHAR(32) NOT NULL, -- operator, field, window, neutralization
|
|
|
feature_name VARCHAR(128) NOT NULL,
|
|
|
feature_value INTEGER DEFAULT 1, -- 1=存在，窗口值存具体数字
|
|
|
created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
|
|
|
FOREIGN KEY (alpha_id) REFERENCES alpha_success(id) ON DELETE CASCADE
|
|
|
)
|
|
|
""")
|
|
|
|
|
|
# 索引（加速统计查询）
|
|
|
cursor.execute("CREATE INDEX IF NOT EXISTS idx_feature_type_name ON alpha_feature_long(feature_type, feature_name)")
|
|
|
cursor.execute("CREATE INDEX IF NOT EXISTS idx_feature_alpha_id ON alpha_feature_long(alpha_id)")
|
|
|
|
|
|
# =========================================================
|
|
|
# 表3: feature_statistics（物化统计表 - 反馈核心）
|
|
|
# 存储每个特征的增益值，供生成器使用
|
|
|
# =========================================================
|
|
|
cursor.execute("""
|
|
|
CREATE TABLE IF NOT EXISTS feature_statistics (
|
|
|
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
|
feature_type VARCHAR(32) NOT NULL,
|
|
|
feature_name VARCHAR(128) NOT NULL,
|
|
|
gain_value REAL, -- 增益值 = 特征平均分 - 全局平均分
|
|
|
avg_score REAL, -- 包含该特征的表达式平均分
|
|
|
global_avg_score REAL, -- 统计时的全局平均分
|
|
|
sample_count INTEGER, -- 包含该特征的表达式数量
|
|
|
positive_effect INTEGER DEFAULT 0, -- 1=正向特征(gain>0.05), -1=负向特征(gain<-0.05)
|
|
|
is_active INTEGER DEFAULT 1, -- 是否启用反馈
|
|
|
calculated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
|
|
|
UNIQUE(feature_type, feature_name)
|
|
|
)
|
|
|
""")
|
|
|
|
|
|
cursor.execute("CREATE INDEX IF NOT EXISTS idx_stats_positive ON feature_statistics(positive_effect)")
|
|
|
cursor.execute("CREATE INDEX IF NOT EXISTS idx_stats_gain ON feature_statistics(gain_value DESC)")
|
|
|
|
|
|
# =========================================================
|
|
|
# 表4: alpha_failure（失败样本 - 不兼容组合）
|
|
|
# =========================================================
|
|
|
cursor.execute("""
|
|
|
CREATE TABLE IF NOT EXISTS alpha_failure (
|
|
|
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
|
operator VARCHAR(64) NOT NULL,
|
|
|
field VARCHAR(64) NOT NULL,
|
|
|
expression TEXT,
|
|
|
error_type VARCHAR(32),
|
|
|
created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP
|
|
|
)
|
|
|
""")
|
|
|
|
|
|
cursor.execute("CREATE INDEX IF NOT EXISTS idx_failure_combo ON alpha_failure(operator, field)")
|
|
|
|
|
|
# =========================================================
|
|
|
# 表5: operator_field_blacklist（黑名单 - 从不兼容统计生成）
|
|
|
# =========================================================
|
|
|
cursor.execute("""
|
|
|
CREATE TABLE IF NOT EXISTS operator_field_blacklist (
|
|
|
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
|
operator VARCHAR(64) NOT NULL,
|
|
|
field VARCHAR(64) NOT NULL,
|
|
|
fail_count INTEGER DEFAULT 0,
|
|
|
total_attempts INTEGER DEFAULT 0,
|
|
|
fail_rate REAL DEFAULT 0.0,
|
|
|
is_active INTEGER DEFAULT 1,
|
|
|
created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
|
|
|
updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
|
|
|
UNIQUE(operator, field)
|
|
|
)
|
|
|
""")
|
|
|
|
|
|
cursor.execute("CREATE INDEX IF NOT EXISTS idx_blacklist_active ON operator_field_blacklist(is_active)")
|
|
|
|
|
|
# =========================================================
|
|
|
# 表6: generation_bias（生成偏向配置 - 反馈到生成器）
|
|
|
# =========================================================
|
|
|
cursor.execute("""
|
|
|
CREATE TABLE IF NOT EXISTS generation_bias (
|
|
|
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
|
feature_type VARCHAR(32) NOT NULL,
|
|
|
feature_name VARCHAR(128) NOT NULL,
|
|
|
bias_weight REAL DEFAULT 1.0, -- >1 正向，<1 负向，1 中性
|
|
|
gain_value REAL,
|
|
|
sample_count INTEGER,
|
|
|
is_active INTEGER DEFAULT 1,
|
|
|
created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
|
|
|
updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
|
|
|
UNIQUE(feature_type, feature_name)
|
|
|
)
|
|
|
""")
|
|
|
|
|
|
cursor.execute("CREATE INDEX IF NOT EXISTS idx_bias_weight ON generation_bias(bias_weight DESC)")
|
|
|
|
|
|
conn.commit()
|
|
|
print("✅ 所有表创建成功")
|
|
|
print(f"📁 数据库路径: {DB_PATH}")
|
|
|
print(f"\n📊 表结构:")
|
|
|
print(f" - alpha_success: 主表，存储表达式和回测结果")
|
|
|
print(f" - alpha_feature_long: 长表，每个特征一行")
|
|
|
print(f" - feature_statistics: 物化统计，特征增益值")
|
|
|
print(f" - alpha_failure: 失败样本记录")
|
|
|
print(f" - operator_field_blacklist: 不兼容黑名单")
|
|
|
print(f" - generation_bias: 生成偏向配置")
|
|
|
|
|
|
|
|
|
def show_table_info(conn):
|
|
|
"""显示各表记录数"""
|
|
|
cursor = conn.cursor()
|
|
|
tables = ['alpha_success', 'alpha_feature_long', 'feature_statistics',
|
|
|
'alpha_failure', 'operator_field_blacklist', 'generation_bias']
|
|
|
|
|
|
print("\n📋 当前表记录数:")
|
|
|
for table in tables:
|
|
|
cursor.execute(f"SELECT COUNT(*) FROM {table}")
|
|
|
count = cursor.fetchone()[0]
|
|
|
print(f" - {table}: {count:,}")
|
|
|
|
|
|
|
|
|
if __name__ == "__main__":
|
|
|
# 检查数据库是否存在
|
|
|
if DB_PATH.exists():
|
|
|
print(f"⚠️ 数据库已存在: {DB_PATH}")
|
|
|
response = input("是否删除并重建？(y/N): ").strip().lower()
|
|
|
if response == 'y':
|
|
|
os.remove(DB_PATH)
|
|
|
print("🗑️ 已删除旧数据库")
|
|
|
else:
|
|
|
print("❌ 已取消操作")
|
|
|
exit(0)
|
|
|
|
|
|
# 创建数据库和表
|
|
|
conn = get_connection()
|
|
|
try:
|
|
|
create_tables(conn)
|
|
|
show_table_info(conn)
|
|
|
finally:
|
|
|
conn.close()
|
|
|
|
|
|
print("\n✅ 初始化完成！下一步：导入数据并提取特征") |
