++

data_sets/get_datasets_local/keys_text.txt

@@ -0,0 +1 @@
+["implied_volatility", "iv", "volatility", "call", "put", "option", "skew", "strike", "moneyness", "vix", "variance", "delta", "gamma", "vega", "theta", "atm", "otm", "itm", "surface", "term", "expiry", "risk_reversal", "butterfly", "spread", "premium", "volume", "open_interest", "bid", "ask", "mid", "spread", "ratio", "percentile", "rank", "zscore", "decay", "momentum", "trend", "sum", "mean", "std", "corr", "beta", "residual", "resid", "regression", "factor", "alpha", "exposure", "neutralized", "industry", "sector", "market_cap", "volume", "liquidity", "turnover", "float", "short_interest", "borrow_fee", "dividend", "earnings", "surprise", "revision", "estimate", "actual", "guidance", "sentiment", "news", "analyst", "rating", "target", "recommendation", "upgrade", "downgrade", "initiation", "coverage", "momentum", "reversal", "value", "growth", "quality", "leverage", "profitability", "efficiency", "solvency", "liquidity", "accruals", "investment", "intangibles", "f_score", "z_score", "o_score", "m_score", "g_score", "p_score"]

data_sets/get_datasets_local/load_data_sets.py

@@ -0,0 +1,129 @@
+# -*- coding: utf-8 -*-
+import os
+import csv
+from collections import defaultdict
+
+
+def load_csv_data(file_path):
+    """
+    加载CSV文件数据
+    
+    Args:
+        file_path: CSV文件路径
+    
+    Returns:
+        数据列表，每个元素是一个字典
+    """
+    if not os.path.exists(file_path):
+        print(f"文件不存在: {file_path}")
+        return []
+    
+    data = []
+    with open(file_path, 'r', encoding='utf-8') as f:
+        reader = csv.DictReader(f)  # 使用DictReader方便按字段名访问
+        for row in reader:
+            data.append(row)
+    
+    print(f"成功加载 {len(data)} 条数据")
+    return data
+
+
+def group_by_dataset(data):
+    """
+    按dataset_name字段对数据进行分组
+    
+    Args:
+        data: 数据列表
+    
+    Returns:
+        字典，键为dataset_name，值为该数据集下的数据列表
+    """
+    grouped_data = defaultdict(list)
+    
+    for item in data:
+        dataset = item.get('dataset_name', '未分类数据集')
+        grouped_data[dataset].append(item)
+    
+    return grouped_data
+
+
+def create_dataset_files(grouped_data):
+    """
+    为每个数据集创建txt文件
+    
+    Args:
+        grouped_data: 按数据集分组的数据
+    """
+    # 创建dataset文件夹
+    dataset_dir = "dataset"
+    if not os.path.exists(dataset_dir):
+        os.makedirs(dataset_dir)
+        print(f"创建文件夹: {dataset_dir}")
+    
+    # 按数据集名称排序
+    sorted_datasets = sorted(grouped_data.keys())
+    
+    for dataset in sorted_datasets:
+        # 生成安全的文件名（替换特殊字符）
+        safe_dataset_name = dataset.replace('/', '_').replace('\\', '_').replace(':', '_').replace('*', '_').replace('?', '_').replace('"', '_').replace('<', '_').replace('>', '_').replace('|', '_')
+        file_path = os.path.join(dataset_dir, f"{safe_dataset_name}.txt")
+        
+        count = len(grouped_data[dataset])
+        
+        with open(file_path, 'w', encoding='utf-8') as f:
+            f.write(f"数据集: {dataset} (共 {count} 条)\n")
+            f.write("=" * 60 + "\n\n")
+            
+            # 写入该数据集下的所有name，按name排序
+            sorted_items = sorted(grouped_data[dataset], key=lambda x: x['name'])
+            for item in sorted_items:
+                f.write(f"- {item['name']}\n")
+        
+        print(f"已创建: {file_path}")
+
+
+def print_datasets_with_names(grouped_data):
+    """
+    打印数据集和对应的name到控制台
+    
+    Args:
+        grouped_data: 按数据集分组的数据
+    """
+    # 按数据集名称排序
+    sorted_datasets = sorted(grouped_data.keys())
+    
+    for dataset in sorted_datasets:
+        count = len(grouped_data[dataset])
+        print(f"\n数据集: {dataset} (共 {count} 条)")
+        print("-" * 60)
+        
+        # 打印该数据集下的所有name
+        sorted_items = sorted(grouped_data[dataset], key=lambda x: x['name'])
+        for item in sorted_items:
+            print(f"  - {item['name']}")
+
+
+def main():
+    """
+    主函数
+    """
+    # CSV文件路径
+    csv_file_path = "all_data_combined.csv"  # 请替换为实际的文件路径
+    
+    # 1. 加载数据
+    data = load_csv_data(csv_file_path)
+    if not data:
+        return
+    
+    # 2. 按数据集分组
+    grouped_data = group_by_dataset(data)
+    
+    # 3. 创建数据集文件
+    create_dataset_files(grouped_data)
+    
+    # 4. 打印到控制台
+    print_datasets_with_names(grouped_data)
+
+
+if __name__ == "__main__":
+    main()

data_sets/get_datasets_local/seach_data_sets.py

@@ -0,0 +1,121 @@
+# -*- coding: utf-8 -*-
+import os
+import jieba
+import csv
+
+
+def process_text(text):
+    """
+    使用jieba分词并过滤不需要的字符
+    """
+    filter_list = ['\n', '\t', '\r', '\b', '\f', '\v', '：', '的', '或', '10', '天', '了', '可', '是', '该', '，', ' ', '、', '让', '和', '集',
+                   '/', '日', '在', '（', '_', '-', ')', '(', '上', '距', '与', '比', '下', '及', '）', '...', '；', '%', '&', '+', ',', '.',
+                   ':', ';', '<', '=', '>', '?', '[', ']', '|', '—', '。'
+                   ]
+
+    text_list = jieba.lcut(text)
+    results = []
+    for tl in text_list:
+        should_include = True
+        for fl in filter_list:
+            if fl == tl:
+                should_include = False
+                break
+        if should_include:
+            results.append(tl)
+
+    if results:
+        return list(set(results))  # 去重
+    else:
+        return None
+
+
+def search_data_sets_by_keywords(csv_file_path, keywords):
+    """
+    根据关键词搜索csv文件中的匹配项
+    
+    Args:
+        csv_file_path: CSV文件路径
+        keywords: 关键词列表
+    
+    Returns:
+        匹配的数据集列表
+    """
+    if not os.path.exists(csv_file_path):
+        print(f"文件不存在: {csv_file_path}")
+        return []
+
+    data_dict = {}  # 使用字典来存储，以id为键去重
+
+    with open(csv_file_path, 'r', encoding='utf-8') as f:
+        reader = csv.reader(f)
+        for row in reader:
+            # 检查每一行的第12列(索引11)和第13列(索引12)是否包含任意关键词
+            for key in keywords:
+                if key in row[11] or key in row[12]:
+                    item_id = row[0]
+                    # 如果id不存在，或者想要保留第一个出现的记录
+                    if item_id not in data_dict:
+                        data_dict[item_id] = {
+                            'id': item_id,
+                            'data_set_name': row[1],
+                            'description': row[2],
+                            'description_cn': row[11],
+                        }
+
+    # 将字典的值转换为列表
+    return list(data_dict.values())
+
+
+def extract_keywords_from_text(text_file_path):
+    """
+    从文本文件中提取关键词
+    
+    Args:
+        text_file_path: 文本文件路径
+    
+    Returns:
+        提取的关键词列表
+    """
+    if not os.path.exists(text_file_path):
+        print(f"文件不存在: {text_file_path}")
+        return None
+    
+    with open(text_file_path, 'r', encoding='utf-8') as f:
+        text_list = [line.strip() for line in f if line.strip()]
+        if not text_list:
+            print('关键词文本无数据')
+            return None
+
+        # 将所有文本合并并用分号连接，然后进行处理
+        result_str = process_text(';'.join(text_list))
+        
+        if result_str:
+            print(f'关键词提取结果: {result_str}')
+            return result_str
+        else:
+            return None
+
+
+def main():
+    keys_text_path = "keys_text.txt"
+    keywords = extract_keywords_from_text(keys_text_path)
+    
+    if not keywords:
+        print("无法提取关键词")
+        return
+    
+    csv_file_path = "all_data_combined.csv"
+    matched_data_sets = search_data_sets_by_keywords(csv_file_path, keywords)
+    
+    print(f'从数据集中提取了 {len(matched_data_sets)} 条匹配数据')
+    
+    for data_set in matched_data_sets:
+        print(f"数据集: {data_set['data_set_name']}")
+        print(f"英文描述: {data_set['description']}")
+        print(f"中文描述: {data_set['description_cn']}")
+        print("-" * 50)
+
+
+if __name__ == "__main__":
+    main()