diff --git a/category_files/analyst.json b/category_files/analyst.json new file mode 100644 index 0000000..6ef4758 --- /dev/null +++ b/category_files/analyst.json @@ -0,0 +1,852 @@ +[ + { + "id": "analyst10", + "region": "USA", + "universe": "TOP3000" + }, + { + "id": "analyst11", + "region": "USA", + "universe": "TOP3000" + }, + { + "id": "analyst14", + "region": "USA", + "universe": "TOP3000" + }, + { + "id": "analyst15", + "region": "USA", + "universe": "TOP3000" + }, + { + "id": "analyst16", + "region": "USA", + "universe": "TOP3000" + }, + { + "id": "analyst39", + "region": "USA", + "universe": "TOP3000" + }, + { + "id": "analyst4", + "region": "USA", + "universe": "TOP3000" + }, + { + "id": "analyst40", + "region": "USA", + "universe": "TOP3000" + }, + { + "id": "analyst44", + "region": "USA", + "universe": "TOP3000" + }, + { + "id": "analyst45", + "region": "USA", + "universe": "TOP3000" + }, + { + "id": "analyst49", + "region": "USA", + "universe": "TOP3000" + }, + { + "id": "analyst52", + "region": "USA", + "universe": "TOP3000" + }, + { + "id": "analyst69", + "region": "USA", + "universe": "TOP3000" + }, + { + "id": "analyst7", + "region": "USA", + "universe": "TOP3000" + }, + { + "id": "analyst82", + "region": "USA", + "universe": "TOP3000" + }, + { + "id": "analyst83", + "region": "USA", + "universe": "TOP3000" + }, + { + "id": "model211", + "region": "USA", + "universe": "TOP3000" + }, + { + "id": "model52", + "region": "USA", + "universe": "TOP3000" + }, + { + "id": "news87", + "region": "USA", + "universe": "TOP3000" + }, + { + "id": "analyst10", + "region": "USA", + "universe": "TOP1000" + }, + { + "id": "analyst11", + "region": "USA", + "universe": "TOP1000" + }, + { + "id": "analyst14", + "region": "USA", + "universe": "TOP1000" + }, + { + "id": "analyst15", + "region": "USA", + "universe": "TOP1000" + }, + { + "id": "analyst16", + "region": "USA", + "universe": "TOP1000" + }, + { + "id": "analyst39", + "region": "USA", + "universe": "TOP1000" + }, + { + "id": "analyst4", + "region": "USA", + "universe": "TOP1000" + }, + { + "id": "analyst40", + "region": "USA", + "universe": "TOP1000" + }, + { + "id": "analyst44", + "region": "USA", + "universe": "TOP1000" + }, + { + "id": "analyst45", + "region": "USA", + "universe": "TOP1000" + }, + { + "id": "analyst49", + "region": "USA", + "universe": "TOP1000" + }, + { + "id": "analyst52", + "region": "USA", + "universe": "TOP1000" + }, + { + "id": "analyst69", + "region": "USA", + "universe": "TOP1000" + }, + { + "id": "analyst7", + "region": "USA", + "universe": "TOP1000" + }, + { + "id": "analyst82", + "region": "USA", + "universe": "TOP1000" + }, + { + "id": "analyst83", + "region": "USA", + "universe": "TOP1000" + }, + { + "id": "model211", + "region": "USA", + "universe": "TOP1000" + }, + { + "id": "model52", + "region": "USA", + "universe": "TOP1000" + }, + { + "id": "news87", + "region": "USA", + "universe": "TOP1000" + }, + { + "id": "analyst10", + "region": "USA", + "universe": "TOP500" + }, + { + "id": "analyst11", + "region": "USA", + "universe": "TOP500" + }, + { + "id": "analyst14", + "region": "USA", + "universe": "TOP500" + }, + { + "id": "analyst15", + "region": "USA", + "universe": "TOP500" + }, + { + "id": "analyst16", + "region": "USA", + "universe": "TOP500" + }, + { + "id": "analyst39", + "region": "USA", + "universe": "TOP500" + }, + { + "id": "analyst4", + "region": "USA", + "universe": "TOP500" + }, + { + "id": "analyst40", + "region": "USA", + "universe": "TOP500" + }, + { + "id": "analyst44", + "region": "USA", + "universe": "TOP500" + }, + { + "id": "analyst45", + "region": "USA", + "universe": "TOP500" + }, + { + "id": "analyst49", + "region": "USA", + "universe": "TOP500" + }, + { + "id": "analyst52", + "region": "USA", + "universe": "TOP500" + }, + { + "id": "analyst69", + "region": "USA", + "universe": "TOP500" + }, + { + "id": "analyst7", + "region": "USA", + "universe": "TOP500" + }, + { + "id": "analyst82", + "region": "USA", + "universe": "TOP500" + }, + { + "id": "analyst83", + "region": "USA", + "universe": "TOP500" + }, + { + "id": "model211", + "region": "USA", + "universe": "TOP500" + }, + { + "id": "model52", + "region": "USA", + "universe": "TOP500" + }, + { + "id": "news87", + "region": "USA", + "universe": "TOP500" + }, + { + "id": "analyst10", + "region": "USA", + "universe": "TOP200" + }, + { + "id": "analyst11", + "region": "USA", + "universe": "TOP200" + }, + { + "id": "analyst14", + "region": "USA", + "universe": "TOP200" + }, + { + "id": "analyst15", + "region": "USA", + "universe": "TOP200" + }, + { + "id": "analyst16", + "region": "USA", + "universe": "TOP200" + }, + { + "id": "analyst39", + "region": "USA", + "universe": "TOP200" + }, + { + "id": "analyst4", + "region": "USA", + "universe": "TOP200" + }, + { + "id": "analyst40", + "region": "USA", + "universe": "TOP200" + }, + { + "id": "analyst44", + "region": "USA", + "universe": "TOP200" + }, + { + "id": "analyst45", + "region": "USA", + "universe": "TOP200" + }, + { + "id": "analyst49", + "region": "USA", + "universe": "TOP200" + }, + { + "id": "analyst52", + "region": "USA", + "universe": "TOP200" + }, + { + "id": "analyst69", + "region": "USA", + "universe": "TOP200" + }, + { + "id": "analyst7", + "region": "USA", + "universe": "TOP200" + }, + { + "id": "analyst82", + "region": "USA", + "universe": "TOP200" + }, + { + "id": "analyst83", + "region": "USA", + "universe": "TOP200" + }, + { + "id": "model211", + "region": "USA", + "universe": "TOP200" + }, + { + "id": "model52", + "region": "USA", + "universe": "TOP200" + }, + { + "id": "news87", + "region": "USA", + "universe": "TOP200" + }, + { + "id": "analyst10", + "region": "USA", + "universe": "ILLIQUID_MINVOL1M" + }, + { + "id": "analyst11", + "region": "USA", + "universe": "ILLIQUID_MINVOL1M" + }, + { + "id": "analyst14", + "region": "USA", + "universe": "ILLIQUID_MINVOL1M" + }, + { + "id": "analyst15", + "region": "USA", + "universe": "ILLIQUID_MINVOL1M" + }, + { + "id": "analyst16", + "region": "USA", + "universe": "ILLIQUID_MINVOL1M" + }, + { + "id": "analyst39", + "region": "USA", + "universe": "ILLIQUID_MINVOL1M" + }, + { + "id": "analyst4", + "region": "USA", + "universe": "ILLIQUID_MINVOL1M" + }, + { + "id": "analyst44", + "region": "USA", + "universe": "ILLIQUID_MINVOL1M" + }, + { + "id": "analyst45", + "region": "USA", + "universe": "ILLIQUID_MINVOL1M" + }, + { + "id": "analyst49", + "region": "USA", + "universe": "ILLIQUID_MINVOL1M" + }, + { + "id": "analyst52", + "region": "USA", + "universe": "ILLIQUID_MINVOL1M" + }, + { + "id": "analyst69", + "region": "USA", + "universe": "ILLIQUID_MINVOL1M" + }, + { + "id": "analyst7", + "region": "USA", + "universe": "ILLIQUID_MINVOL1M" + }, + { + "id": "analyst82", + "region": "USA", + "universe": "ILLIQUID_MINVOL1M" + }, + { + "id": "analyst83", + "region": "USA", + "universe": "ILLIQUID_MINVOL1M" + }, + { + "id": "model211", + "region": "USA", + "universe": "ILLIQUID_MINVOL1M" + }, + { + "id": "news87", + "region": "USA", + "universe": "ILLIQUID_MINVOL1M" + }, + { + "id": "analyst10", + "region": "USA", + "universe": "TOPSP500" + }, + { + "id": "analyst11", + "region": "USA", + "universe": "TOPSP500" + }, + { + "id": "analyst14", + "region": "USA", + "universe": "TOPSP500" + }, + { + "id": "analyst15", + "region": "USA", + "universe": "TOPSP500" + }, + { + "id": "analyst16", + "region": "USA", + "universe": "TOPSP500" + }, + { + "id": "analyst39", + "region": "USA", + "universe": "TOPSP500" + }, + { + "id": "analyst4", + "region": "USA", + "universe": "TOPSP500" + }, + { + "id": "analyst40", + "region": "USA", + "universe": "TOPSP500" + }, + { + "id": "analyst44", + "region": "USA", + "universe": "TOPSP500" + }, + { + "id": "analyst45", + "region": "USA", + "universe": "TOPSP500" + }, + { + "id": "analyst49", + "region": "USA", + "universe": "TOPSP500" + }, + { + "id": "analyst52", + "region": "USA", + "universe": "TOPSP500" + }, + { + "id": "analyst69", + "region": "USA", + "universe": "TOPSP500" + }, + { + "id": "analyst7", + "region": "USA", + "universe": "TOPSP500" + }, + { + "id": "analyst83", + "region": "USA", + "universe": "TOPSP500" + }, + { + "id": "model211", + "region": "USA", + "universe": "TOPSP500" + }, + { + "id": "model52", + "region": "USA", + "universe": "TOPSP500" + }, + { + "id": "news87", + "region": "USA", + "universe": "TOPSP500" + }, + { + "id": "analyst10", + "region": "GLB", + "universe": "TOP3000" + }, + { + "id": "analyst11", + "region": "GLB", + "universe": "TOP3000" + }, + { + "id": "analyst14", + "region": "GLB", + "universe": "TOP3000" + }, + { + "id": "analyst15", + "region": "GLB", + "universe": "TOP3000" + }, + { + "id": "analyst39", + "region": "GLB", + "universe": "TOP3000" + }, + { + "id": "analyst44", + "region": "GLB", + "universe": "TOP3000" + }, + { + "id": "analyst45", + "region": "GLB", + "universe": "TOP3000" + }, + { + "id": "analyst46", + "region": "GLB", + "universe": "TOP3000" + }, + { + "id": "analyst69", + "region": "GLB", + "universe": "TOP3000" + }, + { + "id": "analyst10", + "region": "EUR", + "universe": "ILLIQUID_MINVOL1M" + }, + { + "id": "analyst11", + "region": "EUR", + "universe": "ILLIQUID_MINVOL1M" + }, + { + "id": "analyst14", + "region": "EUR", + "universe": "ILLIQUID_MINVOL1M" + }, + { + "id": "analyst15", + "region": "EUR", + "universe": "ILLIQUID_MINVOL1M" + }, + { + "id": "analyst4", + "region": "EUR", + "universe": "ILLIQUID_MINVOL1M" + }, + { + "id": "analyst44", + "region": "EUR", + "universe": "ILLIQUID_MINVOL1M" + }, + { + "id": "analyst45", + "region": "EUR", + "universe": "ILLIQUID_MINVOL1M" + }, + { + "id": "analyst46", + "region": "EUR", + "universe": "ILLIQUID_MINVOL1M" + }, + { + "id": "analyst48", + "region": "EUR", + "universe": "ILLIQUID_MINVOL1M" + }, + { + "id": "analyst69", + "region": "EUR", + "universe": "ILLIQUID_MINVOL1M" + }, + { + "id": "analyst7", + "region": "EUR", + "universe": "ILLIQUID_MINVOL1M" + }, + { + "id": "analyst84", + "region": "EUR", + "universe": "ILLIQUID_MINVOL1M" + }, + { + "id": "sentiment12", + "region": "EUR", + "universe": "ILLIQUID_MINVOL1M" + }, + { + "id": "analyst11", + "region": "ASI", + "universe": "ILLIQUID_MINVOL1M" + }, + { + "id": "analyst14", + "region": "ASI", + "universe": "ILLIQUID_MINVOL1M" + }, + { + "id": "analyst15", + "region": "ASI", + "universe": "ILLIQUID_MINVOL1M" + }, + { + "id": "analyst16", + "region": "ASI", + "universe": "ILLIQUID_MINVOL1M" + }, + { + "id": "analyst4", + "region": "ASI", + "universe": "ILLIQUID_MINVOL1M" + }, + { + "id": "analyst45", + "region": "ASI", + "universe": "ILLIQUID_MINVOL1M" + }, + { + "id": "analyst48", + "region": "ASI", + "universe": "ILLIQUID_MINVOL1M" + }, + { + "id": "analyst69", + "region": "ASI", + "universe": "ILLIQUID_MINVOL1M" + }, + { + "id": "analyst10", + "region": "TWN", + "universe": "TOP500" + }, + { + "id": "analyst11", + "region": "TWN", + "universe": "TOP500" + }, + { + "id": "analyst14", + "region": "TWN", + "universe": "TOP500" + }, + { + "id": "analyst15", + "region": "TWN", + "universe": "TOP500" + }, + { + "id": "analyst16", + "region": "TWN", + "universe": "TOP500" + }, + { + "id": "analyst21", + "region": "TWN", + "universe": "TOP500" + }, + { + "id": "analyst39", + "region": "TWN", + "universe": "TOP500" + }, + { + "id": "analyst4", + "region": "TWN", + "universe": "TOP500" + }, + { + "id": "analyst44", + "region": "TWN", + "universe": "TOP500" + }, + { + "id": "analyst45", + "region": "TWN", + "universe": "TOP500" + }, + { + "id": "analyst46", + "region": "TWN", + "universe": "TOP500" + }, + { + "id": "analyst48", + "region": "TWN", + "universe": "TOP500" + }, + { + "id": "analyst81", + "region": "TWN", + "universe": "TOP500" + }, + { + "id": "analyst9", + "region": "TWN", + "universe": "TOP500" + }, + { + "id": "analyst92", + "region": "TWN", + "universe": "TOP500" + }, + { + "id": "analyst94", + "region": "TWN", + "universe": "TOP500" + }, + { + "id": "model52", + "region": "TWN", + "universe": "TOP500" + }, + { + "id": "other411", + "region": "TWN", + "universe": "TOP500" + }, + { + "id": "other423", + "region": "TWN", + "universe": "TOP500" + }, + { + "id": "analyst11", + "region": "IND", + "universe": "TOP500" + }, + { + "id": "analyst39", + "region": "IND", + "universe": "TOP500" + }, + { + "id": "analyst4", + "region": "IND", + "universe": "TOP500" + }, + { + "id": "analyst44", + "region": "IND", + "universe": "TOP500" + }, + { + "id": "analyst45", + "region": "IND", + "universe": "TOP500" + }, + { + "id": "analyst46", + "region": "IND", + "universe": "TOP500" + }, + { + "id": "analyst48", + "region": "IND", + "universe": "TOP500" + }, + { + "id": "analyst81", + "region": "IND", + "universe": "TOP500" + }, + { + "id": "analyst83", + "region": "IND", + "universe": "TOP500" + }, + { + "id": "analyst9", + "region": "IND", + "universe": "TOP500" + } +] \ No newline at end of file diff --git a/category_files/broker.json b/category_files/broker.json new file mode 100644 index 0000000..9f6dfbb --- /dev/null +++ b/category_files/broker.json @@ -0,0 +1,7 @@ +[ + { + "id": "broker1", + "region": "ASI", + "universe": "MINVOL1M" + } +] \ No newline at end of file diff --git a/category_files/earnings.json b/category_files/earnings.json new file mode 100644 index 0000000..e6ae5a4 --- /dev/null +++ b/category_files/earnings.json @@ -0,0 +1,37 @@ +[ + { + "id": "earnings6", + "region": "USA", + "universe": "ILLIQUID_MINVOL1M" + }, + { + "id": "earnings7", + "region": "USA", + "universe": "ILLIQUID_MINVOL1M" + }, + { + "id": "earnings6", + "region": "GLB", + "universe": "MINVOL1M" + }, + { + "id": "earnings7", + "region": "GLB", + "universe": "MINVOL1M" + }, + { + "id": "earnings6", + "region": "EUR", + "universe": "ILLIQUID_MINVOL1M" + }, + { + "id": "earnings6", + "region": "ASI", + "universe": "ILLIQUID_MINVOL1M" + }, + { + "id": "earnings6", + "region": "ASI", + "universe": "MINVOL1M" + } +] \ No newline at end of file diff --git a/category_files/fundamental.json b/category_files/fundamental.json new file mode 100644 index 0000000..71e20ce --- /dev/null +++ b/category_files/fundamental.json @@ -0,0 +1,337 @@ +[ + { + "id": "fundamental1", + "region": "USA", + "universe": "ILLIQUID_MINVOL1M" + }, + { + "id": "fundamental110", + "region": "USA", + "universe": "ILLIQUID_MINVOL1M" + }, + { + "id": "fundamental13", + "region": "USA", + "universe": "ILLIQUID_MINVOL1M" + }, + { + "id": "fundamental14", + "region": "USA", + "universe": "ILLIQUID_MINVOL1M" + }, + { + "id": "fundamental17", + "region": "USA", + "universe": "ILLIQUID_MINVOL1M" + }, + { + "id": "fundamental2", + "region": "USA", + "universe": "ILLIQUID_MINVOL1M" + }, + { + "id": "fundamental22", + "region": "USA", + "universe": "ILLIQUID_MINVOL1M" + }, + { + "id": "fundamental23", + "region": "USA", + "universe": "ILLIQUID_MINVOL1M" + }, + { + "id": "fundamental25", + "region": "USA", + "universe": "ILLIQUID_MINVOL1M" + }, + { + "id": "fundamental28", + "region": "USA", + "universe": "ILLIQUID_MINVOL1M" + }, + { + "id": "fundamental3", + "region": "USA", + "universe": "ILLIQUID_MINVOL1M" + }, + { + "id": "fundamental31", + "region": "USA", + "universe": "ILLIQUID_MINVOL1M" + }, + { + "id": "fundamental45", + "region": "USA", + "universe": "ILLIQUID_MINVOL1M" + }, + { + "id": "fundamental6", + "region": "USA", + "universe": "ILLIQUID_MINVOL1M" + }, + { + "id": "fundamental69", + "region": "USA", + "universe": "ILLIQUID_MINVOL1M" + }, + { + "id": "fundamental7", + "region": "USA", + "universe": "ILLIQUID_MINVOL1M" + }, + { + "id": "fundamental72", + "region": "USA", + "universe": "ILLIQUID_MINVOL1M" + }, + { + "id": "fundamental89", + "region": "USA", + "universe": "ILLIQUID_MINVOL1M" + }, + { + "id": "fundamental90", + "region": "USA", + "universe": "ILLIQUID_MINVOL1M" + }, + { + "id": "other395", + "region": "USA", + "universe": "ILLIQUID_MINVOL1M" + }, + { + "id": "other401", + "region": "USA", + "universe": "ILLIQUID_MINVOL1M" + }, + { + "id": "fundamental1", + "region": "GLB", + "universe": "MINVOL1M" + }, + { + "id": "fundamental13", + "region": "GLB", + "universe": "MINVOL1M" + }, + { + "id": "fundamental17", + "region": "GLB", + "universe": "MINVOL1M" + }, + { + "id": "fundamental22", + "region": "GLB", + "universe": "MINVOL1M" + }, + { + "id": "fundamental23", + "region": "GLB", + "universe": "MINVOL1M" + }, + { + "id": "fundamental25", + "region": "GLB", + "universe": "MINVOL1M" + }, + { + "id": "fundamental28", + "region": "GLB", + "universe": "MINVOL1M" + }, + { + "id": "fundamental31", + "region": "GLB", + "universe": "MINVOL1M" + }, + { + "id": "fundamental44", + "region": "GLB", + "universe": "MINVOL1M" + }, + { + "id": "fundamental6", + "region": "GLB", + "universe": "MINVOL1M" + }, + { + "id": "fundamental72", + "region": "GLB", + "universe": "MINVOL1M" + }, + { + "id": "fundamental86", + "region": "GLB", + "universe": "MINVOL1M" + }, + { + "id": "model56", + "region": "GLB", + "universe": "MINVOL1M" + }, + { + "id": "fundamental1", + "region": "EUR", + "universe": "ILLIQUID_MINVOL1M" + }, + { + "id": "fundamental17", + "region": "EUR", + "universe": "ILLIQUID_MINVOL1M" + }, + { + "id": "fundamental22", + "region": "EUR", + "universe": "ILLIQUID_MINVOL1M" + }, + { + "id": "fundamental23", + "region": "EUR", + "universe": "ILLIQUID_MINVOL1M" + }, + { + "id": "fundamental25", + "region": "EUR", + "universe": "ILLIQUID_MINVOL1M" + }, + { + "id": "fundamental28", + "region": "EUR", + "universe": "ILLIQUID_MINVOL1M" + }, + { + "id": "fundamental31", + "region": "EUR", + "universe": "ILLIQUID_MINVOL1M" + }, + { + "id": "fundamental44", + "region": "EUR", + "universe": "ILLIQUID_MINVOL1M" + }, + { + "id": "fundamental6", + "region": "EUR", + "universe": "ILLIQUID_MINVOL1M" + }, + { + "id": "fundamental72", + "region": "EUR", + "universe": "ILLIQUID_MINVOL1M" + }, + { + "id": "fundamental89", + "region": "EUR", + "universe": "ILLIQUID_MINVOL1M" + }, + { + "id": "other395", + "region": "EUR", + "universe": "ILLIQUID_MINVOL1M" + }, + { + "id": "fundamental1", + "region": "ASI", + "universe": "ILLIQUID_MINVOL1M" + }, + { + "id": "fundamental17", + "region": "ASI", + "universe": "ILLIQUID_MINVOL1M" + }, + { + "id": "fundamental21", + "region": "ASI", + "universe": "ILLIQUID_MINVOL1M" + }, + { + "id": "fundamental22", + "region": "ASI", + "universe": "ILLIQUID_MINVOL1M" + }, + { + "id": "fundamental23", + "region": "ASI", + "universe": "ILLIQUID_MINVOL1M" + }, + { + "id": "fundamental28", + "region": "ASI", + "universe": "ILLIQUID_MINVOL1M" + }, + { + "id": "fundamental4", + "region": "ASI", + "universe": "ILLIQUID_MINVOL1M" + }, + { + "id": "fundamental6", + "region": "ASI", + "universe": "ILLIQUID_MINVOL1M" + }, + { + "id": "fundamental94", + "region": "ASI", + "universe": "ILLIQUID_MINVOL1M" + }, + { + "id": "other466", + "region": "ASI", + "universe": "ILLIQUID_MINVOL1M" + }, + { + "id": "fundamental1", + "region": "ASI", + "universe": "MINVOL1M" + }, + { + "id": "fundamental109", + "region": "ASI", + "universe": "MINVOL1M" + }, + { + "id": "fundamental17", + "region": "ASI", + "universe": "MINVOL1M" + }, + { + "id": "fundamental21", + "region": "ASI", + "universe": "MINVOL1M" + }, + { + "id": "fundamental22", + "region": "ASI", + "universe": "MINVOL1M" + }, + { + "id": "fundamental23", + "region": "ASI", + "universe": "MINVOL1M" + }, + { + "id": "fundamental28", + "region": "ASI", + "universe": "MINVOL1M" + }, + { + "id": "fundamental4", + "region": "ASI", + "universe": "MINVOL1M" + }, + { + "id": "fundamental6", + "region": "ASI", + "universe": "MINVOL1M" + }, + { + "id": "fundamental94", + "region": "ASI", + "universe": "MINVOL1M" + }, + { + "id": "other466", + "region": "ASI", + "universe": "MINVOL1M" + } +] \ No newline at end of file diff --git a/get_category.py b/get_category.py new file mode 100644 index 0000000..b8e3762 --- /dev/null +++ b/get_category.py @@ -0,0 +1,111 @@ +# -*- coding: utf-8 -*- +import os +import json +import random +import time +import httpx +from httpx import BasicAuth + + +class CategoryDownloader: + def __init__(self): + self.base_api_url = 'https://api.worldquantbrain.com' + self.client = self.login() + + def login(self): + """登录并返回客户端实例""" + username, password = "jack0210_@hotmail.com", "!QAZ2wsx+0913" + client = httpx.Client(auth=BasicAuth(username, password)) + + try: + response = client.post(f'{self.base_api_url}/authentication') + print(f"登录状态: {response.status_code}") + + if response.status_code in [200, 201]: + print("登录成功!") + return client + else: + print(f"登录失败: {response.json()}") + return None + except Exception as e: + print(f"登录过程中出现错误: {e}") + return None + + def fetch_category_data(self, category, delay, instrumentType, region_list, universe_list): + """获取分类数据并保存到JSON文件""" + results = [] + + # 创建category_files文件夹 + output_dir = "category_files" + if not os.path.exists(output_dir): + os.makedirs(output_dir) + print(f"已创建文件夹: {output_dir}") + + if self.client: + for region in region_list: + for universe in universe_list: + url = f'https://api.worldquantbrain.com/data-sets?category={category}&delay={delay}&instrumentType={instrumentType}&limit=50&offset=0®ion={region}&universe={universe}' + print(f"请求URL: {url}") + + try: + response = self.client.get(url) + + if response.status_code == 200: + data = response.json() + + if data.get('count', 0) > 0: + for item in data.get('results', []): + result_item = { + 'id': item.get('id', ''), + 'region': item.get('region', ''), + 'universe': item.get('universe', '') + } + results.append(result_item) + else: + print(f"请求失败: {response.status_code}") + + except Exception as e: + print(f"请求过程中出现错误: {e}") + + time.sleep(random.uniform(5, 8)) + + # 保存到JSON文件 + filename = os.path.join(output_dir, f"{category}.json") + with open(filename, 'w', encoding='utf-8') as f: + json.dump(results, f, indent=2, ensure_ascii=False) + + print(f"数据已保存到: {filename}") + print(f"总共找到 {len(results)} 条记录") + + return results + + +if __name__ == "__main__": + downloader = CategoryDownloader() + + if downloader.client: + category = 'fundamental' + delay = '1' + instrumentType = 'EQUITY' + + region_list = [ + 'USA', + 'GLB', + 'EUR', + 'ASI', + 'CHN', + 'KOR', + 'TWN', + 'IND' + ] + universe_list = [ + # 'TOP3000', + # 'TOP1000', + # 'TOP500', + # 'TOP200', + # 'TOPSP500', + 'ILLIQUID_MINVOL1M', + 'MINVOL1M' + ] + + downloader.fetch_category_data(category, delay, instrumentType, region_list, universe_list) diff --git a/main.py b/main.py index 146a258..0b01979 100644 --- a/main.py +++ b/main.py @@ -1,3 +1,4 @@ +# -*- coding: utf-8 -*- import os import json import random @@ -45,7 +46,7 @@ class DataSetDownloader: print(f"results 长度: {len(data.get('results', []))}") print(f"响应键: {list(data.keys())}") - def _build_params(self, data_set_id, offset=0, limit=50): + def _build_params(self, data_set_id, region, universe, offset=0, limit=50): """构建请求参数""" return { 'dataset.id': data_set_id, @@ -53,8 +54,8 @@ class DataSetDownloader: 'instrumentType': 'EQUITY', 'limit': limit, 'offset': offset, - 'region': 'USA', - 'universe': 'TOP3000' + 'region': region, + 'universe': universe } def _process_item(self, item): @@ -76,7 +77,8 @@ class DataSetDownloader: """批量处理数据""" return [self._process_item(item) for item in raw_data] - def download_data_set(self, endpoint, data_set_id): + def download_data_set(self, data_set_id, region, universe): + endpoint = 'data-fields' """下载数据集""" # 检查登录状态 if not self.client: @@ -88,7 +90,7 @@ class DataSetDownloader: # 获取数据总数 url = f"{self.base_api_url}/{endpoint}" - params = self._build_params(data_set_id, limit=1) + params = self._build_params(data_set_id, region, universe, limit=1) response = self.client.get(url, params=params) data = response.json() @@ -108,32 +110,51 @@ class DataSetDownloader: for offset in range(0, total_count, limit): time.sleep(random.uniform(1.0, 1.5)) - params = self._build_params(data_set_id, offset, limit) + params = self._build_params(data_set_id, region, universe, offset, limit) # 修正参数 print(f"📥 下载进度: {offset}/{total_count} ({offset / total_count * 100:.1f}%)") - try: - response = self.client.get(url, params=params) + results = [] - if response.status_code == 200: - data = response.json() - results = data.get('results', []) + retry = 3 + while retry > 0: + try: + response = self.client.get(url, params=params) - print(f"✅ 本页获取到 {len(results)} 条记录") - all_data.extend(results) + if response.status_code == 200: + data = response.json() + results = data.get('results', []) - if len(results) < limit: - print("🎯 到达数据末尾") - break - else: - print(f"❌ 请求失败: {response.status_code}") - break + print(f"✅ 本页获取到 {len(results)} 条记录") + all_data.extend(results) - time.sleep(random.uniform(1, 2)) + # 成功时退出重试循环 + break - except Exception as e: - print(f"❌ 下载过程中出错: {e}") + else: + print(f"❌ 请求失败: {response.status_code}") + retry -= 1 + if retry > 0: + print(f"🔄 重试中... ({retry}次剩余)") + time.sleep(random.uniform(2, 3)) + + except Exception as e: + print(f"❌ 下载过程中出错: {e}") + retry -= 1 + if retry > 0: + print(f"🔄 重试中... ({retry}次剩余)") + time.sleep(random.uniform(2, 3)) + + # 如果重试用完仍失败，跳过当前offset继续下一个 + if retry == 0: + print(f"⚠️ 跳过 offset {offset}") + continue + + if len(results) < limit: + print("🎯 到达数据末尾") break + time.sleep(random.uniform(4, 6)) + # 处理数据 print("🔄 处理数据中...") processed_data = self._process_data(all_data) @@ -143,7 +164,7 @@ class DataSetDownloader: os.makedirs(output_dir, exist_ok=True) # 保存为CSV - output_file = os.path.join(output_dir, f"{data_set_id}_{endpoint}.csv") + output_file = os.path.join(output_dir, f"{data_set_id}_{region.lower()}_{universe.lower()}.csv") if processed_data: fieldnames = list(processed_data[0].keys()) @@ -160,8 +181,21 @@ if __name__ == "__main__": downloader = DataSetDownloader() if downloader.client: - endpoint = 'data-fields' - data_set_id = 'socialmedia8' - downloader.download_data_set(endpoint, data_set_id) + plan_to_download = [ + { + 'id': 'analyst10', + 'region': 'USA', + 'universe': 'TOP3000' + }, + { + 'id': 'analyst10', + 'region': 'GLB', + 'universe': 'TOP3000' + } + ] + + for item in plan_to_download: + downloader.download_data_set(item['id'], item['region'], item['universe']) + time.sleep(random.uniform(20, 30)) else: print("❌ 登录失败，无法下载数据") \ No newline at end of file diff --git a/reference_fields/all_data_combined.csv b/reference_fields_bak/all_data_combined.csv similarity index 100% rename from reference_fields/all_data_combined.csv rename to reference_fields_bak/all_data_combined.csv diff --git a/reference_fields/analyst4_data-fields.csv b/reference_fields_bak/analyst4_data-fields.csv similarity index 100% rename from reference_fields/analyst4_data-fields.csv rename to reference_fields_bak/analyst4_data-fields.csv diff --git a/reference_fields/fundamental2_data-fields.csv b/reference_fields_bak/fundamental2_data-fields.csv similarity index 100% rename from reference_fields/fundamental2_data-fields.csv rename to reference_fields_bak/fundamental2_data-fields.csv diff --git a/reference_fields/fundamental6_data-fields.csv b/reference_fields_bak/fundamental6_data-fields.csv similarity index 100% rename from reference_fields/fundamental6_data-fields.csv rename to reference_fields_bak/fundamental6_data-fields.csv diff --git a/reference_fields/model16_data-fields.csv b/reference_fields_bak/model16_data-fields.csv similarity index 100% rename from reference_fields/model16_data-fields.csv rename to reference_fields_bak/model16_data-fields.csv diff --git a/reference_fields/model51_data-fields.csv b/reference_fields_bak/model51_data-fields.csv similarity index 100% rename from reference_fields/model51_data-fields.csv rename to reference_fields_bak/model51_data-fields.csv diff --git a/reference_fields/news12_data-fields.csv b/reference_fields_bak/news12_data-fields.csv similarity index 100% rename from reference_fields/news12_data-fields.csv rename to reference_fields_bak/news12_data-fields.csv diff --git a/reference_fields/news18_data-fields.csv b/reference_fields_bak/news18_data-fields.csv similarity index 100% rename from reference_fields/news18_data-fields.csv rename to reference_fields_bak/news18_data-fields.csv diff --git a/reference_fields/option8_data-fields.csv b/reference_fields_bak/option8_data-fields.csv similarity index 100% rename from reference_fields/option8_data-fields.csv rename to reference_fields_bak/option8_data-fields.csv diff --git a/reference_fields/option9_data-fields.csv b/reference_fields_bak/option9_data-fields.csv similarity index 100% rename from reference_fields/option9_data-fields.csv rename to reference_fields_bak/option9_data-fields.csv diff --git a/reference_fields/pv13_data-fields.csv b/reference_fields_bak/pv13_data-fields.csv similarity index 100% rename from reference_fields/pv13_data-fields.csv rename to reference_fields_bak/pv13_data-fields.csv diff --git a/reference_fields/pv1_data-fields.csv b/reference_fields_bak/pv1_data-fields.csv similarity index 100% rename from reference_fields/pv1_data-fields.csv rename to reference_fields_bak/pv1_data-fields.csv diff --git a/reference_fields/socialmedia12_data-fields.csv b/reference_fields_bak/socialmedia12_data-fields.csv similarity index 100% rename from reference_fields/socialmedia12_data-fields.csv rename to reference_fields_bak/socialmedia12_data-fields.csv diff --git a/reference_fields/socialmedia8_data-fields.csv b/reference_fields_bak/socialmedia8_data-fields.csv similarity index 100% rename from reference_fields/socialmedia8_data-fields.csv rename to reference_fields_bak/socialmedia8_data-fields.csv diff --git a/reference_fields/univ1_data-fields.csv b/reference_fields_bak/univ1_data-fields.csv similarity index 100% rename from reference_fields/univ1_data-fields.csv rename to reference_fields_bak/univ1_data-fields.csv