Jack 3 weeks ago
parent d1001731b5
commit 8cb03f0a4e
  1. 852
      category_files/analyst.json
  2. 7
      category_files/broker.json
  3. 37
      category_files/earnings.json
  4. 337
      category_files/fundamental.json
  5. 111
      get_category.py
  6. 86
      main.py
  7. 0
      reference_fields_bak/all_data_combined.csv
  8. 0
      reference_fields_bak/analyst4_data-fields.csv
  9. 0
      reference_fields_bak/fundamental2_data-fields.csv
  10. 0
      reference_fields_bak/fundamental6_data-fields.csv
  11. 0
      reference_fields_bak/model16_data-fields.csv
  12. 0
      reference_fields_bak/model51_data-fields.csv
  13. 0
      reference_fields_bak/news12_data-fields.csv
  14. 0
      reference_fields_bak/news18_data-fields.csv
  15. 0
      reference_fields_bak/option8_data-fields.csv
  16. 0
      reference_fields_bak/option9_data-fields.csv
  17. 0
      reference_fields_bak/pv13_data-fields.csv
  18. 0
      reference_fields_bak/pv1_data-fields.csv
  19. 0
      reference_fields_bak/socialmedia12_data-fields.csv
  20. 0
      reference_fields_bak/socialmedia8_data-fields.csv
  21. 0
      reference_fields_bak/univ1_data-fields.csv

@ -0,0 +1,852 @@
[
{
"id": "analyst10",
"region": "USA",
"universe": "TOP3000"
},
{
"id": "analyst11",
"region": "USA",
"universe": "TOP3000"
},
{
"id": "analyst14",
"region": "USA",
"universe": "TOP3000"
},
{
"id": "analyst15",
"region": "USA",
"universe": "TOP3000"
},
{
"id": "analyst16",
"region": "USA",
"universe": "TOP3000"
},
{
"id": "analyst39",
"region": "USA",
"universe": "TOP3000"
},
{
"id": "analyst4",
"region": "USA",
"universe": "TOP3000"
},
{
"id": "analyst40",
"region": "USA",
"universe": "TOP3000"
},
{
"id": "analyst44",
"region": "USA",
"universe": "TOP3000"
},
{
"id": "analyst45",
"region": "USA",
"universe": "TOP3000"
},
{
"id": "analyst49",
"region": "USA",
"universe": "TOP3000"
},
{
"id": "analyst52",
"region": "USA",
"universe": "TOP3000"
},
{
"id": "analyst69",
"region": "USA",
"universe": "TOP3000"
},
{
"id": "analyst7",
"region": "USA",
"universe": "TOP3000"
},
{
"id": "analyst82",
"region": "USA",
"universe": "TOP3000"
},
{
"id": "analyst83",
"region": "USA",
"universe": "TOP3000"
},
{
"id": "model211",
"region": "USA",
"universe": "TOP3000"
},
{
"id": "model52",
"region": "USA",
"universe": "TOP3000"
},
{
"id": "news87",
"region": "USA",
"universe": "TOP3000"
},
{
"id": "analyst10",
"region": "USA",
"universe": "TOP1000"
},
{
"id": "analyst11",
"region": "USA",
"universe": "TOP1000"
},
{
"id": "analyst14",
"region": "USA",
"universe": "TOP1000"
},
{
"id": "analyst15",
"region": "USA",
"universe": "TOP1000"
},
{
"id": "analyst16",
"region": "USA",
"universe": "TOP1000"
},
{
"id": "analyst39",
"region": "USA",
"universe": "TOP1000"
},
{
"id": "analyst4",
"region": "USA",
"universe": "TOP1000"
},
{
"id": "analyst40",
"region": "USA",
"universe": "TOP1000"
},
{
"id": "analyst44",
"region": "USA",
"universe": "TOP1000"
},
{
"id": "analyst45",
"region": "USA",
"universe": "TOP1000"
},
{
"id": "analyst49",
"region": "USA",
"universe": "TOP1000"
},
{
"id": "analyst52",
"region": "USA",
"universe": "TOP1000"
},
{
"id": "analyst69",
"region": "USA",
"universe": "TOP1000"
},
{
"id": "analyst7",
"region": "USA",
"universe": "TOP1000"
},
{
"id": "analyst82",
"region": "USA",
"universe": "TOP1000"
},
{
"id": "analyst83",
"region": "USA",
"universe": "TOP1000"
},
{
"id": "model211",
"region": "USA",
"universe": "TOP1000"
},
{
"id": "model52",
"region": "USA",
"universe": "TOP1000"
},
{
"id": "news87",
"region": "USA",
"universe": "TOP1000"
},
{
"id": "analyst10",
"region": "USA",
"universe": "TOP500"
},
{
"id": "analyst11",
"region": "USA",
"universe": "TOP500"
},
{
"id": "analyst14",
"region": "USA",
"universe": "TOP500"
},
{
"id": "analyst15",
"region": "USA",
"universe": "TOP500"
},
{
"id": "analyst16",
"region": "USA",
"universe": "TOP500"
},
{
"id": "analyst39",
"region": "USA",
"universe": "TOP500"
},
{
"id": "analyst4",
"region": "USA",
"universe": "TOP500"
},
{
"id": "analyst40",
"region": "USA",
"universe": "TOP500"
},
{
"id": "analyst44",
"region": "USA",
"universe": "TOP500"
},
{
"id": "analyst45",
"region": "USA",
"universe": "TOP500"
},
{
"id": "analyst49",
"region": "USA",
"universe": "TOP500"
},
{
"id": "analyst52",
"region": "USA",
"universe": "TOP500"
},
{
"id": "analyst69",
"region": "USA",
"universe": "TOP500"
},
{
"id": "analyst7",
"region": "USA",
"universe": "TOP500"
},
{
"id": "analyst82",
"region": "USA",
"universe": "TOP500"
},
{
"id": "analyst83",
"region": "USA",
"universe": "TOP500"
},
{
"id": "model211",
"region": "USA",
"universe": "TOP500"
},
{
"id": "model52",
"region": "USA",
"universe": "TOP500"
},
{
"id": "news87",
"region": "USA",
"universe": "TOP500"
},
{
"id": "analyst10",
"region": "USA",
"universe": "TOP200"
},
{
"id": "analyst11",
"region": "USA",
"universe": "TOP200"
},
{
"id": "analyst14",
"region": "USA",
"universe": "TOP200"
},
{
"id": "analyst15",
"region": "USA",
"universe": "TOP200"
},
{
"id": "analyst16",
"region": "USA",
"universe": "TOP200"
},
{
"id": "analyst39",
"region": "USA",
"universe": "TOP200"
},
{
"id": "analyst4",
"region": "USA",
"universe": "TOP200"
},
{
"id": "analyst40",
"region": "USA",
"universe": "TOP200"
},
{
"id": "analyst44",
"region": "USA",
"universe": "TOP200"
},
{
"id": "analyst45",
"region": "USA",
"universe": "TOP200"
},
{
"id": "analyst49",
"region": "USA",
"universe": "TOP200"
},
{
"id": "analyst52",
"region": "USA",
"universe": "TOP200"
},
{
"id": "analyst69",
"region": "USA",
"universe": "TOP200"
},
{
"id": "analyst7",
"region": "USA",
"universe": "TOP200"
},
{
"id": "analyst82",
"region": "USA",
"universe": "TOP200"
},
{
"id": "analyst83",
"region": "USA",
"universe": "TOP200"
},
{
"id": "model211",
"region": "USA",
"universe": "TOP200"
},
{
"id": "model52",
"region": "USA",
"universe": "TOP200"
},
{
"id": "news87",
"region": "USA",
"universe": "TOP200"
},
{
"id": "analyst10",
"region": "USA",
"universe": "ILLIQUID_MINVOL1M"
},
{
"id": "analyst11",
"region": "USA",
"universe": "ILLIQUID_MINVOL1M"
},
{
"id": "analyst14",
"region": "USA",
"universe": "ILLIQUID_MINVOL1M"
},
{
"id": "analyst15",
"region": "USA",
"universe": "ILLIQUID_MINVOL1M"
},
{
"id": "analyst16",
"region": "USA",
"universe": "ILLIQUID_MINVOL1M"
},
{
"id": "analyst39",
"region": "USA",
"universe": "ILLIQUID_MINVOL1M"
},
{
"id": "analyst4",
"region": "USA",
"universe": "ILLIQUID_MINVOL1M"
},
{
"id": "analyst44",
"region": "USA",
"universe": "ILLIQUID_MINVOL1M"
},
{
"id": "analyst45",
"region": "USA",
"universe": "ILLIQUID_MINVOL1M"
},
{
"id": "analyst49",
"region": "USA",
"universe": "ILLIQUID_MINVOL1M"
},
{
"id": "analyst52",
"region": "USA",
"universe": "ILLIQUID_MINVOL1M"
},
{
"id": "analyst69",
"region": "USA",
"universe": "ILLIQUID_MINVOL1M"
},
{
"id": "analyst7",
"region": "USA",
"universe": "ILLIQUID_MINVOL1M"
},
{
"id": "analyst82",
"region": "USA",
"universe": "ILLIQUID_MINVOL1M"
},
{
"id": "analyst83",
"region": "USA",
"universe": "ILLIQUID_MINVOL1M"
},
{
"id": "model211",
"region": "USA",
"universe": "ILLIQUID_MINVOL1M"
},
{
"id": "news87",
"region": "USA",
"universe": "ILLIQUID_MINVOL1M"
},
{
"id": "analyst10",
"region": "USA",
"universe": "TOPSP500"
},
{
"id": "analyst11",
"region": "USA",
"universe": "TOPSP500"
},
{
"id": "analyst14",
"region": "USA",
"universe": "TOPSP500"
},
{
"id": "analyst15",
"region": "USA",
"universe": "TOPSP500"
},
{
"id": "analyst16",
"region": "USA",
"universe": "TOPSP500"
},
{
"id": "analyst39",
"region": "USA",
"universe": "TOPSP500"
},
{
"id": "analyst4",
"region": "USA",
"universe": "TOPSP500"
},
{
"id": "analyst40",
"region": "USA",
"universe": "TOPSP500"
},
{
"id": "analyst44",
"region": "USA",
"universe": "TOPSP500"
},
{
"id": "analyst45",
"region": "USA",
"universe": "TOPSP500"
},
{
"id": "analyst49",
"region": "USA",
"universe": "TOPSP500"
},
{
"id": "analyst52",
"region": "USA",
"universe": "TOPSP500"
},
{
"id": "analyst69",
"region": "USA",
"universe": "TOPSP500"
},
{
"id": "analyst7",
"region": "USA",
"universe": "TOPSP500"
},
{
"id": "analyst83",
"region": "USA",
"universe": "TOPSP500"
},
{
"id": "model211",
"region": "USA",
"universe": "TOPSP500"
},
{
"id": "model52",
"region": "USA",
"universe": "TOPSP500"
},
{
"id": "news87",
"region": "USA",
"universe": "TOPSP500"
},
{
"id": "analyst10",
"region": "GLB",
"universe": "TOP3000"
},
{
"id": "analyst11",
"region": "GLB",
"universe": "TOP3000"
},
{
"id": "analyst14",
"region": "GLB",
"universe": "TOP3000"
},
{
"id": "analyst15",
"region": "GLB",
"universe": "TOP3000"
},
{
"id": "analyst39",
"region": "GLB",
"universe": "TOP3000"
},
{
"id": "analyst44",
"region": "GLB",
"universe": "TOP3000"
},
{
"id": "analyst45",
"region": "GLB",
"universe": "TOP3000"
},
{
"id": "analyst46",
"region": "GLB",
"universe": "TOP3000"
},
{
"id": "analyst69",
"region": "GLB",
"universe": "TOP3000"
},
{
"id": "analyst10",
"region": "EUR",
"universe": "ILLIQUID_MINVOL1M"
},
{
"id": "analyst11",
"region": "EUR",
"universe": "ILLIQUID_MINVOL1M"
},
{
"id": "analyst14",
"region": "EUR",
"universe": "ILLIQUID_MINVOL1M"
},
{
"id": "analyst15",
"region": "EUR",
"universe": "ILLIQUID_MINVOL1M"
},
{
"id": "analyst4",
"region": "EUR",
"universe": "ILLIQUID_MINVOL1M"
},
{
"id": "analyst44",
"region": "EUR",
"universe": "ILLIQUID_MINVOL1M"
},
{
"id": "analyst45",
"region": "EUR",
"universe": "ILLIQUID_MINVOL1M"
},
{
"id": "analyst46",
"region": "EUR",
"universe": "ILLIQUID_MINVOL1M"
},
{
"id": "analyst48",
"region": "EUR",
"universe": "ILLIQUID_MINVOL1M"
},
{
"id": "analyst69",
"region": "EUR",
"universe": "ILLIQUID_MINVOL1M"
},
{
"id": "analyst7",
"region": "EUR",
"universe": "ILLIQUID_MINVOL1M"
},
{
"id": "analyst84",
"region": "EUR",
"universe": "ILLIQUID_MINVOL1M"
},
{
"id": "sentiment12",
"region": "EUR",
"universe": "ILLIQUID_MINVOL1M"
},
{
"id": "analyst11",
"region": "ASI",
"universe": "ILLIQUID_MINVOL1M"
},
{
"id": "analyst14",
"region": "ASI",
"universe": "ILLIQUID_MINVOL1M"
},
{
"id": "analyst15",
"region": "ASI",
"universe": "ILLIQUID_MINVOL1M"
},
{
"id": "analyst16",
"region": "ASI",
"universe": "ILLIQUID_MINVOL1M"
},
{
"id": "analyst4",
"region": "ASI",
"universe": "ILLIQUID_MINVOL1M"
},
{
"id": "analyst45",
"region": "ASI",
"universe": "ILLIQUID_MINVOL1M"
},
{
"id": "analyst48",
"region": "ASI",
"universe": "ILLIQUID_MINVOL1M"
},
{
"id": "analyst69",
"region": "ASI",
"universe": "ILLIQUID_MINVOL1M"
},
{
"id": "analyst10",
"region": "TWN",
"universe": "TOP500"
},
{
"id": "analyst11",
"region": "TWN",
"universe": "TOP500"
},
{
"id": "analyst14",
"region": "TWN",
"universe": "TOP500"
},
{
"id": "analyst15",
"region": "TWN",
"universe": "TOP500"
},
{
"id": "analyst16",
"region": "TWN",
"universe": "TOP500"
},
{
"id": "analyst21",
"region": "TWN",
"universe": "TOP500"
},
{
"id": "analyst39",
"region": "TWN",
"universe": "TOP500"
},
{
"id": "analyst4",
"region": "TWN",
"universe": "TOP500"
},
{
"id": "analyst44",
"region": "TWN",
"universe": "TOP500"
},
{
"id": "analyst45",
"region": "TWN",
"universe": "TOP500"
},
{
"id": "analyst46",
"region": "TWN",
"universe": "TOP500"
},
{
"id": "analyst48",
"region": "TWN",
"universe": "TOP500"
},
{
"id": "analyst81",
"region": "TWN",
"universe": "TOP500"
},
{
"id": "analyst9",
"region": "TWN",
"universe": "TOP500"
},
{
"id": "analyst92",
"region": "TWN",
"universe": "TOP500"
},
{
"id": "analyst94",
"region": "TWN",
"universe": "TOP500"
},
{
"id": "model52",
"region": "TWN",
"universe": "TOP500"
},
{
"id": "other411",
"region": "TWN",
"universe": "TOP500"
},
{
"id": "other423",
"region": "TWN",
"universe": "TOP500"
},
{
"id": "analyst11",
"region": "IND",
"universe": "TOP500"
},
{
"id": "analyst39",
"region": "IND",
"universe": "TOP500"
},
{
"id": "analyst4",
"region": "IND",
"universe": "TOP500"
},
{
"id": "analyst44",
"region": "IND",
"universe": "TOP500"
},
{
"id": "analyst45",
"region": "IND",
"universe": "TOP500"
},
{
"id": "analyst46",
"region": "IND",
"universe": "TOP500"
},
{
"id": "analyst48",
"region": "IND",
"universe": "TOP500"
},
{
"id": "analyst81",
"region": "IND",
"universe": "TOP500"
},
{
"id": "analyst83",
"region": "IND",
"universe": "TOP500"
},
{
"id": "analyst9",
"region": "IND",
"universe": "TOP500"
}
]

@ -0,0 +1,7 @@
[
{
"id": "broker1",
"region": "ASI",
"universe": "MINVOL1M"
}
]

@ -0,0 +1,37 @@
[
{
"id": "earnings6",
"region": "USA",
"universe": "ILLIQUID_MINVOL1M"
},
{
"id": "earnings7",
"region": "USA",
"universe": "ILLIQUID_MINVOL1M"
},
{
"id": "earnings6",
"region": "GLB",
"universe": "MINVOL1M"
},
{
"id": "earnings7",
"region": "GLB",
"universe": "MINVOL1M"
},
{
"id": "earnings6",
"region": "EUR",
"universe": "ILLIQUID_MINVOL1M"
},
{
"id": "earnings6",
"region": "ASI",
"universe": "ILLIQUID_MINVOL1M"
},
{
"id": "earnings6",
"region": "ASI",
"universe": "MINVOL1M"
}
]

@ -0,0 +1,337 @@
[
{
"id": "fundamental1",
"region": "USA",
"universe": "ILLIQUID_MINVOL1M"
},
{
"id": "fundamental110",
"region": "USA",
"universe": "ILLIQUID_MINVOL1M"
},
{
"id": "fundamental13",
"region": "USA",
"universe": "ILLIQUID_MINVOL1M"
},
{
"id": "fundamental14",
"region": "USA",
"universe": "ILLIQUID_MINVOL1M"
},
{
"id": "fundamental17",
"region": "USA",
"universe": "ILLIQUID_MINVOL1M"
},
{
"id": "fundamental2",
"region": "USA",
"universe": "ILLIQUID_MINVOL1M"
},
{
"id": "fundamental22",
"region": "USA",
"universe": "ILLIQUID_MINVOL1M"
},
{
"id": "fundamental23",
"region": "USA",
"universe": "ILLIQUID_MINVOL1M"
},
{
"id": "fundamental25",
"region": "USA",
"universe": "ILLIQUID_MINVOL1M"
},
{
"id": "fundamental28",
"region": "USA",
"universe": "ILLIQUID_MINVOL1M"
},
{
"id": "fundamental3",
"region": "USA",
"universe": "ILLIQUID_MINVOL1M"
},
{
"id": "fundamental31",
"region": "USA",
"universe": "ILLIQUID_MINVOL1M"
},
{
"id": "fundamental45",
"region": "USA",
"universe": "ILLIQUID_MINVOL1M"
},
{
"id": "fundamental6",
"region": "USA",
"universe": "ILLIQUID_MINVOL1M"
},
{
"id": "fundamental69",
"region": "USA",
"universe": "ILLIQUID_MINVOL1M"
},
{
"id": "fundamental7",
"region": "USA",
"universe": "ILLIQUID_MINVOL1M"
},
{
"id": "fundamental72",
"region": "USA",
"universe": "ILLIQUID_MINVOL1M"
},
{
"id": "fundamental89",
"region": "USA",
"universe": "ILLIQUID_MINVOL1M"
},
{
"id": "fundamental90",
"region": "USA",
"universe": "ILLIQUID_MINVOL1M"
},
{
"id": "other395",
"region": "USA",
"universe": "ILLIQUID_MINVOL1M"
},
{
"id": "other401",
"region": "USA",
"universe": "ILLIQUID_MINVOL1M"
},
{
"id": "fundamental1",
"region": "GLB",
"universe": "MINVOL1M"
},
{
"id": "fundamental13",
"region": "GLB",
"universe": "MINVOL1M"
},
{
"id": "fundamental17",
"region": "GLB",
"universe": "MINVOL1M"
},
{
"id": "fundamental22",
"region": "GLB",
"universe": "MINVOL1M"
},
{
"id": "fundamental23",
"region": "GLB",
"universe": "MINVOL1M"
},
{
"id": "fundamental25",
"region": "GLB",
"universe": "MINVOL1M"
},
{
"id": "fundamental28",
"region": "GLB",
"universe": "MINVOL1M"
},
{
"id": "fundamental31",
"region": "GLB",
"universe": "MINVOL1M"
},
{
"id": "fundamental44",
"region": "GLB",
"universe": "MINVOL1M"
},
{
"id": "fundamental6",
"region": "GLB",
"universe": "MINVOL1M"
},
{
"id": "fundamental72",
"region": "GLB",
"universe": "MINVOL1M"
},
{
"id": "fundamental86",
"region": "GLB",
"universe": "MINVOL1M"
},
{
"id": "model56",
"region": "GLB",
"universe": "MINVOL1M"
},
{
"id": "fundamental1",
"region": "EUR",
"universe": "ILLIQUID_MINVOL1M"
},
{
"id": "fundamental17",
"region": "EUR",
"universe": "ILLIQUID_MINVOL1M"
},
{
"id": "fundamental22",
"region": "EUR",
"universe": "ILLIQUID_MINVOL1M"
},
{
"id": "fundamental23",
"region": "EUR",
"universe": "ILLIQUID_MINVOL1M"
},
{
"id": "fundamental25",
"region": "EUR",
"universe": "ILLIQUID_MINVOL1M"
},
{
"id": "fundamental28",
"region": "EUR",
"universe": "ILLIQUID_MINVOL1M"
},
{
"id": "fundamental31",
"region": "EUR",
"universe": "ILLIQUID_MINVOL1M"
},
{
"id": "fundamental44",
"region": "EUR",
"universe": "ILLIQUID_MINVOL1M"
},
{
"id": "fundamental6",
"region": "EUR",
"universe": "ILLIQUID_MINVOL1M"
},
{
"id": "fundamental72",
"region": "EUR",
"universe": "ILLIQUID_MINVOL1M"
},
{
"id": "fundamental89",
"region": "EUR",
"universe": "ILLIQUID_MINVOL1M"
},
{
"id": "other395",
"region": "EUR",
"universe": "ILLIQUID_MINVOL1M"
},
{
"id": "fundamental1",
"region": "ASI",
"universe": "ILLIQUID_MINVOL1M"
},
{
"id": "fundamental17",
"region": "ASI",
"universe": "ILLIQUID_MINVOL1M"
},
{
"id": "fundamental21",
"region": "ASI",
"universe": "ILLIQUID_MINVOL1M"
},
{
"id": "fundamental22",
"region": "ASI",
"universe": "ILLIQUID_MINVOL1M"
},
{
"id": "fundamental23",
"region": "ASI",
"universe": "ILLIQUID_MINVOL1M"
},
{
"id": "fundamental28",
"region": "ASI",
"universe": "ILLIQUID_MINVOL1M"
},
{
"id": "fundamental4",
"region": "ASI",
"universe": "ILLIQUID_MINVOL1M"
},
{
"id": "fundamental6",
"region": "ASI",
"universe": "ILLIQUID_MINVOL1M"
},
{
"id": "fundamental94",
"region": "ASI",
"universe": "ILLIQUID_MINVOL1M"
},
{
"id": "other466",
"region": "ASI",
"universe": "ILLIQUID_MINVOL1M"
},
{
"id": "fundamental1",
"region": "ASI",
"universe": "MINVOL1M"
},
{
"id": "fundamental109",
"region": "ASI",
"universe": "MINVOL1M"
},
{
"id": "fundamental17",
"region": "ASI",
"universe": "MINVOL1M"
},
{
"id": "fundamental21",
"region": "ASI",
"universe": "MINVOL1M"
},
{
"id": "fundamental22",
"region": "ASI",
"universe": "MINVOL1M"
},
{
"id": "fundamental23",
"region": "ASI",
"universe": "MINVOL1M"
},
{
"id": "fundamental28",
"region": "ASI",
"universe": "MINVOL1M"
},
{
"id": "fundamental4",
"region": "ASI",
"universe": "MINVOL1M"
},
{
"id": "fundamental6",
"region": "ASI",
"universe": "MINVOL1M"
},
{
"id": "fundamental94",
"region": "ASI",
"universe": "MINVOL1M"
},
{
"id": "other466",
"region": "ASI",
"universe": "MINVOL1M"
}
]

@ -0,0 +1,111 @@
# -*- coding: utf-8 -*-
import os
import json
import random
import time
import httpx
from httpx import BasicAuth
class CategoryDownloader:
def __init__(self):
self.base_api_url = 'https://api.worldquantbrain.com'
self.client = self.login()
def login(self):
"""登录并返回客户端实例"""
username, password = "jack0210_@hotmail.com", "!QAZ2wsx+0913"
client = httpx.Client(auth=BasicAuth(username, password))
try:
response = client.post(f'{self.base_api_url}/authentication')
print(f"登录状态: {response.status_code}")
if response.status_code in [200, 201]:
print("登录成功!")
return client
else:
print(f"登录失败: {response.json()}")
return None
except Exception as e:
print(f"登录过程中出现错误: {e}")
return None
def fetch_category_data(self, category, delay, instrumentType, region_list, universe_list):
"""获取分类数据并保存到JSON文件"""
results = []
# 创建category_files文件夹
output_dir = "category_files"
if not os.path.exists(output_dir):
os.makedirs(output_dir)
print(f"已创建文件夹: {output_dir}")
if self.client:
for region in region_list:
for universe in universe_list:
url = f'https://api.worldquantbrain.com/data-sets?category={category}&delay={delay}&instrumentType={instrumentType}&limit=50&offset=0&region={region}&universe={universe}'
print(f"请求URL: {url}")
try:
response = self.client.get(url)
if response.status_code == 200:
data = response.json()
if data.get('count', 0) > 0:
for item in data.get('results', []):
result_item = {
'id': item.get('id', ''),
'region': item.get('region', ''),
'universe': item.get('universe', '')
}
results.append(result_item)
else:
print(f"请求失败: {response.status_code}")
except Exception as e:
print(f"请求过程中出现错误: {e}")
time.sleep(random.uniform(5, 8))
# 保存到JSON文件
filename = os.path.join(output_dir, f"{category}.json")
with open(filename, 'w', encoding='utf-8') as f:
json.dump(results, f, indent=2, ensure_ascii=False)
print(f"数据已保存到: {filename}")
print(f"总共找到 {len(results)} 条记录")
return results
if __name__ == "__main__":
downloader = CategoryDownloader()
if downloader.client:
category = 'fundamental'
delay = '1'
instrumentType = 'EQUITY'
region_list = [
'USA',
'GLB',
'EUR',
'ASI',
'CHN',
'KOR',
'TWN',
'IND'
]
universe_list = [
# 'TOP3000',
# 'TOP1000',
# 'TOP500',
# 'TOP200',
# 'TOPSP500',
'ILLIQUID_MINVOL1M',
'MINVOL1M'
]
downloader.fetch_category_data(category, delay, instrumentType, region_list, universe_list)

@ -1,3 +1,4 @@
# -*- coding: utf-8 -*-
import os
import json
import random
@ -45,7 +46,7 @@ class DataSetDownloader:
print(f"results 长度: {len(data.get('results', []))}")
print(f"响应键: {list(data.keys())}")
def _build_params(self, data_set_id, offset=0, limit=50):
def _build_params(self, data_set_id, region, universe, offset=0, limit=50):
"""构建请求参数"""
return {
'dataset.id': data_set_id,
@ -53,8 +54,8 @@ class DataSetDownloader:
'instrumentType': 'EQUITY',
'limit': limit,
'offset': offset,
'region': 'USA',
'universe': 'TOP3000'
'region': region,
'universe': universe
}
def _process_item(self, item):
@ -76,7 +77,8 @@ class DataSetDownloader:
"""批量处理数据"""
return [self._process_item(item) for item in raw_data]
def download_data_set(self, endpoint, data_set_id):
def download_data_set(self, data_set_id, region, universe):
endpoint = 'data-fields'
"""下载数据集"""
# 检查登录状态
if not self.client:
@ -88,7 +90,7 @@ class DataSetDownloader:
# 获取数据总数
url = f"{self.base_api_url}/{endpoint}"
params = self._build_params(data_set_id, limit=1)
params = self._build_params(data_set_id, region, universe, limit=1)
response = self.client.get(url, params=params)
data = response.json()
@ -108,32 +110,51 @@ class DataSetDownloader:
for offset in range(0, total_count, limit):
time.sleep(random.uniform(1.0, 1.5))
params = self._build_params(data_set_id, offset, limit)
params = self._build_params(data_set_id, region, universe, offset, limit) # 修正参数
print(f"📥 下载进度: {offset}/{total_count} ({offset / total_count * 100:.1f}%)")
try:
response = self.client.get(url, params=params)
results = []
if response.status_code == 200:
data = response.json()
results = data.get('results', [])
retry = 3
while retry > 0:
try:
response = self.client.get(url, params=params)
print(f"✅ 本页获取到 {len(results)} 条记录")
all_data.extend(results)
if response.status_code == 200:
data = response.json()
results = data.get('results', [])
if len(results) < limit:
print("🎯 到达数据末尾")
break
else:
print(f"❌ 请求失败: {response.status_code}")
break
print(f"✅ 本页获取到 {len(results)} 条记录")
all_data.extend(results)
time.sleep(random.uniform(1, 2))
# 成功时退出重试循环
break
except Exception as e:
print(f"❌ 下载过程中出错: {e}")
else:
print(f"❌ 请求失败: {response.status_code}")
retry -= 1
if retry > 0:
print(f"🔄 重试中... ({retry}次剩余)")
time.sleep(random.uniform(2, 3))
except Exception as e:
print(f"❌ 下载过程中出错: {e}")
retry -= 1
if retry > 0:
print(f"🔄 重试中... ({retry}次剩余)")
time.sleep(random.uniform(2, 3))
# 如果重试用完仍失败,跳过当前offset继续下一个
if retry == 0:
print(f" 跳过 offset {offset}")
continue
if len(results) < limit:
print("🎯 到达数据末尾")
break
time.sleep(random.uniform(4, 6))
# 处理数据
print("🔄 处理数据中...")
processed_data = self._process_data(all_data)
@ -143,7 +164,7 @@ class DataSetDownloader:
os.makedirs(output_dir, exist_ok=True)
# 保存为CSV
output_file = os.path.join(output_dir, f"{data_set_id}_{endpoint}.csv")
output_file = os.path.join(output_dir, f"{data_set_id}_{region.lower()}_{universe.lower()}.csv")
if processed_data:
fieldnames = list(processed_data[0].keys())
@ -160,8 +181,21 @@ if __name__ == "__main__":
downloader = DataSetDownloader()
if downloader.client:
endpoint = 'data-fields'
data_set_id = 'socialmedia8'
downloader.download_data_set(endpoint, data_set_id)
plan_to_download = [
{
'id': 'analyst10',
'region': 'USA',
'universe': 'TOP3000'
},
{
'id': 'analyst10',
'region': 'GLB',
'universe': 'TOP3000'
}
]
for item in plan_to_download:
downloader.download_data_set(item['id'], item['region'], item['universe'])
time.sleep(random.uniform(20, 30))
else:
print("❌ 登录失败,无法下载数据")
Loading…
Cancel
Save