wqb
/
FieldDownloader
2
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

++

Browse Source
main
Jack 3 weeks ago
parent d1001731b5
commit 8cb03f0a4e
21 changed files with 1404 additions and 26 deletions
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Show Stats Download Patch File Download Diff File
  1. 852
      category_files/analyst.json
  2. 7
      category_files/broker.json
  3. 37
      category_files/earnings.json
  4. 337
      category_files/fundamental.json
  5. 111
      get_category.py
  6. 86
      main.py
  7. 0
      reference_fields_bak/all_data_combined.csv
  8. 0
      reference_fields_bak/analyst4_data-fields.csv
  9. 0
      reference_fields_bak/fundamental2_data-fields.csv
  10. 0
      reference_fields_bak/fundamental6_data-fields.csv
  11. 0
      reference_fields_bak/model16_data-fields.csv
  12. 0
      reference_fields_bak/model51_data-fields.csv
  13. 0
      reference_fields_bak/news12_data-fields.csv
  14. 0
      reference_fields_bak/news18_data-fields.csv
  15. 0
      reference_fields_bak/option8_data-fields.csv
  16. 0
      reference_fields_bak/option9_data-fields.csv
  17. 0
      reference_fields_bak/pv13_data-fields.csv
  18. 0
      reference_fields_bak/pv1_data-fields.csv
  19. 0
      reference_fields_bak/socialmedia12_data-fields.csv
  20. 0
      reference_fields_bak/socialmedia8_data-fields.csv
  21. 0
      reference_fields_bak/univ1_data-fields.csv

852
category_files/analyst.json
Unescape View File

@ -0,0 +1,852 @@
[
{
"id": "analyst10",
"region": "USA",
"universe": "TOP3000"
},
{
"id": "analyst11",
"region": "USA",
"universe": "TOP3000"
},
{
"id": "analyst14",
"region": "USA",
"universe": "TOP3000"
},
{
"id": "analyst15",
"region": "USA",
"universe": "TOP3000"
},
{
"id": "analyst16",
"region": "USA",
"universe": "TOP3000"
},
{
"id": "analyst39",
"region": "USA",
"universe": "TOP3000"
},
{
"id": "analyst4",
"region": "USA",
"universe": "TOP3000"
},
{
"id": "analyst40",
"region": "USA",
"universe": "TOP3000"
},
{
"id": "analyst44",
"region": "USA",
"universe": "TOP3000"
},
{
"id": "analyst45",
"region": "USA",
"universe": "TOP3000"
},
{
"id": "analyst49",
"region": "USA",
"universe": "TOP3000"
},
{
"id": "analyst52",
"region": "USA",
"universe": "TOP3000"
},
{
"id": "analyst69",
"region": "USA",
"universe": "TOP3000"
},
{
"id": "analyst7",
"region": "USA",
"universe": "TOP3000"
},
{
"id": "analyst82",
"region": "USA",
"universe": "TOP3000"
},
{
"id": "analyst83",
"region": "USA",
"universe": "TOP3000"
},
{
"id": "model211",
"region": "USA",
"universe": "TOP3000"
},
{
"id": "model52",
"region": "USA",
"universe": "TOP3000"
},
{
"id": "news87",
"region": "USA",
"universe": "TOP3000"
},
{
"id": "analyst10",
"region": "USA",
"universe": "TOP1000"
},
{
"id": "analyst11",
"region": "USA",
"universe": "TOP1000"
},
{
"id": "analyst14",
"region": "USA",
"universe": "TOP1000"
},
{
"id": "analyst15",
"region": "USA",
"universe": "TOP1000"
},
{
"id": "analyst16",
"region": "USA",
"universe": "TOP1000"
},
{
"id": "analyst39",
"region": "USA",
"universe": "TOP1000"
},
{
"id": "analyst4",
"region": "USA",
"universe": "TOP1000"
},
{
"id": "analyst40",
"region": "USA",
"universe": "TOP1000"
},
{
"id": "analyst44",
"region": "USA",
"universe": "TOP1000"
},
{
"id": "analyst45",
"region": "USA",
"universe": "TOP1000"
},
{
"id": "analyst49",
"region": "USA",
"universe": "TOP1000"
},
{
"id": "analyst52",
"region": "USA",
"universe": "TOP1000"
},
{
"id": "analyst69",
"region": "USA",
"universe": "TOP1000"
},
{
"id": "analyst7",
"region": "USA",
"universe": "TOP1000"
},
{
"id": "analyst82",
"region": "USA",
"universe": "TOP1000"
},
{
"id": "analyst83",
"region": "USA",
"universe": "TOP1000"
},
{
"id": "model211",
"region": "USA",
"universe": "TOP1000"
},
{
"id": "model52",
"region": "USA",
"universe": "TOP1000"
},
{
"id": "news87",
"region": "USA",
"universe": "TOP1000"
},
{
"id": "analyst10",
"region": "USA",
"universe": "TOP500"
},
{
"id": "analyst11",
"region": "USA",
"universe": "TOP500"
},
{
"id": "analyst14",
"region": "USA",
"universe": "TOP500"
},
{
"id": "analyst15",
"region": "USA",
"universe": "TOP500"
},
{
"id": "analyst16",
"region": "USA",
"universe": "TOP500"
},
{
"id": "analyst39",
"region": "USA",
"universe": "TOP500"
},
{
"id": "analyst4",
"region": "USA",
"universe": "TOP500"
},
{
"id": "analyst40",
"region": "USA",
"universe": "TOP500"
},
{
"id": "analyst44",
"region": "USA",
"universe": "TOP500"
},
{
"id": "analyst45",
"region": "USA",
"universe": "TOP500"
},
{
"id": "analyst49",
"region": "USA",
"universe": "TOP500"
},
{
"id": "analyst52",
"region": "USA",
"universe": "TOP500"
},
{
"id": "analyst69",
"region": "USA",
"universe": "TOP500"
},
{
"id": "analyst7",
"region": "USA",
"universe": "TOP500"
},
{
"id": "analyst82",
"region": "USA",
"universe": "TOP500"
},
{
"id": "analyst83",
"region": "USA",
"universe": "TOP500"
},
{
"id": "model211",
"region": "USA",
"universe": "TOP500"
},
{
"id": "model52",
"region": "USA",
"universe": "TOP500"
},
{
"id": "news87",
"region": "USA",
"universe": "TOP500"
},
{
"id": "analyst10",
"region": "USA",
"universe": "TOP200"
},
{
"id": "analyst11",
"region": "USA",
"universe": "TOP200"
},
{
"id": "analyst14",
"region": "USA",
"universe": "TOP200"
},
{
"id": "analyst15",
"region": "USA",
"universe": "TOP200"
},
{
"id": "analyst16",
"region": "USA",
"universe": "TOP200"
},
{
"id": "analyst39",
"region": "USA",
"universe": "TOP200"
},
{
"id": "analyst4",
"region": "USA",
"universe": "TOP200"
},
{
"id": "analyst40",
"region": "USA",
"universe": "TOP200"
},
{
"id": "analyst44",
"region": "USA",
"universe": "TOP200"
},
{
"id": "analyst45",
"region": "USA",
"universe": "TOP200"
},
{
"id": "analyst49",
"region": "USA",
"universe": "TOP200"
},
{
"id": "analyst52",
"region": "USA",
"universe": "TOP200"
},
{
"id": "analyst69",
"region": "USA",
"universe": "TOP200"
},
{
"id": "analyst7",
"region": "USA",
"universe": "TOP200"
},
{
"id": "analyst82",
"region": "USA",
"universe": "TOP200"
},
{
"id": "analyst83",
"region": "USA",
"universe": "TOP200"
},
{
"id": "model211",
"region": "USA",
"universe": "TOP200"
},
{
"id": "model52",
"region": "USA",
"universe": "TOP200"
},
{
"id": "news87",
"region": "USA",
"universe": "TOP200"
},
{
"id": "analyst10",
"region": "USA",
"universe": "ILLIQUID_MINVOL1M"
},
{
"id": "analyst11",
"region": "USA",
"universe": "ILLIQUID_MINVOL1M"
},
{
"id": "analyst14",
"region": "USA",
"universe": "ILLIQUID_MINVOL1M"
},
{
"id": "analyst15",
"region": "USA",
"universe": "ILLIQUID_MINVOL1M"
},
{
"id": "analyst16",
"region": "USA",
"universe": "ILLIQUID_MINVOL1M"
},
{
"id": "analyst39",
"region": "USA",
"universe": "ILLIQUID_MINVOL1M"
},
{
"id": "analyst4",
"region": "USA",
"universe": "ILLIQUID_MINVOL1M"
},
{
"id": "analyst44",
"region": "USA",
"universe": "ILLIQUID_MINVOL1M"
},
{
"id": "analyst45",
"region": "USA",
"universe": "ILLIQUID_MINVOL1M"
},
{
"id": "analyst49",
"region": "USA",
"universe": "ILLIQUID_MINVOL1M"
},
{
"id": "analyst52",
"region": "USA",
"universe": "ILLIQUID_MINVOL1M"
},
{
"id": "analyst69",
"region": "USA",
"universe": "ILLIQUID_MINVOL1M"
},
{
"id": "analyst7",
"region": "USA",
"universe": "ILLIQUID_MINVOL1M"
},
{
"id": "analyst82",
"region": "USA",
"universe": "ILLIQUID_MINVOL1M"
},
{
"id": "analyst83",
"region": "USA",
"universe": "ILLIQUID_MINVOL1M"
},
{
"id": "model211",
"region": "USA",
"universe": "ILLIQUID_MINVOL1M"
},
{
"id": "news87",
"region": "USA",
"universe": "ILLIQUID_MINVOL1M"
},
{
"id": "analyst10",
"region": "USA",
"universe": "TOPSP500"
},
{
"id": "analyst11",
"region": "USA",
"universe": "TOPSP500"
},
{
"id": "analyst14",
"region": "USA",
"universe": "TOPSP500"
},
{
"id": "analyst15",
"region": "USA",
"universe": "TOPSP500"
},
{
"id": "analyst16",
"region": "USA",
"universe": "TOPSP500"
},
{
"id": "analyst39",
"region": "USA",
"universe": "TOPSP500"
},
{
"id": "analyst4",
"region": "USA",
"universe": "TOPSP500"
},
{
"id": "analyst40",
"region": "USA",
"universe": "TOPSP500"
},
{
"id": "analyst44",
"region": "USA",
"universe": "TOPSP500"
},
{
"id": "analyst45",
"region": "USA",
"universe": "TOPSP500"
},
{
"id": "analyst49",
"region": "USA",
"universe": "TOPSP500"
},
{
"id": "analyst52",
"region": "USA",
"universe": "TOPSP500"
},
{
"id": "analyst69",
"region": "USA",
"universe": "TOPSP500"
},
{
"id": "analyst7",
"region": "USA",
"universe": "TOPSP500"
},
{
"id": "analyst83",
"region": "USA",
"universe": "TOPSP500"
},
{
"id": "model211",
"region": "USA",
"universe": "TOPSP500"
},
{
"id": "model52",
"region": "USA",
"universe": "TOPSP500"
},
{
"id": "news87",
"region": "USA",
"universe": "TOPSP500"
},
{
"id": "analyst10",
"region": "GLB",
"universe": "TOP3000"
},
{
"id": "analyst11",
"region": "GLB",
"universe": "TOP3000"
},
{
"id": "analyst14",
"region": "GLB",
"universe": "TOP3000"
},
{
"id": "analyst15",
"region": "GLB",
"universe": "TOP3000"
},
{
"id": "analyst39",
"region": "GLB",
"universe": "TOP3000"
},
{
"id": "analyst44",
"region": "GLB",
"universe": "TOP3000"
},
{
"id": "analyst45",
"region": "GLB",
"universe": "TOP3000"
},
{
"id": "analyst46",
"region": "GLB",
"universe": "TOP3000"
},
{
"id": "analyst69",
"region": "GLB",
"universe": "TOP3000"
},
{
"id": "analyst10",
"region": "EUR",
"universe": "ILLIQUID_MINVOL1M"
},
{
"id": "analyst11",
"region": "EUR",
"universe": "ILLIQUID_MINVOL1M"
},
{
"id": "analyst14",
"region": "EUR",
"universe": "ILLIQUID_MINVOL1M"
},
{
"id": "analyst15",
"region": "EUR",
"universe": "ILLIQUID_MINVOL1M"
},
{
"id": "analyst4",
"region": "EUR",
"universe": "ILLIQUID_MINVOL1M"
},
{
"id": "analyst44",
"region": "EUR",
"universe": "ILLIQUID_MINVOL1M"
},
{
"id": "analyst45",
"region": "EUR",
"universe": "ILLIQUID_MINVOL1M"
},
{
"id": "analyst46",
"region": "EUR",
"universe": "ILLIQUID_MINVOL1M"
},
{
"id": "analyst48",
"region": "EUR",
"universe": "ILLIQUID_MINVOL1M"
},
{
"id": "analyst69",
"region": "EUR",
"universe": "ILLIQUID_MINVOL1M"
},
{
"id": "analyst7",
"region": "EUR",
"universe": "ILLIQUID_MINVOL1M"
},
{
"id": "analyst84",
"region": "EUR",
"universe": "ILLIQUID_MINVOL1M"
},
{
"id": "sentiment12",
"region": "EUR",
"universe": "ILLIQUID_MINVOL1M"
},
{
"id": "analyst11",
"region": "ASI",
"universe": "ILLIQUID_MINVOL1M"
},
{
"id": "analyst14",
"region": "ASI",
"universe": "ILLIQUID_MINVOL1M"
},
{
"id": "analyst15",
"region": "ASI",
"universe": "ILLIQUID_MINVOL1M"
},
{
"id": "analyst16",
"region": "ASI",
"universe": "ILLIQUID_MINVOL1M"
},
{
"id": "analyst4",
"region": "ASI",
"universe": "ILLIQUID_MINVOL1M"
},
{
"id": "analyst45",
"region": "ASI",
"universe": "ILLIQUID_MINVOL1M"
},
{
"id": "analyst48",
"region": "ASI",
"universe": "ILLIQUID_MINVOL1M"
},
{
"id": "analyst69",
"region": "ASI",
"universe": "ILLIQUID_MINVOL1M"
},
{
"id": "analyst10",
"region": "TWN",
"universe": "TOP500"
},
{
"id": "analyst11",
"region": "TWN",
"universe": "TOP500"
},
{
"id": "analyst14",
"region": "TWN",
"universe": "TOP500"
},
{
"id": "analyst15",
"region": "TWN",
"universe": "TOP500"
},
{
"id": "analyst16",
"region": "TWN",
"universe": "TOP500"
},
{
"id": "analyst21",
"region": "TWN",
"universe": "TOP500"
},
{
"id": "analyst39",
"region": "TWN",
"universe": "TOP500"
},
{
"id": "analyst4",
"region": "TWN",
"universe": "TOP500"
},
{
"id": "analyst44",
"region": "TWN",
"universe": "TOP500"
},
{
"id": "analyst45",
"region": "TWN",
"universe": "TOP500"
},
{
"id": "analyst46",
"region": "TWN",
"universe": "TOP500"
},
{
"id": "analyst48",
"region": "TWN",
"universe": "TOP500"
},
{
"id": "analyst81",
"region": "TWN",
"universe": "TOP500"
},
{
"id": "analyst9",
"region": "TWN",
"universe": "TOP500"
},
{
"id": "analyst92",
"region": "TWN",
"universe": "TOP500"
},
{
"id": "analyst94",
"region": "TWN",
"universe": "TOP500"
},
{
"id": "model52",
"region": "TWN",
"universe": "TOP500"
},
{
"id": "other411",
"region": "TWN",
"universe": "TOP500"
},
{
"id": "other423",
"region": "TWN",
"universe": "TOP500"
},
{
"id": "analyst11",
"region": "IND",
"universe": "TOP500"
},
{
"id": "analyst39",
"region": "IND",
"universe": "TOP500"
},
{
"id": "analyst4",
"region": "IND",
"universe": "TOP500"
},
{
"id": "analyst44",
"region": "IND",
"universe": "TOP500"
},
{
"id": "analyst45",
"region": "IND",
"universe": "TOP500"
},
{
"id": "analyst46",
"region": "IND",
"universe": "TOP500"
},
{
"id": "analyst48",
"region": "IND",
"universe": "TOP500"
},
{
"id": "analyst81",
"region": "IND",
"universe": "TOP500"
},
{
"id": "analyst83",
"region": "IND",
"universe": "TOP500"
},
{
"id": "analyst9",
"region": "IND",
"universe": "TOP500"
}
]

7
category_files/broker.json
Unescape View File

@ -0,0 +1,7 @@
[
{
"id": "broker1",
"region": "ASI",
"universe": "MINVOL1M"
}
]

37
category_files/earnings.json
Unescape View File

@ -0,0 +1,37 @@
[
{
"id": "earnings6",
"region": "USA",
"universe": "ILLIQUID_MINVOL1M"
},
{
"id": "earnings7",
"region": "USA",
"universe": "ILLIQUID_MINVOL1M"
},
{
"id": "earnings6",
"region": "GLB",
"universe": "MINVOL1M"
},
{
"id": "earnings7",
"region": "GLB",
"universe": "MINVOL1M"
},
{
"id": "earnings6",
"region": "EUR",
"universe": "ILLIQUID_MINVOL1M"
},
{
"id": "earnings6",
"region": "ASI",
"universe": "ILLIQUID_MINVOL1M"
},
{
"id": "earnings6",
"region": "ASI",
"universe": "MINVOL1M"
}
]

337
category_files/fundamental.json
Unescape View File

@ -0,0 +1,337 @@
[
{
"id": "fundamental1",
"region": "USA",
"universe": "ILLIQUID_MINVOL1M"
},
{
"id": "fundamental110",
"region": "USA",
"universe": "ILLIQUID_MINVOL1M"
},
{
"id": "fundamental13",
"region": "USA",
"universe": "ILLIQUID_MINVOL1M"
},
{
"id": "fundamental14",
"region": "USA",
"universe": "ILLIQUID_MINVOL1M"
},
{
"id": "fundamental17",
"region": "USA",
"universe": "ILLIQUID_MINVOL1M"
},
{
"id": "fundamental2",
"region": "USA",
"universe": "ILLIQUID_MINVOL1M"
},
{
"id": "fundamental22",
"region": "USA",
"universe": "ILLIQUID_MINVOL1M"
},
{
"id": "fundamental23",
"region": "USA",
"universe": "ILLIQUID_MINVOL1M"
},
{
"id": "fundamental25",
"region": "USA",
"universe": "ILLIQUID_MINVOL1M"
},
{
"id": "fundamental28",
"region": "USA",
"universe": "ILLIQUID_MINVOL1M"
},
{
"id": "fundamental3",
"region": "USA",
"universe": "ILLIQUID_MINVOL1M"
},
{
"id": "fundamental31",
"region": "USA",
"universe": "ILLIQUID_MINVOL1M"
},
{
"id": "fundamental45",
"region": "USA",
"universe": "ILLIQUID_MINVOL1M"
},
{
"id": "fundamental6",
"region": "USA",
"universe": "ILLIQUID_MINVOL1M"
},
{
"id": "fundamental69",
"region": "USA",
"universe": "ILLIQUID_MINVOL1M"
},
{
"id": "fundamental7",
"region": "USA",
"universe": "ILLIQUID_MINVOL1M"
},
{
"id": "fundamental72",
"region": "USA",
"universe": "ILLIQUID_MINVOL1M"
},
{
"id": "fundamental89",
"region": "USA",
"universe": "ILLIQUID_MINVOL1M"
},
{
"id": "fundamental90",
"region": "USA",
"universe": "ILLIQUID_MINVOL1M"
},
{
"id": "other395",
"region": "USA",
"universe": "ILLIQUID_MINVOL1M"
},
{
"id": "other401",
"region": "USA",
"universe": "ILLIQUID_MINVOL1M"
},
{
"id": "fundamental1",
"region": "GLB",
"universe": "MINVOL1M"
},
{
"id": "fundamental13",
"region": "GLB",
"universe": "MINVOL1M"
},
{
"id": "fundamental17",
"region": "GLB",
"universe": "MINVOL1M"
},
{
"id": "fundamental22",
"region": "GLB",
"universe": "MINVOL1M"
},
{
"id": "fundamental23",
"region": "GLB",
"universe": "MINVOL1M"
},
{
"id": "fundamental25",
"region": "GLB",
"universe": "MINVOL1M"
},
{
"id": "fundamental28",
"region": "GLB",
"universe": "MINVOL1M"
},
{
"id": "fundamental31",
"region": "GLB",
"universe": "MINVOL1M"
},
{
"id": "fundamental44",
"region": "GLB",
"universe": "MINVOL1M"
},
{
"id": "fundamental6",
"region": "GLB",
"universe": "MINVOL1M"
},
{
"id": "fundamental72",
"region": "GLB",
"universe": "MINVOL1M"
},
{
"id": "fundamental86",
"region": "GLB",
"universe": "MINVOL1M"
},
{
"id": "model56",
"region": "GLB",
"universe": "MINVOL1M"
},
{
"id": "fundamental1",
"region": "EUR",
"universe": "ILLIQUID_MINVOL1M"
},
{
"id": "fundamental17",
"region": "EUR",
"universe": "ILLIQUID_MINVOL1M"
},
{
"id": "fundamental22",
"region": "EUR",
"universe": "ILLIQUID_MINVOL1M"
},
{
"id": "fundamental23",
"region": "EUR",
"universe": "ILLIQUID_MINVOL1M"
},
{
"id": "fundamental25",
"region": "EUR",
"universe": "ILLIQUID_MINVOL1M"
},
{
"id": "fundamental28",
"region": "EUR",
"universe": "ILLIQUID_MINVOL1M"
},
{
"id": "fundamental31",
"region": "EUR",
"universe": "ILLIQUID_MINVOL1M"
},
{
"id": "fundamental44",
"region": "EUR",
"universe": "ILLIQUID_MINVOL1M"
},
{
"id": "fundamental6",
"region": "EUR",
"universe": "ILLIQUID_MINVOL1M"
},
{
"id": "fundamental72",
"region": "EUR",
"universe": "ILLIQUID_MINVOL1M"
},
{
"id": "fundamental89",
"region": "EUR",
"universe": "ILLIQUID_MINVOL1M"
},
{
"id": "other395",
"region": "EUR",
"universe": "ILLIQUID_MINVOL1M"
},
{
"id": "fundamental1",
"region": "ASI",
"universe": "ILLIQUID_MINVOL1M"
},
{
"id": "fundamental17",
"region": "ASI",
"universe": "ILLIQUID_MINVOL1M"
},
{
"id": "fundamental21",
"region": "ASI",
"universe": "ILLIQUID_MINVOL1M"
},
{
"id": "fundamental22",
"region": "ASI",
"universe": "ILLIQUID_MINVOL1M"
},
{
"id": "fundamental23",
"region": "ASI",
"universe": "ILLIQUID_MINVOL1M"
},
{
"id": "fundamental28",
"region": "ASI",
"universe": "ILLIQUID_MINVOL1M"
},
{
"id": "fundamental4",
"region": "ASI",
"universe": "ILLIQUID_MINVOL1M"
},
{
"id": "fundamental6",
"region": "ASI",
"universe": "ILLIQUID_MINVOL1M"
},
{
"id": "fundamental94",
"region": "ASI",
"universe": "ILLIQUID_MINVOL1M"
},
{
"id": "other466",
"region": "ASI",
"universe": "ILLIQUID_MINVOL1M"
},
{
"id": "fundamental1",
"region": "ASI",
"universe": "MINVOL1M"
},
{
"id": "fundamental109",
"region": "ASI",
"universe": "MINVOL1M"
},
{
"id": "fundamental17",
"region": "ASI",
"universe": "MINVOL1M"
},
{
"id": "fundamental21",
"region": "ASI",
"universe": "MINVOL1M"
},
{
"id": "fundamental22",
"region": "ASI",
"universe": "MINVOL1M"
},
{
"id": "fundamental23",
"region": "ASI",
"universe": "MINVOL1M"
},
{
"id": "fundamental28",
"region": "ASI",
"universe": "MINVOL1M"
},
{
"id": "fundamental4",
"region": "ASI",
"universe": "MINVOL1M"
},
{
"id": "fundamental6",
"region": "ASI",
"universe": "MINVOL1M"
},
{
"id": "fundamental94",
"region": "ASI",
"universe": "MINVOL1M"
},
{
"id": "other466",
"region": "ASI",
"universe": "MINVOL1M"
}
]

111
get_category.py
Unescape View File

@ -0,0 +1,111 @@
# -*- coding: utf-8 -*-
import os
import json
import random
import time
import httpx
from httpx import BasicAuth
class CategoryDownloader:
def __init__(self):
self.base_api_url = 'https://api.worldquantbrain.com'
self.client = self.login()
def login(self):
"""登录并返回客户端实例"""
username, password = "jack0210_@hotmail.com", "!QAZ2wsx+0913"
client = httpx.Client(auth=BasicAuth(username, password))
try:
response = client.post(f'{self.base_api_url}/authentication')
print(f"登录状态: {response.status_code}")
if response.status_code in [200, 201]:
print("登录成功!")
return client
else:
print(f"登录失败: {response.json()}")
return None
except Exception as e:
print(f"登录过程中出现错误: {e}")
return None
def fetch_category_data(self, category, delay, instrumentType, region_list, universe_list):
"""获取分类数据并保存到JSON文件"""
results = []
# 创建category_files文件夹
output_dir = "category_files"
if not os.path.exists(output_dir):
os.makedirs(output_dir)
print(f"已创建文件夹: {output_dir}")
if self.client:
for region in region_list:
for universe in universe_list:
url = f'https://api.worldquantbrain.com/data-sets?category={category}&delay={delay}&instrumentType={instrumentType}&limit=50&offset=0&region={region}&universe={universe}'
print(f"请求URL: {url}")
try:
response = self.client.get(url)
if response.status_code == 200:
data = response.json()
if data.get('count', 0) > 0:
for item in data.get('results', []):
result_item = {
'id': item.get('id', ''),
'region': item.get('region', ''),
'universe': item.get('universe', '')
}
results.append(result_item)
else:
print(f"请求失败: {response.status_code}")
except Exception as e:
print(f"请求过程中出现错误: {e}")
time.sleep(random.uniform(5, 8))
# 保存到JSON文件
filename = os.path.join(output_dir, f"{category}.json")
with open(filename, 'w', encoding='utf-8') as f:
json.dump(results, f, indent=2, ensure_ascii=False)
print(f"数据已保存到: {filename}")
print(f"总共找到 {len(results)} 条记录")
return results
if __name__ == "__main__":
downloader = CategoryDownloader()
if downloader.client:
category = 'fundamental'
delay = '1'
instrumentType = 'EQUITY'
region_list = [
'USA',
'GLB',
'EUR',
'ASI',
'CHN',
'KOR',
'TWN',
'IND'
]
universe_list = [
# 'TOP3000',
# 'TOP1000',
# 'TOP500',
# 'TOP200',
# 'TOPSP500',
'ILLIQUID_MINVOL1M',
'MINVOL1M'
]
downloader.fetch_category_data(category, delay, instrumentType, region_list, universe_list)

86
main.py
Unescape View File

@ -1,3 +1,4 @@
# -*- coding: utf-8 -*-
import os import os
import json import json
import random import random
@ -45,7 +46,7 @@ class DataSetDownloader:
print(f"results 长度: {len(data.get('results', []))}") print(f"results 长度: {len(data.get('results', []))}")
print(f"响应键: {list(data.keys())}") print(f"响应键: {list(data.keys())}")
def _build_params(self, data_set_id, offset=0, limit=50): def _build_params(self, data_set_id, region, universe, offset=0, limit=50):
"""构建请求参数""" """构建请求参数"""
return { return {
'dataset.id': data_set_id, 'dataset.id': data_set_id,
@ -53,8 +54,8 @@ class DataSetDownloader:
'instrumentType': 'EQUITY', 'instrumentType': 'EQUITY',
'limit': limit, 'limit': limit,
'offset': offset, 'offset': offset,
'region': 'USA', 'region': region,
'universe': 'TOP3000' 'universe': universe
} }
def _process_item(self, item): def _process_item(self, item):
@ -76,7 +77,8 @@ class DataSetDownloader:
"""批量处理数据""" """批量处理数据"""
return [self._process_item(item) for item in raw_data] return [self._process_item(item) for item in raw_data]
def download_data_set(self, endpoint, data_set_id): def download_data_set(self, data_set_id, region, universe):
endpoint = 'data-fields'
"""下载数据集""" """下载数据集"""
# 检查登录状态 # 检查登录状态
if not self.client: if not self.client:
@ -88,7 +90,7 @@ class DataSetDownloader:
# 获取数据总数 # 获取数据总数
url = f"{self.base_api_url}/{endpoint}" url = f"{self.base_api_url}/{endpoint}"
params = self._build_params(data_set_id, limit=1) params = self._build_params(data_set_id, region, universe, limit=1)
response = self.client.get(url, params=params) response = self.client.get(url, params=params)
data = response.json() data = response.json()
@ -108,32 +110,51 @@ class DataSetDownloader:
for offset in range(0, total_count, limit): for offset in range(0, total_count, limit):
time.sleep(random.uniform(1.0, 1.5)) time.sleep(random.uniform(1.0, 1.5))
params = self._build_params(data_set_id, offset, limit) params = self._build_params(data_set_id, region, universe, offset, limit) # 修正参数
print(f"📥 下载进度: {offset}/{total_count} ({offset / total_count * 100:.1f}%)") print(f"📥 下载进度: {offset}/{total_count} ({offset / total_count * 100:.1f}%)")
try: results = []
response = self.client.get(url, params=params)
if response.status_code == 200: retry = 3
data = response.json() while retry > 0:
results = data.get('results', []) try:
response = self.client.get(url, params=params)
print(f"✅ 本页获取到 {len(results)} 条记录") if response.status_code == 200:
all_data.extend(results) data = response.json()
results = data.get('results', [])
if len(results) < limit: print(f"✅ 本页获取到 {len(results)} 条记录")
print("🎯 到达数据末尾") all_data.extend(results)
break
else:
print(f"❌ 请求失败: {response.status_code}")
break
time.sleep(random.uniform(1, 2)) # 成功时退出重试循环
break
except Exception as e: else:
print(f"❌ 下载过程中出错: {e}") print(f"❌ 请求失败: {response.status_code}")
retry -= 1
if retry > 0:
print(f"🔄 重试中... ({retry}次剩余)")
time.sleep(random.uniform(2, 3))
except Exception as e:
print(f"❌ 下载过程中出错: {e}")
retry -= 1
if retry > 0:
print(f"🔄 重试中... ({retry}次剩余)")
time.sleep(random.uniform(2, 3))
# 如果重试用完仍失败，跳过当前offset继续下一个
if retry == 0:
print(f"⚠️ 跳过 offset {offset}")
continue
if len(results) < limit:
print("🎯 到达数据末尾")
break break
time.sleep(random.uniform(4, 6))
# 处理数据 # 处理数据
print("🔄 处理数据中...") print("🔄 处理数据中...")
processed_data = self._process_data(all_data) processed_data = self._process_data(all_data)
@ -143,7 +164,7 @@ class DataSetDownloader:
os.makedirs(output_dir, exist_ok=True) os.makedirs(output_dir, exist_ok=True)
# 保存为CSV # 保存为CSV
output_file = os.path.join(output_dir, f"{data_set_id}_{endpoint}.csv") output_file = os.path.join(output_dir, f"{data_set_id}_{region.lower()}_{universe.lower()}.csv")
if processed_data: if processed_data:
fieldnames = list(processed_data[0].keys()) fieldnames = list(processed_data[0].keys())
@ -160,8 +181,21 @@ if __name__ == "__main__":
downloader = DataSetDownloader() downloader = DataSetDownloader()
if downloader.client: if downloader.client:
endpoint = 'data-fields' plan_to_download = [
data_set_id = 'socialmedia8' {
downloader.download_data_set(endpoint, data_set_id) 'id': 'analyst10',
'region': 'USA',
'universe': 'TOP3000'
},
{
'id': 'analyst10',
'region': 'GLB',
'universe': 'TOP3000'
}
]
for item in plan_to_download:
downloader.download_data_set(item['id'], item['region'], item['universe'])
time.sleep(random.uniform(20, 30))
else: else:
print("❌ 登录失败，无法下载数据") print("❌ 登录失败，无法下载数据")

0
reference_fields/all_data_combined.csv → reference_fields_bak/all_data_combined.csv
Unescape View File

unable to load file from base commit

0
reference_fields/analyst4_data-fields.csv → reference_fields_bak/analyst4_data-fields.csv
Unescape View File

unable to load file from base commit

0
reference_fields/fundamental2_data-fields.csv → reference_fields_bak/fundamental2_data-fields.csv
Unescape View File

unable to load file from base commit

0
reference_fields/fundamental6_data-fields.csv → reference_fields_bak/fundamental6_data-fields.csv
Unescape View File

unable to load file from base commit

0
reference_fields/model16_data-fields.csv → reference_fields_bak/model16_data-fields.csv
Unescape View File

unable to load file from base commit

0
reference_fields/model51_data-fields.csv → reference_fields_bak/model51_data-fields.csv
Unescape View File

unable to load file from base commit

0
reference_fields/news12_data-fields.csv → reference_fields_bak/news12_data-fields.csv
Unescape View File

unable to load file from base commit

0
reference_fields/news18_data-fields.csv → reference_fields_bak/news18_data-fields.csv
Unescape View File

unable to load file from base commit

0
reference_fields/option8_data-fields.csv → reference_fields_bak/option8_data-fields.csv
Unescape View File

unable to load file from base commit

0
reference_fields/option9_data-fields.csv → reference_fields_bak/option9_data-fields.csv
Unescape View File

unable to load file from base commit

0
reference_fields/pv13_data-fields.csv → reference_fields_bak/pv13_data-fields.csv
Unescape View File

unable to load file from base commit

0
reference_fields/pv1_data-fields.csv → reference_fields_bak/pv1_data-fields.csv
Unescape View File

unable to load file from base commit

0
reference_fields/socialmedia12_data-fields.csv → reference_fields_bak/socialmedia12_data-fields.csv
Unescape View File

unable to load file from base commit

0
reference_fields/socialmedia8_data-fields.csv → reference_fields_bak/socialmedia8_data-fields.csv
Unescape View File

unable to load file from base commit

0
reference_fields/univ1_data-fields.csv → reference_fields_bak/univ1_data-fields.csv
Unescape View File

unable to load file from base commit
Write Preview
Loading…
Cancel
Save