wqb
/
FieldDownloader
2
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

++

Browse Source
main
Jack 3 weeks ago
parent 8cb03f0a4e
commit 129e328f8a
17 changed files with 8498 additions and 21 deletions
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Show Stats Download Patch File Download Diff File
  1. 52
      category_files/imbalance.json
  2. 47
      category_files/insiders.json
  3. 207
      category_files/institutions.json
  4. 87
      category_files/macro.json
  5. 2322
      category_files/model.json
  6. 1302
      category_files/news.json
  7. 137
      category_files/option.json
  8. 1087
      category_files/other.json
  9. 1102
      category_files/pv.json
  10. 307
      category_files/risk.json
  11. 337
      category_files/sentiment.json
  12. 227
      category_files/shortinterest.json
  13. 92
      category_files/socialmedia.json
  14. 33
      get_category.py
  15. 60
      main.py
  16. 45
      new_db.py
  17. 1075
      reference_fields/analyst10_usa_top3000.csv

52
category_files/imbalance.json
Unescape View File

@ -0,0 +1,52 @@
[
{
"id": "imbalance5",
"region": "USA",
"universe": "TOP3000"
},
{
"id": "imbalance5",
"region": "USA",
"universe": "TOP1000"
},
{
"id": "imbalance5",
"region": "USA",
"universe": "TOP500"
},
{
"id": "imbalance5",
"region": "USA",
"universe": "TOP200"
},
{
"id": "imbalance5",
"region": "USA",
"universe": "TOPSP500"
},
{
"id": "imbalance5",
"region": "USA",
"universe": "ILLIQUID_MINVOL1M"
},
{
"id": "imbalance5",
"region": "EUR",
"universe": "ILLIQUID_MINVOL1M"
},
{
"id": "imbalance5",
"region": "ASI",
"universe": "ILLIQUID_MINVOL1M"
},
{
"id": "imbalance5",
"region": "TWN",
"universe": "TOP500"
},
{
"id": "imbalance5",
"region": "IND",
"universe": "TOP500"
}
]

47
category_files/insiders.json
Unescape View File

@ -0,0 +1,47 @@
[
{
"id": "insiders3",
"region": "USA",
"universe": "TOP3000"
},
{
"id": "insiders3",
"region": "USA",
"universe": "TOP1000"
},
{
"id": "insiders3",
"region": "USA",
"universe": "TOP500"
},
{
"id": "insiders3",
"region": "USA",
"universe": "TOP200"
},
{
"id": "insiders3",
"region": "USA",
"universe": "TOPSP500"
},
{
"id": "insiders3",
"region": "USA",
"universe": "ILLIQUID_MINVOL1M"
},
{
"id": "insiders5",
"region": "TWN",
"universe": "TOP500"
},
{
"id": "insiders1",
"region": "IND",
"universe": "TOP500"
},
{
"id": "insiders5",
"region": "IND",
"universe": "TOP500"
}
]

207
category_files/institutions.json
Unescape View File

@ -0,0 +1,207 @@
[
{
"id": "institutions18",
"region": "USA",
"universe": "TOP3000"
},
{
"id": "institutions20",
"region": "USA",
"universe": "TOP3000"
},
{
"id": "institutions4",
"region": "USA",
"universe": "TOP3000"
},
{
"id": "institutions5",
"region": "USA",
"universe": "TOP3000"
},
{
"id": "institutions6",
"region": "USA",
"universe": "TOP3000"
},
{
"id": "institutions7",
"region": "USA",
"universe": "TOP3000"
},
{
"id": "institutions18",
"region": "USA",
"universe": "TOP1000"
},
{
"id": "institutions20",
"region": "USA",
"universe": "TOP1000"
},
{
"id": "institutions4",
"region": "USA",
"universe": "TOP1000"
},
{
"id": "institutions5",
"region": "USA",
"universe": "TOP1000"
},
{
"id": "institutions6",
"region": "USA",
"universe": "TOP1000"
},
{
"id": "institutions7",
"region": "USA",
"universe": "TOP1000"
},
{
"id": "institutions18",
"region": "USA",
"universe": "TOP500"
},
{
"id": "institutions20",
"region": "USA",
"universe": "TOP500"
},
{
"id": "institutions4",
"region": "USA",
"universe": "TOP500"
},
{
"id": "institutions5",
"region": "USA",
"universe": "TOP500"
},
{
"id": "institutions6",
"region": "USA",
"universe": "TOP500"
},
{
"id": "institutions7",
"region": "USA",
"universe": "TOP500"
},
{
"id": "institutions18",
"region": "USA",
"universe": "TOP200"
},
{
"id": "institutions20",
"region": "USA",
"universe": "TOP200"
},
{
"id": "institutions4",
"region": "USA",
"universe": "TOP200"
},
{
"id": "institutions5",
"region": "USA",
"universe": "TOP200"
},
{
"id": "institutions6",
"region": "USA",
"universe": "TOP200"
},
{
"id": "institutions7",
"region": "USA",
"universe": "TOP200"
},
{
"id": "institutions18",
"region": "USA",
"universe": "TOPSP500"
},
{
"id": "institutions20",
"region": "USA",
"universe": "TOPSP500"
},
{
"id": "institutions4",
"region": "USA",
"universe": "TOPSP500"
},
{
"id": "institutions5",
"region": "USA",
"universe": "TOPSP500"
},
{
"id": "institutions6",
"region": "USA",
"universe": "TOPSP500"
},
{
"id": "institutions7",
"region": "USA",
"universe": "TOPSP500"
},
{
"id": "institutions18",
"region": "USA",
"universe": "ILLIQUID_MINVOL1M"
},
{
"id": "institutions20",
"region": "USA",
"universe": "ILLIQUID_MINVOL1M"
},
{
"id": "institutions4",
"region": "USA",
"universe": "ILLIQUID_MINVOL1M"
},
{
"id": "institutions5",
"region": "USA",
"universe": "ILLIQUID_MINVOL1M"
},
{
"id": "institutions6",
"region": "USA",
"universe": "ILLIQUID_MINVOL1M"
},
{
"id": "institutions7",
"region": "USA",
"universe": "ILLIQUID_MINVOL1M"
},
{
"id": "institutions6",
"region": "GLB",
"universe": "TOP3000"
},
{
"id": "institutions4",
"region": "EUR",
"universe": "ILLIQUID_MINVOL1M"
},
{
"id": "institutions6",
"region": "ASI",
"universe": "ILLIQUID_MINVOL1M"
},
{
"id": "institutions6",
"region": "TWN",
"universe": "TOP500"
},
{
"id": "institutions6",
"region": "IND",
"universe": "TOP500"
}
]

87
category_files/macro.json
Unescape View File

@ -0,0 +1,87 @@
[
{
"id": "macro63",
"region": "USA",
"universe": "TOP3000"
},
{
"id": "other551",
"region": "USA",
"universe": "TOP3000"
},
{
"id": "macro63",
"region": "USA",
"universe": "TOP1000"
},
{
"id": "other551",
"region": "USA",
"universe": "TOP1000"
},
{
"id": "macro63",
"region": "USA",
"universe": "TOP500"
},
{
"id": "other551",
"region": "USA",
"universe": "TOP500"
},
{
"id": "macro63",
"region": "USA",
"universe": "TOP200"
},
{
"id": "other551",
"region": "USA",
"universe": "TOP200"
},
{
"id": "macro63",
"region": "USA",
"universe": "TOPSP500"
},
{
"id": "other551",
"region": "USA",
"universe": "TOPSP500"
},
{
"id": "macro63",
"region": "USA",
"universe": "ILLIQUID_MINVOL1M"
},
{
"id": "other551",
"region": "USA",
"universe": "ILLIQUID_MINVOL1M"
},
{
"id": "other551",
"region": "GLB",
"universe": "TOP3000"
},
{
"id": "macro63",
"region": "ASI",
"universe": "ILLIQUID_MINVOL1M"
},
{
"id": "macro63",
"region": "TWN",
"universe": "TOP500"
},
{
"id": "other551",
"region": "TWN",
"universe": "TOP500"
},
{
"id": "macro63",
"region": "IND",
"universe": "TOP500"
}
]

2322
category_files/model.json
View File

File diff suppressed because it is too large Load Diff

1302
category_files/news.json
View File

File diff suppressed because it is too large Load Diff

137
category_files/option.json
Unescape View File

@ -0,0 +1,137 @@
[
{
"id": "option3",
"region": "USA",
"universe": "TOP3000"
},
{
"id": "option40",
"region": "USA",
"universe": "TOP3000"
},
{
"id": "option8",
"region": "USA",
"universe": "TOP3000"
},
{
"id": "option9",
"region": "USA",
"universe": "TOP3000"
},
{
"id": "option3",
"region": "USA",
"universe": "TOP1000"
},
{
"id": "option40",
"region": "USA",
"universe": "TOP1000"
},
{
"id": "option8",
"region": "USA",
"universe": "TOP1000"
},
{
"id": "option9",
"region": "USA",
"universe": "TOP1000"
},
{
"id": "option3",
"region": "USA",
"universe": "TOP500"
},
{
"id": "option40",
"region": "USA",
"universe": "TOP500"
},
{
"id": "option8",
"region": "USA",
"universe": "TOP500"
},
{
"id": "option9",
"region": "USA",
"universe": "TOP500"
},
{
"id": "option3",
"region": "USA",
"universe": "TOP200"
},
{
"id": "option40",
"region": "USA",
"universe": "TOP200"
},
{
"id": "option8",
"region": "USA",
"universe": "TOP200"
},
{
"id": "option9",
"region": "USA",
"universe": "TOP200"
},
{
"id": "option3",
"region": "USA",
"universe": "TOPSP500"
},
{
"id": "option40",
"region": "USA",
"universe": "TOPSP500"
},
{
"id": "option8",
"region": "USA",
"universe": "TOPSP500"
},
{
"id": "option9",
"region": "USA",
"universe": "TOPSP500"
},
{
"id": "option3",
"region": "USA",
"universe": "ILLIQUID_MINVOL1M"
},
{
"id": "option40",
"region": "USA",
"universe": "ILLIQUID_MINVOL1M"
},
{
"id": "option8",
"region": "USA",
"universe": "ILLIQUID_MINVOL1M"
},
{
"id": "option9",
"region": "USA",
"universe": "ILLIQUID_MINVOL1M"
},
{
"id": "option37",
"region": "TWN",
"universe": "TOP500"
},
{
"id": "option1",
"region": "IND",
"universe": "TOP500"
},
{
"id": "option30",
"region": "IND",
"universe": "TOP500"
}
]

1087
category_files/other.json
View File

File diff suppressed because it is too large Load Diff

1102
category_files/pv.json
View File

File diff suppressed because it is too large Load Diff

307
category_files/risk.json
Unescape View File

@ -0,0 +1,307 @@
[
{
"id": "risk59",
"region": "USA",
"universe": "TOP3000"
},
{
"id": "risk60",
"region": "USA",
"universe": "TOP3000"
},
{
"id": "risk62",
"region": "USA",
"universe": "TOP3000"
},
{
"id": "risk65",
"region": "USA",
"universe": "TOP3000"
},
{
"id": "risk70",
"region": "USA",
"universe": "TOP3000"
},
{
"id": "risk72",
"region": "USA",
"universe": "TOP3000"
},
{
"id": "risk59",
"region": "USA",
"universe": "TOP1000"
},
{
"id": "risk60",
"region": "USA",
"universe": "TOP1000"
},
{
"id": "risk62",
"region": "USA",
"universe": "TOP1000"
},
{
"id": "risk65",
"region": "USA",
"universe": "TOP1000"
},
{
"id": "risk70",
"region": "USA",
"universe": "TOP1000"
},
{
"id": "risk72",
"region": "USA",
"universe": "TOP1000"
},
{
"id": "risk59",
"region": "USA",
"universe": "TOP500"
},
{
"id": "risk60",
"region": "USA",
"universe": "TOP500"
},
{
"id": "risk62",
"region": "USA",
"universe": "TOP500"
},
{
"id": "risk65",
"region": "USA",
"universe": "TOP500"
},
{
"id": "risk70",
"region": "USA",
"universe": "TOP500"
},
{
"id": "risk72",
"region": "USA",
"universe": "TOP500"
},
{
"id": "risk59",
"region": "USA",
"universe": "TOP200"
},
{
"id": "risk60",
"region": "USA",
"universe": "TOP200"
},
{
"id": "risk62",
"region": "USA",
"universe": "TOP200"
},
{
"id": "risk65",
"region": "USA",
"universe": "TOP200"
},
{
"id": "risk70",
"region": "USA",
"universe": "TOP200"
},
{
"id": "risk72",
"region": "USA",
"universe": "TOP200"
},
{
"id": "risk59",
"region": "USA",
"universe": "TOPSP500"
},
{
"id": "risk60",
"region": "USA",
"universe": "TOPSP500"
},
{
"id": "risk62",
"region": "USA",
"universe": "TOPSP500"
},
{
"id": "risk65",
"region": "USA",
"universe": "TOPSP500"
},
{
"id": "risk70",
"region": "USA",
"universe": "TOPSP500"
},
{
"id": "risk72",
"region": "USA",
"universe": "TOPSP500"
},
{
"id": "risk59",
"region": "USA",
"universe": "ILLIQUID_MINVOL1M"
},
{
"id": "risk60",
"region": "USA",
"universe": "ILLIQUID_MINVOL1M"
},
{
"id": "risk62",
"region": "USA",
"universe": "ILLIQUID_MINVOL1M"
},
{
"id": "risk65",
"region": "USA",
"universe": "ILLIQUID_MINVOL1M"
},
{
"id": "risk70",
"region": "USA",
"universe": "ILLIQUID_MINVOL1M"
},
{
"id": "risk72",
"region": "USA",
"universe": "ILLIQUID_MINVOL1M"
},
{
"id": "risk60",
"region": "GLB",
"universe": "TOP3000"
},
{
"id": "risk66",
"region": "GLB",
"universe": "TOP3000"
},
{
"id": "risk68",
"region": "GLB",
"universe": "TOP3000"
},
{
"id": "risk70",
"region": "GLB",
"universe": "TOP3000"
},
{
"id": "risk60",
"region": "EUR",
"universe": "ILLIQUID_MINVOL1M"
},
{
"id": "risk62",
"region": "EUR",
"universe": "ILLIQUID_MINVOL1M"
},
{
"id": "risk68",
"region": "EUR",
"universe": "ILLIQUID_MINVOL1M"
},
{
"id": "risk70",
"region": "EUR",
"universe": "ILLIQUID_MINVOL1M"
},
{
"id": "risk72",
"region": "EUR",
"universe": "ILLIQUID_MINVOL1M"
},
{
"id": "risk59",
"region": "ASI",
"universe": "ILLIQUID_MINVOL1M"
},
{
"id": "risk60",
"region": "ASI",
"universe": "ILLIQUID_MINVOL1M"
},
{
"id": "risk68",
"region": "ASI",
"universe": "ILLIQUID_MINVOL1M"
},
{
"id": "risk70",
"region": "ASI",
"universe": "ILLIQUID_MINVOL1M"
},
{
"id": "risk72",
"region": "ASI",
"universe": "ILLIQUID_MINVOL1M"
},
{
"id": "risk59",
"region": "TWN",
"universe": "TOP500"
},
{
"id": "risk60",
"region": "TWN",
"universe": "TOP500"
},
{
"id": "risk68",
"region": "TWN",
"universe": "TOP500"
},
{
"id": "risk70",
"region": "TWN",
"universe": "TOP500"
},
{
"id": "risk71",
"region": "TWN",
"universe": "TOP500"
},
{
"id": "risk72",
"region": "TWN",
"universe": "TOP500"
},
{
"id": "risk88",
"region": "TWN",
"universe": "TOP500"
},
{
"id": "risk68",
"region": "IND",
"universe": "TOP500"
},
{
"id": "risk70",
"region": "IND",
"universe": "TOP500"
},
{
"id": "risk71",
"region": "IND",
"universe": "TOP500"
},
{
"id": "risk88",
"region": "IND",
"universe": "TOP500"
}
]

337
category_files/sentiment.json
Unescape View File

@ -0,0 +1,337 @@
[
{
"id": "other553",
"region": "USA",
"universe": "TOP3000"
},
{
"id": "sentiment1",
"region": "USA",
"universe": "TOP3000"
},
{
"id": "sentiment21",
"region": "USA",
"universe": "TOP3000"
},
{
"id": "sentiment22",
"region": "USA",
"universe": "TOP3000"
},
{
"id": "sentiment23",
"region": "USA",
"universe": "TOP3000"
},
{
"id": "sentiment26",
"region": "USA",
"universe": "TOP3000"
},
{
"id": "sentiment27",
"region": "USA",
"universe": "TOP3000"
},
{
"id": "sentiment7",
"region": "USA",
"universe": "TOP3000"
},
{
"id": "other553",
"region": "USA",
"universe": "TOP1000"
},
{
"id": "sentiment1",
"region": "USA",
"universe": "TOP1000"
},
{
"id": "sentiment21",
"region": "USA",
"universe": "TOP1000"
},
{
"id": "sentiment22",
"region": "USA",
"universe": "TOP1000"
},
{
"id": "sentiment23",
"region": "USA",
"universe": "TOP1000"
},
{
"id": "sentiment26",
"region": "USA",
"universe": "TOP1000"
},
{
"id": "sentiment27",
"region": "USA",
"universe": "TOP1000"
},
{
"id": "sentiment7",
"region": "USA",
"universe": "TOP1000"
},
{
"id": "other553",
"region": "USA",
"universe": "TOP500"
},
{
"id": "sentiment1",
"region": "USA",
"universe": "TOP500"
},
{
"id": "sentiment21",
"region": "USA",
"universe": "TOP500"
},
{
"id": "sentiment22",
"region": "USA",
"universe": "TOP500"
},
{
"id": "sentiment23",
"region": "USA",
"universe": "TOP500"
},
{
"id": "sentiment26",
"region": "USA",
"universe": "TOP500"
},
{
"id": "sentiment27",
"region": "USA",
"universe": "TOP500"
},
{
"id": "sentiment7",
"region": "USA",
"universe": "TOP500"
},
{
"id": "other553",
"region": "USA",
"universe": "TOP200"
},
{
"id": "sentiment1",
"region": "USA",
"universe": "TOP200"
},
{
"id": "sentiment21",
"region": "USA",
"universe": "TOP200"
},
{
"id": "sentiment22",
"region": "USA",
"universe": "TOP200"
},
{
"id": "sentiment23",
"region": "USA",
"universe": "TOP200"
},
{
"id": "sentiment26",
"region": "USA",
"universe": "TOP200"
},
{
"id": "sentiment27",
"region": "USA",
"universe": "TOP200"
},
{
"id": "sentiment7",
"region": "USA",
"universe": "TOP200"
},
{
"id": "other553",
"region": "USA",
"universe": "TOPSP500"
},
{
"id": "sentiment1",
"region": "USA",
"universe": "TOPSP500"
},
{
"id": "sentiment21",
"region": "USA",
"universe": "TOPSP500"
},
{
"id": "sentiment22",
"region": "USA",
"universe": "TOPSP500"
},
{
"id": "sentiment23",
"region": "USA",
"universe": "TOPSP500"
},
{
"id": "sentiment26",
"region": "USA",
"universe": "TOPSP500"
},
{
"id": "sentiment27",
"region": "USA",
"universe": "TOPSP500"
},
{
"id": "sentiment7",
"region": "USA",
"universe": "TOPSP500"
},
{
"id": "other553",
"region": "USA",
"universe": "ILLIQUID_MINVOL1M"
},
{
"id": "sentiment21",
"region": "USA",
"universe": "ILLIQUID_MINVOL1M"
},
{
"id": "sentiment22",
"region": "USA",
"universe": "ILLIQUID_MINVOL1M"
},
{
"id": "sentiment23",
"region": "USA",
"universe": "ILLIQUID_MINVOL1M"
},
{
"id": "sentiment26",
"region": "USA",
"universe": "ILLIQUID_MINVOL1M"
},
{
"id": "sentiment27",
"region": "USA",
"universe": "ILLIQUID_MINVOL1M"
},
{
"id": "other553",
"region": "GLB",
"universe": "TOP3000"
},
{
"id": "sentiment21",
"region": "GLB",
"universe": "TOP3000"
},
{
"id": "sentiment22",
"region": "GLB",
"universe": "TOP3000"
},
{
"id": "sentiment26",
"region": "GLB",
"universe": "TOP3000"
},
{
"id": "other553",
"region": "EUR",
"universe": "ILLIQUID_MINVOL1M"
},
{
"id": "sentiment21",
"region": "EUR",
"universe": "ILLIQUID_MINVOL1M"
},
{
"id": "sentiment26",
"region": "EUR",
"universe": "ILLIQUID_MINVOL1M"
},
{
"id": "sentiment27",
"region": "EUR",
"universe": "ILLIQUID_MINVOL1M"
},
{
"id": "sentiment7",
"region": "EUR",
"universe": "ILLIQUID_MINVOL1M"
},
{
"id": "other553",
"region": "ASI",
"universe": "ILLIQUID_MINVOL1M"
},
{
"id": "sentiment21",
"region": "ASI",
"universe": "ILLIQUID_MINVOL1M"
},
{
"id": "sentiment23",
"region": "ASI",
"universe": "ILLIQUID_MINVOL1M"
},
{
"id": "sentiment26",
"region": "ASI",
"universe": "ILLIQUID_MINVOL1M"
},
{
"id": "other553",
"region": "TWN",
"universe": "TOP500"
},
{
"id": "sentiment21",
"region": "TWN",
"universe": "TOP500"
},
{
"id": "sentiment23",
"region": "TWN",
"universe": "TOP500"
},
{
"id": "sentiment26",
"region": "TWN",
"universe": "TOP500"
},
{
"id": "sentiment27",
"region": "TWN",
"universe": "TOP500"
},
{
"id": "sentiment21",
"region": "IND",
"universe": "TOP500"
},
{
"id": "sentiment23",
"region": "IND",
"universe": "TOP500"
},
{
"id": "sentiment26",
"region": "IND",
"universe": "TOP500"
}
]

227
category_files/shortinterest.json
Unescape View File

@ -0,0 +1,227 @@
[
{
"id": "shortinterest10",
"region": "USA",
"universe": "TOP3000"
},
{
"id": "shortinterest2",
"region": "USA",
"universe": "TOP3000"
},
{
"id": "shortinterest24",
"region": "USA",
"universe": "TOP3000"
},
{
"id": "shortinterest29",
"region": "USA",
"universe": "TOP3000"
},
{
"id": "shortinterest3",
"region": "USA",
"universe": "TOP3000"
},
{
"id": "shortinterest36",
"region": "USA",
"universe": "TOP3000"
},
{
"id": "shortinterest43",
"region": "USA",
"universe": "TOP3000"
},
{
"id": "shortinterest10",
"region": "USA",
"universe": "TOP1000"
},
{
"id": "shortinterest2",
"region": "USA",
"universe": "TOP1000"
},
{
"id": "shortinterest24",
"region": "USA",
"universe": "TOP1000"
},
{
"id": "shortinterest29",
"region": "USA",
"universe": "TOP1000"
},
{
"id": "shortinterest3",
"region": "USA",
"universe": "TOP1000"
},
{
"id": "shortinterest36",
"region": "USA",
"universe": "TOP1000"
},
{
"id": "shortinterest43",
"region": "USA",
"universe": "TOP1000"
},
{
"id": "shortinterest10",
"region": "USA",
"universe": "TOP500"
},
{
"id": "shortinterest2",
"region": "USA",
"universe": "TOP500"
},
{
"id": "shortinterest24",
"region": "USA",
"universe": "TOP500"
},
{
"id": "shortinterest29",
"region": "USA",
"universe": "TOP500"
},
{
"id": "shortinterest3",
"region": "USA",
"universe": "TOP500"
},
{
"id": "shortinterest36",
"region": "USA",
"universe": "TOP500"
},
{
"id": "shortinterest43",
"region": "USA",
"universe": "TOP500"
},
{
"id": "shortinterest10",
"region": "USA",
"universe": "TOP200"
},
{
"id": "shortinterest2",
"region": "USA",
"universe": "TOP200"
},
{
"id": "shortinterest24",
"region": "USA",
"universe": "TOP200"
},
{
"id": "shortinterest29",
"region": "USA",
"universe": "TOP200"
},
{
"id": "shortinterest3",
"region": "USA",
"universe": "TOP200"
},
{
"id": "shortinterest36",
"region": "USA",
"universe": "TOP200"
},
{
"id": "shortinterest43",
"region": "USA",
"universe": "TOP200"
},
{
"id": "shortinterest10",
"region": "USA",
"universe": "TOPSP500"
},
{
"id": "shortinterest2",
"region": "USA",
"universe": "TOPSP500"
},
{
"id": "shortinterest24",
"region": "USA",
"universe": "TOPSP500"
},
{
"id": "shortinterest29",
"region": "USA",
"universe": "TOPSP500"
},
{
"id": "shortinterest3",
"region": "USA",
"universe": "TOPSP500"
},
{
"id": "shortinterest36",
"region": "USA",
"universe": "TOPSP500"
},
{
"id": "shortinterest43",
"region": "USA",
"universe": "TOPSP500"
},
{
"id": "shortinterest2",
"region": "USA",
"universe": "ILLIQUID_MINVOL1M"
},
{
"id": "shortinterest29",
"region": "USA",
"universe": "ILLIQUID_MINVOL1M"
},
{
"id": "shortinterest3",
"region": "USA",
"universe": "ILLIQUID_MINVOL1M"
},
{
"id": "shortinterest36",
"region": "USA",
"universe": "ILLIQUID_MINVOL1M"
},
{
"id": "shortinterest43",
"region": "USA",
"universe": "ILLIQUID_MINVOL1M"
},
{
"id": "shortinterest3",
"region": "EUR",
"universe": "ILLIQUID_MINVOL1M"
},
{
"id": "shortinterest5",
"region": "ASI",
"universe": "ILLIQUID_MINVOL1M"
},
{
"id": "shortinterest3",
"region": "TWN",
"universe": "TOP500"
},
{
"id": "shortinterest5",
"region": "TWN",
"universe": "TOP500"
},
{
"id": "shortinterest5",
"region": "IND",
"universe": "TOP500"
}
]

92
category_files/socialmedia.json
Unescape View File

@ -0,0 +1,92 @@
[
{
"id": "socialmedia12",
"region": "USA",
"universe": "TOP3000"
},
{
"id": "socialmedia8",
"region": "USA",
"universe": "TOP3000"
},
{
"id": "socialmedia12",
"region": "USA",
"universe": "TOP1000"
},
{
"id": "socialmedia8",
"region": "USA",
"universe": "TOP1000"
},
{
"id": "socialmedia12",
"region": "USA",
"universe": "TOP500"
},
{
"id": "socialmedia8",
"region": "USA",
"universe": "TOP500"
},
{
"id": "socialmedia12",
"region": "USA",
"universe": "TOP200"
},
{
"id": "socialmedia8",
"region": "USA",
"universe": "TOP200"
},
{
"id": "socialmedia12",
"region": "USA",
"universe": "TOPSP500"
},
{
"id": "socialmedia8",
"region": "USA",
"universe": "TOPSP500"
},
{
"id": "socialmedia12",
"region": "USA",
"universe": "ILLIQUID_MINVOL1M"
},
{
"id": "socialmedia8",
"region": "USA",
"universe": "ILLIQUID_MINVOL1M"
},
{
"id": "socialmedia12",
"region": "GLB",
"universe": "TOP3000"
},
{
"id": "socialmedia5",
"region": "GLB",
"universe": "TOP3000"
},
{
"id": "socialmedia12",
"region": "EUR",
"universe": "ILLIQUID_MINVOL1M"
},
{
"id": "socialmedia12",
"region": "ASI",
"universe": "ILLIQUID_MINVOL1M"
},
{
"id": "other100",
"region": "TWN",
"universe": "TOP500"
},
{
"id": "socialmedia39",
"region": "TWN",
"universe": "TOP500"
}
]

33
get_category.py
Unescape View File

@ -84,7 +84,26 @@ if __name__ == "__main__":
downloader = CategoryDownloader()
if downloader.client:
category = 'fundamental'
# category_list = [
# 'analyst',
# 'broker',
# 'earnings',
# 'fundamental',
# 'imbalance',
# 'insiders',
# 'institutions',
# 'macro',
# 'model',
# 'news',
# 'option',
# 'other',
# 'pv',
# 'risk',
# 'sentiment',
# 'shortinterest',
# 'socialmedia'
# ]
category = 'socialmedia'
delay = '1'
instrumentType = 'EQUITY'
@ -99,13 +118,13 @@ if __name__ == "__main__":
'IND'
]
universe_list = [
# 'TOP3000',
# 'TOP1000',
# 'TOP500',
# 'TOP200',
# 'TOPSP500',
'TOP3000',
'TOP1000',
'TOP500',
'TOP200',
'TOPSP500',
'ILLIQUID_MINVOL1M',
'MINVOL1M'
# 'MINVOL1M'
]
downloader.fetch_category_data(category, delay, instrumentType, region_list, universe_list)

60
main.py
Unescape View File

@ -7,6 +7,29 @@ import csv
import httpx
from httpx import BasicAuth
def read_category_json(filename):
# 构建文件路径
file_path = os.path.join('category_files', filename)
try:
# 读取并解析 JSON 文件
with open(file_path, 'r', encoding='utf-8') as f:
data = json.load(f)
print(f"成功读取文件：{file_path}")
return data
except FileNotFoundError:
print(f"错误：文件不存在 - {file_path}")
return None
except json.JSONDecodeError:
print(f"错误：JSON 格式错误 - {file_path}")
return None
except PermissionError:
print(f"错误：没有文件读取权限 - {file_path}")
return None
except Exception as e:
print(f"读取文件时发生未知错误：{e}")
return None
class DataSetDownloader:
def __init__(self):
@ -153,7 +176,7 @@ class DataSetDownloader:
print("🎯 到达数据末尾")
break
time.sleep(random.uniform(4, 6))
time.sleep(random.uniform(10, 15))
# 处理数据
print("🔄 处理数据中...")
@ -178,22 +201,31 @@ class DataSetDownloader:
if __name__ == "__main__":
# category_list = [
# 'analyst',
# 'broker',
# 'earnings',
# 'fundamental',
# 'imbalance',
# 'insiders',
# 'institutions',
# 'macro',
# 'model',
# 'news',
# 'option',
# 'other',
# 'pv',
# 'risk',
# 'sentiment',
# 'shortinterest',
# 'socialmedia'
# ]
plan_to_download = read_category_json('analyst.json')
downloader = DataSetDownloader()
if downloader.client:
plan_to_download = [
{
'id': 'analyst10',
'region': 'USA',
'universe': 'TOP3000'
},
{
'id': 'analyst10',
'region': 'GLB',
'universe': 'TOP3000'
}
]
for item in plan_to_download:
downloader.download_data_set(item['id'], item['region'], item['universe'])
time.sleep(random.uniform(20, 30))

45
new_db.py
Unescape View File

@ -0,0 +1,45 @@
import sqlite3
import os
def create_database():
db_path = os.path.join(os.getcwd(), 'data_sets.db')
conn = sqlite3.connect(db_path)
cursor = conn.cursor()
create_category_table(cursor)
create_datasets_table(cursor)
conn.commit()
conn.close()
def create_category_table(cursor):
cursor.execute('''
CREATE TABLE IF NOT EXISTS category (
name TEXT,
base_url TEXT,
category_id TEXT,
region TEXT,
universe TEXT,
delay TEXT,
downloaded TEXT
)
''')
def create_datasets_table(cursor):
cursor.execute('''
CREATE TABLE IF NOT EXISTS data_sets (
name TEXT,
description TEXT,
dataset_id TEXT,
dataset_name TEXT,
category_id TEXT,
category_name TEXT,
region TEXT,
delay TEXT,
universe TEXT,
type TEXT
)
''')
if __name__ == "__main__":
create_database()

1075
reference_fields/analyst10_usa_top3000.csv
View File

File diff suppressed because it is too large Load Diff
Write Preview
Loading…
Cancel
Save