Initial EMWeb API implementation
Add financial summary and quarterly endpoints with project configuration and documentation.
This commit is contained in:
@@ -0,0 +1,88 @@
|
||||
"""东方财富 CHOICE API 季度财务摘要抓取工具。"""
|
||||
from __future__ import annotations
|
||||
|
||||
import pandas as pd
|
||||
import requests
|
||||
|
||||
from utils.fin_summary import BASE_URL, HEADERS, TIMEOUT
|
||||
|
||||
|
||||
_REPORT_NAME = "RPT_HSF9_FIN_SUMMARYQUARTERNEW"
|
||||
_COLUMNS = "HSF9_FIN_SUMMARYQUARTER"
|
||||
|
||||
|
||||
def _build_filter(
|
||||
secucode: str,
|
||||
years: int,
|
||||
report_dates: list[str] | None,
|
||||
) -> str:
|
||||
if report_dates is not None:
|
||||
dates = ",".join(f"'{date}'" for date in report_dates)
|
||||
return f'(secuCode="{secucode}")(REPORTDATE in ({dates}))'
|
||||
|
||||
today = pd.Timestamp.today()
|
||||
start = (today - pd.DateOffset(years=years)).strftime("%Y-%m-%d")
|
||||
end = today.strftime("%Y-%m-%d")
|
||||
return (
|
||||
f'(secuCode="{secucode}")'
|
||||
f"((REPORTDATE>='{start}')(REPORTDATE<='{end}')"
|
||||
'(dateType in ("1","2","3","4","7"))(|IS_NEWEST="1"))'
|
||||
'(TYPE_CODE in ("1"))'
|
||||
)
|
||||
|
||||
|
||||
def _fetch_one(
|
||||
secucode: str,
|
||||
years: int,
|
||||
report_dates: list[str] | None,
|
||||
) -> pd.DataFrame:
|
||||
params = {
|
||||
"reportName": _REPORT_NAME,
|
||||
"columns": _COLUMNS,
|
||||
"filter": _build_filter(secucode, years, report_dates),
|
||||
"pageNumber": "1",
|
||||
"pageSize": "500",
|
||||
"sortTypes": "-1,1" if report_dates is None else "-1",
|
||||
"sortColumns": "REPORTDATE,TYPE" if report_dates is None else "REPORTDATE",
|
||||
"source": "CHOICE",
|
||||
"client": "SW",
|
||||
}
|
||||
response = requests.post(BASE_URL, headers=HEADERS, data=params, timeout=TIMEOUT)
|
||||
response.raise_for_status()
|
||||
payload = response.json()
|
||||
if not payload.get("success"):
|
||||
raise RuntimeError(f"{secucode}: 接口返回失败: {payload}")
|
||||
|
||||
rows = payload.get("result", {}).get("data") or []
|
||||
df = pd.DataFrame(rows)
|
||||
if not df.empty:
|
||||
df["SECUCODE"] = secucode
|
||||
return df
|
||||
|
||||
|
||||
def get_fin_quarterly_summary(
|
||||
secucodes: list[str],
|
||||
years: int = 3,
|
||||
report_dates: list[str] | None = None,
|
||||
) -> pd.DataFrame:
|
||||
"""抓取指定股票的季度财务摘要并按股票代码和报告期去重、倒序排列。"""
|
||||
codes = [code.strip() for code in secucodes if code.strip()]
|
||||
if not codes:
|
||||
return pd.DataFrame()
|
||||
|
||||
frames: list[pd.DataFrame] = []
|
||||
for code in codes:
|
||||
df = _fetch_one(code, years, report_dates)
|
||||
if not df.empty:
|
||||
frames.append(df)
|
||||
|
||||
if not frames:
|
||||
return pd.DataFrame()
|
||||
|
||||
result = pd.concat(frames, ignore_index=True)
|
||||
if {"SECUCODE", "REPORT_DATE"}.issubset(result.columns):
|
||||
result = result.drop_duplicates(["SECUCODE", "REPORT_DATE"], keep="first")
|
||||
result = result.sort_values(
|
||||
["SECUCODE", "REPORT_DATE"], ascending=[True, False]
|
||||
)
|
||||
return result.reset_index(drop=True)
|
||||
@@ -0,0 +1,358 @@
|
||||
"""
|
||||
东方财富 CHOICE API 财务摘要(报告期) 抓取工具
|
||||
- 输入: 股票代码列表 (如 ['603233.SH', '600519.SH'])
|
||||
- 输出: pandas DataFrame (合并报表 / 报告期 / 全部财务摘要指标)
|
||||
- 不需要登录, 不需要 cookie, 无文件落盘
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import time
|
||||
import random
|
||||
from typing import Iterable
|
||||
|
||||
import requests
|
||||
import pandas as pd
|
||||
from tqdm import tqdm
|
||||
|
||||
|
||||
# ---- SSL 自动修复 ----
|
||||
# 这台机器是游戏机, Python 环境 CA 证书不全, 跑 HTTPS 会挂. 脚本启动时自动检测,
|
||||
# 用系统/Reqable/Charles 装过的证书顶上, 不行就 fallback 关验证 (自用脚本, 数据非敏感).
|
||||
def _auto_fix_ssl() -> bool:
|
||||
"""尝试自动修复 SSL 证书验证, 返回是否已修复."""
|
||||
import os
|
||||
import ssl
|
||||
try:
|
||||
requests.get("https://www.baidu.com", timeout=5)
|
||||
return True # 没事, 直接返回
|
||||
except requests.exceptions.SSLError:
|
||||
pass
|
||||
|
||||
candidates = [
|
||||
# 1) Reqable 装的根证书 (用户刚装好)
|
||||
os.path.expandvars(r"%USERPROFILE%\.reqable\ca\reqable-ca.crt"),
|
||||
os.path.expandvars(r"%USERPROFILE%\AppData\Roaming\reqable\ca\reqable-ca.crt"),
|
||||
# 2) 系统证书
|
||||
r"C:\Windows\System32\curl-ca-bundle.crt",
|
||||
# 3) Charles
|
||||
os.path.expandvars(r"%USERPROFILE%\AppData\Roaming\Charles\ca-bundle.crt"),
|
||||
# 4) 系统 cert store
|
||||
]
|
||||
# 先看下环境变量 SSL_CERT_FILE
|
||||
env_cert = os.environ.get("SSL_CERT_FILE")
|
||||
if env_cert and os.path.exists(env_cert):
|
||||
candidates.insert(0, env_cert)
|
||||
|
||||
for cert in candidates:
|
||||
if cert and os.path.exists(cert):
|
||||
os.environ["REQUESTS_CA_BUNDLE"] = cert
|
||||
os.environ["SSL_CERT_FILE"] = cert
|
||||
try:
|
||||
requests.get("https://www.baidu.com", timeout=5, verify=cert)
|
||||
print(f"[SSL] 使用证书: {cert}")
|
||||
return True
|
||||
except Exception:
|
||||
continue
|
||||
|
||||
# 实在不行, 关掉验证 (仅本脚本内)
|
||||
print("[SSL] 没找到可信证书, 已临时关闭验证 (仅本脚本)")
|
||||
import urllib3
|
||||
urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)
|
||||
requests.Session().verify = False
|
||||
# patch 掉所有新 Session
|
||||
_orig_init = requests.Session.__init__
|
||||
|
||||
def _patched_init(self, *a, **kw):
|
||||
_orig_init(self, *a, **kw)
|
||||
self.verify = False
|
||||
|
||||
requests.Session.__init__ = _patched_init # type: ignore[assignment]
|
||||
return False
|
||||
|
||||
|
||||
_AUTO_FIX_SSL = _auto_fix_ssl()
|
||||
|
||||
|
||||
# ---- 常量 ----
|
||||
BASE_URL = "https://datacenter-choice.eastmoney.com/api/data/v1/get"
|
||||
REFERER = "https://emchoicew.eastmoney.com/"
|
||||
SECURITY_LIST_URL = "https://datacenter.eastmoney.com/securities/api/data/v1/get"
|
||||
TIMEOUT = 15
|
||||
SLEEP_RANGE = (0.4, 1.0) # 单次请求间隔, 礼貌抓取
|
||||
|
||||
HEADERS = {
|
||||
"User-Agent": (
|
||||
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) "
|
||||
"AppleWebKit/537.36 (KHTML, like Gecko) "
|
||||
"Chrome/124.0.0.0 Safari/537.36"
|
||||
),
|
||||
"Accept": "application/json",
|
||||
"Accept-Language": "zh-CN,zh;q=0.9",
|
||||
"Origin": "https://emchoicew.eastmoney.com",
|
||||
"Referer": REFERER,
|
||||
"sec-ch-ua": '"Chromium";v="124", "Not-A.Brand";v="99"',
|
||||
"sec-ch-ua-mobile": "?0",
|
||||
"sec-ch-ua-platform": '"Windows"',
|
||||
}
|
||||
|
||||
|
||||
# ---- 1. 股票代码 -> ORG_CODE 映射 (可选) ----
|
||||
_CONFIRMED_ORG_CODES = {
|
||||
"603233.SH": "10500736",
|
||||
}
|
||||
|
||||
|
||||
def _fetch_org_codes(secucodes: Iterable[str]) -> dict[str, str]:
|
||||
"""通过东财搜索接口批量拿到 ORG_CODE. 不保证成功 (接口路径不稳定).
|
||||
支持 '603233.SH' / 'sh603233' / '603233' 几种写法.
|
||||
"""
|
||||
secucodes = list(secucodes)
|
||||
norm_map: dict[str, str] = {}
|
||||
queries: list[str] = []
|
||||
|
||||
for code in secucodes:
|
||||
c = code.strip()
|
||||
if "." in c:
|
||||
num = c.split(".")[0]
|
||||
elif len(c) == 6 and c.isdigit():
|
||||
num = c
|
||||
elif len(c) >= 6:
|
||||
num = c[-6:]
|
||||
else:
|
||||
raise ValueError(f"无法识别股票代码: {code}")
|
||||
norm_map[num] = code
|
||||
if c in _CONFIRMED_ORG_CODES:
|
||||
continue
|
||||
queries.append(num)
|
||||
|
||||
out: dict[str, str] = {
|
||||
code: _CONFIRMED_ORG_CODES[code.strip()]
|
||||
for code in secucodes
|
||||
if code.strip() in _CONFIRMED_ORG_CODES
|
||||
}
|
||||
for num in queries:
|
||||
try:
|
||||
r = requests.get(
|
||||
"https://searchapi.eastmoney.com/api/suggest/get",
|
||||
params={
|
||||
"input": num,
|
||||
"type": "14",
|
||||
"token": "D43BF722C8E33BDC906FB84D85E326E8",
|
||||
"count": "5",
|
||||
},
|
||||
headers=HEADERS,
|
||||
timeout=TIMEOUT,
|
||||
)
|
||||
r.raise_for_status()
|
||||
data = r.json().get("QuotationCodeTable", {}).get("Data") or []
|
||||
if not data:
|
||||
continue
|
||||
# 注意: 这个接口返回的是 InnerCode, 不是 CHOICE 接口要的 ORG_CODE.
|
||||
# ORG_CODE 需要走 CHOICE 接口, 但其报表名没文档化. 这里把 InnerCode
|
||||
# 作为 fallback 字段返回, 调用方如果已经知道 ORG_CODE, 直接传 org_map.
|
||||
out[norm_map[num]] = data[0].get("InnerCode", "")
|
||||
except Exception: # noqa: BLE001
|
||||
continue
|
||||
|
||||
missing = [c for c in secucodes if c not in out]
|
||||
if missing:
|
||||
raise ValueError(f"以下股票代码基础信息查不到 (请手动提供 ORG_CODE): {missing}")
|
||||
return out
|
||||
|
||||
|
||||
# ---- 2. 拉取单只股票财务摘要 ----
|
||||
def _fetch_one(
|
||||
secucode: str,
|
||||
org_code: str,
|
||||
date_type_codes: str = "6", # 6=年报, 1=一季报, 5=三季报, 2=中报
|
||||
years: int = 10,
|
||||
is_newest: bool = True,
|
||||
) -> pd.DataFrame:
|
||||
"""拉一只股票的财务摘要."""
|
||||
# 日期范围: 当前年 - years 到今天
|
||||
today = pd.Timestamp.today()
|
||||
start = f"{today.year - years}-01-01"
|
||||
end = today.strftime("%Y-%m-%d")
|
||||
|
||||
type_codes_clause = f"(DATE_TYPE_CODE in ({date_type_codes}))"
|
||||
is_newest_clause = '(IS_NEWEST="1")' if is_newest else ""
|
||||
|
||||
filter_ = (
|
||||
f'(ORG_CODE="{org_code}")'
|
||||
f'(TYPE_CODE in ("2"))'
|
||||
f'((DATE_TYPE_NEW in ("1"))'
|
||||
f'((REPORT_DATE>=\'{start}\')(REPORT_DATE<=\'{end}\'){type_codes_clause})'
|
||||
f'{is_newest_clause})'
|
||||
)
|
||||
|
||||
# #region debug-point A:input-filter
|
||||
def _debug_report(hypothesis_id: str, msg: str, data: dict) -> None:
|
||||
try:
|
||||
import json
|
||||
import urllib.request
|
||||
|
||||
request = urllib.request.Request(
|
||||
"http://127.0.0.1:7777/event",
|
||||
data=json.dumps({
|
||||
"sessionId": "fin-summary-empty",
|
||||
"runId": "pre-fix",
|
||||
"hypothesisId": hypothesis_id,
|
||||
"location": "fin_summary.py:_fetch_one",
|
||||
"msg": f"[DEBUG] {msg}",
|
||||
"data": data,
|
||||
"ts": int(time.time() * 1000),
|
||||
}).encode(),
|
||||
headers={"Content-Type": "application/json"},
|
||||
)
|
||||
urllib.request.urlopen(request, timeout=1).close()
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
_debug_report("A", "_fetch_one input and filter constructed", {
|
||||
"secucode": secucode,
|
||||
"org_code": org_code,
|
||||
"date_type_codes": date_type_codes,
|
||||
"years": years,
|
||||
"is_newest": is_newest,
|
||||
"filter": filter_,
|
||||
})
|
||||
# #endregion
|
||||
|
||||
params = {
|
||||
"reportName": "RPT_CUSTOM_ORGF9_FIN_SUMMARY",
|
||||
"columns": "FIN_INDICATOR1",
|
||||
"filter": filter_,
|
||||
"pageNumber": "1",
|
||||
"pageSize": "500",
|
||||
"sortTypes": "-1,-1,1",
|
||||
"sortColumns": "STD_REPORT_DATE,TYPE_CODE,DATE_TYPE_NEW",
|
||||
"source": "CHOICE",
|
||||
"client": "SW",
|
||||
}
|
||||
|
||||
r = requests.get(BASE_URL, headers=HEADERS, params=params, timeout=TIMEOUT)
|
||||
r.raise_for_status()
|
||||
payload = r.json()
|
||||
|
||||
# #region debug-point B:http-response
|
||||
_debug_report("B", "_fetch_one HTTP response", {
|
||||
"status": r.status_code,
|
||||
"success": payload.get("success"),
|
||||
"message": payload.get("message"),
|
||||
"result_count": (payload.get("result") or {}).get("count"),
|
||||
"data_rows": (payload.get("result") or {}).get("data") or [],
|
||||
})
|
||||
# #endregion
|
||||
|
||||
if not payload.get("success"):
|
||||
raise RuntimeError(f"{secucode}: 接口返回失败: {payload}")
|
||||
|
||||
rows = payload.get("result", {}).get("data") or []
|
||||
df = pd.DataFrame(rows)
|
||||
if df.empty:
|
||||
return df
|
||||
df["SECUCODE"] = secucode # 冗余一列方便合并
|
||||
return df
|
||||
|
||||
|
||||
# ---- 3. 主入口 ----
|
||||
def get_fin_summary(
|
||||
secucodes: list[str] | dict[str, str],
|
||||
years: int = 10,
|
||||
date_type_codes: str = "6",
|
||||
is_newest: bool = True,
|
||||
verbose: bool = True,
|
||||
) -> pd.DataFrame:
|
||||
"""批量拉取财务摘要, 返回单个合并 DataFrame.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
secucodes : list[str] | dict[str, str]
|
||||
两种用法:
|
||||
1) list[str]: 只传股票代码 (如 ['603233.SH', '600519.SH']), 自动查询.
|
||||
2) dict[str, str]: {股票代码: ORG_CODE}, 跳过查询, 推荐 (最稳).
|
||||
years : int
|
||||
拉取多少年的历史 (默认 10 年).
|
||||
date_type_codes : str
|
||||
报告期类型, 逗号分隔. 该接口支持的代码:
|
||||
'1'=一季报, '5'=中报, '6'=年报. (三季报不在此接口)
|
||||
默认 '6' 只取年报; 想全要就传 '1,5,6'.
|
||||
is_newest : bool
|
||||
是否只要最新合并报表 (默认 True).
|
||||
|
||||
Returns
|
||||
-------
|
||||
pd.DataFrame
|
||||
合并后的财务摘要数据, 列以原始 API 返回为准.
|
||||
"""
|
||||
if isinstance(secucodes, dict):
|
||||
org_map = dict(secucodes)
|
||||
codes = list(secucodes.keys())
|
||||
else:
|
||||
codes = list(secucodes)
|
||||
org_map = None
|
||||
|
||||
if not codes:
|
||||
return pd.DataFrame()
|
||||
|
||||
# 1) 拿 ORG_CODE (如果没传)
|
||||
if org_map is None:
|
||||
try:
|
||||
org_map = _fetch_org_codes(codes)
|
||||
except ValueError:
|
||||
if verbose:
|
||||
print(
|
||||
"[HINT] 自动查询 ORG_CODE 失败. 请改用 dict 方式传入:\n"
|
||||
" get_fin_summary({'603233.SH': '10500736', ...})\n"
|
||||
" ORG_CODE 可以在 Reqable 里抓包切换股票时拿到."
|
||||
)
|
||||
return pd.DataFrame()
|
||||
|
||||
# 2) 逐只抓
|
||||
frames: list[pd.DataFrame] = []
|
||||
iterator = tqdm(codes, desc="抓取财务摘要") if verbose else codes
|
||||
for code in iterator:
|
||||
org = org_map.get(code)
|
||||
if not org:
|
||||
if verbose:
|
||||
print(f"[WARN] {code} 缺少 ORG_CODE, 跳过.")
|
||||
continue
|
||||
try:
|
||||
df = _fetch_one(
|
||||
code, org,
|
||||
date_type_codes=date_type_codes,
|
||||
years=years,
|
||||
is_newest=is_newest,
|
||||
)
|
||||
if not df.empty:
|
||||
frames.append(df)
|
||||
except Exception as e: # noqa: BLE001
|
||||
if verbose:
|
||||
print(f"[WARN] {code} 抓取失败: {e}")
|
||||
# 礼貌延时
|
||||
time.sleep(random.uniform(*SLEEP_RANGE))
|
||||
|
||||
if not frames:
|
||||
return pd.DataFrame()
|
||||
|
||||
out = pd.concat(frames, ignore_index=True)
|
||||
return out
|
||||
|
||||
|
||||
# ---- 直接运行示例 ----
|
||||
if __name__ == "__main__":
|
||||
# 演示: 拉这几只的财务摘要
|
||||
# 推荐直接传 ORG_CODE (在 Reqable 里抓包切换股票时能看到, 比如 603233 -> 10500736)
|
||||
sample = {
|
||||
"603233.SH": "10500736", # 大参林
|
||||
# "600519.SH": "...", # 贵州茅台
|
||||
# "000001.SZ": "...", # 平安银行
|
||||
}
|
||||
df = get_fin_summary(sample, years=5, date_type_codes="1,5,6")
|
||||
print(f"共 {len(df)} 行, {len(df.columns)} 列")
|
||||
if not df.empty:
|
||||
cols = ["SECUCODE", "REPORT_DATE", "DATE_TYPE", "OPERATE_INCOME",
|
||||
"NETPROFIT", "DILUTED_ROE", "SALE_GPR"]
|
||||
cols = [c for c in cols if c in df.columns]
|
||||
print(df[cols].to_string())
|
||||
Reference in New Issue
Block a user