toor/mirosociety
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

fix simulation reporting and research resilience

Browse Source
This commit is contained in:
toor
2026-09-28 15:43:23 +08:00
parent 23297be2d7
commit a428214c53
25 changed files with 1488 additions and 244 deletions
Download Patch File Download Diff File Expand all files Collapse all files
backend/app/services/narrator.py
+250 -61
View File
@@ -12,7 +12,9 @@ from app.constants import TIMES_OF_DAY
if TYPE_CHECKING:
from app.db.store import SimulationStore
logger = logging.getLogger(__name__)
logger = logging.getLogger(__name__)
REPORT_SCHEMA_VERSION = 2
NARRATE_SYSTEM = """You are a dry, observational narrator documenting life in {world_name}.
Rules of this world: {rules}
@@ -51,7 +53,11 @@ Given structured metric stats, epoch summaries, and agent data, produce:
1. Per-metric explanation: One sentence explaining WHY each metric reached its final value. Reference specific events or behaviors.
2. Executive verdict: Should the user proceed (go), proceed with caution (caution), or fundamentally rethink (rethink)?
3. Confidence assessment: Based on simulation length and agent count.
4. 3-5 emergent insights: Things that were NOT predictable from the simulation setup alone. Each must reference specific agents, days, or action patterns as evidence.
4. 3-5 emergent insights: Things that were NOT predictable from the simulation setup alone. Each must reference specific agents, days, or action patterns as evidence.
An insight without at least one concrete agent, day, action, or metric value is
not an insight. Omit it rather than guessing. Do not present a causal claim
when the supplied data only shows correlation.
Return JSON:
{{
@@ -102,7 +108,7 @@ Return JSON:
]
}}"""
ACTION_SYNTHESIS_SYSTEM = """You are a strategy consultant synthesizing findings from a simulation of {world_name}.
ACTION_SYNTHESIS_SYSTEM = """You are a strategy consultant synthesizing findings from a simulation of {world_name}.
Rules: {rules}
Given the executive brief, metric interpretations, insights, and segment analysis, produce:
@@ -124,18 +130,77 @@ Return JSON:
"risks": [
{{"risk": "Description", "severity": "high" | "medium" | "low"}}
],
"second_order_effects": ["Effect description"]
}}"""
class Narrator:
"second_order_effects": ["Effect description"]
}}"""
SYSTEMATIC_SUMMARY_SYSTEM = """You are the senior historian and audit writer for {world_name}.
Rules of this world: {rules}
Write a traceable summary of the completed simulation. The supplied material
is the complete evidence available; do not invent events, motivations, people,
or metric changes that are not present in it.
Return JSON with exactly these fields:
{{
"summary": "A concise 2-4 paragraph account of the main story",
"chapters": [
{{"title": "time period", "body": "what happened, why it mattered", "evidence_days": [1, 2]}}
],
"key_figures": [
{{"name": "existing agent name", "arc": "choice and consequence", "evidence_days": [1, 2]}}
],
"contradictions": [
{{"surface": "observable reading", "underlying": "supported interpretation", "evidence": "specific metric/action/day"}}
]
}}
Requirements:
- Every claim must cite a supplied metric value, existing agent name, action, or day.
- Distinguish observed facts from interpretation.
- If the data does not answer a question, say that it is not covered.
- Keep the prose concrete and avoid atmospheric filler."""
class Narrator:
def __init__(self, llm: LLMClient, forecast=None, locale: str = DEFAULT_LOCALE):
self.llm = llm
self._forecast = forecast
self._locale = locale
def set_locale(self, locale: str) -> None:
self._locale = locale
def set_locale(self, locale: str) -> None:
self._locale = locale
async def _generate_report_json(
self,
*,
system: str,
user: str,
max_tokens: int,
purpose: str,
degraded_sections: list[str],
) -> dict:
"""Run an optional report pass without making it a single point of failure.
A report is a post-processing artifact. Provider policy errors (for
example MiniMax 422 content screening) or malformed JSON should
degrade one section, not discard an otherwise complete simulation.
"""
try:
response = await self.llm.generate(
system=system,
user=user,
json_mode=True,
max_tokens=max_tokens,
purpose=purpose,
)
parsed = parse_json(response, purpose=purpose)
if isinstance(parsed, dict) and parsed:
return parsed
except Exception as exc:
logger.warning("报告子分析失败，将使用确定性结果（用途=%s）：%s", purpose, exc)
if purpose not in degraded_sections:
degraded_sections.append(purpose)
return {}
async def narrate_round(
self,
@@ -273,13 +338,28 @@ class Narrator:
# event loop — we don't need an extra gather limit. Running 12 epochs
# serially could take 4+ minutes for a 365-day sim; concurrent runs
# collapse this to ~30s (the wall time of the slowest call).
epoch_summaries = await asyncio.gather(*(
self.summarize_epoch(
blueprint.name, blueprint.rules, ep["narrs"],
ep["m_start"], ep["m_end"], ep["start_day"], ep["end_day"],
)
for ep in pending_epochs
))
epoch_results = await asyncio.gather(*(
self.summarize_epoch(
blueprint.name, blueprint.rules, ep["narrs"],
ep["m_start"], ep["m_end"], ep["start_day"], ep["end_day"],
)
for ep in pending_epochs
), return_exceptions=True)
degraded_sections: list[str] = []
epoch_summaries = []
for ep, result in zip(pending_epochs, epoch_results):
if isinstance(result, Exception):
section = f"报告时期总结 Day {ep['start_day']}-{ep['end_day']}"
degraded_sections.append(section)
logger.warning("%s失败，将使用指标摘要：%s", section, result)
result = (
f"Days {ep['start_day']}-{ep['end_day']}: "
"LLM summary unavailable; the period is represented by the recorded metrics. "
f"Stability {ep['m_start'].stability:.2f}->{ep['m_end'].stability:.2f}, "
f"trust {ep['m_start'].trust:.2f}->{ep['m_end'].trust:.2f}, "
f"conflict {ep['m_start'].conflict:.2f}->{ep['m_end'].conflict:.2f}."
)
epoch_summaries.append(result)
epochs = [
{"days": f"{ep['start_day']}-{ep['end_day']}", "summary": summary}
for ep, summary in zip(pending_epochs, epoch_summaries)
@@ -291,10 +371,20 @@ class Narrator:
from app.services.report_analyzer import ReportAnalyzer
metric_stats = ReportAnalyzer.compute_metric_stats(metrics_history, is_market)
agent_stats = ReportAnalyzer.compute_agent_stats(agents, actions)
segment_stats = ReportAnalyzer.compute_segment_stats(agent_stats)
inflection_points = ReportAnalyzer.detect_inflection_points(metrics_history)
confidence = ReportAnalyzer.compute_confidence(metrics_history, len(agents))
agent_stats = ReportAnalyzer.compute_agent_stats(agents, actions)
segment_stats = ReportAnalyzer.compute_segment_stats(agent_stats)
inflection_points = ReportAnalyzer.detect_inflection_points(metrics_history)
confidence = ReportAnalyzer.compute_confidence(metrics_history, len(agents))
tension_event_count = int(await store.get_meta(simulation_id, "tension_event_count") or 0)
tension_event_records = await store.get_meta(simulation_id, "tension_event_records") or []
fallback_count = int(await store.get_meta(simulation_id, "citizen_generation_fallback_count") or 0)
data_gaps = ReportAnalyzer.detect_data_gaps(
metrics_history,
actions,
is_market,
tension_event_count=tension_event_count,
fallback_count=fallback_count,
)
# --- Pass 1b: Darts-based analysis ---
forecast_analysis = {}
@@ -363,10 +453,17 @@ class Narrator:
+ (f"\n\nCausal analysis:\n{causal_summary}" if causal_summary else "")
)
response_p2 = await self.llm.generate(system=system_p2, user=user_p2, json_mode=True, max_tokens=2000, purpose="报告指标分析")
pass2 = parse_json(response_p2, purpose="报告指标分析")
explanations = pass2.get("metric_explanations", {})
pass2 = await self._generate_report_json(
system=system_p2,
user=user_p2,
max_tokens=2000,
purpose="报告指标分析",
degraded_sections=degraded_sections,
)
explanations = pass2.get("metric_explanations", {})
if not isinstance(explanations, dict):
explanations = {}
scorecard = []
for s in metric_stats:
scorecard.append({
@@ -397,26 +494,53 @@ class Narrator:
) + f"\n\n{tail}"
user_p3 = f"Segment data:\n{segment_input}"
response_p3 = await self.llm.generate(system=system_p3, user=user_p3, json_mode=True, max_tokens=1500, purpose="报告分群分析")
pass3 = parse_json(response_p3, purpose="报告分群分析")
llm_segments = {s["name"]: s for s in pass3.get("segments", [])}
segments_final = []
for seg in segment_stats:
llm_seg = llm_segments.get(seg["name"], {})
segments_final.append({
"name": seg["name"],
"adoption_pct": seg["adoption_pct"],
"funnel": seg["funnel"],
"reaction": llm_seg.get("reaction", ""),
"top_objection": llm_seg.get("top_objection", ""),
"champion": llm_seg.get("champion", {"name": "", "why": ""}),
"representative_quote": llm_seg.get("representative_quote", ""),
})
# --- Pass 4: Action synthesis ---
executive_brief = pass2.get("executive_brief", {})
insights = pass2.get("insights", [])
pass3 = await self._generate_report_json(
system=system_p3,
user=user_p3,
max_tokens=1500,
purpose="报告分群分析",
degraded_sections=degraded_sections,
)
llm_segments = {
s["name"]: s for s in pass3.get("segments", [])
if isinstance(s, dict) and s.get("name")
}
segments_final = []
for seg in segment_stats:
llm_seg = llm_segments.get(seg["name"], {})
if llm_seg:
reaction = llm_seg.get("reaction", "")
top_objection = llm_seg.get("top_objection", "")
champion = llm_seg.get("champion", {"name": "", "why": ""})
representative_quote = llm_seg.get("representative_quote", "")
else:
reaction = (
f"结构化数据记录该分群最终采用率为 {seg['adoption_pct']}%，"
f"倡导者 {seg['advocate_count']} 人；未生成可靠的语义层解释。"
)
top_objection = "未能从结构化数据可靠识别主要异议。"
champion = {"name": "", "why": "未生成分群语义分析。"}
representative_quote = ""
segments_final.append({
"name": seg["name"],
"adoption_pct": seg["adoption_pct"],
"funnel": seg["funnel"],
"reaction": reaction,
"top_objection": top_objection,
"champion": champion,
"representative_quote": representative_quote,
})
# --- Pass 4: Action synthesis ---
executive_brief_raw = pass2.get("executive_brief", {})
executive_brief = executive_brief_raw if isinstance(executive_brief_raw, dict) else {}
insights = [
insight for insight in pass2.get("insights", [])
if isinstance(insight, dict)
and isinstance(insight.get("evidence"), dict)
and any(insight["evidence"].get(key) for key in ("agents", "days", "actions", "metrics"))
]
synthesis_context = (
f"Executive verdict: {executive_brief.get('verdict', 'unknown')} — {executive_brief.get('headline', '')}\n\n"
@@ -437,10 +561,68 @@ class Narrator:
rules="; ".join(blueprint.rules),
) + f"\n\n{tail}"
response_p4 = await self.llm.generate(system=system_p4, user=synthesis_context, json_mode=True, max_tokens=1500, purpose="报告行动建议")
pass4 = parse_json(response_p4, purpose="报告行动建议")
# --- Assemble final report ---
pass4 = await self._generate_report_json(
system=system_p4,
user=synthesis_context,
max_tokens=1500,
purpose="报告行动建议",
degraded_sections=degraded_sections,
)
# --- Pass 5: Systematic, evidence-linked summary ---
top_agents = sorted(agent_stats, key=lambda item: item["total_actions"], reverse=True)[:8]
systematic_context = (
f"Timeline epochs:\n{epoch_text}\n\n"
f"Metric scorecard:\n{metrics_summary}\n\n"
f"Inflection points:\n{inflection_summary or '(none)'}\n\n"
f"Top agents:\n" + "\n".join(
f"- {a['name']} ({a['role']}): {a['total_actions']} actions; "
f"speeches={a['speeches'][-2:]}; thoughts={a['thoughts'][-2:]}"
for a in top_agents
) + "\n\n"
f"Segments:\n" + "\n".join(
f"- {s['name']}: {s['adoption_pct']}% adoption; reaction={s.get('reaction', '')}; "
f"objection={s.get('top_objection', '')}"
for s in segments_final
) + "\n\n"
f"Insights:\n" + "\n".join(
f"- {i.get('title', '')}: {i.get('description', '')}; evidence={i.get('evidence', {})}"
for i in insights
) + "\n\n"
f"Risks and actions:\n{pass4.get('risks', [])}\n{pass4.get('action_items', [])}\n\n"
f"Tension events:\n{tension_event_records}\n\n"
f"Deterministic data gaps:\n{data_gaps}\n\n"
f"Simulation metadata: days={total_days}, agents={len(agents)}, actions={len(actions)}, "
f"tension_events={tension_event_count}, fallback_citizens={fallback_count}"
)
head, tail = _language_directive(self._locale)
system_p5 = head + "\n\n" + SYSTEMATIC_SUMMARY_SYSTEM.format(
world_name=blueprint.name,
rules="; ".join(blueprint.rules),
) + f"\n\n{tail}"
systematic_summary = {
"summary": executive_brief.get("summary", ""),
"chapters": [],
"key_figures": [],
"contradictions": [],
}
try:
response_p5 = await self.llm.generate(
system=system_p5,
user=systematic_context,
json_mode=True,
max_tokens=2200,
purpose="报告系统性总结",
)
pass5_raw = parse_json(response_p5, purpose="报告系统性总结")
if isinstance(pass5_raw, dict):
for key in ("summary", "chapters", "key_figures", "contradictions"):
if key in pass5_raw:
systematic_summary[key] = pass5_raw[key]
except Exception as e:
logger.warning("报告系统性总结生成失败，将使用已有摘要：%s", e)
# --- Assemble final report ---
key_moments = []
for p in inflection_points[:10]:
key_moments.append({
@@ -449,28 +631,35 @@ class Narrator:
"description": f"{p['label']} shifted from {p['value_before']:.2f} to {p['value_after']:.2f}",
})
report = {
"executive_brief": executive_brief,
report = {
"report_schema_version": REPORT_SCHEMA_VERSION,
"executive_brief": executive_brief,
"scorecard": scorecard,
"insights": insights,
"segments": segments_final,
"action_items": pass4.get("action_items", []),
"risks": pass4.get("risks", []),
"second_order_effects": pass4.get("second_order_effects", []),
"narrative": {
"summary": executive_brief.get("summary", ""),
"action_items": pass4.get("action_items", []),
"risks": pass4.get("risks", []),
"second_order_effects": pass4.get("second_order_effects", []),
"systematic_summary": systematic_summary,
"data_gaps": data_gaps,
"tension_events": tension_event_records,
"narrative": {
"summary": systematic_summary.get("summary") or executive_brief.get("summary", ""),
"key_moments": key_moments,
"surprise": next((i["description"] for i in insights if i.get("type") == "surprise"), ""),
"factions": [],
},
"metrics_history": metrics_history,
"meta": {
"meta": {
"agent_count": len(agents),
"total_days": total_days,
"total_actions": len(actions),
"rules": blueprint.rules,
"world_name": blueprint.name,
},
"total_actions": len(actions),
"fallback_citizens": fallback_count,
"rules": blueprint.rules,
"world_name": blueprint.name,
"locale": self._locale,
"degraded_sections": degraded_sections,
},
"forecasts": forecast_analysis,
"causal_map": causal_map,
"counterfactuals": counterfactual_data,