fix simulation reporting and research resilience

2026-09-28 15:43:23 +08:00
parent 23297be2d7
commit a428214c53
25 changed files with 1488 additions and 244 deletions
@@ -17,12 +17,16 @@ METRIC_LABELS = {
    "purchase_intent": "Purchase Intent",
    "word_of_mouth": "Word of Mouth",
    "churn_risk": "Churn Risk",
-    "adoption_rate": "Adoption Rate",
-}
+    "adoption_rate": "Adoption Rate",
+    "information_spread": "Information Spread",
+    "echo_chamber_index": "Echo Chamber Index",
+    "rumor_distortion": "Rumor Distortion",
+}

 INVERSE_METRICS = {"conflict", "churn_risk"}

-MARKET_METRICS = {"brand_sentiment", "purchase_intent", "word_of_mouth", "churn_risk", "adoption_rate"}
+MARKET_METRICS = {"brand_sentiment", "purchase_intent", "word_of_mouth", "churn_risk", "adoption_rate"}
+INFORMATION_METRICS = {"information_spread", "echo_chamber_index", "rumor_distortion"}

 SOCIAL_METRICS = {"stability", "prosperity", "trust", "freedom", "conflict"}

@@ -56,9 +60,11 @@ class ReportAnalyzer:
        if not metrics_history:
            return []

-        active_keys = set(SOCIAL_METRICS)
-        if is_market:
-            active_keys |= MARKET_METRICS
+        # Metrics are persisted as a complete WorldMetrics snapshot. Keep the
+        # report scorecard complete as well; hiding market or information
+        # fields based on the scenario made readers mistake omitted fields for
+        # missing data.
+        active_keys = set(METRIC_LABELS)

        results = []
        for key in active_keys:
@@ -86,7 +92,91 @@ class ReportAnalyzer:
                "inverse": inverse,
            })

-        return sorted(results, key=lambda x: x["metric"])
+        return sorted(results, key=lambda x: x["metric"])
+
+    @staticmethod
+    def detect_data_gaps(
+        metrics_history: list[dict],
+        actions: list[ActionEntry],
+        is_market: bool,
+        tension_event_count: int = 0,
+        fallback_count: int = 0,
+    ) -> list[dict]:
+        """Find deterministic limitations worth surfacing in a report.
+
+        These are observations about this run, not claims that the simulator
+        is universally incapable of producing the signal. LLM prose can then
+        explain the finding without inventing a missing data source.
+        """
+        gaps: list[dict] = []
+        if not metrics_history:
+            return [{
+                "key": "metrics_history",
+                "gap": "No metric history was recorded.",
+                "why_it_matters": "The report cannot establish trends or inflection points.",
+                "suggested_fix": "Check persistence and rerun the simulation.",
+            }]
+
+        action_types = {a.action_type.value for a in actions}
+        values_by_key = {
+            key: [float(row.get(key, 0.5)) for row in metrics_history]
+            for key in METRIC_LABELS
+        }
+
+        def add(key: str, gap: str, why: str, fix: str) -> None:
+            gaps.append({"key": key, "gap": gap, "why_it_matters": why, "suggested_fix": fix})
+
+        if fallback_count > 0:
+            add(
+                "citizen_generation",
+                f"{fallback_count} citizen(s) entered through degraded generation.",
+                "Their initial identity fields may be less representative than normally generated citizens.",
+                "Retry cast generation and inspect the affected residents before treating their behavior as evidence.",
+            )
+
+        if is_market and all(abs(v) < 1e-9 for v in values_by_key["adoption_rate"]):
+            add(
+                "adoption_rate",
+                "Adoption rate stayed at 0.00 for the whole run.",
+                "The report cannot infer an adoption trajectory from this simulation.",
+                "Increase the horizon or verify that purchase/proposal actions can occur.",
+            )
+        if "DEFECT" not in action_types and "ABANDON" not in action_types:
+            add(
+                "defection_or_abandonment",
+                "No DEFECT or ABANDON action occurred.",
+                "Churn, defection, and trust-loss conclusions are weakly evidenced.",
+                "Run a longer or more disruptive scenario, or report this as a limitation.",
+            )
+        if "PROPOSE_RULE" not in action_types and not is_market:
+            add(
+                "rule_evolution",
+                "No PROPOSE_RULE action occurred.",
+                "Institutional or rule-evolution claims are not supported by observed actions.",
+                "Include a rule-change pressure or extend the simulation horizon.",
+            )
+        if tension_event_count > 0:
+            add(
+                "tension_events",
+                f"{tension_event_count} tension event(s) occurred but are not represented as a first-class report section.",
+                "Readers may miss event-driven causes of later metric changes.",
+                "Include event descriptions and affected days in the report timeline.",
+            )
+
+        flat_keys = ["trust", "conflict", "information_spread"]
+        if is_market:
+            flat_keys.append("purchase_intent")
+        for key in flat_keys:
+            values = values_by_key[key]
+            if len(values) >= 3 and max(values) - min(values) < 0.005:
+                add(
+                    key,
+                    f"{METRIC_LABELS[key]} stayed effectively flat ({values[0]:.2f} → {values[-1]:.2f}).",
+                    "A flat series does not show whether the mechanism is stable or simply inactive at this horizon.",
+                    "Inspect the triggering actions and compare with a longer or counterfactual run.",
+                )
+
+        return gaps

    @staticmethod
    def compute_agent_stats(agents: list[AgentPersona], actions: list[ActionEntry]) -> list[dict]: