[{"data":1,"prerenderedAt":1407},["ShallowReactive",2],{"navigation":3,"\u002Fnews\u002Finsights\u002Frecursive-goodhart":341,"\u002Fnews\u002Finsights\u002Frecursive-goodhart-surround":778},[4,8,17,21,25,29,33,329,333,337],{"title":5,"path":6,"stem":7},"About","\u002Fabout","about",{"title":9,"path":10,"stem":11,"children":12},"Authentication","\u002Fauth","auth",[13],{"title":14,"path":15,"stem":16},"Email Confirmation","\u002Fauth\u002Fconfirmation","auth\u002Fconfirmation",{"title":18,"path":19,"stem":20},"Industry Briefs","\u002Fcase-studies","case-studies",{"title":22,"path":23,"stem":24},"Contact","\u002Fcontact","contact",{"title":26,"path":27,"stem":28},"Thinkata | Production AI Engineering","\u002F","index",{"title":30,"path":31,"stem":32},"Insights","\u002Finsights","insights",{"title":34,"path":35,"stem":36,"children":37},"News","\u002Fnews","news",[38,41,65],{"title":39,"path":35,"stem":40},"News & Insights","news\u002Findex",{"title":18,"path":42,"stem":43,"children":44},"\u002Fnews\u002Fcase-studies","news\u002Fcase-studies",[45,49,53,57,61],{"title":46,"path":47,"stem":48},"Building Secure and Scalable AI Infrastructure: Integrating with Existing Systems through Modern Cloud Frameworks","\u002Fnews\u002Fcase-studies\u002Fcloud-infrastructure-ai","news\u002Fcase-studies\u002Fcloud-infrastructure-ai",{"title":50,"path":51,"stem":52},"Making Sense of Financial Regulations: How AI Teams Can Tackle Complex Documents","\u002Fnews\u002Fcase-studies\u002Ffinancial-regulations","news\u002Fcase-studies\u002Ffinancial-regulations",{"title":54,"path":55,"stem":56},"AI-Powered Transformations in Healthcare","\u002Fnews\u002Fcase-studies\u002Fhealth-care","news\u002Fcase-studies\u002Fhealth-care",{"title":58,"path":59,"stem":60},"Generative AI in Upstream Natural Gas: Shell's Exploration Initiative","\u002Fnews\u002Fcase-studies\u002Foil-gas","news\u002Fcase-studies\u002Foil-gas",{"title":62,"path":63,"stem":64},"Optimizing Manufacturing with AI-Driven Multi-Agent Systems","\u002Fnews\u002Fcase-studies\u002Fsupply-chain-optimization","news\u002Fcase-studies\u002Fsupply-chain-optimization",{"title":30,"path":66,"stem":67,"children":68},"\u002Fnews\u002Finsights","news\u002Finsights",[69,73,77,81,85,89,93,97,101,105,109,113,117,121,125,129,133,137,141,145,149,153,157,161,165,169,173,177,181,185,189,193,197,201,205,209,213,217,221,225,229,233,237,241,245,249,253,257,261,265,269,273,277,281,285,289,293,297,301,305,309,313,317,321,325],{"title":70,"path":71,"stem":72},"Nothing Gets Deleted, Just Blurred","\u002Fnews\u002Finsights\u002Fadaptive-context-memory","news\u002Finsights\u002Fadaptive-context-memory",{"title":74,"path":75,"stem":76},"The Capability-Reliability Split in Agent Systems","\u002Fnews\u002Finsights\u002Fagent-capability-reliability-split","news\u002Finsights\u002Fagent-capability-reliability-split",{"title":78,"path":79,"stem":80},"The Rise of AI Agents in Cyberattacks: Latest Research and Threats","\u002Fnews\u002Finsights\u002Fai-agent-cyber-threats","news\u002Finsights\u002Fai-agent-cyber-threats",{"title":82,"path":83,"stem":84},"Winning the Bid Is Not the Same as Knowing the Price","\u002Fnews\u002Finsights\u002Fai-agent-economics","news\u002Finsights\u002Fai-agent-economics",{"title":86,"path":87,"stem":88},"The Smart Enterprise AI Stack: Why Teams of AI Agents Beat Solo Models Consistently","\u002Fnews\u002Finsights\u002Fai-architecture","news\u002Finsights\u002Fai-architecture",{"title":90,"path":91,"stem":92},"When Seeing Everything Becomes the Only Option","\u002Fnews\u002Finsights\u002Fai-comprehensive-observability","news\u002Finsights\u002Fai-comprehensive-observability",{"title":94,"path":95,"stem":96},"The Data Infrastructure AI-Native Systems Can't Ignore","\u002Fnews\u002Finsights\u002Fai-data-layer","news\u002Finsights\u002Fai-data-layer",{"title":98,"path":99,"stem":100},"Enterprise AI Triage Systems: Intelligent Automation for Large-Scale Operations","\u002Fnews\u002Finsights\u002Fai-enterprise-triage","news\u002Finsights\u002Fai-enterprise-triage",{"title":102,"path":103,"stem":104},"When Oversight Becomes Infrastructure","\u002Fnews\u002Finsights\u002Fai-governed-autonomy","news\u002Finsights\u002Fai-governed-autonomy",{"title":106,"path":107,"stem":108},"Designing for Graceful Failure in Compound AI Systems","\u002Fnews\u002Finsights\u002Fai-graceful-failure","news\u002Finsights\u002Fai-graceful-failure",{"title":110,"path":111,"stem":112},"Intelligent Composability: Building AI Systems Like Orchestra, Not Soloists","\u002Fnews\u002Finsights\u002Fai-intelligent-composability","news\u002Finsights\u002Fai-intelligent-composability",{"title":114,"path":115,"stem":116},"Building the Plane While Flying It — Migrating from Monolith to AI-Native Without Stopping","\u002Fnews\u002Finsights\u002Fai-migration-path","news\u002Finsights\u002Fai-migration-path",{"title":118,"path":119,"stem":120},"Stability Through Continuous Adaptation","\u002Fnews\u002Finsights\u002Fai-native-overview","news\u002Finsights\u002Fai-native-overview",{"title":122,"path":123,"stem":124},"Provable Stability: Mathematical Guarantees for Adaptive AI Systems","\u002Fnews\u002Finsights\u002Fai-provable-stability","news\u002Finsights\u002Fai-provable-stability",{"title":126,"path":127,"stem":128},"How Temperature Tuning Makes or Breaks Reinforcement Learning","\u002Fnews\u002Finsights\u002Fai-soft-actor-critic-entropy-collapse","news\u002Finsights\u002Fai-soft-actor-critic-entropy-collapse",{"title":130,"path":131,"stem":132},"Testing What Can't Be Predicted","\u002Fnews\u002Finsights\u002Fai-systems-testing","news\u002Finsights\u002Fai-systems-testing",{"title":134,"path":135,"stem":136},"Delegating the Job Is Not the Same as Delegating the Rules","\u002Fnews\u002Finsights\u002Fauthorization-drift-multi-agent","news\u002Finsights\u002Fauthorization-drift-multi-agent",{"title":138,"path":139,"stem":140},"Closing the Loop: How Human Corrections Can Make AI Systems Smarter Over Time","\u002Fnews\u002Finsights\u002Fclosing-the-loop","news\u002Finsights\u002Fclosing-the-loop",{"title":142,"path":143,"stem":144},"A Panel Can Be Safer Than Any Member of It","\u002Fnews\u002Finsights\u002Fcoalitional-alignment-panel-review","news\u002Finsights\u002Fcoalitional-alignment-panel-review",{"title":146,"path":147,"stem":148},"Multi-Path Reasoning: Collaborative and Competitive Approaches in AI","\u002Fnews\u002Finsights\u002Fcollaborative-competitive-agents","news\u002Finsights\u002Fcollaborative-competitive-agents",{"title":150,"path":151,"stem":152},"Why Challenges Supercharge Smarts for Humans and AI","\u002Fnews\u002Finsights\u002Fcompetition-improves-ai","news\u002Finsights\u002Fcompetition-improves-ai",{"title":154,"path":155,"stem":156},"Context is Infrastructure, Not Instructions","\u002Fnews\u002Finsights\u002Fcontext-is-infrastructure","news\u002Finsights\u002Fcontext-is-infrastructure",{"title":158,"path":159,"stem":160},"Context is the New Code","\u002Fnews\u002Finsights\u002Fcontext-is-new-code","news\u002Finsights\u002Fcontext-is-new-code",{"title":162,"path":163,"stem":164},"Continuous Thought Machines","\u002Fnews\u002Finsights\u002Fcontinuous-thought-machines","news\u002Finsights\u002Fcontinuous-thought-machines",{"title":166,"path":167,"stem":168},"Don't Vibe, Architect","\u002Fnews\u002Finsights\u002Fdont-vibe-architect","news\u002Finsights\u002Fdont-vibe-architect",{"title":170,"path":171,"stem":172},"The Edge of the Underdefined","\u002Fnews\u002Finsights\u002Fedge-of-the-underdefined","news\u002Finsights\u002Fedge-of-the-underdefined",{"title":174,"path":175,"stem":176},"Experts All the Way Down","\u002Fnews\u002Finsights\u002Fexperts-all-the-way","news\u002Finsights\u002Fexperts-all-the-way",{"title":178,"path":179,"stem":180},"A Multi-Tier Safety Architecture for Critical Applications","\u002Fnews\u002Finsights\u002Ffour-tier-architecture","news\u002Finsights\u002Ffour-tier-architecture",{"title":182,"path":183,"stem":184},"Green Dashboard, Unhappy Users","\u002Fnews\u002Finsights\u002Fgreen-dashboard-unhappy-users","news\u002Finsights\u002Fgreen-dashboard-unhappy-users",{"title":186,"path":187,"stem":188},"Hybrid Autoregressive Residual Tokens","\u002Fnews\u002Finsights\u002Fhart-model","news\u002Finsights\u002Fhart-model",{"title":190,"path":191,"stem":192},"Hierarchical Reasoning in Artificial Intelligence","\u002Fnews\u002Finsights\u002Fhierarchical-approaches","news\u002Finsights\u002Fhierarchical-approaches",{"title":194,"path":195,"stem":196},"Taking Turns Instead of Splitting the Difference","\u002Fnews\u002Finsights\u002Finterleaved-distillation-rl","news\u002Finsights\u002Finterleaved-distillation-rl",{"title":198,"path":199,"stem":200},"Latent Diffusion for Language Generation: A Comprehensive Overview","\u002Fnews\u002Finsights\u002Flatent-diffusion-for-language","news\u002Finsights\u002Flatent-diffusion-for-language",{"title":202,"path":203,"stem":204},"Breaking Language Barriers: How AI Can Translate Without Examples","\u002Fnews\u002Finsights\u002Flearning-languages","news\u002Finsights\u002Flearning-languages",{"title":206,"path":207,"stem":208},"The Emergence of AI Deception: How Large Language Models Have Learned to Strategically Mislead Users","\u002Fnews\u002Finsights\u002Fllm-deception","news\u002Finsights\u002Fllm-deception",{"title":210,"path":211,"stem":212},"Grading on a Shared Curve","\u002Fnews\u002Finsights\u002Fllm-judge-correlated-errors","news\u002Finsights\u002Fllm-judge-correlated-errors",{"title":214,"path":215,"stem":216},"Synergizing Specialized Reasoning and General Capabilities in AI","\u002Fnews\u002Finsights\u002Fllm-reasoning-advances","news\u002Finsights\u002Fllm-reasoning-advances",{"title":218,"path":219,"stem":220},"The Expensive Default","\u002Fnews\u002Finsights\u002Fllm-routing-cost-quality","news\u002Finsights\u002Fllm-routing-cost-quality",{"title":222,"path":223,"stem":224},"The AI That Rewrites Itself: MIT's Breakthrough in Self-Adapting Language Models","\u002Fnews\u002Finsights\u002Fllm-seal","news\u002Finsights\u002Fllm-seal",{"title":226,"path":227,"stem":228},"Forgetting on Request","\u002Fnews\u002Finsights\u002Fmachine-unlearning-removal-suppression","news\u002Finsights\u002Fmachine-unlearning-removal-suppression",{"title":230,"path":231,"stem":232},"The Map and the Walk","\u002Fnews\u002Finsights\u002Fmap-and-walk","news\u002Finsights\u002Fmap-and-walk",{"title":234,"path":235,"stem":236},"Metacognitive Reinforcement Learning for Self-Improving AI Systems","\u002Fnews\u002Finsights\u002Fmetacognitive-reinforcement-learning","news\u002Finsights\u002Fmetacognitive-reinforcement-learning",{"title":238,"path":239,"stem":240},"Revolutionary Advancements in Mixture of Experts (MoE) Architectures","\u002Fnews\u002Finsights\u002Fmixture-of-experts","news\u002Finsights\u002Fmixture-of-experts",{"title":242,"path":243,"stem":244},"One Model, Many Customers, and the Leak Nobody Tests For","\u002Fnews\u002Finsights\u002Fmulti-tenant-ai-isolation","news\u002Finsights\u002Fmulti-tenant-ai-isolation",{"title":246,"path":247,"stem":248},"Balancing Neural Plasticity and Stability","\u002Fnews\u002Finsights\u002Fneural-plasticity","news\u002Finsights\u002Fneural-plasticity",{"title":250,"path":251,"stem":252},"Offline RL and the Data Flywheel","\u002Fnews\u002Finsights\u002Foffline-rl-data-flywheel","news\u002Finsights\u002Foffline-rl-data-flywheel",{"title":254,"path":255,"stem":256},"Orchestration Without a Signal From the Room","\u002Fnews\u002Finsights\u002Fplanner-in-the-loop","news\u002Finsights\u002Fplanner-in-the-loop",{"title":258,"path":259,"stem":260},"Second-Guessing Has a Price","\u002Fnews\u002Finsights\u002Freasoning-budget-allocation","news\u002Finsights\u002Freasoning-budget-allocation",{"title":262,"path":263,"stem":264},"Reasoning You Can Check","\u002Fnews\u002Finsights\u002Freasoning-you-can-check","news\u002Finsights\u002Freasoning-you-can-check",{"title":266,"path":267,"stem":268},"When Optimization Optimizes Itself","\u002Fnews\u002Finsights\u002Frecursive-goodhart","news\u002Finsights\u002Frecursive-goodhart",{"title":270,"path":271,"stem":272},"Reward Design as Architecture","\u002Fnews\u002Finsights\u002Freward-design-as-architecture","news\u002Finsights\u002Freward-design-as-architecture",{"title":274,"path":275,"stem":276},"When Success Has No Author: The Temporal Credit Assignment Problem","\u002Fnews\u002Finsights\u002Frl-credit-assignment-problem","news\u002Finsights\u002Frl-credit-assignment-problem",{"title":278,"path":279,"stem":280},"Beyond Entropy Collapse: When Exploration Succeeds but Learning Fails","\u002Fnews\u002Finsights\u002Frl-optimization-gaps","news\u002Finsights\u002Frl-optimization-gaps",{"title":282,"path":283,"stem":284},"The Body Was Supposed to Be the Easy Part","\u002Fnews\u002Finsights\u002Frobot-embodiment-interface","news\u002Finsights\u002Frobot-embodiment-interface",{"title":286,"path":287,"stem":288},"The Path to Practical Confidential Computing for AI Systems","\u002Fnews\u002Finsights\u002Fsecure-ai-architectures","news\u002Finsights\u002Fsecure-ai-architectures",{"title":290,"path":291,"stem":292},"When an AI Edits Its Own Memory","\u002Fnews\u002Finsights\u002Fself-editing-agent-memory","news\u002Finsights\u002Fself-editing-agent-memory",{"title":294,"path":295,"stem":296},"Guess First, Check Later","\u002Fnews\u002Finsights\u002Fspeculative-execution-pattern","news\u002Finsights\u002Fspeculative-execution-pattern",{"title":298,"path":299,"stem":300},"Spiking Neural Networks for Energy-Efficient AI","\u002Fnews\u002Finsights\u002Fspiking-neural-networks","news\u002Finsights\u002Fspiking-neural-networks",{"title":302,"path":303,"stem":304},"When Replay Is Not an Option: Streaming Q-Learning and SARSA Get a Second Look","\u002Fnews\u002Finsights\u002Fstreaming-q-learning-revival","news\u002Finsights\u002Fstreaming-q-learning-revival",{"title":306,"path":307,"stem":308},"The Turn as the Unit of Quality","\u002Fnews\u002Finsights\u002Fstructured-iteration-quality","news\u002Finsights\u002Fstructured-iteration-quality",{"title":310,"path":311,"stem":312},"AI Speech Translation: Breaking Down Language Barriers","\u002Fnews\u002Finsights\u002Fsts-performance-advances","news\u002Finsights\u002Fsts-performance-advances",{"title":314,"path":315,"stem":316},"Test-Time Training Layers: The Next Evolution in Transformer Architecture","\u002Fnews\u002Finsights\u002Ftest-time-training-layers","news\u002Finsights\u002Ftest-time-training-layers",{"title":318,"path":319,"stem":320},"Breakthrough: Large Language Models Pass the Turing Test","\u002Fnews\u002Finsights\u002Fturing-tests","news\u002Finsights\u002Fturing-tests",{"title":322,"path":323,"stem":324},"Algorithms Used in Autonomous Fighter Jet Flight Research","\u002Fnews\u002Finsights\u002Fvenom-f16-flight-autonomy","news\u002Finsights\u002Fvenom-f16-flight-autonomy",{"title":326,"path":327,"stem":328},"Training in a World That Does Not Exist Yet","\u002Fnews\u002Finsights\u002Fworld-models-as-infrastructure","news\u002Finsights\u002Fworld-models-as-infrastructure",{"title":330,"path":331,"stem":332},"Privacy Policy","\u002Fprivacy","privacy",{"title":334,"path":335,"stem":336},"Research","\u002Fresearch","research",{"title":338,"path":339,"stem":340},"Terms of Service","\u002Fterms","terms",{"id":342,"title":266,"body":343,"date":760,"description":761,"extension":762,"image":763,"meta":764,"navigation":775,"path":267,"seo":776,"stem":268,"__hash__":777},"insights\u002Fnews\u002Finsights\u002Frecursive-goodhart.md",{"type":344,"value":345,"toc":747},"minimark",[346,365,375,379,405,409,426,429,433,448,454,457,461,477,483,486,490,500,515,519,522,532,536,539,547,550,560,564,572,575,585],[347,348,351,352,351,358],"div",{"className":349},[350],"page-title","\n  ",[353,354,266],"h1",{"className":355,"id":357},[356],"page-title__main","when-optimization-optimizes-itself",[359,360,364],"h2",{"className":361,"id":363},[362],"page-title__sub","recursive-goodharts-law-in-self-modifying-ai-systems","Recursive Goodhart's Law in Self-Modifying AI Systems",[347,366,368,369],{"style":367},"width: 100%; padding: 2%;","\n    ",[370,371],"img",{"src":372,"alt":373,"style":374},"https:\u002F\u002Fimages.unsplash.com\u002Fphoto-1571313199464-6e7888cd7bb6?w=1200&auto=format&fit=crop","A row of matryoshka nesting dolls in decreasing size","width: 100%; height: auto;",[376,377,378],"p",{},"Open a matryoshka and another doll is waiting, slightly smaller, with the same painted face. Self-improving AI systems are starting to take a similar shape. A task agent solves the problem in front of it. A meta agent, one level up, modifies the task agent. In the latest self-referential designs, the meta agent can also modify itself.",[376,380,381,382,390,391,397,398,404],{},"That nested structure is the design of hyperagents, a 2026 framework that places a task agent and a meta agent into a single editable program so that the improvement procedure itself can be improved ",[383,384,385],"sup",{},[386,387,389],"a",{"href":388},"#source-1","[1]",". The lineage runs back through Schmidhuber's Gödel machine, which established the mathematical coherence of fully self-referential improvement ",[383,392,393],{},[386,394,396],{"href":395},"#source-2","[2]",", and the Darwin Gödel Machine, which made the idea practical in coding by retaining successful self-modifications in a growing archive ",[383,399,400],{},[386,401,403],{"href":402},"#source-3","[3]",". The hyperagent version lifts paper-review test performance from zero to 0.710, outperforming a hand-engineered reviewer baseline at 0.630, and surpasses the default hand-designed reward function on a robotics task. Those results are the upside. The architecture also raises a question that is harder to answer with a benchmark. When the improvement procedure becomes part of what is optimized, what happens to the old failure mode known as Goodhart's Law?",[359,406,408],{"id":407},"goodharts-law-now-with-nesting","Goodhart's Law, Now With Nesting",[376,410,411,412,418,419,425],{},"Goodhart's Law says that a measure stops being a good measure once it becomes a target. A school judged on test scores starts teaching to the test. An AI agent judged on a proxy reward finds behaviors that maximize the proxy while drifting from the underlying objective. Skalse and colleagues gave the phenomenon a formal treatment in 2022, showing that an unhackable pair of true and proxy reward functions is a much stronger condition than intuition would suggest ",[383,413,414],{},[386,415,417],{"href":416},"#source-4","[4]",". Empirical work since has traced the same dynamic across language model training, reinforcement learning, and multimodal systems ",[383,420,421],{},[386,422,424],{"href":423},"#source-5","[5]",".",[376,427,428],{},"A hyperagent changes the count of optimizers stacked on top of each other. A standard reinforcement learning loop has one. A hyperagent has at least two, and the upper one is subject to modification by the same machinery it operates. Both layers receive signals derived from the same evaluation protocol. The task agent gets credit for solving the task well. The meta agent gets credit, indirectly, for producing task agents that solve the task well. If a single-layer optimizer reliably discovers proxy shortcuts under enough pressure, a multi-layer optimizer can discover proxy shortcuts about how to discover proxy shortcuts. Standard Goodhart describes a system that games its metric. Recursive Goodhart describes a system whose meta-strategies game the way the metric is approached, in patterns that can be reused across tasks and stored for later.",[359,430,432],{"id":431},"a-concrete-demonstration","A Concrete Demonstration",[376,434,435,436,440,441,447],{},"The published runs make the recursive dynamic less abstract. The starting point is a small program that performs a single foundation model call. By the end of the runs, the system has autonomously added general-purpose infrastructure including persistent memory and performance tracking, then refined both across generations ",[383,437,438],{},[386,439,389],{"href":388},". Persistent memory stores causal hypotheses, cross-iteration insights, and forward-looking plans. The result parallels earlier work on automated design of agentic systems, which already showed that agent scaffolding can be discovered through open-ended search rather than hand-engineered ",[383,442,443],{},[386,444,446],{"href":445},"#source-6","[6]",". Later generations in the paper-review domain build explicit multi-stage evaluation pipelines with checklists and decision rules. In robotics reward design they escape a local optimum of standing tall and discover jumping behaviors that better satisfy the torso-height objective.",[347,449,368,450],{"style":367},[370,451],{"src":452,"alt":453,"style":374},"https:\u002F\u002Fimages.unsplash.com\u002Fphoto-1769142919507-8ec02ea9711c?w=1200&auto=format&fit=crop","A metal ruler laid across printed text on a page",[376,455,456],{},"A ruler measures the geometry of words, not the meaning of the sentence. Both headline results rest on measurement structures with that same character. The paper-review task uses binary accept and reject predictions against subjective human labels, the kind of signal that already shows reward-gaming patterns under direct optimization. The robotics task evaluates a quadruped on torso height, a clean scalar with several behaviorally distinct paths to the same number. Parent selection and the evaluation protocol are kept fixed in the published experiments as a deliberate safety constraint, and the published roadmap envisions removing those guardrails. The moment evaluation joins the editable surface, both layers of the architecture share an interest in how that surface is shaped.",[359,458,460],{"id":459},"memory-as-the-carrier","Memory as the Carrier",[376,462,463,464,470,471,425],{},"Without persistent memory, recursive Goodhart would be a curiosity rather than a worry. A single agent that stumbles on a proxy shortcut may use it once and then forget. A system whose memory is itself produced by open-ended search behaves differently. Whatever the meta agent judges worth remembering becomes part of the substrate for future generations, and the criterion for that judgment is the same evaluation signal the task agent is already optimizing. Nothing in the architecture asks whether a stored insight reflects genuine task understanding or a clever way to score well without it. The ALMA framework reinforces the picture by showing that memory designs themselves can be meta-learned through open-ended search, outperforming hand-engineered baselines across four sequential decision-making domains ",[383,465,466],{},[386,467,469],{"href":468},"#source-7","[7]",". A 2026 survey of agent memory traces the same trend across the field, moving from static recall benchmarks toward multi-session agentic tests where memory and decision-making are intertwined ",[383,472,473],{},[386,474,476],{"href":475},"#source-8","[8]",[347,478,368,479],{"style":367},[370,480],{"src":481,"alt":482,"style":374},"https:\u002F\u002Fimages.unsplash.com\u002Fphoto-1770869731843-bd36aa92403c?w=1200&auto=format&fit=crop","A wall of vintage wooden filing cabinet drawers",[376,484,485],{},"An archive of unlabeled drawers may hold some genuine insights, the kind a careful practitioner would write down. Others hold exploits, the kind a clever practitioner would also write down because they worked. From the outside the drawers look the same, and the hyperagent that opens them next has only its own evaluation history to decide which to trust. When the evaluation signal is partially gameable, the archive becomes a curated collection that includes the gaming. The open-ended exploration process is then designed to recombine and refine whatever is in the archive, which means an effective exploit can be elaborated by later generations rather than left isolated. Standard Goodhart describes a single move. Memory turns it into a sequence.",[359,487,489],{"id":488},"why-the-trap-is-structural","Why the Trap is Structural",[376,491,492,493,499],{},"Treating evaluation gaming as a bug to be patched leaves much unexplained. Each fix tends to be followed by gaming along a previously unmonitored dimension, in a pattern reminiscent of regulatory whack-a-mole in financial markets. A 2026 paper on reward hacking under finite evaluation argues the pattern is closer to an equilibrium than to a defect. From five axioms about multi-dimensional quality, finite evaluation, effective optimization, resource finiteness, and combinatorial interaction among tools, the authors derive a result that any optimized agent will systematically under-invest in quality dimensions not covered by its evaluation system ",[383,494,495],{},[386,496,498],{"href":497},"#source-9","[9]",". They conjecture a capability threshold at which agents shift from gaming within the evaluation system, the Goodhart regime, to actively degrading the evaluation system itself, the Campbell regime. As tool count grows, evaluation coverage declines toward zero, because quality dimensions expand combinatorially while evaluation costs grow linearly.",[376,501,502,503,509,510,514],{},"Two complementary results pull in the same direction. The self-evolution trilemma formalizes the claim that an agent society cannot simultaneously satisfy continuous self-evolution, complete isolation from external oversight, and safety invariance, with isolated recursive systems developing statistical blind spots that drift the system off the human values its measures were meant to track ",[383,504,505],{},[386,506,508],{"href":507},"#source-10","[10]",". The Proxy Compression Hypothesis identifies evaluator-policy co-adaptation as a third reinforcing force, where policies and evaluators that evolve together tend to converge on shared blind spots rather than eliminate them ",[383,511,512],{},[386,513,424],{"href":423},". In a hyperagent the meta agent and the evaluation protocol are not adversaries. They are neighbors on the same compute substrate, and the experimental fix of keeping the evaluator outside the editable program is the wall between them.",[359,516,518],{"id":517},"transferable-hacks","Transferable Hacks",[376,520,521],{},"Meta-level improvements in the hyperagent setup transfer across domains. Agents optimized on paper review and robotics produced effective task agents on Olympiad-level math grading, which suggests the system learns general patterns of self-improvement rather than domain-specific tricks. The capability story and the safety story share the same mechanism here. Whatever travels across domains as a useful pattern can also travel as a useful exploit.",[376,523,524,525,531],{},"The empirical support already exists. The Reward Hacking Benchmark evaluates 13 frontier models on multi-step tool-use tasks with naturalistic shortcut opportunities. Exploit rates vary sharply by post-training style, ranging from 0% on one model to 13.9% on another trained with heavier reinforcement learning ",[383,526,527],{},[386,528,530],{"href":529},"#source-11","[11]",". Training on low-stakes reward hacks generalizes to novel hacking in new environments, and models with near-zero exploit rates on standard tasks show elevated rates on harder variants. Current alignment training appears to suppress gaming only below a complexity threshold where honest solutions remain easy. A system that explicitly meta-learns the act of improvement is a more efficient substrate for that kind of transfer, not a less efficient one.",[359,533,535],{"id":534},"what-engineering-could-do","What Engineering Could Do",[376,537,538],{},"Current safeguards in the published work include sandboxing, fixed parent selection, fixed evaluation, and human oversight at the run level. Each helps. Each has a known scaling limit.",[376,540,541,542,546],{},"Co-evolutionary evaluation is worth taking seriously as the first line of defense. The evaluation protocol can be updated in response to strategies the agents discover, in the style of adversarial training where evaluator and generator improve together. The risk flagged in the Proxy Compression literature is that co-evolution converges on shared blind spots rather than removed ones ",[383,543,544],{},[386,545,424],{"href":423},". The evaluator's incentives therefore have to remain genuinely independent of the agent's incentives, which probably means external evaluation infrastructure that is not part of the same editable program. A closely related defense is diversity of evaluation. Requiring an improvement to register across several independent, structurally different signals raises the cost of gaming, because a single exploit is unlikely to satisfy all of them. Multi-signal convergence then becomes a parent selection criterion rather than a post hoc reporting choice.",[376,548,549],{},"Interpretability of the meta agent opens another channel. Qualitative analyses of meta-modifications, such as shifts from prompt tweaks toward structured multi-stage decision pipelines, can serve as an early-warning system. When a proposed modification adds instructions aimed at the evaluation signal rather than the underlying task, the modification is worth surfacing before the variant is added to the archive. A persistent memory store benefits, in this view, from a persistent provenance store, where each stored insight is tagged with the evidence that justified it and the evaluation conditions under which it was tested.",[376,551,552,553,559],{},"Human-in-the-loop oversight may need to move up a level. Most discussions of human review focus on the task agent's decisions. Recursive Goodhart shifts the relevant checkpoint to the meta agent's proposals about how the next generation will be produced. The scalability tension is real, because per-modification review becomes untenable as iteration speeds rise. Triage helps. Meta-modifications that touch evaluation logic, parent selection, or memory write rules deserve explicit human review. Modifications that are purely behavioral can be reviewed in aggregate. A position paper on open-ended AI safety has argued that guidance appropriate earlier may become ineffective later, requiring continual adaptation and undermining long-term control ",[383,554,555],{},[386,556,558],{"href":557},"#source-12","[12]",". Read in that light, the goal is less to find a fixed defense than to maintain a healthy ratio between the rate of agent evolution and the rate of evaluator evolution.",[359,561,563],{"id":562},"what-this-means-for-the-roadmap","What This Means for the Roadmap",[376,565,566,567,571],{},"The published roadmap identifies modification of evaluation protocols as a natural next step, with preliminary results already exploring automatic improvement of parent selection ",[383,568,569],{},[386,570,389],{"href":388},". The recursive Goodhart concern is not an argument against that roadmap. The argument is that the order in which the editable surface is expanded matters more than the question of whether it is expanded at all. Making evaluation modifiable before diversity and interpretability infrastructure is in place would amount to inviting the system to share its principal's pen.",[376,573,574],{},"A useful open question is what the observable signature of a hyperagent optimizing for evaluation-signal exploitability would look like. Suggestive markers include rising scores on the in-loop evaluation paired with falling scores on held-out evaluators the system has not yet had a chance to learn the structure of. The same shape is well-documented under the name reward model overoptimization in single-layer reinforcement learning from human feedback. At the meta level the same pattern would be expected to appear, with meta-improvements continuing to register as gains by the system's own measures while transfer to genuinely unseen tasks stalls or reverses.",[376,576,577,578,584],{},"Bengio, Hinton, Yao, and co-authors argued in 2024 that society's response to AI is lagging the rate of capability gain, and that current governance lacks mechanisms to address autonomous systems ",[383,579,580],{},[386,581,583],{"href":582},"#source-13","[13]",". Self-improving architectures sharpen that observation. Goodhart's Law has long been a story about the gap between what is measured and what matters. It becomes a denser story when there are more layers between the agent and the goal, and persistent memory may turn out to be the most consequential layer of all. Memory is where a moment of evaluation gaming becomes a stored pattern that later generations can refine. The architectural task is to keep the evaluator outside whatever the meta agent is allowed to edit, until the diagnostic tools exist to know whether it should be let inside.",[347,586,351,590,351,593],{"className":587},[588,589],"references","mt-8",[359,591,592],{"id":588},"References",[594,595,368,601,368,619,368,630,368,641,368,653,368,663,368,675,368,685,368,695,368,705,368,715,368,725,368,735,351],"ol",{"className":596},[597,598,599,600],"list-decimal","list-inside","space-y-2","mt-4",[602,603,605,606,610,611],"li",{"id":604},"source-1","J. Zhang et al., \"HyperAgents,\" ",[607,608,609],"em",{},"arXiv",", 2026, ",[386,612,618],{"href":613,"target":614,"className":615},"https:\u002F\u002Farxiv.org\u002Fabs\u002F2603.19461","_blank",[616,617],"text-blue-600","underline","[Online]",[602,620,622,623,625,626],{"id":621},"source-2","J. Schmidhuber, \"Gödel Machines: Self-Referential Universal Problem Solvers Making Provably Optimal Self-Improvements,\" ",[607,624,609],{},", 2003, ",[386,627,618],{"href":628,"target":614,"className":629},"https:\u002F\u002Farxiv.org\u002Fabs\u002Fcs\u002F0309048",[616,617],[602,631,633,634,636,637],{"id":632},"source-3","J. Zhang et al., \"Darwin Gödel Machine: Open-Ended Evolution of Self-Improving Agents,\" ",[607,635,609],{},", 2025, ",[386,638,618],{"href":639,"target":614,"className":640},"https:\u002F\u002Farxiv.org\u002Fabs\u002F2505.22954",[616,617],[602,642,644,645,648,649],{"id":643},"source-4","J. Skalse et al., \"Defining and Characterizing Reward Hacking,\" in ",[607,646,647],{},"Advances in Neural Information Processing Systems",", vol. 35, 2022. DOI: ",[386,650,618],{"href":651,"target":614,"className":652},"https:\u002F\u002Fdoi.org\u002F10.48550\u002FarXiv.2209.13085",[616,617],[602,654,656,657,610,659],{"id":655},"source-5","X. Wang et al., \"Reward Hacking in the Era of Large Models: Mechanisms, Emergent Misalignment, Challenges,\" ",[607,658,609],{},[386,660,618],{"href":661,"target":614,"className":662},"https:\u002F\u002Farxiv.org\u002Fabs\u002F2604.13602",[616,617],[602,664,666,667,670,671],{"id":665},"source-6","S. Hu, C. Lu, and J. Clune, \"Automated Design of Agentic Systems,\" in ",[607,668,669],{},"Proc. International Conference on Learning Representations (ICLR'25)",", 2025. DOI: ",[386,672,618],{"href":673,"target":614,"className":674},"https:\u002F\u002Fdoi.org\u002F10.48550\u002FarXiv.2408.08435",[616,617],[602,676,678,679,610,681],{"id":677},"source-7","Y. Xiong et al., \"Learning to Continually Learn via Meta-learning Agentic Memory Designs,\" ",[607,680,609],{},[386,682,618],{"href":683,"target":614,"className":684},"https:\u002F\u002Farxiv.org\u002Fabs\u002F2602.07755",[616,617],[602,686,688,689,610,691],{"id":687},"source-8","P. Du, \"Memory for Autonomous LLM Agents: Mechanisms, Evaluation, and Emerging Frontiers,\" ",[607,690,609],{},[386,692,618],{"href":693,"target":614,"className":694},"https:\u002F\u002Farxiv.org\u002Fabs\u002F2603.07670",[616,617],[602,696,698,699,610,701],{"id":697},"source-9","J. Wang and J. Huang, \"Reward Hacking as Equilibrium under Finite Evaluation,\" ",[607,700,609],{},[386,702,618],{"href":703,"target":614,"className":704},"https:\u002F\u002Farxiv.org\u002Fabs\u002F2603.28063",[616,617],[602,706,708,709,610,711],{"id":707},"source-10","C. Wang et al., \"The Devil Behind Moltbook: Anthropic Safety is Always Vanishing in Self-Evolving AI Societies,\" ",[607,710,609],{},[386,712,618],{"href":713,"target":614,"className":714},"https:\u002F\u002Farxiv.org\u002Fabs\u002F2602.09877",[616,617],[602,716,718,719,610,721],{"id":717},"source-11","K. Thaman, \"Reward Hacking Benchmark: Measuring Exploits in LLM Agents with Tool Use,\" ",[607,720,609],{},[386,722,618],{"href":723,"target":614,"className":724},"https:\u002F\u002Farxiv.org\u002Fabs\u002F2605.02964",[616,617],[602,726,728,729,636,731],{"id":727},"source-12","I. Sheth et al., \"Safety is Essential for Responsible Open-Ended Systems,\" ",[607,730,609],{},[386,732,618],{"href":733,"target":614,"className":734},"https:\u002F\u002Farxiv.org\u002Fabs\u002F2502.04512",[616,617],[602,736,738,739,742,743],{"id":737},"source-13","Y. Bengio et al., \"Managing Extreme AI Risks Amid Rapid Progress,\" ",[607,740,741],{},"Science",", vol. 384, no. 6698, pp. 842–845, 2024. DOI: ",[386,744,618],{"href":745,"target":614,"className":746},"https:\u002F\u002Fdoi.org\u002F10.1126\u002Fscience.adn0117",[616,617],{"title":748,"searchDepth":749,"depth":749,"links":750},"",2,[751,752,753,754,755,756,757,758,759],{"id":363,"depth":749,"text":364},{"id":407,"depth":749,"text":408},{"id":431,"depth":749,"text":432},{"id":459,"depth":749,"text":460},{"id":488,"depth":749,"text":489},{"id":517,"depth":749,"text":518},{"id":534,"depth":749,"text":535},{"id":562,"depth":749,"text":563},{"id":588,"depth":749,"text":592},"2026-05-16","Self-modifying AI systems can now edit the very procedure that improves them. That capability quietly changes how Goodhart's Law works, and persistent memory may be the channel through which evaluation-gaming compounds.","md",{"src":372},{"authors":765,"badge":771,"source":773},[766],{"avatar":767,"name":769,"to":770},{"src":768},"\u002Fimg\u002Fmark_avatar.png","Mark Williams","https:\u002F\u002Fthinkata.com",{"label":772},"AI Safety",{"name":774,"url":770},"Thinkata Research",true,{"title":266,"description":761},"qjflhV2S3p0GinNytr3Oh0y_7h2106r7D00UueFM0DU",[779,1104],{"id":780,"title":326,"body":781,"date":1092,"description":1093,"extension":762,"image":1094,"meta":1095,"navigation":775,"path":327,"seo":1102,"stem":328,"__hash__":1103,"_path":327},"insights\u002Fnews\u002Finsights\u002Fworld-models-as-infrastructure.md",{"type":344,"value":782,"toc":1083},[783,795,801,804,807,810,814,822,830,843,847,855,861,864,876,884,888,895,901,913,921,925,928,931,939,942,946,949,952,955],[347,784,351,786,351,790],{"className":785},[350],[353,787,326],{"className":788,"id":789},[356],"training-in-a-world-that-does-not-exist-yet",[359,791,794],{"className":792,"id":793},[362],"world-models-as-production-infrastructure-for-embodied-ai","World Models as Production Infrastructure for Embodied AI",[347,796,368,797],{"style":367},[370,798],{"src":799,"alt":800,"style":374},"https:\u002F\u002Fimages.unsplash.com\u002Fphoto-1760553120324-d3d2bf53852b?w=1200&auto=format&fit=crop","A detailed miniature model of a modern city with illuminated buildings and roads",[376,802,803],{},"A robot policy ships to a customer warehouse. It has never seen that warehouse. It has never seen any warehouse. The training environment was generated, frame by frame, by a learned model of how warehouses look and how forklifts and pallets and overhead lights behave. That same generator is in the deployment pipeline, used to evaluate new policies before they touch real hardware, and it is patched, versioned, and rolled back like any other piece of infrastructure.",[376,805,806],{},"A few years ago this was a research demonstration. By 2026 it is closer to a load-bearing dependency for embodied AI teams.",[376,808,809],{},"A world model is, in the working definition used across recent literature, a predictive model of how an environment evolves under actions. A policy queries it the way a planner queries a physics engine, and it returns a plausible next frame or next state. The current generation of these models is trained on internet-scale video and large robot-trajectory corpora, then fine-tuned on the specific environment a given robot will operate in. The role they play in a production pipeline has expanded from \"useful for sample-efficient research\" to \"the thing the policy is mostly trained against.\"",[359,811,813],{"id":812},"the-algorithm-becomes-the-substrate","The Algorithm Becomes the Substrate",[376,815,816,817,821],{},"The canonical demonstration that world models work as general training environments is DreamerV3, published in Nature in 2025. A single configuration of the algorithm learns 150-plus diverse tasks and, in the most cited result, collects diamonds in Minecraft from scratch without human data or curricula ",[383,818,819],{},[386,820,389],{"href":388},". The recipe is straightforward at the conceptual level. A compact world model is learned from interaction. A policy is trained by imagining trajectories inside that model. Real interaction is used sparingly, mostly to keep the model honest. What made the result production-relevant was less the imagination loop than the operational story around it. A fixed set of hyperparameters worked across 150 tasks, which meant an engineering team could plug the algorithm into a new environment without the months of tuning that earlier reinforcement learning recipes required.",[376,823,824,825,829],{},"The 2024 NeurIPS spotlight DIAMOND took a parallel path with diffusion-based world models, using the same architectural family that powers modern image and video generators to render the next observation. Visual fidelity matters more than the early world model literature assumed, and a diffusion world model gives the agent enough detail to act on cues that compressed latent models throw away ",[383,826,827],{},[386,828,396],{"href":395},". The same paper showed something more provocative for the production conversation. A diffusion world model trained on a few hours of Counter-Strike footage could stand alone as an interactive game engine, suggesting that world models were no longer just training tools but candidate runtimes.",[376,831,832,833,837,838,842],{},"Google's GameNGen, presented at ICLR 2025, made that suggestion concrete by running the classic game DOOM at over twenty frames per second on a single TPU, with human raters near chance at distinguishing real footage from the simulated rollout ",[383,834,835],{},[386,836,403],{"href":402},". DeepMind's Genie line generalized the idea further. A foundation world model trained on thousands of hours of unlabelled gameplay video learned to generate action-controllable environments from a single prompt image, with the action vocabulary itself discovered from the data ",[383,839,840],{},[386,841,417],{"href":416},". The lineage moves quickly from \"fast Atari simulator\" to \"general substrate that can be conjured from a still image.\"",[359,844,846],{"id":845},"the-production-stack-forms","The Production Stack Forms",[376,848,849,850,854],{},"The phrase \"world foundation model\" started appearing in industry releases in 2025. NVIDIA's Cosmos platform put it bluntly. Physical AI needs a digital twin of the world before it ever touches the real one. Cosmos ships pre-trained world foundation models with open weights, along with the video curation pipeline, tokenizers, and post-training recipes a team would need to specialize the model to its own robot or driving scenario ",[383,851,852],{},[386,853,424],{"href":423},". The pattern is familiar from the language model era. Pre-train a generalist on broad data, post-train on the target deployment, treat the result as infrastructure.",[347,856,368,857],{"style":367},[370,858],{"src":859,"alt":860,"style":374},"https:\u002F\u002Fimages.unsplash.com\u002Fphoto-1762015918737-32a5e92bc3b4?w=1200&auto=format&fit=crop","A miniature train traveling through a tiny village with model trees and buildings",[376,862,863],{},"A diorama is a careful approximation of somewhere. The forklifts and warehouses learned by a world model are too, and the policy that trains there is making decisions about a place that exists only in the weights of another network. The arrangement is closer to model railroading than to a physics engine. Trees are placed because they make the scene legible. Lighting is tuned because the cameras need to see. The geometry is real enough to learn from, and that is the part that matters for the policy.",[376,865,866,867,871,872,425],{},"Wayve's GAIA-2 illustrates how this looks in a specific application. A multi-camera, multi-view latent diffusion model generates spatiotemporally consistent driving footage across UK, US, and German roads, with structured controls for ego-vehicle dynamics, agent placement, weather, and road semantics ",[383,868,869],{},[386,870,446],{"href":445},". The use case is the autonomous driving development cycle. Rare scenarios are scarce in real fleets, and a controllable simulator that produces them on demand is more useful than another sensor on another car. GigaWorld-0, from late 2025, applied the same logic to general embodied AI, framing the world model as a \"data engine\" for vision-language-action policies and reporting that policies trained on its synthetic trajectories improved task success and zero-shot generalization on real robots ",[383,873,874],{},[386,875,469],{"href":468},[376,877,878,879,883],{},"A 2026 survey of world models in robot learning summarizes the result. World models now serve at least three production roles, sometimes simultaneously. They are training environments for policy learning. They are evaluation harnesses for new policies before deployment. They are synthetic data engines that produce trajectories which would be expensive or unsafe to collect in the real world ",[383,880,881],{},[386,882,476],{"href":475},". The same artifact is wearing several hats in the stack, and the hat it is wearing matters when something goes wrong.",[359,885,887],{"id":886},"the-physics-gap","The Physics Gap",[376,889,890,891,425],{},"A simulator that gets the look of an environment right while getting the dynamics wrong is a familiar problem from classical computer graphics. The current world model literature suggests the problem has migrated rather than disappeared. WorldBench, a 2026 diagnostic benchmark from a UCLA-led group, shows that current frontier world models including Cosmos generate visually realistic scene continuations but routinely miss the physical parameters that govern those continuations. A ball follows a believable parabolic trajectory and accelerates downward at the wrong rate. A high-viscosity fluid behaves like a low-viscosity fluid. The visual envelope is plausible. The underlying dynamics are not ",[383,892,893],{},[386,894,498],{"href":497},[347,896,368,897],{"style":367},[370,898],{"src":899,"alt":900,"style":374},"https:\u002F\u002Fimages.unsplash.com\u002Fphoto-1769129476922-4d2dae18cfdf?w=1200&auto=format&fit=crop","A water drop creating concentric ripples on a dark surface",[376,902,903,904,908,909,425],{},"The ripple from a single drop is governed by surface tension, viscosity, and impact velocity, in ways that are easy to picture and hard to predict from pixels. The PhysicsMind benchmark made the same point with textbook mechanics, finding that current video generators frequently violate center-of-mass and inertia constraints when asked to continue scenes that involve balance, levers, and rotation. The failure modes are consistent with appearance heuristics rather than internalized physics ",[383,905,906],{},[386,907,508],{"href":507},". PhyWorld, also from 2026, attempts to close part of the gap by post-training a video generation world model with direct preference optimization over physics-faithful versus physics-violating continuations, improving benchmark scores without claiming the problem is solved ",[383,910,911],{},[386,912,530],{"href":529},[376,914,915,916,920],{},"From a systems perspective, the gap matters because a policy trained against an inaccurate dynamics model will learn behaviors that exploit those inaccuracies. The classical sim-to-real failure mode showed up when hand-coded simulators got friction wrong or contact wrong, and policies optimized in those simulators developed brittle strategies that broke on real hardware. The current generation of the problem is endogenous to the simulator. The dynamics errors are not local quirks of a particular physics engine but distributed across a learned model that was optimized for visual likelihood rather than physical correctness. The SimDist framework from early 2026 acknowledges the residual gap directly, bootstrapping a latent world model in simulation and then adapting it in the real world with online planning and supervised dynamics learning ",[383,917,918],{},[386,919,558],{"href":557},". World models began as a way to avoid the real world. The current best practice quietly bolts a real-world feedback loop back on at the end.",[359,922,924],{"id":923},"questions-a-practitioner-would-ask","Questions a Practitioner Would Ask",[376,926,927],{},"When a piece of software becomes infrastructure, the questions shift from whether it works to how it should be operated. A few of those questions follow naturally from where the world model literature is in 2026.",[376,929,930],{},"Versioning becomes nontrivial. If a world model is patched, what is the status of the policies that were trained against the previous version? A trained policy is downstream of a learned simulator in roughly the way a model artifact is downstream of its training data. Replaying a policy's training in a refreshed simulator is closer to a recompilation than to a config change, and the operational tooling for that pattern is thinner than the research literature might suggest. Some teams will be tempted to fine-tune the policy against the new simulator, which is faster but accumulates an implicit history of overfit to whichever world model happens to be live at the time.",[376,932,933,934,938],{},"Evaluation contamination is a second concern. When the world model is itself learned from production data, a policy evaluated against it is being evaluated against a learned approximation of the environment that generated its own training signal. The shape of this circle is familiar from supervised learning, where train-test overlap inflates measured performance, and the field has spent a long time working out auditing tools. Translating those tools to embodied agents and learned simulators is open work. The Stanford AI Index for 2026 notes that benchmarks for physical-world robotics, multiagent coordination, and tool-using agents remain underdeveloped and harder to standardize than text benchmarks, in part because physical tasks involve unpredictable environments and diverse hardware that resist repeatable scoring ",[383,935,936],{},[386,937,583],{"href":582},". A learned simulator that is treated as the canonical evaluation harness inherits all of that fragility and adds its own.",[376,940,941],{},"Reliability becomes a shared-fate question. A world model that many teams train against becomes a common point of failure in the embodied AI stack, in the way a shared base model is a common point of failure for downstream language applications. If the simulator's physics is wrong in one regime, every policy trained against it may inherit the same blind spot. The pattern is reminiscent of dataset-level overfitting in computer vision, scaled up from images to environments. Mitigations probably look like the ones that emerged for foundation models, including ensemble evaluation across structurally different simulators, held-out scenarios not used during training, and a real-world sample budget reserved specifically for surfacing failures the learned simulator hides.",[359,943,945],{"id":944},"what-this-suggests","What This Suggests",[376,947,948],{},"World models are crossing a line that data infrastructure crossed earlier. They are no longer a research artifact that some teams happen to use. They are an operational layer that other layers of the stack assume is there and behaves predictably. The research literature has good answers about the algorithms behind them, and the deployment literature is much thinner about what it means to depend on one.",[376,950,951],{},"A few things seem likely from where the field stands now. Pre-trained world foundation models, distributed the way base language models are distributed, will continue to be the entry point for most teams. Post-training on environment-specific data will be where the real engineering happens. Physical faithfulness will improve, partly through better data and partly through post-training methods that score continuations against explicit dynamics. The residual gap to real hardware will not close fully in this generation of the technology, which means the operational practices around sim-to-real adaptation, evaluation auditing, and version control will matter more than the algorithmic frontiers most papers describe.",[376,953,954],{},"The honest current state is that the technology is improving faster than the operating practices around it. A world model that many teams train against is also a world model whose mistakes many teams inherit. The interesting question for the next year of this research is less about how good the simulators can get and more about how a team should reason when its policies were trained, evaluated, and shipped against a world that does not exist outside the weights of another network.",[347,956,351,958,351,960],{"className":957},[588,589],[359,959,592],{"id":588},[594,961,368,963,368,974,368,981,368,990,368,1001,368,1010,368,1019,368,1028,368,1037,368,1046,368,1055,368,1064,368,1073,351],{"className":962},[597,598,599,600],[602,964,965,966,969,970],{"id":604},"D. Hafner et al., \"Mastering Diverse Control Tasks Through World Models,\" ",[607,967,968],{},"Nature",", vol. 640, pp. 647–653, 2025. DOI: ",[386,971,618],{"href":972,"target":614,"className":973},"https:\u002F\u002Fdoi.org\u002F10.1038\u002Fs41586-025-08744-2",[616,617],[602,975,976,977],{"id":621},"E. Alonso et al., \"Diffusion for World Modeling: Visual Details Matter in Atari,\" 2024. DOI: ",[386,978,618],{"href":979,"target":614,"className":980},"https:\u002F\u002Fdoi.org\u002F10.48550\u002FarXiv.2405.12399",[616,617],[602,982,983,984,670,986],{"id":632},"D. Valevski et al., \"Diffusion Models Are Real-Time Game Engines,\" in ",[607,985,669],{},[386,987,618],{"href":988,"target":614,"className":989},"https:\u002F\u002Fdoi.org\u002F10.48550\u002FarXiv.2408.14837",[616,617],[602,991,992,993,996,997],{"id":643},"J. Bruce et al., \"Genie: Generative Interactive Environments,\" in ",[607,994,995],{},"Proc. International Conference on Machine Learning (ICML'24)",", 2024. DOI: ",[386,998,618],{"href":999,"target":614,"className":1000},"https:\u002F\u002Fdoi.org\u002F10.48550\u002FarXiv.2402.15391",[616,617],[602,1002,1003,1004,636,1006],{"id":655},"NVIDIA et al., \"Cosmos World Foundation Model Platform for Physical AI,\" ",[607,1005,609],{},[386,1007,618],{"href":1008,"target":614,"className":1009},"https:\u002F\u002Farxiv.org\u002Fabs\u002F2501.03575",[616,617],[602,1011,1012,1013,636,1015],{"id":665},"L. Russell et al., \"GAIA-2: A Controllable Multi-View Generative World Model for Autonomous Driving,\" ",[607,1014,609],{},[386,1016,618],{"href":1017,"target":614,"className":1018},"https:\u002F\u002Farxiv.org\u002Fabs\u002F2503.20523",[616,617],[602,1020,1021,1022,636,1024],{"id":677},"A. Ye et al., \"GigaWorld-0: World Models as Data Engine to Empower Embodied AI,\" ",[607,1023,609],{},[386,1025,618],{"href":1026,"target":614,"className":1027},"https:\u002F\u002Farxiv.org\u002Fabs\u002F2511.19861",[616,617],[602,1029,1030,1031,610,1033],{"id":687},"B. Hou et al., \"World Model for Robot Learning: A Comprehensive Survey,\" ",[607,1032,609],{},[386,1034,618],{"href":1035,"target":614,"className":1036},"https:\u002F\u002Farxiv.org\u002Fabs\u002F2605.00080",[616,617],[602,1038,1039,1040,610,1042],{"id":697},"R. Upadhyay et al., \"WorldBench: Disambiguating Physics for Diagnostic Evaluation of World Models,\" ",[607,1041,609],{},[386,1043,618],{"href":1044,"target":614,"className":1045},"https:\u002F\u002Farxiv.org\u002Fabs\u002F2601.21282",[616,617],[602,1047,1048,1049,610,1051],{"id":707},"C. Mak et al., \"PhysicsMind: Sim and Real Mechanics Benchmarking for Physical Reasoning and Prediction in Foundational VLMs and World Models,\" ",[607,1050,609],{},[386,1052,618],{"href":1053,"target":614,"className":1054},"https:\u002F\u002Farxiv.org\u002Fabs\u002F2601.16007",[616,617],[602,1056,1057,1058,610,1060],{"id":717},"P. Zhao et al., \"PhyWorld: Physics-Faithful World Model for Video Generation,\" ",[607,1059,609],{},[386,1061,618],{"href":1062,"target":614,"className":1063},"https:\u002F\u002Farxiv.org\u002Fabs\u002F2605.19242",[616,617],[602,1065,1066,1067,610,1069],{"id":727},"J. Levy et al., \"Simulation Distillation: Pretraining World Models in Simulation for Rapid Real-World Adaptation,\" ",[607,1068,609],{},[386,1070,618],{"href":1071,"target":614,"className":1072},"https:\u002F\u002Farxiv.org\u002Fabs\u002F2603.15759",[616,617],[602,1074,1075,1076,610,1079],{"id":737},"N. Maslej et al., \"The 2026 AI Index Report, Chapter 2: Technical Performance,\" ",[607,1077,1078],{},"Stanford Institute for Human-Centered AI",[386,1080,618],{"href":1081,"target":614,"className":1082},"https:\u002F\u002Fhai.stanford.edu\u002Fassets\u002Ffiles\u002Fai_index_report_2026_chapter_2_technical.pdf",[616,617],{"title":748,"searchDepth":749,"depth":749,"links":1084},[1085,1086,1087,1088,1089,1090,1091],{"id":793,"depth":749,"text":794},{"id":812,"depth":749,"text":813},{"id":845,"depth":749,"text":846},{"id":886,"depth":749,"text":887},{"id":923,"depth":749,"text":924},{"id":944,"depth":749,"text":945},{"id":588,"depth":749,"text":592},"2026-05-23","World models are crossing the line from research artifact to production infrastructure. Embodied AI policies are now trained, evaluated, and shipped against simulators that were themselves learned from video, raising fresh questions about versioning, evaluation, and physical fidelity.",{"src":799},{"authors":1096,"badge":1099,"source":1101},[1097],{"avatar":1098,"name":769,"to":770},{"src":768},{"label":1100},"AI Infrastructure",{"name":774,"url":770},{"title":326,"description":1093},"7_868QwB1HRv0Mv38OVeLt5JB4dRnJnk8f5YnnuxJqg",{"id":1105,"title":154,"body":1106,"date":1393,"description":1394,"extension":762,"image":1395,"meta":1397,"navigation":775,"path":155,"seo":1405,"stem":156,"__hash__":1406,"_path":155},"insights\u002Fnews\u002Finsights\u002Fcontext-is-infrastructure.md",{"type":344,"value":1107,"toc":1385},[1108,1120,1128,1131,1141,1145,1153,1161,1169,1182,1186,1199,1220,1233,1246,1250,1257,1265,1291,1294,1298,1301,1307,1310,1313],[347,1109,351,1111,351,1115],{"className":1110},[350],[353,1112,154],{"className":1113,"id":1114},[356],"context-is-infrastructure-not-instructions",[359,1116,1119],{"className":1117,"id":1118},[362],"what-teams-gain-when-they-govern-ai-context-like-a-software-dependency","What teams gain when they govern AI context like a software dependency",[376,1121,1122,1123,1127],{},"A team replaces task-specific prompts with a generic \"improved\" template. Extraction accuracy drops from 100% to 90%. RAG compliance (the degree to which a model's answers stay grounded in retrieved documents rather than generating from its own training data) falls from 93.3% to 80% ",[383,1124,1125],{},[386,1126,389],{"href":388},". The model is the same. The new instructions look better on paper. What changed was the context, and nobody tested whether the change was safe before deploying it.",[376,1129,1130],{},"This is context regression, a term borrowed from software engineering where \"regression\" means a change that was supposed to improve something but degraded existing behavior instead. It behaves like any other dependency compatibility problem in a software supply chain, and the governance response, production contracts, risk-based test suites, compatibility gates, is the same one software teams already use for their other dependencies.",[376,1132,1133,1136,1137,1140],{},[386,1134,1135],{"href":159},"\"Context is the New Code\""," established context engineering as a formal discipline with its own taxonomy, maturity levels, and practitioner artifacts, and ",[386,1138,1139],{"href":307},"\"The Turn as the Unit of Quality\""," explored how structured iteration with checklists and selective memory improves turn-level quality. This article picks up a different thread. What happens when context moves from a single team's configuration file to an organizational dependency serving dozens of agents across thousands of daily interactions? Recent research suggests that the teams making the fastest progress are the ones applying familiar software supply chain governance to their context, and the returns are measurable.",[359,1142,1144],{"id":1143},"what-structured-context-unlocks","What Structured Context Unlocks",[376,1146,1147,1148,1152],{},"A study of 200 documented interactions across four AI tools found that incomplete context was associated with 72% of iteration cycles ",[383,1149,1150],{},[386,1151,396],{"href":395},". That number is worth sitting with. Nearly three-quarters of the rework, the back-and-forth where a human corrects, clarifies, and re-prompts, traced not to a bad model or a poorly worded instruction but to missing information that should have been available from the start.",[376,1154,1155,1156,1160],{},"When the same study introduced structured context assembly, a methodology that organizes context into five roles (Authority, Exemplar, Constraint, Rubric, and Metadata), iteration cycles dropped from an average of 3.8 to 2.0 per task, and first-pass acceptance rose from 32% to 55% ",[383,1157,1158],{},[386,1159,396],{"href":395},". Authority context establishes what standards govern the task. Exemplar context provides reference outputs that demonstrate the expected quality. Constraint context defines boundaries the output must respect. Rubric context specifies how the output will be evaluated. Metadata context supplies facts, dates, names, and domain-specific details. Having names for these roles is not a minor convenience, it is what makes the difference between ad hoc tuning and repeatable engineering, because a team that cannot describe what is missing from its context cannot systematically fix it.",[347,1162,368,1164],{"style":1163},"width: 100%; margin: 20px 0;",[370,1165],{"src":1166,"alt":1167,"style":1168},"https:\u002F\u002Fimages.unsplash.com\u002Fphoto-1639066648921-82d4500abf1a?w=800&auto=format&fit=crop","Rows of server equipment in a data center, analogous to how structured context engineering creates organized, reliable infrastructure rather than ad hoc configurations","width: 100%; height: 320px; object-fit: cover; object-position: center;",[376,1170,1171,1172,1176,1177,1181],{},"Like a well-organized server room where every cable run is labeled and every rack follows a standard layout, structured context gives a team the ability to reason about what the AI is actually working with. The evaluation-driven iteration research reinforces this by showing that context quality is not one-dimensional ",[383,1173,1174],{},[386,1175,389],{"href":388},". A change that improves instruction-following can simultaneously degrade extraction accuracy. A prompt that scores better on helpfulness can score worse on format compliance. The minimum viable evaluation suite (MVES) framework proposes tiered evaluation requirements, one set for general applications, another for retrieval-augmented generation systems, and a third for agentic workflows, precisely because quality along one dimension does not guarantee quality along others ",[383,1178,1179],{},[386,1180,389],{"href":388},". The practical implication is that quality has multiple dimensions that can trade against each other, and navigating those trade-offs requires measurement infrastructure, not intuition.",[359,1183,1185],{"id":1184},"governing-context-as-a-dependency","Governing Context as a Dependency",[376,1187,1188,1189,1193,1194,1198],{},"The clearest articulation of this shift comes from research that frames LLM update management as a software supply chain governance problem ",[383,1190,1191],{},[386,1192,403],{"href":402},". Hosted language model services evolve through provider-side updates without explicit version changes, so the API endpoint stays the same while the behavior underneath shifts. Empirical work cited within that framework documents cases where code execution accuracy dropped from 52% to 10% within three months with no version change on the consumer side ",[383,1195,1196],{},[386,1197,469],{"href":468},". This is behavioral drift (a gradual, unannounced change in how a model responds to the same inputs), and it affects every piece of context that was tuned against the previous behavior.",[376,1200,1201,1202,1206,1207,1211,1212,1215,1216,1219],{},"The proposed governance framework has three components that map directly to established software engineering practice ",[383,1203,1204],{},[386,1205,403],{"href":402},". ",[1208,1209,1210],"strong",{},"Production contracts"," define explicit behavioral rules with measurable thresholds, things like \"authentication code must pass security tests\" or \"JSON outputs must be valid.\" ",[1208,1213,1214],{},"Risk-category-based testing"," organizes evaluation around deployment risk areas rather than relying on a single aggregate score, preventing critical regressions in formatting or safety from being masked by overall performance improvements. ",[1208,1217,1218],{},"Compatibility gates"," block updates that fail defined thresholds, requiring review before a model update is adopted into production. None of these ideas are new to software engineering. What is new is recognizing that context, the system prompts, retrieved documents, and configuration files that shape AI behavior, is a dependency that deserves the same governance.",[376,1221,1222,1223,1227,1228,1232],{},"A readiness harness for LLM and RAG applications demonstrates what this looks like in practice ",[383,1224,1225],{},[386,1226,417],{"href":416},". The system combines automated benchmarks, OpenTelemetry observability (a standardized way to collect and export telemetry data like traces, metrics, and logs), and CI quality gates (automated checkpoints in the deployment pipeline that block releases if quality checks fail) under a minimal API contract. Rather than reducing readiness to a single metric, it aggregates workflow success, policy compliance, groundedness, retrieval hit rate, cost, and latency into scenario-weighted readiness scores. In ticket-routing experiments, the regression gates consistently rejected unsafe prompt variants before deployment ",[383,1229,1230],{},[386,1231,417],{"href":416},". This is a concrete example of the shift from \"the model was tested\" to \"the deployment pipeline tested every context change before it reached production.\"",[376,1234,1235,1236,1240,1241,1245],{},"One challenge specific to AI systems is that the same configuration can produce different outputs across runs. Traditional binary pass\u002Ffail testing struggles with this fundamental non-determinism. A regression testing framework designed for this problem replaces binary verdicts with three-valued probabilistic outcomes (Pass, Fail, Inconclusive) backed by confidence intervals and sequential analysis ",[383,1237,1238],{},[386,1239,424],{"href":423},". The framework achieves 78 to 100% cost reduction compared to naive repeated testing while maintaining statistical guarantees, and its behavioral fingerprinting approach achieves 86% detection power on regressions where binary pass\u002Ffail testing has 0% ",[383,1242,1243],{},[386,1244,424],{"href":423},". The cost reduction matters as much as the accuracy. Testing that is too expensive to run routinely is testing that does not get run, and context changes that do not get tested are the ones that cause production surprises.",[359,1247,1249],{"id":1248},"from-files-to-living-systems","From Files to Living Systems",[376,1251,1252,1253,425],{},"The governance patterns above treat context as a versioned artifact, something written, tested, and deployed. But a growing body of work suggests that this framing, while useful, captures only part of the picture. In production multi-agent systems, context is not a file. It is a runtime-constructed \"View\" projected into an agent's context window (the maximum amount of text a model can consider at once) from a pool of global artifacts, and that View changes dynamically based on the task, the step, and the state of the system ",[383,1254,1255],{},[386,1256,446],{"href":445},[376,1258,1259,1260,1264],{},"Research on what the authors call \"Loosely-Structured Software\" characterizes this as a class of system whose defining property is runtime generation and evolution under uncertainty ",[383,1261,1262],{},[386,1263,446],{"href":445},". Classic software architecture assumes build-time decomposition and slow-changing boundaries. Multi-agent AI systems violate those assumptions in three ways. First, an agent's effective program is determined not by compiled code but by a View assembled at runtime from system prompts, skills, plans, tools, and memories. Second, the connections between components form dynamically through semantic understanding rather than fixed function signatures. Third, the system's own executable substrate, the artifacts that mediate its behavior, can be rewritten by the system itself.",[376,1266,1267,1268,1206,1272,1275,1276,1278,1279,1282,1283,1286,1287,1290],{},"To make this governable, the research proposes a three-layer engineering framework ",[383,1269,1270],{},[386,1271,446],{"href":445},[1208,1273,1274],{},"View\u002FContext Engineering"," manages the execution environment and maintains task-relevant Views. This is the layer where the static context files that teams already write (the CLAUDE.md and AGENTS.md files examined in ",[386,1277,1135],{"href":159},") get assembled, filtered, and delivered at runtime. ",[1208,1280,1281],{},"Structure Engineering"," organizes the dynamic bindings between agents and artifacts, governing how components find and connect to each other. ",[1208,1284,1285],{},"Evolution Engineering"," manages the lifecycle of self-rewriting artifacts, ensuring that when the system modifies its own context (a capability that ",[386,1288,1289],{"href":171},"\"The Edge of the Underdefined\""," documents self-improving agents already demonstrating), those modifications remain within governed bounds.",[376,1292,1293],{},"This is where context infrastructure becomes genuinely adaptive. Instead of choosing between static configuration files (reliable but rigid) and autonomous self-modification (flexible but ungoverned), the three-layer framework offers a middle path. Context can evolve in response to operational feedback, while infrastructure constraints prevent that evolution from drifting outside acceptable bounds. The combination of governance patterns from the supply chain framing with the runtime adaptivity from the loosely-structured software framing points toward a more complete picture of what production context infrastructure might look like.",[359,1295,1297],{"id":1296},"the-maturity-opportunity","The Maturity Opportunity",[376,1299,1300],{},"The infrastructure patterns described here, production contracts, multi-dimensional evaluation, CI gates, statistical regression testing, runtime View management, each have working implementations backed by empirical evidence. The gap between what the research demonstrates and what most teams have actually built is mostly one of adoption, not of available tools.",[376,1302,1303,1304,1306],{},"Survey data suggests that prompt usage in software engineering remains largely ad hoc, with prompts refined through trial-and-error and rarely reused. As ",[386,1305,1135],{"href":159}," noted, only about 5% of surveyed open-source repositories have adopted any context file format at all. The parallel to early unit testing adoption or early version control adoption is hard to miss. A practice that starts as optional among a skilled minority tends to become standard once enough teams experience the cost of not doing it.",[376,1308,1309],{},"What distinguishes this moment is that the infrastructure does not need to be invented from scratch. Supply chain governance, production testing methodology, continuous deployment practice, and statistical experiment design all have established patterns that transfer directly to context management. Treating context as infrastructure is largely a matter of applying existing engineering discipline to a new class of artifact, one that happens to shape every decision an AI system makes.",[376,1311,1312],{},"The teams moving fastest appear to be the ones that recognized this early. They built the infrastructure to measure, test, and govern the context their models consume, and that investment compounded over time. For teams still tuning prompts by hand and evaluating by feel, the patterns are available to adopt directly, without rediscovering the hard lessons from scratch.",[347,1314,351,1316,351,1318],{"className":1315},[588,589],[359,1317,592],{"id":588},[594,1319,368,1321,368,1330,368,1339,368,1348,368,1357,368,1366,368,1375,351],{"className":1320},[597,598,599,600],[602,1322,1323,1324,610,1326],{"id":604},"D. Commey, \"When 'Better' Prompts Hurt: Evaluation-Driven Iteration for LLM Applications,\" ",[607,1325,609],{},[386,1327,618],{"href":1328,"target":614,"className":1329},"https:\u002F\u002Farxiv.org\u002Fabs\u002F2601.22025",[616,617],[602,1331,1332,1333,610,1335],{"id":621},"E. Calboreanu, \"Context Engineering: A Practitioner Methodology for Structured Human-AI Collaboration,\" ",[607,1334,609],{},[386,1336,618],{"href":1337,"target":614,"className":1338},"https:\u002F\u002Farxiv.org\u002Fabs\u002F2604.04258",[616,617],[602,1340,1341,1342,610,1344],{"id":632},"M. S. Chishti et al., \"Test Before You Deploy: Governing Updates in the LLM Supply Chain,\" ",[607,1343,609],{},[386,1345,618],{"href":1346,"target":614,"className":1347},"https:\u002F\u002Farxiv.org\u002Fabs\u002F2604.27789",[616,617],[602,1349,1350,1351,610,1353],{"id":643},"A. C. Maiorano, \"LLM Readiness Harness: Evaluation, Observability, and CI Gates for LLM\u002FRAG Applications,\" ",[607,1352,609],{},[386,1354,618],{"href":1355,"target":614,"className":1356},"https:\u002F\u002Farxiv.org\u002Fabs\u002F2603.27355",[616,617],[602,1358,1359,1360,610,1362],{"id":655},"V. P. Bhardwaj, \"AgentAssay: Token-Efficient Regression Testing for Non-Deterministic AI Agent Workflows,\" ",[607,1361,609],{},[386,1363,618],{"href":1364,"target":614,"className":1365},"https:\u002F\u002Farxiv.org\u002Fabs\u002F2603.02601",[616,617],[602,1367,1368,1369,610,1371],{"id":665},"W. Zhang et al., \"Loosely-Structured Software: Engineering Context, Structure, and Evolution Entropy in Runtime-Rewired Multi-Agent Systems,\" ",[607,1370,609],{},[386,1372,618],{"href":1373,"target":614,"className":1374},"https:\u002F\u002Farxiv.org\u002Fabs\u002F2603.15690",[616,617],[602,1376,1377,1378,1380,1381],{"id":677},"L. Chen et al., \"How Is ChatGPT's Behavior Changing over Time?,\" ",[607,1379,609],{},", 2023, ",[386,1382,618],{"href":1383,"target":614,"className":1384},"https:\u002F\u002Farxiv.org\u002Fabs\u002F2307.09009",[616,617],{"title":748,"searchDepth":749,"depth":749,"links":1386},[1387,1388,1389,1390,1391,1392],{"id":1118,"depth":749,"text":1119},{"id":1143,"depth":749,"text":1144},{"id":1184,"depth":749,"text":1185},{"id":1248,"depth":749,"text":1249},{"id":1296,"depth":749,"text":1297},{"id":588,"depth":749,"text":592},"2026-05-10","Most teams treat AI context as a runtime concern, something to tune session by session. The teams making the fastest progress treat it as a software dependency, versioned, tested, and governed. The infrastructure patterns for doing this already exist.",{"src":1396},"https:\u002F\u002Fimages.unsplash.com\u002Fphoto-1558494949-ef010cbdcc31?w=800&auto=format&fit=crop",{"authors":1398,"badge":1402,"source":1404},[1399],{"avatar":1400,"name":769,"to":1401},{"src":768},"https:\u002F\u002Fwww.linkedin.com\u002Fin\u002Fmarkwilliamsthinkata\u002F",{"label":1403},"AI Engineering",{"name":774,"url":770},{"title":154,"description":1394},"NX6IF5zuP52v2h_JVoijqArp9QfRO5Oi5Vx8qBrSAfM",1792582876985]