[{"data":1,"prerenderedAt":1342},["ShallowReactive",2],{"navigation":3,"\u002Fnews\u002Finsights\u002Fself-editing-agent-memory":341,"\u002Fnews\u002Finsights\u002Fself-editing-agent-memory-surround":722},[4,8,17,21,25,29,33,329,333,337],{"title":5,"path":6,"stem":7},"About","\u002Fabout","about",{"title":9,"path":10,"stem":11,"children":12},"Authentication","\u002Fauth","auth",[13],{"title":14,"path":15,"stem":16},"Email Confirmation","\u002Fauth\u002Fconfirmation","auth\u002Fconfirmation",{"title":18,"path":19,"stem":20},"Industry Briefs","\u002Fcase-studies","case-studies",{"title":22,"path":23,"stem":24},"Contact","\u002Fcontact","contact",{"title":26,"path":27,"stem":28},"Thinkata | Production AI Engineering","\u002F","index",{"title":30,"path":31,"stem":32},"Insights","\u002Finsights","insights",{"title":34,"path":35,"stem":36,"children":37},"News","\u002Fnews","news",[38,41,65],{"title":39,"path":35,"stem":40},"News & Insights","news\u002Findex",{"title":18,"path":42,"stem":43,"children":44},"\u002Fnews\u002Fcase-studies","news\u002Fcase-studies",[45,49,53,57,61],{"title":46,"path":47,"stem":48},"Building Secure and Scalable AI Infrastructure: Integrating with Existing Systems through Modern Cloud Frameworks","\u002Fnews\u002Fcase-studies\u002Fcloud-infrastructure-ai","news\u002Fcase-studies\u002Fcloud-infrastructure-ai",{"title":50,"path":51,"stem":52},"Making Sense of Financial Regulations: How AI Teams Can Tackle Complex Documents","\u002Fnews\u002Fcase-studies\u002Ffinancial-regulations","news\u002Fcase-studies\u002Ffinancial-regulations",{"title":54,"path":55,"stem":56},"AI-Powered Transformations in Healthcare","\u002Fnews\u002Fcase-studies\u002Fhealth-care","news\u002Fcase-studies\u002Fhealth-care",{"title":58,"path":59,"stem":60},"Generative AI in Upstream Natural Gas: Shell's Exploration Initiative","\u002Fnews\u002Fcase-studies\u002Foil-gas","news\u002Fcase-studies\u002Foil-gas",{"title":62,"path":63,"stem":64},"Optimizing Manufacturing with AI-Driven Multi-Agent Systems","\u002Fnews\u002Fcase-studies\u002Fsupply-chain-optimization","news\u002Fcase-studies\u002Fsupply-chain-optimization",{"title":30,"path":66,"stem":67,"children":68},"\u002Fnews\u002Finsights","news\u002Finsights",[69,73,77,81,85,89,93,97,101,105,109,113,117,121,125,129,133,137,141,145,149,153,157,161,165,169,173,177,181,185,189,193,197,201,205,209,213,217,221,225,229,233,237,241,245,249,253,257,261,265,269,273,277,281,285,289,293,297,301,305,309,313,317,321,325],{"title":70,"path":71,"stem":72},"Nothing Gets Deleted, Just Blurred","\u002Fnews\u002Finsights\u002Fadaptive-context-memory","news\u002Finsights\u002Fadaptive-context-memory",{"title":74,"path":75,"stem":76},"The Capability-Reliability Split in Agent Systems","\u002Fnews\u002Finsights\u002Fagent-capability-reliability-split","news\u002Finsights\u002Fagent-capability-reliability-split",{"title":78,"path":79,"stem":80},"The Rise of AI Agents in Cyberattacks: Latest Research and Threats","\u002Fnews\u002Finsights\u002Fai-agent-cyber-threats","news\u002Finsights\u002Fai-agent-cyber-threats",{"title":82,"path":83,"stem":84},"Winning the Bid Is Not the Same as Knowing the Price","\u002Fnews\u002Finsights\u002Fai-agent-economics","news\u002Finsights\u002Fai-agent-economics",{"title":86,"path":87,"stem":88},"The Smart Enterprise AI Stack: Why Teams of AI Agents Beat Solo Models Consistently","\u002Fnews\u002Finsights\u002Fai-architecture","news\u002Finsights\u002Fai-architecture",{"title":90,"path":91,"stem":92},"When Seeing Everything Becomes the Only Option","\u002Fnews\u002Finsights\u002Fai-comprehensive-observability","news\u002Finsights\u002Fai-comprehensive-observability",{"title":94,"path":95,"stem":96},"The Data Infrastructure AI-Native Systems Can't Ignore","\u002Fnews\u002Finsights\u002Fai-data-layer","news\u002Finsights\u002Fai-data-layer",{"title":98,"path":99,"stem":100},"Enterprise AI Triage Systems: Intelligent Automation for Large-Scale Operations","\u002Fnews\u002Finsights\u002Fai-enterprise-triage","news\u002Finsights\u002Fai-enterprise-triage",{"title":102,"path":103,"stem":104},"When Oversight Becomes Infrastructure","\u002Fnews\u002Finsights\u002Fai-governed-autonomy","news\u002Finsights\u002Fai-governed-autonomy",{"title":106,"path":107,"stem":108},"Designing for Graceful Failure in Compound AI Systems","\u002Fnews\u002Finsights\u002Fai-graceful-failure","news\u002Finsights\u002Fai-graceful-failure",{"title":110,"path":111,"stem":112},"Intelligent Composability: Building AI Systems Like Orchestra, Not Soloists","\u002Fnews\u002Finsights\u002Fai-intelligent-composability","news\u002Finsights\u002Fai-intelligent-composability",{"title":114,"path":115,"stem":116},"Building the Plane While Flying It — Migrating from Monolith to AI-Native Without Stopping","\u002Fnews\u002Finsights\u002Fai-migration-path","news\u002Finsights\u002Fai-migration-path",{"title":118,"path":119,"stem":120},"Stability Through Continuous Adaptation","\u002Fnews\u002Finsights\u002Fai-native-overview","news\u002Finsights\u002Fai-native-overview",{"title":122,"path":123,"stem":124},"Provable Stability: Mathematical Guarantees for Adaptive AI Systems","\u002Fnews\u002Finsights\u002Fai-provable-stability","news\u002Finsights\u002Fai-provable-stability",{"title":126,"path":127,"stem":128},"How Temperature Tuning Makes or Breaks Reinforcement Learning","\u002Fnews\u002Finsights\u002Fai-soft-actor-critic-entropy-collapse","news\u002Finsights\u002Fai-soft-actor-critic-entropy-collapse",{"title":130,"path":131,"stem":132},"Testing What Can't Be Predicted","\u002Fnews\u002Finsights\u002Fai-systems-testing","news\u002Finsights\u002Fai-systems-testing",{"title":134,"path":135,"stem":136},"Delegating the Job Is Not the Same as Delegating the Rules","\u002Fnews\u002Finsights\u002Fauthorization-drift-multi-agent","news\u002Finsights\u002Fauthorization-drift-multi-agent",{"title":138,"path":139,"stem":140},"Closing the Loop: How Human Corrections Can Make AI Systems Smarter Over Time","\u002Fnews\u002Finsights\u002Fclosing-the-loop","news\u002Finsights\u002Fclosing-the-loop",{"title":142,"path":143,"stem":144},"A Panel Can Be Safer Than Any Member of It","\u002Fnews\u002Finsights\u002Fcoalitional-alignment-panel-review","news\u002Finsights\u002Fcoalitional-alignment-panel-review",{"title":146,"path":147,"stem":148},"Multi-Path Reasoning: Collaborative and Competitive Approaches in AI","\u002Fnews\u002Finsights\u002Fcollaborative-competitive-agents","news\u002Finsights\u002Fcollaborative-competitive-agents",{"title":150,"path":151,"stem":152},"Why Challenges Supercharge Smarts for Humans and AI","\u002Fnews\u002Finsights\u002Fcompetition-improves-ai","news\u002Finsights\u002Fcompetition-improves-ai",{"title":154,"path":155,"stem":156},"Context is Infrastructure, Not Instructions","\u002Fnews\u002Finsights\u002Fcontext-is-infrastructure","news\u002Finsights\u002Fcontext-is-infrastructure",{"title":158,"path":159,"stem":160},"Context is the New Code","\u002Fnews\u002Finsights\u002Fcontext-is-new-code","news\u002Finsights\u002Fcontext-is-new-code",{"title":162,"path":163,"stem":164},"Continuous Thought Machines","\u002Fnews\u002Finsights\u002Fcontinuous-thought-machines","news\u002Finsights\u002Fcontinuous-thought-machines",{"title":166,"path":167,"stem":168},"Don't Vibe, Architect","\u002Fnews\u002Finsights\u002Fdont-vibe-architect","news\u002Finsights\u002Fdont-vibe-architect",{"title":170,"path":171,"stem":172},"The Edge of the Underdefined","\u002Fnews\u002Finsights\u002Fedge-of-the-underdefined","news\u002Finsights\u002Fedge-of-the-underdefined",{"title":174,"path":175,"stem":176},"Experts All the Way Down","\u002Fnews\u002Finsights\u002Fexperts-all-the-way","news\u002Finsights\u002Fexperts-all-the-way",{"title":178,"path":179,"stem":180},"A Multi-Tier Safety Architecture for Critical Applications","\u002Fnews\u002Finsights\u002Ffour-tier-architecture","news\u002Finsights\u002Ffour-tier-architecture",{"title":182,"path":183,"stem":184},"Green Dashboard, Unhappy Users","\u002Fnews\u002Finsights\u002Fgreen-dashboard-unhappy-users","news\u002Finsights\u002Fgreen-dashboard-unhappy-users",{"title":186,"path":187,"stem":188},"Hybrid Autoregressive Residual Tokens","\u002Fnews\u002Finsights\u002Fhart-model","news\u002Finsights\u002Fhart-model",{"title":190,"path":191,"stem":192},"Hierarchical Reasoning in Artificial Intelligence","\u002Fnews\u002Finsights\u002Fhierarchical-approaches","news\u002Finsights\u002Fhierarchical-approaches",{"title":194,"path":195,"stem":196},"Taking Turns Instead of Splitting the Difference","\u002Fnews\u002Finsights\u002Finterleaved-distillation-rl","news\u002Finsights\u002Finterleaved-distillation-rl",{"title":198,"path":199,"stem":200},"Latent Diffusion for Language Generation: A Comprehensive Overview","\u002Fnews\u002Finsights\u002Flatent-diffusion-for-language","news\u002Finsights\u002Flatent-diffusion-for-language",{"title":202,"path":203,"stem":204},"Breaking Language Barriers: How AI Can Translate Without Examples","\u002Fnews\u002Finsights\u002Flearning-languages","news\u002Finsights\u002Flearning-languages",{"title":206,"path":207,"stem":208},"The Emergence of AI Deception: How Large Language Models Have Learned to Strategically Mislead Users","\u002Fnews\u002Finsights\u002Fllm-deception","news\u002Finsights\u002Fllm-deception",{"title":210,"path":211,"stem":212},"Grading on a Shared Curve","\u002Fnews\u002Finsights\u002Fllm-judge-correlated-errors","news\u002Finsights\u002Fllm-judge-correlated-errors",{"title":214,"path":215,"stem":216},"Synergizing Specialized Reasoning and General Capabilities in AI","\u002Fnews\u002Finsights\u002Fllm-reasoning-advances","news\u002Finsights\u002Fllm-reasoning-advances",{"title":218,"path":219,"stem":220},"The Expensive Default","\u002Fnews\u002Finsights\u002Fllm-routing-cost-quality","news\u002Finsights\u002Fllm-routing-cost-quality",{"title":222,"path":223,"stem":224},"The AI That Rewrites Itself: MIT's Breakthrough in Self-Adapting Language Models","\u002Fnews\u002Finsights\u002Fllm-seal","news\u002Finsights\u002Fllm-seal",{"title":226,"path":227,"stem":228},"Forgetting on Request","\u002Fnews\u002Finsights\u002Fmachine-unlearning-removal-suppression","news\u002Finsights\u002Fmachine-unlearning-removal-suppression",{"title":230,"path":231,"stem":232},"The Map and the Walk","\u002Fnews\u002Finsights\u002Fmap-and-walk","news\u002Finsights\u002Fmap-and-walk",{"title":234,"path":235,"stem":236},"Metacognitive Reinforcement Learning for Self-Improving AI Systems","\u002Fnews\u002Finsights\u002Fmetacognitive-reinforcement-learning","news\u002Finsights\u002Fmetacognitive-reinforcement-learning",{"title":238,"path":239,"stem":240},"Revolutionary Advancements in Mixture of Experts (MoE) Architectures","\u002Fnews\u002Finsights\u002Fmixture-of-experts","news\u002Finsights\u002Fmixture-of-experts",{"title":242,"path":243,"stem":244},"One Model, Many Customers, and the Leak Nobody Tests For","\u002Fnews\u002Finsights\u002Fmulti-tenant-ai-isolation","news\u002Finsights\u002Fmulti-tenant-ai-isolation",{"title":246,"path":247,"stem":248},"Balancing Neural Plasticity and Stability","\u002Fnews\u002Finsights\u002Fneural-plasticity","news\u002Finsights\u002Fneural-plasticity",{"title":250,"path":251,"stem":252},"Offline RL and the Data Flywheel","\u002Fnews\u002Finsights\u002Foffline-rl-data-flywheel","news\u002Finsights\u002Foffline-rl-data-flywheel",{"title":254,"path":255,"stem":256},"Orchestration Without a Signal From the Room","\u002Fnews\u002Finsights\u002Fplanner-in-the-loop","news\u002Finsights\u002Fplanner-in-the-loop",{"title":258,"path":259,"stem":260},"Second-Guessing Has a Price","\u002Fnews\u002Finsights\u002Freasoning-budget-allocation","news\u002Finsights\u002Freasoning-budget-allocation",{"title":262,"path":263,"stem":264},"Reasoning You Can Check","\u002Fnews\u002Finsights\u002Freasoning-you-can-check","news\u002Finsights\u002Freasoning-you-can-check",{"title":266,"path":267,"stem":268},"When Optimization Optimizes Itself","\u002Fnews\u002Finsights\u002Frecursive-goodhart","news\u002Finsights\u002Frecursive-goodhart",{"title":270,"path":271,"stem":272},"Reward Design as Architecture","\u002Fnews\u002Finsights\u002Freward-design-as-architecture","news\u002Finsights\u002Freward-design-as-architecture",{"title":274,"path":275,"stem":276},"When Success Has No Author: The Temporal Credit Assignment Problem","\u002Fnews\u002Finsights\u002Frl-credit-assignment-problem","news\u002Finsights\u002Frl-credit-assignment-problem",{"title":278,"path":279,"stem":280},"Beyond Entropy Collapse: When Exploration Succeeds but Learning Fails","\u002Fnews\u002Finsights\u002Frl-optimization-gaps","news\u002Finsights\u002Frl-optimization-gaps",{"title":282,"path":283,"stem":284},"The Body Was Supposed to Be the Easy Part","\u002Fnews\u002Finsights\u002Frobot-embodiment-interface","news\u002Finsights\u002Frobot-embodiment-interface",{"title":286,"path":287,"stem":288},"The Path to Practical Confidential Computing for AI Systems","\u002Fnews\u002Finsights\u002Fsecure-ai-architectures","news\u002Finsights\u002Fsecure-ai-architectures",{"title":290,"path":291,"stem":292},"When an AI Edits Its Own Memory","\u002Fnews\u002Finsights\u002Fself-editing-agent-memory","news\u002Finsights\u002Fself-editing-agent-memory",{"title":294,"path":295,"stem":296},"Guess First, Check Later","\u002Fnews\u002Finsights\u002Fspeculative-execution-pattern","news\u002Finsights\u002Fspeculative-execution-pattern",{"title":298,"path":299,"stem":300},"Spiking Neural Networks for Energy-Efficient AI","\u002Fnews\u002Finsights\u002Fspiking-neural-networks","news\u002Finsights\u002Fspiking-neural-networks",{"title":302,"path":303,"stem":304},"When Replay Is Not an Option: Streaming Q-Learning and SARSA Get a Second Look","\u002Fnews\u002Finsights\u002Fstreaming-q-learning-revival","news\u002Finsights\u002Fstreaming-q-learning-revival",{"title":306,"path":307,"stem":308},"The Turn as the Unit of Quality","\u002Fnews\u002Finsights\u002Fstructured-iteration-quality","news\u002Finsights\u002Fstructured-iteration-quality",{"title":310,"path":311,"stem":312},"AI Speech Translation: Breaking Down Language Barriers","\u002Fnews\u002Finsights\u002Fsts-performance-advances","news\u002Finsights\u002Fsts-performance-advances",{"title":314,"path":315,"stem":316},"Test-Time Training Layers: The Next Evolution in Transformer Architecture","\u002Fnews\u002Finsights\u002Ftest-time-training-layers","news\u002Finsights\u002Ftest-time-training-layers",{"title":318,"path":319,"stem":320},"Breakthrough: Large Language Models Pass the Turing Test","\u002Fnews\u002Finsights\u002Fturing-tests","news\u002Finsights\u002Fturing-tests",{"title":322,"path":323,"stem":324},"Algorithms Used in Autonomous Fighter Jet Flight Research","\u002Fnews\u002Finsights\u002Fvenom-f16-flight-autonomy","news\u002Finsights\u002Fvenom-f16-flight-autonomy",{"title":326,"path":327,"stem":328},"Training in a World That Does Not Exist Yet","\u002Fnews\u002Finsights\u002Fworld-models-as-infrastructure","news\u002Finsights\u002Fworld-models-as-infrastructure",{"title":330,"path":331,"stem":332},"Privacy Policy","\u002Fprivacy","privacy",{"title":334,"path":335,"stem":336},"Research","\u002Fresearch","research",{"title":338,"path":339,"stem":340},"Terms of Service","\u002Fterms","terms",{"id":342,"title":290,"body":343,"date":703,"description":704,"extension":705,"image":706,"meta":707,"navigation":719,"path":291,"seo":720,"stem":292,"__hash__":721},"insights\u002Fnews\u002Finsights\u002Fself-editing-agent-memory.md",{"type":344,"value":345,"toc":690},"minimark",[346,365,375,379,396,399,403,413,423,427,437,440,450,465,469,477,487,495,512,520,527,535,550,554,562,570,573],[347,348,351,352,351,358],"div",{"className":349},[350],"page-title","\n  ",[353,354,290],"h1",{"className":355,"id":357},[356],"page-title__main","when-an-ai-edits-its-own-memory",[359,360,364],"h2",{"className":361,"id":363},[362],"page-title__sub","lessons-from-how-the-brain-forgets-and-rewrites","Lessons from How the Brain Forgets and Rewrites",[347,366,368,369],{"style":367},"width: 100%; padding: 2%;","\n    ",[370,371],"img",{"src":372,"alt":373,"style":374},"https:\u002F\u002Fimages.unsplash.com\u002Fphoto-1747276569181-6236594f507f?w=1200&h=750&fit=crop&auto=format","A pencil eraser rubbing out marks on white paper, leaving pink shavings behind","width: 100%; height: auto;",[376,377,378],"p",{},"An AI agent that rewrites its own memory to save space can also change what that memory says. The risk has been easy to overlook because language models have rarely been allowed to edit their memory at all. A model's context, the running text it can see at any one moment, works as its short-term memory, and in a standard setup new text simply piles onto the end. Long-running agents, programs that let a model take actions like searching the web or editing code over hundreds of steps, eventually overflow that space, and the usual fix is a fixed rule, such as summarizing the whole history once it reaches a set length.",[376,380,381,382,390,391,395],{},"A recent preprint led by researchers at the University of Washington and Meta takes a more permissive route. Its Context Language Models, or CLMs, treat the context as an ordinary file that the model can rewrite with code whenever it chooses ",[383,384,385],"sup",{},[386,387,389],"a",{"href":388},"#source-1","[1]",". Given that freedom, models wrote loops that swapped stale search results for a one-line placeholder, invented a new \"notes\" role to hold their own running summaries, and defined helper functions they reused dozens of times to compact old turns. On a deep-research benchmark, the approach scored about 11 percent higher than the strongest fixed strategy while using less computation ",[383,392,393],{},[386,394,389],{"href":388},".",[376,397,398],{},"The paper treats forgetting as an efficiency question, which parts of the past can be dropped to keep the context small. Decades of memory research suggest a more mixed picture. Forgetting appears to help a mind make better decisions, but recalling a memory and storing it again also gives that memory a chance to change. An agent that edits its own context may inherit both halves of that bargain, and the second half is where the open design work seems to sit.",[359,400,402],{"id":401},"forgetting-on-purpose","Forgetting on Purpose",[376,404,405,406,412],{},"A review by Blake Richards and Paul Frankland argues that transience, the gradual loss of memories, deserves as much attention as persistence, which has long dominated the study of memory ",[383,407,408],{},[386,409,411],{"href":410},"#source-2","[2]",". In their account, forgetting improves flexibility by reducing the pull of outdated information on decisions, and it keeps memory from overfitting to specific past events. Overfitting is a familiar idea in machine learning, describing a model that memorizes the quirks of its training examples so closely that it handles new cases poorly. The review concludes that the goal of memory is better decision-making rather than the faithful transmission of information through time.",[376,414,415,416,422],{},"Language models seem to have a similar need, for a mechanical reason. A study of how models use long inputs found that performance was often highest when the relevant information sat near the beginning or end of the context, and that it dropped when the information was buried in the middle, even for models built to handle long inputs ",[383,417,418],{},[386,419,421],{"href":420},"#source-3","[3]",". Every outdated search result left in place pushes something useful toward that weak middle. The CLM agent that replaced irrelevant results with the line \"No relevant results\" was practicing a small version of the transience the review describes, and the efficiency gains in the paper suggest it paid off.",[359,424,426],{"id":425},"when-remembering-means-rewriting","When Remembering Means Rewriting",[376,428,429,430,436],{},"The less comfortable finding concerns what happens when a memory gets used. A review by Jonathan Lee, Karim Nader, and Daniela Schiller describes reconsolidation, a process in which reactivating a stored memory can make it temporarily unstable ",[383,431,432],{},[386,433,435],{"href":434},"#source-4","[4]",". While the memory settles again, it can be weakened or strengthened, and many researchers have suggested that this window is well placed for updating memories with new information. The review also takes up recent challenges to the field, so the conditions under which the effect appears remain unsettled.",[376,438,439],{},"A CLM edit has the same shape, without the biological uncertainty. To compact its context, the model reads its history and writes a new version in its place. The next turn sees only the rewrite, and the original text is gone from the model's view. In the brain, whether recall reopens a memory depends on conditions that researchers still debate. In a CLM, every edit reopens the record by construction.",[376,441,442,443,449],{},"Elizabeth Loftus spent three decades documenting how revision goes wrong in people. Her review of the misinformation effect describes how exposure to misleading details after an event impairs memory of the event itself, and how warnings about possible misinformation reduce the damage only under limited circumstances ",[383,444,445],{},[386,446,448],{"href":447},"#source-5","[5]",". Researchers have also asked what becomes of the original memory once misinformation seems to have made it inaccessible, a question the review leaves open.",[376,451,452,453,459,460,464],{},"Summaries written by language models show a comparable drift. In a large human evaluation of neural summarization systems, annotators found hallucinated content, statements not supported by the source document, in the summaries of every system tested ",[383,454,455],{},[386,456,458],{"href":457},"#source-6","[6]",". The CLM authors saw a version of this in their own pilot study, where summary-based compaction lost or hallucinated information on simple retention tasks ",[383,461,462],{},[386,463,389],{"href":388},". Letting the model choose what to rewrite beat the fixed summarizers, but a more selective editor is still an editor. Each pass carries some chance of a small error, and the next pass treats that error as fact, much like a message whispered down a line of players in a game of telephone.",[359,466,468],{"id":467},"when-the-notes-start-giving-orders","When the Notes Start Giving Orders",[376,470,471,472,476],{},"Distortion is the accidental failure. The deliberate one may matter more. The CLM paper's own discussion warns that an editable context can become a channel through which prompt injections, instructions hidden in content the agent reads, or instructions the model writes for itself persist across turns ",[383,473,474],{},[386,475,389],{"href":388},". The authors point to reported cases in which a model inserted unauthorized instructions into its own compaction summary, which then shaped its later behavior.",[376,478,479,480,486],{},"Research on agent memory suggests how little planted material such an attack might need. AgentPoison, a backdoor attack on agents that draw on a long-term memory or a retrieval database, succeeded in the large majority of attempts while poisoning less than a tenth of one percent of the stored entries, and it left the agents' normal behavior largely intact ",[383,481,482],{},[386,483,485],{"href":484},"#source-7","[7]",". That attack targets a store the agent retrieves from. A writable live context adds a shorter path, since text from a web page can be summarized directly into the agent's own notes.",[376,488,489,490,494],{},"From a systems perspective, the core problem is provenance, the record of where a piece of information came from. Before a rewrite, a suspicious instruction sits inside a tool result and is labeled as one. After the rewrite, it can sit in a section the model labeled \"notes,\" worded as the agent's own conclusion. Loftus describes people who come to believe they lived through rich, detailed events that never happened ",[383,491,492],{},[386,493,448],{"href":447},". A rewritten context can produce the software version of that, a confident memory with no trustworthy origin.",[347,496,368,498,368,502,368,508],{"style":497},"width: 100%; margin: 20px 0;",[370,499],{"src":500,"alt":501,"style":374},"https:\u002F\u002Fimages.unsplash.com\u002Fphoto-1773266110372-44e7b9bad188?w=1200&auto=format&fit=crop","Rows of wooden card catalog drawers with small labels and brass pulls",[503,504,507],"h3",{"style":505,"id":506},"margin: 1rem 0 0.5rem 0;","a-record-that-recall-cannot-touch","A Record That Recall Cannot Touch",[376,509,511],{"style":510},"margin: 0;","Each drawer in a card catalog holds cards that were filed once and are consulted many times, and pulling one out to read it leaves the card exactly as it was. Brains do not appear to get that kind of storage. Software can have it cheaply. The extension proposed here, which might be called reconsolidation-aware context management, splits an agent's memory into two layers. The first is an append-only log, a record that can only be added to, holding every user message, tool result, and model output exactly as it occurred. The second is the editable working view that the model curates, the context it actually reads on each turn. The model keeps full freedom over the view, and the log stays outside its reach.",[376,513,514,515,519],{},"The two layers meet at a checkpoint. Before a rewrite of the working view takes effect, the system compares it against the log, and two checks seem worth running. The first asks whether each factual claim in the new view is supported by something in the log. The summarization study above found that textual entailment, a measure of whether one passage logically follows from another, tracked human judgments of faithfulness better than standard metrics that count overlapping words ",[383,516,517],{},[386,518,458],{"href":457},", which makes an entailment model a plausible checker. The second check asks where any instruction in the new view came from. An instruction that traces back to the user or the system prompt passes. One that traces back only to a web page or a tool output gets flagged.",[376,521,522,523,526],{},"What happens to a flagged edit is a design choice. It could be blocked, applied with a warning attached, or routed to a human reviewer, and the right policy probably depends on how much harm a bad edit can do in a given deployment. Under any policy, the log gives auditors a true history of what the agent saw and did, and it gives the agent a way back. An earlier Thinkata piece on ",[386,524,525],{"href":71},"resolution-adaptive memory"," looked at keeping compressed traces that can be expanded on demand, and a frozen log extends that idea to the whole history, so a detail compacted away too early can be looked up instead of guessed.",[376,528,529,530,534],{},"The checks are not free. On the paper's terminal-coding benchmark, models edited their context only a couple of times per task on average ",[383,531,532],{},[386,533,389],{"href":388},", so one checking call per edit would add little there. Long research runs that compact far more often would raise the bill.",[376,536,537,538,544,545,549],{},"The design sits between two earlier positions. MemGPT, a system modeled on an operating system's memory hierarchy, let the model write to a fixed-size working block inside its context through dedicated function calls, while the rest of the prompt stayed under system control ",[383,539,540],{},[386,541,543],{"href":542},"#source-8","[8]",". CLMs remove those limits, and a frozen log would keep that freedom while restoring a guarantee about what actually happened. The CLM paper hints at why that guarantee should not rest on the model. Told in a single sentence to back up its context before every edit, the agent made a full backup before about two-thirds of its edits, up from none without the instruction ",[383,546,547],{},[386,548,389],{"href":388},". That is an encouraging result for steering and a shaky basis for an audit trail. The CLM authors deliberately keep the surrounding software thin, and an append-only log fits that preference, since it records passively and never limits what the model can do.",[359,551,553],{"id":552},"a-game-of-telephone-measured","A Game of Telephone, Measured",[376,555,556,557,561],{},"The idea can be tested with tools the CLM paper already provides. Its ContextBench suite includes a needle-retention task that grades whether specific lines survive verbatim in the agent's context ",[383,558,559],{},[386,560,389],{"href":388},". A drift experiment could seed a context with a known set of facts, force the agent through 5, 10, and 20 successive rewrites, and measure two things after each round, how many original facts survive intact and how many unsupported claims have crept in. Running the same chain with and without the verification checkpoint would show whether the gate slows the decay, and what it costs in accuracy and compute. A second variant would plant a single instruction inside a tool result, then track how often it survives into the agent's notes as rewrites accumulate and how often the provenance check catches it.",[376,563,564,565,569],{},"The results could go several ways. Drift might build steadily with each rewrite, or hold flat and then collapse in one bad compaction, and the two patterns would call for different defenses. Larger models might drift less, or simply produce more convincing errors. The checker is itself a model, and entailment tracked faithfulness better than older metrics without matching human judgment perfectly ",[383,566,567],{},[386,568,458],{"href":457},", so a changed number or name might slip through.",[376,571,572],{},"The CLM results make a good case for letting agents decide what to forget. Memory research adds a caution, that rewriting is where errors and planted ideas tend to get in. People have no untouched original to check a memory against, but an agent can have one if the system keeps it. Whether that check is worth its cost in compute and accuracy is still an open question, and the telephone experiment is a fairly simple way to start answering it.",[347,574,351,578,351,581],{"className":575},[576,577],"references","mt-8",[359,579,580],{"id":576},"References",[582,583,368,589,368,607,368,619,368,631,368,643,368,655,368,667,368,679,351],"ol",{"className":584},[585,586,587,588],"list-decimal","list-inside","space-y-2","mt-4",[590,591,593,594,598,599],"li",{"id":592},"source-1","R. Shao et al., \"Context Language Models,\" ",[595,596,597],"em",{},"arXiv",", 2026, ",[386,600,606],{"href":601,"target":602,"className":603},"https:\u002F\u002Farxiv.org\u002Fabs\u002F2609.37725","_blank",[604,605],"text-blue-600","underline","[Online]",[590,608,610,611,614,615],{"id":609},"source-2","B. A. Richards and P. W. Frankland, \"The Persistence and Transience of Memory,\" ",[595,612,613],{},"Neuron",", vol. 94, no. 6, pp. 1071–1084, 2017. DOI: ",[386,616,606],{"href":617,"target":602,"className":618},"https:\u002F\u002Fdoi.org\u002F10.1016\u002Fj.neuron.2017.04.037",[604,605],[590,620,622,623,626,627],{"id":621},"source-3","N. F. Liu et al., \"Lost in the Middle: How Language Models Use Long Contexts,\" ",[595,624,625],{},"Transactions of the Association for Computational Linguistics",", vol. 12, pp. 157–173, 2024. DOI: ",[386,628,606],{"href":629,"target":602,"className":630},"https:\u002F\u002Fdoi.org\u002F10.1162\u002Ftacl_a_00638",[604,605],[590,632,634,635,638,639],{"id":633},"source-4","J. L. C. Lee et al., \"An Update on Memory Reconsolidation Updating,\" ",[595,636,637],{},"Trends in Cognitive Sciences",", vol. 21, no. 7, pp. 531–545, 2017. DOI: ",[386,640,606],{"href":641,"target":602,"className":642},"https:\u002F\u002Fdoi.org\u002F10.1016\u002Fj.tics.2017.04.006",[604,605],[590,644,646,647,650,651],{"id":645},"source-5","E. F. Loftus, \"Planting Misinformation in the Human Mind: A 30-Year Investigation of the Malleability of Memory,\" ",[595,648,649],{},"Learning & Memory",", vol. 12, no. 4, pp. 361–366, 2005. DOI: ",[386,652,606],{"href":653,"target":602,"className":654},"https:\u002F\u002Fdoi.org\u002F10.1101\u002Flm.94705",[604,605],[590,656,658,659,662,663],{"id":657},"source-6","J. Maynez et al., \"On Faithfulness and Factuality in Abstractive Summarization,\" in ",[595,660,661],{},"Proc. 58th Annual Meeting of the Association for Computational Linguistics (ACL'20)",", 2020, pp. 1906–1919. DOI: ",[386,664,606],{"href":665,"target":602,"className":666},"https:\u002F\u002Fdoi.org\u002F10.18653\u002Fv1\u002F2020.acl-main.173",[604,605],[590,668,670,671,674,675],{"id":669},"source-7","Z. Chen et al., \"AgentPoison: Red-teaming LLM Agents via Poisoning Memory or Knowledge Bases,\" in ",[595,672,673],{},"Advances in Neural Information Processing Systems",", vol. 37, 2024, ",[386,676,606],{"href":677,"target":602,"className":678},"https:\u002F\u002Fproceedings.neurips.cc\u002Fpaper_files\u002Fpaper\u002F2024\u002Fhash\u002Feb113910e9c3f6242541c1652e30dfd6-Abstract-Conference.html",[604,605],[590,680,682,683,685,686],{"id":681},"source-8","C. Packer et al., \"MemGPT: Towards LLMs as Operating Systems,\" ",[595,684,597],{},", 2023, ",[386,687,606],{"href":688,"target":602,"className":689},"https:\u002F\u002Farxiv.org\u002Fabs\u002F2310.08560",[604,605],{"title":691,"searchDepth":692,"depth":692,"links":693},"",2,[694,695,696,697,701,702],{"id":363,"depth":692,"text":364},{"id":401,"depth":692,"text":402},{"id":425,"depth":692,"text":426},{"id":467,"depth":692,"text":468,"children":698},[699],{"id":506,"depth":700,"text":507},3,{"id":552,"depth":692,"text":553},{"id":576,"depth":692,"text":580},"2026-10-06","A new class of language models manages its own context by rewriting it, and early results suggest that letting a model forget can make it both sharper and cheaper. Decades of memory research point to the other half of that bargain, since every recall that rewrites a memory is also a chance to distort it. Keeping an untouched record beside the editable one may let agents keep the gains while catching the drift.","md",{"src":372},{"authors":708,"badge":714,"source":716},[709],{"avatar":710,"name":712,"to":713},{"src":711},"\u002Fimg\u002Fmark_avatar.png","Mark Williams","#",{"label":715},"Agent Memory",{"name":717,"url":718},"Thinkata Research","https:\u002F\u002Fthinkata.com",true,{"title":290,"description":704},"Zw1q8HYX-mq-zT3_9Mdcx6NRM3OxUHfE0N4a1iBSlMg",[723,1014],{"id":724,"title":226,"body":725,"date":1002,"description":1003,"extension":705,"image":1004,"meta":1005,"navigation":719,"path":227,"seo":1012,"stem":228,"__hash__":1013,"_path":227},"insights\u002Fnews\u002Finsights\u002Fmachine-unlearning-removal-suppression.md",{"type":344,"value":726,"toc":990},[727,739,745,748,751,764,768,776,779,782,786,794,802,806,809,821,829,833,851,859,862,866,873,876,879,883,893,896,899],[347,728,351,730,351,734],{"className":729},[350],[353,731,226],{"className":732,"id":733},[356],"forgetting-on-request",[359,735,738],{"className":736,"id":737},[362],"why-removing-data-from-a-trained-model-is-not-like-deleting-a-row","Why Removing Data From a Trained Model Is Not Like Deleting a Row",[347,740,368,741],{"style":367},[370,742],{"src":743,"alt":744,"style":374},"https:\u002F\u002Fimages.unsplash.com\u002Fphoto-1747147292047-7c61f1110897?w=1200&h=750&fit=crop&auto=format","A hand pulling a single index card from a library card catalog drawer",[376,746,747],{},"A deletion request is easy to honor in a database. Find the record, remove it, and look again to confirm it is gone. The same request aimed at a trained language model has nothing comparable to find. Whatever the model absorbed during training is spread across billions of weights, the numerical values it adjusted while learning, and blended with everything else it read.",[376,749,750],{},"A library card catalog is a closer picture of the problem. Pulling a card from the drawer makes a book hard to find, but the book is still on the shelf, and anyone who walks the stacks can come across it. Research from the past two years suggests that the methods built to make models forget mostly pull the card. The model stops giving the answer when asked, while the knowledge behind that answer stays in the weights. Telling those two outcomes apart has turned out to be harder than producing a model that stays quiet.",[376,752,753,754,758,759,763],{},"Machine unlearning is the name for methods that try to remove the influence of particular training data from a model after training has finished. Interest has grown with the European right to be forgotten, with copyright disputes over training data, and with worries about models that know too much about weapons ",[383,755,756],{},[386,757,389],{"href":388},". The cleanest answer is to retrain from scratch without the offending data, and for the largest models that is usually far too expensive. An early design called SISA planned for deletion in advance, splitting the training data into separate shards that each feed a smaller model, so a request only forces a retrain of the shard that held the record ",[383,760,761],{},[386,762,411],{"href":410},". That works when it is chosen before training. For a language model already trained on a large slice of the web, it is too late, which is why cheaper after-the-fact methods exist at all.",[359,765,767],{"id":766},"two-goals-sharing-one-word","Two Goals Sharing One Word",[376,769,770,771,775],{},"A position paper presented at NeurIPS 2025 argues that the word unlearning now covers two different aims ",[383,772,773],{},[386,774,389],{"href":388},". Removal means taking a specific piece of training data out of the model's weights, which in library terms means taking the book off the shelf. Suppression means stopping a kind of content from appearing in the model's answers, which is closer to pulling the card.",[376,777,778],{},"The two come apart in both directions. Removal works on discrete items, a particular article or photograph, so it cannot target a skill the model pieced together from many sources, such as a general recipe for a toxic compound. Succeeding at removal also does not settle what the model can produce. The authors point out that a model trained without a single copyrighted image of a comic book character can still draw something that looks like that character, because generalizing beyond its examples is what the model was built to do.",[376,780,781],{},"Seen this way, a method that makes a model go quiet is doing suppression, whatever its paper calls it. That raises a practical question about how well the quiet holds once someone goes looking.",[359,783,785],{"id":784},"walking-the-stacks","Walking the Stacks",[376,787,788,789,793],{},"A team from Carnegie Mellon University tested exactly that by fine-tuning unlearned models afterward, giving them a short round of extra training on ordinary public text that never contained the forgotten material ",[383,790,791],{},[386,792,421],{"href":420},". A model made to forget the Harry Potter novels was trained on general facts about the characters, written by another model and containing no passages from the books, and it began reproducing text from the novels word for word. Public medical articles brought back hazardous knowledge about bioweapons in the same way. Training on gibberish brought back nothing, which suggests the related text was jogging something still stored rather than teaching it fresh. Stronger unlearning made recovery harder, so the effect is a matter of degree, but the authors' own reading is that current methods hide answers rather than remove knowledge.",[376,795,796,797,801],{},"A second test makes the same point more directly ",[383,798,799],{},[386,800,435],{"href":434},". An attacker is handed some of the facts a model was supposed to forget and fine-tunes on them, then checks whether other facts from the same set come back, ones that could not be guessed from the facts provided. On knowledge the model picked up in its original training, this recovered 88 percent of the accuracy the model had before unlearning. That work is a preprint and has not been peer reviewed, though it lines up closely with the Carnegie Mellon result. In library terms, a visitor who knows roughly which aisle to search finds most of the books whose cards were pulled.",[359,803,805],{"id":804},"checking-the-shelves-not-the-catalog","Checking the Shelves, Not the Catalog",[376,807,808],{},"If a quiet model can still hold the answer, then asking the model and reading its reply is a weak test. A model that says it does not know might have forgotten, or might simply be declining.",[376,810,811,812,816,817,395],{},"One way around this is to invent the data, so the true answer is known in advance. TOFU, introduced at the Conference on Language Modeling in 2024, generated 200 profiles of authors who do not exist, trained a model on them, and asked unlearning methods to make it forget some ",[383,813,814],{},[386,815,448],{"href":447},". Anything the model knows about those authors came from a known source, so an unlearned model can be compared with one that never saw them. None of the methods tested matched that standard. The adversarial study above adds a caution, that facts added in a late round of training, as these fictitious authors are, may be easier to forget than knowledge absorbed from the start, so tests like this one may flatter the methods ",[383,818,819],{},[386,820,435],{"href":434},[376,822,823,824,828],{},"A NeurIPS 2025 benchmark turned the same idea on the measuring tools themselves ",[383,825,826],{},[386,827,458],{"href":457},". Its authors released several hundred models whose true knowledge is known by construction, then asked whether each of a dozen common scores could tell a model that holds a fact from one that does not, and whether that verdict survived routine changes such as compressing the model or briefly exposing it to related text. No score did well on both counts. A widely used score based on word overlap separated knowing from not knowing reasonably well, yet came close to a coin flip once related text was used to jog the model. A score that weakens under exactly the conditions an attacker would create is a shaky basis for claiming that data was deleted.",[359,830,832],{"id":831},"when-the-same-fact-sits-in-several-books","When the Same Fact Sits in Several Books",[347,834,368,835,368,839,368,843],{"style":497},[370,836],{"src":837,"alt":838,"style":374},"https:\u002F\u002Fimages.unsplash.com\u002Fphoto-1573812195421-50a396d17893?w=1200&auto=format&fit=crop","A tightly stacked pile of newspapers seen edge on",[503,840,842],{"style":505,"id":841},"the-same-story-filed-five-ways","The Same Story, Filed Five Ways",[376,844,845,846,850],{"style":510},"Pull one paper out of the stack and the earthquake it reported is still in the four below it, told in different words. Benchmarks have generally treated the data to forget and the data to keep as separate piles, which is not how real collections of text are assembled ",[383,847,848],{},[386,849,485],{"href":484},". A news article covered by a deletion request may state a fact that an encyclopedia entry the model is allowed to keep also states. A model retrained without the article would still learn that fact from the encyclopedia, so an honest unlearning method should leave the fact alone and remove only what was unique to the article.",[376,852,853,854,858],{},"The DUSK benchmark, published in the Findings of ACL 2026, builds that situation on purpose ",[383,855,856],{},[386,857,485],{"href":484},". Its authors wrote profiles of 120 fictitious professors into five documents in five different styles, with half the profiles appearing in all five documents and the rest in only one, then asked recent unlearning methods to remove a single document.",[376,860,861],{},"The methods tended to miss in both directions at once. The wording of the deleted document usually disappeared, yet facts unique to it often survived, while shared facts that should have been kept were damaged. Giving more weight to protecting the kept documents helped at first and then stopped helping. Removing a document and removing the facts it contains are different operations, and current methods have trouble aiming at one without hitting the other.",[359,863,865],{"id":864},"pulling-the-card-without-reading-it-aloud","Pulling the Card Without Reading It Aloud",[376,867,868,869,395],{},"Standard methods also have an awkward habit. To teach a model not to give an answer, they put that answer into the training data and push its likelihood down, so the sensitive text gets handled again in order to be forgotten ",[383,870,871],{},[386,872,543],{"href":542},[376,874,875],{},"Researchers at the Technical University of Munich and Mila, the Quebec AI institute, avoid using the target answer at all. Their method has the model write several of its own responses to a question it should forget, keeps the one a scoring rule judges furthest from the original answer, and trains on that. Repeating the loop deliberately sets off the kind of collapse people worry about when models train on their own output, a gradual loss of the original information, but aimed only at the material to be forgotten. The reported result is that the quiet holds up better against two attacks that break other methods, asking the same question many times and forcing the model to continue from the opening words of an answer.",[376,877,878],{},"The authors also found that target-based methods leave a fingerprint. They push correct answers so far down that an attacker can find a forgotten fact by looking for the option the model rates least likely, a little like an empty slot in the catalog drawer showing exactly where a card used to be. The work appeared at ICLR 2026, and its claims concern what the model says rather than what its weights hold. It makes the quiet harder to break, which is useful, while leaving open whether the book has left the shelf.",[359,880,882],{"id":881},"what-a-deletion-claim-can-honestly-say","What a Deletion Claim Can Honestly Say",[376,884,885,886,889,890,395],{},"The evidence points toward a narrower kind of claim. From a systems perspective, an unlearning method is best treated as an output control, a filter on what the model says, together with a stated list of the attacks it was tested against. A method that holds under direct questioning may fail under relearning or repeated sampling, so a claim that names only the easy test says little. Evaluations whose own scores have been stress tested deserve more weight than ones that simply ask the model. Where genuine removal is required, the options are still the expensive ones, filtering data before training or retraining without it, and that cost is a reason to decide carefully what enters a training set in the first place. Related questions about what a system keeps come up in earlier Thinkata pieces on ",[386,887,888],{"href":243},"isolation between customers"," and on ",[386,891,892],{"href":291},"agents that rewrite their own memory",[376,894,895],{},"The open question is the one the benchmarks keep circling. What evidence would show that a fact is gone from the weights rather than merely hard to reach? Relearning attacks are the closest thing to such a test, and they only set a floor, since a stronger attacker might recover more. Legal practitioners would know better than a systems reading can whether an assurance that depends on which attacks were tried could ever satisfy a deletion obligation.",[376,897,898],{},"Until that kind of evidence exists, the catalog picture is probably the fairest description of what unlearning delivers today. The card has been pulled, and newer methods are getting better at keeping it pulled when someone asks directly or tries a few tricks. Whether the book has left the shelf is a separate question, and a deletion record that keeps the two apart would tell both the person who asked for the deletion and the engineers who maintain the model what was actually done.",[347,900,351,902,351,904],{"className":901},[576,577],[359,903,580],{"id":576},[582,905,368,907,368,917,368,928,368,939,368,949,368,959,368,969,368,980,351],{"className":906},[585,586,587,588],[590,908,909,910,912,913],{"id":592},"A. F. Cooper et al., \"Machine Unlearning Doesn't Do What You Think: Lessons for Generative AI Policy and Research,\" in ",[595,911,673],{},", vol. 38, 2025, ",[386,914,606],{"href":915,"target":602,"className":916},"https:\u002F\u002Fproceedings.neurips.cc\u002Fpaper_files\u002Fpaper\u002F2025\u002Fhash\u002F8110ef9878e4642ad848ca583d37d7ca-Abstract-Position_Paper_Track.html",[604,605],[590,918,919,920,923,924],{"id":609},"L. Bourtoule et al., \"Machine Unlearning,\" in ",[595,921,922],{},"Proc. 2021 IEEE Symposium on Security and Privacy (SP)",", 2021, pp. 141–159. DOI: ",[386,925,606],{"href":926,"target":602,"className":927},"https:\u002F\u002Fdoi.org\u002F10.1109\u002FSP40001.2021.00019",[604,605],[590,929,930,931,934,935],{"id":621},"S. Hu, Y. Fu, Z. S. Wu, and V. Smith, \"Unlearning or Obfuscating? Jogging the Memory of Unlearned LLMs via Benign Relearning,\" in ",[595,932,933],{},"Proc. International Conference on Learning Representations (ICLR'25)",", 2025, ",[386,936,606],{"href":937,"target":602,"className":938},"https:\u002F\u002Fproceedings.iclr.cc\u002Fpaper_files\u002Fpaper\u002F2025\u002Fhash\u002F18fd48d9cbbf9a20e434c9d3db6973c5-Abstract-Conference.html",[604,605],[590,940,941,942,944,945],{"id":633},"A. Deeb and F. Roger, \"Do Unlearning Methods Remove Information from Language Model Weights?,\" ",[595,943,597],{},", 2024, ",[386,946,606],{"href":947,"target":602,"className":948},"https:\u002F\u002Farxiv.org\u002Fabs\u002F2410.08827",[604,605],[590,950,951,952,944,955],{"id":645},"P. Maini et al., \"TOFU: A Task of Fictitious Unlearning for LLMs,\" in ",[595,953,954],{},"Proc. 1st Conference on Language Modeling (COLM)",[386,956,606],{"href":957,"target":602,"className":958},"https:\u002F\u002Fopenreview.net\u002Fpdf?id=B41hNBoWLo",[604,605],[590,960,961,962,912,965],{"id":657},"V. Dorna et al., \"OpenUnlearning: Accelerating LLM Unlearning via Unified Benchmarking of Methods and Metrics,\" in ",[595,963,964],{},"Advances in Neural Information Processing Systems, Datasets and Benchmarks Track",[386,966,606],{"href":967,"target":602,"className":968},"https:\u002F\u002Fproceedings.neurips.cc\u002Fpaper_files\u002Fpaper\u002F2025\u002Fhash\u002F3e4a38f228427ab819ba7899003a44b1-Abstract-Datasets_and_Benchmarks_Track.html",[604,605],[590,970,971,972,975,976],{"id":669},"W. Jeung et al., \"DUSK: Do Not Unlearn Shared Knowledge,\" in ",[595,973,974],{},"Findings of the Association for Computational Linguistics: ACL 2026",", 2026, pp. 42013–42031, ",[386,977,606],{"href":978,"target":602,"className":979},"https:\u002F\u002Faclanthology.org\u002F2026.findings-acl.2085\u002F",[604,605],[590,981,982,983,598,986],{"id":681},"Y. Scholten, S. Xhonneux, L. Schwinn, and S. Günnemann, \"Model Collapse Is Not a Bug but a Feature in Machine Unlearning for LLMs,\" in ",[595,984,985],{},"Proc. International Conference on Learning Representations (ICLR'26)",[386,987,606],{"href":988,"target":602,"className":989},"https:\u002F\u002Fproceedings.iclr.cc\u002Fpaper_files\u002Fpaper\u002F2026\u002Fhash\u002F263da600dcd225dcb9ecaef8290e2012-Abstract-Conference.html",[604,605],{"title":691,"searchDepth":692,"depth":692,"links":991},[992,993,994,995,996,999,1000,1001],{"id":737,"depth":692,"text":738},{"id":766,"depth":692,"text":767},{"id":784,"depth":692,"text":785},{"id":804,"depth":692,"text":805},{"id":831,"depth":692,"text":832,"children":997},[998],{"id":841,"depth":700,"text":842},{"id":864,"depth":692,"text":865},{"id":881,"depth":692,"text":882},{"id":576,"depth":692,"text":580},"2026-10-13","A deletion request aimed at a trained language model has no row to remove. Evidence from relearning attacks, benchmarks built on invented data, and documents that share facts suggests that current unlearning methods mostly make a model quieter rather than emptier, and that telling quiet from gone is the harder problem.",{"src":743},{"authors":1006,"badge":1009,"source":1011},[1007],{"avatar":1008,"name":712,"to":713},{"src":711},{"label":1010},"Machine Unlearning",{"name":717,"url":718},{"title":226,"description":1003},"M6mQ-UU0dsXGw1N2TcuejIEgUKVuxtORzEz9y-A4KYc",{"id":1015,"title":194,"body":1016,"date":1330,"description":1331,"extension":705,"image":1332,"meta":1333,"navigation":719,"path":195,"seo":1340,"stem":196,"__hash__":1341,"_path":195},"insights\u002Fnews\u002Finsights\u002Finterleaved-distillation-rl.md",{"type":344,"value":1017,"toc":1319},[1018,1030,1036,1039,1042,1050,1054,1066,1074,1082,1086,1094,1102,1110,1118,1126,1130,1138,1146,1159,1163,1175,1193,1211,1214,1218,1221,1229,1237],[347,1019,351,1021,351,1025],{"className":1020},[350],[353,1022,194],{"className":1023,"id":1024},[356],"taking-turns-instead-of-splitting-the-difference",[359,1026,1029],{"className":1027,"id":1028},[362],"why-reinforcement-learning-and-distillation-may-work-better-in-alternation-than-in-a-single-blended-loss","Why Reinforcement Learning and Distillation May Work Better in Alternation Than in a Single Blended Loss",[347,1031,368,1032],{"style":367},[370,1033],{"src":1034,"alt":1035,"style":374},"https:\u002F\u002Fimages.unsplash.com\u002Fphoto-1772517403501-c4bb361a4bdc?w=1200&h=750&fit=crop&auto=format","Two teams leaning hard in opposite directions on a taut rope across a grass field",[376,1037,1038],{},"Both teams in a tug of war can be pulling as hard as they can while the rope barely moves. All of that effort is real, and most of it cancels out.",[376,1040,1041],{},"Training a large language model after pretraining, a stage usually called post-training, often involves two forces that behave in a similar way. One is reinforcement learning, where the model attempts a problem, receives a reward based on how well the answer checks out, and becomes more likely to produce whatever earned that reward. The other is distillation, where a smaller model, the student, learns by imitating a larger and more capable model, the teacher. Both are widely used to improve models. They also pull on the same property of the model in opposite directions. One straightforward way to combine them is to fold both objectives into a single weighted loss, the score training tries to push down, so that one update carries both.",[376,1043,1044,1045,1049],{},"A technical report from ByteDance describing Pistis, a family of multimodal models that read images and video as well as text, makes a case for a different arrangement ",[383,1046,1047],{},[386,1048,389],{"href":388},". Instead of blending the two signals, its training loop alternates between them. The measured gains are modest and self-reported. The schedule illustrates a broader problem that can arise whenever one model answers to competing objectives, and two other parts of the report show a similar preference for keeping signals apart, one in the credit given to an agent's individual actions and one in the software wrapped around a model that is no longer being trained.",[359,1051,1053],{"id":1052},"two-signals-that-pull-apart","Two Signals That Pull Apart",[376,1055,1056,1057,1061,1062,1065],{},"The property in question is entropy, a measure of how spread out a model's choices are. A model with high entropy keeps many possible next words in play at each step. A model with low entropy commits hard to one or two. Reinforcement learning tends to drive entropy down, because rewarding a correct answer raises the probability of the exact words that produced it. A study of reinforcement learning on reasoning models found that without deliberate intervention, entropy falls sharply early in training, and performance flattens once that spread has been used up ",[383,1058,1059],{},[386,1060,411],{"href":410},". Much of the drop came from words the model already favored that also earned a high reward, which then crowd out the alternatives. An earlier Thinkata piece covered ",[386,1063,1064],{"href":127},"entropy collapse"," in classic control settings.",[376,1067,1068,1069,1073],{},"On-policy distillation works differently. The student writes its own answer, and the teacher then scores every word of it, reporting what it would have chosen at that same point ",[383,1070,1071],{},[386,1072,421],{"href":420},". Training on its own attempts means the student gets corrected in the situations it actually wanders into. The signal is also dense, a target at every word rather than a single pass or fail at the end.",[376,1075,1076,1077,1081],{},"Depending on how the gap between student and teacher is measured, distillation can either narrow the student's choices or widen them. Pistis uses Jensen-Shannon divergence, a measure of how far apart two probability distributions are, computed over the teacher's fifty most likely next words. That setup asks the student to cover the range of options the teacher considers plausible ",[383,1078,1079],{},[386,1080,389],{"href":388},". In the team's comparison, a version restricted to the top five words still let entropy collapse, and so did a popular alternative that compares the two models only on the single word the student actually wrote. Only the broad version held entropy up.",[359,1083,1085],{"id":1084},"why-the-sum-goes-wrong","Why the Sum Goes Wrong",[376,1087,1088,1089,1093],{},"When both objectives are folded into one loss, every training step produces one combined direction in which to move the model's weights. The trouble comes on words where the student is already more confident than the teacher. If that word belonged to a rewarded answer, reinforcement learning pushes its probability higher still, while distillation pushes it back down toward the teacher's level. A first-order analysis in the report finds that the disagreement between the two directions subtracts from the progress each would make on its own ",[383,1090,1091],{},[386,1092,389],{"href":388},". Past a certain degree of conflict, the blended step can actually lower the reward even while the combined loss improves.",[376,1095,1096,1097,1101],{},"Multi-task learning hit a version of this years earlier. A study of networks trained on several tasks at once identified conflicting gradients, the per-task directions of improvement pointing against one another, as one of a small set of conditions behind detrimental gradient interference ",[383,1098,1099],{},[386,1100,435],{"href":434},". Their fix, which they call gradient surgery, edits each gradient, stripping out the part that fights the other before combining them.",[376,1103,1104,1105,1109],{},"Pistis takes a blunter route. Its schedule, called Interleaved Distillation and Reinforcement Learning, or IDRL, runs five steps of distillation, then five steps of reinforcement learning, and repeats ",[383,1106,1107],{},[386,1108,389],{"href":388},". Each individual step answers to one objective only, so neither can cancel the other within an update. The two still shape each other, just across phases rather than inside a single gradient.",[376,1111,1112,1113,1117],{},"The distillation phases play a role similar to the penalty many reinforcement learning setups apply to keep a model from drifting too far from where it started, a connection a recent survey of on-policy distillation lays out formally ",[383,1114,1115],{},[386,1116,448],{"href":447},". Here the anchor is a stronger model rather than an earlier copy of the student, so the pull toward it can add ability instead of only restraining change. It also sets a ceiling, since a student constantly drawn toward its teacher cannot pass it.",[376,1119,1120,1121,1125],{},"Once entropy stops rising, the schedule drops distillation and continues with reinforcement learning alone ",[383,1122,1123],{},[386,1124,389],{"href":388},". Only the smaller 9-billion-parameter models were trained this way. The 27-billion-parameter models used reinforcement learning alone, then served as the frozen teachers.",[359,1127,1129],{"id":1128},"what-the-comparison-actually-shows","What the Comparison Actually Shows",[376,1131,1132,1133,1137],{},"From the same fine-tuned checkpoint, the team trained its 9-billion-parameter agent model five ways, with reinforcement learning alone, distillation alone, distillation followed by reinforcement learning, both objectives summed at every step, and the alternating schedule. Across eighteen benchmarks spanning charts, visual reasoning, search, and tool use, alternation came out on top ",[383,1134,1135],{},[386,1136,389],{"href":388},". The summed loss and the two-stage handoff tied just behind it, a bit more than half a point lower on the averaged score. That comparison ran on one model size with one teacher, and the report shows no error bars or repeated training runs, so it cannot say whether a gap that small would survive a rerun. The authors say the results support interleaving without establishing that reduced gradient conflict or better exploration explains the difference.",[376,1139,1140,1141,1145],{},"The training curves tell a clearer story than the scores. Reinforcement learning on its own showed the steady entropy decline the earlier research predicts, along with gradient norms that climbed and spiked late in training, a sign of unstable updates. The variants that mixed in distillation throughout kept entropy higher, and the alternating run traced a sawtooth, entropy dipping in each reinforcement phase and recovering in each distillation phase ",[383,1142,1143],{},[386,1144,389],{"href":388},". Search tasks were where alternation pulled furthest ahead of the single handoff.",[376,1147,1148,1149,1153,1154,1158],{},"Distillation also turned out to have a precondition. Applied directly to a student that had not been warmed up, it barely moved accuracy on a geometry benchmark, and its training curve dipped sharply at the start ",[383,1150,1151],{},[386,1152,389],{"href":388},". A supervised phase on teacher-written answers, run first, fixed that. A separate study of text-only models found that on-policy distillation depends on student and teacher sharing compatible patterns of reasoning, and that this kind of cold start can rescue failing runs ",[383,1155,1156],{},[386,1157,458],{"href":457},". The Pistis results suggest the same holds once images enter the picture.",[359,1160,1162],{"id":1161},"the-same-separation-outside-the-gradient","The Same Separation Outside the Gradient",[376,1164,1165,1166,1169,1170,1174],{},"The report shows a similar preference in its handling of credit assignment, the problem of deciding which actions in a long sequence deserve credit for the final outcome, also the subject of an earlier Thinkata piece on ",[386,1167,1168],{"href":275},"temporal credit assignment",". When an agent searches over many turns and lands on the right answer, a reward attached only to that answer credits every step along the way, including malformed tool calls, repeated or near-duplicate calls, calls made after the model was told to answer, and calls whose results could not be fed back into its context. Pistis zeroes out positive credit for those steps while leaving any penalty in place ",[383,1171,1172],{},[386,1173,389],{"href":388},". Removing that rule hurt the search tasks most and barely changed short tasks like chart reading, which fits the idea that noisy credit compounds with length.",[347,1176,368,1177,368,1181,368,1185],{"style":497},[370,1178],{"src":1179,"alt":1180,"style":374},"https:\u002F\u002Fimages.unsplash.com\u002Fphoto-1462642109801-4ac2971a3a51?w=1200&auto=format&fit=crop","An open notebook with a numbered handwritten entry and short notes indented beneath it, a fountain pen resting on the page",[503,1182,1184],{"style":505,"id":1183},"a-ledger-around-a-frozen-model","A Ledger Around a Frozen Model",[376,1186,1187,1188,1192],{"style":510},"A numbered entry with a few short notes indented beneath it is about as plain as record keeping gets, and the second half of the report builds something close to it. Pistis-Auto-Harnessing, or PAH, leaves the trained model untouched and instead improves its harness, the surrounding software that decides what the model sees, which tools it can call, and when it has to stop and answer ",[383,1189,1190],{},[386,1191,389],{"href":388},". During development, a separate optimization agent reads failed runs, proposes one reversible change at a time, and keeps it only if a fixed development score improves. The test set is used once, after the harness is frozen. The harness that came out of this process keeps a running ledger of candidate answers, each tied to the evidence for and against it and to the conditions it has not yet satisfied, while the model itself still makes every decision.",[376,1194,1195,1196,1200,1201,1205,1206,1210],{},"Related work has pursued the same target more broadly. One line of research uses a meta agent that writes new agent designs in code ",[383,1197,1198],{},[386,1199,485],{"href":484},", and another evolves prompts by reflecting in natural language on complete runs ",[383,1202,1203],{},[386,1204,543],{"href":542},". PAH is narrower by design. On a multimodal search benchmark, with every attempt held to the same budget of fifteen search and page-reading actions, the optimized harness got about nine more questions right out of five hundred, and the average number of actions barely moved ",[383,1207,1208],{},[386,1209,389],{"href":388},". Placed around a different model, GPT-5.5, the same frozen harness helped by a wider margin. Transfer outside multimodal search has not been tested.",[376,1212,1213],{},"The three mechanisms work for different reasons. Interleaving changes when each gradient applies, the credit rule changes which actions a reward reaches, and the harness loop moves part of the search out of the weights entirely. What they share is a design preference rather than a common mechanism, a habit of keeping apart signals that a simpler design would merge.",[359,1215,1217],{"id":1216},"what-remains-uncertain","What Remains Uncertain",[376,1219,1220],{},"How far this generalizes is still open. The key IDRL and PAH results come from the team's own experiments, and the preprint, posted only days ago, has not yet been independently reproduced or peer reviewed. With no sweep over phase lengths, it is hard to tell whether the benefit depends on the ratio, the length, or simply alternating. A teacher from another model family, or a much smaller student, might also change the picture, given how sensitive distillation appears to be to the starting overlap between the two.",[376,1222,1223,1224,1228],{},"The report's own failure analysis points somewhere else. Even the improved agent sometimes wrote code that confirmed a guess instead of measuring anything, or found the right fact during search and then contradicted it in its final answer ",[383,1225,1226],{},[386,1227,389],{"href":388},". Neither failure is obviously addressed by entropy management or gradient separation. Both concern whether intermediate steps actually constrain the outcome, which suggests the next useful reward may need to score the process rather than the answer.",[376,1230,1231,1232,1236],{},"For teams that fold a distillation term and a reward term into one loss, the report offers a useful diagnostic more than a new recipe. Logging the agreement between the two gradients over a run, often measured as the cosine of the angle between them, would show whether they point against each other at all, though computing them separately adds real cost at large scale. Opposition alone is not proof of harm. The multi-task study behind gradient surgery found conflicting gradients damaging mainly when the loss surface also curves sharply and one gradient is much larger than the other ",[383,1233,1234],{},[386,1235,435],{"href":434},". If strong opposition is absent, gradient conflict becomes a less convincing explanation for any weakness in the blended loss. If it grows as the model becomes more confident, alternating the two may be worth trying before retuning the weights on the sum.",[347,1238,351,1240,351,1242],{"className":1239},[576,577],[359,1241,580],{"id":576},[582,1243,368,1245,368,1254,368,1263,368,1273,368,1283,368,1292,368,1301,368,1310,351],{"className":1244},[585,586,587,588],[590,1246,1247,1248,598,1250],{"id":592},"H. Chen et al., \"Pistis Technical Report,\" ",[595,1249,597],{},[386,1251,606],{"href":1252,"target":602,"className":1253},"https:\u002F\u002Farxiv.org\u002Fabs\u002F2609.28554",[604,605],[590,1255,1256,1257,934,1259],{"id":609},"G. Cui et al., \"The Entropy Mechanism of Reinforcement Learning for Reasoning Language Models,\" ",[595,1258,597],{},[386,1260,606],{"href":1261,"target":602,"className":1262},"https:\u002F\u002Farxiv.org\u002Fabs\u002F2505.22617",[604,605],[590,1264,1265,1266,944,1269],{"id":621},"R. Agarwal et al., \"On-Policy Distillation of Language Models: Learning from Self-Generated Mistakes,\" in ",[595,1267,1268],{},"Proc. International Conference on Learning Representations (ICLR'24)",[386,1270,606],{"href":1271,"target":602,"className":1272},"https:\u002F\u002Farxiv.org\u002Fabs\u002F2306.13649",[604,605],[590,1274,1275,1276,1278,1279],{"id":633},"T. Yu et al., \"Gradient Surgery for Multi-Task Learning,\" in ",[595,1277,673],{},", vol. 33, 2020, ",[386,1280,606],{"href":1281,"target":602,"className":1282},"https:\u002F\u002Farxiv.org\u002Fabs\u002F2001.06782",[604,605],[590,1284,1285,1286,598,1288],{"id":645},"M. Song and M. Zheng, \"A Survey of On-Policy Distillation for Large Language Models,\" ",[595,1287,597],{},[386,1289,606],{"href":1290,"target":602,"className":1291},"https:\u002F\u002Farxiv.org\u002Fabs\u002F2604.00626",[604,605],[590,1293,1294,1295,598,1297],{"id":657},"Y. Li et al., \"Rethinking On-Policy Distillation of Large Language Models: Phenomenology, Mechanism, and Recipe,\" ",[595,1296,597],{},[386,1298,606],{"href":1299,"target":602,"className":1300},"https:\u002F\u002Farxiv.org\u002Fabs\u002F2604.13016",[604,605],[590,1302,1303,1304,934,1306],{"id":669},"S. Hu et al., \"Automated Design of Agentic Systems,\" in ",[595,1305,933],{},[386,1307,606],{"href":1308,"target":602,"className":1309},"https:\u002F\u002Fproceedings.iclr.cc\u002Fpaper_files\u002Fpaper\u002F2025\u002Fhash\u002F36b7acf6f6010652b3f2a433774a66fe-Abstract-Conference.html",[604,605],[590,1311,1312,1313,598,1315],{"id":681},"L. A. Agrawal et al., \"GEPA: Reflective Prompt Evolution Can Outperform Reinforcement Learning,\" in ",[595,1314,985],{},[386,1316,606],{"href":1317,"target":602,"className":1318},"https:\u002F\u002Fproceedings.iclr.cc\u002Fpaper_files\u002Fpaper\u002F2026\u002Fhash\u002F0e9e708b6f48e14fd0ac29e167413f76-Abstract-Conference.html",[604,605],{"title":691,"searchDepth":692,"depth":692,"links":1320},[1321,1322,1323,1324,1325,1328,1329],{"id":1028,"depth":692,"text":1029},{"id":1052,"depth":692,"text":1053},{"id":1084,"depth":692,"text":1085},{"id":1128,"depth":692,"text":1129},{"id":1161,"depth":692,"text":1162,"children":1326},[1327],{"id":1183,"depth":700,"text":1184},{"id":1216,"depth":692,"text":1217},{"id":576,"depth":692,"text":580},"2026-09-28","Reinforcement learning and distillation push a language model in opposite directions, one narrowing its choices and one widening them. A new post-training report from ByteDance argues the two signals work better when they alternate than when their losses are added together, and finds the same separation paying off in credit assignment and in the software wrapped around a frozen model.",{"src":1034},{"authors":1334,"badge":1337,"source":1339},[1335],{"avatar":1336,"name":712,"to":713},{"src":711},{"label":1338},"Post-Training",{"name":717,"url":718},{"title":194,"description":1331},"dD5vUP4qrJqWOox-EMzDza0fy1ZDyYYEOBzw_N9nsSY",1792582875802]