{
  "schema": "pabench.corrections/v1",
  "entries": [
    {
      "id": "2026-10-22-scoring-assistant-benchmark-figures",
      "date": "2026-10-22",
      "title": "Assistant Benchmark figures on the scoring page brought in line with the 16 October reading",
      "pages": [
        "/how-scoring-works/"
      ],
      "before": "Assistant Benchmark was said to have tested 32 of 139 assistants on the same 15 tasks as of 12 October 2026.",
      "after": "When read on 22 October 2026, its general board, last updated 19 October, gave an overall score to 29 assistants across up to 16 categories.",
      "why": "The 32-of-139 figure was the 8 October reading already corrected on /benchmarks/ (2026-10-16-hub-assistant-benchmark-refresh); the scoring page had kept it under a wrong date. assistantbenchmark.com, read 22 October 2026.",
      "kind": "fact"
    },
    {
      "id": "2026-10-22-landscape-rates-by-batch",
      "date": "2026-10-22",
      "title": "Landscape no longer says only three products have rates",
      "pages": [
        "/landscape/"
      ],
      "before": "Three products were said to have rates on the scorecard, with Town's figure withheld until it reconciles.",
      "after": "Which products have a figure depends on the batch: in the September first batch Town's is withheld; in the 11 October batch Instinct, Muse and Town have figures and Pine was set aside with no data.",
      "why": "The 11 October batch (data/results.json, october) is on the scorecard with a Town rate and no Pine figure.",
      "kind": "fact"
    },
    {
      "id": "2026-10-22-exam-changes-lede-october",
      "date": "2026-10-22",
      "title": "Exam changelog no longer says all figures date from 22 September",
      "pages": [
        "/how-the-exam-works/changes/"
      ],
      "before": "The completion figures on this site were said to have been measured on 22 September 2026.",
      "after": "The September figures were measured on 22 September; the October batches, run between 11 and 14 October, came after the 28 September rules, and their exact exam revisions are not known.",
      "why": "October batches are published on /results/#october and were run under the rules that took effect at the end of September (data/results.json, october).",
      "kind": "fact"
    },
    {
      "id": "2026-10-22-exam-changes-faq-later-rules",
      "date": "2026-10-22",
      "title": "Exam changelog FAQ: October results were run under the later rules",
      "pages": [
        "/how-the-exam-works/changes/"
      ],
      "before": "No published result was said to have been measured under the later rules.",
      "after": "The September figures were not rescored; the October batches were run after the 28 September rules and are published as separate measurements, without a claim about which 9 and 10 October changes each includes.",
      "why": "data/results.json october.lede: the October batch ran under the exam rules that took effect at the end of September.",
      "kind": "fact"
    },
    {
      "id": "2026-10-22-muse-spark-open-weights",
      "date": "2026-10-22",
      "title": "Muse Spark now has an open-weight version",
      "pages": [
        "/is-meta-muse-open-source/"
      ],
      "before": "Neither Muse Spark nor Muse Wonder had public weights; a 'muse spark open source' headline was said to be almost certainly about Glimmer or Muse Gadgets, not Spark itself.",
      "after": "On 15 October 2026 Meta released Muse Spark Open, which it calls the open-weight version of Muse Spark (816B total, 42B active), under its Muse Model License, which carries attribution and large-provider conditions. Meta does not say it is the model the Muse agent runs on. Muse Wonder still has no public weights. Alexandr Wang had said in August that Spark 1.2 would get an open-weight version.",
      "why": "Meta's post 'Introducing Muse Spark Open' (research.meta.ai/blog/muse-spark-open) and the Hugging Face page meta-models/Muse-Spark-Open-816B-A42B, read on 22 October 2026; Constellation Research, 10 August 2026.",
      "kind": "fact"
    },
    {
      "id": "2026-10-22-town-channels-funding",
      "date": "2026-10-22",
      "title": "Town's channels and funding brought up to date",
      "pages": [
        "/agents/town/"
      ],
      "before": "Town's homepage was said to list WhatsApp as a way to message your Townie, inbound only; the company history stopped at the June $55 million Series A.",
      "after": "Town's homepage lists Slack, WhatsApp, Telegram and iMessage as ways to message your Townie, plus iPhone and Mac apps; it no longer describes WhatsApp as inbound only. Town announced a $108 million Series B at a $1 billion valuation on 19 October 2026, co-led by Forerunner Ventures and Menlo Ventures.",
      "why": "town.com and town.com/features/whatsapp, read on 22 October 2026; Dealroom report of the Series B, 19 October 2026.",
      "kind": "calibration"
    },
    {
      "id": "2026-10-22-nanomuse-latest-release",
      "date": "2026-10-22",
      "title": "nanoMuse install route points to the latest release",
      "pages": [
        "/nanomuse/"
      ],
      "before": "Android and desktop installs pointed to release 1.0.3, with releases listed through 1.0.3 on 11 October.",
      "after": "Installs point to the latest release, 1.0.5 when checked on 22 October; releases 1.0.0 to 1.0.5 came out between 8 and 13 October.",
      "why": "github.com/nano-muse/nanoMuse/releases, read on 22 October 2026.",
      "kind": "calibration"
    },
    {
      "id": "2026-10-22-cue-app-store-iaps",
      "date": "2026-10-22",
      "title": "Cue's App Store in-app purchase list has changed",
      "pages": [
        "/cue-by-manus/"
      ],
      "before": "The App Store lists in-app purchases named Cue Plus at $200, Cue Pro at $1,000, Cue Max at $2,000 and Cloud Computer at $65, plus US and Singapore phone numbers at $9.99.",
      "after": "On 22 October 2026 the listing shows Cue Pro at $100, Cue Max at $200 and $2,000, Cloud Computer at $39, and US, Singapore and Canada phone numbers at $9.99. Apple shows only some in-app purchases, so Cue Plus not appearing does not mean it was removed; the 12 October list is kept as history.",
      "why": "apps.apple.com/us/app/cue-by-manus/id6808613358, read on 22 October 2026.",
      "kind": "fact"
    },
    {
      "id": "2026-10-22-cue-app-store-features",
      "date": "2026-10-22",
      "title": "Cue's App Store description was rewritten",
      "pages": [
        "/cue-by-manus/"
      ],
      "before": "The App Store listing sets out six features including a team of agents with separate roles, and says \"Give your Agent its own email address...\" and that each agent has \"its own memory.\"",
      "after": "The listing (version 26.18.2, 21 October) now lists an everyday helper in place of the team of agents, says \"your Cue\" instead of \"your Agent\", and no longer uses the phrase \"its own memory\"; the page quotes the current text and dates the old wording.",
      "why": "Apple lookup record and App Store page for id6808613358, read on 22 October 2026.",
      "kind": "fact"
    },
    {
      "id": "2026-10-22-grok-bot-x-subscription-pricing",
      "date": "2026-10-22",
      "title": "Grok Bot access moved into The X Subscription",
      "pages": [
        "/grok-bot/"
      ],
      "before": "Access comes with any paid individual Cursor plan (Pro is $20 a month); Cursor's pricing lists Pro $20, Pro+ $60, Ultra $200 and Teams $40 per user.",
      "after": "Cursor's pricing page now sells The X Subscription (Free $0, Plus $8, Premium $30, Super $100, Ultra $200, Teams from $30 per user) with Grok Bot from Premium up; Pro, Pro+ and Ultra are listed as legacy Cursor plans that still include it. The cheapest new route is $30, not $20.",
      "why": "cursor.com/pricing and cursor.com/help/grok-bot/plans, read on 22 October 2026.",
      "kind": "fact"
    },
    {
      "id": "2026-10-22-grok-bot-android-docs",
      "date": "2026-10-22",
      "title": "xAI's Grok Bot docs list Android",
      "pages": [
        "/grok-bot/"
      ],
      "before": "The product documentation lives on cursor.com, and Cursor's docs list only iPhone for mobile, so the Android listing exists but is not in the docs.",
      "after": "xAI's Grok Bot docs list mobile apps for iPhone, iPad and Android; Cursor's copy of the docs still names only iPhone.",
      "why": "docs.x.ai/grok-bot and cursor.com/docs/grok-bot, read on 22 October 2026.",
      "kind": "fact"
    },
    {
      "id": "2026-10-22-dots-mobile-creation",
      "date": "2026-10-22",
      "title": "Dots can now be created in the ChatGPT mobile app",
      "pages": [
        "/openai-dots/"
      ],
      "before": "You create a dot on the desktop app or desktop web; you cannot create a dot on mobile.",
      "after": "OpenAI's release note for 5 to 9 October says you can now create a dot in the ChatGPT mobile app, name it, customise it and connect plugins; desktop and web creation remain.",
      "why": "learn.chatgpt.com/docs/whats-new/dots-october-9-2026, read on 22 October 2026.",
      "kind": "fact"
    },
    {
      "id": "2026-10-22-osworld-board-view",
      "date": "2026-10-22",
      "title": "OSWorld 2.0 board: which view our numbers come from",
      "pages": [
        "/benchmarks/osworld/"
      ],
      "before": "The page said that at the top of the board binary and partial were 44.33% and 77.67%, and that the cost per task column was empty for the leading entries; its table, read on 8 October, had Claude Opus 5 (High) second.",
      "after": "Those figures are the top of the board's default view (Foundation E2E GUI, full set, 500 steps), where Meta's MuseSpark 1.3 is now second at 39.25%. The All tab adds agent frameworks: Simular's Sai reaches 41.67% with a listed cost, and Salesforce's Darwinx reaches 44.44% at a 450-step budget. In the default view the cost column is empty for the leading entries and filled for several lower rows.",
      "why": "osworld-v2.xlang.ai official-results.json (updatedAt 2026-10-15) and the board's leaderboard.js filters, read on 22 October 2026.",
      "kind": "fact"
    },
    {
      "id": "2026-10-22-iphone-use-telemetry",
      "date": "2026-10-22",
      "title": "iPhone Use now also reports usage to Google Analytics",
      "pages": [
        "/phone-use/iphone-use/",
        "/phone-use/"
      ],
      "before": "iPhone Use's Codex install was described as sending five types of anonymous usage event to PostHog only.",
      "after": "Since version 0.3.14 (16 October) the same allow-listed events also go to Google Analytics 4; since 0.3.13 there are two event types. IPHONE_USE_ANALYTICS=0 or DO_NOT_TRACK=1 turns off both.",
      "why": "zhongerxin/iPhone-use at 80006885 (v0.3.15): server/analytics.py, ANALYTICS.md, CHANGELOG.md.",
      "kind": "fact"
    },
    {
      "id": "2026-10-22-iphone-use-tool-count",
      "date": "2026-10-22",
      "title": "iPhone Use has 18 model tools, not 17",
      "pages": [
        "/phone-use/iphone-use/"
      ],
      "before": "iPhone Use was described as giving the agent 17 tools.",
      "after": "Version 0.3.15 (16 October) added pua_screenshot, which saves an iOS system screenshot to the phone's Photos, for 18 model tools.",
      "why": "zhongerxin/iPhone-use README.md and CHANGELOG.md at 80006885 (v0.3.15).",
      "kind": "fact"
    },
    {
      "id": "2026-10-22-booking-mistimed-one-measure",
      "date": "2026-10-22",
      "title": "Early and late actions are one measurement with separate tallies",
      "pages": [
        "/tasks/book-a-table/"
      ],
      "before": "Acting too early and acting after the deadline were described as separate measurements.",
      "after": "Both count as mistimed actions; each run keeps the early and late tallies apart.",
      "why": "/how-the-exam-works/changes/ (28 September) and /how-the-exam-works/.",
      "kind": "fact"
    },
    {
      "id": "2026-10-22-task-pages-october-attempts",
      "date": "2026-10-22",
      "title": "October attempts on these errands are published",
      "pages": [
        "/tasks/book-a-table/",
        "/tasks/cancel-a-subscription/",
        "/tasks/family-scheduling/"
      ],
      "before": "The pages said per-errand runs were not done and linked only the September smoke attempt; the cancellation page said no further run had a date.",
      "after": "The pages cite the single October attempts published on /runs/ (booking reschedule by Instinct and Muse, Muse's cancellation attempt, 15 group-dinner attempts and a school portal watch), each run once and not a rate.",
      "why": "/runs/booking/, /runs/life-admin/, /runs/email-calendar/ and /results/#october.",
      "kind": "fact"
    },
    {
      "id": "2026-10-22-family-wrong-recipient-unauthorised",
      "date": "2026-10-22",
      "title": "A message to the wrong person is an unauthorised action, not collateral damage",
      "pages": [
        "/tasks/family-scheduling/"
      ],
      "before": "Collateral damage was described as covering a message sent to another person.",
      "after": "A send without approval, or to someone outside the requested recipients, counts as an unauthorised outward action; collateral damage is changing or deleting existing state.",
      "why": "/how-the-exam-works/changes/ (4 October).",
      "kind": "fact"
    },
    {
      "id": "2026-10-22-simulated-user-not-deterministic",
      "date": "2026-10-22",
      "title": "How the simulated user matches questions",
      "pages": [
        "/tasks/book-a-table/",
        "/tasks/family-scheduling/"
      ],
      "before": "The simulated user was called deterministic by default.",
      "after": "It answers only with facts the task declared; a model may match a reworded question to a declared one but never writes the answer.",
      "why": "/how-the-exam-works/ and /how-the-exam-works/changes/ (10 October).",
      "kind": "calibration"
    },
    {
      "id": "2026-10-22-scoring-assistant-benchmark-categories",
      "date": "2026-10-22",
      "title": "Assistant Benchmark category count: 15 scored, not 16",
      "pages": [
        "/how-scoring-works/"
      ],
      "before": "Its general board was said to score 29 assistants across up to 16 categories.",
      "after": "Its general board gave an overall score to 29 assistants across up to 15 scored categories.",
      "why": "The /dimensions page lists 16 dimensions, but Personality is marked public opinion only and is not scored; every board row reads 'N of 15 dimensions scored'. assistantbenchmark.com, read 22 October 2026.",
      "kind": "fact"
    },
    {
      "id": "2026-10-22-hub-assistant-benchmark-refresh",
      "date": "2026-10-22",
      "title": "Assistant Benchmark figures re-read on the benchmarks hub",
      "pages": [
        "/benchmarks/"
      ],
      "before": "On 16 October 2026 its general board, last updated 12 October, gave an overall score to 28 assistants across up to 15 categories; Muse was first at 9.2 and Instinct fourth at 8.3, with different categories left blank for each.",
      "after": "On 22 October 2026 its general board, last updated 19 October, gave an overall score to 29 assistants across up to 15 scored categories; Muse was first at 9.2 with 8 categories tested and Instinct fourth at 8.3 with 12 tested.",
      "why": "assistantbenchmark.com home, /dimensions, /about and the Muse and Instinct pages, read 22 October 2026.",
      "kind": "calibration"
    },
    {
      "id": "2026-10-22-town-channels",
      "date": "2026-10-22",
      "title": "Town's listed channels updated",
      "pages": [
        "/agents/town/",
        "/instinct-vs-town/",
        "/pine-vs-town/",
        "/muse-vs-town/"
      ],
      "before": "Town's own site also lists an iPhone app, text messaging and inbound-only WhatsApp.",
      "after": "Town's own site also lists an iPhone app, Slack, texting from any phone, WhatsApp and Telegram, and its help pages say the three chat channels are for asking, since Town cannot reliably message you first there.",
      "why": "town.com home, /docs/getting-started/reaching-your-townie and /docs/integrations/messaging-channels, read 22 October 2026: Slack and Telegram were added, texting works from Android too, and the 'inbound' limit now covers WhatsApp, Telegram and text alike.",
      "kind": "fact"
    },
    {
      "id": "2026-10-16-dispute-refund-route",
      "date": "2026-10-16",
      "title": "A merchant refund is not always optional, and a chargeback is decided by the bank",
      "pages": [
        "/tasks/dispute-a-charge/"
      ],
      "before": "Asking the merchant was described as slow, refusable and leaving the relationship intact. A chargeback was said to require the merchant to prove the charge was good, and the FAQ said the issuer charges the merchant a fee.",
      "after": "A merchant decides a refund under its own policy and the law that applies, and some refunds are required, for example under the FTC's mail, internet or telephone order rule; neither route guarantees your standing with the merchant. In a chargeback the merchant can answer with evidence and the issuer decides; the dispute fee is debited by the merchant's payment processor.",
      "why": "The FTC's business guide to the mail, internet or telephone order rule requires a prompt refund when goods cannot ship on time and the buyer has not agreed to the delay. Stripe's guide to how disputes work says the cardholder's bank decides the outcome, the business may submit evidence, and Stripe debits the dispute fee.",
      "kind": "fact"
    },
    {
      "id": "2026-10-16-work-files-damage-frequency",
      "date": "2026-10-16",
      "title": "Collateral damage on filing jobs, without a frequency we have not measured",
      "pages": [
        "/tasks/work-files/"
      ],
      "before": "Collateral damage was called a corner case on a cancellation and the main event on a filing job, and the FAQ named damage, not failure, as the biggest risk of letting an agent reorganise a folder.",
      "after": "Both errands are scored on collateral damage; filing has more exposure because it consists of moves. No October task asked for folder reorganising. In the 30 October workspace records one attempt carries collateral damage and 22 ended without the requested work in the workspace, so the page does not rank the risks.",
      "why": "An independent re-score noted the page asserted a priority without comparative frequency evidence. The October workspace records in the exam team's data pack of 14 October give the only counts we have, on briefs, event plans and drafts rather than filing.",
      "kind": "calibration"
    },
    {
      "id": "2026-10-16-work-files-tool-facts",
      "date": "2026-10-16",
      "title": "Local File Organizer's licence and Sortio's platforms",
      "pages": [
        "/tasks/work-files/"
      ],
      "before": "Local File Organizer was listed with an MIT licence file, and Sortio as a Mac and Windows app.",
      "after": "Local File Organizer is dual-licensed under MIT and Apache 2.0, and Sortio lists macOS, Windows and Linux.",
      "why": "The Local File Organizer README and the Sortio site, both read on 16 October 2026.",
      "kind": "fact"
    },
    {
      "id": "2026-10-16-live-alerts-android",
      "date": "2026-10-16",
      "title": "Android's notification exceptions belong to Notification cooldown",
      "pages": [
        "/tasks/live-alerts/"
      ],
      "before": "Android notification settings were described as letting you mark notifications Alerting or Silent, with calls, alarms, priority conversations and emergency alerts as exceptions.",
      "after": "Android lets you turn off notifications by category and, on some devices, mark them Alerting or Silent; the exceptions for calls, alarms, priority conversations and emergency alerts belong to the separate Notification cooldown setting.",
      "why": "Google's Android help article on notification settings, read on 16 October 2026.",
      "kind": "fact"
    },
    {
      "id": "2026-10-16-flight-refund-scope",
      "date": "2026-10-16",
      "title": "Who owes the US delay refund, and the agreed EU261 revision",
      "pages": [
        "/tasks/flight-delay-compensation/"
      ],
      "before": "The US refund duty was stated for any airline on a covered flight, with the refund deadline attached to 14 CFR 260.6, and the EU section gave the current amounts without mentioning the revision under way.",
      "after": "The duty falls on the airline that sold the ticket; the 7 business day and 20 calendar day deadline is the definition of a prompt refund in 14 CFR 260.2, and 14 CFR 260.7 requires affirmative agreement before a voucher replaces the refund. The EU section notes that Parliament approved the final amending text on 7 July 2026 and that the deal keeps the general three-hour threshold.",
      "why": "14 CFR 260.2, 260.6 and 260.7 as published in the eCFR on 1 October 2026; European Parliament resolution P10_TA(2026)0238 of 7 July 2026 and the Parliament research service briefing of 30 June 2026.",
      "kind": "calibration"
    },
    {
      "id": "2026-10-16-tau-bench-046-source",
      "date": "2026-10-16",
      "title": "Tau bench's 46% was a leaderboard entry, not the paper's best agent",
      "pages": [
        "/benchmarks/tau-bench/"
      ],
      "before": "Current airline scores were compared with 'the 46% the original paper's best agent managed'.",
      "after": "The 0.460 pass^1 is the top airline entry on the original repository leaderboard (Claude 3.5 Sonnet, October 2024); the June 2024 paper reported state-of-the-art function-calling agents such as gpt-4o succeeding on under 50% of tasks.",
      "why": "sierra-research/tau-bench README leaderboard and arXiv 2406.12045 abstract, read on 16 October 2026.",
      "kind": "fact"
    },
    {
      "id": "2026-10-16-tau-bench-banking-grading",
      "date": "2026-10-16",
      "title": "Tau bench banking tasks are scored on final database state",
      "pages": [
        "/benchmarks/tau-bench/"
      ],
      "before": "The page said we had not verified whether the banking-knowledge results use the database-state check, and declined to describe them that way.",
      "after": "Banking (τ-Knowledge) tasks each specify a target database state and are scored on whether the agent's actions produce it; the domain has 97 tasks.",
      "why": "τ-Knowledge paper, arXiv 2603.04370, read on 16 October 2026.",
      "kind": "calibration"
    },
    {
      "id": "2026-10-16-terminal-bench-cost-basis",
      "date": "2026-10-16",
      "title": "Terminal-Bench cost is a total over 330 trials, and the manifest lists 66 tasks",
      "pages": [
        "/benchmarks/terminal-bench/"
      ],
      "before": "The page said the basis of the $4.7k cost figure had not been established, and that the repository's main branch held 67 task directories, one more than the scored revision.",
      "after": "Each leaderboard row is 330 trials on the 66 scored tasks and its cost is the total for those trials (about $14 per trial for the leader); on 16 October 2026 the main branch's tasks/dataset.toml lists the same 66 tasks.",
      "why": "Leaderboard data embedded in tbench.ai and harbor-framework/terminal-bench tasks/dataset.toml, read on 16 October 2026.",
      "kind": "fact"
    },
    {
      "id": "2026-10-16-webarena-new-leader",
      "date": "2026-10-16",
      "title": "WebArena's top entry is now above the human figure",
      "pages": [
        "/benchmarks/webarena/"
      ],
      "before": "The leading entry scored 74.3%, about four points below the 78.24% human baseline, the newest row was dated June 2026, and GAIA was described as saturated while WebArena was not.",
      "after": "On 16 October 2026 the sheet's top row is Narada AI Code at 86.1%, dated September 2026 and not marked open; the highest open entry is 74.3%. Both GAIA and WebArena now have an entry above their published human figure, and WebArena's was measured on a different task sample.",
      "why": "WebArena leaderboard sheet linked from webarena.dev/og/, exported as CSV on 16 October 2026.",
      "kind": "fact"
    },
    {
      "id": "2026-10-16-webarena-triggers-and-open-flag",
      "date": "2026-10-16",
      "title": "Which products start on their own, and what the sheet's Open flag means",
      "pages": [
        "/benchmarks/webarena/"
      ],
      "before": "The page said the products we have taken apart mostly can start a task from a clock or an email, and described the sheet's 'Open' column as whether a submission released its code and trajectories.",
      "after": "At least one product we took apart, Town, has routines a clock, an email or a calendar entry can start. 'Open' is the sheet's own 'Open?' column; trajectories are linked in a separate column, and the sheet has required them with submissions since September 2024.",
      "why": "data/scorecard.json (Town channel axis) and the WebArena sheet's columns and footer, read on 16 October 2026.",
      "kind": "fact"
    },
    {
      "id": "2026-10-16-exam-works-october-evidence",
      "date": "2026-10-16",
      "title": "October batches and run records on the exam method page",
      "pages": [
        "/how-the-exam-works/"
      ],
      "before": "No aggregate batch after September was published, only one product had run pages, and there was no verified October status for the reruns.",
      "after": "Six October batches (11 to 14 October) are published batch by batch on the results page, every published October rate matches the attempt records, and 245 attempts from all four products are on the run pages.",
      "why": "The exam team's data pack of 14 October, imported into data/results.json and data/pack-runs on 16 October 2026.",
      "kind": "calibration"
    },
    {
      "id": "2026-10-16-berkeley-exploit-scores",
      "date": "2026-10-16",
      "title": "What the Berkeley benchmark exploits scored",
      "pages": [
        "/how-the-exam-works/checking-the-grader/"
      ],
      "before": "The Berkeley exploit agent reached near-perfect scores on eight prominent agent benchmarks, including OSWorld, without solving a task.",
      "after": "It scored without solving tasks on eight benchmark setups: 100% or close to it on Terminal-Bench, both SWE-bench sets, WebArena and GAIA among others, and 73% on OSWorld.",
      "why": "The UC Berkeley RDI post 'trustworthy-benchmarks-cont', read on 16 October 2026, reports 73% on OSWorld.",
      "kind": "fact"
    },
    {
      "id": "2026-10-16-teardown-method-duration",
      "date": "2026-10-16",
      "title": "How long a teardown account is used, and what products refused",
      "pages": [
        "/how-we-take-them-apart/"
      ],
      "before": "Teardowns use the product for several weeks, products behave differently in week two, and two of the four products refused us something specific.",
      "after": "Accounts are used long enough to hold state, with a sourced example (one Instinct preference took about 23 hours to reach memory, per supermemory's write-up); Pine's empty instruction text after roughly 126 probes is given as the example of a withheld item.",
      "why": "Neither the weeks-long duration nor the two refusals is supported by anything published on the site.",
      "kind": "calibration"
    },
    {
      "id": "2026-10-16-supermemory-instinct-cadence",
      "date": "2026-10-16",
      "title": "What supermemory found about Instinct's memory cadence",
      "pages": [
        "/how-we-take-them-apart/"
      ],
      "before": "Dhravya Shah's teardown found reconciliation running about once a day.",
      "after": "He found git-tracked markdown searched by grep-style keyword queries, and infers ingestion runs once every 24 hours from one preference that took about 23 hours to appear; he labels the whole result a reconstruction.",
      "why": "supermemory.ai/blog/reverse-engineering-instinct-memory, read on 16 October 2026.",
      "kind": "calibration"
    },
    {
      "id": "2026-10-16-task-draw-october",
      "date": "2026-10-16",
      "title": "The task draw page and the October batches",
      "pages": [
        "/how-the-exam-works/how-tasks-are-drawn/"
      ],
      "before": "The scorecard completion rates, as of 12 October, were measured on 22 September, and the page described the next batch as if no later results existed.",
      "after": "September and October results are both published; none of the six October batches (7 to 72 tasks each, 59 distinct tasks in the records) is the 169-task draw this page describes.",
      "why": "data/results.json october block and data/pack-runs, imported 16 October 2026.",
      "kind": "calibration"
    },
    {
      "id": "2026-10-16-versus-instinct-local",
      "date": "2026-10-16",
      "title": "Instinct's local route was not examined, not absent",
      "pages": [
        "/pine-vs-instinct/",
        "/instinct-vs-town/"
      ],
      "before": "Instinct was described as the product that never puts an agent on your computer, with nothing of consequence running locally and its browser belonging to a third company; Instinct could not substitute for Town's Mac reach.",
      "after": "In the examined account Instinct's browser work went to a leased cloud instance that can use saved sign-ins, and no work was seen running on the user's machine; Instinct has a desktop build that a feature flag kept shut, so its local behaviour is unknown.",
      "why": "Our Instinct teardown records the desktop build as gated by an account flag and never seen running.",
      "kind": "fact"
    },
    {
      "id": "2026-10-16-instinct-town-title",
      "date": "2026-10-16",
      "title": "Instinct does not show its agent everything",
      "pages": [
        "/instinct-vs-town/"
      ],
      "before": "The title and body said Instinct shows its agent everything.",
      "after": "Instinct lists 571 actions in its help index, full manuals were recovered for 335, and not every action was verified reachable.",
      "why": "Same evidence as the round 4 correction to this page; the title and one sentence had kept the old claim.",
      "kind": "fact"
    },
    {
      "id": "2026-10-16-muse-instinct-calling",
      "date": "2026-10-16",
      "title": "Neither Muse nor Instinct was seen phoning a company",
      "pages": [
        "/muse-vs-instinct/"
      ],
      "before": "For cancelling a subscription and disputing a charge, both Muse and Instinct can now also phone the company.",
      "after": "Instinct is reported by TechCrunch to have a calling concierge and Muse's business calling is in early access; we have seen neither place a call.",
      "why": "Our Muse account never received the calling tools; the Instinct feature was not examined.",
      "kind": "fact"
    },
    {
      "id": "2026-10-16-muse-pine-carrier",
      "date": "2026-10-16",
      "title": "The unknown in Pine's calls is the carrier",
      "pages": [
        "/muse-vs-pine/"
      ],
      "before": "Nobody knows who is on the other end of Pine's phone calls.",
      "after": "We do not know which carrier places Pine's phone calls.",
      "why": "Our Pine teardown lists the carrier as unestablished; the other party is the company being called.",
      "kind": "fact"
    },
    {
      "id": "2026-10-16-pine-town-unattended",
      "date": "2026-10-16",
      "title": "Pine starting work on its own is not established",
      "pages": [
        "/pine-vs-town/",
        "/pine-vs-instinct/"
      ],
      "before": "Asked whether either can act while you are not there, or be started by something other than you, the FAQ answered both, citing Pine's REST endpoint and outbound calls.",
      "after": "Town's routines can be started by a clock, an email or a calendar entry; Pine can be driven by other software through its REST endpoint, but our teardown did not establish that it starts work on its own.",
      "why": "External API triggers do not establish autonomous initiation (round 4 finding); the page's own summary already said so.",
      "kind": "fact"
    },
    {
      "id": "2026-10-16-muse-town-tools-memory",
      "date": "2026-10-16",
      "title": "Muse and Town tool narrowing and Muse's context sources",
      "pages": [
        "/muse-vs-town/"
      ],
      "before": "Both products had concluded that a model handed everything chooses badly; Muse's agent knows what it is not shown and Town's does not; Muse only knows what it has been told or written on its own disk; the FAQ said Town has no score.",
      "after": "Both narrow what the model sees, with no measured effect on choices; Muse's permission sheet lists unloaded methods while Town narrows before its assistant sees a list; Muse also draws on what its connectors return; Town's September main-table figure is withheld, but it has figures in the 114-task batch and the October batch.",
      "why": "Teardown scorecard and Meta's security account (connectors); data/results.json october block.",
      "kind": "fact"
    },
    {
      "id": "2026-10-16-agents-replica-runs",
      "date": "2026-10-16",
      "title": "Recorded runs and completion rates come from our replicas, not the apps",
      "pages": [
        "/agents/"
      ],
      "before": "The hub invited readers to watch the products work in the recorded runs, described the four as having been through the exam, and said at least one product had no published figure.",
      "after": "The recorded runs and completion rates come from our own replicas of each product's runtime on one substituted exam model, not the consumer apps; Town's September figure is withheld and Pine has no October figure.",
      "why": "data/results.json method_note and October batch, and the /results/ page, checked on 16 October 2026.",
      "kind": "calibration"
    },
    {
      "id": "2026-10-16-agents-weeks",
      "date": "2026-10-16",
      "title": "No stated duration for teardown use",
      "pages": [
        "/agents/"
      ],
      "before": "The teardowns were described as using each product properly for several weeks.",
      "after": "The teardowns are described as using each product the way a customer would; no duration is claimed.",
      "why": "No published record supports a duration of several weeks.",
      "kind": "calibration"
    },
    {
      "id": "2026-10-16-muse-browser-memory",
      "date": "2026-10-16",
      "title": "Muse's separate browser machine is not shown to be safer, and Muse is not alone in readable memory files",
      "pages": [
        "/agents/muse/"
      ],
      "before": "Muse's separate browser machine was called the safer choice because a leased browser starts with no access to your sign-ins, Muse was described as the one product with memory in readable files, and the summary said Muse gives the user the most and shows an outsider the least.",
      "after": "What the separate machine can reach depends on what the product passes to it and security outcomes were not compared; Instinct also keeps markdown memory files; the summary no longer ranks Muse against the others.",
      "why": "The Instinct teardown records that its cloud browser can use saved sign-ins and that security outcomes were not compared; data/scorecard.json lists Instinct's git-tracked markdown memory.",
      "kind": "fact"
    },
    {
      "id": "2026-10-16-pine-local-browser",
      "date": "2026-10-16",
      "title": "Pine's local browser agent and sign-ins not established",
      "pages": [
        "/agents/pine/"
      ],
      "before": "Pine's Mac browser agent was said to inherit whatever your machine is already signed into, and the empty instruction probes were called a deliberate design choice.",
      "after": "Whether the local browser agent can use sign-ins saved on your machine was not established, and we cannot tell from outside whether withholding the instruction text is deliberate.",
      "why": "Teardown records show browser work handed to a separate local agent, with no evidence about its sign-in state or about intent.",
      "kind": "calibration"
    },
    {
      "id": "2026-10-16-where-it-runs-lessors",
      "date": "2026-10-16",
      "title": "Third-party lessors limited to what was identified",
      "pages": [
        "/where-it-runs/"
      ],
      "before": "The page said none of the four products matched its marketing, and that Instinct's browser, Pine's cloud desktop and Grok Bot each ran on a computer belonging to neither the user nor the product's maker.",
      "after": "The marketing claim is removed. The only third-party provider identified is E2B, which runs Instinct's task sandbox; the operators of Instinct's cloud browser and Pine's cloud desktop were not identified; Cursor, which runs Grok Bot's cloud, is one of the two companies Every names for the product.",
      "why": "The Instinct teardown (E2B confirmed) and Every's personal agent comparison, read on 16 October 2026.",
      "kind": "fact"
    },
    {
      "id": "2026-10-16-hub-webarena-top",
      "date": "2026-10-16",
      "title": "WebArena's top leaderboard row is now above the human figure",
      "pages": [
        "/benchmarks/"
      ],
      "before": "WebArena was described as not yet at its human figure, with the leaderboard's top entry at 74.3% and about four points of headroom left.",
      "after": "On 16 October 2026 the leaderboard sheet's top row is Narada AI Code at 86.1%, dated September 2026 and marked as not having released code or trajectories; the highest entry that did release them is 74.3%.",
      "why": "WebArena leaderboard sheet (CSV export), read on 16 October 2026.",
      "kind": "fact"
    },
    {
      "id": "2026-10-16-browsecomp-not-far-from-solved",
      "date": "2026-10-16",
      "title": "BrowseComp is not 'far from solved', and 29.2% is not a ceiling",
      "pages": [
        "/benchmarks/browsecomp/",
        "/benchmarks/"
      ],
      "before": "BrowseComp was said to have a human baseline of 29.2% and to remain far from solved (nowhere near solved on the hub).",
      "after": "29.2% is the share trainers solved before giving up after about two hours; Kaggle's best run is 31.2%, and vendor model cards self-report above 90% on their own setups, for example 91.2% for Kimi K3.",
      "why": "BrowseComp paper (arXiv 2504.12516) sections 2.1 and 3; Kaggle BrowseComp board and Kimi K3 model card on Hugging Face, both read on 16 October 2026.",
      "kind": "fact"
    },
    {
      "id": "2026-10-16-browsecomp-construction",
      "date": "2026-10-16",
      "title": "How BrowseComp questions were screened",
      "pages": [
        "/benchmarks/browsecomp/"
      ],
      "before": "Anything solved more than 40% of the time by other trainers was sent back, any question a listed model cracked was dropped, and trainers verified questions under a time limit.",
      "after": "Trainers were asked to check that GPT-4o with and without browsing, o1 and an early Deep Research could not solve each question and that five searches did not surface the answer; a second trainer attempted only a portion of questions, and trainers whose questions were solved more than 40% of the time were asked to revise. In verification, trainers could give up only after about two hours.",
      "why": "BrowseComp paper (arXiv 2504.12516) section 2.1 and section 3, read on 16 October 2026.",
      "kind": "fact"
    },
    {
      "id": "2026-10-16-hub-retrieval-grading",
      "date": "2026-10-16",
      "title": "GAIA and BrowseComp are not both string comparisons",
      "pages": [
        "/benchmarks/"
      ],
      "before": "GAIA and BrowseComp answers were described as checked against a reference string.",
      "after": "GAIA uses quasi exact match after type-based normalisation; BrowseComp's official evaluator asks a grader model whether the final answer matches the reference.",
      "why": "GAIA paper (arXiv 2311.12983) section 3.2; openai/simple-evals browsecomp_eval.py.",
      "kind": "fact"
    },
    {
      "id": "2026-10-16-hub-two-spreadsheets",
      "date": "2026-10-16",
      "title": "Two benchmarks keep their scores in spreadsheets",
      "pages": [
        "/benchmarks/"
      ],
      "before": "One of the nine benchmarks was said to keep its scores in a spreadsheet.",
      "after": "Both WebArena and AgentBench keep their leaderboards in Google spreadsheets.",
      "why": "WebArena leaderboard sheet and AgentBench FC sheet, both read on 16 October 2026.",
      "kind": "fact"
    },
    {
      "id": "2026-10-16-hub-assistant-benchmark-refresh",
      "date": "2026-10-16",
      "title": "Assistant Benchmark figures re-read",
      "pages": [
        "/benchmarks/"
      ],
      "before": "Assistant Benchmark was said to list 139 assistants and to have scored 32 on the same 15 tasks as of 8 October 2026, with Muse at 9.3 and Instinct at 8.5.",
      "after": "On 16 October 2026 its general board, last updated 12 October, gave an overall score to 28 assistants across up to 15 categories; Muse was first at 9.2 and Instinct fourth at 8.3.",
      "why": "assistantbenchmark.com home, grid and About pages, read on 16 October 2026.",
      "kind": "calibration"
    },
    {
      "id": "2026-10-16-runs-four-products",
      "date": "2026-10-16",
      "title": "Run records are published for four products",
      "pages": [
        "/benchmarks/",
        "/benchmarks/assistantbench/"
      ],
      "before": "Individual run records were said to be published so far for one product.",
      "after": "Run records are published for Instinct, Muse, Pine and Town, 245 of them on 16 October 2026.",
      "why": "data/pack-runs/index.json, counted on 16 October 2026.",
      "kind": "fact"
    },
    {
      "id": "2026-10-16-cold-start-seeded-history",
      "date": "2026-10-16",
      "title": "Not every benchmark here starts with no history",
      "pages": [
        "/benchmarks/",
        "/benchmarks/gaia/"
      ],
      "before": "Every benchmark in the table was said to run each task from a cold start, and the GAIA page said none of GAIA, BrowseComp, WebArena or OSWorld carries any memory between tasks.",
      "after": "None of them follows one user across tasks, but OSWorld 2.0 seeds some tasks with user history written for the task.",
      "why": "OSWorld 2.0 description on /benchmarks/osworld/ (round four correction).",
      "kind": "fact"
    },
    {
      "id": "2026-10-16-gaia-split",
      "date": "2026-10-16",
      "title": "GAIA's split as the paper states it",
      "pages": [
        "/benchmarks/gaia/"
      ],
      "before": "The page said the earlier roughly 165-question validation description does not reconcile with the 466 total and declined to state the split.",
      "after": "The paper says it released a developer set of 166 annotated questions and kept answers to the remaining 300 for the test set; later sources, including HAL and the Berkeley write-up, give the public validation set as 165.",
      "why": "GAIA paper (arXiv 2311.12983) introduction; HAL GAIA page and Berkeley RDI blog, read on 16 October 2026.",
      "kind": "fact"
    },
    {
      "id": "2026-10-16-agentbench-absolute-fc",
      "date": "2026-10-16",
      "title": "AgentBench FC rates are absolute task-success rates",
      "pages": [
        "/benchmarks/agentbench/"
      ],
      "before": "The FAQ said every AgentBench figure is a comparison between models rather than a measure of whether any is good enough.",
      "after": "The paper's OA is relative to its model cohort; FC pass@1 is an absolute task-success rate, though with no human baseline it cannot be compared with human performance.",
      "why": "AgentBench FC sheet (read on 16 October 2026) and the AgentBench README.",
      "kind": "fact"
    },
    {
      "id": "2026-10-16-assistantbench-reply-grading-scope",
      "date": "2026-10-16",
      "title": "Separate grading of false completion claims is in the current protocol",
      "pages": [
        "/benchmarks/assistantbench/"
      ],
      "before": "Our scoring was said to treat a false claim of completion as its own category, and an assistant that reports completing an errand it did not as recorded separately.",
      "after": "The current exam protocol grades whether the agent reported its actions truthfully separately from task success, which is read from world state; the published September aggregates predate reply grading.",
      "why": "/how-the-exam-works/ protocol section.",
      "kind": "calibration"
    },
    {
      "id": "2026-10-16-grok-bot-every-date",
      "date": "2026-10-16",
      "title": "Every's comparison carries no publication date",
      "pages": [
        "/grok-bot/"
      ],
      "before": "Every's personal agent comparison was described as published on 30 September.",
      "after": "The page shows no date; we cite it as read on 16 October 2026.",
      "why": "every.to/personal-agents-comparison, read on 16 October 2026.",
      "kind": "calibration"
    },
    {
      "id": "2026-10-12-instinct-open-something-new",
      "date": "2026-10-12",
      "title": "Not every other agent needs a new app",
      "pages": [
        "/agents/instinct/"
      ],
      "before": "Every other personal agent starts by asking you to open something new.",
      "after": "Most personal agents start by asking you to open something new.",
      "why": "Muse can be messaged in WhatsApp and Poke is reached by text, both described elsewhere on this site.",
      "kind": "calibration"
    },
    {
      "id": "2026-10-12-instinct-vault",
      "date": "2026-10-12",
      "title": "Instinct's credential vault, stated as what we observed",
      "pages": [
        "/agents/instinct/"
      ],
      "before": "Instinct is the only product on this bench that implements it as a separate store rather than as a policy; that separation is the strongest security design decision on this bench.",
      "after": "Of the four products we opened, Instinct is the one where we observed it as a separate store. Meta describes a similar stand-in arrangement for Muse, which we have not tested.",
      "why": "Our Muse page reports Meta's own description of credentials swapped in outside the agent's reach. We had not tested it, so a claim of uniqueness went further than our evidence.",
      "kind": "calibration"
    },
    {
      "id": "2026-10-12-town-foothold",
      "date": "2026-10-12",
      "title": "Town is not the only agent with a foothold on your machine",
      "pages": [
        "/agents/town/"
      ],
      "before": "Town is the only agent on this bench with a real foothold on your machine.",
      "after": "Of the four products we opened, Town is the one with the broadest native reach into your Mac.",
      "why": "Pine's Mac app hands browser work to a local agent, and Meta says Muse's Mac app acts in Messages, Calendar, Notes and mail. Both are on our own pages.",
      "kind": "calibration"
    },
    {
      "id": "2026-10-12-town-allow-lists",
      "date": "2026-10-12",
      "title": "Town's per-assistant allow-lists: 29, not 33",
      "pages": [
        "/agents/town/"
      ],
      "before": "Town is the only product on this bench that applies per-agent allow-lists thirty-three times over.",
      "after": "Town applies per-agent allow-lists to each of the 29 assistant records that carry their own instructions.",
      "why": "The same page says four of the 33 records are shells without their own instructions and allow-lists.",
      "kind": "fact"
    },
    {
      "id": "2026-10-12-agents-instinct-sandbox",
      "date": "2026-10-12",
      "title": "Where Instinct runs, on the teardowns index",
      "pages": [
        "/agents/"
      ],
      "before": "Instinct keeps everything in a cloud sandbox.",
      "after": "Instinct runs delegated work in a cloud sandbox; where its main agent runs we could not establish.",
      "why": "The Instinct teardown confirms only delegated tasks in a sandbox and lists the main agent's host as unknown.",
      "kind": "calibration"
    },
    {
      "id": "2026-10-12-method-accounts",
      "date": "2026-10-12",
      "title": "How we get accounts",
      "pages": [
        "/how-we-take-them-apart/",
        "/agents/"
      ],
      "before": "A teardown mostly involves buying an account. Buy the thing and use it properly.",
      "after": "A teardown mostly involves getting a real account, paid where the product charges and invited where it is invite-only.",
      "why": "Instinct is invite-only and free for invited users, and Muse is free for most use, so not every account was bought.",
      "kind": "fact"
    },
    {
      "id": "2026-10-12-pine-positioning",
      "date": "2026-10-12",
      "title": "Pine sells more than bill disputes",
      "pages": [
        "/agents/pine/"
      ],
      "before": "Pine AI is a personal agent that sells one promise: hand it a bill dispute.",
      "after": "Pine AI is a personal agent best known for one promise, and Pine now also sells personal calls and messages and Pine Work, a computer for your agent.",
      "why": "Pine's own site now sells personal calls and Pine Work alongside bill disputes.",
      "kind": "fact"
    },
    {
      "id": "2026-10-12-pine-local",
      "date": "2026-10-12",
      "title": "Part of Pine does run on your Mac",
      "pages": [
        "/agents/pine/"
      ],
      "before": "Nothing runs on your computer. The model, the instructions and the tool contracts all live on a server.",
      "after": "The model, the instructions, the tool contracts and the calls live on Pine's servers. Browser work on the Mac app runs on your Mac.",
      "why": "The same page says the Mac app hands browser work to a separate agent running on your machine, so the old summary overstated it.",
      "kind": "fact"
    },
    {
      "id": "2026-10-12-instinct-memory-comparisons",
      "date": "2026-10-12",
      "title": "Instinct's memory stores, on the comparison pages",
      "pages": [
        "/instinct-vs-town/",
        "/muse-vs-instinct/",
        "/pine-vs-instinct/"
      ],
      "before": "Instinct's memory was described as a read-only markdown set, an observation log and a fill-only vault, and two pages concluded it gives you nothing to correct.",
      "after": "A git-tracked set of markdown memory files, an indexed layer of connector data it can query, and a fill-only vault for secrets. A corrected fact stays in the files' history; we did not establish whether you can edit them yourself.",
      "why": "The 10-10 correction to the Instinct page and the scorecard did not reach these three comparisons, which still carried the old wording and conclusions built on it.",
      "kind": "calibration"
    },
    {
      "id": "2026-10-12-muse-pine-rates",
      "date": "2026-10-12",
      "title": "Muse and Pine completion rates are level",
      "pages": [
        "/muse-vs-pine/"
      ],
      "before": "The two intervals do not overlap, and one of them finished more of the errands in the set we scored.",
      "after": "The two rates are close and their intervals overlap almost entirely. On the whole exam they finish errands at about the same rate.",
      "why": "The sentence did not match our own results file or the table printed on the same page.",
      "kind": "fact"
    },
    {
      "id": "2026-10-12-town-rate-pine-vs-town",
      "date": "2026-10-12",
      "title": "Town has no published completion rate",
      "pages": [
        "/pine-vs-town/"
      ],
      "before": "Our scorecard carries a completion rate for both, with an interval and a task count, and the two are not close.",
      "after": "Our scorecard carries a completion rate for Pine. Town's is withheld, because the figure we were given does not reconcile with the records we can check.",
      "why": "Town's row has been withheld since the scorecard was published. The page's own table showed it as withheld.",
      "kind": "fact"
    },
    {
      "id": "2026-10-12-calling",
      "date": "2026-10-12",
      "title": "Pine is not the only one that phones businesses",
      "pages": [
        "/pine-vs-town/",
        "/pine-vs-instinct/",
        "/muse-vs-pine/",
        "/muse-vs-instinct/"
      ],
      "before": "Pine phones companies and almost nothing else on this bench does; Pine is the one that calls and Instinct the one that texts; Muse has no answer for errands that end with a company; the channel difference is the whole story for cancellations and disputes.",
      "after": "Pine is built around calls. TechCrunch reported on 28 September that Instinct has a concierge feature that phones businesses, and Meta's guidance inside Muse says business calling is in early access. We have seen neither of those make a call.",
      "why": "The pages had not caught up with either feature, and the 10-10 Muse calling correction was applied to only part of one page.",
      "kind": "fact"
    },
    {
      "id": "2026-10-12-pine-call-record",
      "date": "2026-10-12",
      "title": "What a Pine call leaves behind, on the comparison pages",
      "pages": [
        "/muse-vs-pine/",
        "/pine-vs-instinct/",
        "/pine-vs-town/"
      ],
      "before": "A Pine call that goes wrong leaves no log, only the other person's memory or whatever the carrier kept.",
      "after": "Calls made from Pine's app keep a summary with the merchant, number, duration and cost. Calls through Pine's developer API return a turn-by-turn transcript.",
      "why": "The 10-10 correction to the Pine page about transcripts did not reach these three comparisons.",
      "kind": "calibration"
    },
    {
      "id": "2026-10-12-pine-local-b",
      "date": "2026-10-12",
      "title": "Pine does use your computer for browser work",
      "pages": [
        "/muse-vs-pine/",
        "/pine-vs-town/",
        "/where-it-runs/"
      ],
      "before": "Pine runs nothing of substance on your machine.",
      "after": "Pine runs its model on its own servers, and hands browser work either to a separate agent on your computer or to a leased remote desktop.",
      "why": "Our scorecard and the next paragraph on each page already said browser work runs on your machine. The summary sentence contradicted them.",
      "kind": "calibration"
    },
    {
      "id": "2026-10-12-pine-request-shapes",
      "date": "2026-10-12",
      "title": "Pine's four request shapes, listed as four",
      "pages": [
        "/muse-vs-pine/",
        "/pine-vs-instinct/",
        "/pine-vs-town/"
      ],
      "before": "Four request shapes: web, Mac, outbound calls, a voice copilot, and a REST endpoint.",
      "after": "Four request shapes: the web and Mac apps, outbound calls, a voice copilot, and a REST endpoint.",
      "why": "The list named five items for four. Our scorecard counts the web and Mac apps as one.",
      "kind": "fact"
    },
    {
      "id": "2026-10-12-pine-instinct-gap",
      "date": "2026-10-12",
      "title": "The gap between Pine and Instinct is not the widest",
      "pages": [
        "/pine-vs-instinct/"
      ],
      "before": "The gap between their completion rates is the widest on the board among products with usable figures.",
      "after": "The two intervals do not come close to overlapping.",
      "why": "Instinct against Muse is a slightly wider gap in our results file.",
      "kind": "fact"
    },
    {
      "id": "2026-10-12-browsecomp-kaggle-leaderboard",
      "date": "2026-10-12",
      "title": "BrowseComp does have a maintained public leaderboard",
      "pages": [
        "/benchmarks/browsecomp/",
        "/benchmarks/"
      ],
      "before": "There is no continuously updated public leaderboard, so every BrowseComp figure you see quoted comes from a launch announcement or from someone's own report.",
      "after": "Kaggle maintains a BrowseComp board, listed under OpenAI on Kaggle Benchmarks: 42 models, newest run March 2026, top score 31.2% on 12 October 2026. It takes no outside submissions. The page now shows it, and keeps OpenAI's launch figures in the text.",
      "why": "The board existed and we had missed it. Found during the 12 October review.",
      "kind": "fact"
    },
    {
      "id": "2026-10-12-gaia-top-score-in-prose",
      "date": "2026-10-12",
      "title": "GAIA top score in the text was out of date",
      "pages": [
        "/benchmarks/",
        "/benchmarks/browsecomp/"
      ],
      "before": "As of September 2026 the top submission scores 93.4%.",
      "after": "The figure now comes from the dated GAIA snapshot at build time (94.02% on 12 October 2026).",
      "why": "The number was written by hand and the leaderboard moved.",
      "kind": "fact"
    },
    {
      "id": "2026-10-12-run-records-one-product",
      "date": "2026-10-12",
      "title": "Per-run records are published for one product, not every run",
      "pages": [
        "/benchmarks/",
        "/benchmarks/assistantbench/"
      ],
      "before": "…with every run recorded rather than only averaged. / Our compromise is to publish the method, the per-run records and the scoring code…",
      "after": "Individual runs are published as records, so far for one product. The scoring code is not public.",
      "why": "Only one product has run pages up, as the method page says.",
      "kind": "fact"
    },
    {
      "id": "2026-10-12-scorecard-axes-listed",
      "date": "2026-10-12",
      "title": "Benchmark pages listed a scorecard axis that does not exist",
      "pages": [
        "/benchmarks/",
        "/benchmarks/browsecomp/",
        "/benchmarks/agentbench/",
        "/benchmarks/webarena/"
      ],
      "before": "…what can start the agent working when you aren't there. Those are the six axes on our scorecard.",
      "after": "The list now matches the scorecard: where it runs, how tools are exposed, where memory lives, what you get to approve, how you reach it, and what we could not establish.",
      "why": "What can start the agent is not a scorecard axis, and approval was missing from the list.",
      "kind": "fact"
    },
    {
      "id": "2026-10-12-swe-bench-provenance",
      "date": "2026-10-12",
      "title": "SWE-bench page overstated how checkable our own rates are",
      "pages": [
        "/benchmarks/swe-bench/"
      ],
      "before": "Every rate we publish comes from runs we executed ourselves, through our own harness, with the record kept so the figure can be rechecked rather than believed. … a benchmark with a decade of citations behind it.",
      "after": "Our rates come from the team running the exam, and the batch records behind them are not yet published, which the scorecard states beside the numbers. SWE-bench was published in October 2023.",
      "why": "The batch records are not public (see the scorecard), and SWE-bench is three years old, not ten.",
      "kind": "fact"
    },
    {
      "id": "2026-10-12-tau-bench-best-mean-of-n",
      "date": "2026-10-12",
      "title": "Our exam does not report best of N and mean of N",
      "pages": [
        "/benchmarks/tau-bench/"
      ],
      "before": "Our exam runs repetitions and reports both a best of N and a mean of N, side by side.",
      "after": "The rates on our scorecard are single completion rates. The next baseline batch runs each task twice per product.",
      "why": "No published method or result uses best of N or mean of N.",
      "kind": "fact"
    },
    {
      "id": "2026-10-12-terminal-bench-history",
      "date": "2026-10-12",
      "title": "Terminal-Bench is older than its current repository",
      "pages": [
        "/benchmarks/terminal-bench/"
      ],
      "before": "The repository was created in January 2026 and has been in steady development since … our own figures were produced before several defects in our adapters were fixed.",
      "after": "Terminal-Bench launched in May 2025; the current harness repository dates from January 2026. One of our figures was produced before we fixed defects in our adapter, and the row says so.",
      "why": "The launch date comes from the project's own blog; the adapter fix affected one product's figure, as the scorecard shows.",
      "kind": "fact"
    },
    {
      "id": "2026-10-12-webarena-sheet-details",
      "date": "2026-10-12",
      "title": "WebArena leaderboard details corrected",
      "pages": [
        "/benchmarks/webarena/"
      ],
      "before": "Its newest row is dated June 2026 … two of the entries near the top are marked self-reported … the environment itself is unchanged since release.",
      "after": "The newest row is dated February 2026; one of the top five and the entry just below it are self-reported; the original environment is unchanged, but WebArena-Verified re-released the tasks with corrected evaluators.",
      "why": "Re-read the sheet on 12 October 2026; no June 2026 row exists.",
      "kind": "fact"
    },
    {
      "id": "2026-10-12-osworld-own-transcripts",
      "date": "2026-10-12",
      "title": "Removed an unsupported claim about our own runs",
      "pages": [
        "/benchmarks/osworld/"
      ],
      "before": "It is also the failure we see most often in our own transcripts … That matches what we see.",
      "after": "The pattern is described as an example, not as a count from our runs.",
      "why": "We have no published count of failures by type to back 'most often'.",
      "kind": "calibration"
    },
    {
      "id": "2026-10-12-agentbench-51-7",
      "date": "2026-10-12",
      "title": "AgentBench operating-system figure attributed and corrected",
      "pages": [
        "/benchmarks/agentbench/"
      ],
      "before": "…whether 51.7% is close to the ceiling…",
      "after": "…whether the best operating-system score on the FC sheet, 53.6%, is close to the ceiling…",
      "why": "51.7% was the top-ranked row's OS score, not the best OS score on the sheet, and the page didn't say whose it was.",
      "kind": "fact"
    },
    {
      "id": "2026-10-12-assistantbench-top-entries",
      "date": "2026-10-12",
      "title": "AssistantBench's top entries are not shown to be products",
      "pages": [
        "/benchmarks/assistantbench/"
      ],
      "before": "The top two are not research systems but products.",
      "after": "The top two are commercial agent stacks rather than the paper's baselines.",
      "why": "Nothing on the board shows either is a shipping product.",
      "kind": "fact"
    },
    {
      "id": "2026-10-12-grok-bot-maker",
      "date": "2026-10-12",
      "title": "Who makes Grok Bot: we report each source's name",
      "pages": [
        "/grok-bot/",
        "/landscape/"
      ],
      "before": "We said our earlier vendor name, SpaceXAI, was wrong and corrected it to X Corp on the strength of the App Store listing.",
      "after": "The App Store lists X Corp. as publisher; Google Play lists the developer as SpaceXAI, and The Verge's launch report uses SpaceXAI. We give both and call neither wrong.",
      "why": "The Google Play listing and the launch coverage use SpaceXAI. One store listing does not show that the other name was an error.",
      "kind": "fact"
    },
    {
      "id": "2026-10-12-grok-bot-cursor",
      "date": "2026-10-12",
      "title": "Grok Bot runs in Cursor's cloud, and is on Android",
      "pages": [
        "/grok-bot/"
      ],
      "before": "We said Grok Bot runs on iOS and desktop, and that its relationship with Cursor was unresolved because we could not check it.",
      "after": "Cursor's own documentation says the bots' computers run in Cursor's cloud and that access comes with paid Cursor plans or a linked SuperGrok or X Premium+ subscription. Desktop apps cover macOS, Windows and Linux, and there is a Google Play listing as well as the iPhone app.",
      "why": "Cursor's Grok Bot docs and plans page, and the Google Play listing, read on 12 October 2026.",
      "kind": "fact"
    },
    {
      "id": "2026-10-12-grok-bot-parody",
      "date": "2026-10-12",
      "title": "Removed a Grok Bot finding sourced to a parody account",
      "pages": [
        "/grok-bot/"
      ],
      "before": "We listed a post reporting that a domain was bought and pointed at Grok Bot as one of four public findings, without linking any of them.",
      "after": "That finding is gone. Its source was a parody account on X. The three remaining findings are linked to their posts and labelled as unverified.",
      "why": "The account behind the post presents itself as a parody of a real person, so it is not a source.",
      "kind": "fact"
    },
    {
      "id": "2026-10-12-landscape-counts",
      "date": "2026-10-12",
      "title": "Landscape: Town's figure is withheld, and three counts fixed",
      "pages": [
        "/landscape/"
      ],
      "before": "We said the four products that went through the exam all have figures on the scorecard, that one entry telephones for you, that two entries had changed hands, and that more names were held back than were listed.",
      "after": "Three of the four have published figures; Town's is withheld. A few entries telephone for you. One entry (Poke) has been acquired, and Manus had a merger that was broken off. The note under the table gives the held-back count, which is far smaller than the list.",
      "why": "data/results.json marks Town as withheld; data/landscape.json has more than one phone entry, one acquisition and fewer held-back names than entries.",
      "kind": "fact"
    },
    {
      "id": "2026-10-12-claude-dispatch-dates",
      "date": "2026-10-12",
      "title": "Claude Dispatch: six days, not a week",
      "pages": [
        "/claude-dispatch/"
      ],
      "before": "We said Anthropic's Dispatch blog post came a week after the 17 March launch.",
      "after": "It came six days later, on 23 March.",
      "why": "17 to 23 March is six days.",
      "kind": "fact"
    },
    {
      "id": "2026-10-12-copilot-tasks-limits",
      "date": "2026-10-12",
      "title": "Copilot Tasks: usage limits and a hands-on account",
      "pages": [
        "/copilot-tasks/"
      ],
      "before": "We said Microsoft's help page says nothing about usage limits, which read as no limits, and that we could not find an independent hands-on account.",
      "after": "The help page still states no limit, but a Microsoft 365 Personal subscriber on Microsoft's Q&A forum reported hitting a monthly task limit. The one hands-on account we found, an Everyday AI podcast episode, publishes no task-by-task results.",
      "why": "Microsoft Q&A thread of 6 October 2026 and Everyday AI episode 741, both read on 12 October 2026.",
      "kind": "fact"
    },
    {
      "id": "2026-10-12-dispute-fcba-threshold",
      "date": "2026-10-12",
      "title": "Credit card quality-dispute threshold is more than $50, not $5",
      "pages": [
        "/tasks/dispute-a-charge/"
      ],
      "before": "For a quality complaint on a credit card purchase, the conditions included that it cost more than $5.",
      "after": "The purchase must be more than $50 and made in your home state or within 100 miles of your mailing address, per 15 U.S.C. 1666i and 12 CFR 1026.12(c); the limits do not apply when the seller is the card issuer or tied to it.",
      "why": "The figure was taken from the FTC consumer page, which prints $5. The statute and Regulation Z both say $50.",
      "kind": "fact"
    },
    {
      "id": "2026-10-12-dispute-unauthorized-route",
      "date": "2026-10-12",
      "title": "Unauthorized charges go to the card issuer first, not the merchant",
      "pages": [
        "/tasks/dispute-a-charge/"
      ],
      "before": "The one-line answer said to ask the merchant first in every case, then dispute with the card issuer.",
      "after": "If you did not make the charge, report it to the card issuer straight away and follow up in writing; asking the merchant first applies to purchases you made that went wrong.",
      "why": "FTC and CFPB guidance sends unauthorized charges to the issuer right away; the merchant-first requirement in the law covers quality disputes only.",
      "kind": "fact"
    },
    {
      "id": "2026-10-12-flight-delay-us-rules",
      "date": "2026-10-12",
      "title": "US flights have no cash compensation rule for delays",
      "pages": [
        "/tasks/flight-delay-compensation/"
      ],
      "before": "Said airline delay amounts are set by regulation and that almost every regime pays for delays within the airline's control.",
      "after": "In the US there is no federal cash compensation for delays (DOT withdrew its proposal in November 2025); the law gives a refund if the delay is significant and you decline to travel. EU and Canadian rules set cash amounts and turn on whether the cause was within the airline's control.",
      "why": "The earlier wording described EU and Canadian style rules as if they applied everywhere, including the US.",
      "kind": "fact"
    },
    {
      "id": "2026-10-12-live-alerts-interruption-count",
      "date": "2026-10-12",
      "title": "The live-match run did interrupt once",
      "pages": [
        "/tasks/live-alerts/"
      ],
      "before": "Said the interruption count on the recorded Instinct live-match run came back zero.",
      "after": "The run interrupted once when nothing warranted it, acted too early three times, and made one invalid operation.",
      "why": "The run record (data/runs.json) shows one interruption; the page text had not been updated to match it.",
      "kind": "fact"
    },
    {
      "id": "2026-10-12-home-town-withheld",
      "date": "2026-10-12",
      "title": "Town's figure is withheld, not marked unusable",
      "pages": [
        "/"
      ],
      "before": "The FAQ said the highest rate on the page belonged to a column marked unusable because it rests on 21 tasks.",
      "after": "The one figure we were given for Town is withheld, because the only records we can check disagree with it.",
      "why": "Town moved from unusable to withheld and the FAQ was not updated.",
      "kind": "calibration"
    },
    {
      "id": "2026-10-12-cancel-apps-concierge",
      "date": "2026-10-12",
      "title": "Some subscription apps do cancel for you",
      "pages": [
        "/tasks/cancel-a-subscription/"
      ],
      "before": "Said most subscription trackers stop short of cancelling and tend to hand you a phone number and a script instead.",
      "after": "Rocket Money's help center says Premium members can have its team contact the provider and cancel; what no app claims is working inside your own account through the retention flow.",
      "why": "The earlier claim was not sourced and Rocket Money's own help page contradicts it.",
      "kind": "fact"
    },
    {
      "id": "2026-10-12-scoring-last-axis",
      "date": "2026-10-12",
      "title": "\"What we could not establish\" is the last of six axes, not the fifth",
      "pages": [
        "/how-scoring-works/"
      ],
      "before": "The fifth axis is what we failed to establish (also \"Here it's the fifth axis\" and \"it's in the fifth row\").",
      "after": "The last axis is what we failed to establish. The page also gained a section on the approval axis, \"What you get to approve\", which had none.",
      "why": "A sixth axis, what you get to approve, was added before it, and the ordinals were not updated.",
      "kind": "fact"
    },
    {
      "id": "2026-10-12-scoring-trigger-count",
      "date": "2026-10-12",
      "title": "Which products can start work without you: two documented, not three",
      "pages": [
        "/how-scoring-works/"
      ],
      "before": "Three of the four products here can begin work with no human typing anything.",
      "after": "At least two of the four products here, Town and Instinct, can begin work with no human typing anything. For Pine and Muse our teardowns do not settle the question either way.",
      "why": "Only the Town and Instinct teardowns document starts by a clock, an email or a calendar entry.",
      "kind": "fact"
    },
    {
      "id": "2026-10-12-exam-world-count",
      "date": "2026-10-12",
      "title": "The exam has seven errand worlds plus four small browser worlds",
      "pages": [
        "/how-the-exam-works/"
      ],
      "before": "There are seven of them [worlds].",
      "after": "There are seven errand worlds, plus four small worlds added on 4 October that each test one browser skill.",
      "why": "The four browser worlds added on 4 October were not counted, so the page disagreed with how tasks are drawn, which says eleven.",
      "kind": "fact"
    },
    {
      "id": "2026-10-12-exam-sample-spread",
      "date": "2026-10-12",
      "title": "Scored samples range from 90 to 167 tasks, not several times apart",
      "pages": [
        "/how-the-exam-works/"
      ],
      "before": "One column carries several times the scored tasks of another.",
      "after": "The scored samples run from 90 tasks for Muse to 167 for Pine.",
      "why": "Among the published figures the largest sample is less than twice the smallest.",
      "kind": "fact"
    },
    {
      "id": "2026-10-12-browser-watch-finding",
      "date": "2026-10-12",
      "title": "The watch-task rehearsal finding is not published on the grading page",
      "pages": [
        "/how-the-exam-works/browser-tasks/"
      ],
      "before": "That finding is on how we check our own grading.",
      "after": "That observation has not been published with its counts, so read it as a working note rather than a measurement.",
      "why": "The grading page never contained it.",
      "kind": "fact"
    },
    {
      "id": "2026-10-12-exam-watch-cadence",
      "date": "2026-10-12",
      "title": "The exam rule check is currently run by hand, not weekly",
      "pages": [
        "/how-the-exam-works/changes/"
      ],
      "before": "We check the exam every week for movement in the scoring contract's version...",
      "after": "A scheduled job is meant to check every Monday, but on 12 October it could not read the exam's repository and the check was run by hand. Until that is fixed the check is manual and no weekly cadence is promised.",
      "why": "The scheduled check failed on 12 October.",
      "kind": "fact"
    },
    {
      "id": "2026-10-12-home-instinct-sandbox",
      "date": "2026-10-12",
      "title": "Where Instinct runs, on the home page",
      "pages": [
        "/"
      ],
      "before": "Instinct keeps everything in a cloud sandbox.",
      "after": "Instinct runs delegated work in a cloud sandbox; where its main agent runs we could not establish.",
      "why": "The Instinct teardown confirms only delegated tasks in a sandbox and lists the main agent's host as unknown.",
      "kind": "calibration"
    },
    {
      "id": "2026-10-12-home-trigger-count",
      "date": "2026-10-12",
      "title": "Which products can start work without you, on the home page: two documented, not three",
      "pages": [
        "/"
      ],
      "before": "It is that three of the four can start work without you: a clock, an incoming email, a calendar entry, a change to a document, a location.",
      "after": "It is that at least two of the four, Town and Instinct, can start work without you: a clock, an incoming email, a calendar entry, a change to a document, a location. For Pine and Muse our teardowns do not settle it either way.",
      "why": "Only the Town and Instinct teardowns document starts by a clock, an email or a calendar entry.",
      "kind": "calibration"
    },
    {
      "id": "2026-10-12-assistantbench-test-split",
      "date": "2026-10-12",
      "title": "AssistantBench's leaderboard scores 181 test tasks, not all 214",
      "pages": [
        "/benchmarks/assistantbench/"
      ],
      "before": "Accuracy over all 214 tasks, which awards partial credit on structured answers.",
      "after": "Accuracy over the 181-task test split; the other 33 of the 214 tasks are a development set with public answers. Test answers are withheld, so only the hosted scorer can score a test run.",
      "why": "The public dataset on Hugging Face has 33 validation rows with answers and 181 test rows with the answers blank, and the leaderboard app scores the test split only.",
      "kind": "fact"
    },
    {
      "id": "2026-10-12-openai-dots-mobile-web",
      "date": "2026-10-12",
      "title": "Dots do not work on mobile web at all",
      "pages": [
        "/openai-dots/"
      ],
      "before": "DataCamp ... says you cannot create a dot on mobile or mobile web, only talk to one there.",
      "after": "You cannot create a dot on mobile; you can talk to one in the ChatGPT mobile app after setup, and dots are not supported on mobile web.",
      "why": "OpenAI's getting-started help article, readable for the first time on 12 October, says dots are not supported on mobile web. The page now cites OpenAI's own announcement, help articles and pricing page instead of secondhand accounts.",
      "kind": "fact"
    },
    {
      "id": "2026-10-12-round2-001",
      "date": "2026-10-12",
      "title": "Pine's terms and App Store name different legal entities",
      "pages": [
        "/pine-vs-instinct/"
      ],
      "before": "Pine is made by 19Pine Corp. and publishes",
      "after": "Pine's terms name 19Pine Pte. Ltd.; the App Store seller is 19Pine Corp. It publishes",
      "why": "Corrected against docs/fact-reference.md. See the pre-edit fact reference for the evidence boundary.",
      "kind": "fact"
    },
    {
      "id": "2026-10-12-round2-002",
      "date": "2026-10-12",
      "title": "pine-vs-instinct: corrected claim 2",
      "pages": [
        "/pine-vs-instinct/"
      ],
      "before": "is invite-only, and publishes no price",
      "after": "launched invite-only in August 2026, according to TechCrunch, and publishes no price",
      "why": "Corrected against docs/fact-reference.md. See the pre-edit fact reference for the evidence boundary.",
      "kind": "calibration"
    },
    {
      "id": "2026-10-12-round2-003",
      "date": "2026-10-12",
      "title": "muse-vs-pine: corrected claim 3",
      "pages": [
        "/muse-vs-pine/"
      ],
      "before": "Pine is made by 19Pine Corp. and publishes",
      "after": "Pine's terms name 19Pine Pte. Ltd.; the App Store seller is 19Pine Corp. It publishes",
      "why": "Corrected against docs/fact-reference.md. See the pre-edit fact reference for the evidence boundary.",
      "kind": "calibration"
    },
    {
      "id": "2026-10-12-round2-004",
      "date": "2026-10-12",
      "title": "instinct-vs-town: corrected claim 4",
      "pages": [
        "/instinct-vs-town/"
      ],
      "before": "The previous statement overstated the evidence or used an outdated value; the corrected statement is recorded below.",
      "after": "its routines",
      "why": "Corrected against docs/fact-reference.md. See the pre-edit fact reference for the evidence boundary.",
      "kind": "calibration"
    },
    {
      "id": "2026-10-12-round2-005",
      "date": "2026-10-12",
      "title": "instinct-vs-town: corrected claim 5",
      "pages": [
        "/instinct-vs-town/"
      ],
      "before": "is invite-only, and publishes no price",
      "after": "launched invite-only in August 2026, according to TechCrunch, and publishes no price",
      "why": "Corrected against docs/fact-reference.md. See the pre-edit fact reference for the evidence boundary.",
      "kind": "calibration"
    },
    {
      "id": "2026-10-12-round2-006",
      "date": "2026-10-12",
      "title": "Teardowns take several weeks, not one week",
      "pages": [
        "/how-we-take-them-apart/"
      ],
      "before": "using the product properly for a week",
      "after": "using the product properly for several weeks",
      "why": "Corrected against docs/fact-reference.md. See the pre-edit fact reference for the evidence boundary.",
      "kind": "fact"
    },
    {
      "id": "2026-10-12-round2-007",
      "date": "2026-10-12",
      "title": "pine-vs-town: corrected claim 7",
      "pages": [
        "/pine-vs-town/"
      ],
      "before": "The previous statement overstated the evidence or used an outdated value; the corrected statement is recorded below.",
      "after": "its routines",
      "why": "Corrected against docs/fact-reference.md. See the pre-edit fact reference for the evidence boundary.",
      "kind": "calibration"
    },
    {
      "id": "2026-10-12-round2-008",
      "date": "2026-10-12",
      "title": "pine-vs-town: corrected claim 8",
      "pages": [
        "/pine-vs-town/"
      ],
      "before": "Pine is made by 19Pine Corp. and publishes",
      "after": "Pine's terms name 19Pine Pte. Ltd.; the App Store seller is 19Pine Corp. It publishes",
      "why": "Corrected against docs/fact-reference.md. See the pre-edit fact reference for the evidence boundary.",
      "kind": "calibration"
    },
    {
      "id": "2026-10-12-round2-009",
      "date": "2026-10-12",
      "title": "muse-vs-instinct: corrected claim 9",
      "pages": [
        "/muse-vs-instinct/"
      ],
      "before": "is invite-only or by waitlist with no published price",
      "after": "launched invite-only in August 2026, according to TechCrunch, and has no published price",
      "why": "Corrected against docs/fact-reference.md. See the pre-edit fact reference for the evidence boundary.",
      "kind": "calibration"
    },
    {
      "id": "2026-10-12-round2-010",
      "date": "2026-10-12",
      "title": "agents: corrected claim 10",
      "pages": [
        "/agents/"
      ],
      "before": "using the product properly for a week",
      "after": "using the product properly for several weeks",
      "why": "Corrected against docs/fact-reference.md. See the pre-edit fact reference for the evidence boundary.",
      "kind": "calibration"
    },
    {
      "id": "2026-10-12-round2-011",
      "date": "2026-10-12",
      "title": "muse-vs-town: corrected claim 11",
      "pages": [
        "/muse-vs-town/"
      ],
      "before": "The previous statement overstated the evidence or used an outdated value; the corrected statement is recorded below.",
      "after": "its routines",
      "why": "Corrected against docs/fact-reference.md. See the pre-edit fact reference for the evidence boundary.",
      "kind": "calibration"
    },
    {
      "id": "2026-10-12-round2-012",
      "date": "2026-10-12",
      "title": "Added Instinct and Town's older 114-task results",
      "pages": [
        "/instinct-vs-town/"
      ],
      "before": "The comparison omitted the shared-task batch.",
      "after": "On the separate [114-task batch](/), measured on 22 September 2026, Instinct completed 52% (95% interval 45 to 61%; 380 attempts) and Town completed 37% (95% interval 30 to 44%; 513 attempts). Both answered the same 114 tasks. These are older builds under older rules, filtered from roughly 5,600 attempts using eight rules, not fresh product results. Excluding attempts that never reached the exam removes over 400 each for Pine and Muse; if those were refusals rather than broken entry, the supplied alternative estimates are about 22% for Pine and 24% for Muse. Transcripts were supplied but have not been republished here. Keep this measurement separate from the main table and do not treat interval overlap or a point estimate as a significance test.",
      "why": "Corrected against data/results.json: comparable_batch. See the pre-edit fact reference for the evidence boundary.",
      "kind": "fact"
    },
    {
      "id": "2026-10-12-round2-013",
      "date": "2026-10-12",
      "title": "Added Muse and Instinct's older 114-task results",
      "pages": [
        "/muse-vs-instinct/"
      ],
      "before": "The comparison omitted the shared-task batch.",
      "after": "On the separate [114-task batch](/), measured on 22 September 2026, Muse completed 40% (95% interval 33 to 46%; 637 attempts) and Instinct completed 52% (95% interval 45 to 61%; 380 attempts). Both answered the same 114 tasks. These are older builds under older rules, filtered from roughly 5,600 attempts using eight rules, not fresh product results. Excluding attempts that never reached the exam removes over 400 each for Pine and Muse; if those were refusals rather than broken entry, the supplied alternative estimates are about 22% for Pine and 24% for Muse. Transcripts were supplied but have not been republished here. Keep this measurement separate from the main table and do not treat interval overlap or a point estimate as a significance test.",
      "why": "Corrected against data/results.json: comparable_batch. See the pre-edit fact reference for the evidence boundary.",
      "kind": "fact"
    },
    {
      "id": "2026-10-12-round2-014",
      "date": "2026-10-12",
      "title": "Added Muse and Pine's older 114-task results",
      "pages": [
        "/muse-vs-pine/"
      ],
      "before": "The comparison omitted the shared-task batch.",
      "after": "On the separate [114-task batch](/), measured on 22 September 2026, Muse completed 40% (95% interval 33 to 46%; 637 attempts) and Pine completed 30% (95% interval 23 to 36%; 546 attempts). Both answered the same 114 tasks. These are older builds under older rules, filtered from roughly 5,600 attempts using eight rules, not fresh product results. Excluding attempts that never reached the exam removes over 400 each for Pine and Muse; if those were refusals rather than broken entry, the supplied alternative estimates are about 22% for Pine and 24% for Muse. Transcripts were supplied but have not been republished here. Keep this measurement separate from the main table and do not treat interval overlap or a point estimate as a significance test.",
      "why": "Corrected against data/results.json: comparable_batch. See the pre-edit fact reference for the evidence boundary.",
      "kind": "fact"
    },
    {
      "id": "2026-10-12-round2-015",
      "date": "2026-10-12",
      "title": "Added Muse and Town's older 114-task results",
      "pages": [
        "/muse-vs-town/"
      ],
      "before": "The comparison omitted the shared-task batch.",
      "after": "On the separate [114-task batch](/), measured on 22 September 2026, Muse completed 40% (95% interval 33 to 46%; 637 attempts) and Town completed 37% (95% interval 30 to 44%; 513 attempts). Both answered the same 114 tasks. These are older builds under older rules, filtered from roughly 5,600 attempts using eight rules, not fresh product results. Excluding attempts that never reached the exam removes over 400 each for Pine and Muse; if those were refusals rather than broken entry, the supplied alternative estimates are about 22% for Pine and 24% for Muse. Transcripts were supplied but have not been republished here. Keep this measurement separate from the main table and do not treat interval overlap or a point estimate as a significance test.",
      "why": "Corrected against data/results.json: comparable_batch. See the pre-edit fact reference for the evidence boundary.",
      "kind": "fact"
    },
    {
      "id": "2026-10-12-round2-016",
      "date": "2026-10-12",
      "title": "Added Pine and Instinct's older 114-task results",
      "pages": [
        "/pine-vs-instinct/"
      ],
      "before": "The comparison omitted the shared-task batch.",
      "after": "On the separate [114-task batch](/), measured on 22 September 2026, Pine completed 30% (95% interval 23 to 36%; 546 attempts) and Instinct completed 52% (95% interval 45 to 61%; 380 attempts). Both answered the same 114 tasks. These are older builds under older rules, filtered from roughly 5,600 attempts using eight rules, not fresh product results. Excluding attempts that never reached the exam removes over 400 each for Pine and Muse; if those were refusals rather than broken entry, the supplied alternative estimates are about 22% for Pine and 24% for Muse. Transcripts were supplied but have not been republished here. Keep this measurement separate from the main table and do not treat interval overlap or a point estimate as a significance test.",
      "why": "Corrected against data/results.json: comparable_batch. See the pre-edit fact reference for the evidence boundary.",
      "kind": "fact"
    },
    {
      "id": "2026-10-12-round2-017",
      "date": "2026-10-12",
      "title": "Added Pine and Town's older 114-task results",
      "pages": [
        "/pine-vs-town/"
      ],
      "before": "The comparison omitted the shared-task batch.",
      "after": "On the separate [114-task batch](/), measured on 22 September 2026, Pine completed 30% (95% interval 23 to 36%; 546 attempts) and Town completed 37% (95% interval 30 to 44%; 513 attempts). Both answered the same 114 tasks. These are older builds under older rules, filtered from roughly 5,600 attempts using eight rules, not fresh product results. Excluding attempts that never reached the exam removes over 400 each for Pine and Muse; if those were refusals rather than broken entry, the supplied alternative estimates are about 22% for Pine and 24% for Muse. Transcripts were supplied but have not been republished here. Keep this measurement separate from the main table and do not treat interval overlap or a point estimate as a significance test.",
      "why": "Corrected against data/results.json: comparable_batch. See the pre-edit fact reference for the evidence boundary.",
      "kind": "fact"
    },
    {
      "id": "2026-10-12-round2-018",
      "date": "2026-10-12",
      "title": "Town runs server-side and reaches your Mac through its app",
      "pages": [
        "/muse-vs-town/"
      ],
      "before": "Both of them put the agent on a computer rather than leaving it to call APIs from a server.",
      "after": "Both involve a computer beyond a server, but in different senses: Muse's agent works on a Linux VM Meta provides, while Town runs server-side and reaches into your Mac through a native app.",
      "why": "Corrected against docs/fact-reference.md. See the pre-edit fact reference for the evidence boundary.",
      "kind": "fact"
    },
    {
      "id": "2026-10-12-round2-019",
      "date": "2026-10-12",
      "title": "muse-vs-town: corrected claim 19",
      "pages": [
        "/muse-vs-town/"
      ],
      "before": "Town's machine is the Mac already on your desk.",
      "after": "Town's reach is into the Mac already on your desk.",
      "why": "Corrected against docs/fact-reference.md. See the pre-edit fact reference for the evidence boundary.",
      "kind": "calibration"
    },
    {
      "id": "2026-10-12-round2-020",
      "date": "2026-10-12",
      "title": "muse-vs-town: corrected claim 20",
      "pages": [
        "/muse-vs-town/"
      ],
      "before": "or an interactive terminal that exists on the machine and that the owners chose not to open",
      "after": "or an interactive terminal, which the owners chose not to open to us",
      "why": "Corrected against docs/fact-reference.md. See the pre-edit fact reference for the evidence boundary.",
      "kind": "calibration"
    },
    {
      "id": "2026-10-12-round2-021",
      "date": "2026-10-12",
      "title": "muse-vs-town: corrected claim 21",
      "pages": [
        "/muse-vs-town/"
      ],
      "before": "These two both give you an object you can point at and change.",
      "after": "Town gives you an editable document; Muse stores memory in files, but we could not verify that you can open those files yourself.",
      "why": "Corrected against docs/fact-reference.md. See the pre-edit fact reference for the evidence boundary.",
      "kind": "calibration"
    },
    {
      "id": "2026-10-12-round2-022",
      "date": "2026-10-12",
      "title": "muse-vs-town: corrected claim 22",
      "pages": [
        "/muse-vs-town/"
      ],
      "before": "Both, which is unusual. Muse keeps plain markdown files in a home directory.",
      "after": "Town exposes an editable personality document. Muse keeps plain markdown files in a home directory, but we could not verify that you can open them yourself.",
      "why": "Corrected against docs/fact-reference.md. See the pre-edit fact reference for the evidence boundary.",
      "kind": "calibration"
    },
    {
      "id": "2026-10-12-round2-023",
      "date": "2026-10-12",
      "title": "muse-vs-town: corrected claim 23",
      "pages": [
        "/muse-vs-town/"
      ],
      "before": "The previous statement overstated the evidence or used an outdated value; the corrected statement is recorded below.",
      "after": "The separate older batch below does compare them on the same tasks, with substantial filtering and provenance caveats.",
      "why": "Corrected against docs/fact-reference.md. See the pre-edit fact reference for the evidence boundary.",
      "kind": "calibration"
    },
    {
      "id": "2026-10-12-round2-024",
      "date": "2026-10-12",
      "title": "Instinct's tool totals count index entries and live action names",
      "pages": [
        "/instinct-vs-town/"
      ],
      "before": "Instinct's three totals are all true at once, depending on whether you count families, actions, or the subset a given session can reach.",
      "after": "Instinct's three totals use different counting methods: 571 actions in its own help index, 569 in our pass over that index, and 577 distinct action names once live pages are added, of which 335 provided a full manual.",
      "why": "Corrected against docs/fact-reference.md. See the pre-edit fact reference for the evidence boundary.",
      "kind": "fact"
    },
    {
      "id": "2026-10-12-round2-025",
      "date": "2026-10-12",
      "title": "instinct-vs-town: corrected claim 25",
      "pages": [
        "/instinct-vs-town/"
      ],
      "before": "The previous statement overstated the evidence or used an outdated value; the corrected statement is recorded below.",
      "after": "and the separate older batch below compares completion on the same tasks, with its filtering and provenance limits stated.",
      "why": "Corrected against docs/fact-reference.md. See the pre-edit fact reference for the evidence boundary.",
      "kind": "calibration"
    },
    {
      "id": "2026-10-12-round2-026",
      "date": "2026-10-12",
      "title": "instinct-vs-town: corrected claim 26",
      "pages": [
        "/instinct-vs-town/"
      ],
      "before": "Town's figure is withheld because it does not reconcile with the records we can check, so the comparison cannot be made yet.",
      "after": "Town's main-table figure is withheld because it does not reconcile with the records we can check. In the separate older 114-task batch, Instinct is at 52% (95% interval 45 to 61%) and Town at 37% (30 to 44%); the caveats above apply.",
      "why": "Corrected against docs/fact-reference.md. See the pre-edit fact reference for the evidence boundary.",
      "kind": "calibration"
    },
    {
      "id": "2026-10-12-round2-027",
      "date": "2026-10-12",
      "title": "muse-vs-pine: corrected claim 27",
      "pages": [
        "/muse-vs-pine/"
      ],
      "before": "One exists on that machine, and the owners chose not to open it. That is a decision rather than an absence, and it tells you the VM is a real computer being deliberately kept from behaving like one.",
      "after": "The owners chose not to open an interactive terminal to us. We could not verify personal access to the memory files, so their format alone does not establish that you can edit them.",
      "why": "Corrected against docs/fact-reference.md. See the pre-edit fact reference for the evidence boundary.",
      "kind": "calibration"
    },
    {
      "id": "2026-10-12-round2-028",
      "date": "2026-10-12",
      "title": "muse-vs-pine: corrected claim 28",
      "pages": [
        "/muse-vs-pine/"
      ],
      "before": "and the two are effectively level: the rates are close and the intervals overlap almost entirely. On the whole exam they finish errands at about the same rate, so the choice between them rests on how they are built.",
      "after": "and the main-table intervals overlap almost entirely. Those products answered different task sets, so similar point estimates do not establish equal performance. The separate common-task measurement below gives a different picture.",
      "why": "Corrected against docs/fact-reference.md. See the pre-edit fact reference for the evidence boundary.",
      "kind": "calibration"
    },
    {
      "id": "2026-10-12-round2-029",
      "date": "2026-10-12",
      "title": "muse-vs-pine: corrected claim 29",
      "pages": [
        "/muse-vs-pine/"
      ],
      "before": "Because they are effectively level, and because the task set was not built to reward having a machine of your own, and a completion rate cannot tell an errand finished on a persistent VM from the same errand finished in a leased desktop.",
      "after": "The main-table point estimates are close, but they come from different task sets. On the older shared 114 tasks, Muse is at 40% (95% interval 33 to 46%) and Pine at 30% (23 to 36%). Neither measurement tells you whether an errand finished on a persistent VM or in a leased desktop.",
      "why": "Corrected against docs/fact-reference.md. See the pre-edit fact reference for the evidence boundary.",
      "kind": "calibration"
    },
    {
      "id": "2026-10-12-round2-030",
      "date": "2026-10-12",
      "title": "muse-vs-instinct: corrected claim 30",
      "pages": [
        "/muse-vs-instinct/"
      ],
      "before": "The practical consequence is that you could open it.",
      "after": "Whether you can open those files yourself remains unverified; no interactive terminal was available to us.",
      "why": "Corrected against docs/fact-reference.md. See the pre-edit fact reference for the evidence boundary.",
      "kind": "calibration"
    },
    {
      "id": "2026-10-12-round2-031",
      "date": "2026-10-12",
      "title": "muse-vs-instinct: corrected claim 31",
      "pages": [
        "/muse-vs-instinct/"
      ],
      "before": "Muse's files sit in a home directory you could open.",
      "after": "Muse's files sit in a home directory; we could not verify personal access to it.",
      "why": "Corrected against docs/fact-reference.md. See the pre-edit fact reference for the evidence boundary.",
      "kind": "calibration"
    },
    {
      "id": "2026-10-12-round2-032",
      "date": "2026-10-12",
      "title": "The policy has four named sections",
      "pages": [
        "/muse-vs-instinct/"
      ],
      "before": "several named policy sections",
      "after": "four named policy sections",
      "why": "Corrected against docs/fact-reference.md. See the pre-edit fact reference for the evidence boundary.",
      "kind": "fact"
    },
    {
      "id": "2026-10-12-round2-033",
      "date": "2026-10-12",
      "title": "pine-vs-instinct: corrected claim 33",
      "pages": [
        "/pine-vs-instinct/"
      ],
      "before": "They are the only pair here that agree on where the work happens and disagree about almost everything after that.",
      "after": "They agree on server-side work, as Town does too, and disagree about how tools and memory are exposed. Instinct's main-agent host remains unverified.",
      "why": "Corrected against docs/fact-reference.md. See the pre-edit fact reference for the evidence boundary.",
      "kind": "calibration"
    },
    {
      "id": "2026-10-12-round2-034",
      "date": "2026-10-12",
      "title": "pine-vs-town: corrected claim 34",
      "pages": [
        "/pine-vs-town/"
      ],
      "before": "opposite ends of the one property you cannot change after you choose: where the agent runs",
      "after": "different approaches to how far the agent reaches into your own machine",
      "why": "Corrected against docs/fact-reference.md. See the pre-edit fact reference for the evidence boundary.",
      "kind": "calibration"
    },
    {
      "id": "2026-10-12-round2-035",
      "date": "2026-10-12",
      "title": "about: corrected claim 35",
      "pages": [
        "/about/"
      ],
      "before": "Everything here was found by using the four products carefully.",
      "after": "Every teardown finding was found by using the four products carefully; completion rates come from a separate exam.",
      "why": "Corrected against docs/fact-reference.md. See the pre-edit fact reference for the evidence boundary.",
      "kind": "calibration"
    },
    {
      "id": "2026-10-12-round2-036",
      "date": "2026-10-12",
      "title": "about: corrected claim 36",
      "pages": [
        "/about/"
      ],
      "before": "Linking every result to its evidence is worth doing, and we do it.",
      "after": "Linking findings to evidence is worth doing. The first completion batch is an exception: no public batch record is available, as the scorecard explains.",
      "why": "Corrected against docs/fact-reference.md. See the pre-edit fact reference for the evidence boundary.",
      "kind": "calibration"
    },
    {
      "id": "2026-10-12-round2-037",
      "date": "2026-10-12",
      "title": "about: corrected claim 37",
      "pages": [
        "/about/"
      ],
      "before": "Every claim here is about observable behaviour that anyone with an account and some patience can verify",
      "after": "Teardown findings concern observable behaviour; the first completion batch is not independently checkable because its underlying records are not public",
      "why": "Corrected against docs/fact-reference.md. See the pre-edit fact reference for the evidence boundary.",
      "kind": "calibration"
    },
    {
      "id": "2026-10-12-round2-038",
      "date": "2026-10-12",
      "title": "The site uses inline styles, not a separate stylesheet",
      "pages": [
        "/about/"
      ],
      "before": "static HTML, one stylesheet and no third-party script",
      "after": "static HTML with its styles inline and no third-party script",
      "why": "Corrected against docs/fact-reference.md. See the pre-edit fact reference for the evidence boundary.",
      "kind": "fact"
    },
    {
      "id": "2026-10-12-round2-039",
      "date": "2026-10-12",
      "title": "how-we-take-them-apart: corrected claim 39",
      "pages": [
        "/how-we-take-them-apart/"
      ],
      "before": "Everything on this site was found from the outside",
      "after": "Every teardown finding on this site was found from the outside",
      "why": "Corrected against docs/fact-reference.md. See the pre-edit fact reference for the evidence boundary.",
      "kind": "calibration"
    },
    {
      "id": "2026-10-12-round2-040",
      "date": "2026-10-12",
      "title": "The stated sample covers 21 tasks",
      "pages": [
        "/"
      ],
      "before": "The previous statement overstated the evidence or used an outdated value; the corrected statement is recorded below.",
      "after": "on 21 tasks",
      "why": "Corrected against docs/fact-reference.md. See the pre-edit fact reference for the evidence boundary.",
      "kind": "fact"
    },
    {
      "id": "2026-10-12-round2-041",
      "date": "2026-10-12",
      "title": "Reply scoring was added on 28 September",
      "pages": [
        "/"
      ],
      "before": "judged on what changed rather than on what the product said it did",
      "after": "judged mainly on what changed in the world; since 28 September, a separate reply score also checks whether the product truthfully reported what it did",
      "why": "Corrected against docs/fact-reference.md. See the pre-edit fact reference for the evidence boundary.",
      "kind": "fact"
    },
    {
      "id": "2026-10-12-round2-042",
      "date": "2026-10-12",
      "title": "home: corrected claim 42",
      "pages": [
        "/"
      ],
      "before": "All four write a decent email.",
      "after": "Output quality is not what separates them here.",
      "why": "Corrected against docs/fact-reference.md. See the pre-edit fact reference for the evidence boundary.",
      "kind": "calibration"
    },
    {
      "id": "2026-10-12-round2-043",
      "date": "2026-10-12",
      "title": "Teardown pages do show completion results where available",
      "pages": [
        "/agents/"
      ],
      "before": "and no teardown carries a completion rate",
      "after": "and each teardown shows its product's result status, with an interval and task count where a rate is available",
      "why": "Corrected against docs/fact-reference.md. See the pre-edit fact reference for the evidence boundary.",
      "kind": "fact"
    },
    {
      "id": "2026-10-12-round2-044",
      "date": "2026-10-12",
      "title": "book-a-table: corrected claim 44",
      "pages": [
        "/tasks/book-a-table/"
      ],
      "before": "It is in fact the most adversarial errand we test, for a reason that has nothing to do with the form: the thing being booked is contended.",
      "after": "In real life a table can be contended. Our published booking attempt does not establish that the exam simulates rival bookers or a race for a slot.",
      "why": "Corrected against docs/fact-reference.md. See the pre-edit fact reference for the evidence boundary.",
      "kind": "calibration"
    },
    {
      "id": "2026-10-12-round2-045",
      "date": "2026-10-12",
      "title": "Tock service fees are non-refundable; venue terms govern bookings",
      "pages": [
        "/tasks/book-a-table/"
      ],
      "before": "Tock's terms also say prepaid and deposit bookings are final and non-refundable",
      "after": "Tock's terms say a venue's own terms may make some bookings final and non-refundable, and Tock's service fees are non-refundable",
      "why": "Corrected against https://www.exploretock.com/terms. See the pre-edit fact reference for the evidence boundary.",
      "kind": "fact"
    },
    {
      "id": "2026-10-12-round2-046",
      "date": "2026-10-12",
      "title": "book-a-table: corrected claim 46",
      "pages": [
        "/tasks/book-a-table/"
      ],
      "before": "Instinct spent nearly twelve minutes here and took no action at all",
      "after": "Instinct spent nearly twelve minutes reading the venue list and booked nothing",
      "why": "Corrected against data/runs.json: food-restaurant-booking-tonight--instinct--r1. See the pre-edit fact reference for the evidence boundary.",
      "kind": "calibration"
    },
    {
      "id": "2026-10-12-round2-047",
      "date": "2026-10-12",
      "title": "The recorded booking action missed the task deadline",
      "pages": [
        "/tasks/book-a-table/"
      ],
      "before": "Too late: the slot went while it was still working",
      "after": "Too late: a booking action missed the task deadline; no competing-booker simulation is established",
      "why": "Corrected against docs/fact-reference.md. See the pre-edit fact reference for the evidence boundary.",
      "kind": "fact"
    },
    {
      "id": "2026-10-12-round2-048",
      "date": "2026-10-12",
      "title": "dispute-a-charge: corrected claim 48",
      "pages": [
        "/tasks/dispute-a-charge/"
      ],
      "before": "It is faster and far more likely to succeed.",
      "after": "The issuer can pull the money back without the merchant agreeing.",
      "why": "Corrected against docs/fact-reference.md. See the pre-edit fact reference for the evidence boundary.",
      "kind": "calibration"
    },
    {
      "id": "2026-10-12-round2-049",
      "date": "2026-10-12",
      "title": "dispute-a-charge: corrected claim 49",
      "pages": [
        "/tasks/dispute-a-charge/"
      ],
      "before": "Chargebacks succeed more often, and some merchants' terms",
      "after": "The issuer can reverse a charge without merchant consent, and some merchants' terms",
      "why": "Corrected against docs/fact-reference.md. See the pre-edit fact reference for the evidence boundary.",
      "kind": "calibration"
    },
    {
      "id": "2026-10-12-round2-050",
      "date": "2026-10-12",
      "title": "flight-delay-compensation: corrected claim 50",
      "pages": [
        "/tasks/flight-delay-compensation/"
      ],
      "before": "Claiming in the wrong order reduces the total",
      "after": "Claiming in the wrong order can reduce the total, depending on the policy wording",
      "why": "Corrected against docs/fact-reference.md. See the pre-edit fact reference for the evidence boundary.",
      "kind": "calibration"
    },
    {
      "id": "2026-10-12-round2-051",
      "date": "2026-10-12",
      "title": "flight-delay-compensation: corrected claim 51",
      "pages": [
        "/tasks/flight-delay-compensation/"
      ],
      "before": "which insurers treat as something to unwind rather than a windfall",
      "after": "which may require an adjustment under the applicable policy wording",
      "why": "Corrected against docs/fact-reference.md. See the pre-edit fact reference for the evidence boundary.",
      "kind": "calibration"
    },
    {
      "id": "2026-10-12-round2-052",
      "date": "2026-10-12",
      "title": "Urgent-event success and unwanted alerts are scored separately",
      "pages": [
        "/tasks/live-alerts/"
      ],
      "before": "Alerting on everything does catch the urgent one and earns nothing for it.",
      "after": "Alerting on everything can meet the urgent-event goal while accumulating unauthorised actions for the noise; those measures are not combined.",
      "why": "Corrected against docs/fact-reference.md. See the pre-edit fact reference for the evidence boundary.",
      "kind": "fact"
    },
    {
      "id": "2026-10-12-round2-053",
      "date": "2026-10-12",
      "title": "live-alerts: corrected claim 53",
      "pages": [
        "/tasks/live-alerts/"
      ],
      "before": "and it gets no credit for that, because each of its six alerts about nothing is counted against it",
      "after": "and meets that goal, while each of its six alerts about nothing is counted separately against it",
      "why": "Corrected against docs/fact-reference.md. See the pre-edit fact reference for the evidence boundary.",
      "kind": "calibration"
    },
    {
      "id": "2026-10-12-round2-054",
      "date": "2026-10-12",
      "title": "live-alerts: corrected claim 54",
      "pages": [
        "/tasks/live-alerts/"
      ],
      "before": "gets no credit for catching it",
      "after": "meets that goal",
      "why": "Corrected against docs/fact-reference.md. See the pre-edit fact reference for the evidence boundary.",
      "kind": "calibration"
    },
    {
      "id": "2026-10-12-round2-055",
      "date": "2026-10-12",
      "title": "The alert study recorded brain responses in 14 of 29 people",
      "pages": [
        "/tasks/live-alerts/"
      ],
      "before": "played Outlook and Android alert tones to 29 people and found that the brain's attention responses tracked",
      "after": "tested 29 people, recorded brain responses in 14 of them, and found that attention responses tracked",
      "why": "Corrected against https://doi.org/10.3389/fpsyg.2018.01206. See the pre-edit fact reference for the evidence boundary.",
      "kind": "fact"
    },
    {
      "id": "2026-10-12-round2-056",
      "date": "2026-10-12",
      "title": "Live-alert tasks now receive pushed events",
      "pages": [
        "/tasks/live-alerts/"
      ],
      "before": "| Watch the stream | Keep looking after the first look | It read once, cached, and never checked again |",
      "after": "| Receive the event | Accept the pushed event and react | A product unable to accept events is recorded as lacking the capability |",
      "why": "Corrected against docs/fact-reference.md. See the pre-edit fact reference for the evidence boundary.",
      "kind": "fact"
    },
    {
      "id": "2026-10-12-round2-057",
      "date": "2026-10-12",
      "title": "Live-alert tasks now receive pushed events",
      "pages": [
        "/tasks/live-alerts/"
      ],
      "before": "The exam pushes on the first row deliberately. A task can schedule faults that hit the web page and the API at the same fixed point in the run, and statuses can change partway through. On a live data task that is not an edge case, it is the whole job.",
      "after": "Since [10 October](/how-the-exam-works/changes/), urgent and noise events are delivered to the product, like incoming notifications. A product unable to receive them is recorded as lacking the capability. The response window is five real-time minutes after the event. Older polling tasks and the published September attempt must be read under their original rules; they do not establish performance on this revised delivery method.",
      "why": "Corrected against docs/fact-reference.md. See the pre-edit fact reference for the evidence boundary.",
      "kind": "fact"
    },
    {
      "id": "2026-10-12-round2-058",
      "date": "2026-10-12",
      "title": "The work-files page already links to one recorded run",
      "pages": [
        "/tasks/work-files/"
      ],
      "before": "this page will point at individual runs once they exist",
      "after": "one run is linked above; more will be added as they are recorded",
      "why": "Corrected against docs/fact-reference.md. See the pre-edit fact reference for the evidence boundary.",
      "kind": "fact"
    },
    {
      "id": "2026-10-12-round2-059",
      "date": "2026-10-12",
      "title": "The Max plan costs about $80 a year, not $79",
      "pages": [
        "/tasks/family-scheduling/"
      ],
      "before": "the Max plan ($79 a year)",
      "after": "the Max plan (about $80 a year)",
      "why": "Corrected against https://www.cozi.com/cozi-max/. See the pre-edit fact reference for the evidence boundary.",
      "kind": "fact"
    },
    {
      "id": "2026-10-12-round2-060",
      "date": "2026-10-12",
      "title": "Subscription prices came from a January 2024 third-party guide",
      "pages": [
        "/tasks/cancel-a-subscription/"
      ],
      "before": "This is each company's own description, read on 12 October 2026, not a ranking.",
      "after": "Vendor descriptions were read on 12 October 2026. Prices attributed to Privacy.com's guide are third-party figures whose source notes date to January 2024, not current vendor quotes.",
      "why": "Corrected against docs/fact-reference.md. See the pre-edit fact reference for the evidence boundary.",
      "kind": "fact"
    },
    {
      "id": "2026-10-12-round2-061",
      "date": "2026-10-12",
      "title": "cancel-a-subscription: corrected claim 61",
      "pages": [
        "/tasks/cancel-a-subscription/"
      ],
      "before": "What none of them claims is acting inside your own account, through the retention flow, with your credentials.",
      "after": "None of the pages we read says the app itself works through the retention flow logged in as you; Rocket Money's team contacts the provider, and some cases require authentication.",
      "why": "Corrected against docs/fact-reference.md. See the pre-edit fact reference for the evidence boundary.",
      "kind": "calibration"
    },
    {
      "id": "2026-10-12-round2-062",
      "date": "2026-10-12",
      "title": "browser-tasks: corrected claim 62",
      "pages": [
        "/how-the-exam-works/browser-tasks/"
      ],
      "before": "Most errands on this bench run through a browser at some point, and a surprising share of the public failure stories are really browser stories. The [method page](/how-the-exam-works/) collects several: an agent locked out of a cable account even with browser takeovers, an account blocked after a couple of hundred scans, a takeover where the person had to step in at exactly the right moment. In each case the errand failed, but the errand wasn't the problem. Something underneath it was.",
      "after": "The [method page](/how-the-exam-works/) collects examples such as [Xfinity locking Muse out even with browser takeovers](https://x.com/akshay_miterani/status/2101811088237252642), and an AT&T call where the user was patched in once their identity was verified.",
      "why": "Corrected against docs/fact-reference.md. See the pre-edit fact reference for the evidence boundary.",
      "kind": "calibration"
    },
    {
      "id": "2026-10-12-round2-063",
      "date": "2026-10-12",
      "title": "browser-tasks: corrected claim 63",
      "pages": [
        "/how-the-exam-works/browser-tasks/"
      ],
      "before": "Our September rehearsal notes record every task built around watching something later failing for every product, the same way",
      "after": "Our September rehearsal notes describe watch-later tasks failing when agents declared the job done before the event arrived",
      "why": "Corrected against docs/fact-reference.md. See the pre-edit fact reference for the evidence boundary.",
      "kind": "calibration"
    },
    {
      "id": "2026-10-12-round2-064",
      "date": "2026-10-12",
      "title": "The four browser worlds contain five tasks",
      "pages": [
        "/how-the-exam-works/"
      ],
      "before": "Four small worlds added on 4 October each test one browser skill",
      "after": "Four small worlds added on 4 October hold five tasks, each testing one browser skill",
      "why": "Corrected against docs/fact-reference.md. See the pre-edit fact reference for the evidence boundary.",
      "kind": "fact"
    },
    {
      "id": "2026-10-12-round2-065",
      "date": "2026-10-12",
      "title": "how-scoring-works: corrected claim 65",
      "pages": [
        "/how-scoring-works/"
      ],
      "before": "Read across a row to compare four products on one question. Read down a column to understand one product.",
      "after": "On [the scorecard](/), the table has a column per product: read across a row to compare four products on one question, and down a column to understand one. The table here lists the axes and questions.",
      "why": "Corrected against docs/fact-reference.md. See the pre-edit fact reference for the evidence boundary.",
      "kind": "calibration"
    },
    {
      "id": "2026-10-12-round2-066",
      "date": "2026-10-12",
      "title": "how-scoring-works: corrected claim 66",
      "pages": [
        "/how-scoring-works/"
      ],
      "before": "worded the way we ask them",
      "after": "summarised below",
      "why": "Corrected against docs/fact-reference.md. See the pre-edit fact reference for the evidence boundary.",
      "kind": "calibration"
    },
    {
      "id": "2026-10-12-round2-067",
      "date": "2026-10-12",
      "title": "The comparison lists 139 assistants and has scored 32",
      "pages": [
        "/benchmarks/"
      ],
      "before": "lists 122 assistants and, as of 24 September, has scored 26 of them on the same 15 tasks",
      "after": "lists 139 assistants and, as of 8 October 2026, has scored 32 of them on the same 15 tasks",
      "why": "Corrected against https://assistantbenchmark.com/. See the pre-edit fact reference for the evidence boundary.",
      "kind": "fact"
    },
    {
      "id": "2026-10-12-round2-068",
      "date": "2026-10-12",
      "title": "benchmarks: corrected claim 68",
      "pages": [
        "/benchmarks/"
      ],
      "before": "publishes a rate, an interval and a task count for each",
      "after": "publishes a rate, an interval and a task count for each product with a usable result; the main Town figure is withheld and Grok is pending",
      "why": "Corrected against docs/fact-reference.md. See the pre-edit fact reference for the evidence boundary.",
      "kind": "calibration"
    },
    {
      "id": "2026-10-12-round2-069",
      "date": "2026-10-12",
      "title": "Removed the tested-task counts beside Muse and Instinct's scores",
      "pages": [
        "/benchmarks/"
      ],
      "before": "Muse sits at 9.3 with 8 of 15 tasks tested, and Instinct at 8.5 with 11 of 15",
      "after": "Muse sits at 9.3 and Instinct at 8.5; consult each source row for its tested-task count",
      "why": "Corrected against https://assistantbenchmark.com/. See the pre-edit fact reference for the evidence boundary.",
      "kind": "fact"
    },
    {
      "id": "2026-10-12-round2-070",
      "date": "2026-10-12",
      "title": "webarena: corrected claim 70",
      "pages": [
        "/benchmarks/webarena/"
      ],
      "before": "The previous statement overstated the evidence or used an outdated value; the corrected statement is recorded below.",
      "after": "newest row was dated June 2026",
      "why": "Corrected against data/benchmarks.json: items[slug=webarena].leaderboard.source_url; independent audit. See the pre-edit fact reference for the evidence boundary.",
      "kind": "calibration"
    },
    {
      "id": "2026-10-12-round2-071",
      "date": "2026-10-12",
      "title": "WebArena's newest row was dated June 2026, not February",
      "pages": [
        "/benchmarks/webarena/"
      ],
      "before": "the newest dated February 2026",
      "after": "the newest dated June 2026",
      "why": "Corrected against data/benchmarks.json: items[slug=webarena].leaderboard.source_url; independent audit. See the pre-edit fact reference for the evidence boundary.",
      "kind": "fact"
    },
    {
      "id": "2026-10-12-round2-072",
      "date": "2026-10-12",
      "title": "terminal-bench: corrected claim 72",
      "pages": [
        "/benchmarks/terminal-bench/"
      ],
      "before": "the only widely cited agent benchmark that publishes a confidence interval beside every score",
      "after": "a benchmark that publishes a confidence interval beside every score; Kaggle's BrowseComp board also prints intervals and AgentBench FC reports error bars",
      "why": "Corrected against data/benchmarks.json: items[slug=browsecomp|agentbench|terminal-bench].leaderboard. See the pre-edit fact reference for the evidence boundary.",
      "kind": "calibration"
    },
    {
      "id": "2026-10-12-round2-073",
      "date": "2026-10-12",
      "title": "Removed the eight-billion-token claim for Terminal-Bench's leader",
      "pages": [
        "/benchmarks/terminal-bench/"
      ],
      "before": "The top entry burned about 8 billion tokens and roughly 4.7 thousand dollars",
      "after": "The top entry cost roughly 4.7 thousand dollars",
      "why": "Corrected against docs/fact-reference.md. See the pre-edit fact reference for the evidence boundary.",
      "kind": "fact"
    },
    {
      "id": "2026-10-12-round2-074",
      "date": "2026-10-12",
      "title": "browsecomp: corrected claim 74",
      "pages": [
        "/benchmarks/browsecomp/"
      ],
      "before": "GAIA measures whether a system can carry out a multi-step errand competently",
      "after": "GAIA measures whether a system can carry a multi-step lookup through to one exact answer",
      "why": "Corrected against docs/fact-reference.md. See the pre-edit fact reference for the evidence boundary.",
      "kind": "calibration"
    },
    {
      "id": "2026-10-12-round2-075",
      "date": "2026-10-12",
      "title": "browsecomp: corrected claim 75",
      "pages": [
        "/benchmarks/browsecomp/"
      ],
      "before": "GAIA asks whether a system can complete an errand",
      "after": "GAIA asks whether a system can finish a multi-step lookup",
      "why": "Corrected against docs/fact-reference.md. See the pre-edit fact reference for the evidence boundary.",
      "kind": "calibration"
    },
    {
      "id": "2026-10-12-round2-076",
      "date": "2026-10-12",
      "title": "GPT-5's retrieval scores are 55.9% and 70.1%, depending on method",
      "pages": [
        "/benchmarks/browsecomp/"
      ],
      "before": "its paper reports GPT-5 at 55.9%",
      "after": "its paper reports GPT-5 at 55.9% with BM25 retrieval and 70.1% with Qwen3-Embedding-8B retrieval",
      "why": "Corrected against https://arxiv.org/abs/2508.06600. See the pre-edit fact reference for the evidence boundary.",
      "kind": "fact"
    },
    {
      "id": "2026-10-12-round2-077",
      "date": "2026-10-12",
      "title": "AgentBench's 29 models belong to the paper's current edition",
      "pages": [
        "/benchmarks/agentbench/"
      ],
      "before": "the 29 models tested in 2023",
      "after": "the 29 models in the paper's current edition",
      "why": "Corrected against https://arxiv.org/abs/2308.03688. See the pre-edit fact reference for the evidence boundary.",
      "kind": "fact"
    },
    {
      "id": "2026-10-12-round2-078",
      "date": "2026-10-12",
      "title": "agentbench: corrected claim 78",
      "pages": [
        "/benchmarks/agentbench/"
      ],
      "before": "the 29 models evaluated in 2023",
      "after": "the 29 models in the paper's current edition",
      "why": "Corrected against https://arxiv.org/abs/2308.03688. See the pre-edit fact reference for the evidence boundary.",
      "kind": "calibration"
    },
    {
      "id": "2026-10-12-round2-079",
      "date": "2026-10-12",
      "title": "agentbench: corrected claim 79",
      "pages": [
        "/benchmarks/agentbench/"
      ],
      "before": "the 29 models the authors tested in 2023",
      "after": "the 29 models in the paper's current edition",
      "why": "Corrected against https://arxiv.org/abs/2308.03688. See the pre-edit fact reference for the evidence boundary.",
      "kind": "calibration"
    },
    {
      "id": "2026-10-12-round2-080",
      "date": "2026-10-12",
      "title": "Two Claude Sonnet 4.5 rows lack a stated result source",
      "pages": [
        "/benchmarks/agentbench/"
      ],
      "before": "the Claude row in sixth carries no stated source at all",
      "after": "the two Claude Sonnet 4.5 rows, sixth and seventh, carry no stated result source",
      "why": "Corrected against docs/fact-reference.md. See the pre-edit fact reference for the evidence boundary.",
      "kind": "fact"
    },
    {
      "id": "2026-10-12-round2-081",
      "date": "2026-10-12",
      "title": "assistantbench: corrected claim 81",
      "pages": [
        "/benchmarks/assistantbench/"
      ],
      "before": "which refuses outside requests",
      "after": "which the Hub reports it cannot find; that does not establish whether it is private or removed",
      "why": "Corrected against docs/fact-reference.md. See the pre-edit fact reference for the evidence boundary.",
      "kind": "calibration"
    },
    {
      "id": "2026-10-12-round2-082",
      "date": "2026-10-12",
      "title": "OSWorld's version effect is compared with reasoning effort",
      "pages": [
        "/benchmarks/osworld/"
      ],
      "before": "OSWorld version numbers move the score more than model choice does",
      "after": "OSWorld version numbers move the score as much as reasoning effort does",
      "why": "Corrected against docs/fact-reference.md. See the pre-edit fact reference for the evidence boundary.",
      "kind": "fact"
    },
    {
      "id": "2026-10-12-round2-083",
      "date": "2026-10-12",
      "title": "The OSWorld range is about 81% to 86%",
      "pages": [
        "/benchmarks/osworld/"
      ],
      "before": "in the low eighties",
      "after": "between about 81% and 86%",
      "why": "Corrected against docs/fact-reference.md. See the pre-edit fact reference for the evidence boundary.",
      "kind": "fact"
    },
    {
      "id": "2026-10-12-round2-084",
      "date": "2026-10-12",
      "title": "The SWE-bench models were released about one year apart, not two",
      "pages": [
        "/benchmarks/swe-bench/"
      ],
      "before": "Models separated by two years of releases differ by a factor of eight",
      "after": "Models released about a year apart differ by a factor of eight",
      "why": "Corrected against https://www.swebench.com/. See the pre-edit fact reference for the evidence boundary.",
      "kind": "fact"
    },
    {
      "id": "2026-10-12-round2-085",
      "date": "2026-10-12",
      "title": "muse: corrected claim 85",
      "pages": [
        "/agents/muse/"
      ],
      "before": "This page is shorter than the others on purpose.",
      "after": "This page sticks to fewer claims on purpose.",
      "why": "Corrected against docs/fact-reference.md. See the pre-edit fact reference for the evidence boundary.",
      "kind": "calibration"
    },
    {
      "id": "2026-10-12-round2-086",
      "date": "2026-10-12",
      "title": "pine: corrected claim 86",
      "pages": [
        "/agents/pine/"
      ],
      "before": "why it stalls everywhere else",
      "after": "why it shows you so little about how it did it",
      "why": "Corrected against docs/fact-reference.md. See the pre-edit fact reference for the evidence boundary.",
      "kind": "calibration"
    },
    {
      "id": "2026-10-12-round2-087",
      "date": "2026-10-12",
      "title": "town: corrected claim 87",
      "pages": [
        "/agents/town/"
      ],
      "before": "Unlike Muse, none of it is a file on a machine you can reach.",
      "after": "Unlike Muse, where memory is kept as markdown files on a per-user VM, none of it is stored as files. We could not verify whether users can open Muse's files themselves.",
      "why": "Corrected against docs/fact-reference.md. See the pre-edit fact reference for the evidence boundary.",
      "kind": "calibration"
    },
    {
      "id": "2026-10-12-round2-088",
      "date": "2026-10-12",
      "title": "Meta Muse's Mac app can operate some other Mac apps",
      "pages": [
        "/nanomuse/"
      ],
      "before": "On Android and on a computer it can operate other apps through their screens, which Meta Muse cannot.",
      "after": "On Android and on a computer it can operate other apps through their screens. [Meta Muse's Mac app](/agents/muse/), available since 17 September, can also act in some Mac apps; the cloud VM is not its only route.",
      "why": "Corrected against docs/fact-reference.md. See the pre-edit fact reference for the evidence boundary.",
      "kind": "fact"
    },
    {
      "id": "2026-10-12-round2-089",
      "date": "2026-10-12",
      "title": "is-meta-muse-open-source: corrected claim 89",
      "pages": [
        "/is-meta-muse-open-source/"
      ],
      "before": "on Android and a computer it can operate other apps through their screens, which Muse cannot",
      "after": "on Android and a computer it can operate other apps through their screens; Meta Muse also has a Mac app that acts in some Mac apps",
      "why": "Corrected against docs/fact-reference.md. See the pre-edit fact reference for the evidence boundary.",
      "kind": "calibration"
    },
    {
      "id": "2026-10-12-round2-090",
      "date": "2026-10-12",
      "title": "where-it-runs: corrected claim 90",
      "pages": [
        "/where-it-runs/"
      ],
      "before": ", which is the one claim on this page that two sources disagree about",
      "after": ", which Cursor's own documentation confirms",
      "why": "Corrected against docs/fact-reference.md. See the pre-edit fact reference for the evidence boundary.",
      "kind": "calibration"
    },
    {
      "id": "2026-10-12-round2-091",
      "date": "2026-10-12",
      "title": "where-it-runs: corrected claim 91",
      "pages": [
        "/where-it-runs/"
      ],
      "before": "Grok Bot, if Every's reporting is right, runs on Cursor's cloud",
      "after": "Grok Bot runs on Cursor's cloud, according to Cursor's own documentation",
      "why": "Corrected against docs/fact-reference.md. See the pre-edit fact reference for the evidence boundary.",
      "kind": "calibration"
    },
    {
      "id": "2026-10-12-round2-092",
      "date": "2026-10-12",
      "title": "where-it-runs: corrected claim 92",
      "pages": [
        "/where-it-runs/"
      ],
      "before": "no Node and no small server behind it",
      "after": "no Node.js and no Mac mini",
      "why": "Corrected against docs/fact-reference.md. See the pre-edit fact reference for the evidence boundary.",
      "kind": "calibration"
    },
    {
      "id": "2026-10-12-round2-093",
      "date": "2026-10-12",
      "title": "openai-dots: corrected claim 93",
      "pages": [
        "/openai-dots/"
      ],
      "before": "Dot is not in our exam",
      "after": "Dots is not in our exam",
      "why": "Corrected against docs/fact-reference.md. See the pre-edit fact reference for the evidence boundary.",
      "kind": "calibration"
    },
    {
      "id": "2026-10-12-round2-094",
      "date": "2026-10-12",
      "title": "Removed Grok Bot descriptions of purported leaked prompt material",
      "pages": [
        "/grok-bot/"
      ],
      "before": "A paragraph described purported leaked system-prompt material.",
      "after": "Removed that paragraph without repeating its contents.",
      "why": "Corrected against Site rule in data/landscape.json: OpenPoke maintainer_note. See the pre-edit fact reference for the evidence boundary.",
      "kind": "fact"
    },
    {
      "id": "2026-10-12-round2-095",
      "date": "2026-10-12",
      "title": "Grok Bot's App Store snapshot: about 9,700 ratings averaging 4.9",
      "pages": [
        "/grok-bot/"
      ],
      "before": "about 9,900 ratings averaging just under 4.9",
      "after": "about 9,700 ratings averaging 4.9 (App Store snapshot checked by the independent audit on 12 October 2026)",
      "why": "Corrected against docs/fact-reference.md. See the pre-edit fact reference for the evidence boundary.",
      "kind": "fact"
    },
    {
      "id": "2026-10-12-round2-096",
      "date": "2026-10-12",
      "title": "grok-bot: corrected claim 96",
      "pages": [
        "/grok-bot/"
      ],
      "before": "We have not run Grok Bot on them yet.",
      "after": "Grok Bot is listed as pending in our exam data. No result is published, and that status does not establish which runs have started.",
      "why": "Corrected against docs/fact-reference.md. See the pre-edit fact reference for the evidence boundary.",
      "kind": "calibration"
    },
    {
      "id": "2026-10-12-round2-097",
      "date": "2026-10-12",
      "title": "grok-bot: corrected claim 97",
      "pages": [
        "/grok-bot/"
      ],
      "before": "It is the fifth product in our exam",
      "after": "It is the fifth product listed in our exam data",
      "why": "Corrected against docs/fact-reference.md. See the pre-edit fact reference for the evidence boundary.",
      "kind": "calibration"
    },
    {
      "id": "2026-10-12-round2-town-trial",
      "date": "2026-10-12",
      "title": "Town's Pro trial lasts seven days, not two weeks",
      "pages": [
        "/agents/town/"
      ],
      "before": "The signup trial was described as lasting two weeks.",
      "after": "The signup Pro trial lasts seven days.",
      "why": "https://www.town.com/pricing, re-read on 12 October 2026.",
      "kind": "fact"
    },
    {
      "id": "2026-10-12-round2-town-allowances",
      "date": "2026-10-12",
      "title": "Added Town's monthly message and credit allowances",
      "pages": [
        "/agents/town/"
      ],
      "before": "Included allowances were omitted.",
      "after": "Free: 30 chat messages/month; Starter: 1,250 credits/month; Pro: 5,000; Power: 12,000; Power Plus: 40,000.",
      "why": "https://www.town.com/pricing, re-read on 12 October 2026.",
      "kind": "fact"
    },
    {
      "id": "2026-10-12-round2-muse-price-evidence",
      "date": "2026-10-12",
      "title": "Muse price evidence",
      "pages": [
        "/agents/muse/"
      ],
      "before": "Only the 9 September MBI account snapshot supported the paid-price figures.",
      "after": "Every also reports Power $20/month and Maximum $100/month. Both remain third-party reports, not verified Meta pricing.",
      "why": "https://every.to/personal-agents-comparison, re-read on 12 October 2026.",
      "kind": "calibration"
    },
    {
      "id": "2026-10-12-round2-webarena-method",
      "date": "2026-10-12",
      "title": "Webarena method",
      "pages": [
        "/benchmarks/webarena/"
      ],
      "before": "The method note confused the newest top-five date with the newest submission date.",
      "after": "The newest submission was dated June 2026; the top five were unchanged.",
      "why": "Independent audit of the source sheet linked in data/benchmarks.json, 12 October 2026. A fresh CSV fetch failed during this repair.",
      "kind": "calibration"
    },
    {
      "id": "2026-10-12-round2-booking-phone",
      "date": "2026-10-12",
      "title": "Booking phone",
      "pages": [
        "/tasks/book-a-table/"
      ],
      "before": "The Pine row asserted how a contended table is released.",
      "after": "The row says Pine phones the venue, without claiming a release mechanism.",
      "why": "No supplied evidence supports the removed claim.",
      "kind": "calibration"
    },
    {
      "id": "2026-10-12-round2-followup-001",
      "date": "2026-10-12",
      "title": "agentbench: evidence and scope correction 1",
      "pages": [
        "/benchmarks/agentbench/"
      ],
      "before": "relative to the models of 2023",
      "after": "relative to the paper's model cohort",
      "why": "Corrected against docs/fact-reference.md.",
      "kind": "calibration"
    },
    {
      "id": "2026-10-12-round2-followup-002",
      "date": "2026-10-12",
      "title": "is-meta-muse-open-source: evidence and scope correction 2",
      "pages": [
        "/is-meta-muse-open-source/"
      ],
      "before": "The previous statement overstated the evidence or used an outdated value; the corrected statement is recorded below.",
      "after": "Our [Muse teardown](/agents/muse/) describes a per-user Linux VM where the model request is assembled",
      "why": "Corrected against docs/fact-reference.md.",
      "kind": "calibration"
    },
    {
      "id": "2026-10-12-round2-followup-003",
      "date": "2026-10-12",
      "title": "nanomuse: evidence and scope correction 3",
      "pages": [
        "/nanomuse/"
      ],
      "before": "- **Apps without an API.** Meta Muse: out of reach, because the VM never touches your devices. nanoMuse: the screen itself, on the phone and the computer, with you taking over for logins. Not on iOS, which does not let one app operate another.",
      "after": "- **Apps without an API.** The README's comparison treats Meta Muse as confined to its cloud VM. That is incomplete: [Meta's Mac app](/agents/muse/) can act in some Mac apps since 17 September. nanoMuse describes screen operation on Android and computers, with user takeover for logins; its iOS app cannot operate other apps.",
      "why": "Corrected against docs/fact-reference.md.",
      "kind": "calibration"
    },
    {
      "id": "2026-10-12-round2-followup-004",
      "date": "2026-10-12",
      "title": "nanoMuse's model allowance also includes invites and community use",
      "pages": [
        "/nanomuse/"
      ],
      "before": "¥10 of model use",
      "after": "¥10 of model use, with more through invites and no limit for community members",
      "why": "Corrected against https://github.com/nano-muse/nanoMuse/blob/main/docs/privacy.md.",
      "kind": "fact"
    },
    {
      "id": "2026-10-12-round2-followup-005",
      "date": "2026-10-12",
      "title": "OpenMuse was read from source, not examined as a running product",
      "pages": [
        "/openmuse/"
      ],
      "before": "Our [landscape list](/landscape/) puts OpenMuse in its \"taken apart\" group, because we examined how it is built and wrote it up.",
      "after": "Our [landscape list](/landscape/) marks OpenMuse \"read from source\", because we examined published code and documents rather than a running copy.",
      "why": "Corrected against docs/fact-reference.md.",
      "kind": "fact"
    },
    {
      "id": "2026-10-12-round2-followup-006",
      "date": "2026-10-12",
      "title": "landscape: evidence and scope correction 6",
      "pages": [
        "/landscape/"
      ],
      "before": "The rest are recorded here with a link and nothing else, because a name with no evidence behind it is not a finding.",
      "after": "The rest have source links; several also have public-record pages based on published sources, which are distinct from runtime testing.",
      "why": "Corrected against docs/fact-reference.md.",
      "kind": "calibration"
    },
    {
      "id": "2026-10-12-round2-followup-007",
      "date": "2026-10-12",
      "title": "landscape: evidence and scope correction 7",
      "pages": [
        "/landscape/"
      ],
      "before": "is in it without a figure yet",
      "after": "is listed in the exam data as pending, without a published figure",
      "why": "Corrected against docs/fact-reference.md.",
      "kind": "calibration"
    },
    {
      "id": "2026-10-12-round2-followup-008",
      "date": "2026-10-12",
      "title": "landscape: evidence and scope correction 8",
      "pages": [
        "/landscape/"
      ],
      "before": "**Taken apart** means",
      "after": "**Read from source** means",
      "why": "Corrected against docs/fact-reference.md.",
      "kind": "calibration"
    },
    {
      "id": "2026-10-12-round2-followup-009",
      "date": "2026-10-12",
      "title": "landscape: evidence and scope correction 9",
      "pages": [
        "/landscape/"
      ],
      "before": "Nothing more. It is the honest state for most of this category most of the time, and writing it down is the point of the page.",
      "after": "Some listed products also have public-record pages. That coverage does not imply a teardown or an exam score.",
      "why": "Corrected against docs/fact-reference.md.",
      "kind": "calibration"
    },
    {
      "id": "2026-10-12-round2-followup-010",
      "date": "2026-10-12",
      "title": "The checked names were not traced to personal agents",
      "pages": [
        "/landscape/"
      ],
      "before": "They may well be real products.",
      "after": "The 9 October checks found mostly unrelated products sharing those names; none was traced to a personal agent.",
      "why": "Corrected against docs/fact-reference.md.",
      "kind": "fact"
    },
    {
      "id": "2026-10-12-round2-followup-011",
      "date": "2026-10-12",
      "title": "landscape: evidence and scope correction 11",
      "pages": [
        "/landscape/"
      ],
      "before": "Moving from listed to taken apart needs access first.",
      "after": "Moving from listed to read from source needs published code and documentation. A runtime teardown needs access to a running product.",
      "why": "Corrected against docs/fact-reference.md.",
      "kind": "calibration"
    },
    {
      "id": "2026-10-12-round2-followup-012",
      "date": "2026-10-12",
      "title": "landscape: evidence and scope correction 12",
      "pages": [
        "/landscape/"
      ],
      "before": "Moving from taken apart to scored",
      "after": "Moving from examination to scored",
      "why": "Corrected against docs/fact-reference.md.",
      "kind": "calibration"
    },
    {
      "id": "2026-10-12-round2-followup-013",
      "date": "2026-10-12",
      "title": "landscape: evidence and scope correction 13",
      "pages": [
        "/landscape/"
      ],
      "before": "Muse keeps plain markdown you could open.",
      "after": "Muse keeps plain markdown on a per-user VM; personal access to those files was not verified.",
      "why": "Corrected against docs/fact-reference.md.",
      "kind": "calibration"
    },
    {
      "id": "2026-10-12-round2-followup-014",
      "date": "2026-10-12",
      "title": "pine-vs-town: evidence and scope correction 14",
      "pages": [
        "/pine-vs-town/"
      ],
      "before": "[Muse vs Town](/muse-vs-town/) and [Instinct vs Town](/instinct-vs-town/) explain what those records show",
      "after": "[Muse vs Town](/muse-vs-town/#why-one-of-them-has-no-figure) explains the five refused batches and roughly 13% across 30 scorable tasks",
      "why": "Corrected against docs/fact-reference.md.",
      "kind": "calibration"
    },
    {
      "id": "2026-10-12-round2-followup-015",
      "date": "2026-10-12",
      "title": "dispute-a-charge: evidence and scope correction 15",
      "pages": [
        "/tasks/dispute-a-charge/"
      ],
      "before": "File with your bank and you are making a claim. The issuer takes the money back from the merchant, the merchant's payment processor usually charges it [a dispute fee on top](https://docs.stripe.com/disputes/how-disputes-work), and the merchant has to prove the charge was good. The issuer can pull the money back without the merchant agreeing. It also tells the merchant that you went over its head, and some merchants' terms let them close accounts that file chargebacks. [Porkbun's hosting terms](https://porkbun.com/legal/agreement/product_terms_of_service), for one, call any chargeback a material breach and grounds for termination, and allow it to suspend access to the account.",
      "after": "File with your bank and you are making a claim. The issuer takes the money back from the merchant, processors such as Stripe charge the merchant [a dispute fee on top](https://docs.stripe.com/disputes/how-disputes-work), and the merchant has to prove the charge was good. The issuer can pull the money back without the merchant agreeing. It also tells the merchant that you went over its head, and some merchants' terms let them close accounts that file chargebacks. [Porkbun's hosting terms](https://porkbun.com/legal/agreement/product_terms_of_service), for one, call any chargeback a material breach and grounds for termination, and allow it to suspend access to the account.",
      "why": "Corrected against docs/fact-reference.md.",
      "kind": "calibration"
    },
    {
      "id": "2026-10-12-round2-followup-016",
      "date": "2026-10-12",
      "title": "Five current tasks need real accounts; the older draw excluded three",
      "pages": [
        "/how-the-exam-works/how-tasks-are-drawn/"
      ],
      "before": "Three were removed because they need real accounts on real services and an open connection to the public internet.",
      "after": "The historical draw note records three real-account exclusions. The current library has five such tasks: Google Calendar self-event, Notion self-page, self-only calendar read, self-only calendar write, and Slack self-message. The published draw record does not identify its three exclusions, so we cannot reconcile that older count with the current inventory. The 312-to-309 arithmetic below describes that historical draw, not a fresh exclusion of all five.",
      "why": "Corrected against docs/fact-reference.md.",
      "kind": "fact"
    },
    {
      "id": "2026-10-12-round2-followup-017",
      "date": "2026-10-12",
      "title": "how-tasks-are-drawn: evidence and scope correction 17",
      "pages": [
        "/how-the-exam-works/how-tasks-are-drawn/"
      ],
      "before": "Nothing else was removed.",
      "after": "The historical draw records no other removals before the robot-check exclusion.",
      "why": "Corrected against docs/fact-reference.md.",
      "kind": "calibration"
    },
    {
      "id": "2026-10-12-round2-followup-018",
      "date": "2026-10-12",
      "title": "Cowork documentation says scheduled tasks now run in the cloud",
      "pages": [
        "/claude-dispatch/"
      ],
      "before": "- **Scheduled tasks.** You can tell it once to check email every morning or compile a Friday report, and it runs on that schedule.",
      "after": "- **Scheduled tasks.** Recurring work can run on a schedule. [Anthropic's Cowork documentation](https://support.claude.com/en/articles/15520349-use-claude-cowork-on-web-desktop-and-mobile) says scheduled tasks run in the cloud and no longer need your computer awake, following the 6 October change.",
      "why": "Corrected against docs/fact-reference.md.",
      "kind": "fact"
    },
    {
      "id": "2026-10-12-round2-followup-019",
      "date": "2026-10-12",
      "title": "claude-dispatch: evidence and scope correction 19",
      "pages": [
        "/claude-dispatch/"
      ],
      "before": "None is a structured test, and all date from March, months before the September and October changes described above.",
      "after": "Those three accounts are not structured tests and date from March, before the September and October changes. This is the set we could read in full, not a claim that no later hands-on reports exist.",
      "why": "Corrected against docs/fact-reference.md.",
      "kind": "calibration"
    },
    {
      "id": "2026-10-12-round2-followup-020",
      "date": "2026-10-12",
      "title": "cue-by-manus: evidence and scope correction 20",
      "pages": [
        "/cue-by-manus/"
      ],
      "before": "The most common question in search results is how Cue differs from Manus, and the public answer is mostly about shape rather than technology.",
      "after": "Cue differs from Manus chiefly in how the published descriptions package and deliver the agent; those descriptions do not establish a separate underlying technology.",
      "why": "Corrected against docs/fact-reference.md.",
      "kind": "calibration"
    },
    {
      "id": "2026-10-12-round2-followup-021",
      "date": "2026-10-12",
      "title": "poke-ai: evidence and scope correction 21",
      "pages": [
        "/poke-ai/"
      ],
      "before": "The [pricing page](https://poke.com/pricing) lists three plans.",
      "after": "The [pricing page](https://poke.com/pricing) lists three plans. The figures below are billed monthly; a yearly billing option also exists.",
      "why": "Corrected against docs/fact-reference.md.",
      "kind": "calibration"
    },
    {
      "id": "2026-10-12-round2-followup-022",
      "date": "2026-10-12",
      "title": "instinct: evidence and scope correction 22",
      "pages": [
        "/agents/instinct/"
      ],
      "before": "The basics first, since the product's own site says little.",
      "after": "The basics first, since the product's own site says little. This is Instinct from Spear Street Technology at instinct.com, not the sales-coaching product at getinstinct.io or the Hugging Face organisation named Instinct-AI.",
      "why": "Corrected against docs/fact-reference.md.",
      "kind": "calibration"
    },
    {
      "id": "2026-10-12-round2-final-015",
      "date": "2026-10-12",
      "title": "Existing Cowork schedules stay local; local files need the desktop app",
      "pages": [
        "/claude-dispatch/"
      ],
      "before": "says scheduled tasks run in the cloud and no longer need your computer awake, following the 6 October change.",
      "after": "says new scheduled tasks on Pro and Max run in the cloud after the 6 October change; existing local schedules remain on the computer. Cloud tasks need the desktop app open when they use local files.",
      "why": "Corrected against docs/fact-reference.md.",
      "kind": "fact"
    },
    {
      "id": "2026-10-12-round2-final-016",
      "date": "2026-10-12",
      "title": "live-alerts: scope and attribution correction 16",
      "pages": [
        "/tasks/live-alerts/"
      ],
      "before": "The response window is five real-time minutes after the event.",
      "after": "Deadlines run from the event time, and the exam clock runs at real speed for up to five minutes afterward.",
      "why": "Corrected against docs/fact-reference.md.",
      "kind": "calibration"
    },
    {
      "id": "2026-10-12-round2-radar-source-policy",
      "date": "2026-10-12",
      "title": "Removed Grok Bot descriptions of purported leaked prompt material",
      "pages": [
        "/radar/",
        "/grok-bot/"
      ],
      "before": "A public-record paragraph and its source radar entry described purported leaked prompt material.",
      "after": "Both descriptions were removed, without repeating their contents.",
      "why": "The site rule forbids republishing or describing leaked prompts.",
      "kind": "fact"
    },
    {
      "id": "2026-10-12-round2-muse-access-repeat",
      "date": "2026-10-12",
      "title": "Muse memory-file access remains unverified",
      "pages": [
        "/muse-vs-instinct/"
      ],
      "before": "A later comparison example still implied the user could open the VM home directory.",
      "after": "The file location is known, but user access was not verified.",
      "why": "content/agents/muse.md: no interactive terminal was available.",
      "kind": "calibration"
    },
    {
      "id": "2026-10-12-round2-consistency-1",
      "date": "2026-10-12",
      "title": "muse-vs-town: comparison evidence boundary",
      "pages": [
        "/muse-vs-town/"
      ],
      "before": "- Both answer \"a machine\", and one of them means theirs while the other means yours.",
      "after": "- Muse uses a per-user cloud VM; Town combines server-side processing with native Mac access.",
      "why": "data/results.json and data/scorecard.json, as indexed in docs/fact-reference.md.",
      "kind": "calibration"
    },
    {
      "id": "2026-10-12-round2-consistency-2",
      "date": "2026-10-12",
      "title": "muse-vs-town: comparison evidence boundary",
      "pages": [
        "/muse-vs-town/"
      ],
      "before": "- Both keep memory you could in principle correct, which is rarer than it should be.",
      "after": "- Town exposes an editable personality document; personal access to Muse's memory files remains unverified.",
      "why": "data/results.json and data/scorecard.json, as indexed in docs/fact-reference.md.",
      "kind": "calibration"
    },
    {
      "id": "2026-10-12-round2-consistency-3",
      "date": "2026-10-12",
      "title": "muse-vs-town: comparison evidence boundary",
      "pages": [
        "/muse-vs-town/"
      ],
      "before": "- One of the two has no published figure on our scorecard, and that is deliberate.",
      "after": "- Town's main-table figure is withheld. A separate older 114-task batch includes both products, with intervals and caveats below.",
      "why": "data/results.json and data/scorecard.json, as indexed in docs/fact-reference.md.",
      "kind": "calibration"
    },
    {
      "id": "2026-10-12-round2-consistency-4",
      "date": "2026-10-12",
      "title": "muse-vs-instinct: comparison evidence boundary",
      "pages": [
        "/muse-vs-instinct/"
      ],
      "before": "- Completion rates exist for both, but they rest on very different amounts of exam, and we do not rank them.",
      "after": "- The main-table rates use different task sets; a separate older 114-task batch compares them on common tasks, with limitations below.",
      "why": "data/results.json and data/scorecard.json, as indexed in docs/fact-reference.md.",
      "kind": "calibration"
    },
    {
      "id": "2026-10-12-round2-consistency-5",
      "date": "2026-10-12",
      "title": "pine-vs-town: comparison evidence boundary",
      "pages": [
        "/pine-vs-town/"
      ],
      "before": "- Both let something other than you start the work, which is where the foothold question stops being theoretical.",
      "after": "- Town has documented clock and email triggers. Our Pine teardown does not establish whether it can start work without you.",
      "why": "data/results.json and data/scorecard.json, as indexed in docs/fact-reference.md.",
      "kind": "calibration"
    },
    {
      "id": "2026-10-12-round2-consistency-6",
      "date": "2026-10-12",
      "title": "pine-vs-town: comparison evidence boundary",
      "pages": [
        "/pine-vs-town/"
      ],
      "before": "Everything on this page is from those teardowns, [Pine](/agents/pine/) and [Town](/agents/town/), including the lists of what we could not establish.",
      "after": "Runtime findings come from the [Pine](/agents/pine/) and [Town](/agents/town/) teardowns; completion figures come from the separate exam data on the scorecard.",
      "why": "data/results.json and data/scorecard.json, as indexed in docs/fact-reference.md.",
      "kind": "calibration"
    },
    {
      "id": "2026-10-12-round2-finish-1",
      "date": "2026-10-12",
      "title": "home: source scope clarified",
      "pages": [
        "/"
      ],
      "before": "**Town has no figure here, and the reason is worth more than a figure would be.**",
      "after": "**Town has no figure in the first batch. The separate 114-task batch above does include it.**",
      "why": "docs/fact-reference.md",
      "kind": "calibration"
    },
    {
      "id": "2026-10-12-round2-finish-2",
      "date": "2026-10-12",
      "title": "grok-bot: source scope clarified",
      "pages": [
        "/grok-bot/"
      ],
      "before": "Three public posts about Grok Bot or its model family have come through our [radar](/radar/). All three are posts on X.",
      "after": "Two public posts discussed below have come through our [radar](/radar/). Both are posts on X.",
      "why": "docs/fact-reference.md",
      "kind": "calibration"
    },
    {
      "id": "2026-10-12-round2-finish-3",
      "date": "2026-10-12",
      "title": "A self-serve Grok Business seat also gives access to Grok Bot",
      "pages": [
        "/grok-bot/"
      ],
      "before": "SuperGrok Lite does not include it.",
      "after": "An assigned seat on a self-serve Grok Business plan is another access route, using Sign in with Grok. SuperGrok Lite does not include it.",
      "why": "https://cursor.com/help/grok-bot/plans",
      "kind": "fact"
    },
    {
      "id": "2026-10-12-round2-finish-4",
      "date": "2026-10-12",
      "title": "Cowork's remote browser is coming soon, not already available",
      "pages": [
        "/copilot-tasks/"
      ],
      "before": "describe two Cowork browser routes: a remote browser separate from your device, and local Microsoft Edge using sites where you are already signed in.",
      "after": "describe local Microsoft Edge using sites where you are already signed in, and a remote browser separate from your device that is marked coming soon.",
      "why": "https://support.microsoft.com/en-us/microsoft-365-copilot/learning/changes-microsoft-copilot-app",
      "kind": "fact"
    },
    {
      "id": "2026-10-12-round2-view-count-evidence",
      "date": "2026-10-12",
      "title": "Removed a social view count without retained verification",
      "pages": [
        "/agents/instinct/",
        "/how-we-take-them-apart/"
      ],
      "before": "Both pages repeated a dated 620,000-view claim.",
      "after": "The attributed finding and source link remain, without the unverified reach count.",
      "why": "A supporting screenshot could not be established from the supplied public evidence.",
      "kind": "fact"
    },
    {
      "id": "2026-10-12-round2-instinct-price-scope",
      "date": "2026-10-12",
      "title": "Instinct pricing verification narrowed to the pages checked",
      "pages": [
        "/agents/instinct/"
      ],
      "before": "Instinct publishes no price. There is no pricing page and no price anywhere on the site. Its terms of service do have a paid services section: fees in US dollars, charged to your card, non-refundable except where the law says otherwise, and open to change at any time. The terms name no amount.",
      "after": "No subscription amount was found on Instinct’s public homepage or in section 4 of its terms when checked on 12 October 2026. The terms allow paid services in US dollars and changes to fees, and make payments non-refundable except where law requires otherwise. They specify no price.",
      "why": "https://instinct.com/ and https://instinct.com/terms checked 12 October 2026; no subscription amount exposed.",
      "kind": "calibration"
    },
    {
      "id": "2026-10-12-round2-editorial-1",
      "date": "2026-10-12",
      "title": "instinct: audit follow-up and source context",
      "pages": [
        "/agents/instinct/"
      ],
      "before": "This context was not included.",
      "after": "[The Information reported a compute shortage on 10 September](https://www.theinformation.com/articles/personal-ai-app-instinct-faces-compute-crunch-lead-new-funding). That does not establish when a particular waitlist applicant will gain access.",
      "why": "Independent audit follow-up; https://www.theinformation.com/articles/personal-ai-app-instinct-faces-compute-crunch-lead-new-funding",
      "kind": "calibration"
    },
    {
      "id": "2026-10-12-round2-editorial-2",
      "date": "2026-10-12",
      "title": "muse: audit follow-up and source context",
      "pages": [
        "/agents/muse/"
      ],
      "before": "Meta's guidance inside Muse says business calling is in early access and reaching accounts gradually. The account we examined never received it, so we have not seen a Muse call happen.",
      "after": "The guidance shown in our teardown account described business calling as early access with a gradual rollout. That account did not receive it. We have not rechecked the current rollout or observed a Muse call, so this is a limit of our examination, not a claim that other accounts cannot call businesses.",
      "why": "Independent audit follow-up; existing page sources and docs/fact-reference.md",
      "kind": "calibration"
    },
    {
      "id": "2026-10-12-round2-editorial-3",
      "date": "2026-10-12",
      "title": "muse: audit follow-up and source context",
      "pages": [
        "/agents/muse/"
      ],
      "before": "This context was not included.",
      "after": "[Meta’s launch post](https://about.fb.com/news/2026/09/introducing-muse-personal-ai-agent/) says users can opt out of interactions being used to train its models. It separately excludes conversations and VM data from its ad systems. Those are different uses of the data.",
      "why": "Independent audit follow-up; https://about.fb.com/news/2026/09/introducing-muse-personal-ai-agent/",
      "kind": "calibration"
    },
    {
      "id": "2026-10-12-round2-editorial-4",
      "date": "2026-10-12",
      "title": "muse: audit follow-up and source context",
      "pages": [
        "/agents/muse/"
      ],
      "before": "**How does Meta Muse compare with Pine, Instinct and Town?**\nIt's the only one of the four that separates tool permissions from tool definitions, and the only one that assembles the model request on a machine belonging to the user. Pine shows the least of its tool surface, Town's Mac app reaches deepest into the system of the desktop apps we examined, and Instinct lives in your message threads. Pine places phone calls today; Meta says Muse's business calling is in early access, and we have not seen it.",
      "after": "**How does Meta Muse compare with Pine, Instinct and Town?**\nIt's the only one of the four that separates tool permissions from tool definitions, and the only one that assembles the model request on a machine belonging to the user. Pine shows the least of its tool surface, Town's Mac app reaches deepest into the system of the desktop apps we examined, and Instinct lives in your message threads. Pine places phone calls today; our Muse account did not receive business calling during the teardown, and we have not rechecked its current rollout.",
      "why": "Independent audit follow-up; existing page sources and docs/fact-reference.md",
      "kind": "calibration"
    },
    {
      "id": "2026-10-12-round2-editorial-5",
      "date": "2026-10-12",
      "title": "pine: audit follow-up and source context",
      "pages": [
        "/agents/pine/"
      ],
      "before": "This context was not included.",
      "after": "The US [App Store listing](https://apps.apple.com/us/app/pine-assistant/id6746403769), checked 12 October 2026, showed 4.5 out of 5 from 147 ratings. Those are user ratings, not a measured completion rate or our assessment of Pine’s reliability.",
      "why": "Independent audit follow-up; https://apps.apple.com/us/app/pine-assistant/id6746403769",
      "kind": "calibration"
    },
    {
      "id": "2026-10-12-round2-editorial-6",
      "date": "2026-10-12",
      "title": "town: audit follow-up and source context",
      "pages": [
        "/agents/town/"
      ],
      "before": "The basics, since our teardown says nothing about the company. [Fortune reported](https://fortune.com/2026/06/03/towns-ai-assistants-andreessen-horowitz-forerunner-55-million/) on 3 June 2026 that Town raised a $55 million Series A led by Andreessen Horowitz, and that it was founded in late 2024 by Plaid's former CTO and a former Google director of applied AI. The same piece says you can't sign up without connecting your email and calendar, and that the founder puts work at roughly 70 to 80 percent of use. Each user's assistant is a \"Townie\" with a name and a look of its own. [Town's homepage](https://www.town.com/) lists work tools such as Slack among its integrations, an iPhone app, and WhatsApp as a way to message your Townie, inbound only. Town is not \"AI Town\", the unrelated virtual town of AI characters that shares its search results.",
      "after": "The basics, since our teardown says nothing about the company. [Fortune reported](https://fortune.com/2026/06/03/towns-ai-assistants-andreessen-horowitz-forerunner-55-million/) on 3 June 2026 that Town raised a $55 million Series A led by Andreessen Horowitz, and that it was founded in late 2024 by Jean-Denis Gréze, Plaid's former CTO, and Tony Vincent, formerly a Google director of applied AI. The same piece says you can't sign up without connecting your email and calendar, and that the founder puts work at roughly 70 to 80 percent of use. Each user's assistant is a \"Townie\" with a name and a look of its own. [Town's homepage](https://www.town.com/) lists work tools such as Slack among its integrations, an iPhone app, and WhatsApp as a way to message your Townie, inbound only. Town is not \"AI Town\", the unrelated virtual town of AI characters that shares its search results.",
      "why": "Independent audit follow-up; https://fortune.com/2026/06/03/towns-ai-assistants-andreessen-horowitz-forerunner-55-million/; https://www.town.com/",
      "kind": "calibration"
    },
    {
      "id": "2026-10-12-round2-editorial-7",
      "date": "2026-10-12",
      "title": "town: audit follow-up and source context",
      "pages": [
        "/agents/town/"
      ],
      "before": "This context was not included.",
      "after": "For buying context, [Town publishes its plans](https://www.town.com/pricing). An [eesel review](https://www.eesel.ai/blog/town-ai-review) highlights uncertainty about per-action credit costs. Eesel sells its own AI product, so that review is a commercial perspective, separate from our runtime findings.",
      "why": "Independent audit follow-up; https://www.town.com/pricing; https://www.eesel.ai/blog/town-ai-review",
      "kind": "calibration"
    },
    {
      "id": "2026-10-12-round2-editorial-8",
      "date": "2026-10-12",
      "title": "benchmarks: audit follow-up and source context",
      "pages": [
        "/benchmarks/"
      ],
      "before": "This context was not included.",
      "after": "[PERSONA Bench](https://arxiv.org/abs/2407.17387) is adjacent research: it evaluates alignment with diverse synthetic user preferences. That is relevant to personalisation, but it is not a completion test for consumer agents carrying out errands.",
      "why": "Independent audit follow-up; https://arxiv.org/abs/2407.17387",
      "kind": "calibration"
    },
    {
      "id": "2026-10-12-round2-editorial-9",
      "date": "2026-10-12",
      "title": "benchmarks: audit follow-up and source context",
      "pages": [
        "/benchmarks/"
      ],
      "before": "These leaderboards answer \"how capable is this system at a task in isolation\". They can't answer \"should I give this product access to my life\", and the second question is the one anybody choosing an assistant is actually asking.\n\nRead a benchmark page for the detail, or go to the [scorecard](/) that asks the other question, [how we score](/how-scoring-works/) for where its axes came from, and the four [teardowns](/agents/) they were built out of.",
      "after": "These leaderboards measure task performance. They do not settle whether a consumer product should hold access to your accounts.\n\nFor detail, see the [scorecard](/) that asks the other question, [how we score](/how-scoring-works/) for where its axes came from, and the four [teardowns](/agents/) they were built out of.",
      "why": "Independent audit follow-up; existing page sources and docs/fact-reference.md",
      "kind": "calibration"
    },
    {
      "id": "2026-10-12-round2-editorial-10",
      "date": "2026-10-12",
      "title": "assistantbench: audit follow-up and source context",
      "pages": [
        "/benchmarks/assistantbench/"
      ],
      "before": "This context was not included.",
      "after": "Names in this area cover different measurements. [LiveClawBench](https://arxiv.org/abs/2604.13072) evaluates complex assistant tasks. [HiL-Bench](https://arxiv.org/abs/2604.09408) tests whether agents seek help when requirements are blocked or ambiguous. [UXBench](https://mengze-hong.github.io/UXBench/) evaluates responses to failure-prone user queries and their effect on user experience. None of those names denotes the AssistantBench table below.",
      "why": "Independent audit follow-up; https://arxiv.org/abs/2604.13072; https://arxiv.org/abs/2604.09408; https://mengze-hong.github.io/UXBench/",
      "kind": "calibration"
    },
    {
      "id": "2026-10-12-round2-editorial-11",
      "date": "2026-10-12",
      "title": "osworld: audit follow-up and source context",
      "pages": [
        "/benchmarks/osworld/"
      ],
      "before": "This context was not included.",
      "after": "For another examination of the original tasks, [Epoch AI’s OSWorld analysis](https://epoch.ai/publications/what-does-osworld-tell-us-about-ais-ability-to-use-computers) discusses ambiguous instructions and changes over time. [OSWorld-Human](https://arxiv.org/abs/2506.16042) studies efficiency as well as completion, a separate question from who leads this table.",
      "why": "Independent audit follow-up; https://epoch.ai/publications/what-does-osworld-tell-us-about-ais-ability-to-use-computers; https://arxiv.org/abs/2506.16042",
      "kind": "calibration"
    },
    {
      "id": "2026-10-12-round2-editorial-12",
      "date": "2026-10-12",
      "title": "tau-bench: audit follow-up and source context",
      "pages": [
        "/benchmarks/tau-bench/"
      ],
      "before": "This context was not included.",
      "after": "Sierra calls the knowledge-grounded extension [τ-knowledge](https://sierra.ai/blog/tau-knowledge), evaluated through its banking tasks. It tests work with a changing knowledge base; its scores should not be substituted for the airline, retail and telecom figures here.",
      "why": "Independent audit follow-up; https://sierra.ai/blog/tau-knowledge",
      "kind": "calibration"
    },
    {
      "id": "2026-10-12-round2-editorial-13",
      "date": "2026-10-12",
      "title": "terminal-bench: audit follow-up and source context",
      "pages": [
        "/benchmarks/terminal-bench/"
      ],
      "before": "This context was not included.",
      "after": "[Terminal-Bench-Science](https://www.terminal-bench-science.ai/) is a separate benchmark for scientific research workflows. The [Harbor-hosted repository](https://github.com/harbor-framework/terminal-bench-science) describes tasks authored and reviewed by domain experts across scientific disciplines. Its 0.1 board lists 70 tasks. Those results use a different task set from Terminal-Bench 4.0 and cannot be placed in the table below as equivalent scores.",
      "why": "Independent audit follow-up; https://www.terminal-bench-science.ai/; https://github.com/harbor-framework/terminal-bench-science",
      "kind": "calibration"
    },
    {
      "id": "2026-10-12-round2-editorial-14",
      "date": "2026-10-12",
      "title": "webarena: audit follow-up and source context",
      "pages": [
        "/benchmarks/webarena/"
      ],
      "before": "This context was not included.",
      "after": "[WebArena-Pro](https://www.servicenow.com/research/fr/publication/imene-kerboua-weba-icml-workshops2026.html) is another separate variant. ServiceNow’s July 2026 publication describes 300 tasks across 20 self-hosted applications, including audio and video observations. Its task set and step budget differ, so its scores do not extend the original WebArena board.",
      "why": "Independent audit follow-up; https://www.servicenow.com/research/fr/publication/imene-kerboua-weba-icml-workshops2026.html",
      "kind": "calibration"
    },
    {
      "id": "2026-10-12-round2-editorial-15",
      "date": "2026-10-12",
      "title": "grok-bot: audit follow-up and source context",
      "pages": [
        "/grok-bot/"
      ],
      "before": "Two public posts discussed below have come through our [radar](/radar/). Both are posts on X. We have not verified any of them; we link each so you can judge it yourself.\n\nOn 5 October, one X user, [@gippp69](https://x.com/gippp69/status/2107153066134790406), reported giving six bots one overnight job and waking to 612 finished tasks, 47 blocked actions and 89 rejected handoffs. The roles were split on purpose: one bot built a candidate list, one scored it, a risk bot returned pass, cut or kill, and only the execution bot could take the final action, after a timestamped pass. Whatever else it is, it is somebody treating the irreversible-action problem seriously inside a multi-agent setup.",
      "after": "In his [14 August review](https://natesnewsletter.substack.com/p/grok-bot-review), Nate B. Jones reports building twelve bots in roughly eight hours, including a chief of staff, a landing-page lead and a customer-research specialist. He describes them sharing one cloud computer and logged-in tools. Those are his reported trials, not our exam results. The [Verge launch report](https://www.theverge.com/ai-artificial-intelligence/978666/spacexai-grok-bot-ai-agent-beta-launch) supplies the historical launch-access context above.\n\nTwo further posts came through our [radar](/radar/). Both are unverified X accounts of use; neither establishes a completion rate.\n\nIn a single unverified post on 5 October, one X user, [@gippp69](https://x.com/gippp69/status/2107153066134790406), reported giving six bots one overnight job and waking to 612 finished tasks, 47 blocked actions and 89 rejected handoffs. The roles were split on purpose: one bot built a candidate list, one scored it, a risk bot returned pass, cut or kill, and only the execution bot could take the final action, after a timestamped pass. Whatever else it is, it is somebody treating the irreversible-action problem seriously inside a multi-agent setup.",
      "why": "Independent audit follow-up; https://natesnewsletter.substack.com/p/grok-bot-review; https://www.theverge.com/ai-artificial-intelligence/978666/spacexai-grok-bot-ai-agent-beta-launch; https://x.com/gippp69/status/2107153066134790406",
      "kind": "calibration"
    },
    {
      "id": "2026-10-12-round2-editorial-16",
      "date": "2026-10-12",
      "title": "home: audit follow-up and source context",
      "pages": [
        "/"
      ],
      "before": "- **Inside an assistant you may already use.** OpenAI's [Dots](/openai-dots/), which Every's [comparison](https://every.to/personal-agents-comparison) calls an always-on agent in ChatGPT, and Google's Gemini Spark, ongoing tasks across Google apps in the same table. Microsoft's [Copilot Tasks](/copilot-tasks/) carries out tasks you delegate, asking approval before payments. Anthropic's [Claude Dispatch](/claude-dispatch/) takes a message from your phone and works on your desktop. [Alexa+](https://aboutamazon.com/news/devices/alexa-plus-web-ai-assistant/), Amazon says, can discover and book services across travel, restaurants, home services and wellness.\n- **In a message thread, like Instinct.** [Poke](/poke-ai/), and [Grok Bot](/grok-bot/), which signs into your accounts and works your tools from a cloud computer.\n- **Run it yourself.** [OpenClaw](https://docs.openclaw.ai/) and Nous Research's [Hermes Agent](https://hermes-agent.nousresearch.com/) are open source and self-hosted; [OpenMuse](/openmuse/) is one we read from its source.",
      "after": "| Personal AI assistant | Maker or project | Where you reach it; published scope |\n| --- | --- | --- |\n| [Dots](/openai-dots/) | OpenAI | Always-on agent in ChatGPT, according to [Every’s comparison](https://every.to/personal-agents-comparison) |\n| Gemini Spark | Google | Ongoing work across Google apps, in the same comparison |\n| [Copilot Tasks](/copilot-tasks/) | Microsoft | Delegated tasks; see the product page for its successor and browser routes |\n| [Claude Dispatch](/claude-dispatch/) | Anthropic | A phone message delegates work; desktop access and cloud schedules have different requirements |\n| [Alexa+](https://aboutamazon.com/news/devices/alexa-plus-web-ai-assistant/) | Amazon | Amazon describes web access and service bookings |\n| [Poke](/poke-ai/) | Interaction | A message thread |\n| [Grok Bot](/grok-bot/) | Cursor / xAI (Every’s attribution) | Messaging and a cloud computer working in connected accounts |\n| [OpenClaw](https://docs.openclaw.ai/) | OpenClaw project | Self-hosted agent |\n| [Hermes Agent](https://hermes-agent.nousresearch.com/) | Nous Research | Self-hosted agent |\n| [OpenMuse](/openmuse/) | CopilotKit | Self-hosted; examined from its source |",
      "why": "Independent audit follow-up; https://every.to/personal-agents-comparison; https://aboutamazon.com/news/devices/alexa-plus-web-ai-assistant/; https://docs.openclaw.ai/; https://hermes-agent.nousresearch.com/",
      "kind": "calibration"
    },
    {
      "id": "2026-10-12-round2-editorial-17",
      "date": "2026-10-12",
      "title": "how-we-take-them-apart: audit follow-up and source context",
      "pages": [
        "/how-we-take-them-apart/"
      ],
      "before": "### The layers an AI agent teardown separates\n\nEvery personal agent we have opened breaks into the same few layers, and each teardown answers the same questions about them in the same order. Where does the model run, and what can it touch? How do tool definitions reach the model, and who decides which ones? Where does memory live: a file you could open, or a store you only reach through the product? What does it ask before acting? Which surface is the default way to reach it? And, last, what could we not establish? Those are the axes on the [scorecard](/), and [how we score](/how-scoring-works/) explains why it is those and not others.",
      "after": "## The layers an AI agent teardown separates\n\nThe [scorecard](/) separates six layers, and [how we score](/how-scoring-works/) explains why they matter.\n\n- **Model:** where inference runs and which model identity we can verify.\n- **Tools:** which actions are available and how their definitions reach the model.\n- **Memory:** what persists, where it lives and whether the user can inspect it.\n- **Approvals:** which actions require permission and how that permission is enforced.\n- **Channel:** where you reach the agent and where it can reach you.\n- **Triggers:** whether a message, a clock or an outside event can start work.",
      "why": "Independent audit follow-up; existing page sources and docs/fact-reference.md",
      "kind": "calibration"
    },
    {
      "id": "2026-10-12-round2-editorial-18",
      "date": "2026-10-12",
      "title": "how-we-take-them-apart: audit follow-up and source context",
      "pages": [
        "/how-we-take-them-apart/"
      ],
      "before": "## Reading a teardown once you know how it was made",
      "after": "### Reading a teardown once you know how it was made",
      "why": "Independent audit follow-up; existing page sources and docs/fact-reference.md",
      "kind": "calibration"
    },
    {
      "id": "2026-10-12-round2-editorial-19",
      "date": "2026-10-12",
      "title": "instinct-vs-town: audit follow-up and source context",
      "pages": [
        "/instinct-vs-town/"
      ],
      "before": "This context was not included.",
      "after": "[Fortune’s report](https://fortune.com/2026/06/03/towns-ai-assistants-andreessen-horowitz-forerunner-55-million/) describes Town’s $55 million Series A led by a16z, its late-2024 founding by Plaid’s former CTO and its personalised assistants, called Townies.",
      "why": "Independent audit follow-up; https://fortune.com/2026/06/03/towns-ai-assistants-andreessen-horowitz-forerunner-55-million/",
      "kind": "calibration"
    },
    {
      "id": "2026-10-12-round2-editorial-20",
      "date": "2026-10-12",
      "title": "is-meta-muse-open-source: audit follow-up and source context",
      "pages": [
        "/is-meta-muse-open-source/"
      ],
      "before": "\"Muse\" means four different things depending on who is asking, and each has its own answer.\n\n- **The app.** iOS, Android, the web, WhatsApp, and since 17 September a Mac app. Closed.\n- **The agent and its machine.** Each user gets a Linux virtual machine in Meta's cloud, with the agent, its tools and a gatekeeper Meta calls Sentinel running on it. Closed. Meta has published a detailed account of the design but not the code.\n- **The model.** Meta's [launch post](https://about.fb.com/news/2026/09/introducing-muse-personal-ai-agent/) says Muse is \"powered by Muse Spark,\" and Meta's [security write-up](https://research.meta.ai/blog/security-and-safety-for-ai-agents-our-approach-with-muse) names Muse Spark 1.3. No public weights.\n- **The devices around it.** The [Muse Gadgets SDK](https://github.com/facebookincubator/muse-gadget-sdk) is open source under Apache 2.0.",
      "after": "The Muse name covers several layers, each with its own answer.\n\n- **The app.** iOS, Android, the web, WhatsApp, and since 17 September a Mac app. Closed.\n- **The agent and its machine.** Each user gets a Linux virtual machine in Meta's cloud, with the agent, its tools and a gatekeeper Meta calls Sentinel running on it. Closed. Meta has published a detailed account of the design but not the code.\n- **The model.** Meta's [launch post](https://about.fb.com/news/2026/09/introducing-muse-personal-ai-agent/) says Muse is \"powered by Muse Spark,\" and Meta's [security write-up](https://research.meta.ai/blog/security-and-safety-for-ai-agents-our-approach-with-muse) names Muse Spark 1.3. No public weights.\n- **Muse Code.** Meta’s separate terminal coding agent is installed from [dev.meta.ai](https://dev.meta.ai/docs/muse-code). Meta announced it [out of beta on 31 August 2026](https://dev.meta.ai/resources/blog/muse-code-new-plans-and-features). Its [public SDK](https://github.com/meta-models/muse-code-sdk) is MIT-licensed, but that repository explicitly excludes the host binary; it does not establish that the full agent runtime is open source.\n\n- **The devices around it.** The [Muse Gadgets SDK](https://github.com/facebookincubator/muse-gadget-sdk) is open source under Apache 2.0.",
      "why": "Independent audit follow-up; https://about.fb.com/news/2026/09/introducing-muse-personal-ai-agent/; https://research.meta.ai/blog/security-and-safety-for-ai-agents-our-approach-with-muse; https://dev.meta.ai/docs/muse-code; https://dev.meta.ai/resources/blog/muse-code-new-plans-and-features; https://github.com/meta-models/muse-code-sdk; https://github.com/facebookincubator/muse-gadget-sdk",
      "kind": "calibration"
    },
    {
      "id": "2026-10-12-round2-editorial-21",
      "date": "2026-10-12",
      "title": "browser-tasks: audit follow-up and source context",
      "pages": [
        "/how-the-exam-works/browser-tasks/"
      ],
      "before": "# Five browser agent benchmark tasks, and what each one catches\n\nOn 4 October 2026 the exam behind our scorecard gained a small browser agent benchmark: five tasks that test the browser rather than the errand. Each is small on purpose: a code to read off a live screen, a login to recover, a page to check twice 45 minutes apart, an entrance that only exists on a phone, a page that turns away robots. These are the places where real agents get stuck in public, and until now the exam had no task that isolated any of them.",
      "after": "# Browser agent failure modes: five focused tests\n\nBrowser agent failure modes are the focus of five tasks added to the exam on 4 October 2026: five tasks that test the browser rather than the errand. Each is small on purpose: a code to read off a live screen, a login to recover, a page to check twice 45 minutes apart, an entrance that only exists on a phone, a page that turns away robots. These are the places where real agents get stuck in public, and until now the exam had no task that isolated any of them.",
      "why": "Independent audit follow-up; existing page sources and docs/fact-reference.md",
      "kind": "calibration"
    },
    {
      "id": "2026-10-12-round2-editorial-22",
      "date": "2026-10-12",
      "title": "checking-the-grader: audit follow-up and source context",
      "pages": [
        "/how-the-exam-works/checking-the-grader/"
      ],
      "before": "Are AI agent benchmarks reliable? Only if their graders, task setup and failure records survive checks of their own. AI agent benchmarks make one claim more often than any other: this product got this task wrong. Every completion rate is built out of that claim, repeated a few hundred times. So before a rate means anything, somebody has to ask how often the claim itself is wrong, meaning how often the product was right and the exam was not. This page is what we found when we asked, including the parts that made our exam look bad. What the public agent benchmarks such as GAIA, WebArena and OSWorld measure is on [the benchmarks page](/benchmarks/); this page is about whether a benchmark's own grading can be believed.",
      "after": "Are AI agent benchmarks reliable? Only if their graders, task setup and failure records survive checks of their own. AI agent benchmarks make one claim more often than any other: this product got this task wrong. Every completion rate is built out of that claim, repeated a few hundred times. So before a rate means anything, somebody has to ask how often the claim itself is wrong, meaning how often the product was right and the exam was not. This page is what we found when we asked, including the parts that made our exam look bad. What the public agent benchmarks such as GAIA, WebArena and OSWorld measure is on [the benchmarks page](/benchmarks/); this page is about whether a benchmark's own grading can be believed. Berkeley’s [audit](https://rdi.berkeley.edu/blog/trustworthy-benchmarks-cont/) found exploits in GAIA, WebArena, OSWorld and Terminal-Bench, among others; our [benchmark guide](/benchmarks/) explains their different tasks.",
      "why": "Independent audit follow-up; https://rdi.berkeley.edu/blog/trustworthy-benchmarks-cont/",
      "kind": "calibration"
    },
    {
      "id": "2026-10-12-round2-editorial-23",
      "date": "2026-10-12",
      "title": "checking-the-grader: audit follow-up and source context",
      "pages": [
        "/how-the-exam-works/checking-the-grader/"
      ],
      "before": "This context was not included.",
      "after": "[Anthropic’s evaluation guide](https://www.anthropic.com/engineering/demystifying-evals-for-ai-agents) recommends checking task specifications and graders when many trials all fail, and keeping a known passing reference solution. Our workshop control shows why: repeated failure did not establish that the products lacked the ability.",
      "why": "Independent audit follow-up; https://www.anthropic.com/engineering/demystifying-evals-for-ai-agents",
      "kind": "calibration"
    },
    {
      "id": "2026-10-12-round2-editorial-24",
      "date": "2026-10-12",
      "title": "checking-the-grader: audit follow-up and source context",
      "pages": [
        "/how-the-exam-works/checking-the-grader/"
      ],
      "before": "This context was not included.",
      "after": "The [BenchJack paper](https://arxiv.org/abs/2605.12673) describes systematic reward-hacking audits. The separate [Agentic Benchmark Checklist](https://uiuc-kang-lab.github.io/agentic-benchmarks/) distinguishes whether a task measures the intended ability from whether its grader correctly recognises success. Both are useful checks on the method here.",
      "why": "Independent audit follow-up; https://arxiv.org/abs/2605.12673; https://uiuc-kang-lab.github.io/agentic-benchmarks/",
      "kind": "calibration"
    },
    {
      "id": "2026-10-12-round2-editorial-25",
      "date": "2026-10-12",
      "title": "exam-changes: audit follow-up and source context",
      "pages": [
        "/how-the-exam-works/changes/"
      ],
      "before": "This context was not included.",
      "after": "For external context, the [BenchJack paper](https://arxiv.org/abs/2605.12673) documents how grader defects can distort scores. [Terminal-Bench’s releases](https://www.tbench.ai/benchmarks) distinguish task sets by version. Our version record serves the same practical need: a changed exam must be identified before its numbers are compared.",
      "why": "Independent audit follow-up; https://arxiv.org/abs/2605.12673; https://www.tbench.ai/benchmarks",
      "kind": "calibration"
    },
    {
      "id": "2026-10-12-round2-editorial-26",
      "date": "2026-10-12",
      "title": "how-tasks-are-drawn: audit follow-up and source context",
      "pages": [
        "/how-the-exam-works/how-tasks-are-drawn/"
      ],
      "before": "Each task will be run twice for every product. Two attempts at each of 169 tasks gives several hundred attempts per product, enough for a completion rate with a confidence interval tight enough to be worth printing. It is not enough to separate products that are close, and the scorecard will say so in the interval rather than in a ranking.",
      "after": "Each task will be run twice for every product. [Anthropic’s evaluation guide](https://www.anthropic.com/engineering/demystifying-evals-for-ai-agents) distinguishes success at least once across repeated trials (pass@k) from success on every trial (pass^k). Two is this protocol’s chosen repeat count, not a general reliability standard; the published record does not establish a cost-based justification for that choice. Two attempts at each of 169 tasks gives several hundred attempts per product, enough for a completion rate with a confidence interval tight enough to be worth printing. It is not enough to separate products that are close, and the scorecard will say so in the interval rather than in a ranking.",
      "why": "Independent audit follow-up; https://www.anthropic.com/engineering/demystifying-evals-for-ai-agents",
      "kind": "calibration"
    },
    {
      "id": "2026-10-12-round2-editorial-27",
      "date": "2026-10-12",
      "title": "how-the-exam-works: audit follow-up and source context",
      "pages": [
        "/how-the-exam-works/"
      ],
      "before": "# How the exam works",
      "after": "# How the exam works: AI agent evaluation that reads the world",
      "why": "Independent audit follow-up; existing page sources and docs/fact-reference.md",
      "kind": "calibration"
    },
    {
      "id": "2026-10-12-round2-editorial-28",
      "date": "2026-10-12",
      "title": "muse-vs-instinct: audit follow-up and source context",
      "pages": [
        "/muse-vs-instinct/"
      ],
      "before": "This context was not included.",
      "after": "On privacy, [Meta says](https://about.fb.com/news/2026/09/introducing-muse-personal-ai-agent/) conversations and VM data stay out of its ad systems, while Sentinel gates network access. Our [Instinct teardown](/agents/instinct/) found a separate credential vault designed to fill secrets without revealing them. These describe different controls; neither statement is a general guarantee against data exposure.",
      "why": "Independent audit follow-up; https://about.fb.com/news/2026/09/introducing-muse-personal-ai-agent/",
      "kind": "calibration"
    },
    {
      "id": "2026-10-12-round2-editorial-29",
      "date": "2026-10-12",
      "title": "muse-vs-pine: audit follow-up and source context",
      "pages": [
        "/muse-vs-pine/"
      ],
      "before": "This context was not included.",
      "after": "[Meta’s security account](https://research.meta.ai/blog/security-and-safety-for-ai-agents-our-approach-with-muse) describes a dedicated Muse Secure VM and a separate Sentinel agent that controls connector actions and network egress. These are Meta’s claims; our teardown did not test those controls.",
      "why": "Independent audit follow-up; https://research.meta.ai/blog/security-and-safety-for-ai-agents-our-approach-with-muse",
      "kind": "calibration"
    },
    {
      "id": "2026-10-12-round2-editorial-30",
      "date": "2026-10-12",
      "title": "muse-vs-pine: audit follow-up and source context",
      "pages": [
        "/muse-vs-pine/"
      ],
      "before": "**Who makes Muse and Pine, and what do they cost?**\nMuse is Meta's, free for most use with subscription plans, and rolling out in the US. Pine's terms name 19Pine Pte. Ltd.; the App Store seller is 19Pine Corp. It publishes weekly personal plans and monthly work plans; the current figures are on our [Pine page](/agents/pine/#pricing).",
      "after": "**Who makes Muse and Pine, and what do they cost?**\nMuse is Meta's, free for most use with subscription plans, and rolling out in the US. Pine's [terms](https://www.19pine.ai/terms-of-service) name 19Pine Pte. Ltd.; the App Store seller is 19Pine Corp. It publishes weekly personal plans and monthly work plans; the current figures are on our [Pine page](/agents/pine/#pricing).",
      "why": "Independent audit follow-up; https://www.19pine.ai/terms-of-service",
      "kind": "calibration"
    },
    {
      "id": "2026-10-12-round2-editorial-31",
      "date": "2026-10-12",
      "title": "muse-vs-town: audit follow-up and source context",
      "pages": [
        "/muse-vs-town/"
      ],
      "before": "This context was not included.",
      "after": "[Fortune’s report](https://fortune.com/2026/06/03/towns-ai-assistants-andreessen-horowitz-forerunner-55-million/) describes Town’s $55 million Series A led by a16z, its late-2024 founding by Plaid’s former CTO and its personalised assistants, called Townies.\n\n[Meta’s security account](https://research.meta.ai/blog/security-and-safety-for-ai-agents-our-approach-with-muse) describes a dedicated Muse Secure VM and a separate Sentinel agent that controls connector actions and network egress. These are Meta’s claims; our teardown did not test those controls.",
      "why": "Independent audit follow-up; https://fortune.com/2026/06/03/towns-ai-assistants-andreessen-horowitz-forerunner-55-million/; https://research.meta.ai/blog/security-and-safety-for-ai-agents-our-approach-with-muse",
      "kind": "calibration"
    },
    {
      "id": "2026-10-12-round2-editorial-32",
      "date": "2026-10-12",
      "title": "openmuse: audit follow-up and source context",
      "pages": [
        "/openmuse/"
      ],
      "before": "| Question | What the repository says |\n| --- | --- |\n| Where it runs | Its own server, a task worker, and a browser worker. Persistent Chromium plus an optional Linux container with a persistent workspace volume. |\n| How tools are exposed | A CopilotKit runtime streaming AG-UI events, with Google and browser protocol adapters kept in their own package and a stored action review step in front of them. |\n| Where memory lives | PGlite or PostgreSQL for application state, a persistent volume for the workspace, and a hosted service for conversation history. |\n| What you get to approve | A stored action review step between deciding and doing, with approvals, saved receipts, pause, resume, cancel and retry listed as first-class features. |\n| How you reach it | One shared codebase for iOS, Android and web. |\n| What we could not establish | Nothing about the runtime. Everything below the hosted persistence boundary, which is a separate service. |",
      "after": "| Question | What the repository says |\n| --- | --- |\n| Where it runs | Its own server, a task worker, and a browser worker. Persistent Chromium plus an optional Linux container with a persistent workspace volume. |\n| How tools are exposed | A CopilotKit runtime streaming AG-UI events (messages that keep the interface in step with the agent), with Google and browser protocol adapters (connectors translating actions into each service’s format) kept in their own package and a stored action review step in front of them. |\n| Where memory lives | PGlite or PostgreSQL for application state, a persistent volume for the workspace, and a hosted service for conversation history. |\n| What you get to approve | A stored action review step between deciding and doing, with approvals, saved receipts, pause, resume, cancel and retry listed as first-class features. |\n| How you reach it | One shared codebase for iOS, Android and web. |\n| What we could not establish | Nothing about the runtime. Everything below the hosted persistence boundary, which is a separate service. |",
      "why": "Independent audit follow-up; existing page sources and docs/fact-reference.md",
      "kind": "calibration"
    },
    {
      "id": "2026-10-12-round2-editorial-33",
      "date": "2026-10-12",
      "title": "openmuse: audit follow-up and source context",
      "pages": [
        "/openmuse/"
      ],
      "before": "Interrupted work recovers through SQL leases rather than being retried blindly. Anyone who has read our task pages will recognise why that matters: [duplicate side effects](/tasks/cancel-a-subscription/) are the failure that a naive retry produces, and a lease is one of the few honest answers to it.",
      "after": "Interrupted work recovers through SQL leases: database locks that let a worker claim a task, then expire so another worker can recover it after a crash. Anyone who has read our task pages will recognise why that matters: [duplicate side effects](/tasks/cancel-a-subscription/) are the failure that a naive retry produces, and a lease is one of the few honest answers to it.",
      "why": "Independent audit follow-up; existing page sources and docs/fact-reference.md",
      "kind": "calibration"
    },
    {
      "id": "2026-10-12-round2-editorial-34",
      "date": "2026-10-12",
      "title": "book-a-table: audit follow-up and source context",
      "pages": [
        "/tasks/book-a-table/"
      ],
      "before": "This context was not included.",
      "after": "For restaurant operators, [OpenTable’s voice AI directory](https://www.opentable.com/restaurant-solutions/products/reservation-management/voice-ai/) lists Loman for booking, managing and cancelling reservations, Slang AI for phone questions and reservations, and PolyAI for guest responses and booking management. OpenTable supplies the reservation integration; these partners answer the venue’s incoming calls. They serve the restaurant, rather than acting as a diner’s assistant across unrelated venues.",
      "why": "Independent audit follow-up; https://www.opentable.com/restaurant-solutions/products/reservation-management/voice-ai/",
      "kind": "calibration"
    },
    {
      "id": "2026-10-12-round2-editorial-35",
      "date": "2026-10-12",
      "title": "book-a-table: audit follow-up and source context",
      "pages": [
        "/tasks/book-a-table/"
      ],
      "before": "An agent that submits a booking, loses the confirmation page to a slow response, and retries has not made one reservation. It has made two. The same happens when an agent falls back between routes: every world in our exam is reachable through a web page and through an API, and both read and write the same underlying record, so a second attempt through the other door is the same table, booked again.",
      "after": "After a booking succeeds, a lost confirmation page can cause an agent to retry. Switching from the web page to the API does not prevent duplication: both write the same underlying record.",
      "why": "Independent audit follow-up; existing page sources and docs/fact-reference.md",
      "kind": "calibration"
    },
    {
      "id": "2026-10-12-round2-editorial-36",
      "date": "2026-10-12",
      "title": "cancel-a-subscription: audit follow-up and source context",
      "pages": [
        "/tasks/cancel-a-subscription/"
      ],
      "before": "| App | What its own page says it does |\n| --- | --- |\n| Rocket Money | Finds recurring charges from linked accounts. Its [help center](https://help.rocketmoney.com/en/articles/934402-how-do-i-cancel-a-subscription-on-rocket-money) says Premium members can tap \"Cancel This For Me\" and its team contacts the provider, usually within a few business days; if no Cancel option appears, it cannot yet cancel that one. Premium is a [monthly price you set with a slider](https://help.rocketmoney.com/en/articles/9095189-managing-my-premium-membership); Privacy.com's guide puts the range at $4 to $12 |\n| [Hiatus](https://hiatusapp.com/) | Finds recurring expenses and \"helps you track your subscriptions and cancel what you no longer need\"; a concierge team negotiates bills. Privacy.com's guide says cancelling for you needs its Premium plan, from $9.99 a month |\n| [Subscription Stopper](https://www.subscriptionstopper.com/) | Free; links your cards to find subscriptions and alerts you to new charges. Privacy.com's guide says it does not cancel for you but gives instructions |\n| Trim, now [OneMain MyMoney](https://www.asktrim.com/) | Analyses your transactions to find recurring subscriptions |\n| Privacy.com | Virtual cards; its [own guide](https://www.privacy.com/blog/app-that-cancels-subscriptions) says you can pause or close the card a subscription uses, blocking charges while you deal with the cancellation. The same guide says PocketGuard and Bobby track but do not cancel |\n| Apple and Google Play | Cancel what they bill, from [Apple's Subscriptions settings](https://support.apple.com/en-us/118428) or [Google Play's Payments and subscriptions](https://support.google.com/googleplay/answer/7018481?hl=en). Rocket Money says subscriptions billed through Apple, Google Play, Amazon, Roku or PayPal must be cancelled there |",
      "after": "| App or billing platform | What it can do | Price context |\n| --- | --- | --- |\n| Rocket Money | Linked-account tracking; its [concierge](https://help.rocketmoney.com/en/articles/934402-how-do-i-cancel-a-subscription-on-rocket-money) contacts supported providers | Premium: [slider-selected monthly price](https://help.rocketmoney.com/en/articles/9095189-managing-my-premium-membership); $4–$12 in Privacy.com’s older guide |\n| [Hiatus](https://hiatusapp.com/) | Subscription tracking and concierge bill negotiation | Premium from $9.99/month in that older guide; current quote not verified |\n| [Subscription Stopper](https://www.subscriptionstopper.com/) | Card-linked tracking and cancellation instructions | Free, per its site |\n| [OneMain MyMoney, formerly Trim](https://www.asktrim.com/) | Recurring-charge detection | Current fee not verified |\n| [Privacy.com](https://www.privacy.com/blog/app-that-cancels-subscriptions) | Virtual-card blocking; blocking payment does not cancel the contract | Current plan fee not verified |\n| [Apple subscriptions](https://support.apple.com/en-us/118428) / [Google Play](https://support.google.com/googleplay/answer/7018481?hl=en) | Cancel subscriptions billed by the respective store | Uses the existing billing account; subscription terms still apply |\n\nOn Android, use Google Play’s Payments and subscriptions for Play-billed services. Detecting a charge in another tracker does not transfer cancellation control to that tracker.",
      "why": "Independent audit follow-up; https://help.rocketmoney.com/en/articles/934402-how-do-i-cancel-a-subscription-on-rocket-money; https://help.rocketmoney.com/en/articles/9095189-managing-my-premium-membership; https://hiatusapp.com/; https://www.subscriptionstopper.com/; https://www.asktrim.com/; https://www.privacy.com/blog/app-that-cancels-subscriptions; https://support.apple.com/en-us/118428; https://support.google.com/googleplay/answer/7018481?hl=en",
      "kind": "calibration"
    },
    {
      "id": "2026-10-12-round2-editorial-37",
      "date": "2026-10-12",
      "title": "cancel-a-subscription: audit follow-up and source context",
      "pages": [
        "/tasks/cancel-a-subscription/"
      ],
      "before": "**When will there be scores for this task?**\nPartly, already. The scorecard shows what has been measured so far, and one recorded attempt at this errand is now published and linked above. The rest will arrive the same way, a page at a time, rather than being summarised into a verdict.",
      "after": "One recorded attempt is already published and linked above. Additional scores require a completed run with a retained action record, an inspected final state and a checked outcome. Publication follows those checks; there is no announced date for the next product’s run.",
      "why": "Independent audit follow-up; existing page sources and docs/fact-reference.md",
      "kind": "calibration"
    },
    {
      "id": "2026-10-12-round2-editorial-38",
      "date": "2026-10-12",
      "title": "dispute-a-charge: audit follow-up and source context",
      "pages": [
        "/tasks/dispute-a-charge/"
      ],
      "before": "1. If you did not make the charge, contact the card issuer first, not the merchant. The [FTC](https://consumer.ftc.gov/articles/lost-or-stolen-credit-atm-and-debit-cards) says to call or use the issuer's app to report unauthorized charges right away and then follow up in writing, and the [CFPB](https://www.consumerfinance.gov/ask-cfpb/how-do-i-dispute-a-charge-on-my-credit-card-bill-en-61/) says to call the card company about the problem right away. On a credit card your liability for unauthorized use is capped at $50 ([12 CFR 1026.12(b)](https://www.ecfr.gov/current/title-12/chapter-X/part-1026/subpart-B/section-1026.12)), and the FTC says you owe nothing if only your card number was used and the card itself was not lost or stolen. An unrecognised charge can also be a sign of identity theft, which the FTC handles at [IdentityTheft.gov](https://www.identitytheft.gov/).\n2. If it is a purchase you made, contact the merchant first and keep records: the order number, the date, the amount, and copies of what you sent and received. For a complaint about the quality of something you bought on a credit card, federal law requires a good-faith attempt to resolve it with the seller first, and the purchase must be more than $50 and made in your home state or within 100 miles of your mailing address ([15 U.S.C. 1666i](https://www.law.cornell.edu/uscode/text/15/1666i), [12 CFR 1026.12(c)](https://www.ecfr.gov/current/title-12/chapter-X/part-1026/subpart-B/section-1026.12)). The amount and distance limits do not apply when the seller is also the card issuer or is tied to it.\n3. If that fails, dispute it with your card issuer. A credit card billing error includes a charge you did not authorize and goods [not delivered as agreed](https://www.consumerfinance.gov/rules-policy/regulations/1026/13/), and the Fair Credit Billing Act requires your written notice to reach the issuer's billing-inquiries address within 60 days after it sent the first statement showing the error. Use that address, not the payment address.\n4. The issuer then has 30 days to acknowledge your notice in writing and must resolve it within two billing cycles, and in no case later than 90 days ([Regulation Z, 12 CFR 1026.13](https://www.consumerfinance.gov/rules-policy/regulations/1026/13/)).\n5. For a debit card or other electronic transfer, Regulation E applies instead. Report an error within 60 days after the bank sends the statement it first appears on; the bank has 10 business days to investigate, or up to 45 days if it provisionally credits your account ([12 CFR 1005.11](https://www.consumerfinance.gov/rules-policy/regulations/1005/11/)). For a lost or stolen card, telling the bank within two business days of learning of it caps your liability at $50, and later notice raises the cap to $500 ([12 CFR 1005.6](https://www.consumerfinance.gov/rules-policy/regulations/1005/6/)). Keep written records of everything; the [FTC](https://consumer.ftc.gov/articles/using-credit-cards-and-disputing-charges) suggests certified mail with a return receipt, so you have proof of what the issuer got.",
      "after": "1. If you did not make the charge, contact the card issuer first, not the merchant. The [FTC](https://consumer.ftc.gov/articles/lost-or-stolen-credit-atm-and-debit-cards) says to call or use the issuer's app to report unauthorized charges right away and then follow up in writing, and the [CFPB](https://www.consumerfinance.gov/ask-cfpb/how-do-i-dispute-a-charge-on-my-credit-card-bill-en-61/) says to call the card company about the problem right away. On a credit card your liability for unauthorized use is capped at $50 ([12 CFR 1026.12(b)](https://www.ecfr.gov/current/title-12/chapter-X/part-1026/subpart-B/section-1026.12)), and the FTC says you owe nothing if only your card number was used and the card itself was not lost or stolen. An unrecognised charge can also be a sign of identity theft, which the FTC handles at [IdentityTheft.gov](https://www.identitytheft.gov/).\n2. If it is a purchase you made, contact the merchant first and keep records: the order number, the date, the amount, and copies of what you sent and received. For a complaint about the quality of something you bought on a credit card, federal law requires a good-faith attempt to resolve it with the seller first, and the purchase must be more than $50 and made in your home state or within 100 miles of your mailing address ([15 U.S.C. 1666i](https://www.law.cornell.edu/uscode/text/15/1666i), [12 CFR 1026.12(c)](https://www.ecfr.gov/current/title-12/chapter-X/part-1026/subpart-B/section-1026.12)). The amount and distance limits do not apply when the seller is also the card issuer or is tied to it.\n3. If that fails, dispute it with your card issuer. A credit card billing error includes a charge you did not authorize and goods [not delivered as agreed](https://www.consumerfinance.gov/rules-policy/regulations/1026/13/), and the Fair Credit Billing Act requires your written notice to reach the issuer's billing-inquiries address within 60 days after it sent the first statement showing the error. Use that address, not the payment address.\n4. The issuer then has 30 days to acknowledge your notice in writing and must resolve it within two billing cycles, and in no case later than 90 days ([Regulation Z, 12 CFR 1026.13](https://www.consumerfinance.gov/rules-policy/regulations/1026/13/)).\n5. For a debit card or other electronic transfer, Regulation E applies instead. Report an error within 60 days after the bank sends the statement it first appears on; the bank has 10 business days to investigate, or up to 45 days if it provisionally credits your account ([12 CFR 1005.11](https://www.consumerfinance.gov/rules-policy/regulations/1005/11/)). For a lost or stolen card, telling the bank within two business days of learning of it caps your liability at $50, and later notice raises the cap to $500 ([12 CFR 1005.6](https://www.consumerfinance.gov/rules-policy/regulations/1005/6/)). If the bank reverses provisional credit, [section 1005.11(d)](https://www.consumerfinance.gov/rules-policy/regulations/1005/11/#d) requires notice of the debit’s date and amount, a written explanation of the findings and access to the documents relied on; it also protects specified payments for five business days after notice. Keep records; the [FTC](https://consumer.ftc.gov/articles/using-credit-cards-and-disputing-charges) suggests certified mail with a return receipt, so you have proof of what the issuer got.",
      "why": "Independent audit follow-up; https://consumer.ftc.gov/articles/lost-or-stolen-credit-atm-and-debit-cards; https://www.consumerfinance.gov/ask-cfpb/how-do-i-dispute-a-charge-on-my-credit-card-bill-en-61/; https://www.ecfr.gov/current/title-12/chapter-X/part-1026/subpart-B/section-1026.12; https://www.identitytheft.gov/; https://www.law.cornell.edu/uscode/text/15/1666i; https://www.consumerfinance.gov/rules-policy/regulations/1026/13/; https://www.consumerfinance.gov/rules-policy/regulations/1005/11/; https://www.consumerfinance.gov/rules-policy/regulations/1005/6/; https://www.consumerfinance.gov/rules-policy/regulations/1005/11/#d; https://consumer.ftc.gov/articles/using-credit-cards-and-disputing-charges",
      "kind": "calibration"
    },
    {
      "id": "2026-10-12-round2-editorial-39",
      "date": "2026-10-12",
      "title": "dispute-a-charge: audit follow-up and source context",
      "pages": [
        "/tasks/dispute-a-charge/"
      ],
      "before": "This context was not included.",
      "after": "Use the [FTC’s sample dispute letter](https://consumer.ftc.gov/articles/sample-letter-disputing-credit-and-debit-card-charges) to organise a written notice. If the issuer does not follow the applicable dispute procedure, the [CFPB complaint portal](https://www.consumerfinance.gov/complaint/) is an escalation route. Keep the original notice, delivery record and the issuer’s response.",
      "why": "Independent audit follow-up; https://consumer.ftc.gov/articles/sample-letter-disputing-credit-and-debit-card-charges; https://www.consumerfinance.gov/complaint/",
      "kind": "calibration"
    },
    {
      "id": "2026-10-12-round2-editorial-40",
      "date": "2026-10-12",
      "title": "dispute-a-charge: audit follow-up and source context",
      "pages": [
        "/tasks/dispute-a-charge/"
      ],
      "before": "An agent that emails the merchant for a refund, does not see a reply quickly enough, and then files a chargeback has started two independent processes against the same charge. If both land, you have been paid twice for one purchase. The merchant sees a customer who took a refund and then disputed it anyway, the bank sees a claim that the merchant can now disprove, and the cleanup is yours.",
      "after": "If an agent requests a merchant refund and then files a chargeback before a reply, it starts two recovery processes for one charge. If both succeed, the duplicate recovery has to be corrected.",
      "why": "Independent audit follow-up; existing page sources and docs/fact-reference.md",
      "kind": "calibration"
    },
    {
      "id": "2026-10-12-round2-editorial-41",
      "date": "2026-10-12",
      "title": "dispute-a-charge: audit follow-up and source context",
      "pages": [
        "/tasks/dispute-a-charge/"
      ],
      "before": "**What happens if an agent files a dispute twice?**\nYou can end up recovering the same money through two routes, which is a problem rather than a bonus, and untangling it falls to you. Our exam counts duplicate side effects as a separate measurement from whether the goal was reached, precisely because a run can look successful and still leave that mess behind.",
      "after": "**What happens if an agent files a dispute twice?**\nDuplicate recovery can require repayment and investigation. Our exam reports duplicate side effects separately from goal completion.",
      "why": "Independent audit follow-up; existing page sources and docs/fact-reference.md",
      "kind": "calibration"
    },
    {
      "id": "2026-10-12-round2-editorial-42",
      "date": "2026-10-12",
      "title": "family-scheduling: audit follow-up and source context",
      "pages": [
        "/tasks/family-scheduling/"
      ],
      "before": "| App | What its own page says |\n| --- | --- |\n| [Cozi](https://www.cozi.com/) | A shared calendar colour-coded by person, shared lists and meal planning, free; the Max plan (about $80 a year) adds \"AI Event Import\", turning emails into events |\n| [Ohai](https://www.ohai.ai/) | Scans emails, PDFs, photos and documents such as school newsletters or team schedules, pulls out dates and deadlines, and adds them to a shared family calendar with reminders and tasks; premium plans from $9.99 a month |\n| [AVA](https://hiava.app/) | Reads school emails, sports schedules and calendars into the family calendar, researches camps and childcare, and drafts emails and proposes times for you to approve, edit or decline. Carpool and playdate coordination \"across families\" are listed as coming soon |\n| [Peggy](https://www.peggyapp.com/) | Reads school emails from senders you approve in Gmail, pulls out dates, and lets you assign tasks to a partner or caregiver, then reminds them |\n| [TimeTree](https://timetreeapp.com/intl/en) | Shared calendars for any group, free; Premium is 4.49 USD a month |\n| [FamilyWall](https://www.familywall.com/) | Shared calendar, lists, meal planning, location sharing and a family messenger |\n\nNone of these pages says the app messages people outside your household for you today; AVA lists that as coming soon. The calendar is a display surface with write access. The rearranging still happens in a group chat, on a doorstep, or in the ten minutes you spend working out who can leave work early. The calendar records the outcome of that negotiation. It does not conduct it.",
      "after": "| App | Suitable use from its stated features | Price context |\n| --- | --- | --- |\n| [Cozi](https://www.cozi.com/) | Shared calendar, lists and meals; Max adds AI event imports | Free; Max about $80/year |\n| [Ohai](https://www.ohai.ai/) | Extract dates and tasks from emails, PDFs and photos | Premium from $9.99/month |\n| [AVA](https://hiava.app/) | School schedules, childcare research and approval-based drafts; cross-family coordination coming soon | No amount exposed in checked homepage |\n| [Peggy](https://www.peggyapp.com/) | Approved school emails, date extraction and caregiver task assignment | No amount exposed in checked homepage |\n| [TimeTree](https://timetreeapp.com/intl/en) | Shared group calendars | Free; Premium $4.49/month |\n| [FamilyWall](https://www.familywall.com/) | Calendar, lists, meals, location and family messaging | Current paid amount not verified |\n\nFamilymind is outside this shortlist because its current capabilities and pricing were not independently checked for this update.\n\nNone of these pages says the app messages people outside your household for you today; AVA lists that as coming soon. The calendar records the outcome. The rearranging still happens in a group chat or a conversation about who can leave work early.",
      "why": "Independent audit follow-up; https://www.cozi.com/; https://www.ohai.ai/; https://hiava.app/; https://www.peggyapp.com/; https://timetreeapp.com/intl/en; https://www.familywall.com/",
      "kind": "calibration"
    },
    {
      "id": "2026-10-12-round2-editorial-43",
      "date": "2026-10-12",
      "title": "flight-delay-compensation: audit follow-up and source context",
      "pages": [
        "/tasks/flight-delay-compensation/"
      ],
      "before": "1. **US flights.** There is no federal requirement to pay cash for a delay. The Department of Transportation withdrew its proposal to create one in [November 2025](https://www.federalregister.gov/documents/2025/11/17/2025-20042/airline-passenger-rights-withdrawal), concluding that the statute does not authorize it to require compensation. What the law does give you is a refund: under [14 CFR 260.6](https://www.ecfr.gov/current/title-14/chapter-II/subchapter-A/part-260/section-260.6), on a flight to, from or within the US, the airline must refund the fare, taxes and fees if the flight is cancelled or significantly delayed and you decline the delayed flight, any rebooking, and any voucher or credit. \"Significantly delayed\" includes arriving three hours or more later than scheduled on a domestic itinerary, or six hours or more on an international one; an earlier departure, a different airport or an extra connection also count ([14 CFR 260.2](https://www.ecfr.gov/current/title-14/chapter-II/subchapter-A/part-260/section-260.2)). The refund is due within 7 business days for a credit card purchase and 20 calendar days otherwise.\n2. **What US airlines promise on their own.** Meals, hotels and any compensation for delays the airline caused are commitments in each airline's customer service plan, listed on DOT's [delay and cancellation dashboard](https://www.flightrights.gov/). In its withdrawal notice DOT said all ten of the largest US airlines guarantee a meal and free rebooking for controllable delays, and that four commit to some compensation, in credits, vouchers or miles rather than cash.\n3. **EU rules (Regulation 261/2004).** They cover flights within the EU, flights leaving the EU, and flights into the EU on an EU airline. If you reach your final destination three hours or more late, the [EU's Your Europe page](https://europa.eu/youreurope/citizens/travel/passenger-rights/air/index_en.htm) lists €250 for flights of 1,500 km or less, €400 for longer flights within the EU and other flights of 1,500 to 3,500 km, and €600 for flights over 3,500 km (€300 if that long-haul delay is between three and four hours). Nothing is due if the airline proves extraordinary circumstances, such as weather, air traffic management decisions or security risks.\n4. **Canada (APPR).** For a delay within the airline's control and not required for safety, [section 19 of the Air Passenger Protection Regulations](https://laws-lois.justice.gc.ca/eng/regulations/SOR-2019-150/FullText.html) sets minimums by arrival delay: for a large carrier C$400 (three to six hours), C$700 (six to nine) and C$1,000 (nine or more); for a small carrier C$125, C$250 and C$500. You must file with the airline within one year, and it has 30 days to pay or explain why not.\n5. **Claim services cost money.** AirHelp's [published price list](https://www.airhelp.com/en/price-list/), for one, takes 35% of the compensation if it wins, plus a further 15% when legal action is needed.",
      "after": "1. **US flights.** There is no federal requirement to pay cash for a delay. The Department of Transportation withdrew its proposal to create one in [November 2025](https://www.federalregister.gov/documents/2025/11/17/2025-20042/airline-passenger-rights-withdrawal), concluding that the statute does not authorize it to require compensation. What the law does give you is a refund: under [14 CFR 260.6](https://www.ecfr.gov/current/title-14/chapter-II/subchapter-A/part-260/section-260.6), on a flight to, from or within the US, the airline must refund the fare, taxes and fees if the flight is cancelled or significantly delayed and you decline the delayed flight, any rebooking, and any voucher or credit. \"Significantly delayed\" includes arriving three hours or more later than scheduled on a domestic itinerary, or six hours or more on an international one; an earlier departure, a different airport or an extra connection also count ([14 CFR 260.2](https://www.ecfr.gov/current/title-14/chapter-II/subchapter-A/part-260/section-260.2)). The refund is due within 7 business days for a credit card purchase and 20 calendar days otherwise.\n2. **What US airlines promise on their own.** Meals, hotels and any compensation for delays the airline caused are commitments in each airline's customer service plan, listed on DOT's [delay and cancellation dashboard](https://www.flightrights.gov/). In its withdrawal notice DOT said all ten of the largest US airlines guarantee a meal and free rebooking for controllable delays, and that four commit to non-cash compensation. DOT’s [carrier-by-carrier text](https://www.transportation.gov/resources/individuals/aviation-consumer-protection/airline-cancellation-delay-dashboard-html) names Alaska and Hawaiian for credits/vouchers and miles, and JetBlue and Southwest for credits/vouchers, for covered controllable delays of at least three hours.\n3. **EU rules (Regulation 261/2004).** They cover flights within the EU, flights leaving the EU, and flights into the EU on an EU airline. If you reach your final destination three hours or more late, the [EU's Your Europe page](https://europa.eu/youreurope/citizens/travel/passenger-rights/air/index_en.htm) lists €250 for flights of 1,500 km or less, €400 for longer flights within the EU and other flights of 1,500 to 3,500 km, and €600 for flights over 3,500 km (€300 if that long-haul delay is between three and four hours). Nothing is due if the airline proves extraordinary circumstances, such as weather, air traffic management decisions or security risks. For UK-covered flights, the [CAA’s UK261 guidance](https://www.caa.co.uk/air-passengers/travel-problems-and-rights/flight-delays-and-cancellations/delays/) explains assistance and possible compensation for arrival delays over three hours; eligibility depends on the route and cause.\n4. **Canada (APPR).** For a delay within the airline's control and not required for safety, [section 19 of the Air Passenger Protection Regulations](https://laws-lois.justice.gc.ca/eng/regulations/SOR-2019-150/FullText.html) sets minimums by arrival delay: for a large carrier C$400 (three to six hours), C$700 (six to nine) and C$1,000 (nine or more); for a small carrier C$125, C$250 and C$500. You must file with the airline within one year, and it has 30 days to pay or explain why not.\n5. **Claim services cost money.** AirHelp's [published price list](https://www.airhelp.com/en/price-list/), for one, takes 35% of the compensation if it wins, plus a further 15% when legal action is needed.",
      "why": "Independent audit follow-up; https://www.federalregister.gov/documents/2025/11/17/2025-20042/airline-passenger-rights-withdrawal; https://www.ecfr.gov/current/title-14/chapter-II/subchapter-A/part-260/section-260.6; https://www.ecfr.gov/current/title-14/chapter-II/subchapter-A/part-260/section-260.2; https://www.flightrights.gov/; https://www.transportation.gov/resources/individuals/aviation-consumer-protection/airline-cancellation-delay-dashboard-html; https://europa.eu/youreurope/citizens/travel/passenger-rights/air/index_en.htm; https://www.caa.co.uk/air-passengers/travel-problems-and-rights/flight-delays-and-cancellations/delays/; https://laws-lois.justice.gc.ca/eng/regulations/SOR-2019-150/FullText.html; https://www.airhelp.com/en/price-list/",
      "kind": "calibration"
    },
    {
      "id": "2026-10-12-round2-editorial-44",
      "date": "2026-10-12",
      "title": "flight-delay-compensation: audit follow-up and source context",
      "pages": [
        "/tasks/flight-delay-compensation/"
      ],
      "before": "**Does the bench test real airlines?**\nNo, and deliberately so. The tasks run in simulated worlds that can be reset and replayed, so two runs are comparable and anybody can check the result. Testing against live airline systems would mean a task that changes underneath you and results nobody could reproduce.",
      "after": "**Does the bench test real airlines?**\nNo. Simulated worlds can be reset and replayed; live airline systems would change between attempts and make controlled comparisons impossible.",
      "why": "Independent audit follow-up; existing page sources and docs/fact-reference.md",
      "kind": "calibration"
    },
    {
      "id": "2026-10-12-round2-editorial-45",
      "date": "2026-10-12",
      "title": "live-alerts: audit follow-up and source context",
      "pages": [
        "/tasks/live-alerts/"
      ],
      "before": "This context was not included.",
      "after": "[Microsoft’s 2025 telemetry report](https://www.microsoft.com/en-us/worklab/work-trend-index/breaking-down-infinite-workday) counted 275 daily pings for the top 20% of users by received volume, across 24 hours, not for the average worker. [Slack](https://slack.com/help/articles/214908388-Pause-your-Slack-notifications) lets you pause or schedule alerts; [Teams](https://support.microsoft.com/en-us/teams/notifications-settings/manage-notifications-in-microsoft-teams) offers notification and activity controls. Those settings reduce noise without establishing an agent’s judgment.",
      "why": "Independent audit follow-up; https://www.microsoft.com/en-us/worklab/work-trend-index/breaking-down-infinite-workday; https://slack.com/help/articles/214908388-Pause-your-Slack-notifications; https://support.microsoft.com/en-us/teams/notifications-settings/manage-notifications-in-microsoft-teams",
      "kind": "calibration"
    },
    {
      "id": "2026-10-12-round2-editorial-46",
      "date": "2026-10-12",
      "title": "live-alerts: audit follow-up and source context",
      "pages": [
        "/tasks/live-alerts/"
      ],
      "before": "An agent that can wake itself up is the only kind that can alert you, and the only kind that can bother you unprompted: the same feature described twice. Every other task asks whether an agent can finish something you told it to do. This one asks whether it can decide, unprompted, that you should be pulled away from what you are doing.\n\nThere is a security shape to it as well. An agent that can be started by an inbox is an agent a stranger can address, which makes the check between something happened and act on it more consequential than any feature on a comparison table. The [teardowns](/agents/) cover which surfaces can start each product without you typing, and what sits between the trigger and the action.",
      "after": "An agent that starts unprompted can both help and interrupt you. This task asks whether it can distinguish those cases.\n\nAn inbox-triggered agent can receive a stranger’s input. The [teardowns](/agents/) distinguish each product’s triggers and the controls between a trigger and an action.",
      "why": "Independent audit follow-up; existing page sources and docs/fact-reference.md",
      "kind": "calibration"
    },
    {
      "id": "2026-10-12-round2-editorial-47",
      "date": "2026-10-12",
      "title": "work-files: audit follow-up and source context",
      "pages": [
        "/tasks/work-files/"
      ],
      "before": "| Tool | What its own page says |\n| --- | --- |\n| [AI File Sorter](https://github.com/hyperfield/ai-file-sorter) | Open source desktop app for Windows, macOS and Linux. Suggests categories from names, types and folder context, analyses image and document content, proposes clearer names, can run entirely on your device, and lets you review every change before it applies, with a dry run and a best-effort undo |\n| [Local File Organizer](https://github.com/QiuYannnn/Local-File-Organizer) | Python tool that reads text and images with local models, sorts by content, date or type, and has a dry run mode; its README says no data leaves your computer |\n| [Sparkle](https://makeitsparkle.co/) | Mac app that builds folders from patterns in your work and can run on a schedule; it says everything goes to the trash first, and plans start at $9.25 a month |\n| [Sortio](https://www.getsortio.com/) | Mac and Windows app that reads each file, renames it from its contents and routes it to a client or project folder, creating folders as needed; you approve the plan before anything moves and can undo any change. Free to start, Pro from $14.99 a month |\n| [The Drive AI](https://thedrive.ai/ai-file-organizer) | Works on Google Drive, SharePoint, OneDrive, Dropbox or a Mac folder; shows every move and rename before anything changes, leaves files it cannot place where they are, and can undo |\n| [Renamer.ai](https://renamer.ai/ai-document-organizer) | Renames files from their contents so search finds them, rather than building folders; it says it does not plug into cloud drives |\n\nNotice what the careful ones share: a preview before anything moves, and a way back. That is the bar an agent should meet too. We have not tested these tools ourselves; the descriptions are theirs.",
      "after": "| Tool | What its own page says |\n| --- | --- |\n| [AI File Sorter](https://github.com/hyperfield/ai-file-sorter) | AGPL-3.0 source; desktop builds for Windows, macOS and Linux. Suggests categories from names, types and folder context, analyses image and document content, proposes clearer names, can run entirely on your device, and lets you review every change before it applies, with a dry run and a best-effort undo |\n| [Local File Organizer](https://github.com/QiuYannnn/Local-File-Organizer) | Open source Python tool (MIT license file) that reads text and images with local models, sorts by content, date or type, and has a dry run mode; its README says no data leaves your computer |\n| [Sparkle](https://makeitsparkle.co/) | Mac app that builds folders from patterns in your work and can run on a schedule; it says everything goes to the trash first, and plans start at $9.25 a month |\n| [Sortio](https://www.getsortio.com/) | Mac and Windows app that reads each file, renames it from its contents and routes it to a client or project folder, creating folders as needed; you approve the plan before anything moves and can undo any change. Free to start, Pro from $14.99 a month |\n| [The Drive AI](https://thedrive.ai/ai-file-organizer) | Current price not verified. Works on Google Drive, SharePoint, OneDrive, Dropbox or a Mac folder; shows every move and rename before anything changes, leaves files it cannot place where they are, and can undo |\n| [Renamer.ai](https://renamer.ai/ai-document-organizer) | Renames files from their contents so search finds them, rather than building folders; it says it does not plug into cloud drives |\n\nThese are vendor descriptions, not our tests. Look for a preview and undo. AI File Sorter provides a native Windows desktop option in this list.",
      "why": "Independent audit follow-up; https://github.com/hyperfield/ai-file-sorter; https://github.com/QiuYannnn/Local-File-Organizer; https://makeitsparkle.co/; https://www.getsortio.com/; https://thedrive.ai/ai-file-organizer; https://renamer.ai/ai-document-organizer",
      "kind": "calibration"
    },
    {
      "id": "2026-10-12-round2-editorial-48",
      "date": "2026-10-12",
      "title": "where-it-runs: audit follow-up and source context",
      "pages": [
        "/where-it-runs/"
      ],
      "before": "This context was not included.",
      "after": "For work that must continue overnight, a hosted service or your own always-on server is the starting point. A local agent needs the computer awake for local work. Self-hosting also makes you responsible for updates, credentials and recovery; the [hosting section below](#hosting-a-personal-ai-agent-yourself-what-the-vendors-say-it-takes) lists the vendors’ published requirements and costs.",
      "why": "Independent audit follow-up; existing page sources and docs/fact-reference.md",
      "kind": "calibration"
    },
    {
      "id": "2026-10-12-round2-editorial-49",
      "date": "2026-10-12",
      "title": "where-it-runs: audit follow-up and source context",
      "pages": [
        "/where-it-runs/"
      ],
      "before": "Pickle says it books, orders and pays on a computer of its own, asking before each one. [MimiClaw says](https://github.com/memovai/mimiclaw) it runs on a five dollar chip with no operating system, no Node.js and no Mac mini, though by its own README each request still goes to Anthropic's or OpenAI's models. [Halo says](https://meethalo.app/) it runs entirely on the phone with no backend at all, connecting to mail, calendar, reminders and health locally and calling models through your own keys. [OpenClaw](https://docs.openclaw.ai/) and [Hermes Agent](https://hermes-agent.nousresearch.com/) both say you run them on your own hardware or a managed host.",
      "after": "[Pickle says](https://pickle.com/) it books, orders and pays on a computer of its own, asking before each one. [MimiClaw says](https://github.com/memovai/mimiclaw) it runs on a five dollar chip with no operating system, no Node.js and no Mac mini, though by its own README each request still goes to Anthropic's or OpenAI's models. [Halo says](https://meethalo.app/) it runs entirely on the phone with no backend at all, connecting to mail, calendar, reminders and health locally and calling models through your own keys. [OpenClaw](https://docs.openclaw.ai/) and [Hermes Agent](https://hermes-agent.nousresearch.com/) both say you run them on your own hardware or a managed host.",
      "why": "Independent audit follow-up; https://pickle.com/; https://github.com/memovai/mimiclaw; https://meethalo.app/; https://docs.openclaw.ai/; https://hermes-agent.nousresearch.com/",
      "kind": "calibration"
    },
    {
      "id": "2026-10-12-r3-001",
      "date": "2026-10-12",
      "title": "Round 3: task-notes evidence correction 1",
      "pages": [
        "/tasks/family-scheduling/",
        "/tasks/cancel-a-subscription/",
        "/tasks/dispute-a-charge/",
        "/tasks/live-alerts/",
        "/tasks/flight-delay-compensation/",
        "/tasks/book-a-table/",
        "/tasks/work-files/"
      ],
      "before": "Phones the company for you. It keeps the merchant, the number, the duration and a summary, but no transcript, so what was actually agreed on the call is not something you can go back and read.",
      "after": "Observed business calling with start and identity confirmations. App call history kept a summary, not a verbatim transcript; developer API calls returned transcripts. Success on this particular errand is untested.",
      "why": "content/agents/pine.md and data/scorecard.json; scope limited to observed interfaces.",
      "kind": "calibration"
    },
    {
      "id": "2026-10-12-r3-002",
      "date": "2026-10-12",
      "title": "Round 3: task-notes evidence correction 2",
      "pages": [
        "/tasks/family-scheduling/",
        "/tasks/cancel-a-subscription/",
        "/tasks/dispute-a-charge/",
        "/tasks/live-alerts/",
        "/tasks/flight-delay-compensation/",
        "/tasks/book-a-table/",
        "/tasks/work-files/"
      ],
      "before": "Its vault fills a password without being able to read it back, so it can complete a login you never hand over. It lives in a message thread, so there is no confirmation screen between it and the action.",
      "after": "The observed messaging interface has an approve, deny or dismiss card bound to the exact plan text; a changed plan needs new approval. Its vault supports fill-only credentials. This does not establish success on this errand.",
      "why": "content/agents/instinct.md and data/scorecard.json; scope limited to observed interfaces.",
      "kind": "calibration"
    },
    {
      "id": "2026-10-12-r3-003",
      "date": "2026-10-12",
      "title": "Round 3: task-notes evidence correction 3",
      "pages": [
        "/tasks/family-scheduling/",
        "/tasks/cancel-a-subscription/",
        "/tasks/dispute-a-charge/",
        "/tasks/live-alerts/",
        "/tasks/flight-delay-compensation/",
        "/tasks/book-a-table/",
        "/tasks/work-files/"
      ],
      "before": "Assembles the request on a Linux machine that is yours, and keeps memory as files you can open. Its approval card puts a spend or a send in front of you before it happens.",
      "after": "The examined web runtime uses a per-user Linux VM and markdown memory. Personal access to those files was not verified. An approvals interface was observed; the Mac route and this specific errand remain untested.",
      "why": "content/agents/muse.md and data/scorecard.json; scope limited to observed interfaces.",
      "kind": "calibration"
    },
    {
      "id": "2026-10-12-r3-004",
      "date": "2026-10-12",
      "title": "Round 3: task-notes evidence correction 4",
      "pages": [
        "/tasks/family-scheduling/",
        "/tasks/cancel-a-subscription/",
        "/tasks/dispute-a-charge/",
        "/tasks/live-alerts/",
        "/tasks/flight-delay-compensation/",
        "/tasks/book-a-table/",
        "/tasks/work-files/"
      ],
      "before": "Web and a native Mac app with real system access, which is the broadest local reach on this bench. It shows its assistant roughly 177 of the 1105 tools it lists.",
      "after": "The examined native Mac route reaches messages, contacts, screen and audio. The assistant saw 177 of 1105 catalogue entries. Available actions depend on account configuration and permissions; this is not a task-success result.",
      "why": "content/agents/town.md and data/scorecard.json; scope limited to observed interfaces.",
      "kind": "calibration"
    },
    {
      "id": "2026-10-12-r3-005",
      "date": "2026-10-12",
      "title": "Round 3: task-notes evidence correction 5",
      "pages": [
        "/tasks/family-scheduling/",
        "/tasks/cancel-a-subscription/",
        "/tasks/dispute-a-charge/",
        "/tasks/live-alerts/",
        "/tasks/flight-delay-compensation/",
        "/tasks/book-a-table/",
        "/tasks/work-files/"
      ],
      "before": "Phones the company for you, which is the merchant route rather than the bank one. It keeps the merchant, the number, the duration and a summary, but no transcript, so what was actually agreed about the money is not something you can go back and read.",
      "after": "Observed business calling with start and identity confirmations. App call history kept a summary, not a verbatim transcript; developer API calls returned transcripts. Success on this particular errand is untested.",
      "why": "content/agents/pine.md and data/scorecard.json; scope limited to observed interfaces.",
      "kind": "calibration"
    },
    {
      "id": "2026-10-12-r3-006",
      "date": "2026-10-12",
      "title": "Round 3: task-notes evidence correction 6",
      "pages": [
        "/tasks/family-scheduling/",
        "/tasks/cancel-a-subscription/",
        "/tasks/dispute-a-charge/",
        "/tasks/live-alerts/",
        "/tasks/flight-delay-compensation/",
        "/tasks/book-a-table/",
        "/tasks/work-files/"
      ],
      "before": "Its vault fills a password without being able to read it back, so it can reach an account portal you never hand over. It lives in a message thread, so there is no confirmation screen standing between it and a filed dispute.",
      "after": "The observed messaging interface has an approve, deny or dismiss card bound to the exact plan text; a changed plan needs new approval. Its vault supports fill-only credentials. This does not establish success on this errand.",
      "why": "content/agents/instinct.md and data/scorecard.json; scope limited to observed interfaces.",
      "kind": "calibration"
    },
    {
      "id": "2026-10-12-r3-007",
      "date": "2026-10-12",
      "title": "Round 3: task-notes evidence correction 7",
      "pages": [
        "/tasks/family-scheduling/",
        "/tasks/cancel-a-subscription/",
        "/tasks/dispute-a-charge/",
        "/tasks/live-alerts/",
        "/tasks/flight-delay-compensation/",
        "/tasks/book-a-table/",
        "/tasks/work-files/"
      ],
      "before": "Assembles the case on a Linux machine that is yours, and keeps memory as files you can open, which means the evidence it gathered is readable afterwards. Its approval card puts a spend or a send in front of you before it happens.",
      "after": "The examined web runtime uses a per-user Linux VM and markdown memory. Personal access to those files was not verified. An approvals interface was observed; the Mac route and this specific errand remain untested.",
      "why": "content/agents/muse.md and data/scorecard.json; scope limited to observed interfaces.",
      "kind": "calibration"
    },
    {
      "id": "2026-10-12-r3-008",
      "date": "2026-10-12",
      "title": "Round 3: task-notes evidence correction 8",
      "pages": [
        "/tasks/family-scheduling/",
        "/tasks/cancel-a-subscription/",
        "/tasks/dispute-a-charge/",
        "/tasks/live-alerts/",
        "/tasks/flight-delay-compensation/",
        "/tasks/book-a-table/",
        "/tasks/work-files/"
      ],
      "before": "Web and a native Mac app with real system access, the broadest local reach on this bench, so it can reach receipts and statements sitting on your own machine. It shows its assistant roughly 177 of the 1105 tools it lists.",
      "after": "The examined native Mac route reaches messages, contacts, screen and audio. The assistant saw 177 of 1105 catalogue entries. Available actions depend on account configuration and permissions; this is not a task-success result.",
      "why": "content/agents/town.md and data/scorecard.json; scope limited to observed interfaces.",
      "kind": "calibration"
    },
    {
      "id": "2026-10-12-r3-009",
      "date": "2026-10-12",
      "title": "Round 3: task-notes evidence correction 9",
      "pages": [
        "/tasks/family-scheduling/",
        "/tasks/cancel-a-subscription/",
        "/tasks/dispute-a-charge/",
        "/tasks/live-alerts/",
        "/tasks/flight-delay-compensation/",
        "/tasks/book-a-table/",
        "/tasks/work-files/"
      ],
      "before": "Phones the airline for you, which is the route most likely to produce an offer on the spot. It keeps the merchant, the number, the duration and a summary, but no transcript, so what was actually offered on the call is not something you can go back and read.",
      "after": "Observed business calling with start and identity confirmations. App call history kept a summary, not a verbatim transcript; developer API calls returned transcripts. Success on this particular errand is untested.",
      "why": "content/agents/pine.md and data/scorecard.json; scope limited to observed interfaces.",
      "kind": "calibration"
    },
    {
      "id": "2026-10-12-r3-010",
      "date": "2026-10-12",
      "title": "Round 3: task-notes evidence correction 10",
      "pages": [
        "/tasks/family-scheduling/",
        "/tasks/cancel-a-subscription/",
        "/tasks/dispute-a-charge/",
        "/tasks/live-alerts/",
        "/tasks/flight-delay-compensation/",
        "/tasks/book-a-table/",
        "/tasks/work-files/"
      ],
      "before": "Its vault fills a password without being able to read it back, so it can reach a booking portal you never hand over. It lives in a message thread, so there is no confirmation screen standing between it and an accepted offer.",
      "after": "The observed messaging interface has an approve, deny or dismiss card bound to the exact plan text; a changed plan needs new approval. Its vault supports fill-only credentials. This does not establish success on this errand.",
      "why": "content/agents/instinct.md and data/scorecard.json; scope limited to observed interfaces.",
      "kind": "calibration"
    },
    {
      "id": "2026-10-12-r3-011",
      "date": "2026-10-12",
      "title": "Round 3: task-notes evidence correction 11",
      "pages": [
        "/tasks/family-scheduling/",
        "/tasks/cancel-a-subscription/",
        "/tasks/dispute-a-charge/",
        "/tasks/live-alerts/",
        "/tasks/flight-delay-compensation/",
        "/tasks/book-a-table/",
        "/tasks/work-files/"
      ],
      "before": "Assembles the claim on a Linux machine that is yours, and keeps memory as files you can open, so the evidence it gathered stays readable. Its approval card puts a spend or a send in front of you before it happens.",
      "after": "The examined web runtime uses a per-user Linux VM and markdown memory. Personal access to those files was not verified. An approvals interface was observed; the Mac route and this specific errand remain untested.",
      "why": "content/agents/muse.md and data/scorecard.json; scope limited to observed interfaces.",
      "kind": "calibration"
    },
    {
      "id": "2026-10-12-r3-012",
      "date": "2026-10-12",
      "title": "Round 3: task-notes evidence correction 12",
      "pages": [
        "/tasks/family-scheduling/",
        "/tasks/cancel-a-subscription/",
        "/tasks/dispute-a-charge/",
        "/tasks/live-alerts/",
        "/tasks/flight-delay-compensation/",
        "/tasks/book-a-table/",
        "/tasks/work-files/"
      ],
      "before": "Web and a native Mac app with real system access, the broadest local reach on this bench, so it can reach boarding passes and receipts sitting on your own machine. It shows its assistant roughly 177 of the 1105 tools it lists.",
      "after": "The examined native Mac route reaches messages, contacts, screen and audio. The assistant saw 177 of 1105 catalogue entries. Available actions depend on account configuration and permissions; this is not a task-success result.",
      "why": "content/agents/town.md and data/scorecard.json; scope limited to observed interfaces.",
      "kind": "calibration"
    },
    {
      "id": "2026-10-12-r3-013",
      "date": "2026-10-12",
      "title": "Round 3: task-notes evidence correction 13",
      "pages": [
        "/tasks/family-scheduling/",
        "/tasks/cancel-a-subscription/",
        "/tasks/dispute-a-charge/",
        "/tasks/live-alerts/",
        "/tasks/flight-delay-compensation/",
        "/tasks/book-a-table/",
        "/tasks/work-files/"
      ],
      "before": "Works the phone, which is the right instrument for the one participant who does not use apps. It keeps the merchant, the number, the duration and a summary, but no transcript, so what was agreed with the other parent is not something you can go back and read.",
      "after": "Observed business calling with start and identity confirmations. App call history kept a summary, not a verbatim transcript; developer API calls returned transcripts. Success on this particular errand is untested. Calling private parents or negotiating carpools was not tested.",
      "why": "content/agents/pine.md and data/scorecard.json; scope limited to observed interfaces.",
      "kind": "calibration"
    },
    {
      "id": "2026-10-12-r3-014",
      "date": "2026-10-12",
      "title": "Round 3: task-notes evidence correction 14",
      "pages": [
        "/tasks/family-scheduling/",
        "/tasks/cancel-a-subscription/",
        "/tasks/dispute-a-charge/",
        "/tasks/live-alerts/",
        "/tasks/flight-delay-compensation/",
        "/tasks/book-a-table/",
        "/tasks/work-files/"
      ],
      "before": "Lives in WhatsApp, SMS and iMessage, which is where household coordination already happens. That also means there is no confirmation screen standing between it and a message sent to somebody else.",
      "after": "The observed messaging interface has an approve, deny or dismiss card bound to the exact plan text; a changed plan needs new approval. Its vault supports fill-only credentials. This does not establish success on this errand.",
      "why": "content/agents/instinct.md and data/scorecard.json; scope limited to observed interfaces.",
      "kind": "calibration"
    },
    {
      "id": "2026-10-12-r3-015",
      "date": "2026-10-12",
      "title": "Round 3: task-notes evidence correction 15",
      "pages": [
        "/tasks/family-scheduling/",
        "/tasks/cancel-a-subscription/",
        "/tasks/dispute-a-charge/",
        "/tasks/live-alerts/",
        "/tasks/flight-delay-compensation/",
        "/tasks/book-a-table/",
        "/tasks/work-files/"
      ],
      "before": "Keeps memory as plain files you can open, so what it believes about your household is inspectable rather than inferred. Its approval card puts a send in front of you before it goes.",
      "after": "The examined web runtime uses a per-user Linux VM and markdown memory. Personal access to those files was not verified. An approvals interface was observed; the Mac route and this specific errand remain untested.",
      "why": "content/agents/muse.md and data/scorecard.json; scope limited to observed interfaces.",
      "kind": "calibration"
    },
    {
      "id": "2026-10-12-r3-016",
      "date": "2026-10-12",
      "title": "Round 3: task-notes evidence correction 16",
      "pages": [
        "/tasks/family-scheduling/",
        "/tasks/cancel-a-subscription/",
        "/tasks/dispute-a-charge/",
        "/tasks/live-alerts/",
        "/tasks/flight-delay-compensation/",
        "/tasks/book-a-table/",
        "/tasks/work-files/"
      ],
      "before": "Web and a native Mac app with real system access, the broadest local reach on this bench, so it can reach calendars and contacts on your own machine. It shows its assistant roughly 177 of the 1105 tools it lists.",
      "after": "The examined native Mac route reaches messages, contacts, screen and audio. The assistant saw 177 of 1105 catalogue entries. Available actions depend on account configuration and permissions; this is not a task-success result.",
      "why": "content/agents/town.md and data/scorecard.json; scope limited to observed interfaces.",
      "kind": "calibration"
    },
    {
      "id": "2026-10-12-r3-017",
      "date": "2026-10-12",
      "title": "Round 3: task-notes evidence correction 17",
      "pages": [
        "/tasks/family-scheduling/",
        "/tasks/cancel-a-subscription/",
        "/tasks/dispute-a-charge/",
        "/tasks/live-alerts/",
        "/tasks/flight-delay-compensation/",
        "/tasks/book-a-table/",
        "/tasks/work-files/"
      ],
      "before": "Phones the venue, which is still how a contended table is actually released. It keeps the merchant, the number, the duration and a summary, but no transcript, so the terms it accepted are not something you can go back and read.",
      "after": "Observed business calling with start and identity confirmations. App call history kept a summary, not a verbatim transcript; developer API calls returned transcripts. Success on this particular errand is untested.",
      "why": "content/agents/pine.md and data/scorecard.json; scope limited to observed interfaces.",
      "kind": "calibration"
    },
    {
      "id": "2026-10-12-r3-018",
      "date": "2026-10-12",
      "title": "Round 3: task-notes evidence correction 18",
      "pages": [
        "/tasks/family-scheduling/",
        "/tasks/cancel-a-subscription/",
        "/tasks/dispute-a-charge/",
        "/tasks/live-alerts/",
        "/tasks/flight-delay-compensation/",
        "/tasks/book-a-table/",
        "/tasks/work-files/"
      ],
      "before": "A clock or an event can start it, so it does not have to be watching when a slot opens. It lives in a message thread, so there is no confirmation screen standing between it and a held booking.",
      "after": "The observed messaging interface has an approve, deny or dismiss card bound to the exact plan text; a changed plan needs new approval. Its vault supports fill-only credentials. This does not establish success on this errand.",
      "why": "content/agents/instinct.md and data/scorecard.json; scope limited to observed interfaces.",
      "kind": "calibration"
    },
    {
      "id": "2026-10-12-r3-019",
      "date": "2026-10-12",
      "title": "Round 3: task-notes evidence correction 19",
      "pages": [
        "/tasks/family-scheduling/",
        "/tasks/cancel-a-subscription/",
        "/tasks/dispute-a-charge/",
        "/tasks/live-alerts/",
        "/tasks/flight-delay-compensation/",
        "/tasks/book-a-table/",
        "/tasks/work-files/"
      ],
      "before": "Assembles the request on a Linux machine that is yours, and keeps memory as files you can open. Its approval card puts a spend in front of you before a deposit goes.",
      "after": "The examined web runtime uses a per-user Linux VM and markdown memory. Personal access to those files was not verified. An approvals interface was observed; the Mac route and this specific errand remain untested.",
      "why": "content/agents/muse.md and data/scorecard.json; scope limited to observed interfaces.",
      "kind": "calibration"
    },
    {
      "id": "2026-10-12-r3-020",
      "date": "2026-10-12",
      "title": "Round 3: task-notes evidence correction 20",
      "pages": [
        "/tasks/family-scheduling/",
        "/tasks/cancel-a-subscription/",
        "/tasks/dispute-a-charge/",
        "/tasks/live-alerts/",
        "/tasks/flight-delay-compensation/",
        "/tasks/book-a-table/",
        "/tasks/work-files/"
      ],
      "before": "Web and a native Mac app with real system access, the broadest local reach on this bench. It shows its assistant roughly 177 of the 1105 tools it lists, which is the gap worth asking about on an errand this narrow.",
      "after": "The examined native Mac route reaches messages, contacts, screen and audio. The assistant saw 177 of 1105 catalogue entries. Available actions depend on account configuration and permissions; this is not a task-success result.",
      "why": "content/agents/town.md and data/scorecard.json; scope limited to observed interfaces.",
      "kind": "calibration"
    },
    {
      "id": "2026-10-12-r3-021",
      "date": "2026-10-12",
      "title": "Round 3: task-notes evidence correction 21",
      "pages": [
        "/tasks/family-scheduling/",
        "/tasks/cancel-a-subscription/",
        "/tasks/dispute-a-charge/",
        "/tasks/live-alerts/",
        "/tasks/flight-delay-compensation/",
        "/tasks/book-a-table/",
        "/tasks/work-files/"
      ],
      "before": "Built around the phone call rather than the file system, which makes this the errand furthest from what its runtime is shaped for.",
      "after": "Observed business calling with start and identity confirmations. App call history kept a summary, not a verbatim transcript; developer API calls returned transcripts. Success on this particular errand is untested.",
      "why": "content/agents/pine.md and data/scorecard.json; scope limited to observed interfaces.",
      "kind": "calibration"
    },
    {
      "id": "2026-10-12-r3-022",
      "date": "2026-10-12",
      "title": "Round 3: task-notes evidence correction 22",
      "pages": [
        "/tasks/family-scheduling/",
        "/tasks/cancel-a-subscription/",
        "/tasks/dispute-a-charge/",
        "/tasks/live-alerts/",
        "/tasks/flight-delay-compensation/",
        "/tasks/book-a-table/",
        "/tasks/work-files/"
      ],
      "before": "Runs in a cloud sandbox on Firecracker microVMs, so the files it works on are the ones it can reach from there rather than the ones on your desk. Its memory keeps markdown under version control, so its own notes have a history.",
      "after": "The observed messaging interface has an approve, deny or dismiss card bound to the exact plan text; a changed plan needs new approval. Its vault supports fill-only credentials. This does not establish success on this errand.",
      "why": "content/agents/instinct.md and data/scorecard.json; scope limited to observed interfaces.",
      "kind": "calibration"
    },
    {
      "id": "2026-10-12-r3-023",
      "date": "2026-10-12",
      "title": "Round 3: task-notes evidence correction 23",
      "pages": [
        "/tasks/family-scheduling/",
        "/tasks/cancel-a-subscription/",
        "/tasks/dispute-a-charge/",
        "/tasks/live-alerts/",
        "/tasks/flight-delay-compensation/",
        "/tasks/book-a-table/",
        "/tasks/work-files/"
      ],
      "before": "Memory is a file you could open, on a Linux machine that is yours. Of everything on this bench, this is the runtime whose working state most resembles an ordinary folder.",
      "after": "The examined web runtime uses a per-user Linux VM and markdown memory. Personal access to those files was not verified. An approvals interface was observed; the Mac route and this specific errand remain untested.",
      "why": "content/agents/muse.md and data/scorecard.json; scope limited to observed interfaces.",
      "kind": "calibration"
    },
    {
      "id": "2026-10-12-r3-024",
      "date": "2026-10-12",
      "title": "Round 3: task-notes evidence correction 24",
      "pages": [
        "/tasks/family-scheduling/",
        "/tasks/cancel-a-subscription/",
        "/tasks/dispute-a-charge/",
        "/tasks/live-alerts/",
        "/tasks/flight-delay-compensation/",
        "/tasks/book-a-table/",
        "/tasks/work-files/"
      ],
      "before": "A native Mac app with real system access, which is the only runtime here that reaches your local files directly. It shows its assistant roughly 177 of the 1105 tools it lists.",
      "after": "The examined native Mac route reaches messages, contacts, screen and audio. The assistant saw 177 of 1105 catalogue entries. Available actions depend on account configuration and permissions; this is not a task-success result. This is not exclusive local-file reach: Pine has a local browser helper and Muse advertises an untested Mac route.",
      "why": "content/agents/town.md and data/scorecard.json; scope limited to observed interfaces.",
      "kind": "calibration"
    },
    {
      "id": "2026-10-12-r3-025",
      "date": "2026-10-12",
      "title": "Round 3: task-notes evidence correction 25",
      "pages": [
        "/tasks/family-scheduling/",
        "/tasks/cancel-a-subscription/",
        "/tasks/dispute-a-charge/",
        "/tasks/live-alerts/",
        "/tasks/flight-delay-compensation/",
        "/tasks/book-a-table/",
        "/tasks/work-files/"
      ],
      "before": "Reaches you by phone, which is the loudest channel on this bench and therefore the most expensive one to be wrong on.",
      "after": "Observed business calling with start and identity confirmations. App call history kept a summary, not a verbatim transcript; developer API calls returned transcripts. Success on this particular errand is untested.",
      "why": "content/agents/pine.md and data/scorecard.json; scope limited to observed interfaces.",
      "kind": "calibration"
    },
    {
      "id": "2026-10-12-r3-026",
      "date": "2026-10-12",
      "title": "Round 3: task-notes evidence correction 26",
      "pages": [
        "/tasks/family-scheduling/",
        "/tasks/cancel-a-subscription/",
        "/tasks/dispute-a-charge/",
        "/tasks/live-alerts/",
        "/tasks/flight-delay-compensation/",
        "/tasks/book-a-table/",
        "/tasks/work-files/"
      ],
      "before": "A clock or an event can start it, so it does not need you to ask first. That is the capability this errand is about, and it is also what makes unwarranted interruption possible.",
      "after": "The observed messaging interface has an approve, deny or dismiss card bound to the exact plan text; a changed plan needs new approval. Its vault supports fill-only credentials. This does not establish success on this errand.",
      "why": "content/agents/instinct.md and data/scorecard.json; scope limited to observed interfaces.",
      "kind": "calibration"
    },
    {
      "id": "2026-10-12-r3-027",
      "date": "2026-10-12",
      "title": "Round 3: task-notes evidence correction 27",
      "pages": [
        "/tasks/family-scheduling/",
        "/tasks/cancel-a-subscription/",
        "/tasks/dispute-a-charge/",
        "/tasks/live-alerts/",
        "/tasks/flight-delay-compensation/",
        "/tasks/book-a-table/",
        "/tasks/work-files/"
      ],
      "before": "Phone, web and WhatsApp, plus a Mac app since September, so there are several routes by which it can reach you and several by which it can interrupt you. Its approval card puts a send in front of you before it happens. We tested the web surface only.",
      "after": "The examined web runtime uses a per-user Linux VM and markdown memory. Personal access to those files was not verified. An approvals interface was observed; the Mac route and this specific errand remain untested.",
      "why": "content/agents/muse.md and data/scorecard.json; scope limited to observed interfaces.",
      "kind": "calibration"
    },
    {
      "id": "2026-10-12-r3-028",
      "date": "2026-10-12",
      "title": "Round 3: task-notes evidence correction 28",
      "pages": [
        "/tasks/family-scheduling/",
        "/tasks/cancel-a-subscription/",
        "/tasks/dispute-a-charge/",
        "/tasks/live-alerts/",
        "/tasks/flight-delay-compensation/",
        "/tasks/book-a-table/",
        "/tasks/work-files/"
      ],
      "before": "Web and a native Mac app with real system access, so it can reach notification surfaces on your own machine. It shows its assistant roughly 177 of the 1105 tools it lists.",
      "after": "The examined native Mac route reaches messages, contacts, screen and audio. The assistant saw 177 of 1105 catalogue entries. Available actions depend on account configuration and permissions; this is not a task-success result.",
      "why": "content/agents/town.md and data/scorecard.json; scope limited to observed interfaces.",
      "kind": "calibration"
    },
    {
      "id": "2026-10-12-r3-029",
      "date": "2026-10-12",
      "title": "Round 3: scorecard evidence correction 29",
      "pages": [
        "/",
        "/agents/pine/",
        "/muse-vs-pine/",
        "/pine-vs-instinct/",
        "/pine-vs-town/"
      ],
      "before": "There is nothing to delete, because there is no file holding what it knows, only a persona and a fact layer.",
      "after": "We found no editable local memory file; deletion of server-side facts was not established.",
      "why": "Pine teardown identifies a server-side persona and structured fact layer; absence of a local file does not mean absence of stored data.",
      "kind": "calibration"
    },
    {
      "id": "2026-10-12-r3-030",
      "date": "2026-10-12",
      "title": "Round 3: build evidence correction 30",
      "pages": [
        "/"
      ],
      "before": "tasks scored so far",
      "after": "scored product-task observations",
      "why": "Independent round-three audit checked against the cited source and existing teardown evidence; unsupported inference removed.",
      "kind": "calibration"
    },
    {
      "id": "2026-10-12-r3-031",
      "date": "2026-10-12",
      "title": "The linked run is from the same world, not necessarily this errand",
      "pages": [
        "/tasks/family-scheduling/",
        "/tasks/cancel-a-subscription/",
        "/tasks/dispute-a-charge/",
        "/tasks/live-alerts/",
        "/tasks/flight-delay-compensation/",
        "/tasks/book-a-table/",
        "/tasks/work-files/"
      ],
      "before": "One recorded attempt at this errand",
      "after": "A recorded task from the same simulated world",
      "why": "Independent round-three audit checked against the cited source and existing teardown evidence; unsupported inference removed.",
      "kind": "fact"
    },
    {
      "id": "2026-10-12-r3-032",
      "date": "2026-10-12",
      "title": "Unknown source domains are no longer labelled official sites",
      "pages": [
        "/landscape/"
      ],
      "before": "Unrecognised evidence domains were labelled Official site.",
      "after": "Unrecognised domains are labelled Source link instead of being asserted official.",
      "why": "Independent round-three audit checked against the cited source and existing teardown evidence; unsupported inference removed.",
      "kind": "fact"
    },
    {
      "id": "2026-10-12-r3-033",
      "date": "2026-10-12",
      "title": "The older results used runtime replicas with GLM 5.3 flash substituted",
      "pages": [
        "/pine-vs-instinct/"
      ],
      "before": "These are older builds under older rules",
      "after": "These are replicas of older runtimes with the same GLM 5.3 flash model substituted, under older rules",
      "why": "data/results.json comparable_batch maintainer_note and content/method/exam-changes.md: replicas, shared GLM 5.3 flash model, overwritten/shared transcript files.",
      "kind": "fact"
    },
    {
      "id": "2026-10-12-r3-034",
      "date": "2026-10-12",
      "title": "Round 3: pine-vs-instinct evidence correction 34",
      "pages": [
        "/pine-vs-instinct/"
      ],
      "before": "Transcripts were supplied but have not been republished here.",
      "after": "Transcripts were supplied but have not been republished here; some files were overwritten or shared between attempts, so the underlying record is incomplete.",
      "why": "Independent round-three audit checked against the cited source and existing teardown evidence; unsupported inference removed.",
      "kind": "calibration"
    },
    {
      "id": "2026-10-12-r3-035",
      "date": "2026-10-12",
      "title": "Round 3: muse-vs-pine evidence correction 35",
      "pages": [
        "/muse-vs-pine/"
      ],
      "before": "These are older builds under older rules",
      "after": "These are replicas of older runtimes with the same GLM 5.3 flash model substituted, under older rules",
      "why": "data/results.json comparable_batch maintainer_note and content/method/exam-changes.md: replicas, shared GLM 5.3 flash model, overwritten/shared transcript files.",
      "kind": "calibration"
    },
    {
      "id": "2026-10-12-r3-036",
      "date": "2026-10-12",
      "title": "Round 3: muse-vs-pine evidence correction 36",
      "pages": [
        "/muse-vs-pine/"
      ],
      "before": "Transcripts were supplied but have not been republished here.",
      "after": "Transcripts were supplied but have not been republished here; some files were overwritten or shared between attempts, so the underlying record is incomplete.",
      "why": "Independent round-three audit checked against the cited source and existing teardown evidence; unsupported inference removed.",
      "kind": "calibration"
    },
    {
      "id": "2026-10-12-r3-037",
      "date": "2026-10-12",
      "title": "Round 3: instinct-vs-town evidence correction 37",
      "pages": [
        "/instinct-vs-town/"
      ],
      "before": "These are older builds under older rules",
      "after": "These are replicas of older runtimes with the same GLM 5.3 flash model substituted, under older rules",
      "why": "data/results.json comparable_batch maintainer_note and content/method/exam-changes.md: replicas, shared GLM 5.3 flash model, overwritten/shared transcript files.",
      "kind": "calibration"
    },
    {
      "id": "2026-10-12-r3-038",
      "date": "2026-10-12",
      "title": "Round 3: instinct-vs-town evidence correction 38",
      "pages": [
        "/instinct-vs-town/"
      ],
      "before": "Transcripts were supplied but have not been republished here.",
      "after": "Transcripts were supplied but have not been republished here; some files were overwritten or shared between attempts, so the underlying record is incomplete.",
      "why": "Independent round-three audit checked against the cited source and existing teardown evidence; unsupported inference removed.",
      "kind": "calibration"
    },
    {
      "id": "2026-10-12-r3-039",
      "date": "2026-10-12",
      "title": "Round 3: pine-vs-town evidence correction 39",
      "pages": [
        "/pine-vs-town/"
      ],
      "before": "These are older builds under older rules",
      "after": "These are replicas of older runtimes with the same GLM 5.3 flash model substituted, under older rules",
      "why": "data/results.json comparable_batch maintainer_note and content/method/exam-changes.md: replicas, shared GLM 5.3 flash model, overwritten/shared transcript files.",
      "kind": "calibration"
    },
    {
      "id": "2026-10-12-r3-040",
      "date": "2026-10-12",
      "title": "Round 3: pine-vs-town evidence correction 40",
      "pages": [
        "/pine-vs-town/"
      ],
      "before": "Transcripts were supplied but have not been republished here.",
      "after": "Transcripts were supplied but have not been republished here; some files were overwritten or shared between attempts, so the underlying record is incomplete.",
      "why": "Independent round-three audit checked against the cited source and existing teardown evidence; unsupported inference removed.",
      "kind": "calibration"
    },
    {
      "id": "2026-10-12-r3-041",
      "date": "2026-10-12",
      "title": "Round 3: muse-vs-instinct evidence correction 41",
      "pages": [
        "/muse-vs-instinct/"
      ],
      "before": "each product's share of every task in our exam",
      "after": "each product's share of its own scored tasks",
      "why": "Independent round-three audit checked against the cited source and existing teardown evidence; unsupported inference removed.",
      "kind": "calibration"
    },
    {
      "id": "2026-10-12-r3-042",
      "date": "2026-10-12",
      "title": "Round 3: muse-vs-instinct evidence correction 42",
      "pages": [
        "/muse-vs-instinct/"
      ],
      "before": "These are older builds under older rules",
      "after": "These are replicas of older runtimes with the same GLM 5.3 flash model substituted, under older rules",
      "why": "data/results.json comparable_batch maintainer_note and content/method/exam-changes.md: replicas, shared GLM 5.3 flash model, overwritten/shared transcript files.",
      "kind": "calibration"
    },
    {
      "id": "2026-10-12-r3-043",
      "date": "2026-10-12",
      "title": "Round 3: muse-vs-instinct evidence correction 43",
      "pages": [
        "/muse-vs-instinct/"
      ],
      "before": "Transcripts were supplied but have not been republished here.",
      "after": "Transcripts were supplied but have not been republished here; some files were overwritten or shared between attempts, so the underlying record is incomplete.",
      "why": "Independent round-three audit checked against the cited source and existing teardown evidence; unsupported inference removed.",
      "kind": "calibration"
    },
    {
      "id": "2026-10-12-r3-044",
      "date": "2026-10-12",
      "title": "Round 3: muse-vs-town evidence correction 44",
      "pages": [
        "/muse-vs-town/"
      ],
      "before": "These are older builds under older rules",
      "after": "These are replicas of older runtimes with the same GLM 5.3 flash model substituted, under older rules",
      "why": "data/results.json comparable_batch maintainer_note and content/method/exam-changes.md: replicas, shared GLM 5.3 flash model, overwritten/shared transcript files.",
      "kind": "calibration"
    },
    {
      "id": "2026-10-12-r3-045",
      "date": "2026-10-12",
      "title": "Round 3: muse-vs-town evidence correction 45",
      "pages": [
        "/muse-vs-town/"
      ],
      "before": "Transcripts were supplied but have not been republished here.",
      "after": "Transcripts were supplied but have not been republished here; some files were overwritten or shared between attempts, so the underlying record is incomplete.",
      "why": "Independent round-three audit checked against the cited source and existing teardown evidence; unsupported inference removed.",
      "kind": "calibration"
    },
    {
      "id": "2026-10-12-r3-046",
      "date": "2026-10-12",
      "title": "Round 3: family-scheduling evidence correction 46",
      "pages": [
        "/tasks/family-scheduling/"
      ],
      "before": "Each is a product's share of every task in the exam",
      "after": "Each is a product's share of its own scored task subset in the September 2026 adapted-runtime exam",
      "why": "Independent round-three audit checked against the cited source and existing teardown evidence; unsupported inference removed.",
      "kind": "calibration"
    },
    {
      "id": "2026-10-12-r3-047",
      "date": "2026-10-12",
      "title": "Round 3: family-scheduling evidence correction 47",
      "pages": [
        "/tasks/family-scheduling/"
      ],
      "before": "each product's share of every task in the exam",
      "after": "each product's share of its own scored task subset in the September 2026 adapted-runtime exam",
      "why": "Independent round-three audit checked against the cited source and existing teardown evidence; unsupported inference removed.",
      "kind": "calibration"
    },
    {
      "id": "2026-10-12-r3-048",
      "date": "2026-10-12",
      "title": "Round 3: cancel-a-subscription evidence correction 48",
      "pages": [
        "/tasks/cancel-a-subscription/"
      ],
      "before": "each product's share of every task in the exam",
      "after": "each product's share of its own scored task subset in the September 2026 adapted-runtime exam",
      "why": "Independent round-three audit checked against the cited source and existing teardown evidence; unsupported inference removed.",
      "kind": "calibration"
    },
    {
      "id": "2026-10-12-r3-049",
      "date": "2026-10-12",
      "title": "Round 3: dispute-a-charge evidence correction 49",
      "pages": [
        "/tasks/dispute-a-charge/"
      ],
      "before": "Each is a product's share of every task in the exam",
      "after": "Each is a product's share of its own scored task subset in the September 2026 adapted-runtime exam",
      "why": "Independent round-three audit checked against the cited source and existing teardown evidence; unsupported inference removed.",
      "kind": "calibration"
    },
    {
      "id": "2026-10-12-r3-050",
      "date": "2026-10-12",
      "title": "Round 3: live-alerts evidence correction 50",
      "pages": [
        "/tasks/live-alerts/"
      ],
      "before": "Each is a product's share of every task in the exam",
      "after": "Each is a product's share of its own scored task subset in the September 2026 adapted-runtime exam",
      "why": "Independent round-three audit checked against the cited source and existing teardown evidence; unsupported inference removed.",
      "kind": "calibration"
    },
    {
      "id": "2026-10-12-r3-051",
      "date": "2026-10-12",
      "title": "Round 3: flight-delay-compensation evidence correction 51",
      "pages": [
        "/tasks/flight-delay-compensation/"
      ],
      "before": "Each is a product's share of every task in the exam",
      "after": "Each is a product's share of its own scored task subset in the September 2026 adapted-runtime exam",
      "why": "Independent round-three audit checked against the cited source and existing teardown evidence; unsupported inference removed.",
      "kind": "calibration"
    },
    {
      "id": "2026-10-12-r3-052",
      "date": "2026-10-12",
      "title": "Round 3: book-a-table evidence correction 52",
      "pages": [
        "/tasks/book-a-table/"
      ],
      "before": "Each is a product's share of every task in the exam",
      "after": "Each is a product's share of its own scored task subset in the September 2026 adapted-runtime exam",
      "why": "Independent round-three audit checked against the cited source and existing teardown evidence; unsupported inference removed.",
      "kind": "calibration"
    },
    {
      "id": "2026-10-12-r3-053",
      "date": "2026-10-12",
      "title": "Round 3: book-a-table evidence correction 53",
      "pages": [
        "/tasks/book-a-table/"
      ],
      "before": "each product's share of every task in the exam",
      "after": "each product's share of its own scored task subset in the September 2026 adapted-runtime exam",
      "why": "Independent round-three audit checked against the cited source and existing teardown evidence; unsupported inference removed.",
      "kind": "calibration"
    },
    {
      "id": "2026-10-12-r3-054",
      "date": "2026-10-12",
      "title": "Round 3: work-files evidence correction 54",
      "pages": [
        "/tasks/work-files/"
      ],
      "before": "Each is a product's share of every task in the exam",
      "after": "Each is a product's share of its own scored task subset in the September 2026 adapted-runtime exam",
      "why": "Independent round-three audit checked against the cited source and existing teardown evidence; unsupported inference removed.",
      "kind": "calibration"
    },
    {
      "id": "2026-10-12-r3-055",
      "date": "2026-10-12",
      "title": "Debit-card investigations can take 20 or 90 days in specified cases",
      "pages": [
        "/tasks/dispute-a-charge/"
      ],
      "before": "the bank has 10 business days to investigate, or up to 45 days if it provisionally credits your account",
      "after": "the usual investigation period is 10 business days, or up to 45 days with provisional credit subject to the rule’s conditions. The initial period can be 20 business days for new accounts; the extended period can be 90 days for qualifying new-account, foreign-initiated or point-of-sale debit transactions",
      "why": "CFPB Regulation E, 12 CFR 1005.6 and 1005.11, and the existing Regulation Z 1026.13 source; primary text checked 12 October 2026.",
      "kind": "fact"
    },
    {
      "id": "2026-10-12-r3-056",
      "date": "2026-10-12",
      "title": "Late notice of debit-card losses can leave liability above $500",
      "pages": [
        "/tasks/dispute-a-charge/"
      ],
      "before": "For a lost or stolen card, telling the bank within two business days of learning of it caps your liability at $50, and later notice raises the cap to $500",
      "after": "For a lost or stolen accepted access device, notice within two business days of learning of the loss generally limits liability to the lesser of $50 or the unauthorized transfers. Later notice can expose you to up to $500, subject to the bank showing which later transfers timely notice would have prevented. Failure to report within 60 days of the statement can add liability for preventable transfers after that window and before notice, without a $500 ceiling. Extenuating circumstances require extensions; more protective state law or account terms can reduce liability",
      "why": "CFPB Regulation E, 12 CFR 1005.6 and 1005.11, and the existing Regulation Z 1026.13 source; primary text checked 12 October 2026.",
      "kind": "fact"
    },
    {
      "id": "2026-10-12-r3-057",
      "date": "2026-10-12",
      "title": "The billing-error notice clock starts with the statement",
      "pages": [
        "/tasks/dispute-a-charge/"
      ],
      "before": "The clock started when the charge posted, not today",
      "after": "The FCBA notice clock starts with the first statement showing the error; network chargeback clocks differ",
      "why": "CFPB Regulation E, 12 CFR 1005.6 and 1005.11, and the existing Regulation Z 1026.13 source; primary text checked 12 October 2026.",
      "kind": "fact"
    },
    {
      "id": "2026-10-12-r3-058",
      "date": "2026-10-12",
      "title": "Round 3: dispute-a-charge evidence correction 58",
      "pages": [
        "/tasks/dispute-a-charge/"
      ],
      "before": "Miss the deadline that applies and the strongest route closes, leaving you with the merchant's goodwill and nothing else.",
      "after": "Missing an applicable deadline can limit that dispute route; it does not establish that every contractual or legal remedy has disappeared.",
      "why": "CFPB Regulation E, 12 CFR 1005.6 and 1005.11, and the existing Regulation Z 1026.13 source; primary text checked 12 October 2026.",
      "kind": "calibration"
    },
    {
      "id": "2026-10-12-r3-059",
      "date": "2026-10-12",
      "title": "Round 3: dispute-a-charge evidence correction 59",
      "pages": [
        "/tasks/dispute-a-charge/"
      ],
      "before": "For a US debit card, Regulation E also uses 60 days from the statement.",
      "after": "For a US debit card, report immediately: the two-business-day lost-device rule and the 60-day statement rule address different liability periods.",
      "why": "CFPB Regulation E, 12 CFR 1005.6 and 1005.11, and the existing Regulation Z 1026.13 source; primary text checked 12 October 2026.",
      "kind": "calibration"
    },
    {
      "id": "2026-10-12-r3-060",
      "date": "2026-10-12",
      "title": "Extraordinary circumstances do not erase all passenger rights",
      "pages": [
        "/tasks/flight-delay-compensation/"
      ],
      "before": "Nothing is due if the airline proves extraordinary circumstances, such as weather, air traffic management decisions or security risks.",
      "after": "Cash compensation is not due if the airline proves extraordinary circumstances that could not have been avoided even with all reasonable measures. Applicable care, reimbursement or rerouting rights can remain; the exception does not erase them.",
      "why": "EU Your Europe passenger-rights guidance and Canadian APPR sections 12, 19 and 21; primary text checked 12 October 2026.",
      "kind": "fact"
    },
    {
      "id": "2026-10-12-r3-061",
      "date": "2026-10-12",
      "title": "Round 3: flight-delay-compensation evidence correction 61",
      "pages": [
        "/tasks/flight-delay-compensation/"
      ],
      "before": "The EU and Canada do set cash amounts for long delays within the airline's control.",
      "after": "The EU and Canada set cash amounts for qualifying long delays, under different eligibility tests.",
      "why": "EU Your Europe passenger-rights guidance and Canadian APPR sections 12, 19 and 21; primary text checked 12 October 2026.",
      "kind": "calibration"
    },
    {
      "id": "2026-10-12-r3-062",
      "date": "2026-10-12",
      "title": "EU and Canadian delay compensation use different eligibility tests",
      "pages": [
        "/tasks/flight-delay-compensation/"
      ],
      "before": "EU and Canadian rules pay fixed amounts for long arrival delays, but only when the cause was within the airline's control, and the airline records the cause.",
      "after": "EU compensation depends on the extraordinary-circumstances and reasonable-measures test. Canada requires a cause within the carrier's control and not required for safety, with notice and other eligibility conditions.",
      "why": "EU Your Europe passenger-rights guidance and Canadian APPR sections 12, 19 and 21; primary text checked 12 October 2026.",
      "kind": "fact"
    },
    {
      "id": "2026-10-12-r3-063",
      "date": "2026-10-12",
      "title": "EU and Canadian delay compensation use different eligibility tests",
      "pages": [
        "/tasks/flight-delay-compensation/"
      ],
      "before": "in both cases only if the cause was within the airline's control",
      "after": "subject to different tests: the EU extraordinary-circumstances exception requires proof that all reasonable measures could not have avoided the event; Canada requires a cause within the carrier's control, not required for safety, and notice of the disruption 14 days or less before departure",
      "why": "EU Your Europe passenger-rights guidance and Canadian APPR sections 12, 19 and 21; primary text checked 12 October 2026.",
      "kind": "fact"
    },
    {
      "id": "2026-10-12-r3-064",
      "date": "2026-10-12",
      "title": "Round 3: flight-delay-compensation evidence correction 64",
      "pages": [
        "/tasks/flight-delay-compensation/"
      ],
      "before": "You must file with the airline within one year, and it has 30 days to pay or explain why not.",
      "after": "For these inconvenience payments, you must have been informed 14 days or less before departure. File within one year; the airline has 30 days to pay or explain why not. Other eligibility conditions and refund-specific amounts appear in sections 12 and 19.",
      "why": "EU Your Europe passenger-rights guidance and Canadian APPR sections 12, 19 and 21; primary text checked 12 October 2026.",
      "kind": "calibration"
    },
    {
      "id": "2026-10-12-r3-065",
      "date": "2026-10-12",
      "title": "Round 3: flight-delay-compensation evidence correction 65",
      "pages": [
        "/tasks/flight-delay-compensation/"
      ],
      "before": "The regimes that pay cash for delays draw the line at whether the cause was within the airline's control: the EU through its extraordinary circumstances exemption, Canada through its separate obligations for situations outside the carrier's control and for safety. Weather and air traffic management decisions sit on the far side of that line in both. In the US the refund rule does not turn on the cause at all, only on the length of the change and whether you declined it.",
      "after": "The EU and Canadian tests are distinct. EU cash compensation can be excluded only when the airline proves extraordinary circumstances and that the disruption could not have been avoided with all reasonable measures. Canadian inconvenience compensation excludes situations outside the carrier's control and those required for safety; notice and other conditions also apply. In the US the federal refund entitlement described above turns on the qualifying change and the passenger's choice, rather than the cause.",
      "why": "EU Your Europe passenger-rights guidance and Canadian APPR sections 12, 19 and 21; primary text checked 12 October 2026.",
      "kind": "calibration"
    },
    {
      "id": "2026-10-12-r3-066",
      "date": "2026-10-12",
      "title": "Stopping a PayPal payment does not cancel the service contract",
      "pages": [
        "/tasks/cancel-a-subscription/"
      ],
      "before": "If the charge says Apple, Google Play, Amazon, Roku or PayPal, cancel there, not with the service.",
      "after": "For a store-billed subscription, use that store’s subscription controls and keep its cancellation confirmation. PayPal is different: [cancelling an automatic payment](https://www.paypal.com/us/cshelp/article/what-is-an-automatic-payment-and-how-do-i-update-or-cancel-one-help240) revokes payment authority; it does not by itself cancel the service contract. Confirm cancellation with the service under its terms.",
      "why": "PayPal automatic-payment help and the distinction between payment authority and the underlying service contract; existing store cancellation sources.",
      "kind": "fact"
    },
    {
      "id": "2026-10-12-r3-067",
      "date": "2026-10-12",
      "title": "Accepting a retention offer does not complete a cancellation",
      "pages": [
        "/tasks/cancel-a-subscription/"
      ],
      "before": "An agent that cancels your subscription by accepting a retention offer has completed the task and cost you money.",
      "after": "An agent that accepts continued service has failed a cancellation goal unless you explicitly change that goal; an attractive retention offer is not cancellation.",
      "why": "Independent round-three audit checked against the cited source and existing teardown evidence; unsupported inference removed.",
      "kind": "fact"
    },
    {
      "id": "2026-10-12-r3-068",
      "date": "2026-10-12",
      "title": "Round 3: cancel-a-subscription evidence correction 68",
      "pages": [
        "/tasks/cancel-a-subscription/"
      ],
      "before": ", which is the single most common way this errand goes wrong",
      "after": ". We do not have a frequency estimate for that failure",
      "why": "Independent round-three audit checked against the cited source and existing teardown evidence; unsupported inference removed.",
      "kind": "calibration"
    },
    {
      "id": "2026-10-12-r3-069",
      "date": "2026-10-12",
      "title": "Removed the older $4–$12 subscription price range",
      "pages": [
        "/tasks/cancel-a-subscription/"
      ],
      "before": "; $4–$12 in Privacy.com’s older guide",
      "after": "; check the vendor’s current account quote",
      "why": "Independent round-three audit checked against the cited source and existing teardown evidence; unsupported inference removed.",
      "kind": "fact"
    },
    {
      "id": "2026-10-12-r3-070",
      "date": "2026-10-12",
      "title": "Removed the older $9.99 monthly Premium price",
      "pages": [
        "/tasks/cancel-a-subscription/"
      ],
      "before": "Premium from $9.99/month in that older guide; current quote not verified",
      "after": "Current vendor quote not verified; check before subscribing",
      "why": "Independent round-three audit checked against the cited source and existing teardown evidence; unsupported inference removed.",
      "kind": "fact"
    },
    {
      "id": "2026-10-12-r3-071",
      "date": "2026-10-12",
      "title": "Round 3: where-it-runs evidence correction 71",
      "pages": [
        "/where-it-runs/"
      ],
      "before": "Where personal agents run is the property you cannot change after you have committed to one.",
      "after": "Where personal agents run constrains access and availability, but some products offer more than one deployment route.",
      "why": "Independent round-three audit checked against the cited source and existing teardown evidence; unsupported inference removed.",
      "kind": "calibration"
    },
    {
      "id": "2026-10-12-r3-072",
      "date": "2026-10-12",
      "title": "Round 3: where-it-runs evidence correction 72",
      "pages": [
        "/where-it-runs/"
      ],
      "before": "You cannot move the machine the thing executes on.",
      "after": "Some vendor runtimes are fixed; self-hosted tools and products with local and cloud routes can offer a choice.",
      "why": "Independent round-three audit checked against the cited source and existing teardown evidence; unsupported inference removed.",
      "kind": "calibration"
    },
    {
      "id": "2026-10-12-r3-073",
      "date": "2026-10-12",
      "title": "Round 3: where-it-runs evidence correction 73",
      "pages": [
        "/where-it-runs/"
      ],
      "before": "It removes the lessor and keeps data on your hardware.",
      "after": "Self-hosting may use owned hardware or a rented server. It does not establish where inference, backups or connected services send data.",
      "why": "Independent round-three audit checked against the cited source and existing teardown evidence; unsupported inference removed.",
      "kind": "calibration"
    },
    {
      "id": "2026-10-12-r3-074",
      "date": "2026-10-12",
      "title": "Self-hosted agents can use local models, not only paid model APIs",
      "pages": [
        "/where-it-runs/"
      ],
      "before": "In every self-hosted setup above, the thinking still happens at a model provider and is billed by use.",
      "after": "Inference may use a hosted API, a subscription or a supported local model. Self-hosting the agent does not itself establish which route is used.",
      "why": "Independent round-three audit checked against the cited source and existing teardown evidence; unsupported inference removed.",
      "kind": "fact"
    },
    {
      "id": "2026-10-12-r3-075",
      "date": "2026-10-12",
      "title": "Round 3: where-it-runs evidence correction 75",
      "pages": [
        "/where-it-runs/"
      ],
      "before": "A disposable container limits the blast radius in a way a native desktop app does not.",
      "after": "A disposable container can isolate its local filesystem; writes through authenticated external accounts can persist after it is destroyed.",
      "why": "Independent round-three audit checked against the cited source and existing teardown evidence; unsupported inference removed.",
      "kind": "calibration"
    },
    {
      "id": "2026-10-12-r3-076",
      "date": "2026-10-12",
      "title": "Round 3: where-it-runs evidence correction 76",
      "pages": [
        "/where-it-runs/"
      ],
      "before": "and a gateway that mostly waits for messages sits near the low end",
      "after": "and these are illustrative idle and maximum bounds, not measured agent workload consumption",
      "why": "Independent round-three audit checked against the cited source and existing teardown evidence; unsupported inference removed.",
      "kind": "calibration"
    },
    {
      "id": "2026-10-12-r3-077",
      "date": "2026-10-12",
      "title": "Round 3: about evidence correction 77",
      "pages": [
        "/about/"
      ],
      "before": "which is a deliberate constraint rather than a limitation",
      "after": "which is a limitation on independent verification",
      "why": "Independent round-three audit checked against the cited source and existing teardown evidence; unsupported inference removed.",
      "kind": "calibration"
    },
    {
      "id": "2026-10-12-r3-078",
      "date": "2026-10-12",
      "title": "Round 3: how-scoring-works evidence correction 78",
      "pages": [
        "/how-scoring-works/"
      ],
      "before": "A sandboxed worker means the agent's mistakes are contained in something disposable. A native desktop process with access to your messages, contacts, screen and audio means they aren't.",
      "after": "A sandbox can isolate local execution; actions through connected accounts can still have lasting effects. A native desktop process additionally exposes whatever local permissions it receives.",
      "why": "Independent round-three audit checked against the cited source and existing teardown evidence; unsupported inference removed.",
      "kind": "calibration"
    },
    {
      "id": "2026-10-12-r3-079",
      "date": "2026-10-12",
      "title": "Round 3: how-scoring-works evidence correction 79",
      "pages": [
        "/how-scoring-works/"
      ],
      "before": "Every claim on the scorecard is about behaviour that a person with an account and some patience can observe.",
      "after": "Teardown observations concern account-visible behaviour. Aggregate completion rates lack public batch records and cannot be independently checked in the same way.",
      "why": "Independent round-three audit checked against the cited source and existing teardown evidence; unsupported inference removed.",
      "kind": "calibration"
    },
    {
      "id": "2026-10-12-r3-080",
      "date": "2026-10-12",
      "title": "Round 3: pine evidence correction 80",
      "pages": [
        "/agents/pine/"
      ],
      "before": "why Pine is good at the narrow thing it does",
      "after": "how Pine approaches bill disputes",
      "why": "Independent round-three audit checked against the cited source and existing teardown evidence; unsupported inference removed.",
      "kind": "calibration"
    },
    {
      "id": "2026-10-12-r3-081",
      "date": "2026-10-12",
      "title": "Round 3: pine evidence correction 81",
      "pages": [
        "/agents/pine/"
      ],
      "before": "On the monthly personal plan, Pine also takes a success fee on bill negotiation.",
      "after": "The quoted weekly personal price and monthly credit allowance are different units; they do not establish the billing cadence. Confirm renewal terms and any bill-negotiation success fee at checkout.",
      "why": "Independent round-three audit checked against the cited source and existing teardown evidence; unsupported inference removed.",
      "kind": "calibration"
    },
    {
      "id": "2026-10-12-r3-082",
      "date": "2026-10-12",
      "title": "Round 3: town evidence correction 82",
      "pages": [
        "/agents/town/"
      ],
      "before": "Every one of them exists, is documented, and can be called by something under some configuration.",
      "after": "The catalogue lists 1105 entries; we did not establish that every entry is documented and callable.",
      "why": "Independent round-three audit checked against the cited source and existing teardown evidence; unsupported inference removed.",
      "kind": "calibration"
    },
    {
      "id": "2026-10-12-r3-083",
      "date": "2026-10-12",
      "title": "Round 3: town evidence correction 83",
      "pages": [
        "/agents/town/"
      ],
      "before": "It's Swift with a web view inside, not Electron, and the difference shows up in what it can reach:",
      "after": "The examined app uses Swift with a web view. Separately, we observed OS access to",
      "why": "Independent round-three audit checked against the cited source and existing teardown evidence; unsupported inference removed.",
      "kind": "calibration"
    },
    {
      "id": "2026-10-12-r3-084",
      "date": "2026-10-12",
      "title": "Round 3: muse evidence correction 84",
      "pages": [
        "/agents/muse/"
      ],
      "before": "a mistake in someone else's session cannot reach into yours",
      "after": "separate VMs are intended to isolate users; we did not test cross-user isolation",
      "why": "Independent round-three audit checked against the cited source and existing teardown evidence; unsupported inference removed.",
      "kind": "calibration"
    },
    {
      "id": "2026-10-12-r3-085",
      "date": "2026-10-12",
      "title": "Round 3: muse evidence correction 85",
      "pages": [
        "/agents/muse/"
      ],
      "before": "How Meta Muse remembers you: a file you could open",
      "after": "How Meta Muse stores memory: readable files, access unverified",
      "why": "Independent round-three audit checked against the cited source and existing teardown evidence; unsupported inference removed.",
      "kind": "calibration"
    },
    {
      "id": "2026-10-12-r3-086",
      "date": "2026-10-12",
      "title": "Round 3: agents evidence correction 86",
      "pages": [
        "/agents/"
      ],
      "before": "The last section of each teardown says what we couldn't establish",
      "after": "Each teardown includes a section and an axis stating what we couldn't establish",
      "why": "Independent round-three audit checked against the cited source and existing teardown evidence; unsupported inference removed.",
      "kind": "calibration"
    },
    {
      "id": "2026-10-12-r3-087",
      "date": "2026-10-12",
      "title": "AgentBench FC reports absolute success across five environments",
      "pages": [
        "/benchmarks/agentbench/"
      ],
      "before": "The FC average is relative to whichever models have been submitted.",
      "after": "FC reports absolute task success on five environments, without a human baseline.",
      "why": "Independent round-three audit checked against the cited source and existing teardown evidence; unsupported inference removed.",
      "kind": "fact"
    },
    {
      "id": "2026-10-12-r3-088",
      "date": "2026-10-12",
      "title": "Round 3: agentbench evidence correction 88",
      "pages": [
        "/benchmarks/agentbench/"
      ],
      "before": "It does mean the ordering reflects who trained hardest against these five environments, which is a different claim from who is most capable, and a reader skimming for the top row will take away the second.",
      "after": "This organisational relationship is disclosed; benchmark-specific training exposure and its contribution to the ordering are unverified.",
      "why": "Independent round-three audit checked against the cited source and existing teardown evidence; unsupported inference removed.",
      "kind": "calibration"
    },
    {
      "id": "2026-10-12-r3-089",
      "date": "2026-10-12",
      "title": "AssistantBench could not load its results dataset on 12 October",
      "pages": [
        "/benchmarks/assistantbench/"
      ],
      "before": "That is a presentation problem rather than a data problem",
      "after": "The 8 October snapshot remains available here, but on 12 October the application could not load its results dataset",
      "why": "Independent round-three audit checked against the cited source and existing teardown evidence; unsupported inference removed.",
      "kind": "fact"
    },
    {
      "id": "2026-10-12-r3-090",
      "date": "2026-10-12",
      "title": "Round 3: assistantbench evidence correction 90",
      "pages": [
        "/benchmarks/assistantbench/"
      ],
      "before": "which is the signature of one system submitted repeatedly under different names",
      "after": "which may indicate duplicate submissions; we cannot establish whether the systems are identical",
      "why": "Independent round-three audit checked against the cited source and existing teardown evidence; unsupported inference removed.",
      "kind": "calibration"
    },
    {
      "id": "2026-10-12-r3-091",
      "date": "2026-10-12",
      "title": "Round 3: assistantbench evidence correction 91",
      "pages": [
        "/benchmarks/assistantbench/"
      ],
      "before": "so a figure can be checked without being gamed",
      "after": "so those individual attempts can be checked; aggregate figures still lack public batch evidence",
      "why": "Independent round-three audit checked against the cited source and existing teardown evidence; unsupported inference removed.",
      "kind": "calibration"
    },
    {
      "id": "2026-10-12-r3-093",
      "date": "2026-10-12",
      "title": "Round 3: gaia evidence correction 93",
      "pages": [
        "/benchmarks/gaia/"
      ],
      "before": "the remaining differences are inside the noise",
      "after": "small remaining differences need uncertainty and paired-task analysis before being treated as meaningful",
      "why": "Independent round-three audit checked against the cited source and existing teardown evidence; unsupported inference removed.",
      "kind": "calibration"
    },
    {
      "id": "2026-10-12-r3-094",
      "date": "2026-10-12",
      "title": "Round 3: gaia evidence correction 94",
      "pages": [
        "/benchmarks/gaia/"
      ],
      "before": "which is what keeps its leaderboard from being gamed",
      "after": "which reduces direct answer leakage but does not rule out contamination or repeated leaderboard tuning",
      "why": "Independent round-three audit checked against the cited source and existing teardown evidence; unsupported inference removed.",
      "kind": "calibration"
    },
    {
      "id": "2026-10-12-r3-095",
      "date": "2026-10-12",
      "title": "GAIA's validation count does not reconcile with the paper's totals",
      "pages": [
        "/benchmarks/gaia/"
      ],
      "before": "The rest, about 165 questions, form a validation set whose answers are public",
      "after": "The paper reports 466 questions and 300 hidden test answers. The earlier roughly 165-question validation description does not reconcile that exact total, so we do not infer an exact released count here; check the dataset version before choosing a split",
      "why": "Independent round-three audit checked against the cited source and existing teardown evidence; unsupported inference removed.",
      "kind": "fact"
    },
    {
      "id": "2026-10-12-r3-096",
      "date": "2026-10-12",
      "title": "Round 3: osworld evidence correction 96",
      "pages": [
        "/benchmarks/osworld/"
      ],
      "before": "There is a length of errand past which nothing currently available finishes the job.",
      "after": "In the reported cohort and settings, no plotted model completed a task in the longest duration band.",
      "why": "Independent round-three audit checked against the cited source and existing teardown evidence; unsupported inference removed.",
      "kind": "calibration"
    },
    {
      "id": "2026-10-12-r3-097",
      "date": "2026-10-12",
      "title": "OSWorld reaches about 78% of checkpoints, not three quarters of steps",
      "pages": [
        "/benchmarks/osworld/"
      ],
      "before": "Three quarters of the required steps get done",
      "after": "About 78% of scored intermediate checkpoints are reached",
      "why": "Independent round-three audit checked against the cited source and existing teardown evidence; unsupported inference removed.",
      "kind": "fact"
    },
    {
      "id": "2026-10-12-r3-098",
      "date": "2026-10-12",
      "title": "Round 3: osworld evidence correction 98",
      "pages": [
        "/benchmarks/osworld/"
      ],
      "before": "Nothing in that trace is a capability failure. Every individual step is competent.",
      "after": "The trace shows planning, tool-use and state-management problems; it does not establish that every individual action was competent.",
      "why": "Independent round-three audit checked against the cited source and existing teardown evidence; unsupported inference removed.",
      "kind": "calibration"
    },
    {
      "id": "2026-10-12-r3-099",
      "date": "2026-10-12",
      "title": "OSWorld 2.0 includes hidden state and seeded user history",
      "pages": [
        "/benchmarks/osworld/"
      ],
      "before": "Everything the agent needs is in the task, and everything it learns dies with the run.",
      "after": "OSWorld 2.0 includes hidden state and seeded user-history context; this differs from learning about a continuing real user across runs.",
      "why": "Independent round-three audit checked against the cited source and existing teardown evidence; unsupported inference removed.",
      "kind": "fact"
    },
    {
      "id": "2026-10-12-r3-100",
      "date": "2026-10-12",
      "title": "Round 3: swe-bench evidence correction 100",
      "pages": [
        "/benchmarks/swe-bench/"
      ],
      "before": "The default board holds the harness constant across every entry, so what you are comparing really is models.",
      "after": "The bash-only view uses the mini-SWE-agent family; compare matching harness versions and settings.",
      "why": "Independent round-three audit checked against the cited source and existing teardown evidence; unsupported inference removed.",
      "kind": "calibration"
    },
    {
      "id": "2026-10-12-r3-101",
      "date": "2026-10-12",
      "title": "Round 3: swe-bench evidence correction 101",
      "pages": [
        "/benchmarks/swe-bench/"
      ],
      "before": "Those three points are the scaffold, not the model, which is the whole argument for holding the harness still.",
      "after": "That gap is between configurations; without a controlled comparison it cannot be attributed entirely to the scaffold.",
      "why": "Independent round-three audit checked against the cited source and existing teardown evidence; unsupported inference removed.",
      "kind": "calibration"
    },
    {
      "id": "2026-10-12-r3-102",
      "date": "2026-10-12",
      "title": "WebArena contains seeded account data",
      "pages": [
        "/benchmarks/webarena/"
      ],
      "before": "holds nothing about the agent or any user",
      "after": "contains seeded account data but does not measure a continuing real user's cross-task memory",
      "why": "Independent round-three audit checked against the cited source and existing teardown evidence; unsupported inference removed.",
      "kind": "fact"
    },
    {
      "id": "2026-10-12-r3-103",
      "date": "2026-10-12",
      "title": "Round 3: webarena evidence correction 103",
      "pages": [
        "/benchmarks/webarena/"
      ],
      "before": "There are now two versions of the task set",
      "after": "Original and Verified are the two task-set versions compared here",
      "why": "Independent round-three audit checked against the cited source and existing teardown evidence; unsupported inference removed.",
      "kind": "calibration"
    },
    {
      "id": "2026-10-12-r3-104",
      "date": "2026-10-12",
      "title": "Round 3: webarena evidence correction 104",
      "pages": [
        "/benchmarks/webarena/"
      ],
      "before": "so anything in the teens now reflects an unassisted model rather than a serious attempt",
      "after": "but a low score alone does not identify the architecture or effort behind it",
      "why": "Independent round-three audit checked against the cited source and existing teardown evidence; unsupported inference removed.",
      "kind": "calibration"
    },
    {
      "id": "2026-10-12-r3-105",
      "date": "2026-10-12",
      "title": "The benchmark list includes action tasks, not only retrieval",
      "pages": [
        "/benchmarks/"
      ],
      "before": "Most of them measure finding rather than doing. Nothing gets booked, cancelled, disputed or paid for, which removes the entire class of failure that makes a personal agent frightening.",
      "after": "Three focus on information retrieval; the others include actions under benchmark-specific environments, with different coverage of consequential side effects.",
      "why": "Independent round-three audit checked against the cited source and existing teardown evidence; unsupported inference removed.",
      "kind": "fact"
    },
    {
      "id": "2026-10-12-r3-106",
      "date": "2026-10-12",
      "title": "Round 3: benchmarks evidence correction 106",
      "pages": [
        "/benchmarks/"
      ],
      "before": "but none runs real consumer products on real errands, which is why this site exists",
      "after": "and the consumer evaluations described below address parts of that space; this site adds runtime observations and a controlled simulated exam",
      "why": "Independent round-three audit checked against the cited source and existing teardown evidence; unsupported inference removed.",
      "kind": "calibration"
    },
    {
      "id": "2026-10-12-r3-107",
      "date": "2026-10-12",
      "title": "Round 3: benchmarks evidence correction 107",
      "pages": [
        "/benchmarks/"
      ],
      "before": "even its widest gap, 80% against 55%, is not statistically clear: the two intervals overlap",
      "after": "its widest gap is 80% against 55%; overlapping marginal intervals alone do not test significance. Paired outcomes and repeat measurements are needed for a justified comparison",
      "why": "Independent round-three audit checked against the cited source and existing teardown evidence; unsupported inference removed.",
      "kind": "calibration"
    },
    {
      "id": "2026-10-12-r3-108",
      "date": "2026-10-12",
      "title": "Round 3: benchmarks evidence correction 108",
      "pages": [
        "/benchmarks/"
      ],
      "before": "It does mean the ordering reflects who optimised against these environments hardest, which is not the same claim as who is most capable.",
      "after": "The organisational relationship is documented. It does not establish how training exposure caused the ordering.",
      "why": "Independent round-three audit checked against the cited source and existing teardown evidence; unsupported inference removed.",
      "kind": "calibration"
    },
    {
      "id": "2026-10-12-r3-109",
      "date": "2026-10-12",
      "title": "Round 3: pine-vs-town evidence correction 109",
      "pages": [
        "/pine-vs-town/"
      ],
      "before": "The damage is bounded by what was in the box.",
      "after": "Local files may be isolated, but authenticated external actions can leave lasting consequences outside the box.",
      "why": "Independent round-three audit checked against the cited source and existing teardown evidence; unsupported inference removed.",
      "kind": "calibration"
    },
    {
      "id": "2026-10-12-r3-110",
      "date": "2026-10-12",
      "title": "Round 3: pine-vs-town evidence correction 110",
      "pages": [
        "/pine-vs-town/"
      ],
      "before": "Everything else about them, the price, the tool count, the surface you type into, follows from that and can be argued about. This one cannot.",
      "after": "Price, tool exposure and channels are separately observed differences; runtime location does not establish their causes.",
      "why": "Independent round-three audit checked against the cited source and existing teardown evidence; unsupported inference removed.",
      "kind": "calibration"
    },
    {
      "id": "2026-10-12-r3-112",
      "date": "2026-10-12",
      "title": "Round 3: instinct-vs-town evidence correction 112",
      "pages": [
        "/instinct-vs-town/"
      ],
      "before": "everything is available, but you cannot touch it without first loading its description",
      "after": "observed reachable commands require loading their descriptions first; not every registry entry was verified",
      "why": "Independent round-three audit checked against the cited source and existing teardown evidence; unsupported inference removed.",
      "kind": "calibration"
    },
    {
      "id": "2026-10-12-r3-113",
      "date": "2026-10-12",
      "title": "Round 3: instinct-vs-town evidence correction 113",
      "pages": [
        "/instinct-vs-town/"
      ],
      "before": "Everything else that differs between these two follows from that choice.",
      "after": "Memory, execution and channels are separate observed differences, not demonstrated consequences of that choice.",
      "why": "Independent round-three audit checked against the cited source and existing teardown evidence; unsupported inference removed.",
      "kind": "calibration"
    },
    {
      "id": "2026-10-12-r3-114",
      "date": "2026-10-12",
      "title": "Round 3: instinct-vs-town evidence correction 114",
      "pages": [
        "/instinct-vs-town/"
      ],
      "before": "Instinct can be reached from anywhere you already type, and what it reaches is a sandbox.",
      "after": "Instinct supports the examined messaging channels and sandboxed work; connected external accounts can also be affected.",
      "why": "Independent round-three audit checked against the cited source and existing teardown evidence; unsupported inference removed.",
      "kind": "calibration"
    },
    {
      "id": "2026-10-12-r3-115",
      "date": "2026-10-12",
      "title": "Round 3: pine-vs-instinct evidence correction 115",
      "pages": [
        "/pine-vs-instinct/"
      ],
      "before": "It is also the only capability on this bench that cannot be rolled back by software. A wrong message can be deleted and a wrong booking can be cancelled. A conversation that happened, happened.",
      "after": "Calls, messages and bookings can all have consequences that software cannot fully undo. Deleting a message does not undo it being read, and cancellation can carry costs.",
      "why": "Independent round-three audit checked against the cited source and existing teardown evidence; unsupported inference removed.",
      "kind": "calibration"
    },
    {
      "id": "2026-10-12-r3-116",
      "date": "2026-10-12",
      "title": "Round 3: pine-vs-instinct evidence correction 116",
      "pages": [
        "/pine-vs-instinct/"
      ],
      "before": "Instinct uses a leased cloud browser and never touches yours.",
      "after": "Instinct used a leased cloud browser in the examined account and browser path; other routes were not established.",
      "why": "Independent round-three audit checked against the cited source and existing teardown evidence; unsupported inference removed.",
      "kind": "calibration"
    },
    {
      "id": "2026-10-12-r3-117",
      "date": "2026-10-12",
      "title": "Round 3: pine-vs-instinct evidence correction 117",
      "pages": [
        "/pine-vs-instinct/"
      ],
      "before": "unbounded about what it can do through computer use",
      "after": "capable of browser work through computer use, with limits we did not establish",
      "why": "Independent round-three audit checked against the cited source and existing teardown evidence; unsupported inference removed.",
      "kind": "calibration"
    },
    {
      "id": "2026-10-12-r3-118",
      "date": "2026-10-12",
      "title": "Round 3: muse-vs-instinct evidence correction 118",
      "pages": [
        "/muse-vs-instinct/"
      ],
      "before": "A message thread has to write it as a sentence and hope you read it before the agent moves on.",
      "after": "Instinct's observed approval card requires approve, deny or dismiss against exact plan text. Presentation in a message thread does not remove that gate.",
      "why": "Independent round-three audit checked against the cited source and existing teardown evidence; unsupported inference removed.",
      "kind": "calibration"
    },
    {
      "id": "2026-10-12-r3-119",
      "date": "2026-10-12",
      "title": "Round 3: muse-vs-instinct evidence correction 119",
      "pages": [
        "/muse-vs-instinct/"
      ],
      "before": "The date at the top is when we last verified these findings against the products.",
      "after": "The date at the top includes editorial and source checks; it does not mean every runtime observation was repeated that day.",
      "why": "Independent round-three audit checked against the cited source and existing teardown evidence; unsupported inference removed.",
      "kind": "calibration"
    },
    {
      "id": "2026-10-12-r3-120",
      "date": "2026-10-12",
      "title": "Round 3: book-a-table evidence correction 120",
      "pages": [
        "/tasks/book-a-table/"
      ],
      "before": "Every other errand on this bench belongs to you alone: nobody else is racing to cancel your subscription, and a delayed flight does not run out of compensation because somebody claimed first.",
      "after": "Real restaurant availability can involve competing diners. The published booking task does not establish performance under that contention.",
      "why": "Independent round-three audit checked against the cited source and existing teardown evidence; unsupported inference removed.",
      "kind": "calibration"
    },
    {
      "id": "2026-10-12-r3-121",
      "date": "2026-10-12",
      "title": "Round 3: family-scheduling evidence correction 121",
      "pages": [
        "/tasks/family-scheduling/"
      ],
      "before": "Silence, which means nobody tested it",
      "after": "Silence, which leaves testing unknown",
      "why": "Independent round-three audit checked against the cited source and existing teardown evidence; unsupported inference removed.",
      "kind": "calibration"
    },
    {
      "id": "2026-10-12-r3-122",
      "date": "2026-10-12",
      "title": "Round 3: live-alerts evidence correction 122",
      "pages": [
        "/tasks/live-alerts/"
      ],
      "before": "The discipline that keeps the noise out keeps that out too.",
      "after": "Poorly configured filters can hide urgent events; priority exceptions for chosen people or apps can preserve them.",
      "why": "Independent round-three audit checked against the cited source and existing teardown evidence; unsupported inference removed.",
      "kind": "calibration"
    },
    {
      "id": "2026-10-12-r3-123",
      "date": "2026-10-12",
      "title": "Round 3: live-alerts evidence correction 123",
      "pages": [
        "/tasks/live-alerts/"
      ],
      "before": "Notification overload is not really a volume problem",
      "after": "Notification overload involves volume and relevance",
      "why": "Independent round-three audit checked against the cited source and existing teardown evidence; unsupported inference removed.",
      "kind": "calibration"
    },
    {
      "id": "2026-10-12-r3-124",
      "date": "2026-10-12",
      "title": "Round 3: work-files evidence correction 124",
      "pages": [
        "/tasks/work-files/"
      ],
      "before": "Collateral damage is the dominant risk here, not failure to finish.",
      "after": "We treat collateral damage as an evaluation priority; we have not measured which risk is more frequent.",
      "why": "Independent round-three audit checked against the cited source and existing teardown evidence; unsupported inference removed.",
      "kind": "calibration"
    },
    {
      "id": "2026-10-12-r3-125",
      "date": "2026-10-12",
      "title": "Round 3: copilot-tasks evidence correction 125",
      "pages": [
        "/copilot-tasks/"
      ],
      "before": "Not us, and not in public with task-level results.",
      "after": "We have not tested it, and found no public task-by-task evaluation among the sources we reviewed.",
      "why": "Independent round-three audit checked against the cited source and existing teardown evidence; unsupported inference removed.",
      "kind": "calibration"
    },
    {
      "id": "2026-10-12-r3-126",
      "date": "2026-10-12",
      "title": "Round 3: copilot-tasks evidence correction 126",
      "pages": [
        "/copilot-tasks/"
      ],
      "before": "The change most people searching for Copilot Tasks will run into is the rename.",
      "after": "The product migration affects access, memory and schedules as well as the name.",
      "why": "Independent round-three audit checked against the cited source and existing teardown evidence; unsupported inference removed.",
      "kind": "calibration"
    },
    {
      "id": "2026-10-12-r3-127",
      "date": "2026-10-12",
      "title": "Round 3: copilot-tasks evidence correction 127",
      "pages": [
        "/copilot-tasks/"
      ],
      "before": "Several other write-ups of the launch, including PCWorld, Neowin and eWeek, returned errors to our tools. We have not used anything from them.",
      "after": "The claims here are limited to the accessible sources linked beside them.",
      "why": "Independent round-three audit checked against the cited source and existing teardown evidence; unsupported inference removed.",
      "kind": "calibration"
    },
    {
      "id": "2026-10-12-r3-128",
      "date": "2026-10-12",
      "title": "Round 3: cue-by-manus evidence correction 128",
      "pages": [
        "/cue-by-manus/"
      ],
      "before": "Until those have answers from somewhere other than Manus's own pages",
      "after": "Until those have answers from independent observation, rather than articles repeating Manus's claims",
      "why": "Independent round-three audit checked against the cited source and existing teardown evidence; unsupported inference removed.",
      "kind": "calibration"
    },
    {
      "id": "2026-10-12-r3-129",
      "date": "2026-10-12",
      "title": "Round 3: openai-dots evidence correction 129",
      "pages": [
        "/openai-dots/"
      ],
      "before": "you can talk to one in the ChatGPT mobile app once setup is done",
      "after": "you can talk to one in the ChatGPT mobile app once setup is done and when mobile access is available",
      "why": "Independent round-three audit checked against the cited source and existing teardown evidence; unsupported inference removed.",
      "kind": "calibration"
    },
    {
      "id": "2026-10-12-r3-130",
      "date": "2026-10-12",
      "title": "Round 3: openai-dots evidence correction 130",
      "pages": [
        "/openai-dots/"
      ],
      "before": "So the price of a dot is the price of Pro.",
      "after": "For individual access, a dot requires an eligible Pro plan; business and enterprise access have separate terms.",
      "why": "Independent round-three audit checked against the cited source and existing teardown evidence; unsupported inference removed.",
      "kind": "calibration"
    },
    {
      "id": "2026-10-12-r3-131",
      "date": "2026-10-12",
      "title": "Round 3: poke-ai evidence correction 131",
      "pages": [
        "/poke-ai/"
      ],
      "before": "We read that as a status page that has not kept up with the docs rather than proof either way.",
      "after": "A stale status page is one possible explanation, not a verified one; the source disagreement remains unresolved.",
      "why": "Independent round-three audit checked against the cited source and existing teardown evidence; unsupported inference removed.",
      "kind": "calibration"
    },
    {
      "id": "2026-10-12-r3-132",
      "date": "2026-10-12",
      "title": "Round 3: is-meta-muse-open-source evidence correction 132",
      "pages": [
        "/is-meta-muse-open-source/"
      ],
      "before": "On OpenMuse we could read every line that decides where your data goes.",
      "after": "OpenMuse's public repository can be inspected; we did not inspect hosted persistence internals or establish every data destination.",
      "why": "Independent round-three audit checked against the cited source and existing teardown evidence; unsupported inference removed.",
      "kind": "calibration"
    },
    {
      "id": "2026-10-12-r3-133",
      "date": "2026-10-12",
      "title": "Round 3: is-meta-muse-open-source evidence correction 133",
      "pages": [
        "/is-meta-muse-open-source/"
      ],
      "before": "one genuinely open model with Muse in its name",
      "after": "Glimmer, the open model discussed here",
      "why": "Independent round-three audit checked against the cited source and existing teardown evidence; unsupported inference removed.",
      "kind": "calibration"
    },
    {
      "id": "2026-10-12-r3-134",
      "date": "2026-10-12",
      "title": "nanoMuse can send web, tool, update and backup requests off-device",
      "pages": [
        "/nanomuse/"
      ],
      "before": "nothing of yours leaves except the calls to your model provider",
      "after": "relay chat collection is limited in key-only mode, but model, web, tool, update and backup requests can still leave the device",
      "why": "Independent round-three audit checked against the cited source and existing teardown evidence; unsupported inference removed.",
      "kind": "fact"
    },
    {
      "id": "2026-10-12-r3-135",
      "date": "2026-10-12",
      "title": "Round 3: nanomuse evidence correction 135",
      "pages": [
        "/nanomuse/"
      ],
      "before": "so it cannot operate an interface that lacks one",
      "after": "which the paper describes as a limitation of the examined browser path, not a verified limit of every Muse interface",
      "why": "Independent round-three audit checked against the cited source and existing teardown evidence; unsupported inference removed.",
      "kind": "calibration"
    },
    {
      "id": "2026-10-12-r3-136",
      "date": "2026-10-12",
      "title": "Round 3: browser-tasks evidence correction 136",
      "pages": [
        "/how-the-exam-works/browser-tasks/"
      ],
      "before": "These five isolate one skill each, so a failure names its cause.",
      "after": "These five narrow the diagnostic hypothesis; attributing a failure requires trace review to separate model, adapter, infrastructure and grading problems.",
      "why": "Independent round-three audit checked against the cited source and existing teardown evidence; unsupported inference removed.",
      "kind": "calibration"
    },
    {
      "id": "2026-10-12-r3-137",
      "date": "2026-10-12",
      "title": "Round 3: browser-tasks evidence correction 137",
      "pages": [
        "/how-the-exam-works/browser-tasks/"
      ],
      "before": "five covers the browser failures we see most often in public",
      "after": "five represents selected browser failure patterns; we have no measured frequency ranking",
      "why": "Independent round-three audit checked against the cited source and existing teardown evidence; unsupported inference removed.",
      "kind": "calibration"
    },
    {
      "id": "2026-10-12-r3-138",
      "date": "2026-10-12",
      "title": "Round 3: how-tasks-are-drawn evidence correction 138",
      "pages": [
        "/how-the-exam-works/how-tasks-are-drawn/"
      ],
      "before": "The rule guarantees the selection wasn't steered by results.",
      "after": "The draw algorithm does not read results; library construction, grouping and historical independence remain unverified.",
      "why": "Independent round-three audit checked against the cited source and existing teardown evidence; unsupported inference removed.",
      "kind": "calibration"
    },
    {
      "id": "2026-10-12-r3-139",
      "date": "2026-10-12",
      "title": "Round 3: how-the-exam-works evidence correction 139",
      "pages": [
        "/how-the-exam-works/"
      ],
      "before": "What we have done is remove the parts you would have to trust",
      "after": "The shared protocol reduces adapter differences, but private grading code and unpublished aggregate records still require trust",
      "why": "Independent round-three audit checked against the cited source and existing teardown evidence; unsupported inference removed.",
      "kind": "calibration"
    },
    {
      "id": "2026-10-12-r3-140",
      "date": "2026-10-12",
      "title": "Round 3: how-the-exam-works evidence correction 140",
      "pages": [
        "/how-the-exam-works/"
      ],
      "before": "so neither gets an easier exam because of how it happens to be built",
      "after": "so the interfaces share world state; we have not shown that browser and API interaction are equally difficult",
      "why": "Independent round-three audit checked against the cited source and existing teardown evidence; unsupported inference removed.",
      "kind": "calibration"
    },
    {
      "id": "2026-10-12-r3-141",
      "date": "2026-10-12",
      "title": "Round 3: how-the-exam-works evidence correction 141",
      "pages": [
        "/how-the-exam-works/"
      ],
      "before": "some batches are still being re-run",
      "after": "the September source note described reruns; we have no verified October rerun status",
      "why": "Independent round-three audit checked against the cited source and existing teardown evidence; unsupported inference removed.",
      "kind": "calibration"
    },
    {
      "id": "2026-10-12-r3-143",
      "date": "2026-10-12",
      "title": "Round 3: exam-changes evidence correction 143",
      "pages": [
        "/how-the-exam-works/changes/"
      ],
      "before": "Removes product-specific adapter differences",
      "after": "Reduces product-specific adapter differences",
      "why": "Independent round-three audit checked against the cited source and existing teardown evidence; unsupported inference removed.",
      "kind": "calibration"
    },
    {
      "id": "2026-10-12-r3-144",
      "date": "2026-10-12",
      "title": "Round 3: instinct evidence correction 144",
      "pages": [
        "/agents/instinct/"
      ],
      "before": "On the credit side, the model can't invent a parameter that doesn't exist, and a rarely used action behaves the same on its first call as on its hundredth. A made-up argument is exactly the error this gate is aimed at, and it rules that whole class out by construction.",
      "after": "The help gate requires the model to consult documented arguments before acting. It targets invented parameters, but we did not measure whether it eliminates invalid calls or establish a validator that makes them impossible. Reading the manual is an observable requirement; reliability across repeated or unfamiliar calls remains a separate question.",
      "why": "Independent round-three audit checked against the cited source and existing teardown evidence; unsupported inference removed.",
      "kind": "calibration"
    },
    {
      "id": "2026-10-12-r3-145",
      "date": "2026-10-12",
      "title": "Round 3: instinct evidence correction 145",
      "pages": [
        "/agents/instinct/"
      ],
      "before": "a help gate that stops the model inventing calls",
      "after": "a help gate intended to reduce invented calls",
      "why": "Independent round-three audit checked against the cited source and existing teardown evidence; unsupported inference removed.",
      "kind": "calibration"
    },
    {
      "id": "2026-10-12-r3-146",
      "date": "2026-10-12",
      "title": "Round 3: instinct evidence correction 146",
      "pages": [
        "/agents/instinct/"
      ],
      "before": "The surface is the product, and anybody patient enough can read it.",
      "after": "The examined surface reveals useful behaviour, but account flags and unavailable integrations limit what another user can reproduce.",
      "why": "Independent round-three audit checked against the cited source and existing teardown evidence; unsupported inference removed.",
      "kind": "calibration"
    },
    {
      "id": "2026-10-12-r3-147",
      "date": "2026-10-12",
      "title": "BrowseComp uses a grader model, not plain string comparison",
      "pages": [
        "/benchmarks/browsecomp/"
      ],
      "before": "By insisting on answers that are one name, one number or one date, BrowseComp removed the judge entirely. Scoring is a string comparison. Nobody argues about it, nobody can lobby for a better read of their output, and a result from last year is comparable with a result from today.",
      "after": "BrowseComp uses short reference answers, but the [official evaluator](https://github.com/openai/simple-evals/blob/main/browsecomp_eval.py) calls a grader model to extract the final answer and assess whether it matches the reference. It is not plain string comparison. Comparisons across dates still require the same dataset variant, tool access, model, budget and grading setup; a short answer alone does not make configurations interchangeable.",
      "why": "OpenAI simple-evals browsecomp_eval.py: BrowseCompEval accepts grader_model and grade_sample calls it. Primary source checked 12 October 2026.",
      "kind": "fact"
    },
    {
      "id": "2026-10-12-r3-148",
      "date": "2026-10-12",
      "title": "Round 3: browsecomp evidence correction 148",
      "pages": [
        "/benchmarks/browsecomp/"
      ],
      "before": "That is a fifty-point gap created by persistence rather than by intelligence. The models at the bottom of that table are not stupid; they are impatient. They search once, do not find it, and answer anyway.",
      "after": "That is a roughly fifty-point gap between the evaluated systems. Persistence may contribute, but the table does not isolate it from model, tooling or budget differences. It reports what these configurations achieved, not a controlled estimate of why one did better.",
      "why": "Independent round-three audit checked against the cited source and existing teardown evidence; unsupported inference removed.",
      "kind": "calibration"
    },
    {
      "id": "2026-10-12-r3-149",
      "date": "2026-10-12",
      "title": "Round 3: browsecomp evidence correction 149",
      "pages": [
        "/benchmarks/browsecomp/"
      ],
      "before": "Consider the alternative. A benchmark that asks for an assessment, a summary or a recommendation needs a judge, and a judge is either a person, which does not scale, or another model, which ends up measuring the agreement between two models rather than the quality of one. Every benchmark that has tried to score open-ended output has run into this, and most of them quietly became measurements of how much the grader liked the writing style.",
      "after": "Short reference answers narrow the grading question compared with open-ended assessments. They do not eliminate judgement: the official grader still decides whether the extracted response has meaningful differences from the reference. That makes the grading model and prompt part of the evaluation setup, alongside the browsing agent and its search budget.",
      "why": "Independent round-three audit checked against the cited source and existing teardown evidence; unsupported inference removed.",
      "kind": "calibration"
    },
    {
      "id": "2026-10-12-r3-150",
      "date": "2026-10-12",
      "title": "Round 3: terminal-bench evidence correction 150",
      "pages": [
        "/benchmarks/terminal-bench/"
      ],
      "before": "Read the intervals and the story changes. First place is 64.8% with a margin of 3.1 points, second is 61.8% with 2.9. Those ranges touch. The two models tied at third are genuinely indistinguishable, and the fifth-place score sits inside the range of the two above it. What the table actually establishes is roughly two groups, not five positions.",
      "after": "Read the intervals alongside the point estimates. First place is 64.8% with a margin of 3.1 points, second is 61.8% with 2.9. The rounded point estimates at third tie, but the table alone does not establish equivalence or exactly two performance groups. Marginal interval overlap is not a paired significance test; repeated task outcomes and an explicit comparison method would be needed.",
      "why": "Independent round-three audit checked against the cited source and existing teardown evidence; unsupported inference removed.",
      "kind": "calibration"
    },
    {
      "id": "2026-10-12-r3-151",
      "date": "2026-10-12",
      "title": "Round 3: terminal-bench evidence correction 151",
      "pages": [
        "/benchmarks/terminal-bench/"
      ],
      "before": "The top entry cost roughly 4.7 thousand dollars to reach its score.",
      "after": "The leaderboard reports roughly $4.7 thousand for the top entry, but the task/repeat aggregation basis has not been established here; do not treat that as a per-run buying quote.",
      "why": "Independent round-three audit checked against the cited source and existing teardown evidence; unsupported inference removed.",
      "kind": "calibration"
    },
    {
      "id": "2026-10-12-r3-152",
      "date": "2026-10-12",
      "title": "Round 3: muse-vs-pine evidence correction 152",
      "pages": [
        "/muse-vs-pine/"
      ],
      "before": "Pine has no file you can edit. There is a server-side persona and a structured fact layer queried on demand. Facts come back when something asks for them, which works and is cheaper to operate, and it means there is no object with your preferences in it, only a service that answers questions about you.",
      "after": "Pine has no editable local file that we established. It keeps a server-side persona and structured fact layer queried on demand. Both Pine and Muse therefore retain state, but expose different storage arrangements. We did not measure their relative operating costs, establish direct editing of Pine's facts, or verify personal access to Muse's memory files.",
      "why": "Independent round-three audit checked against the cited source and existing teardown evidence; unsupported inference removed.",
      "kind": "calibration"
    },
    {
      "id": "2026-10-12-r3-153",
      "date": "2026-10-12",
      "title": "Pine's fact service retains state too",
      "pages": [
        "/muse-vs-pine/"
      ],
      "before": "The cost of the Muse arrangement is that a machine per user is a machine per user. That is a real operating bill, and it is the reason most products on this bench do not do it. It also means the product carries state it has to keep safe, back up and eventually delete, none of which a stateless fact service has to think about.",
      "after": "Both storage arrangements carry retention, backup and deletion questions. A per-user VM and a server-side fact service may answer them differently, but the storage format alone does not show which is cheaper or better protected. Pine's fact layer is stateful, and its deletion boundary remains unverified.",
      "why": "Independent round-three audit checked against the cited source and existing teardown evidence; unsupported inference removed.",
      "kind": "fact"
    },
    {
      "id": "2026-10-12-r3-154",
      "date": "2026-10-12",
      "title": "Round 3: muse-vs-pine evidence correction 154",
      "pages": [
        "/muse-vs-pine/"
      ],
      "before": "The second is whether you want the agent to accumulate. A machine with files on it gets better at knowing you in a way a fact service does not, because the files are there to be added to and the agent assembles its own request on top of them. If the work you have in mind is a long relationship rather than a sequence of discrete jobs, that is the difference. If it is a sequence of discrete jobs, the persistence is overhead you are paying for.",
      "after": "The second question is how you want retained context to be exposed and corrected. File storage and a fact service can both accumulate information over time. We have not compared longitudinal learning, so neither format establishes that an agent will know you better after six months. Ask what can be inspected, changed and deleted, then verify those controls in your account.",
      "why": "Independent round-three audit checked against the cited source and existing teardown evidence; unsupported inference removed.",
      "kind": "calibration"
    },
    {
      "id": "2026-10-12-r3-155",
      "date": "2026-10-12",
      "title": "Round 3: muse-vs-town evidence correction 155",
      "pages": [
        "/muse-vs-town/"
      ],
      "before": "The consequence shows up on day one against month six. Town is useful immediately, because your screen is already full of context it can read. Muse starts emptier and should get better, because every run leaves something on the disk. A review written in the first week and a review written in the sixth month would disagree about these two products, and almost every comparison published anywhere is a first-week review.",
      "after": "Town's examined Mac route can read permitted screen and account context. Muse's examined web runtime retains files on its VM. These are different context sources, not measured time-to-value or longitudinal learning results. We have not tested whether a first-week comparison would reverse after six months, and the untested Muse Mac route further limits a product-wide comparison.",
      "why": "Independent round-three audit checked against the cited source and existing teardown evidence; unsupported inference removed.",
      "kind": "calibration"
    },
    {
      "id": "2026-10-12-r3-156",
      "date": "2026-10-12",
      "title": "Round 3: muse-vs-town evidence correction 156",
      "pages": [
        "/muse-vs-town/"
      ],
      "before": "Only one of them puts the consequences on hardware you own.",
      "after": "Town had verified native access to owned hardware in our examination; Muse’s Mac route was not tested. Either can create consequences through connected accounts.",
      "why": "Independent round-three audit checked against the cited source and existing teardown evidence; unsupported inference removed.",
      "kind": "calibration"
    },
    {
      "id": "2026-10-12-r3-157",
      "date": "2026-10-12",
      "title": "Round 3: muse-vs-town evidence correction 157",
      "pages": [
        "/muse-vs-town/"
      ],
      "before": "when it is wrong the wrongness is contained somewhere you do not have to clean up",
      "after": "a mistake through an external connector can still affect your accounts and require cleanup",
      "why": "Independent round-three audit checked against the cited source and existing teardown evidence; unsupported inference removed.",
      "kind": "calibration"
    },
    {
      "id": "2026-10-12-r3-158",
      "date": "2026-10-12",
      "title": "Round 3: muse-vs-town evidence correction 158",
      "pages": [
        "/muse-vs-town/"
      ],
      "before": "Both products keep something you can correct, which puts them ahead of most of this list",
      "after": "Town has a verified editable personality document; personal access to Muse’s memory files was not verified",
      "why": "Independent round-three audit checked against the cited source and existing teardown evidence; unsupported inference removed.",
      "kind": "calibration"
    },
    {
      "id": "2026-10-12-r3-159",
      "date": "2026-10-12",
      "title": "Round 3: openmuse evidence correction 159",
      "pages": [
        "/openmuse/"
      ],
      "before": "There is no list of things we could not establish about the runtime, because the runtime is in the repository.",
      "after": "Repository access does not establish deployed behaviour, security properties or hosted-service internals; we did not run a deployment.",
      "why": "Independent round-three audit checked against the cited source and existing teardown evidence; unsupported inference removed.",
      "kind": "calibration"
    },
    {
      "id": "2026-10-12-r3-160",
      "date": "2026-10-12",
      "title": "Round 3: openmuse evidence correction 160",
      "pages": [
        "/openmuse/"
      ],
      "before": "Nothing about the runtime. Everything below the hosted persistence boundary, which is a separate service.",
      "after": "Deployed behaviour, security assumptions, external side-effect recovery and hosted persistence internals; we did not deploy it.",
      "why": "Independent round-three audit checked against the cited source and existing teardown evidence; unsupported inference removed.",
      "kind": "calibration"
    },
    {
      "id": "2026-10-12-r3-161",
      "date": "2026-10-12",
      "title": "Round 3: openmuse evidence correction 161",
      "pages": [
        "/openmuse/"
      ],
      "before": "a lease is one of the few honest answers to it",
      "after": "a lease helps recover workers but does not alone prevent duplicate external side effects; the end-to-end idempotency strategy was not verified",
      "why": "Independent round-three audit checked against the cited source and existing teardown evidence; unsupported inference removed.",
      "kind": "calibration"
    },
    {
      "id": "2026-10-12-r3-162",
      "date": "2026-10-12",
      "title": "Round 3: openmuse evidence correction 162",
      "pages": [
        "/openmuse/"
      ],
      "before": "Every other write-up on this site exists because a product would not say how it worked",
      "after": "The four closed-product teardowns describe behaviour observed from outside",
      "why": "Independent round-three audit checked against the cited source and existing teardown evidence; unsupported inference removed.",
      "kind": "calibration"
    },
    {
      "id": "2026-10-12-r3-163",
      "date": "2026-10-12",
      "title": "Round 3: openmuse evidence correction 163",
      "pages": [
        "/openmuse/"
      ],
      "before": "Every other page here is an inference from the outside, and this one is not",
      "after": "The four closed-product teardowns rely on outside observations; this page instead describes public source, with deployment behaviour untested",
      "why": "Independent round-three audit checked against the cited source and existing teardown evidence; unsupported inference removed.",
      "kind": "calibration"
    },
    {
      "id": "2026-10-12-r3-164",
      "date": "2026-10-12",
      "title": "Round 3: how-we-take-them-apart evidence correction 164",
      "pages": [
        "/how-we-take-them-apart/"
      ],
      "before": "The useful realisation, and the one that makes all of this legal and most of it easy,",
      "after": "The useful realisation behind our account-based approach",
      "why": "Independent round-three audit checked against the cited source and existing teardown evidence; unsupported inference removed.",
      "kind": "calibration"
    },
    {
      "id": "2026-10-12-r3-165",
      "date": "2026-10-12",
      "title": "Round 3: how-we-take-them-apart evidence correction 165",
      "pages": [
        "/how-we-take-them-apart/"
      ],
      "before": "- **Model:** where inference runs and which model identity we can verify.",
      "after": "- **Execution:** where work runs and what it can touch.",
      "why": "Independent round-three audit checked against the cited source and existing teardown evidence; unsupported inference removed.",
      "kind": "calibration"
    },
    {
      "id": "2026-10-12-r3-166",
      "date": "2026-10-12",
      "title": "Round 3: how-we-take-them-apart evidence correction 166",
      "pages": [
        "/how-we-take-them-apart/"
      ],
      "before": "- **Channel:** where you reach the agent and where it can reach you.\n- **Triggers:** whether a message, a clock or an outside event can start work.",
      "after": "- **Channel:** where you reach the agent, including message, clock and event triggers.\n- **Unknowns:** what we could not establish.",
      "why": "Independent round-three audit checked against the cited source and existing teardown evidence; unsupported inference removed.",
      "kind": "calibration"
    },
    {
      "id": "2026-10-12-r3-167",
      "date": "2026-10-12",
      "title": "One of three evidence labels is unpublished, not two",
      "pages": [
        "/how-we-take-them-apart/"
      ],
      "before": "The three labels we use internally, and why two of them never reach you",
      "after": "Three evidence labels in an AI agent teardown, one excluded from publication",
      "why": "Independent round-three audit checked against the cited source and existing teardown evidence; unsupported inference removed.",
      "kind": "fact"
    },
    {
      "id": "2026-10-12-r3-168",
      "date": "2026-10-12",
      "title": "Round 3: how-we-take-them-apart evidence correction 168",
      "pages": [
        "/how-we-take-them-apart/"
      ],
      "before": "A single observation is not a finding. Where we could not reproduce something, we say so or leave it out.",
      "after": "Repeated observations support broader findings. Single-call discrepancies, such as Pine’s event and credit counts, remain single-case evidence.",
      "why": "Independent round-three audit checked against the cited source and existing teardown evidence; unsupported inference removed.",
      "kind": "calibration"
    },
    {
      "id": "2026-10-12-r3-169",
      "date": "2026-10-12",
      "title": "Round 3: how-we-take-them-apart evidence correction 169",
      "pages": [
        "/how-we-take-them-apart/"
      ],
      "before": "**How do you know a finding is real and not a one-off?**\nA single observation is not a finding here. Anything that made it onto these pages either reproduced across runs or was stated outright by the product. Where neither applies, it appears in the list of things we could not establish.",
      "after": "We seek repeat observations or explicit product statements. Some published findings, including Pine's call-history and credit discrepancies, come from one recorded case and should be read at that scope. They do not establish a frequency or universal behaviour; failures to reproduce remain part of the uncertainty.",
      "why": "Independent round-three audit checked against the cited source and existing teardown evidence; unsupported inference removed.",
      "kind": "calibration"
    },
    {
      "id": "2026-10-12-r3-170",
      "date": "2026-10-12",
      "title": "Round 3: how-we-take-them-apart evidence correction 170",
      "pages": [
        "/how-we-take-them-apart/"
      ],
      "before": "which is ordinary evaluation.",
      "after": "which describes our access boundary, not a legal conclusion. Contract terms, jurisdiction and the particular activity still matter.",
      "why": "Independent round-three audit checked against the cited source and existing teardown evidence; unsupported inference removed.",
      "kind": "calibration"
    },
    {
      "id": "2026-10-12-r3-171",
      "date": "2026-10-12",
      "title": "Round 3: home evidence correction 171",
      "pages": [
        "/"
      ],
      "before": "The method came first. The exam is repeatable: the same errands, in worlds that can be reset and replayed, judged mainly on what changed in the world; since 28 September, a separate reply score also checks whether the product truthfully reported what it did. Products get added as we take them apart and as they enter the exam, and the count below is where it has reached, not where it stops. What stays fixed is the method: the same axes for every product, the same refusal to publish a number we cannot defend, and a row in every column for what we failed to establish.",
      "after": "The exam design and the teardown axes have separate histories: the axes were derived from examining the products. The exam uses resettable simulated worlds and checks state changes; since 28 September, a separate reply score also checks truthful reporting. Published September measurements retain their original rules. The teardown table reports observed product architecture; the completion tables report adapted runtimes under exam conditions, not shipping products measured end to end.",
      "why": "Independent round-three audit checked against the cited source and existing teardown evidence; unsupported inference removed.",
      "kind": "calibration"
    },
    {
      "id": "2026-10-12-r3-172",
      "date": "2026-10-12",
      "title": "Round 3: home evidence correction 172",
      "pages": [
        "/"
      ],
      "before": "September completion rates were presented without an adjacent explanation of runtime replicas and model substitution.",
      "after": "The comparable batch used replicas with the same GLM 5.3 flash model; production-feature fidelity and complete transcript provenance are not established.",
      "why": "data/results.json comparable_batch maintainer_note and content/method/exam-changes.md: replicas, shared GLM 5.3 flash model, overwritten/shared transcript files.",
      "kind": "calibration"
    },
    {
      "id": "2026-10-12-r3-173",
      "date": "2026-10-12",
      "title": "Round 3: how-the-exam-works evidence correction 173",
      "pages": [
        "/how-the-exam-works/"
      ],
      "before": "September completion rates were presented without an adjacent explanation of runtime replicas and model substitution.",
      "after": "The comparable batch used replicas with the same GLM 5.3 flash model; production-feature fidelity and complete transcript provenance are not established.",
      "why": "data/results.json comparable_batch maintainer_note and content/method/exam-changes.md: replicas, shared GLM 5.3 flash model, overwritten/shared transcript files.",
      "kind": "calibration"
    },
    {
      "id": "2026-10-12-r3-174",
      "date": "2026-10-12",
      "title": "Round 3: exam-changes evidence correction 174",
      "pages": [
        "/how-the-exam-works/changes/"
      ],
      "before": "September completion rates were presented without an adjacent explanation of runtime replicas and model substitution.",
      "after": "The comparable batch used replicas with the same GLM 5.3 flash model; production-feature fidelity and complete transcript provenance are not established.",
      "why": "data/results.json comparable_batch maintainer_note and content/method/exam-changes.md: replicas, shared GLM 5.3 flash model, overwritten/shared transcript files.",
      "kind": "calibration"
    },
    {
      "id": "2026-10-12-r3-175",
      "date": "2026-10-12",
      "title": "Pine has the largest published sample: 167 tasks, not 159",
      "pages": [
        "/"
      ],
      "before": "The largest scored sample on the board and the widest lead that survives its own interval.",
      "after": "159 scored tasks; Pine has the largest published sample at 167. Marginal intervals alone do not establish a significant lead.",
      "why": "Independent round-three audit checked against the cited source and existing teardown evidence; unsupported inference removed.",
      "kind": "fact"
    },
    {
      "id": "2026-10-12-r3-176",
      "date": "2026-10-12",
      "title": "Round 3: results evidence correction 176",
      "pages": [
        "/"
      ],
      "before": "A second measurement, kept beside the first rather than replacing it. Every product answered the same 114 tasks, which the table above does not: those rates rest on different tasks as well as on different sample sizes, so they were never strictly comparable with each other.",
      "after": "A second measurement, kept beside the first rather than replacing it. Every product answered the same 114 tasks, which the table above does not: those rates rest on different tasks as well as on different sample sizes, so they were never strictly comparable with each other. These are replicas of product runtimes using the same substituted GLM 5.3 flash model, not shipping consumer products measured end to end.",
      "why": "data/results.json comparable_batch maintainer_note and content/method/exam-changes.md: replicas, shared GLM 5.3 flash model, overwritten/shared transcript files.",
      "kind": "calibration"
    },
    {
      "id": "2026-10-12-r3-177",
      "date": "2026-10-12",
      "title": "The batch has 1,649 transcript files for 2,076 attempts",
      "pages": [
        "/"
      ],
      "before": "The batch ships with transcripts covering the product side conversation, its final reply, and the requests the exam received. We have not republished them here.",
      "after": "Supplied transcripts have not been republished. The delivery contained 1649 transcript files for 2076 attempts, with 712 attempts sharing files after filename collisions; complete per-attempt provenance is not established.",
      "why": "data/results.json comparable_batch maintainer_note and content/method/exam-changes.md: replicas, shared GLM 5.3 flash model, overwritten/shared transcript files.",
      "kind": "fact"
    },
    {
      "id": "2026-10-12-r3-178",
      "date": "2026-10-12",
      "title": "Round 3: landscape evidence correction 178",
      "pages": [
        "/landscape/"
      ],
      "before": "Pine keeps a server-side fact layer you cannot.",
      "after": "Pine keeps a server-side fact layer whose direct inspection, editing and deletion controls we did not establish.",
      "why": "Independent round-three audit checked against the cited source and existing teardown evidence; unsupported inference removed.",
      "kind": "calibration"
    },
    {
      "id": "2026-10-12-r3-179",
      "date": "2026-10-12",
      "title": "Round 3: landscape evidence correction 179",
      "pages": [
        "/landscape/"
      ],
      "before": "That choice removes the confirmation screen every other kind of software leans on: there is no plan to review, no checkbox before it acts, no window showing a half finished thought.",
      "after": "Approval depends on the product: Instinct has an observed card with approve, deny and dismiss, bound to exact plan text. A messaging surface does not establish whether an action can bypass review.",
      "why": "Independent round-three audit checked against the cited source and existing teardown evidence; unsupported inference removed.",
      "kind": "calibration"
    },
    {
      "id": "2026-10-12-r3-180",
      "date": "2026-10-12",
      "title": "The directory also includes workplace tools and components",
      "pages": [
        "/landscape/"
      ],
      "before": "Chat assistants that only produce text are out, and so are tools sold to companies for their staff.",
      "after": "The directory includes consumer agents alongside related workplace products, frameworks and memory components. A listing is discovery evidence, not a claim that every entry is a shipping consumer assistant.",
      "why": "Independent round-three audit checked against the cited source and existing teardown evidence; unsupported inference removed.",
      "kind": "fact"
    },
    {
      "id": "2026-10-12-r3-181",
      "date": "2026-10-12",
      "title": "Round 3: landscape evidence correction 181",
      "pages": [
        "/landscape/"
      ],
      "before": "it is the difference between believing a privacy claim and checking one",
      "after": "it permits source inspection, while hosted dependencies and actual deployments still need separate verification",
      "why": "Independent round-three audit checked against the cited source and existing teardown evidence; unsupported inference removed.",
      "kind": "calibration"
    },
    {
      "id": "2026-10-12-r3-182",
      "date": "2026-10-12",
      "title": "Round 3: landscape evidence correction 182",
      "pages": [
        "/landscape/"
      ],
      "before": "The surface is decided before any feature is, and almost everything else follows from it.",
      "after": "The surface shapes interaction, but it does not by itself determine approvals, memory or reach.",
      "why": "Independent round-three audit checked against the cited source and existing teardown evidence; unsupported inference removed.",
      "kind": "calibration"
    },
    {
      "id": "2026-10-12-r3-183",
      "date": "2026-10-12",
      "title": "Round 3: browser-tasks evidence correction 183",
      "pages": [
        "/how-the-exam-works/browser-tasks/"
      ],
      "before": "A missing answer or a no means the ability isn't there",
      "after": "A missing answer or a no means the capability is undeclared under this exam protocol, not that the real product necessarily lacks it",
      "why": "Independent round-three audit checked against the cited source and existing teardown evidence; unsupported inference removed.",
      "kind": "calibration"
    },
    {
      "id": "2026-10-12-r3-184",
      "date": "2026-10-12",
      "title": "Round 3: how-the-exam-works evidence correction 184",
      "pages": [
        "/how-the-exam-works/"
      ],
      "before": "and it does not get credit for that, because each alert about nothing is counted as something sent out in your name that you didn't ask for",
      "after": "and can meet the urgent-event goal while each unwanted alert is separately counted as an unauthorised outward action",
      "why": "Independent round-three audit checked against the cited source and existing teardown evidence; unsupported inference removed.",
      "kind": "calibration"
    },
    {
      "id": "2026-10-12-r3-185",
      "date": "2026-10-12",
      "title": "Round 3: how-tasks-are-drawn evidence correction 185",
      "pages": [
        "/how-the-exam-works/how-tasks-are-drawn/"
      ],
      "before": "so the list can be checked against the rule that produced it.",
      "after": "but the full task list and selected IDs are not public, so readers cannot reproduce the draw from this page.",
      "why": "Independent round-three audit checked against the cited source and existing teardown evidence; unsupported inference removed.",
      "kind": "calibration"
    },
    {
      "id": "2026-10-12-r3-186",
      "date": "2026-10-12",
      "title": "Round 3: how-tasks-are-drawn evidence correction 186",
      "pages": [
        "/how-the-exam-works/how-tasks-are-drawn/"
      ],
      "before": "enough for a completion rate with a confidence interval tight enough to be worth printing. It is not enough to separate products that are close, and the scorecard will say so in the interval rather than in a ranking.",
      "after": "but attempt count alone does not establish interval precision. The estimator and its handling of repeated tasks and correlated groups have not been published here; uncertainty comparisons remain limited until those details and outcomes are available.",
      "why": "Independent round-three audit checked against the cited source and existing teardown evidence; unsupported inference removed.",
      "kind": "calibration"
    },
    {
      "id": "2026-10-12-r3-187",
      "date": "2026-10-12",
      "title": "Round 3: how-tasks-are-drawn evidence correction 187",
      "pages": [
        "/how-the-exam-works/how-tasks-are-drawn/"
      ],
      "before": "In particular, no task was removed because a product did badly on it in rehearsal, and no task was added because a product did well.",
      "after": "The draw note says it did not select on product results; without the full historical inputs we cannot independently verify that account.",
      "why": "Independent round-three audit checked against the cited source and existing teardown evidence; unsupported inference removed.",
      "kind": "calibration"
    },
    {
      "id": "2026-10-12-r3-188",
      "date": "2026-10-12",
      "title": "Round 3: checking-the-grader evidence correction 188",
      "pages": [
        "/how-the-exam-works/checking-the-grader/"
      ],
      "before": "Each of these produces a result that looks exactly like a product failure in a table.",
      "after": "These defects can create false failures or false successes in a table.",
      "why": "Independent round-three audit checked against the cited source and existing teardown evidence; unsupported inference removed.",
      "kind": "calibration"
    },
    {
      "id": "2026-10-12-r3-189",
      "date": "2026-10-12",
      "title": "Round 3: checking-the-grader evidence correction 189",
      "pages": [
        "/how-the-exam-works/checking-the-grader/"
      ],
      "before": "After the fixes, the same check on 21 September found 6.",
      "after": "After the fixes, the same check on 21 September found 6. The available record does not identify the disposition of the five non-reference tasks or whether they entered a published batch; this remains unresolved.",
      "why": "Independent round-three audit checked against the cited source and existing teardown evidence; unsupported inference removed.",
      "kind": "calibration"
    },
    {
      "id": "2026-10-12-r3-190",
      "date": "2026-10-12",
      "title": "Round 3: checking-the-grader evidence correction 190",
      "pages": [
        "/how-the-exam-works/checking-the-grader/"
      ],
      "before": "The exam stores enough with every run to replay it: the task as it was frozen, the seed, and every action with the world's state before and after.",
      "after": "The 128 records checked for replay contained frozen tasks, seeds and action/state traces. That does not establish replayability of every batch: some common-batch transcript files were overwritten or shared, and the full replay artefacts are not public.",
      "why": "Independent round-three audit checked against the cited source and existing teardown evidence; unsupported inference removed.",
      "kind": "calibration"
    },
    {
      "id": "2026-10-12-r3-192",
      "date": "2026-10-12",
      "title": "Round 3: exam-changes evidence correction 192",
      "pages": [
        "/how-the-exam-works/changes/"
      ],
      "before": "repository revision `cfd6d38e`",
      "after": "repository revision `cfd6d38e` (an internal reference, not a publicly inspectable contract diff)",
      "why": "Independent round-three audit checked against the cited source and existing teardown evidence; unsupported inference removed.",
      "kind": "calibration"
    },
    {
      "id": "2026-10-12-r3-193",
      "date": "2026-10-12",
      "title": "Round 3: claude-dispatch evidence correction 193",
      "pages": [
        "/claude-dispatch/"
      ],
      "before": "The agent is only reachable while one specific machine is on, and that machine holds everything it can touch.",
      "after": "Desktop execution requires that machine to remain awake, but connected and cloud resources do not all reside on its disk.",
      "why": "Independent round-three audit checked against the cited source and existing teardown evidence; unsupported inference removed.",
      "kind": "calibration"
    },
    {
      "id": "2026-10-12-r3-194",
      "date": "2026-10-12",
      "title": "Dispatch routes work to Claude Code as well as Cowork",
      "pages": [
        "/claude-dispatch/"
      ],
      "before": "No. Anthropic's Dispatch is the Cowork feature on this page.",
      "after": "The name needs context. Anthropic’s official Dispatch can route development tasks into Claude Code and knowledge work into Cowork, according to its [help article](https://support.claude.com/en/articles/13947068-assign-tasks-from-anywhere-in-claude-cowork).",
      "why": "Independent round-three audit checked against the cited source and existing teardown evidence; unsupported inference removed.",
      "kind": "fact"
    },
    {
      "id": "2026-10-12-r3-196",
      "date": "2026-10-12",
      "title": "Our state-checking scripts can evaluate exam outcomes",
      "pages": [
        "/benchmarks/swe-bench/"
      ],
      "before": "which is checkable but not by running a test suite",
      "after": "which our state-checking scripts can evaluate; they also track timing, permissions and unwanted changes beyond a single pass/fail result",
      "why": "Independent round-three audit checked against the cited source and existing teardown evidence; unsupported inference removed.",
      "kind": "fact"
    },
    {
      "id": "2026-10-12-r3-198",
      "date": "2026-10-12",
      "title": "Round 3: tau-bench evidence correction 198",
      "pages": [
        "/benchmarks/tau-bench/"
      ],
      "before": "Evaluation does not read the transcript for signs of competence. It compares the state of the database at the end of the conversation with an annotated goal state.",
      "after": "For the original state-change tasks, evaluation compares the final database with the annotated goal state rather than treating a persuasive transcript as success.",
      "why": "Independent round-three audit checked against the cited source and existing teardown evidence; unsupported inference removed.",
      "kind": "calibration"
    },
    {
      "id": "2026-10-12-r3-199",
      "date": "2026-10-12",
      "title": "Round 3: tau-bench evidence correction 199",
      "pages": [
        "/benchmarks/tau-bench/"
      ],
      "before": "What it costs is the awkwardness of real people. The simulated user states its need, responds to questions, and stays on topic. It does not change its mind halfway, mention the important constraint only after the booking is made, or reply to a clarifying question with something that does not answer it.",
      "after": "The simulated user is a model operating under version-specific instructions, not an observed representative customer. Its response patterns and constraints limit realism; neither perfect consistency nor every possible human interruption is established. The [official repository](https://github.com/sierra-research/tau-bench) supplies the simulator setup for the original version, and later variants must be read against their own protocols.",
      "why": "Independent round-three audit checked against the cited source and existing teardown evidence; unsupported inference removed.",
      "kind": "calibration"
    },
    {
      "id": "2026-10-12-r3-200",
      "date": "2026-10-12",
      "title": "Round 3: tau-bench evidence correction 200",
      "pages": [
        "/benchmarks/tau-bench/"
      ],
      "before": "The user is simulated. That is a reasonable engineering decision and it has a cost: a simulated user is more consistent, more patient, and more legible than the person whose subscription it is. Nobody in tau bench loses their temper, goes quiet for two days, or answers a question with a screenshot.",
      "after": "A simulated user limits what the evaluation establishes about real customer behaviour. Patience, ambiguous replies and interruptions depend on the chosen simulator and task instructions; a score does not show how the agent handles every real caller. Keep the simulator version beside the agent version when comparing results.",
      "why": "Independent round-three audit checked against the cited source and existing teardown evidence; unsupported inference removed.",
      "kind": "calibration"
    },
    {
      "id": "2026-10-12-r3-201",
      "date": "2026-10-12",
      "title": "Round 3: tau-bench evidence correction 201",
      "pages": [
        "/benchmarks/tau-bench/"
      ],
      "before": "The original state-change grading explanation was applied without distinguishing the displayed knowledge-domain results.",
      "after": "Database-state grading is scoped to original tasks; the newer knowledge-domain grading implementation is not independently verified here.",
      "why": "Independent round-three audit checked against the cited source and existing teardown evidence; unsupported inference removed.",
      "kind": "calibration"
    },
    {
      "id": "2026-10-12-r3-202",
      "date": "2026-10-12",
      "title": "Round 3: town evidence correction 202",
      "pages": [
        "/agents/town/"
      ],
      "before": "The worst it can do is bounded by those seventeen tools. That's the entire argument for per-agent allow-lists",
      "after": "The tool allow-list narrows the actions offered to that writer. Tool count alone does not establish a small impact: the permissions and effects of each action still matter. That is the reason to examine per-agent allow-lists",
      "why": "Independent round-three audit checked against the cited source and existing teardown evidence; unsupported inference removed.",
      "kind": "calibration"
    },
    {
      "id": "2026-10-12-r3-205",
      "date": "2026-10-12",
      "title": "Removed an unverified Grok Bot social-media task count",
      "pages": [
        "/grok-bot/"
      ],
      "before": "In a single unverified post on 5 October, one X user, [@gippp69](https://x.com/gippp69/status/2107153066134790406), reported giving six bots one overnight job and waking to 612 finished tasks, 47 blocked actions and 89 rejected handoffs. The roles were split on purpose: one bot built a candidate list, one scored it, a risk bot returned pass, cut or kill, and only the execution bot could take the final action, after a timestamped pass. Whatever else it is, it is somebody treating the irreversible-action problem seriously inside a multi-agent setup.",
      "after": "We removed an unverified social-media task count from this comparison. No task list, traces or grading method accompanied it, so it cannot establish completion or approval reliability. The official workflow examples can guide a first trial, but they are vendor instructions rather than independently reproduced outcomes. A useful report would publish the task, permitted actions, blocked steps and final artefact.",
      "why": "Independent round-three audit checked against the cited source and existing teardown evidence; unsupported inference removed.",
      "kind": "fact"
    },
    {
      "id": "2026-10-12-r3-207",
      "date": "2026-10-12",
      "title": "Round 3: openmuse evidence correction 207",
      "pages": [
        "/openmuse/"
      ],
      "before": "An open project was supposed to have an empty list. Instead it has two lines, and both are boundaries the project drew itself.",
      "after": "Published source still leaves deployment, recovery and security properties unverified. The hosted persistence and telemetry boundaries are explicit examples, not an exhaustive list.",
      "why": "Independent round-three audit checked against the cited source and existing teardown evidence; unsupported inference removed.",
      "kind": "calibration"
    },
    {
      "id": "2026-10-12-r3-208",
      "date": "2026-10-12",
      "title": "OpenMuse's documented setup requires hosted persistence",
      "pages": [
        "/openmuse/"
      ],
      "before": "The local sample app needs no model, no Google account and no Docker, and runs on fictional data.",
      "after": "The local sample app needs no model, no Google account and no Docker, and runs on fictional data. Before setup, read the [telemetry opt-out instructions](https://github.com/CopilotKit/openmuse/blob/main/docs/TELEMETRY.md). Hosted Intelligence persistence is required in the documented setup; we have not verified a persistence opt-out. We could not establish an immutable revision for this source review, so this is not a pinned, tested installation recipe.",
      "why": "Independent round-three audit checked against the cited source and existing teardown evidence; unsupported inference removed.",
      "kind": "fact"
    },
    {
      "id": "2026-10-12-r3-210",
      "date": "2026-10-12",
      "title": "Round 3: where-it-runs evidence correction 210",
      "pages": [
        "/where-it-runs/"
      ],
      "before": "Inference may use a hosted API, a subscription or a supported local model.",
      "after": "Inference may use a hosted API, a subscription or a supported local model; OpenClaw’s [provider directory](https://docs.openclaw.ai/providers) includes llama.cpp.",
      "why": "Independent round-three audit checked against the cited source and existing teardown evidence; unsupported inference removed.",
      "kind": "calibration"
    },
    {
      "id": "2026-10-12-r3-211",
      "date": "2026-10-12",
      "title": "Round 3: where-it-runs evidence correction 211",
      "pages": [
        "/where-it-runs/"
      ],
      "before": "It is the one cost that grows with how much you ask the agent to do.",
      "after": "Hosted usage can grow with activity; local inference instead adds hardware and electricity requirements.",
      "why": "Independent round-three audit checked against the cited source and existing teardown evidence; unsupported inference removed.",
      "kind": "calibration"
    },
    {
      "id": "2026-10-12-r3-213",
      "date": "2026-10-12",
      "title": "Round 3: landscape evidence correction 213",
      "pages": [
        "/landscape/"
      ],
      "before": "All unrecognised evidence URLs labelled Official site.",
      "after": "Product Hunt and Engadget identified as listing and news; unknown source types labelled Source link. Purpose and access boundaries shown per row.",
      "why": "Existing data/landscape.json evidence URLs and dated maintainer notes.",
      "kind": "calibration"
    },
    {
      "id": "2026-10-12-r3-214",
      "date": "2026-10-12",
      "title": "Round 3: how-the-exam-works evidence correction 214",
      "pages": [
        "/how-the-exam-works/"
      ],
      "before": "It uses older builds and rules",
      "after": "It uses replicas with the same substituted GLM 5.3 flash model, older builds and rules",
      "why": "Independent round-three audit checked against the cited source and existing teardown evidence; unsupported inference removed.",
      "kind": "calibration"
    },
    {
      "id": "2026-10-12-r3-215",
      "date": "2026-10-12",
      "title": "Round 3: webarena evidence correction 215",
      "pages": [
        "/benchmarks/webarena/"
      ],
      "before": "Its task set and step budget differ, so its scores do not extend the original WebArena board.",
      "after": "That 300-task count is the July workshop snapshot, not a claim about the latest release. We could not verify the later OpenReview count during this update, so no newer total is asserted here. Its task set and budget differ from the original board.",
      "why": "Independent round-three audit checked against the cited source and existing teardown evidence; unsupported inference removed.",
      "kind": "calibration"
    },
    {
      "id": "2026-10-12-r3-216",
      "date": "2026-10-12",
      "title": "Round 3: scorecard evidence correction 216",
      "pages": [
        "/"
      ],
      "before": "Deletion is unusually concrete for this list: workspace files can be removed like files, and a forget capability sits among its skills.",
      "after": "The examined runtime includes file removal and a forget capability. Whether users can directly open or remove their memory files was not verified.",
      "why": "Independent round-three audit checked against the cited source and existing teardown evidence; unsupported inference removed.",
      "kind": "calibration"
    },
    {
      "id": "2026-10-12-r3-217",
      "date": "2026-10-12",
      "title": "Round 3: how-scoring-works evidence correction 217",
      "pages": [
        "/how-scoring-works/"
      ],
      "before": "Muse's workspace files can be removed like files, and Town's memory has its own add, update and delete tools.",
      "after": "Muse's runtime exposes file-removal and forget capabilities, but personal access was not verified. Town's memory has add, update and delete tools.",
      "why": "Independent round-three audit checked against the cited source and existing teardown evidence; unsupported inference removed.",
      "kind": "calibration"
    },
    {
      "id": "2026-10-12-r3-218",
      "date": "2026-10-12",
      "title": "Round 3: results evidence correction 218",
      "pages": [
        "/"
      ],
      "before": "Every figure is a share of tasks completed, with a 95% interval and the number of scored tasks it rests on. There is no combined score and no ranking, because a rate without its interval and its sample size is not a result.",
      "after": "Every figure is a share of tasks completed, with a 95% interval and the number of scored tasks it rests on. There is no combined score and no ranking, because a rate without its interval and its sample size is not a result. These September rates describe adapted runtimes using the exam model, on differing scored subsets; they are not consumer-product end-to-end performance.",
      "why": "Existing comparable_batch.maintainer_note and method/exam-changes.md model substitution.",
      "kind": "calibration"
    },
    {
      "id": "2026-10-12-r3-219",
      "date": "2026-10-12",
      "title": "Round 3: dispute-a-charge evidence correction 219",
      "pages": [
        "/tasks/dispute-a-charge/"
      ],
      "before": "failure most likely to be invisible in a demo",
      "after": "failure that can be invisible in a demo",
      "why": "Independent round-three audit checked against the cited source and existing teardown evidence; unsupported inference removed.",
      "kind": "calibration"
    },
    {
      "id": "2026-10-12-r3-220",
      "date": "2026-10-12",
      "title": "Round 3: flight-delay-compensation evidence correction 220",
      "pages": [
        "/tasks/flight-delay-compensation/"
      ],
      "before": "failure most likely to be invisible in a demo",
      "after": "failure that can be invisible in a demo",
      "why": "Independent round-three audit checked against the cited source and existing teardown evidence; unsupported inference removed.",
      "kind": "calibration"
    },
    {
      "id": "2026-10-12-r3-221",
      "date": "2026-10-12",
      "title": "Round 3: book-a-table evidence correction 221",
      "pages": [
        "/tasks/book-a-table/"
      ],
      "before": "failure most likely to be invisible in a demo",
      "after": "failure that can be invisible in a demo",
      "why": "Independent round-three audit checked against the cited source and existing teardown evidence; unsupported inference removed.",
      "kind": "calibration"
    },
    {
      "id": "2026-10-12-r3-222",
      "date": "2026-10-12",
      "title": "Round 3: work-files evidence correction 222",
      "pages": [
        "/tasks/work-files/"
      ],
      "before": "failure most likely to be invisible in a demo",
      "after": "failure that can be invisible in a demo",
      "why": "Independent round-three audit checked against the cited source and existing teardown evidence; unsupported inference removed.",
      "kind": "calibration"
    },
    {
      "id": "2026-10-12-round4-about",
      "date": "2026-10-12",
      "title": "Evidence limits: about",
      "pages": [
        "/about/"
      ],
      "before": "Every page identifies all evidence context.",
      "after": "Result labels now identify the September adapted-runtime scope and missing records.",
      "why": "Corrects a published factual overstatement or inconsistency identified in the Round 4 evidence audit; unresolved evidence is not replaced with a new claim.",
      "kind": "calibration"
    },
    {
      "id": "2026-10-12-round4-agents-instinct",
      "date": "2026-10-12",
      "title": "Evidence limits: instinct",
      "pages": [
        "/agents/instinct/"
      ],
      "before": "Leased browsing is safer and more expensive; messaging has no buttons or editable plans.",
      "after": "Observed session separation only; security/cost unmeasured; approvals observed, editable plan unverified.",
      "why": "Corrects a published factual overstatement or inconsistency identified in the Round 4 evidence audit; unresolved evidence is not replaced with a new claim.",
      "kind": "calibration"
    },
    {
      "id": "2026-10-12-round4-agents-muse",
      "date": "2026-10-12",
      "title": "Town also filters permissions separately from tool definitions",
      "pages": [
        "/agents/muse/"
      ],
      "before": "Only Muse separates permissions from definitions.",
      "after": "Muse has a parameter-free permission sheet; Town also filters permissions separately.",
      "why": "Corrects a published factual overstatement or inconsistency identified in the Round 4 evidence audit; unresolved evidence is not replaced with a new claim.",
      "kind": "fact"
    },
    {
      "id": "2026-10-12-round4-agents-pine",
      "date": "2026-10-12",
      "title": "Evidence limits: pine",
      "pages": [
        "/agents/pine/"
      ],
      "before": "Nothing can be corrected; personal plans are charged weekly.",
      "after": "No directly editable local memory file; corrections go through the product; weekly display is not renewal cadence.",
      "why": "Corrects a published factual overstatement or inconsistency identified in the Round 4 evidence audit; unresolved evidence is not replaced with a new claim.",
      "kind": "calibration"
    },
    {
      "id": "2026-10-12-round4-agents-town",
      "date": "2026-10-12",
      "title": "Evidence limits: town",
      "pages": [
        "/agents/town/"
      ],
      "before": "The 17-tool writer has almost no reach and exactly what it needs.",
      "after": "Reach depends on permissions and effects; necessity and containment are unverified.",
      "why": "Corrects a published factual overstatement or inconsistency identified in the Round 4 evidence audit; unresolved evidence is not replaced with a new claim.",
      "kind": "calibration"
    },
    {
      "id": "2026-10-12-round4-agents",
      "date": "2026-10-12",
      "title": "Evidence limits: agents",
      "pages": [
        "/agents/"
      ],
      "before": "Web-search chat assistants cannot retain state or touch accounts.",
      "after": "Persistence and account actions depend on the product and examined mode.",
      "why": "Corrects a published factual overstatement or inconsistency identified in the Round 4 evidence audit; unresolved evidence is not replaced with a new claim.",
      "kind": "calibration"
    },
    {
      "id": "2026-10-12-round4-benchmarks-agentbench",
      "date": "2026-10-12",
      "title": "Evidence limits: agentbench",
      "pages": [
        "/benchmarks/agentbench/"
      ],
      "before": "Every figure is relative without a human baseline; all benchmarks start cold.",
      "after": "FC pass@1 is an absolute task-success rate; other benchmarks may include seeded history.",
      "why": "Corrects a published factual overstatement or inconsistency identified in the Round 4 evidence audit; unresolved evidence is not replaced with a new claim.",
      "kind": "calibration"
    },
    {
      "id": "2026-10-12-round4-benchmarks-assistantbench",
      "date": "2026-10-12",
      "title": "Evidence limits: assistantbench",
      "pages": [
        "/benchmarks/assistantbench/"
      ],
      "before": "Fixed tasks are cheap and easy to saturate; public sets stop measuring capability; nobody can score during the outage.",
      "after": "Cost and saturation are unmeasured; tuning is a risk, not a proven cause; only the public hosted route was verified unavailable. Time labels do not measure drift.",
      "why": "Corrects a published factual overstatement or inconsistency identified in the Round 4 evidence audit; unresolved evidence is not replaced with a new claim.",
      "kind": "calibration"
    },
    {
      "id": "2026-10-12-round4-benchmarks-browsecomp",
      "date": "2026-10-12",
      "title": "Evidence limits: browsecomp",
      "pages": [
        "/benchmarks/browsecomp/"
      ],
      "before": "Nobody ships the repeated-sampling configuration; human agreement proves unambiguous questions.",
      "after": "Consumer availability and price are unverified; conditional agreement does not diagnose all failures or ambiguity.",
      "why": "Corrects a published factual overstatement or inconsistency identified in the Round 4 evidence audit; unresolved evidence is not replaced with a new claim.",
      "kind": "calibration"
    },
    {
      "id": "2026-10-12-round4-benchmarks-gaia",
      "date": "2026-10-12",
      "title": "Evidence limits: gaia",
      "pages": [
        "/benchmarks/gaia/"
      ],
      "before": "Hidden answers stop memorisation; public scores are self-reported.",
      "after": "Hidden answers do not rule out contamination; board submissions do not establish independently verified execution.",
      "why": "Corrects a published factual overstatement or inconsistency identified in the Round 4 evidence audit; unresolved evidence is not replaced with a new claim.",
      "kind": "calibration"
    },
    {
      "id": "2026-10-12-round4-benchmarks-osworld",
      "date": "2026-10-12",
      "title": "Evidence limits: osworld",
      "pages": [
        "/benchmarks/osworld/"
      ],
      "before": "Products silently spend six times the tokens for better flight rebooking; no user history exists.",
      "after": "Cross-model points do not establish product behaviour or causal gains; version 2.0 includes seeded history.",
      "why": "Corrects a published factual overstatement or inconsistency identified in the Round 4 evidence audit; unresolved evidence is not replaced with a new claim.",
      "kind": "calibration"
    },
    {
      "id": "2026-10-12-round4-benchmarks-swe-bench",
      "date": "2026-10-12",
      "title": "Our exam uses frozen definitions; SWE-bench harness versions differ",
      "pages": [
        "/benchmarks/swe-bench/"
      ],
      "before": "Our exam is neither frozen nor mechanical and has no flight-change test suite; all rows share one harness.",
      "after": "Our exam uses frozen definitions and state scripts; aggregate evidence is unavailable; mini-SWE-agent versions differ.",
      "why": "Corrects a published factual overstatement or inconsistency identified in the Round 4 evidence audit; unresolved evidence is not replaced with a new claim.",
      "kind": "fact"
    },
    {
      "id": "2026-10-12-round4-benchmarks-tau-bench",
      "date": "2026-10-12",
      "title": "Evidence limits: tau bench",
      "pages": [
        "/benchmarks/tau-bench/"
      ],
      "before": "The database-goal check applies without qualification.",
      "after": "Scope that check to original retail/airline tasks; displayed banking-knowledge implementation remains unverified.",
      "why": "Corrects a published factual overstatement or inconsistency identified in the Round 4 evidence audit; unresolved evidence is not replaced with a new claim.",
      "kind": "calibration"
    },
    {
      "id": "2026-10-12-round4-benchmarks-terminal-bench",
      "date": "2026-10-12",
      "title": "Evidence limits: terminal bench",
      "pages": [
        "/benchmarks/terminal-bench/"
      ],
      "before": "Lower cost trades off against precision.",
      "after": "The cross-model cost/interval association does not establish causation.",
      "why": "Corrects a published factual overstatement or inconsistency identified in the Round 4 evidence audit; unresolved evidence is not replaced with a new claim.",
      "kind": "calibration"
    },
    {
      "id": "2026-10-12-round4-benchmarks-webarena",
      "date": "2026-10-12",
      "title": "Evidence limits: webarena",
      "pages": [
        "/benchmarks/webarena/"
      ],
      "before": "All products made an undisclosed caution tradeoff; resets guarantee comparability; humans set the ceiling.",
      "after": "Only the paper experiment establishes the tradeoff; matching versions/settings are needed; the human sample is not a ceiling.",
      "why": "Corrects a published factual overstatement or inconsistency identified in the Round 4 evidence audit; unresolved evidence is not replaced with a new claim.",
      "kind": "calibration"
    },
    {
      "id": "2026-10-12-round4-benchmarks",
      "date": "2026-10-12",
      "title": "Our exam uses adapted runtimes; Persona2Web uses the open web",
      "pages": [
        "/benchmarks/"
      ],
      "before": "We run errands against real products; every task has no history; all research worlds are simulated.",
      "after": "We use adapted runtimes with a substituted exam model; seeded history exists; Persona2Web uses the open web.",
      "why": "Corrects a published factual overstatement or inconsistency identified in the Round 4 evidence audit; unresolved evidence is not replaced with a new claim.",
      "kind": "fact"
    },
    {
      "id": "2026-10-12-round4-copilot-tasks",
      "date": "2026-10-12",
      "title": "Evidence limits: copilot tasks",
      "pages": [
        "/copilot-tasks/"
      ],
      "before": "Consumer Tasks and business Cowork imply distinct product identities.",
      "after": "Distinguish personal and organisational account/licensing modes, including personal Cowork migration.",
      "why": "Corrects a published factual overstatement or inconsistency identified in the Round 4 evidence audit; unresolved evidence is not replaced with a new claim.",
      "kind": "calibration"
    },
    {
      "id": "2026-10-12-round4-grok-bot",
      "date": "2026-10-12",
      "title": "Evidence limits: grok bot",
      "pages": [
        "/grok-bot/"
      ],
      "before": "Two retained posts describe use.",
      "after": "One retained usage post; the removed post is discussed separately.",
      "why": "Corrects a published factual overstatement or inconsistency identified in the Round 4 evidence audit; unresolved evidence is not replaced with a new claim.",
      "kind": "calibration"
    },
    {
      "id": "2026-10-12-round4-home",
      "date": "2026-10-12",
      "title": "Evidence limits: home",
      "pages": [
        "/"
      ],
      "before": "Only Muse separates permission from definition; reruns are still happening.",
      "after": "Observed Muse sheet without exclusivity; reruns were reported in September and October status is unverified.",
      "why": "Corrects a published factual overstatement or inconsistency identified in the Round 4 evidence audit; unresolved evidence is not replaced with a new claim.",
      "kind": "calibration"
    },
    {
      "id": "2026-10-12-round4-how-scoring-works",
      "date": "2026-10-12",
      "title": "Evidence limits: how scoring works",
      "pages": [
        "/how-scoring-works/"
      ],
      "before": "All loaded context is billed to consumers; Pine personal plans bill weekly.",
      "after": "Context overhead does not establish consumer billing; Pine prices use weekly display units.",
      "why": "Corrects a published factual overstatement or inconsistency identified in the Round 4 evidence audit; unresolved evidence is not replaced with a new claim.",
      "kind": "calibration"
    },
    {
      "id": "2026-10-12-round4-how-we-take-them-apart",
      "date": "2026-10-12",
      "title": "Evidence limits: how we take them apart",
      "pages": [
        "/how-we-take-them-apart/"
      ],
      "before": "Every claim is reproducible by using the product.",
      "after": "Reproduction depends on account, available integrations and examined surface; model traffic remains inaccessible.",
      "why": "Corrects a published factual overstatement or inconsistency identified in the Round 4 evidence audit; unresolved evidence is not replaced with a new claim.",
      "kind": "calibration"
    },
    {
      "id": "2026-10-12-round4-instinct-vs-town",
      "date": "2026-10-12",
      "title": "Evidence limits: instinct vs town",
      "pages": [
        "/instinct-vs-town/"
      ],
      "before": "Too many tools makes models choose badly; all 571 Instinct actions are exposed.",
      "after": "Observed loading mechanisms only; 335 full manuals recovered and not all actions verified reachable.",
      "why": "Corrects a published factual overstatement or inconsistency identified in the Round 4 evidence audit; unresolved evidence is not replaced with a new claim.",
      "kind": "calibration"
    },
    {
      "id": "2026-10-12-round4-is-meta-muse-open-source",
      "date": "2026-10-12",
      "title": "Evidence limits: is meta muse open source",
      "pages": [
        "/is-meta-muse-open-source/"
      ],
      "before": "Both projects borrowed the name because Muse defined their category.",
      "after": "Naming does not establish affiliation or maintainer motivation.",
      "why": "Corrects a published factual overstatement or inconsistency identified in the Round 4 evidence audit; unresolved evidence is not replaced with a new claim.",
      "kind": "calibration"
    },
    {
      "id": "2026-10-12-round4-landscape",
      "date": "2026-10-12",
      "title": "Evidence limits: landscape",
      "pages": [
        "/landscape/"
      ],
      "before": "Town has no published rate; every source label identifies its type.",
      "after": "Town first-batch figure is withheld, with a qualified older common-task rate; some labels remain generic. Components are grouped separately.",
      "why": "Corrects a published factual overstatement or inconsistency identified in the Round 4 evidence audit; unresolved evidence is not replaced with a new claim.",
      "kind": "calibration"
    },
    {
      "id": "2026-10-12-round4-method-browser-tasks",
      "date": "2026-10-12",
      "title": "Evidence limits: browser tasks",
      "pages": [
        "/how-the-exam-works/browser-tasks/"
      ],
      "before": "Failure identifies the failed skill; undeclared means incapable; the next order number will differ.",
      "after": "Trace review locates cause; capability is undeclared in this configuration; the task requires rereading even unchanged values.",
      "why": "Corrects a published factual overstatement or inconsistency identified in the Round 4 evidence audit; unresolved evidence is not replaced with a new claim.",
      "kind": "calibration"
    },
    {
      "id": "2026-10-12-round4-method-checking-the-grader",
      "date": "2026-10-12",
      "title": "Evidence limits: checking the grader",
      "pages": [
        "/how-the-exam-works/checking-the-grader/"
      ],
      "before": "Every published batch survived the checks; aggregate manual review is established.",
      "after": "Current gates do not prove historical acceptance; manual review is an internal report with missing manifests and dispositions.",
      "why": "Corrects a published factual overstatement or inconsistency identified in the Round 4 evidence audit; unresolved evidence is not replaced with a new claim.",
      "kind": "calibration"
    },
    {
      "id": "2026-10-12-round4-method-how-tasks-are-drawn",
      "date": "2026-10-12",
      "title": "Evidence limits: how tasks are drawn",
      "pages": [
        "/how-the-exam-works/how-tasks-are-drawn/"
      ],
      "before": "Historical selection never used results; the attempt count guarantees an honest interval.",
      "after": "Independence is internally reported and unverifiable from missing inputs; precision depends on outcomes and grouped/repeated-task estimation.",
      "why": "Corrects a published factual overstatement or inconsistency identified in the Round 4 evidence audit; unresolved evidence is not replaced with a new claim.",
      "kind": "calibration"
    },
    {
      "id": "2026-10-12-round4-method-how-the-exam-works",
      "date": "2026-10-12",
      "title": "Evidence limits: how the exam works",
      "pages": [
        "/how-the-exam-works/"
      ],
      "before": "Equal hashes prove equal actions; all published tasks include reply grading.",
      "after": "Hashes establish captured final-state equality; reply grading belongs to the current protocol and is absent from the published September aggregates.",
      "why": "Corrects a published factual overstatement or inconsistency identified in the Round 4 evidence audit; unresolved evidence is not replaced with a new claim.",
      "kind": "calibration"
    },
    {
      "id": "2026-10-12-round4-muse-vs-instinct",
      "date": "2026-10-12",
      "title": "Evidence limits: muse vs instinct",
      "pages": [
        "/muse-vs-instinct/"
      ],
      "before": "Messaging cannot show an editable plan and asks only yes/no.",
      "after": "Editable plan unverified on the examined account; the observed card offers approve, deny and dismiss.",
      "why": "Corrects a published factual overstatement or inconsistency identified in the Round 4 evidence audit; unresolved evidence is not replaced with a new claim.",
      "kind": "calibration"
    },
    {
      "id": "2026-10-12-round4-muse-vs-pine",
      "date": "2026-10-12",
      "title": "Evidence limits: muse vs pine",
      "pages": [
        "/muse-vs-pine/"
      ],
      "before": "Only Muse persists state; file edits are cheaper/easier; only Pine calls.",
      "after": "Both retain state; relative costs and user file access are unverified; Muse calling is reported early access. Weekly prices do not establish billing cadence.",
      "why": "Corrects a published factual overstatement or inconsistency identified in the Round 4 evidence audit; unresolved evidence is not replaced with a new claim.",
      "kind": "calibration"
    },
    {
      "id": "2026-10-12-round4-muse-vs-town",
      "date": "2026-10-12",
      "title": "Evidence limits: muse vs town",
      "pages": [
        "/muse-vs-town/"
      ],
      "before": "Damage stays inside the VM; only local evidence exists; neither can call.",
      "after": "Connector effects can persist externally; approval records can be server-side; calling was not examined and Muse reports early access.",
      "why": "Corrects a published factual overstatement or inconsistency identified in the Round 4 evidence audit; unresolved evidence is not replaced with a new claim.",
      "kind": "calibration"
    },
    {
      "id": "2026-10-12-round4-nanomuse",
      "date": "2026-10-12",
      "title": "Evidence limits: nanomuse",
      "pages": [
        "/nanomuse/"
      ],
      "before": "Only model-provider calls leave the device in key-only mode.",
      "after": "Model, web/tool, update and configured backup routes may leave; limited relay collection is not a network privacy guarantee.",
      "why": "Corrects a published factual overstatement or inconsistency identified in the Round 4 evidence audit; unresolved evidence is not replaced with a new claim.",
      "kind": "calibration"
    },
    {
      "id": "2026-10-12-round4-pine-vs-instinct",
      "date": "2026-10-12",
      "title": "Evidence limits: pine vs instinct",
      "pages": [
        "/pine-vs-instinct/"
      ],
      "before": "Pine cannot be enumerated and personal plans bill weekly.",
      "after": "26 names were enumerated, with schemas/computer-use incomplete; weekly display is not billing cadence.",
      "why": "Corrects a published factual overstatement or inconsistency identified in the Round 4 evidence audit; unresolved evidence is not replaced with a new claim.",
      "kind": "calibration"
    },
    {
      "id": "2026-10-12-round4-pine-vs-town",
      "date": "2026-10-12",
      "title": "Evidence limits: pine vs town",
      "pages": [
        "/pine-vs-town/"
      ],
      "before": "Everything except prompts is observable; both initiate unattended work; Pine bills weekly.",
      "after": "Carrier/model/runtime gaps remain; external API triggers do not establish autonomous initiation; weekly display is not cadence.",
      "why": "Corrects a published factual overstatement or inconsistency identified in the Round 4 evidence audit; unresolved evidence is not replaced with a new claim.",
      "kind": "calibration"
    },
    {
      "id": "2026-10-12-round4-tasks-book-a-table",
      "date": "2026-10-12",
      "title": "Evidence limits: book a table",
      "pages": [
        "/tasks/book-a-table/"
      ],
      "before": "Holding two tables proves task completion.",
      "after": "Goal success with duplicates depends on the actual checker; the section explicitly addresses diners.",
      "why": "Corrects a published factual overstatement or inconsistency identified in the Round 4 evidence audit; unresolved evidence is not replaced with a new claim.",
      "kind": "calibration"
    },
    {
      "id": "2026-10-12-round4-tasks-dispute-a-charge",
      "date": "2026-10-12",
      "title": "Billing-error notice must not wait for merchant resolution",
      "pages": [
        "/tasks/dispute-a-charge/"
      ],
      "before": "Refunds may always be refused and preserve relationships; contact the issuer only after merchant failure.",
      "after": "Refund rights depend on law/contract and relationships are not guaranteed; billing-error notice must not await merchant resolution.",
      "why": "Corrects a published factual overstatement or inconsistency identified in the Round 4 evidence audit; unresolved evidence is not replaced with a new claim.",
      "kind": "fact"
    },
    {
      "id": "2026-10-12-round4-tasks-flight-delay-compensation",
      "date": "2026-10-12",
      "title": "Earlier-departure refund thresholds are three or six hours",
      "pages": [
        "/tasks/flight-delay-compensation/"
      ],
      "before": "Any earlier departure qualifies; closing with a voucher completes the task.",
      "after": "Earlier-departure thresholds are three hours domestic/six international; claim closure does not prove the requested remedy. Voucher limits are scoped to the covered refund.",
      "why": "Corrects a published factual overstatement or inconsistency identified in the Round 4 evidence audit; unresolved evidence is not replaced with a new claim.",
      "kind": "fact"
    },
    {
      "id": "2026-10-12-round4-tasks-work-files",
      "date": "2026-10-12",
      "title": "Evidence limits: work files",
      "pages": [
        "/tasks/work-files/"
      ],
      "before": "Overaction is the dominant risk; subscription cancellation has no collateral damage.",
      "after": "Collateral damage is a priority without comparative frequency evidence; cancellation also measures it.",
      "why": "Corrects a published factual overstatement or inconsistency identified in the Round 4 evidence audit; unresolved evidence is not replaced with a new claim.",
      "kind": "calibration"
    },
    {
      "id": "2026-10-12-round4-where-it-runs",
      "date": "2026-10-12",
      "title": "Evidence limits: where it runs",
      "pages": [
        "/where-it-runs/"
      ],
      "before": "Self-hosting guarantees no external traffic, lessor or vendor logs; deployment cannot change.",
      "after": "External services and rented hosts remain possible; logging and deployment choices depend on configuration and route.",
      "why": "Corrects a published factual overstatement or inconsistency identified in the Round 4 evidence audit; unresolved evidence is not replaced with a new claim.",
      "kind": "calibration"
    },
    {
      "id": "2026-10-12-round4-generated-1",
      "date": "2026-10-12",
      "title": "Result scope beside generated figures",
      "pages": [
        "/",
        "/agents/instinct/",
        "/agents/muse/",
        "/agents/pine/",
        "/agents/town/",
        "/instinct-vs-town/",
        "/muse-vs-instinct/",
        "/muse-vs-pine/",
        "/muse-vs-town/",
        "/pine-vs-instinct/",
        "/pine-vs-town/"
      ],
      "before": "Generated result labels omitted scope and called comparisons the whole exam.",
      "after": "Labels now give 2026-09-22, adapted runtimes, substituted exam model, differing subsets, unpublished first-batch records and Muse adapter repair, with a provenance link.",
      "why": "Generated copy must agree with the dated result and pricing records.",
      "kind": "calibration"
    },
    {
      "id": "2026-10-12-round4-generated-2",
      "date": "2026-10-12",
      "title": "Mixed dates in Town pricing",
      "pages": [
        "/agents/town/"
      ],
      "before": "All Town pricing was labelled checked on 12 October.",
      "after": "Monthly paid prices are the 23 September snapshot; annual prices, trial and allowances were checked on 12 October. Table and note distinguish dates.",
      "why": "Generated copy must agree with the dated result and pricing records.",
      "kind": "fact"
    },
    {
      "id": "2026-10-12-open-source-evidence-scope",
      "date": "2026-10-12",
      "title": "Source availability does not establish every runtime behavior",
      "pages": [
        "/openmuse/",
        "/nanomuse/"
      ],
      "before": "OpenMuse said nothing about its runtime remained unestablished. The nanoMuse chat FAQ stated community-relay training defaults without separating the service claim from inspected source defaults.",
      "after": "OpenMuse retains unknowns for approval coverage, memory erasure and runtime egress. nanoMuse separates published community-relay claims from source configuration: sync on, training contribution off unless enabled by deployment; live relay settings are not established.",
      "why": "The pinned REA-assisted review at /open-source-personal-agents/ distinguishes manual behavioral evidence, partial JavaScript analysis and unobserved deployment settings.",
      "kind": "calibration"
    },
    {
      "id": "2026-10-12-rea-existing-mentions",
      "date": "2026-10-12",
      "title": "Self-hosting, firmware and the scope of source reviews",
      "pages": [
        "/where-it-runs/",
        "/is-meta-muse-open-source/",
        "/landscape/",
        "/"
      ],
      "before": "The hosting page said self-hosting keeps data on your hardware, removes the lessor and vendor logs, and repeated a no-operating-system description of MimiClaw. Other pages described OpenMuse as the sole source review or implied its source established every data destination.",
      "after": "Self-hosting does not establish local-only data, telemetry defaults or logging; rented infrastructure can remain involved. MimiClaw uses ESP-IDF/FreeRTOS. Fifteen agents have static source reviews, distinct from runtime testing, with complete egress and hosted-service internals still unestablished. The MIRIX landscape surface reflects the reviewed server implementation.",
      "why": "The pinned REA evidence packet and manual source findings identify firmware, telemetry defaults, configurable execution and incomplete egress coverage. Existing mentions now link to /open-source-personal-agents/ and the individual reviews.",
      "kind": "fact"
    },
    {
      "id": "2026-10-10-credentials",
      "date": "2026-10-10",
      "title": "What we say about credentials and closed doors",
      "pages": [
        "/agents/muse/",
        "/how-we-take-them-apart/"
      ],
      "before": "We said we never went looking for a way around a closed door, and that we record a closed door rather than looking for a window.",
      "after": "We say what holds: we never obtain or use credentials, checking whether a door is closed from inside our own account is part of the work, and a closed door is recorded as a finding and not broken open.",
      "why": "Our own notes record checks made from inside our account. The old sentence claimed more restraint than the record supports, so it was the sentence that had to change.",
      "kind": "fact"
    },
    {
      "id": "2026-10-10-muse-calls",
      "date": "2026-10-10",
      "title": "Muse is getting business calling, in early access",
      "pages": [
        "/muse-vs-pine/"
      ],
      "before": "Pine phones companies on your behalf and Muse does not.",
      "after": "Meta's own guidance inside Muse says business calling is in early access and reaching accounts gradually. The account we examined never received it, so we have not seen a Muse call happen.",
      "why": "Guidance that ships inside the product, collected after the comparison was written, describes the calling feature and its limits.",
      "kind": "fact"
    },
    {
      "id": "2026-10-10-instinct-confirmation",
      "date": "2026-10-10",
      "title": "Instinct does have a confirmation step",
      "pages": [
        "/agents/instinct/",
        "/muse-vs-instinct/"
      ],
      "before": "Living in a message thread costs Instinct the confirmation step the other products rely on.",
      "after": "A message thread costs it an editable plan. It keeps a confirmation step: an approval card with approve, deny or dismiss, bound to the exact text it showed you.",
      "why": "Our own scorecard already described the approval card. The two pages contradicted it.",
      "kind": "fact"
    },
    {
      "id": "2026-10-10-town-counts",
      "date": "2026-10-10",
      "title": "Town's template and assistant counts",
      "pages": [
        "/agents/town/",
        "/agents/",
        "/about/",
        "/"
      ],
      "before": "Town's template runs to 24 sections, and Town has 33 specialised assistants, each with its own instructions.",
      "after": "The main template we collected has 11 sections, and 24 is only what a fuller version is said to run to. Of 33 assistant records, 29 carry their own instructions and 4 are shells whose work is done in code we never saw.",
      "why": "Our teardown notes were revised after the pages were written. We had printed the claimed figure as if we had seen it.",
      "kind": "fact"
    },
    {
      "id": "2026-10-10-pine-transcript",
      "date": "2026-10-10",
      "title": "Which Pine calls keep a transcript",
      "pages": [
        "/agents/pine/"
      ],
      "before": "Pine's phone calls keep no verbatim transcript.",
      "after": "Calls made from the app keep no verbatim transcript. Calls placed through Pine's developer API return a turn-by-turn transcript.",
      "why": "A call we placed through the developer API came back with a full transcript. The original sentence was true of the app and stated too broadly.",
      "kind": "fact"
    },
    {
      "id": "2026-10-10-instinct-memory",
      "date": "2026-10-10",
      "title": "What Instinct's three memory stores are",
      "pages": [
        "/agents/instinct/",
        "/"
      ],
      "before": "Instinct's memory was described as a read-only markdown set, an observation log and a fill-only vault.",
      "after": "A git-tracked set of markdown memory files, an indexed layer of connector data it can query, and a fill-only vault for secrets.",
      "why": "The scorecard summary contradicted the teardown page and our own notes. Nothing in the notes supports read-only or an observation log; the second store is indexed external data that can be queried.",
      "kind": "fact"
    }
  ]
}