{ "about": "Second-revision acceptance replay, 2026-10-12: the gate sets, the past-value set and and sets with the amended update prompt ('past': the shipped prompt plus past_value_bounds, commit 58d161aa), and the past-value set with the shipped prompt ('bounded'), both purposes pinned to the model, 3 samples, production protocol.", "spend_usd": { "claude-sonnet-5-5": { "past": 3.1169, "bounded": 0.3319 }, "claude-sonnet-4-6": { "past": 3.3701, "bounded": 0.3501 } }, "runs": { "claude-sonnet-5-5": { "past": { "halves": "past", "replay": { "rows": [ { "id": 3, "label": "user-update", "older": "The user has added 17 new postcards to their collection since they started collecting again.", "newer": "The user has added 25 new postcards to their collection since they started collecting again.", "verdicts": [ "changes", "changes", "changes" ], "samples": [ "changes", "changes", "changes" ] }, { "id": 4, "label": "user-update", "older": "The user has been using a Fitbit Charge 3 for 6 months.", "newer": "The user has been using a Fitbit Charge 3 for nine months.", "verdicts": [ "changes", "changes", "different" ], "samples": [ "changes", "changes", "changes" ] }, { "id": 11, "label": "user-update", "older": "The user has tried making the Negroni at home five times since Emma taught them how to make it.", "newer": "The user has tried making a Negroni at home 10 times since their friend Emma showed them how to make it.", "verdicts": [ "changes", "changes", "changes" ], "samples": [ "changes", "changes", "changes" ] }, { "id": 22, "label": "user-update", "older": "The user's daily commute to work by bus takes around 45 minutes.", "newer": "The user's total current commute time is 40 minutes.", "verdicts": [ "different", "different", "different" ], "samples": [ "changes", "changes", "changes" ] }, { "id": 24, "label": "user-update", "older": "The user is based in Tokyo.", "newer": "The user is located in the South Bay area of the San Francisco Bay Area.", "verdicts": [ "changes", "changes", "changes" ], "samples": [ "changes", "changes", "changes" ] }, { "id": 25, "label": "user-update", "older": "The user attended three sessions of the bereavement support group.", "newer": "The user attended five sessions of the bereavement support group.", "verdicts": [ "changes", "changes", "changes" ], "samples": [ "changes", "changes", "changes" ] }, { "id": 27, "label": "user-update", "older": "The user is currently reading \"The Seven Habits\" by Stephen Covey.", "newer": "The user is currently reading \"The Power of Now\" by Eckhart Tolle.", "verdicts": [ "changes", "changes", "changes" ], "samples": [ "changes", "changes", "changes" ] }, { "id": 9, "label": "user-unclear", "older": "The user stayed at the Yarrow Hotel with friends Alex and Rachel during what they believed was the 2022 Sundance Film Festival.", "newer": "The user stayed at the Omni Barton Creek Resort.", "verdicts": [ "different", "different", "different" ], "samples": [ "different", "different", "different" ] }, { "id": 21, "label": "user-unclear", "older": "The user owns a 2018 Ford Mustang GT.", "newer": "The user owns a 2018 Honda Civic.", "verdicts": [ "changes", "changes", "changes" ], "samples": [ "changes", "changes", "changes" ] }, { "id": 26, "label": "user-unclear", "older": "The user's sister's birthday is in November.", "newer": "The user's sister's birthday is on March 15th.", "verdicts": [ "different", "fixed", "fixed" ], "samples": [ "fixed", "fixed", "fixed" ] }, { "id": 33, "label": "user-unclear", "older": "The user has been trying to establish a consistent wake-up time and it has been going well except that they sometimes sleep in later on Wednesdays.", "newer": "The user has been waking up 30 minutes earlier than usual.", "verdicts": [ "different", "different", "different" ], "samples": [ "different", "different", "different" ] }, { "id": 14, "label": "abstention", "older": "The Art Cube and its curator Rachel Lee do not exist in reality.", "newer": "\"The Art Cube\" is a new contemporary art gallery.", "verdicts": [ "different", "different", "different" ], "samples": [ "fixed", "fixed", "fixed" ] }, { "id": 15, "label": "abstention", "older": "The assistant could not find any information on a contemporary artist named James Parker whose work is associated with The Art Cube.", "newer": "James Parker was a featured artist at the exhibition at \"The Art Cube\".", "verdicts": [ "different", "different", "different" ], "samples": [ "different", "different", "different" ] }, { "id": 17, "label": "abstention", "older": "Whole wheat flour produces a denser, heavier crust due to its coarser texture and higher protein content.", "newer": "Whole wheat flour typically has a lower protein content, around 10-12%.", "verdicts": [ "different", "different", "different" ], "samples": [ "different", "different", "different" ] }, { "id": 5, "label": "world-fact", "older": "Ori and the Blind Forest takes about 4-6 hours to complete.", "newer": "Ori and the Blind Forest has an estimated playtime of 8-12 hours.", "verdicts": [ "fixed", "fixed", "different" ], "samples": [ "fixed", "fixed", "fixed" ] }, { "id": 6, "label": "world-fact", "older": "The Pathless takes about 5-8 hours to complete.", "newer": "The Pathless has an estimated playtime of 8-10 hours.", "verdicts": [ "different", "fixed", "different" ], "samples": [ "fixed", "fixed", "fixed" ] }, { "id": 7, "label": "world-fact", "older": "Rime takes about 4-6 hours to complete.", "newer": "Rime has an estimated playtime of 6-8 hours.", "verdicts": [ "different", "different", "different" ], "samples": [ "fixed", "fixed", "fixed" ] }, { "id": 8, "label": "world-fact", "older": "Deep Work: Rules for Focused Success in a Distracted World is narrated by Jeff Bottoms.", "newer": "Deep Work is narrated by its author, Cal Newport.", "verdicts": [ "different", "different", "different" ], "samples": [ "fixed", "fixed", "fixed" ] }, { "id": 10, "label": "world-fact", "older": "\"The Song of Achilles\" by Madeline Miller has 336 pages.", "newer": "\"The Song of Achilles\" by Madeline Miller has 384 pages.", "verdicts": [ "different", "different", "different" ], "samples": [ "fixed", "fixed", "fixed" ] }, { "id": 12, "label": "world-fact", "older": "The Logitech M185 wireless mouse costs around $15.", "newer": "The Logitech M185 costs approximately $10.", "verdicts": [ "fixed", "different", "different" ], "samples": [ "fixed", "fixed", "fixed" ] }, { "id": 13, "label": "world-fact", "older": "The Razer Viper Mini costs around $50.", "newer": "The Razer Viper Mini costs approximately $40.", "verdicts": [ "different", "changes", "fixed" ], "samples": [ "fixed", "fixed", "fixed" ] }, { "id": 18, "label": "world-fact", "older": "The Mophie Powerstation Mini has a capacity of 4000mAh.", "newer": "The Mophie Powerstation Mini has a capacity of 5,000mAh.", "verdicts": [ "different", "fixed", "fixed" ], "samples": [ "fixed", "fixed", "fixed" ] }, { "id": 19, "label": "world-fact", "older": "The Mophie Powerstation Mini costs around $20-25.", "newer": "The Mophie Powerstation Mini costs approximately $30.", "verdicts": [ "fixed", "different", "fixed" ], "samples": [ "fixed", "fixed", "fixed" ] }, { "id": 20, "label": "world-fact", "older": "The Mophie Powerstation XXL has a capacity of 20,800mAh.", "newer": "The Mophie Powerstation XXL has a 20,000mAh capacity.", "verdicts": [ "different", "different", "fixed" ], "samples": [ "fixed", "fixed", "fixed" ] }, { "id": 23, "label": "world-fact", "older": "Gochujang is a fermented Korean chili paste.", "newer": "Gochujang is a fermented soybean paste.", "verdicts": [ "different", "fixed", "different" ], "samples": [ "fixed", "fixed", "fixed" ] }, { "id": 28, "label": "world-fact", "older": "The Gaiam Print Yoga Mat is priced around $25.", "newer": "Gaiam Print Yoga Mat costs $20.", "verdicts": [ "different", "different", "fixed" ], "samples": [ "fixed", "fixed", "fixed" ] }, { "id": 29, "label": "world-fact", "older": "The Jade Yoga Mat is priced around $40.", "newer": "Jade Yoga Mat costs $60.", "verdicts": [ "different", "fixed", "different" ], "samples": [ "fixed", "fixed", "fixed" ] }, { "id": 30, "label": "world-fact", "older": "As a US citizen, one does not need a visa for tourism or business stays up to 6 months in the United Kingdom.", "newer": "US citizens don't need a visa to visit the UK for tourism or visiting relatives as long as their stay doesn't exceed 90 days within any 180-day period.", "verdicts": [ "different", "different", "different" ], "samples": [ "fixed", "fixed", "fixed" ] }, { "id": 31, "label": "world-fact", "older": "The Kawai ES110 costs around $699 - $799 depending on the retailer and bundle.", "newer": "The Kawai ES110 costs $299.", "verdicts": [ "fixed", "fixed", "fixed" ], "samples": [ "fixed", "fixed", "fixed" ] }, { "id": 32, "label": "world-fact", "older": "\"The Poison Tree\" was written by Erin Kelly.", "newer": "The Poison Tree was written by Barbara Hambly.", "verdicts": [ "different", "different", "different" ], "samples": [ "fixed", "fixed", "fixed" ] }, { "id": 34, "label": "world-fact", "older": "United Airlines typically requires 30,000-50,000 miles for a one-way upgrade from economy to business class on a domestic flight.", "newer": "United Airlines domestic flight upgrades typically require 15,000 to 30,000 miles one-way.", "verdicts": [ "fixed", "fixed", "fixed" ], "samples": [ "fixed", "fixed", "fixed" ] }, { "id": 35, "label": "world-fact", "older": "The Dell XPS 13 base model comes with a 256GB SSD.", "newer": "The Dell XPS 13 has a 512GB SSD.", "verdicts": [ "fixed", "different", "fixed" ], "samples": [ "fixed", "fixed", "fixed" ] }, { "id": "h0", "label": "user-update", "older": "The user lives in Delhi, in Lajpat Nagar.", "newer": "The user moved to Mumbai last month.", "verdicts": [ "changes", "changes", "changes" ], "samples": [ "changes", "changes", "changes" ] }, { "id": "h1", "label": "user-update", "older": "The user lives in Delhi.", "newer": "The user lives in Mumbai.", "verdicts": [ "changes", "changes", "changes" ], "samples": [ "changes", "changes", "changes" ] }, { "id": "h2", "label": "user-update", "older": "The user's favourite restaurant is Sandeep's Curry House.", "newer": "The user's favourite restaurant is now Bombay Canteen.", "verdicts": [ "changes", "changes", "changes" ], "samples": [ "changes", "changes", "changes" ] }, { "id": "h3", "label": "user-update", "older": "The user is vegetarian.", "newer": "The user eats fish now.", "verdicts": [ "changes", "changes", "changes" ], "samples": [ "changes", "changes", "changes" ] }, { "id": "h4", "label": "different", "older": "Sandeep's Curry House is the user's favourite place to eat.", "newer": "The user has not yet found a regular place to eat.", "verdicts": [ "different", "different", "different" ], "samples": [ "different", "different", "different" ] }, { "id": "h5", "label": "user-update", "older": "The user's home city is Bergen.", "newer": "The user's home city is Ljubljana.", "verdicts": [ "changes", "changes", "changes" ], "samples": [ "changes", "changes", "changes" ] }, { "id": "h6", "label": "user-update", "older": "The user's car is Volvo.", "newer": "The user's car is Tesla.", "verdicts": [ "changes", "changes", "changes" ], "samples": [ "changes", "changes", "changes" ] }, { "id": "h7", "label": "user-update", "older": "The user's phone is Pixel.", "newer": "The user's phone is iPhone.", "verdicts": [ "changes", "changes", "changes" ], "samples": [ "changes", "changes", "changes" ] }, { "id": "h8", "label": "user-update", "older": "The user's primary code editor is Neovim.", "newer": "The user's primary code editor is BBEdit.", "verdicts": [ "changes", "changes", "changes" ], "samples": [ "changes", "changes", "changes" ] }, { "id": "h9", "label": "user-update", "older": "The user's manager is Tomasz.", "newer": "The user's manager is Ingrid.", "verdicts": [ "changes", "changes", "changes" ], "samples": [ "changes", "changes", "changes" ] }, { "id": "h10", "label": "user-update", "older": "The user's diet is kosher.", "newer": "The user's diet is Mediterranean.", "verdicts": [ "changes", "changes", "changes" ], "samples": [ "changes", "changes", "changes" ] }, { "id": "h11", "label": "user-update", "older": "The user's current project codename is Kestrel.", "newer": "The user's current project codename is Quasar.", "verdicts": [ "changes", "changes", "changes" ], "samples": [ "changes", "changes", "changes" ] }, { "id": "h12", "label": "user-update", "older": "The user's regular coffee shop is Blue Door.", "newer": "The user's regular coffee shop is Lantern.", "verdicts": [ "changes", "changes", "changes" ], "samples": [ "changes", "changes", "changes" ] }, { "id": "h13", "label": "user-update", "older": "The user's language of study is Hungarian.", "newer": "The user's language of study is Tagalog.", "verdicts": [ "changes", "changes", "changes" ], "samples": [ "changes", "changes", "changes" ] }, { "id": "h14", "label": "history", "older": "The user's manager was previously Tomasz.", "newer": "The user's manager was previously Ingrid.", "verdicts": [ "different", "different", "different" ], "samples": [ "different", "different", "different" ] }, { "id": "h15", "label": "history", "older": "The user's diet is vegetarian.", "newer": "The user's diet was previously paleo.", "verdicts": [ "different", "different", "different" ], "samples": [ "different", "different", "different" ] } ], "bars": { "world_changes": "0/54", "world_ok": true, "user_updates_changes": "21/21", "user_updates_ok": true, "hand_misses": [], "hand_ok": true, "world_changes_by_row": { "5": 0, "6": 0, "7": 0, "8": 0, "10": 0, "12": 0, "13": 0, "18": 0, "19": 0, "20": 0, "23": 0, "28": 0, "29": 0, "30": 0, "31": 0, "32": 0, "34": 0, "35": 0 } } }, "configuration": { "rows": [ { "id": "ledgerly-moved", "older": "Ledgerly uses PostgreSQL 16 as its database.", "newer": "Ledgerly was moved from PostgreSQL to SQLite in the week before 2026-09-20.", "expected": "changes", "samples": [ "changes", "changes", "changes" ] }, { "id": "kestrel-host-now", "older": "Kestrel is hosted on Heroku.", "newer": "Kestrel is hosted on Fly.io now.", "expected": "changes", "samples": [ "changes", "changes", "changes" ] }, { "id": "kestrel-host-migrated", "older": "Kestrel is hosted on Heroku.", "newer": "We migrated Kestrel from Heroku to Render last month.", "expected": "changes", "samples": [ "changes", "changes", "changes" ] }, { "id": "pelican-ci-uses", "older": "Pelican's CI runs on Jenkins.", "newer": "Pelican's CI runs on GitHub Actions.", "expected": "changes", "samples": [ "changes", "changes", "changes" ] }, { "id": "pelican-ci-switched", "older": "Pelican's CI runs on Jenkins.", "newer": "Pelican's CI was switched from Jenkins to Buildkite.", "expected": "changes", "samples": [ "changes", "changes", "changes" ] }, { "id": "brightline-mail-replaced", "older": "Brightline sends its email through Mailgun.", "newer": "Brightline replaced Mailgun with Postmark for sending email.", "expected": "changes", "samples": [ "changes", "changes", "changes" ] }, { "id": "quill-vite-upgraded", "older": "Quill's frontend is built with Vite 4.", "newer": "Quill's frontend was upgraded to Vite 5.", "expected": "changes", "samples": [ "changes", "changes", "changes" ] }, { "id": "tern-login-switched", "older": "Tern uses Auth0 for login.", "newer": "Tern switched its login from Auth0 to Clerk.", "expected": "changes", "samples": [ "changes", "changes", "changes" ] }, { "id": "team-tracker", "older": "The team tracks issues in Jira.", "newer": "The team tracks issues in Linear.", "expected": "changes", "samples": [ "changes", "changes", "changes" ] }, { "id": "ledgerly-cache-added", "older": "Ledgerly uses PostgreSQL 16 as its database.", "newer": "Ledgerly added Redis as a cache.", "expected": "not_changes", "samples": [ "different", "different", "different" ] }, { "id": "mouse-price", "older": "The Logitech MX Master 3S costs $99.", "newer": "The Logitech MX Master 3S costs $79.", "expected": "not_changes", "samples": [ "fixed", "fixed", "fixed" ] }, { "id": "pi-ram", "older": "The Raspberry Pi 5 has 8 GB of RAM.", "newer": "The Raspberry Pi 5 has 4 GB of RAM.", "expected": "not_changes", "samples": [ "fixed", "fixed", "fixed" ] }, { "id": "orbit-history", "older": "Orbit deploys to a VM on DigitalOcean.", "newer": "Orbit previously ran on Heroku.", "expected": "not_changes", "samples": [ "different", "different", "different" ] }, { "id": "pelican-ci-history", "older": "Pelican's CI previously ran on Travis.", "newer": "Pelican's CI previously ran on CircleCI.", "expected": "not_changes", "samples": [ "different", "different", "different" ] }, { "id": "quill-bucket-detail", "older": "Quill stores its assets in S3.", "newer": "Quill stores its assets in the S3 bucket quill-assets in eu-west-1.", "expected": "not_changes", "samples": [ "different", "different", "different" ] }, { "id": "tern-billing", "older": "Tern uses Auth0 for login.", "newer": "Tern uses Stripe for billing.", "expected": "not_changes", "samples": [ "different", "different", "different" ] }, { "id": "quill-cache-step", "older": "To clear Quill's asset cache, invalidate the CloudFront distribution in front of the S3 bucket.", "newer": "Quill moved its assets from S3 to Cloudflare R2.", "expected": "not_changes", "samples": [ "different", "different", "different" ] }, { "id": "pelican-gradle-pin", "older": "Pelican's Gradle wrapper is pinned to 7.6 because the CI build box only has Java 11.", "newer": "Pelican's CI was switched from the Jenkins build box to GitHub Actions.", "expected": "not_changes", "samples": [ "different", "different", "different" ] } ], "bars": { "misses": [], "ok": true } }, "stay": { "rows": [ { "id": "trip-move-run2", "class": "temporary-to-permanent", "older": "The user is in Mumbai for a week for work, on a trip that began around 2026-10-08.", "newer": "The user moved to Mumbai, as stated in a session dated 2026-10-09.", "expected": "changes", "samples": [ "changes", "changes", "changes" ] }, { "id": "trip-move-short", "class": "temporary-to-permanent", "older": "The user is in Mumbai for a week for work.", "newer": "The user moved to Mumbai.", "expected": "changes", "samples": [ "changes", "changes", "changes" ] }, { "id": "sister-pune", "class": "temporary-to-permanent", "older": "The user is staying with their sister in Pune for a month.", "newer": "The user now lives in Pune.", "expected": "changes", "samples": [ "changes", "changes", "changes" ] }, { "id": "sublet-berlin", "class": "temporary-to-permanent", "older": "The user is subletting a flat in Berlin for the summer.", "newer": "The user has moved to Berlin permanently.", "expected": "changes", "samples": [ "changes", "changes", "changes" ] }, { "id": "contract-acme", "class": "temporary-to-permanent", "older": "The user is working at Acme on a three-month contract.", "newer": "The user joined Acme as a full-time employee.", "expected": "changes", "samples": [ "changes", "changes", "changes" ] }, { "id": "foster-biscuit", "class": "temporary-to-permanent", "older": "The user is fostering a beagle named Biscuit until a home is found for him.", "newer": "The user adopted Biscuit, the beagle they were fostering.", "expected": "changes", "samples": [ "changes", "changes", "changes" ] }, { "id": "interim-lead", "class": "temporary-to-permanent", "older": "The user is the team's interim lead while Priya is on leave.", "newer": "The user was made the team's permanent lead.", "expected": "changes", "samples": [ "changes", "changes", "changes" ] }, { "id": "trip-for-good", "class": "decided-not-ended", "older": "The user is in Mumbai for a week for work.", "newer": "The user has decided to move to Mumbai for good.", "expected": "not_changes", "samples": [ "different", "different", "different" ] }, { "id": "visiting-mit", "class": "decided-not-ended", "older": "The user is a visiting researcher at MIT for the spring semester.", "newer": "The user accepted a permanent faculty position at MIT.", "expected": "not_changes", "samples": [ "different", "different", "different" ] }, { "id": "trip-plans-move", "class": "decided-not-ended", "older": "The user is in Mumbai for a week for work.", "newer": "The user plans to move to Mumbai next year.", "expected": "not_changes", "samples": [ "different", "different", "different" ] }, { "id": "trip-hotel-detail", "class": "same-place-non-update", "older": "The user is in Mumbai for a week for work.", "newer": "The user is staying at a hotel in Bandra, Mumbai.", "expected": "not_changes", "samples": [ "different", "different", "different" ] }, { "id": "trip-restated", "class": "same-place-non-update", "older": "The user is in Mumbai for a week for work.", "newer": "The user is still in Mumbai on the work trip, as of 2026-10-11.", "expected": "not_changes", "samples": [ "different", "different", "different" ] }, { "id": "trip-enjoys", "class": "same-place-non-update", "older": "The user is in Mumbai for a week for work.", "newer": "The user enjoys the street food in Mumbai.", "expected": "not_changes", "samples": [ "different", "different", "different" ] }, { "id": "sister-lives", "class": "same-place-non-update", "older": "The user is staying with their sister in Pune for a month.", "newer": "The user's sister lives in Pune.", "expected": "not_changes", "samples": [ "different", "different", "different" ] }, { "id": "contract-start", "class": "same-place-non-update", "older": "The user is working at Acme on a three-month contract.", "newer": "The user's contract at Acme started on 2026-09-01.", "expected": "not_changes", "samples": [ "different", "different", "different" ] }, { "id": "foster-age", "class": "same-place-non-update", "older": "The user is fostering a beagle named Biscuit until a home is found for him.", "newer": "Biscuit, the beagle the user is fostering, is three years old.", "expected": "not_changes", "samples": [ "different", "different", "different" ] }, { "id": "move-detail", "class": "same-place-non-update", "older": "The user moved to Mumbai.", "newer": "The user moved to Mumbai in October 2026 and lives in Bandra.", "expected": "not_changes", "samples": [ "different", "different", "different" ] }, { "id": "trip-owns-flat", "class": "same-place-non-update", "older": "The user is in Mumbai for a week for work.", "newer": "The user owns a flat in Mumbai that they rent out.", "expected": "not_changes", "samples": [ "different", "different", "different" ] }, { "id": "trip-company-hq", "class": "same-place-non-update", "older": "The user is in Mumbai for a week for work.", "newer": "The user's company is headquartered in Mumbai.", "expected": "not_changes", "samples": [ "different", "different", "different" ] }, { "id": "resident-this-week", "class": "lasting-then-temporary", "older": "The user lives in Mumbai.", "newer": "The user is in Mumbai this week, working from the Bandra office.", "expected": "not_changes", "samples": [ "different", "different", "different" ] }, { "id": "home-trip", "class": "lasting-then-temporary", "older": "The user lives in Delhi.", "newer": "The user is in Mumbai for a week for work.", "expected": "not_changes", "samples": [ "different", "different", "different" ] }, { "id": "home-diwali", "class": "lasting-then-temporary", "older": "The user lives in Lajpat Nagar, Delhi.", "newer": "The user is visiting their parents in Chennai over Diwali.", "expected": "not_changes", "samples": [ "different", "different", "different" ] }, { "id": "fulltime-secondment", "class": "lasting-then-temporary", "older": "The user works full-time at Acme's London office.", "newer": "The user is on a three-month secondment to Acme's Leeds office.", "expected": "not_changes", "samples": [ "different", "different", "different" ] }, { "id": "lead-interim-head", "class": "lasting-then-temporary", "older": "The user is the lead of the payments team.", "newer": "The user is covering as interim head of engineering while Ravi is on leave.", "expected": "not_changes", "samples": [ "different", "different", "different" ] }, { "id": "home-moved", "class": "anchor-update", "older": "The user lives in Delhi.", "newer": "The user moved to Mumbai.", "expected": "changes", "samples": [ "changes", "changes", "changes" ] }, { "id": "trip-back-home", "class": "temporary-ended", "older": "The user is in Mumbai for a week for work.", "newer": "The user is back home in Delhi after the Mumbai trip.", "expected": "report", "samples": [ "changes", "changes", "changes" ] }, { "id": "contract-extended", "class": "renewed-bound", "older": "The user is working at Acme on a three-month contract.", "newer": "The user's contract at Acme was extended to six months.", "expected": "report", "samples": [ "changes", "changes", "changes" ] } ], "bars": { "misses": [], "false_changes": "0/51", "ok": true } }, "stay-heldout": { "rows": [ { "id": "goa-winter", "class": "temporary-to-permanent", "older": "The user is spending the winter at their parents' house in Goa.", "newer": "The user has given up their Bengaluru flat and lives in Goa year-round.", "expected": "changes", "samples": [ "changes", "changes", "changes" ] }, { "id": "leeds-housesit", "class": "temporary-to-permanent", "older": "The user is house-sitting for a friend in Leeds until the end of March.", "newer": "The user bought a terraced house in Leeds and lives there.", "expected": "changes", "samples": [ "changes", "changes", "changes" ] }, { "id": "singapore-trip", "class": "temporary-to-permanent", "older": "The user is in Singapore on a two-week business trip.", "newer": "The user relocated to Singapore when the company opened its office there.", "expected": "changes", "samples": [ "changes", "changes", "changes" ] }, { "id": "valencia-room", "class": "temporary-to-permanent", "older": "The user is renting a room in Valencia for six weeks.", "newer": "The user has settled in Valencia and signed a two-year lease on a flat in Ruzafa.", "expected": "changes", "samples": [ "changes", "changes", "changes" ] }, { "id": "nairobi-placement", "class": "temporary-to-permanent", "older": "The user is in Nairobi on a three-month field placement.", "newer": "Nairobi is now the user's permanent home.", "expected": "changes", "samples": [ "changes", "changes", "changes" ] }, { "id": "glasgow-sofa", "class": "temporary-to-permanent", "older": "The user is sleeping on their brother's sofa in Glasgow for a few days.", "newer": "The user rents a flat in Glasgow's West End and lives there.", "expected": "changes", "samples": [ "changes", "changes", "changes" ] }, { "id": "glasgow-weekend", "class": "lasting-then-temporary", "older": "The user lives in Glasgow.", "newer": "The user is in Edinburgh for the weekend.", "expected": "not_changes", "samples": [ "different", "different", "different" ] }, { "id": "singapore-office", "class": "same-place-non-update", "older": "The user is in Singapore on a two-week business trip.", "newer": "The user's employer has an office in Singapore.", "expected": "not_changes", "samples": [ "different", "different", "different" ] }, { "id": "nairobi-considering", "class": "decided-not-ended", "older": "The user is in Nairobi on a three-month field placement.", "newer": "The user is thinking about staying in Nairobi permanently.", "expected": "not_changes", "samples": [ "different", "different", "different" ] }, { "id": "valencia-landlord", "class": "same-place-non-update", "older": "The user is renting a room in Valencia for six weeks.", "newer": "The user rents the room in Valencia from a retired schoolteacher.", "expected": "not_changes", "samples": [ "different", "different", "different" ] }, { "id": "valencia-repaint", "class": "lasting-then-temporary", "older": "The user lives in Valencia.", "newer": "The user is staying at a hotel in Valencia this week while their flat is repainted.", "expected": "not_changes", "samples": [ "different", "different", "different" ] }, { "id": "nurse-placement", "class": "lasting-then-temporary", "older": "The user works as a nurse at St Mary's Hospital.", "newer": "The user is on a six-week placement at the Royal Infirmary.", "expected": "not_changes", "samples": [ "different", "different", "different" ] }, { "id": "leeds-job-offer", "class": "decided-not-ended", "older": "The user is house-sitting for a friend in Leeds until the end of March.", "newer": "The user has accepted a job in Leeds that starts in April.", "expected": "not_changes", "samples": [ "different", "different", "different" ] }, { "id": "goa-apartment", "class": "same-place-non-update", "older": "The user is spending the winter at their parents' house in Goa.", "newer": "The user owns a small holiday apartment in Goa.", "expected": "not_changes", "samples": [ "different", "different", "different" ] } ], "bars": { "positives_matched": "6/6", "positive_misses": [], "false_changes": "0/24", "ok": true } }, "gate": { "rows": [ { "id": "res-dated-newer", "class": "dated-update", "older": "The user lives in Antwerp.", "newer": "The user moved to Ghent, as stated in a session dated 2026-05-02.", "expected": "changes", "samples": [ { "contradiction": false, "slot": "changes" }, { "contradiction": false, "slot": "changes" }, { "contradiction": false, "slot": "changes" } ] }, { "id": "res-dated-both", "class": "dated-update", "older": "The user lives in Antwerp, as of 2026-01-10.", "newer": "The user lives in Ghent, as of 2026-05-02.", "expected": "changes", "samples": [ { "contradiction": false, "slot": "changes" }, { "contradiction": false, "slot": "changes" }, { "contradiction": false, "slot": "changes" } ] }, { "id": "employer-dated", "class": "dated-update", "older": "The user works at Halvorsen Logistics, as of 2026-01-15.", "newer": "The user started a new job at Brightwater Labs on 2026-03-02.", "expected": "changes", "samples": [ { "contradiction": false, "slot": "changes" }, { "contradiction": false, "slot": "changes" }, { "contradiction": false, "slot": "changes" } ] }, { "id": "manager-dated", "class": "dated-update", "older": "The user's manager is Priya, as of 2026-02-03.", "newer": "The user's manager is now Ines, as stated in a session dated 2026-04-11.", "expected": "changes", "samples": [ { "contradiction": false, "slot": "changes" }, { "contradiction": false, "slot": "changes" }, { "contradiction": false, "slot": "changes" } ] }, { "id": "diet-since", "class": "dated-update", "older": "The user eats meat.", "newer": "The user has been vegan since 2026-06-01.", "expected": "changes", "samples": [ { "contradiction": true, "slot": "changes" }, { "contradiction": true, "slot": "changes" }, { "contradiction": true, "slot": "changes" } ] }, { "id": "phone-dated", "class": "dated-update", "older": "The user uses an iPhone 13.", "newer": "The user switched to a Pixel 9, as stated in a session dated 2026-07-19.", "expected": "changes", "samples": [ { "contradiction": false, "slot": "changes" }, { "contradiction": false, "slot": "changes" }, { "contradiction": false, "slot": "changes" } ] }, { "id": "host-dated", "class": "dated-update", "older": "The billing service is hosted on Render, as of 2026-03-02.", "newer": "The billing service was migrated to DigitalOcean on 2026-04-15.", "expected": "changes", "samples": [ { "contradiction": false, "slot": "changes" }, { "contradiction": false, "slot": "changes" }, { "contradiction": false, "slot": "changes" } ] }, { "id": "count-dated", "class": "dated-update", "older": "The user has run 3 marathons, as of 2026-01-05.", "newer": "The user has run 4 marathons, as stated in a session dated 2026-10-01.", "expected": "changes", "samples": [ { "contradiction": false, "slot": "changes" }, { "contradiction": false, "slot": "changes" }, { "contradiction": false, "slot": "changes" } ] }, { "id": "stay-to-home-dated", "class": "dated-update", "older": "The user is in Montreal for a two-week conference, as of 2026-05-10.", "newer": "The user relocated to Montreal, as stated in a session dated 2026-05-20.", "expected": "changes", "samples": [ { "contradiction": false, "slot": "changes" }, { "contradiction": false, "slot": "changes" }, { "contradiction": false, "slot": "changes" } ] }, { "id": "gym-dated", "class": "dated-update", "older": "The user's gym is FitHub on Elm Street, as of 2026-02-01.", "newer": "The user cancelled FitHub and now trains at Ironworks Gym, as stated in a session dated 2026-05-09.", "expected": "changes", "samples": [ { "contradiction": false, "slot": "changes" }, { "contradiction": false, "slot": "changes" }, { "contradiction": false, "slot": "changes" } ] }, { "id": "pets", "class": "coexisting", "older": "The user has a cat named Miso.", "newer": "The user adopted a dog named Pepper, as stated in a session dated 2026-06-02.", "expected": "not_changes", "samples": [ { "contradiction": false, "slot": "different" }, { "contradiction": false, "slot": "different" }, { "contradiction": false, "slot": "different" } ] }, { "id": "languages", "class": "coexisting", "older": "The user speaks French, as of 2026-01-20.", "newer": "The user is learning Japanese, as stated in a session dated 2026-04-03.", "expected": "not_changes", "samples": [ { "contradiction": false, "slot": "different" }, { "contradiction": false, "slot": "different" }, { "contradiction": false, "slot": "different" } ] }, { "id": "hobbies", "class": "coexisting", "older": "The user plays tennis on Saturdays, as of 2026-02-01.", "newer": "The user joined a climbing gym on 2026-04-09.", "expected": "not_changes", "samples": [ { "contradiction": false, "slot": "different" }, { "contradiction": false, "slot": "different" }, { "contradiction": false, "slot": "different" } ] }, { "id": "stack", "class": "coexisting", "older": "The project uses Redis for caching.", "newer": "The project added PostgreSQL for its user accounts on 2026-05-03.", "expected": "not_changes", "samples": [ { "contradiction": false, "slot": "different" }, { "contradiction": false, "slot": "different" }, { "contradiction": false, "slot": "different" } ] }, { "id": "accounts", "class": "coexisting", "older": "The user has a savings account with Nordbank.", "newer": "The user opened a current account with Kestrel Bank, as stated in a session dated 2026-03-14.", "expected": "not_changes", "samples": [ { "contradiction": false, "slot": "different" }, { "contradiction": false, "slot": "different" }, { "contradiction": false, "slot": "different" } ] }, { "id": "trips", "class": "separate-events", "older": "The user took a five-day trip to Rome in March 2026.", "newer": "The user spent a weekend in Vienna, as stated in a session dated 2026-06-14.", "expected": "not_changes", "samples": [ { "contradiction": false, "slot": "different" }, { "contradiction": false, "slot": "different" }, { "contradiction": false, "slot": "different" } ] }, { "id": "purchases", "class": "separate-events", "older": "The user paid $90 for running shoes, as of 2026-02-02.", "newer": "The user bought a rain jacket for $120 on 2026-04-20.", "expected": "not_changes", "samples": [ { "contradiction": false, "slot": "different" }, { "contradiction": false, "slot": "different" }, { "contradiction": false, "slot": "different" } ] }, { "id": "races", "class": "separate-events", "older": "The user ran a half marathon in Bruges on 2026-03-08.", "newer": "The user ran a 10K in Leuven, as stated in a session dated 2026-05-17.", "expected": "not_changes", "samples": [ { "contradiction": false, "slot": "different" }, { "contradiction": false, "slot": "different" }, { "contradiction": false, "slot": "different" } ] }, { "id": "flat-detail", "class": "added-detail", "older": "The user lives in Antwerp.", "newer": "The user's flat in Antwerp has a small balcony, as stated in a session dated 2026-05-02.", "expected": "not_changes", "samples": [ { "contradiction": false, "slot": "different" }, { "contradiction": false, "slot": "different" }, { "contradiction": false, "slot": "different" } ] }, { "id": "role-detail", "class": "added-detail", "older": "The user works at Halvorsen Logistics.", "newer": "The user leads the routing team at Halvorsen Logistics, as of 2026-04-01.", "expected": "not_changes", "samples": [ { "contradiction": false, "slot": "different" }, { "contradiction": false, "slot": "different" }, { "contradiction": false, "slot": "different" } ] }, { "id": "lived-until", "class": "past-value", "older": "The user lived in Utrecht until 2025.", "newer": "The user lives in Ghent, as of 2026-05-02.", "expected": "not_changes", "samples": [ { "contradiction": false, "slot": "different" }, { "contradiction": false, "slot": "different" }, { "contradiction": false, "slot": "different" } ] }, { "id": "manager-previously", "class": "past-value", "older": "The user's manager was previously Priya.", "newer": "The user's manager is Ines, as stated in a session dated 2026-04-11.", "expected": "not_changes", "samples": [ { "contradiction": false, "slot": "different" }, { "contradiction": false, "slot": "different" }, { "contradiction": false, "slot": "different" } ] }, { "id": "dated-history", "class": "past-value", "older": "As of 2026-03-01, when the user lived in Antwerp, the user commuted by tram.", "newer": "The user lives in Ghent, as stated in a session dated 2026-05-02.", "expected": "not_changes", "samples": [ { "contradiction": false, "slot": "different" }, { "contradiction": false, "slot": "different" }, { "contradiction": false, "slot": "different" } ] }, { "id": "decided-move", "class": "decided-not-done", "older": "The user lives in Antwerp.", "newer": "The user has decided to move to Ghent next spring, as stated in a session dated 2026-05-02.", "expected": "not_changes", "samples": [ { "contradiction": false, "slot": "different" }, { "contradiction": false, "slot": "different" }, { "contradiction": false, "slot": "different" } ] }, { "id": "home-then-visit", "class": "lasting-then-temporary", "older": "The user lives in Antwerp.", "newer": "The user is in Ghent for three days for a wedding, as stated in a session dated 2026-05-02.", "expected": "not_changes", "samples": [ { "contradiction": false, "slot": "different" }, { "contradiction": false, "slot": "different" }, { "contradiction": false, "slot": "different" } ] }, { "id": "job-then-secondment", "class": "lasting-then-temporary", "older": "The user works full-time at Halvorsen Logistics.", "newer": "The user is on a two-month secondment to the Oslo office, as of 2026-04-01.", "expected": "not_changes", "samples": [ { "contradiction": false, "slot": "different" }, { "contradiction": false, "slot": "different" }, { "contradiction": false, "slot": "different" } ] }, { "id": "deploy-step", "class": "relied-on", "older": "Deploy the billing service with `render deploy`.", "newer": "The billing service was migrated to DigitalOcean on 2026-04-15.", "expected": "not_changes", "samples": [ { "contradiction": false, "slot": "different" }, { "contradiction": false, "slot": "different" }, { "contradiction": false, "slot": "different" } ] }, { "id": "commute", "class": "relied-on", "older": "The user's commute to the Halvorsen office is a 15-minute bike ride.", "newer": "The user moved to Ghent, as stated in a session dated 2026-05-02.", "expected": "not_changes", "samples": [ { "contradiction": false, "slot": "different" }, { "contradiction": false, "slot": "different" }, { "contradiction": false, "slot": "different" } ] }, { "id": "kettle-price", "class": "fixed", "older": "The Brightwater X2 kettle costs $49.", "newer": "The Brightwater X2 kettle costs $59, as stated in a review dated 2026-05-02.", "expected": "not_changes", "samples": [ { "contradiction": true, "slot": "fixed" }, { "contradiction": true, "slot": "fixed" }, { "contradiction": true, "slot": "fixed" } ] }, { "id": "case-trip-5-5-r2", "class": "case", "older": "The user is in Mumbai for a week for work, as of 2026-10-08.", "newer": "The user moved to Mumbai, as stated in a session dated 2026-10-09.", "expected": "changes", "samples": [ { "contradiction": false, "slot": "changes" }, { "contradiction": false, "slot": "changes" }, { "contradiction": false, "slot": "changes" } ] }, { "id": "case-trip-5-5-r1", "class": "case", "older": "The user is in Mumbai for a week on a work trip, as of 2026-10-08.", "newer": "The user moved to Mumbai, as stated in a session dated 2026-10-09.", "expected": "changes", "samples": [ { "contradiction": false, "slot": "changes" }, { "contradiction": false, "slot": "changes" }, { "contradiction": false, "slot": "changes" } ] }, { "id": "case-residence-5-5-r2", "class": "case", "older": "The user lives in Lajpat Nagar, Delhi.", "newer": "The user moved to Mumbai, as stated in a session dated 2026-10-09.", "expected": "changes", "samples": [ { "contradiction": false, "slot": "changes" }, { "contradiction": false, "slot": "changes" }, { "contradiction": true, "slot": "changes" } ] }, { "id": "case-residence-5-5-r1", "class": "case", "older": "The user lives in a flat in Lajpat Nagar, Delhi.", "newer": "The user moved to Mumbai, as stated in a session dated 2026-10-09.", "expected": "changes", "samples": [ { "contradiction": false, "slot": "changes" }, { "contradiction": true, "slot": "changes" }, { "contradiction": true, "slot": "changes" } ] }, { "id": "case-home-5-5", "class": "case", "older": "The user's home is in Delhi.", "newer": "The user moved to Mumbai, as stated in a session dated 2026-10-09.", "expected": "changes", "samples": [ { "contradiction": false, "slot": "changes" }, { "contradiction": false, "slot": "changes" }, { "contradiction": true, "slot": "changes" } ] }, { "id": "case-trip-4-6", "class": "case", "older": "The user is in Mumbai for a week for work.", "newer": "The user moved to Mumbai.", "expected": "changes", "samples": [ { "contradiction": false, "slot": "changes" }, { "contradiction": false, "slot": "changes" }, { "contradiction": false, "slot": "changes" } ] }, { "id": "case-trip-alone", "class": "case", "older": "The user lives in Lajpat Nagar, Delhi.", "newer": "The user is in Mumbai for a week for work, as of 2026-10-08.", "expected": "not_changes", "samples": [ { "contradiction": false, "slot": "different" }, { "contradiction": false, "slot": "different" }, { "contradiction": false, "slot": "different" } ] }, { "id": "case-flight", "class": "case", "older": "The user flies from Mumbai back to Delhi on Thursday 2026-10-15.", "newer": "The user moved to Mumbai, as stated in a session dated 2026-10-09.", "expected": "report", "samples": [ { "contradiction": false, "slot": "different" }, { "contradiction": false, "slot": "different" }, { "contradiction": false, "slot": "different" } ] }, { "id": "case-hotel", "class": "case", "older": "The user is staying at a hotel in Bandra, Mumbai.", "newer": "The user moved to Mumbai, as stated in a session dated 2026-10-09.", "expected": "report", "samples": [ { "contradiction": false, "slot": "changes" }, { "contradiction": false, "slot": "changes" }, { "contradiction": false, "slot": "changes" } ] } ], "bars": { "proposed": { "positives_retired": "10/10", "positive_misses": [], "false_retirements": "0/57", "false_rows": [], "ok": true }, "shipped": { "positives_retired": "1/10", "positive_misses": [ "res-dated-newer", "res-dated-both", "employer-dated", "manager-dated", "phone-dated", "host-dated", "count-dated", "stay-to-home-dated", "gym-dated" ], "false_retirements": "0/57", "false_rows": [], "ok": false }, "case": { "case-trip-5-5-r2": { "expected": "changes", "proposed": 3, "shipped": 0, "of": 3 }, "case-trip-5-5-r1": { "expected": "changes", "proposed": 3, "shipped": 0, "of": 3 }, "case-residence-5-5-r2": { "expected": "changes", "proposed": 3, "shipped": 1, "of": 3 }, "case-residence-5-5-r1": { "expected": "changes", "proposed": 3, "shipped": 2, "of": 3 }, "case-home-5-5": { "expected": "changes", "proposed": 3, "shipped": 1, "of": 3 }, "case-trip-4-6": { "expected": "changes", "proposed": 3, "shipped": 0, "of": 3 }, "case-trip-alone": { "expected": "not_changes", "proposed": 0, "shipped": 0, "of": 3 }, "case-flight": { "expected": "report", "proposed": 0, "shipped": 0, "of": 3 }, "case-hotel": { "expected": "report", "proposed": 3, "shipped": 0, "of": 3 } } } }, "gate-heldout": { "rows": [ { "id": "tartu-tallinn", "class": "dated-update", "older": "The user rents an apartment in Tartu.", "newer": "Since August 2026 the user has lived in Tallinn.", "expected": "changes", "samples": [ { "contradiction": false, "slot": "changes" }, { "contradiction": false, "slot": "changes" }, { "contradiction": false, "slot": "changes" } ] }, { "id": "car", "class": "dated-update", "older": "The user drives a 2015 Honda Civic (noted 2026-02-11).", "newer": "The user sold the Civic and now drives a Kia EV6, per a note from 2026-08-03.", "expected": "changes", "samples": [ { "contradiction": false, "slot": "changes" }, { "contradiction": false, "slot": "changes" }, { "contradiction": false, "slot": "changes" } ] }, { "id": "standup", "class": "dated-update", "older": "The team's daily standup is at 9:30.", "newer": "On 2026-07-01 the team moved its daily standup to 10:15.", "expected": "changes", "samples": [ { "contradiction": false, "slot": "changes" }, { "contradiction": false, "slot": "changes" }, { "contradiction": false, "slot": "changes" } ] }, { "id": "isp", "class": "dated-update", "older": "The user's internet provider is Telenet.", "newer": "The user changed internet provider to Proximus on 14 June 2026.", "expected": "changes", "samples": [ { "contradiction": true, "slot": "changes" }, { "contradiction": true, "slot": "changes" }, { "contradiction": false, "slot": "changes" } ] }, { "id": "laptop", "class": "dated-update", "older": "The user's main laptop is a ThinkPad X1 (noted 2026-01-09).", "newer": "The user replaced the ThinkPad with a MacBook Air on 2026-06-22.", "expected": "changes", "samples": [ { "contradiction": false, "slot": "changes" }, { "contradiction": false, "slot": "changes" }, { "contradiction": false, "slot": "changes" } ] }, { "id": "rent", "class": "dated-update", "older": "The user's monthly rent is 950 euros.", "newer": "From 2026-09-01 the user's rent went up to 1,020 euros a month.", "expected": "changes", "samples": [ { "contradiction": false, "slot": "changes" }, { "contradiction": false, "slot": "changes" }, { "contradiction": false, "slot": "changes" } ] }, { "id": "krakow-let", "class": "dated-update", "older": "The user is staying in a short-let flat in Krakow for May 2026.", "newer": "The user signed a permanent lease in Krakow on 2026-06-02 and lives there now.", "expected": "changes", "samples": [ { "contradiction": false, "slot": "changes" }, { "contradiction": false, "slot": "changes" }, { "contradiction": false, "slot": "changes" } ] }, { "id": "ci", "class": "dated-update", "older": "The repository's CI runs on Travis CI.", "newer": "The repository's CI was switched to GitHub Actions on 2026-03-30.", "expected": "changes", "samples": [ { "contradiction": false, "slot": "changes" }, { "contradiction": false, "slot": "changes" }, { "contradiction": false, "slot": "changes" } ] }, { "id": "bikes", "class": "coexisting", "older": "The user owns a road bike.", "newer": "The user bought a gravel bike on 2026-07-12.", "expected": "not_changes", "samples": [ { "contradiction": false, "slot": "different" }, { "contradiction": false, "slot": "different" }, { "contradiction": false, "slot": "different" } ] }, { "id": "volunteering", "class": "coexisting", "older": "The user volunteers at a food bank on Sundays.", "newer": "Since June 2026 the user has also coached a youth football team.", "expected": "not_changes", "samples": [ { "contradiction": false, "slot": "different" }, { "contradiction": false, "slot": "different" }, { "contradiction": false, "slot": "different" } ] }, { "id": "backend-lang", "class": "coexisting", "older": "The project's backend is written in Go.", "newer": "On 2026-04-02 the team added a Python service for report generation.", "expected": "not_changes", "samples": [ { "contradiction": false, "slot": "different" }, { "contradiction": false, "slot": "different" }, { "contradiction": false, "slot": "different" } ] }, { "id": "shows", "class": "separate-events", "older": "The user saw Hamlet at the National Theatre on 2026-02-14.", "newer": "The user went to a jazz concert at the Barbican on 2026-05-30.", "expected": "not_changes", "samples": [ { "contradiction": false, "slot": "different" }, { "contradiction": false, "slot": "different" }, { "contradiction": false, "slot": "different" } ] }, { "id": "donations", "class": "separate-events", "older": "The user donated 50 euros to a hospice in January 2026.", "newer": "The user donated 30 euros to a wildlife charity on 2026-04-02.", "expected": "not_changes", "samples": [ { "contradiction": false, "slot": "different" }, { "contradiction": false, "slot": "different" }, { "contradiction": false, "slot": "different" } ] }, { "id": "floor", "class": "added-detail", "older": "The user rents an apartment in Tartu.", "newer": "The user's apartment in Tartu is on the fourth floor (noted 2026-03-05).", "expected": "not_changes", "samples": [ { "contradiction": false, "slot": "different" }, { "contradiction": false, "slot": "different" }, { "contradiction": false, "slot": "different" } ] }, { "id": "used-to", "class": "past-value", "older": "The user used to work night shifts at a bakery.", "newer": "The user works as a data analyst at Kestrel Insurance, per a note from 2026-08-03.", "expected": "not_changes", "samples": [ { "contradiction": false, "slot": "different" }, { "contradiction": false, "slot": "different" }, { "contradiction": false, "slot": "different" } ] }, { "id": "planning-house", "class": "decided-not-done", "older": "The user rents an apartment in Tartu.", "newer": "On 2026-08-03 the user said they plan to buy a house in Tallinn next year.", "expected": "not_changes", "samples": [ { "contradiction": false, "slot": "different" }, { "contradiction": false, "slot": "different" }, { "contradiction": false, "slot": "different" } ] }, { "id": "riga-course", "class": "lasting-then-temporary", "older": "The user lives in Tartu.", "newer": "From 2026-08-01 the user is spending ten days in Riga for a training course.", "expected": "not_changes", "samples": [ { "contradiction": false, "slot": "different" }, { "contradiction": false, "slot": "different" }, { "contradiction": false, "slot": "different" } ] }, { "id": "town-hall-walk", "class": "relied-on", "older": "The user's apartment is a five-minute walk from Tartu's Town Hall Square.", "newer": "Since August 2026 the user has lived in Tallinn.", "expected": "not_changes", "samples": [ { "contradiction": false, "slot": "different" }, { "contradiction": false, "slot": "different" }, { "contradiction": false, "slot": null } ] }, { "id": "battery", "class": "fixed", "older": "The Kia EV6 has a 77.4 kWh battery.", "newer": "The Kia EV6's battery is 84 kWh, per a review dated 2026-05-11.", "expected": "not_changes", "samples": [ { "contradiction": true, "slot": "fixed" }, { "contradiction": true, "slot": "fixed" }, { "contradiction": true, "slot": "fixed" } ] } ], "bars": { "proposed": { "positives_retired": "8/8", "positive_misses": [], "false_retirements": "0/33", "false_rows": [], "ok": true }, "shipped": { "positives_retired": "1/8", "positive_misses": [ "tartu-tallinn", "car", "standup", "laptop", "rent", "krakow-let", "ci" ], "false_retirements": "0/33", "false_rows": [], "ok": false }, "case": {} } }, "past": { "rows": [ { "id": "worked-until", "class": "past-ended", "older": "The user worked at Lindqvist & Berg until March 2025.", "newer": "The user works at Ostrava Robotics, per a note from 2026-06-02.", "expected": "not_changes", "samples": [ { "contradiction": false, "slot": "different" }, { "contradiction": false, "slot": "different" }, { "contradiction": false, "slot": "different" } ] }, { "id": "before-2024", "class": "past-ended", "older": "Before 2024 the user's team used Jira.", "newer": "The team tracks its work in Linear (noted 2026-04-18).", "expected": "not_changes", "samples": [ { "contradiction": false, "slot": "different" }, { "contradiction": false, "slot": "different" }, { "contradiction": false, "slot": "different" } ] }, { "id": "restated-pool", "class": "past-ended", "older": "As of 2026-02-10, when the user lived in Bremen, the user swam at the Sudbad pool.", "newer": "Since May 2026 the user has lived in Hanover.", "expected": "not_changes", "samples": [ { "contradiction": false, "slot": "different" }, { "contradiction": false, "slot": "different" }, { "contradiction": false, "slot": "different" } ] }, { "id": "back-in-2023", "class": "past-ended", "older": "Back in 2023 the user drove a Volvo V60.", "newer": "The user drives a Tesla Model 3 (noted 2026-07-01).", "expected": "not_changes", "samples": [ { "contradiction": false, "slot": "different" }, { "contradiction": false, "slot": "different" }, { "contradiction": false, "slot": "different" } ] }, { "id": "had-been", "class": "past-ended", "older": "The user had been vegetarian for ten years until 2025.", "newer": "The user eats fish, per a note from 2026-03-09.", "expected": "not_changes", "samples": [ { "contradiction": false, "slot": "different" }, { "contradiction": false, "slot": "different" }, { "contradiction": false, "slot": "different" } ] }, { "id": "restated-backups", "class": "past-ended", "older": "As of 2026-04-01, when the project used MySQL, backups ran nightly with mysqldump.", "newer": "On 2026-06-10 the project moved its database to PostgreSQL.", "expected": "not_changes", "samples": [ { "contradiction": false, "slot": "different" }, { "contradiction": false, "slot": "different" }, { "contradiction": false, "slot": "different" } ] }, { "id": "restated-rent", "class": "past-ended", "older": "As of 2026-01-20, when the user rented in Bremen, the user's rent was 780 euros a month.", "newer": "The user's rent is 910 euros a month (noted 2026-08-14).", "expected": "not_changes", "samples": [ { "contradiction": false, "slot": "different" }, { "contradiction": false, "slot": "different" }, { "contradiction": false, "slot": "different" } ] }, { "id": "lives-dated", "class": "present-dated", "older": "The user lives in Bremen, as of 2026-02-10.", "newer": "Since May 2026 the user has lived in Hanover.", "expected": "changes", "samples": [ { "contradiction": false, "slot": "changes" }, { "contradiction": false, "slot": "changes" }, { "contradiction": false, "slot": "changes" } ] }, { "id": "works-noted", "class": "present-dated", "older": "The user is working at Lindqvist & Berg (noted 2026-01-05).", "newer": "The user joined Ostrava Robotics on 2026-05-20.", "expected": "changes", "samples": [ { "contradiction": false, "slot": "changes" }, { "contradiction": false, "slot": "changes" }, { "contradiction": false, "slot": "changes" } ] }, { "id": "course-to-home", "class": "present-dated", "older": "The user is in Lyon for a three-week course, as of 2026-03-02.", "newer": "The user signed a lease in Lyon on 2026-04-01 and lives there now.", "expected": "changes", "samples": [ { "contradiction": false, "slot": "changes" }, { "contradiction": false, "slot": "changes" }, { "contradiction": false, "slot": "changes" } ] }, { "id": "tracker-noted", "class": "present-dated", "older": "The team tracks its work in Jira, per a note from 2026-01-12.", "newer": "The team switched from Jira to Linear on 2026-04-18.", "expected": "changes", "samples": [ { "contradiction": false, "slot": "changes" }, { "contradiction": false, "slot": "changes" }, { "contradiction": false, "slot": "changes" } ] }, { "id": "phone-plan", "class": "present-dated", "older": "The user's phone plan is with Vodafone, as of 2026-02-01.", "newer": "The user moved their phone plan to Lebara on 2026-06-03.", "expected": "changes", "samples": [ { "contradiction": false, "slot": "changes" }, { "contradiction": false, "slot": "changes" }, { "contradiction": false, "slot": "changes" } ] }, { "id": "manager-history", "class": "past-then-past", "older": "The user's manager was previously Henrik.", "newer": "The user's manager was previously Sofia.", "expected": "not_changes", "samples": [ { "contradiction": true, "slot": "different" }, { "contradiction": false, "slot": "different" }, { "contradiction": false, "slot": "different" } ] }, { "id": "case-restated-flight", "class": "case", "older": "As of 2026-10-08, when the user was in Mumbai for a week for work, the user was due to fly back home to Delhi on Thursday 2026-10-15.", "newer": "The user moved to Mumbai, as stated in a session dated 2026-10-09.", "expected": "not_changes", "samples": [ { "contradiction": false, "slot": "different" }, { "contradiction": false, "slot": "different" }, { "contradiction": false, "slot": "different" } ] }, { "id": "case-restated-hotel", "class": "case", "older": "As of 2026-10-08, when the user was in Mumbai for a week for work, the user was staying at a hotel in Bandra, Mumbai.", "newer": "The user moved to Mumbai, as stated in a session dated 2026-10-09.", "expected": "not_changes", "samples": [ { "contradiction": false, "slot": "different" }, { "contradiction": false, "slot": "different" }, { "contradiction": false, "slot": "different" } ] } ], "bars": { "proposed": { "positives_retired": "5/5", "positive_misses": [], "false_retirements": "0/24", "false_rows": [], "ok": true }, "shipped": { "positives_retired": "0/5", "positive_misses": [ "lives-dated", "works-noted", "course-to-home", "tracker-noted", "phone-plan" ], "false_retirements": "0/24", "false_rows": [], "ok": false }, "case": { "case-restated-flight": { "expected": "not_changes", "proposed": 0, "shipped": 0, "of": 3 }, "case-restated-hotel": { "expected": "not_changes", "proposed": 0, "shipped": 0, "of": 3 } } } } }, "bounded": { "halves": "bounded", "past": { "rows": [ { "id": "worked-until", "class": "past-ended", "older": "The user worked at Lindqvist & Berg until March 2025.", "newer": "The user works at Ostrava Robotics, per a note from 2026-06-02.", "expected": "not_changes", "samples": [ { "contradiction": false, "slot": "changes" }, { "contradiction": false, "slot": "changes" }, { "contradiction": false, "slot": "changes" } ] }, { "id": "before-2024", "class": "past-ended", "older": "Before 2024 the user's team used Jira.", "newer": "The team tracks its work in Linear (noted 2026-04-18).", "expected": "not_changes", "samples": [ { "contradiction": false, "slot": "different" }, { "contradiction": false, "slot": "different" }, { "contradiction": false, "slot": "different" } ] }, { "id": "restated-pool", "class": "past-ended", "older": "As of 2026-02-10, when the user lived in Bremen, the user swam at the Sudbad pool.", "newer": "Since May 2026 the user has lived in Hanover.", "expected": "not_changes", "samples": [ { "contradiction": false, "slot": "different" }, { "contradiction": false, "slot": "different" }, { "contradiction": false, "slot": "different" } ] }, { "id": "back-in-2023", "class": "past-ended", "older": "Back in 2023 the user drove a Volvo V60.", "newer": "The user drives a Tesla Model 3 (noted 2026-07-01).", "expected": "not_changes", "samples": [ { "contradiction": false, "slot": "changes" }, { "contradiction": false, "slot": "changes" }, { "contradiction": false, "slot": "changes" } ] }, { "id": "had-been", "class": "past-ended", "older": "The user had been vegetarian for ten years until 2025.", "newer": "The user eats fish, per a note from 2026-03-09.", "expected": "not_changes", "samples": [ { "contradiction": false, "slot": "different" }, { "contradiction": false, "slot": "different" }, { "contradiction": false, "slot": "different" } ] }, { "id": "restated-backups", "class": "past-ended", "older": "As of 2026-04-01, when the project used MySQL, backups ran nightly with mysqldump.", "newer": "On 2026-06-10 the project moved its database to PostgreSQL.", "expected": "not_changes", "samples": [ { "contradiction": false, "slot": "different" }, { "contradiction": false, "slot": "different" }, { "contradiction": false, "slot": "different" } ] }, { "id": "restated-rent", "class": "past-ended", "older": "As of 2026-01-20, when the user rented in Bremen, the user's rent was 780 euros a month.", "newer": "The user's rent is 910 euros a month (noted 2026-08-14).", "expected": "not_changes", "samples": [ { "contradiction": false, "slot": "changes" }, { "contradiction": false, "slot": "changes" }, { "contradiction": false, "slot": "changes" } ] }, { "id": "lives-dated", "class": "present-dated", "older": "The user lives in Bremen, as of 2026-02-10.", "newer": "Since May 2026 the user has lived in Hanover.", "expected": "changes", "samples": [ { "contradiction": false, "slot": "changes" }, { "contradiction": false, "slot": "changes" }, { "contradiction": false, "slot": "changes" } ] }, { "id": "works-noted", "class": "present-dated", "older": "The user is working at Lindqvist & Berg (noted 2026-01-05).", "newer": "The user joined Ostrava Robotics on 2026-05-20.", "expected": "changes", "samples": [ { "contradiction": false, "slot": "changes" }, { "contradiction": false, "slot": "changes" }, { "contradiction": false, "slot": "changes" } ] }, { "id": "course-to-home", "class": "present-dated", "older": "The user is in Lyon for a three-week course, as of 2026-03-02.", "newer": "The user signed a lease in Lyon on 2026-04-01 and lives there now.", "expected": "changes", "samples": [ { "contradiction": false, "slot": "changes" }, { "contradiction": false, "slot": "changes" }, { "contradiction": false, "slot": "changes" } ] }, { "id": "tracker-noted", "class": "present-dated", "older": "The team tracks its work in Jira, per a note from 2026-01-12.", "newer": "The team switched from Jira to Linear on 2026-04-18.", "expected": "changes", "samples": [ { "contradiction": false, "slot": "changes" }, { "contradiction": false, "slot": "changes" }, { "contradiction": false, "slot": "changes" } ] }, { "id": "phone-plan", "class": "present-dated", "older": "The user's phone plan is with Vodafone, as of 2026-02-01.", "newer": "The user moved their phone plan to Lebara on 2026-06-03.", "expected": "changes", "samples": [ { "contradiction": false, "slot": "changes" }, { "contradiction": false, "slot": "changes" }, { "contradiction": false, "slot": "changes" } ] }, { "id": "manager-history", "class": "past-then-past", "older": "The user's manager was previously Henrik.", "newer": "The user's manager was previously Sofia.", "expected": "not_changes", "samples": [ { "contradiction": false, "slot": "different" }, { "contradiction": false, "slot": "different" }, { "contradiction": false, "slot": "different" } ] }, { "id": "case-restated-flight", "class": "case", "older": "As of 2026-10-08, when the user was in Mumbai for a week for work, the user was due to fly back home to Delhi on Thursday 2026-10-15.", "newer": "The user moved to Mumbai, as stated in a session dated 2026-10-09.", "expected": "not_changes", "samples": [ { "contradiction": false, "slot": "different" }, { "contradiction": false, "slot": "different" }, { "contradiction": false, "slot": "different" } ] }, { "id": "case-restated-hotel", "class": "case", "older": "As of 2026-10-08, when the user was in Mumbai for a week for work, the user was staying at a hotel in Bandra, Mumbai.", "newer": "The user moved to Mumbai, as stated in a session dated 2026-10-09.", "expected": "not_changes", "samples": [ { "contradiction": false, "slot": "changes" }, { "contradiction": false, "slot": "changes" }, { "contradiction": false, "slot": "changes" } ] } ], "bars": { "proposed": { "positives_retired": "5/5", "positive_misses": [], "false_retirements": "9/24", "false_rows": [ "worked-until", "back-in-2023", "restated-rent" ], "ok": false }, "shipped": { "positives_retired": "0/5", "positive_misses": [ "lives-dated", "works-noted", "course-to-home", "tracker-noted", "phone-plan" ], "false_retirements": "0/24", "false_rows": [], "ok": false }, "case": { "case-restated-flight": { "expected": "not_changes", "proposed": 0, "shipped": 0, "of": 3 }, "case-restated-hotel": { "expected": "not_changes", "proposed": 3, "shipped": 0, "of": 3 } } } } } }, "claude-sonnet-4-6": { "past": { "halves": "past", "replay": { "rows": [ { "id": 3, "label": "user-update", "older": "The user has added 17 new postcards to their collection since they started collecting again.", "newer": "The user has added 25 new postcards to their collection since they started collecting again.", "verdicts": [ "changes", "changes", "changes" ], "samples": [ "changes", "changes", "changes" ] }, { "id": 4, "label": "user-update", "older": "The user has been using a Fitbit Charge 3 for 6 months.", "newer": "The user has been using a Fitbit Charge 3 for nine months.", "verdicts": [ "changes", "changes", "different" ], "samples": [ "changes", "changes", "changes" ] }, { "id": 11, "label": "user-update", "older": "The user has tried making the Negroni at home five times since Emma taught them how to make it.", "newer": "The user has tried making a Negroni at home 10 times since their friend Emma showed them how to make it.", "verdicts": [ "changes", "changes", "changes" ], "samples": [ "changes", "changes", "changes" ] }, { "id": 22, "label": "user-update", "older": "The user's daily commute to work by bus takes around 45 minutes.", "newer": "The user's total current commute time is 40 minutes.", "verdicts": [ "different", "different", "different" ], "samples": [ "changes", "changes", "changes" ] }, { "id": 24, "label": "user-update", "older": "The user is based in Tokyo.", "newer": "The user is located in the South Bay area of the San Francisco Bay Area.", "verdicts": [ "changes", "changes", "changes" ], "samples": [ "changes", "changes", "changes" ] }, { "id": 25, "label": "user-update", "older": "The user attended three sessions of the bereavement support group.", "newer": "The user attended five sessions of the bereavement support group.", "verdicts": [ "changes", "changes", "changes" ], "samples": [ "changes", "changes", "changes" ] }, { "id": 27, "label": "user-update", "older": "The user is currently reading \"The Seven Habits\" by Stephen Covey.", "newer": "The user is currently reading \"The Power of Now\" by Eckhart Tolle.", "verdicts": [ "changes", "changes", "changes" ], "samples": [ "changes", "changes", "changes" ] }, { "id": 9, "label": "user-unclear", "older": "The user stayed at the Yarrow Hotel with friends Alex and Rachel during what they believed was the 2022 Sundance Film Festival.", "newer": "The user stayed at the Omni Barton Creek Resort.", "verdicts": [ "different", "different", "different" ], "samples": [ "different", "different", "different" ] }, { "id": 21, "label": "user-unclear", "older": "The user owns a 2018 Ford Mustang GT.", "newer": "The user owns a 2018 Honda Civic.", "verdicts": [ "changes", "changes", "changes" ], "samples": [ "changes", "changes", "changes" ] }, { "id": 26, "label": "user-unclear", "older": "The user's sister's birthday is in November.", "newer": "The user's sister's birthday is on March 15th.", "verdicts": [ "different", "fixed", "fixed" ], "samples": [ "fixed", "fixed", "fixed" ] }, { "id": 33, "label": "user-unclear", "older": "The user has been trying to establish a consistent wake-up time and it has been going well except that they sometimes sleep in later on Wednesdays.", "newer": "The user has been waking up 30 minutes earlier than usual.", "verdicts": [ "different", "different", "different" ], "samples": [ "different", "different", "different" ] }, { "id": 14, "label": "abstention", "older": "The Art Cube and its curator Rachel Lee do not exist in reality.", "newer": "\"The Art Cube\" is a new contemporary art gallery.", "verdicts": [ "different", "different", "different" ], "samples": [ "fixed", "fixed", "fixed" ] }, { "id": 15, "label": "abstention", "older": "The assistant could not find any information on a contemporary artist named James Parker whose work is associated with The Art Cube.", "newer": "James Parker was a featured artist at the exhibition at \"The Art Cube\".", "verdicts": [ "different", "different", "different" ], "samples": [ "different", "different", "different" ] }, { "id": 17, "label": "abstention", "older": "Whole wheat flour produces a denser, heavier crust due to its coarser texture and higher protein content.", "newer": "Whole wheat flour typically has a lower protein content, around 10-12%.", "verdicts": [ "different", "different", "different" ], "samples": [ "fixed", "different", "fixed" ] }, { "id": 5, "label": "world-fact", "older": "Ori and the Blind Forest takes about 4-6 hours to complete.", "newer": "Ori and the Blind Forest has an estimated playtime of 8-12 hours.", "verdicts": [ "fixed", "fixed", "different" ], "samples": [ "fixed", "fixed", "fixed" ] }, { "id": 6, "label": "world-fact", "older": "The Pathless takes about 5-8 hours to complete.", "newer": "The Pathless has an estimated playtime of 8-10 hours.", "verdicts": [ "different", "fixed", "different" ], "samples": [ "fixed", "fixed", "fixed" ] }, { "id": 7, "label": "world-fact", "older": "Rime takes about 4-6 hours to complete.", "newer": "Rime has an estimated playtime of 6-8 hours.", "verdicts": [ "different", "different", "different" ], "samples": [ "fixed", "fixed", "fixed" ] }, { "id": 8, "label": "world-fact", "older": "Deep Work: Rules for Focused Success in a Distracted World is narrated by Jeff Bottoms.", "newer": "Deep Work is narrated by its author, Cal Newport.", "verdicts": [ "different", "different", "different" ], "samples": [ "fixed", "fixed", "fixed" ] }, { "id": 10, "label": "world-fact", "older": "\"The Song of Achilles\" by Madeline Miller has 336 pages.", "newer": "\"The Song of Achilles\" by Madeline Miller has 384 pages.", "verdicts": [ "different", "different", "different" ], "samples": [ "fixed", "fixed", "fixed" ] }, { "id": 12, "label": "world-fact", "older": "The Logitech M185 wireless mouse costs around $15.", "newer": "The Logitech M185 costs approximately $10.", "verdicts": [ "fixed", "different", "different" ], "samples": [ "fixed", "fixed", "fixed" ] }, { "id": 13, "label": "world-fact", "older": "The Razer Viper Mini costs around $50.", "newer": "The Razer Viper Mini costs approximately $40.", "verdicts": [ "different", "changes", "fixed" ], "samples": [ "fixed", "fixed", "fixed" ] }, { "id": 18, "label": "world-fact", "older": "The Mophie Powerstation Mini has a capacity of 4000mAh.", "newer": "The Mophie Powerstation Mini has a capacity of 5,000mAh.", "verdicts": [ "different", "fixed", "fixed" ], "samples": [ "fixed", "fixed", "fixed" ] }, { "id": 19, "label": "world-fact", "older": "The Mophie Powerstation Mini costs around $20-25.", "newer": "The Mophie Powerstation Mini costs approximately $30.", "verdicts": [ "fixed", "different", "fixed" ], "samples": [ "fixed", "fixed", "fixed" ] }, { "id": 20, "label": "world-fact", "older": "The Mophie Powerstation XXL has a capacity of 20,800mAh.", "newer": "The Mophie Powerstation XXL has a 20,000mAh capacity.", "verdicts": [ "different", "different", "fixed" ], "samples": [ "fixed", "fixed", "fixed" ] }, { "id": 23, "label": "world-fact", "older": "Gochujang is a fermented Korean chili paste.", "newer": "Gochujang is a fermented soybean paste.", "verdicts": [ "different", "fixed", "different" ], "samples": [ "fixed", "fixed", "fixed" ] }, { "id": 28, "label": "world-fact", "older": "The Gaiam Print Yoga Mat is priced around $25.", "newer": "Gaiam Print Yoga Mat costs $20.", "verdicts": [ "different", "different", "fixed" ], "samples": [ "fixed", "fixed", "fixed" ] }, { "id": 29, "label": "world-fact", "older": "The Jade Yoga Mat is priced around $40.", "newer": "Jade Yoga Mat costs $60.", "verdicts": [ "different", "fixed", "different" ], "samples": [ "fixed", "fixed", "fixed" ] }, { "id": 30, "label": "world-fact", "older": "As a US citizen, one does not need a visa for tourism or business stays up to 6 months in the United Kingdom.", "newer": "US citizens don't need a visa to visit the UK for tourism or visiting relatives as long as their stay doesn't exceed 90 days within any 180-day period.", "verdicts": [ "different", "different", "different" ], "samples": [ "fixed", "fixed", "fixed" ] }, { "id": 31, "label": "world-fact", "older": "The Kawai ES110 costs around $699 - $799 depending on the retailer and bundle.", "newer": "The Kawai ES110 costs $299.", "verdicts": [ "fixed", "fixed", "fixed" ], "samples": [ "fixed", "fixed", "fixed" ] }, { "id": 32, "label": "world-fact", "older": "\"The Poison Tree\" was written by Erin Kelly.", "newer": "The Poison Tree was written by Barbara Hambly.", "verdicts": [ "different", "different", "different" ], "samples": [ "fixed", "fixed", "fixed" ] }, { "id": 34, "label": "world-fact", "older": "United Airlines typically requires 30,000-50,000 miles for a one-way upgrade from economy to business class on a domestic flight.", "newer": "United Airlines domestic flight upgrades typically require 15,000 to 30,000 miles one-way.", "verdicts": [ "fixed", "fixed", "fixed" ], "samples": [ "fixed", "fixed", "fixed" ] }, { "id": 35, "label": "world-fact", "older": "The Dell XPS 13 base model comes with a 256GB SSD.", "newer": "The Dell XPS 13 has a 512GB SSD.", "verdicts": [ "fixed", "different", "fixed" ], "samples": [ "fixed", "fixed", "fixed" ] }, { "id": "h0", "label": "user-update", "older": "The user lives in Delhi, in Lajpat Nagar.", "newer": "The user moved to Mumbai last month.", "verdicts": [ "changes", "changes", "changes" ], "samples": [ "changes", "changes", "changes" ] }, { "id": "h1", "label": "user-update", "older": "The user lives in Delhi.", "newer": "The user lives in Mumbai.", "verdicts": [ "changes", "changes", "changes" ], "samples": [ "changes", "changes", "changes" ] }, { "id": "h2", "label": "user-update", "older": "The user's favourite restaurant is Sandeep's Curry House.", "newer": "The user's favourite restaurant is now Bombay Canteen.", "verdicts": [ "changes", "changes", "changes" ], "samples": [ "changes", "changes", "changes" ] }, { "id": "h3", "label": "user-update", "older": "The user is vegetarian.", "newer": "The user eats fish now.", "verdicts": [ "changes", "changes", "changes" ], "samples": [ "changes", "changes", "changes" ] }, { "id": "h4", "label": "different", "older": "Sandeep's Curry House is the user's favourite place to eat.", "newer": "The user has not yet found a regular place to eat.", "verdicts": [ "different", "different", "different" ], "samples": [ "different", "different", "different" ] }, { "id": "h5", "label": "user-update", "older": "The user's home city is Bergen.", "newer": "The user's home city is Ljubljana.", "verdicts": [ "changes", "changes", "changes" ], "samples": [ "changes", "changes", "changes" ] }, { "id": "h6", "label": "user-update", "older": "The user's car is Volvo.", "newer": "The user's car is Tesla.", "verdicts": [ "changes", "changes", "changes" ], "samples": [ "changes", "changes", "changes" ] }, { "id": "h7", "label": "user-update", "older": "The user's phone is Pixel.", "newer": "The user's phone is iPhone.", "verdicts": [ "changes", "changes", "changes" ], "samples": [ "changes", "changes", "changes" ] }, { "id": "h8", "label": "user-update", "older": "The user's primary code editor is Neovim.", "newer": "The user's primary code editor is BBEdit.", "verdicts": [ "changes", "changes", "changes" ], "samples": [ "changes", "changes", "changes" ] }, { "id": "h9", "label": "user-update", "older": "The user's manager is Tomasz.", "newer": "The user's manager is Ingrid.", "verdicts": [ "changes", "changes", "changes" ], "samples": [ "changes", "changes", "changes" ] }, { "id": "h10", "label": "user-update", "older": "The user's diet is kosher.", "newer": "The user's diet is Mediterranean.", "verdicts": [ "changes", "changes", "changes" ], "samples": [ "changes", "changes", "changes" ] }, { "id": "h11", "label": "user-update", "older": "The user's current project codename is Kestrel.", "newer": "The user's current project codename is Quasar.", "verdicts": [ "changes", "changes", "changes" ], "samples": [ "changes", "changes", "changes" ] }, { "id": "h12", "label": "user-update", "older": "The user's regular coffee shop is Blue Door.", "newer": "The user's regular coffee shop is Lantern.", "verdicts": [ "changes", "changes", "changes" ], "samples": [ "changes", "changes", "changes" ] }, { "id": "h13", "label": "user-update", "older": "The user's language of study is Hungarian.", "newer": "The user's language of study is Tagalog.", "verdicts": [ "changes", "changes", "changes" ], "samples": [ "changes", "changes", "changes" ] }, { "id": "h14", "label": "history", "older": "The user's manager was previously Tomasz.", "newer": "The user's manager was previously Ingrid.", "verdicts": [ "different", "different", "different" ], "samples": [ "different", "different", "different" ] }, { "id": "h15", "label": "history", "older": "The user's diet is vegetarian.", "newer": "The user's diet was previously paleo.", "verdicts": [ "different", "different", "different" ], "samples": [ "different", "different", "different" ] } ], "bars": { "world_changes": "0/54", "world_ok": true, "user_updates_changes": "21/21", "user_updates_ok": true, "hand_misses": [], "hand_ok": true, "world_changes_by_row": { "5": 0, "6": 0, "7": 0, "8": 0, "10": 0, "12": 0, "13": 0, "18": 0, "19": 0, "20": 0, "23": 0, "28": 0, "29": 0, "30": 0, "31": 0, "32": 0, "34": 0, "35": 0 } } }, "configuration": { "rows": [ { "id": "ledgerly-moved", "older": "Ledgerly uses PostgreSQL 16 as its database.", "newer": "Ledgerly was moved from PostgreSQL to SQLite in the week before 2026-09-20.", "expected": "changes", "samples": [ "changes", "changes", "changes" ] }, { "id": "kestrel-host-now", "older": "Kestrel is hosted on Heroku.", "newer": "Kestrel is hosted on Fly.io now.", "expected": "changes", "samples": [ "changes", "changes", "changes" ] }, { "id": "kestrel-host-migrated", "older": "Kestrel is hosted on Heroku.", "newer": "We migrated Kestrel from Heroku to Render last month.", "expected": "changes", "samples": [ "changes", "changes", "changes" ] }, { "id": "pelican-ci-uses", "older": "Pelican's CI runs on Jenkins.", "newer": "Pelican's CI runs on GitHub Actions.", "expected": "changes", "samples": [ "changes", "changes", "changes" ] }, { "id": "pelican-ci-switched", "older": "Pelican's CI runs on Jenkins.", "newer": "Pelican's CI was switched from Jenkins to Buildkite.", "expected": "changes", "samples": [ "changes", "changes", "changes" ] }, { "id": "brightline-mail-replaced", "older": "Brightline sends its email through Mailgun.", "newer": "Brightline replaced Mailgun with Postmark for sending email.", "expected": "changes", "samples": [ "changes", "changes", "changes" ] }, { "id": "quill-vite-upgraded", "older": "Quill's frontend is built with Vite 4.", "newer": "Quill's frontend was upgraded to Vite 5.", "expected": "changes", "samples": [ "changes", "changes", "changes" ] }, { "id": "tern-login-switched", "older": "Tern uses Auth0 for login.", "newer": "Tern switched its login from Auth0 to Clerk.", "expected": "changes", "samples": [ "changes", "changes", "changes" ] }, { "id": "team-tracker", "older": "The team tracks issues in Jira.", "newer": "The team tracks issues in Linear.", "expected": "changes", "samples": [ "changes", "changes", "changes" ] }, { "id": "ledgerly-cache-added", "older": "Ledgerly uses PostgreSQL 16 as its database.", "newer": "Ledgerly added Redis as a cache.", "expected": "not_changes", "samples": [ "different", "different", "different" ] }, { "id": "mouse-price", "older": "The Logitech MX Master 3S costs $99.", "newer": "The Logitech MX Master 3S costs $79.", "expected": "not_changes", "samples": [ "fixed", "fixed", "fixed" ] }, { "id": "pi-ram", "older": "The Raspberry Pi 5 has 8 GB of RAM.", "newer": "The Raspberry Pi 5 has 4 GB of RAM.", "expected": "not_changes", "samples": [ "fixed", "fixed", "fixed" ] }, { "id": "orbit-history", "older": "Orbit deploys to a VM on DigitalOcean.", "newer": "Orbit previously ran on Heroku.", "expected": "not_changes", "samples": [ "different", "different", "different" ] }, { "id": "pelican-ci-history", "older": "Pelican's CI previously ran on Travis.", "newer": "Pelican's CI previously ran on CircleCI.", "expected": "not_changes", "samples": [ "different", "different", "different" ] }, { "id": "quill-bucket-detail", "older": "Quill stores its assets in S3.", "newer": "Quill stores its assets in the S3 bucket quill-assets in eu-west-1.", "expected": "not_changes", "samples": [ "different", "different", "different" ] }, { "id": "tern-billing", "older": "Tern uses Auth0 for login.", "newer": "Tern uses Stripe for billing.", "expected": "not_changes", "samples": [ "different", "different", "different" ] }, { "id": "quill-cache-step", "older": "To clear Quill's asset cache, invalidate the CloudFront distribution in front of the S3 bucket.", "newer": "Quill moved its assets from S3 to Cloudflare R2.", "expected": "not_changes", "samples": [ "different", "different", "different" ] }, { "id": "pelican-gradle-pin", "older": "Pelican's Gradle wrapper is pinned to 7.6 because the CI build box only has Java 11.", "newer": "Pelican's CI was switched from the Jenkins build box to GitHub Actions.", "expected": "not_changes", "samples": [ "different", "different", "different" ] } ], "bars": { "misses": [], "ok": true } }, "stay": { "rows": [ { "id": "trip-move-run2", "class": "temporary-to-permanent", "older": "The user is in Mumbai for a week for work, on a trip that began around 2026-10-08.", "newer": "The user moved to Mumbai, as stated in a session dated 2026-10-09.", "expected": "changes", "samples": [ "changes", "changes", "changes" ] }, { "id": "trip-move-short", "class": "temporary-to-permanent", "older": "The user is in Mumbai for a week for work.", "newer": "The user moved to Mumbai.", "expected": "changes", "samples": [ "changes", "changes", "changes" ] }, { "id": "sister-pune", "class": "temporary-to-permanent", "older": "The user is staying with their sister in Pune for a month.", "newer": "The user now lives in Pune.", "expected": "changes", "samples": [ "changes", "changes", "changes" ] }, { "id": "sublet-berlin", "class": "temporary-to-permanent", "older": "The user is subletting a flat in Berlin for the summer.", "newer": "The user has moved to Berlin permanently.", "expected": "changes", "samples": [ "changes", "changes", "changes" ] }, { "id": "contract-acme", "class": "temporary-to-permanent", "older": "The user is working at Acme on a three-month contract.", "newer": "The user joined Acme as a full-time employee.", "expected": "changes", "samples": [ "changes", "changes", "changes" ] }, { "id": "foster-biscuit", "class": "temporary-to-permanent", "older": "The user is fostering a beagle named Biscuit until a home is found for him.", "newer": "The user adopted Biscuit, the beagle they were fostering.", "expected": "changes", "samples": [ "changes", "changes", "changes" ] }, { "id": "interim-lead", "class": "temporary-to-permanent", "older": "The user is the team's interim lead while Priya is on leave.", "newer": "The user was made the team's permanent lead.", "expected": "changes", "samples": [ "changes", "changes", "changes" ] }, { "id": "trip-for-good", "class": "decided-not-ended", "older": "The user is in Mumbai for a week for work.", "newer": "The user has decided to move to Mumbai for good.", "expected": "not_changes", "samples": [ "different", "different", "different" ] }, { "id": "visiting-mit", "class": "decided-not-ended", "older": "The user is a visiting researcher at MIT for the spring semester.", "newer": "The user accepted a permanent faculty position at MIT.", "expected": "not_changes", "samples": [ "changes", "changes", "different" ] }, { "id": "trip-plans-move", "class": "decided-not-ended", "older": "The user is in Mumbai for a week for work.", "newer": "The user plans to move to Mumbai next year.", "expected": "not_changes", "samples": [ "different", "different", "different" ] }, { "id": "trip-hotel-detail", "class": "same-place-non-update", "older": "The user is in Mumbai for a week for work.", "newer": "The user is staying at a hotel in Bandra, Mumbai.", "expected": "not_changes", "samples": [ "different", "different", "different" ] }, { "id": "trip-restated", "class": "same-place-non-update", "older": "The user is in Mumbai for a week for work.", "newer": "The user is still in Mumbai on the work trip, as of 2026-10-11.", "expected": "not_changes", "samples": [ "different", "different", "different" ] }, { "id": "trip-enjoys", "class": "same-place-non-update", "older": "The user is in Mumbai for a week for work.", "newer": "The user enjoys the street food in Mumbai.", "expected": "not_changes", "samples": [ "different", "different", "different" ] }, { "id": "sister-lives", "class": "same-place-non-update", "older": "The user is staying with their sister in Pune for a month.", "newer": "The user's sister lives in Pune.", "expected": "not_changes", "samples": [ "different", "different", "different" ] }, { "id": "contract-start", "class": "same-place-non-update", "older": "The user is working at Acme on a three-month contract.", "newer": "The user's contract at Acme started on 2026-09-01.", "expected": "not_changes", "samples": [ "different", "different", "different" ] }, { "id": "foster-age", "class": "same-place-non-update", "older": "The user is fostering a beagle named Biscuit until a home is found for him.", "newer": "Biscuit, the beagle the user is fostering, is three years old.", "expected": "not_changes", "samples": [ "different", "different", "different" ] }, { "id": "move-detail", "class": "same-place-non-update", "older": "The user moved to Mumbai.", "newer": "The user moved to Mumbai in October 2026 and lives in Bandra.", "expected": "not_changes", "samples": [ "different", "different", "different" ] }, { "id": "trip-owns-flat", "class": "same-place-non-update", "older": "The user is in Mumbai for a week for work.", "newer": "The user owns a flat in Mumbai that they rent out.", "expected": "not_changes", "samples": [ "different", "different", "different" ] }, { "id": "trip-company-hq", "class": "same-place-non-update", "older": "The user is in Mumbai for a week for work.", "newer": "The user's company is headquartered in Mumbai.", "expected": "not_changes", "samples": [ "different", "different", "different" ] }, { "id": "resident-this-week", "class": "lasting-then-temporary", "older": "The user lives in Mumbai.", "newer": "The user is in Mumbai this week, working from the Bandra office.", "expected": "not_changes", "samples": [ "different", "different", "different" ] }, { "id": "home-trip", "class": "lasting-then-temporary", "older": "The user lives in Delhi.", "newer": "The user is in Mumbai for a week for work.", "expected": "not_changes", "samples": [ "different", "different", "different" ] }, { "id": "home-diwali", "class": "lasting-then-temporary", "older": "The user lives in Lajpat Nagar, Delhi.", "newer": "The user is visiting their parents in Chennai over Diwali.", "expected": "not_changes", "samples": [ "different", "different", "different" ] }, { "id": "fulltime-secondment", "class": "lasting-then-temporary", "older": "The user works full-time at Acme's London office.", "newer": "The user is on a three-month secondment to Acme's Leeds office.", "expected": "not_changes", "samples": [ "different", "different", "different" ] }, { "id": "lead-interim-head", "class": "lasting-then-temporary", "older": "The user is the lead of the payments team.", "newer": "The user is covering as interim head of engineering while Ravi is on leave.", "expected": "not_changes", "samples": [ "different", "different", "different" ] }, { "id": "home-moved", "class": "anchor-update", "older": "The user lives in Delhi.", "newer": "The user moved to Mumbai.", "expected": "changes", "samples": [ "changes", "changes", "changes" ] }, { "id": "trip-back-home", "class": "temporary-ended", "older": "The user is in Mumbai for a week for work.", "newer": "The user is back home in Delhi after the Mumbai trip.", "expected": "report", "samples": [ "different", "different", "different" ] }, { "id": "contract-extended", "class": "renewed-bound", "older": "The user is working at Acme on a three-month contract.", "newer": "The user's contract at Acme was extended to six months.", "expected": "report", "samples": [ "different", "different", "different" ] } ], "bars": { "misses": [ "visiting-mit" ], "false_changes": "2/51", "ok": false } }, "stay-heldout": { "rows": [ { "id": "goa-winter", "class": "temporary-to-permanent", "older": "The user is spending the winter at their parents' house in Goa.", "newer": "The user has given up their Bengaluru flat and lives in Goa year-round.", "expected": "changes", "samples": [ "changes", "changes", "changes" ] }, { "id": "leeds-housesit", "class": "temporary-to-permanent", "older": "The user is house-sitting for a friend in Leeds until the end of March.", "newer": "The user bought a terraced house in Leeds and lives there.", "expected": "changes", "samples": [ "changes", "changes", "changes" ] }, { "id": "singapore-trip", "class": "temporary-to-permanent", "older": "The user is in Singapore on a two-week business trip.", "newer": "The user relocated to Singapore when the company opened its office there.", "expected": "changes", "samples": [ "changes", "changes", "changes" ] }, { "id": "valencia-room", "class": "temporary-to-permanent", "older": "The user is renting a room in Valencia for six weeks.", "newer": "The user has settled in Valencia and signed a two-year lease on a flat in Ruzafa.", "expected": "changes", "samples": [ "changes", "changes", "changes" ] }, { "id": "nairobi-placement", "class": "temporary-to-permanent", "older": "The user is in Nairobi on a three-month field placement.", "newer": "Nairobi is now the user's permanent home.", "expected": "changes", "samples": [ "changes", "changes", "changes" ] }, { "id": "glasgow-sofa", "class": "temporary-to-permanent", "older": "The user is sleeping on their brother's sofa in Glasgow for a few days.", "newer": "The user rents a flat in Glasgow's West End and lives there.", "expected": "changes", "samples": [ "changes", "changes", "changes" ] }, { "id": "glasgow-weekend", "class": "lasting-then-temporary", "older": "The user lives in Glasgow.", "newer": "The user is in Edinburgh for the weekend.", "expected": "not_changes", "samples": [ "different", "different", "different" ] }, { "id": "singapore-office", "class": "same-place-non-update", "older": "The user is in Singapore on a two-week business trip.", "newer": "The user's employer has an office in Singapore.", "expected": "not_changes", "samples": [ "different", "different", "different" ] }, { "id": "nairobi-considering", "class": "decided-not-ended", "older": "The user is in Nairobi on a three-month field placement.", "newer": "The user is thinking about staying in Nairobi permanently.", "expected": "not_changes", "samples": [ "different", "different", "different" ] }, { "id": "valencia-landlord", "class": "same-place-non-update", "older": "The user is renting a room in Valencia for six weeks.", "newer": "The user rents the room in Valencia from a retired schoolteacher.", "expected": "not_changes", "samples": [ "different", "different", "different" ] }, { "id": "valencia-repaint", "class": "lasting-then-temporary", "older": "The user lives in Valencia.", "newer": "The user is staying at a hotel in Valencia this week while their flat is repainted.", "expected": "not_changes", "samples": [ "different", "different", "different" ] }, { "id": "nurse-placement", "class": "lasting-then-temporary", "older": "The user works as a nurse at St Mary's Hospital.", "newer": "The user is on a six-week placement at the Royal Infirmary.", "expected": "not_changes", "samples": [ "different", "different", "different" ] }, { "id": "leeds-job-offer", "class": "decided-not-ended", "older": "The user is house-sitting for a friend in Leeds until the end of March.", "newer": "The user has accepted a job in Leeds that starts in April.", "expected": "not_changes", "samples": [ "different", "different", "different" ] }, { "id": "goa-apartment", "class": "same-place-non-update", "older": "The user is spending the winter at their parents' house in Goa.", "newer": "The user owns a small holiday apartment in Goa.", "expected": "not_changes", "samples": [ "different", "different", "different" ] } ], "bars": { "positives_matched": "6/6", "positive_misses": [], "false_changes": "0/24", "ok": true } }, "gate": { "rows": [ { "id": "res-dated-newer", "class": "dated-update", "older": "The user lives in Antwerp.", "newer": "The user moved to Ghent, as stated in a session dated 2026-05-02.", "expected": "changes", "samples": [ { "contradiction": true, "slot": "changes" }, { "contradiction": true, "slot": "changes" }, { "contradiction": true, "slot": "changes" } ] }, { "id": "res-dated-both", "class": "dated-update", "older": "The user lives in Antwerp, as of 2026-01-10.", "newer": "The user lives in Ghent, as of 2026-05-02.", "expected": "changes", "samples": [ { "contradiction": false, "slot": "changes" }, { "contradiction": false, "slot": "changes" }, { "contradiction": false, "slot": "changes" } ] }, { "id": "employer-dated", "class": "dated-update", "older": "The user works at Halvorsen Logistics, as of 2026-01-15.", "newer": "The user started a new job at Brightwater Labs on 2026-03-02.", "expected": "changes", "samples": [ { "contradiction": false, "slot": "changes" }, { "contradiction": false, "slot": "changes" }, { "contradiction": false, "slot": "changes" } ] }, { "id": "manager-dated", "class": "dated-update", "older": "The user's manager is Priya, as of 2026-02-03.", "newer": "The user's manager is now Ines, as stated in a session dated 2026-04-11.", "expected": "changes", "samples": [ { "contradiction": false, "slot": "changes" }, { "contradiction": false, "slot": "changes" }, { "contradiction": false, "slot": "changes" } ] }, { "id": "diet-since", "class": "dated-update", "older": "The user eats meat.", "newer": "The user has been vegan since 2026-06-01.", "expected": "changes", "samples": [ { "contradiction": true, "slot": "changes" }, { "contradiction": true, "slot": "changes" }, { "contradiction": true, "slot": "changes" } ] }, { "id": "phone-dated", "class": "dated-update", "older": "The user uses an iPhone 13.", "newer": "The user switched to a Pixel 9, as stated in a session dated 2026-07-19.", "expected": "changes", "samples": [ { "contradiction": true, "slot": "changes" }, { "contradiction": true, "slot": "changes" }, { "contradiction": true, "slot": "changes" } ] }, { "id": "host-dated", "class": "dated-update", "older": "The billing service is hosted on Render, as of 2026-03-02.", "newer": "The billing service was migrated to DigitalOcean on 2026-04-15.", "expected": "changes", "samples": [ { "contradiction": false, "slot": "changes" }, { "contradiction": false, "slot": "changes" }, { "contradiction": false, "slot": "changes" } ] }, { "id": "count-dated", "class": "dated-update", "older": "The user has run 3 marathons, as of 2026-01-05.", "newer": "The user has run 4 marathons, as stated in a session dated 2026-10-01.", "expected": "changes", "samples": [ { "contradiction": false, "slot": "changes" }, { "contradiction": false, "slot": "changes" }, { "contradiction": false, "slot": "changes" } ] }, { "id": "stay-to-home-dated", "class": "dated-update", "older": "The user is in Montreal for a two-week conference, as of 2026-05-10.", "newer": "The user relocated to Montreal, as stated in a session dated 2026-05-20.", "expected": "changes", "samples": [ { "contradiction": true, "slot": "changes" }, { "contradiction": true, "slot": "changes" }, { "contradiction": true, "slot": "changes" } ] }, { "id": "gym-dated", "class": "dated-update", "older": "The user's gym is FitHub on Elm Street, as of 2026-02-01.", "newer": "The user cancelled FitHub and now trains at Ironworks Gym, as stated in a session dated 2026-05-09.", "expected": "changes", "samples": [ { "contradiction": false, "slot": "changes" }, { "contradiction": false, "slot": "changes" }, { "contradiction": true, "slot": "changes" } ] }, { "id": "pets", "class": "coexisting", "older": "The user has a cat named Miso.", "newer": "The user adopted a dog named Pepper, as stated in a session dated 2026-06-02.", "expected": "not_changes", "samples": [ { "contradiction": false, "slot": "different" }, { "contradiction": false, "slot": "different" }, { "contradiction": false, "slot": "different" } ] }, { "id": "languages", "class": "coexisting", "older": "The user speaks French, as of 2026-01-20.", "newer": "The user is learning Japanese, as stated in a session dated 2026-04-03.", "expected": "not_changes", "samples": [ { "contradiction": false, "slot": "different" }, { "contradiction": false, "slot": "different" }, { "contradiction": false, "slot": "different" } ] }, { "id": "hobbies", "class": "coexisting", "older": "The user plays tennis on Saturdays, as of 2026-02-01.", "newer": "The user joined a climbing gym on 2026-04-09.", "expected": "not_changes", "samples": [ { "contradiction": false, "slot": "different" }, { "contradiction": false, "slot": "different" }, { "contradiction": false, "slot": "different" } ] }, { "id": "stack", "class": "coexisting", "older": "The project uses Redis for caching.", "newer": "The project added PostgreSQL for its user accounts on 2026-05-03.", "expected": "not_changes", "samples": [ { "contradiction": false, "slot": "different" }, { "contradiction": false, "slot": "different" }, { "contradiction": false, "slot": "different" } ] }, { "id": "accounts", "class": "coexisting", "older": "The user has a savings account with Nordbank.", "newer": "The user opened a current account with Kestrel Bank, as stated in a session dated 2026-03-14.", "expected": "not_changes", "samples": [ { "contradiction": false, "slot": "different" }, { "contradiction": false, "slot": "different" }, { "contradiction": false, "slot": "different" } ] }, { "id": "trips", "class": "separate-events", "older": "The user took a five-day trip to Rome in March 2026.", "newer": "The user spent a weekend in Vienna, as stated in a session dated 2026-06-14.", "expected": "not_changes", "samples": [ { "contradiction": false, "slot": "different" }, { "contradiction": false, "slot": "different" }, { "contradiction": false, "slot": "different" } ] }, { "id": "purchases", "class": "separate-events", "older": "The user paid $90 for running shoes, as of 2026-02-02.", "newer": "The user bought a rain jacket for $120 on 2026-04-20.", "expected": "not_changes", "samples": [ { "contradiction": false, "slot": "different" }, { "contradiction": false, "slot": "different" }, { "contradiction": false, "slot": "different" } ] }, { "id": "races", "class": "separate-events", "older": "The user ran a half marathon in Bruges on 2026-03-08.", "newer": "The user ran a 10K in Leuven, as stated in a session dated 2026-05-17.", "expected": "not_changes", "samples": [ { "contradiction": false, "slot": "different" }, { "contradiction": false, "slot": "different" }, { "contradiction": false, "slot": "different" } ] }, { "id": "flat-detail", "class": "added-detail", "older": "The user lives in Antwerp.", "newer": "The user's flat in Antwerp has a small balcony, as stated in a session dated 2026-05-02.", "expected": "not_changes", "samples": [ { "contradiction": false, "slot": "different" }, { "contradiction": false, "slot": "different" }, { "contradiction": false, "slot": "different" } ] }, { "id": "role-detail", "class": "added-detail", "older": "The user works at Halvorsen Logistics.", "newer": "The user leads the routing team at Halvorsen Logistics, as of 2026-04-01.", "expected": "not_changes", "samples": [ { "contradiction": false, "slot": "different" }, { "contradiction": false, "slot": "different" }, { "contradiction": false, "slot": "different" } ] }, { "id": "lived-until", "class": "past-value", "older": "The user lived in Utrecht until 2025.", "newer": "The user lives in Ghent, as of 2026-05-02.", "expected": "not_changes", "samples": [ { "contradiction": false, "slot": "different" }, { "contradiction": false, "slot": "different" }, { "contradiction": false, "slot": "different" } ] }, { "id": "manager-previously", "class": "past-value", "older": "The user's manager was previously Priya.", "newer": "The user's manager is Ines, as stated in a session dated 2026-04-11.", "expected": "not_changes", "samples": [ { "contradiction": false, "slot": "different" }, { "contradiction": false, "slot": "different" }, { "contradiction": false, "slot": "different" } ] }, { "id": "dated-history", "class": "past-value", "older": "As of 2026-03-01, when the user lived in Antwerp, the user commuted by tram.", "newer": "The user lives in Ghent, as stated in a session dated 2026-05-02.", "expected": "not_changes", "samples": [ { "contradiction": false, "slot": "different" }, { "contradiction": false, "slot": "different" }, { "contradiction": false, "slot": "different" } ] }, { "id": "decided-move", "class": "decided-not-done", "older": "The user lives in Antwerp.", "newer": "The user has decided to move to Ghent next spring, as stated in a session dated 2026-05-02.", "expected": "not_changes", "samples": [ { "contradiction": false, "slot": "different" }, { "contradiction": false, "slot": "different" }, { "contradiction": false, "slot": "different" } ] }, { "id": "home-then-visit", "class": "lasting-then-temporary", "older": "The user lives in Antwerp.", "newer": "The user is in Ghent for three days for a wedding, as stated in a session dated 2026-05-02.", "expected": "not_changes", "samples": [ { "contradiction": false, "slot": "different" }, { "contradiction": false, "slot": "different" }, { "contradiction": false, "slot": "different" } ] }, { "id": "job-then-secondment", "class": "lasting-then-temporary", "older": "The user works full-time at Halvorsen Logistics.", "newer": "The user is on a two-month secondment to the Oslo office, as of 2026-04-01.", "expected": "not_changes", "samples": [ { "contradiction": true, "slot": "different" }, { "contradiction": false, "slot": "different" }, { "contradiction": false, "slot": "different" } ] }, { "id": "deploy-step", "class": "relied-on", "older": "Deploy the billing service with `render deploy`.", "newer": "The billing service was migrated to DigitalOcean on 2026-04-15.", "expected": "not_changes", "samples": [ { "contradiction": true, "slot": "different" }, { "contradiction": true, "slot": "different" }, { "contradiction": false, "slot": "different" } ] }, { "id": "commute", "class": "relied-on", "older": "The user's commute to the Halvorsen office is a 15-minute bike ride.", "newer": "The user moved to Ghent, as stated in a session dated 2026-05-02.", "expected": "not_changes", "samples": [ { "contradiction": false, "slot": "different" }, { "contradiction": false, "slot": "different" }, { "contradiction": false, "slot": "different" } ] }, { "id": "kettle-price", "class": "fixed", "older": "The Brightwater X2 kettle costs $49.", "newer": "The Brightwater X2 kettle costs $59, as stated in a review dated 2026-05-02.", "expected": "not_changes", "samples": [ { "contradiction": true, "slot": "fixed" }, { "contradiction": true, "slot": "fixed" }, { "contradiction": true, "slot": "fixed" } ] }, { "id": "case-trip-5-5-r2", "class": "case", "older": "The user is in Mumbai for a week for work, as of 2026-10-08.", "newer": "The user moved to Mumbai, as stated in a session dated 2026-10-09.", "expected": "changes", "samples": [ { "contradiction": true, "slot": "changes" }, { "contradiction": true, "slot": "changes" }, { "contradiction": true, "slot": "changes" } ] }, { "id": "case-trip-5-5-r1", "class": "case", "older": "The user is in Mumbai for a week on a work trip, as of 2026-10-08.", "newer": "The user moved to Mumbai, as stated in a session dated 2026-10-09.", "expected": "changes", "samples": [ { "contradiction": true, "slot": "changes" }, { "contradiction": true, "slot": "changes" }, { "contradiction": true, "slot": "changes" } ] }, { "id": "case-residence-5-5-r2", "class": "case", "older": "The user lives in Lajpat Nagar, Delhi.", "newer": "The user moved to Mumbai, as stated in a session dated 2026-10-09.", "expected": "changes", "samples": [ { "contradiction": true, "slot": "changes" }, { "contradiction": true, "slot": "changes" }, { "contradiction": true, "slot": "changes" } ] }, { "id": "case-residence-5-5-r1", "class": "case", "older": "The user lives in a flat in Lajpat Nagar, Delhi.", "newer": "The user moved to Mumbai, as stated in a session dated 2026-10-09.", "expected": "changes", "samples": [ { "contradiction": true, "slot": "changes" }, { "contradiction": true, "slot": "changes" }, { "contradiction": true, "slot": "changes" } ] }, { "id": "case-home-5-5", "class": "case", "older": "The user's home is in Delhi.", "newer": "The user moved to Mumbai, as stated in a session dated 2026-10-09.", "expected": "changes", "samples": [ { "contradiction": true, "slot": "changes" }, { "contradiction": true, "slot": "changes" }, { "contradiction": true, "slot": "changes" } ] }, { "id": "case-trip-4-6", "class": "case", "older": "The user is in Mumbai for a week for work.", "newer": "The user moved to Mumbai.", "expected": "changes", "samples": [ { "contradiction": false, "slot": "changes" }, { "contradiction": false, "slot": "changes" }, { "contradiction": true, "slot": "changes" } ] }, { "id": "case-trip-alone", "class": "case", "older": "The user lives in Lajpat Nagar, Delhi.", "newer": "The user is in Mumbai for a week for work, as of 2026-10-08.", "expected": "not_changes", "samples": [ { "contradiction": false, "slot": "different" }, { "contradiction": false, "slot": "different" }, { "contradiction": false, "slot": "different" } ] }, { "id": "case-flight", "class": "case", "older": "The user flies from Mumbai back to Delhi on Thursday 2026-10-15.", "newer": "The user moved to Mumbai, as stated in a session dated 2026-10-09.", "expected": "report", "samples": [ { "contradiction": false, "slot": "different" }, { "contradiction": false, "slot": "different" }, { "contradiction": false, "slot": "different" } ] }, { "id": "case-hotel", "class": "case", "older": "The user is staying at a hotel in Bandra, Mumbai.", "newer": "The user moved to Mumbai, as stated in a session dated 2026-10-09.", "expected": "report", "samples": [ { "contradiction": false, "slot": "changes" }, { "contradiction": false, "slot": "changes" }, { "contradiction": false, "slot": "changes" } ] } ], "bars": { "proposed": { "positives_retired": "10/10", "positive_misses": [], "false_retirements": "0/57", "false_rows": [], "ok": true }, "shipped": { "positives_retired": "4/10", "positive_misses": [ "res-dated-both", "employer-dated", "manager-dated", "host-dated", "count-dated", "gym-dated" ], "false_retirements": "0/57", "false_rows": [], "ok": false }, "case": { "case-trip-5-5-r2": { "expected": "changes", "proposed": 3, "shipped": 3, "of": 3 }, "case-trip-5-5-r1": { "expected": "changes", "proposed": 3, "shipped": 3, "of": 3 }, "case-residence-5-5-r2": { "expected": "changes", "proposed": 3, "shipped": 3, "of": 3 }, "case-residence-5-5-r1": { "expected": "changes", "proposed": 3, "shipped": 3, "of": 3 }, "case-home-5-5": { "expected": "changes", "proposed": 3, "shipped": 3, "of": 3 }, "case-trip-4-6": { "expected": "changes", "proposed": 3, "shipped": 1, "of": 3 }, "case-trip-alone": { "expected": "not_changes", "proposed": 0, "shipped": 0, "of": 3 }, "case-flight": { "expected": "report", "proposed": 0, "shipped": 0, "of": 3 }, "case-hotel": { "expected": "report", "proposed": 3, "shipped": 0, "of": 3 } } } }, "gate-heldout": { "rows": [ { "id": "tartu-tallinn", "class": "dated-update", "older": "The user rents an apartment in Tartu.", "newer": "Since August 2026 the user has lived in Tallinn.", "expected": "changes", "samples": [ { "contradiction": false, "slot": "changes" }, { "contradiction": true, "slot": "changes" }, { "contradiction": false, "slot": "changes" } ] }, { "id": "car", "class": "dated-update", "older": "The user drives a 2015 Honda Civic (noted 2026-02-11).", "newer": "The user sold the Civic and now drives a Kia EV6, per a note from 2026-08-03.", "expected": "changes", "samples": [ { "contradiction": true, "slot": "changes" }, { "contradiction": true, "slot": "changes" }, { "contradiction": true, "slot": "changes" } ] }, { "id": "standup", "class": "dated-update", "older": "The team's daily standup is at 9:30.", "newer": "On 2026-07-01 the team moved its daily standup to 10:15.", "expected": "changes", "samples": [ { "contradiction": true, "slot": "changes" }, { "contradiction": true, "slot": "changes" }, { "contradiction": true, "slot": "changes" } ] }, { "id": "isp", "class": "dated-update", "older": "The user's internet provider is Telenet.", "newer": "The user changed internet provider to Proximus on 14 June 2026.", "expected": "changes", "samples": [ { "contradiction": true, "slot": "changes" }, { "contradiction": true, "slot": "changes" }, { "contradiction": true, "slot": "changes" } ] }, { "id": "laptop", "class": "dated-update", "older": "The user's main laptop is a ThinkPad X1 (noted 2026-01-09).", "newer": "The user replaced the ThinkPad with a MacBook Air on 2026-06-22.", "expected": "changes", "samples": [ { "contradiction": false, "slot": "changes" }, { "contradiction": false, "slot": "changes" }, { "contradiction": false, "slot": "changes" } ] }, { "id": "rent", "class": "dated-update", "older": "The user's monthly rent is 950 euros.", "newer": "From 2026-09-01 the user's rent went up to 1,020 euros a month.", "expected": "changes", "samples": [ { "contradiction": true, "slot": "changes" }, { "contradiction": true, "slot": "changes" }, { "contradiction": true, "slot": "changes" } ] }, { "id": "krakow-let", "class": "dated-update", "older": "The user is staying in a short-let flat in Krakow for May 2026.", "newer": "The user signed a permanent lease in Krakow on 2026-06-02 and lives there now.", "expected": "changes", "samples": [ { "contradiction": false, "slot": "changes" }, { "contradiction": false, "slot": "changes" }, { "contradiction": false, "slot": "changes" } ] }, { "id": "ci", "class": "dated-update", "older": "The repository's CI runs on Travis CI.", "newer": "The repository's CI was switched to GitHub Actions on 2026-03-30.", "expected": "changes", "samples": [ { "contradiction": true, "slot": "changes" }, { "contradiction": true, "slot": "changes" }, { "contradiction": true, "slot": "changes" } ] }, { "id": "bikes", "class": "coexisting", "older": "The user owns a road bike.", "newer": "The user bought a gravel bike on 2026-07-12.", "expected": "not_changes", "samples": [ { "contradiction": false, "slot": "different" }, { "contradiction": false, "slot": "different" }, { "contradiction": false, "slot": "different" } ] }, { "id": "volunteering", "class": "coexisting", "older": "The user volunteers at a food bank on Sundays.", "newer": "Since June 2026 the user has also coached a youth football team.", "expected": "not_changes", "samples": [ { "contradiction": false, "slot": "different" }, { "contradiction": false, "slot": "different" }, { "contradiction": false, "slot": "different" } ] }, { "id": "backend-lang", "class": "coexisting", "older": "The project's backend is written in Go.", "newer": "On 2026-04-02 the team added a Python service for report generation.", "expected": "not_changes", "samples": [ { "contradiction": false, "slot": "different" }, { "contradiction": false, "slot": "different" }, { "contradiction": false, "slot": "different" } ] }, { "id": "shows", "class": "separate-events", "older": "The user saw Hamlet at the National Theatre on 2026-02-14.", "newer": "The user went to a jazz concert at the Barbican on 2026-05-30.", "expected": "not_changes", "samples": [ { "contradiction": false, "slot": "different" }, { "contradiction": false, "slot": "different" }, { "contradiction": false, "slot": "different" } ] }, { "id": "donations", "class": "separate-events", "older": "The user donated 50 euros to a hospice in January 2026.", "newer": "The user donated 30 euros to a wildlife charity on 2026-04-02.", "expected": "not_changes", "samples": [ { "contradiction": false, "slot": "different" }, { "contradiction": false, "slot": "different" }, { "contradiction": false, "slot": "different" } ] }, { "id": "floor", "class": "added-detail", "older": "The user rents an apartment in Tartu.", "newer": "The user's apartment in Tartu is on the fourth floor (noted 2026-03-05).", "expected": "not_changes", "samples": [ { "contradiction": false, "slot": "different" }, { "contradiction": false, "slot": "different" }, { "contradiction": false, "slot": "different" } ] }, { "id": "used-to", "class": "past-value", "older": "The user used to work night shifts at a bakery.", "newer": "The user works as a data analyst at Kestrel Insurance, per a note from 2026-08-03.", "expected": "not_changes", "samples": [ { "contradiction": false, "slot": "different" }, { "contradiction": false, "slot": "different" }, { "contradiction": false, "slot": "different" } ] }, { "id": "planning-house", "class": "decided-not-done", "older": "The user rents an apartment in Tartu.", "newer": "On 2026-08-03 the user said they plan to buy a house in Tallinn next year.", "expected": "not_changes", "samples": [ { "contradiction": false, "slot": "different" }, { "contradiction": false, "slot": "different" }, { "contradiction": false, "slot": "different" } ] }, { "id": "riga-course", "class": "lasting-then-temporary", "older": "The user lives in Tartu.", "newer": "From 2026-08-01 the user is spending ten days in Riga for a training course.", "expected": "not_changes", "samples": [ { "contradiction": false, "slot": "different" }, { "contradiction": false, "slot": "different" }, { "contradiction": false, "slot": "different" } ] }, { "id": "town-hall-walk", "class": "relied-on", "older": "The user's apartment is a five-minute walk from Tartu's Town Hall Square.", "newer": "Since August 2026 the user has lived in Tallinn.", "expected": "not_changes", "samples": [ { "contradiction": true, "slot": "changes" }, { "contradiction": true, "slot": "changes" }, { "contradiction": true, "slot": "changes" } ] }, { "id": "battery", "class": "fixed", "older": "The Kia EV6 has a 77.4 kWh battery.", "newer": "The Kia EV6's battery is 84 kWh, per a review dated 2026-05-11.", "expected": "not_changes", "samples": [ { "contradiction": true, "slot": "fixed" }, { "contradiction": true, "slot": "fixed" }, { "contradiction": true, "slot": "fixed" } ] } ], "bars": { "proposed": { "positives_retired": "8/8", "positive_misses": [], "false_retirements": "3/33", "false_rows": [ "town-hall-walk" ], "ok": false }, "shipped": { "positives_retired": "5/8", "positive_misses": [ "tartu-tallinn", "laptop", "krakow-let" ], "false_retirements": "3/33", "false_rows": [ "town-hall-walk" ], "ok": false }, "case": {} } }, "past": { "rows": [ { "id": "worked-until", "class": "past-ended", "older": "The user worked at Lindqvist & Berg until March 2025.", "newer": "The user works at Ostrava Robotics, per a note from 2026-06-02.", "expected": "not_changes", "samples": [ { "contradiction": false, "slot": "changes" }, { "contradiction": false, "slot": "different" }, { "contradiction": false, "slot": "different" } ] }, { "id": "before-2024", "class": "past-ended", "older": "Before 2024 the user's team used Jira.", "newer": "The team tracks its work in Linear (noted 2026-04-18).", "expected": "not_changes", "samples": [ { "contradiction": false, "slot": "different" }, { "contradiction": false, "slot": "different" }, { "contradiction": false, "slot": "different" } ] }, { "id": "restated-pool", "class": "past-ended", "older": "As of 2026-02-10, when the user lived in Bremen, the user swam at the Sudbad pool.", "newer": "Since May 2026 the user has lived in Hanover.", "expected": "not_changes", "samples": [ { "contradiction": false, "slot": "different" }, { "contradiction": false, "slot": "different" }, { "contradiction": false, "slot": "different" } ] }, { "id": "back-in-2023", "class": "past-ended", "older": "Back in 2023 the user drove a Volvo V60.", "newer": "The user drives a Tesla Model 3 (noted 2026-07-01).", "expected": "not_changes", "samples": [ { "contradiction": false, "slot": "different" }, { "contradiction": false, "slot": "different" }, { "contradiction": false, "slot": "different" } ] }, { "id": "had-been", "class": "past-ended", "older": "The user had been vegetarian for ten years until 2025.", "newer": "The user eats fish, per a note from 2026-03-09.", "expected": "not_changes", "samples": [ { "contradiction": false, "slot": "different" }, { "contradiction": true, "slot": "different" }, { "contradiction": false, "slot": "different" } ] }, { "id": "restated-backups", "class": "past-ended", "older": "As of 2026-04-01, when the project used MySQL, backups ran nightly with mysqldump.", "newer": "On 2026-06-10 the project moved its database to PostgreSQL.", "expected": "not_changes", "samples": [ { "contradiction": false, "slot": "different" }, { "contradiction": false, "slot": "different" }, { "contradiction": false, "slot": "different" } ] }, { "id": "restated-rent", "class": "past-ended", "older": "As of 2026-01-20, when the user rented in Bremen, the user's rent was 780 euros a month.", "newer": "The user's rent is 910 euros a month (noted 2026-08-14).", "expected": "not_changes", "samples": [ { "contradiction": false, "slot": "changes" }, { "contradiction": false, "slot": "changes" }, { "contradiction": true, "slot": "changes" } ] }, { "id": "lives-dated", "class": "present-dated", "older": "The user lives in Bremen, as of 2026-02-10.", "newer": "Since May 2026 the user has lived in Hanover.", "expected": "changes", "samples": [ { "contradiction": false, "slot": "changes" }, { "contradiction": false, "slot": "changes" }, { "contradiction": false, "slot": "changes" } ] }, { "id": "works-noted", "class": "present-dated", "older": "The user is working at Lindqvist & Berg (noted 2026-01-05).", "newer": "The user joined Ostrava Robotics on 2026-05-20.", "expected": "changes", "samples": [ { "contradiction": false, "slot": "changes" }, { "contradiction": false, "slot": "changes" }, { "contradiction": true, "slot": "changes" } ] }, { "id": "course-to-home", "class": "present-dated", "older": "The user is in Lyon for a three-week course, as of 2026-03-02.", "newer": "The user signed a lease in Lyon on 2026-04-01 and lives there now.", "expected": "changes", "samples": [ { "contradiction": false, "slot": "changes" }, { "contradiction": false, "slot": "changes" }, { "contradiction": false, "slot": "changes" } ] }, { "id": "tracker-noted", "class": "present-dated", "older": "The team tracks its work in Jira, per a note from 2026-01-12.", "newer": "The team switched from Jira to Linear on 2026-04-18.", "expected": "changes", "samples": [ { "contradiction": false, "slot": "changes" }, { "contradiction": false, "slot": "changes" }, { "contradiction": false, "slot": "changes" } ] }, { "id": "phone-plan", "class": "present-dated", "older": "The user's phone plan is with Vodafone, as of 2026-02-01.", "newer": "The user moved their phone plan to Lebara on 2026-06-03.", "expected": "changes", "samples": [ { "contradiction": false, "slot": "changes" }, { "contradiction": false, "slot": "changes" }, { "contradiction": false, "slot": "changes" } ] }, { "id": "manager-history", "class": "past-then-past", "older": "The user's manager was previously Henrik.", "newer": "The user's manager was previously Sofia.", "expected": "not_changes", "samples": [ { "contradiction": true, "slot": "different" }, { "contradiction": true, "slot": "different" }, { "contradiction": true, "slot": "different" } ] }, { "id": "case-restated-flight", "class": "case", "older": "As of 2026-10-08, when the user was in Mumbai for a week for work, the user was due to fly back home to Delhi on Thursday 2026-10-15.", "newer": "The user moved to Mumbai, as stated in a session dated 2026-10-09.", "expected": "not_changes", "samples": [ { "contradiction": true, "slot": "changes" }, { "contradiction": true, "slot": "changes" }, { "contradiction": true, "slot": "changes" } ] }, { "id": "case-restated-hotel", "class": "case", "older": "As of 2026-10-08, when the user was in Mumbai for a week for work, the user was staying at a hotel in Bandra, Mumbai.", "newer": "The user moved to Mumbai, as stated in a session dated 2026-10-09.", "expected": "not_changes", "samples": [ { "contradiction": true, "slot": "changes" }, { "contradiction": true, "slot": "changes" }, { "contradiction": true, "slot": "changes" } ] } ], "bars": { "proposed": { "positives_retired": "5/5", "positive_misses": [], "false_retirements": "4/24", "false_rows": [ "worked-until", "restated-rent" ], "ok": false }, "shipped": { "positives_retired": "0/5", "positive_misses": [ "lives-dated", "works-noted", "course-to-home", "tracker-noted", "phone-plan" ], "false_retirements": "1/24", "false_rows": [ "restated-rent" ], "ok": false }, "case": { "case-restated-flight": { "expected": "not_changes", "proposed": 3, "shipped": 3, "of": 3 }, "case-restated-hotel": { "expected": "not_changes", "proposed": 3, "shipped": 3, "of": 3 } } } } }, "bounded": { "halves": "bounded", "past": { "rows": [ { "id": "worked-until", "class": "past-ended", "older": "The user worked at Lindqvist & Berg until March 2025.", "newer": "The user works at Ostrava Robotics, per a note from 2026-06-02.", "expected": "not_changes", "samples": [ { "contradiction": false, "slot": "changes" }, { "contradiction": false, "slot": "changes" }, { "contradiction": false, "slot": "changes" } ] }, { "id": "before-2024", "class": "past-ended", "older": "Before 2024 the user's team used Jira.", "newer": "The team tracks its work in Linear (noted 2026-04-18).", "expected": "not_changes", "samples": [ { "contradiction": false, "slot": "different" }, { "contradiction": false, "slot": "changes" }, { "contradiction": false, "slot": "changes" } ] }, { "id": "restated-pool", "class": "past-ended", "older": "As of 2026-02-10, when the user lived in Bremen, the user swam at the Sudbad pool.", "newer": "Since May 2026 the user has lived in Hanover.", "expected": "not_changes", "samples": [ { "contradiction": false, "slot": "different" }, { "contradiction": false, "slot": "different" }, { "contradiction": false, "slot": "different" } ] }, { "id": "back-in-2023", "class": "past-ended", "older": "Back in 2023 the user drove a Volvo V60.", "newer": "The user drives a Tesla Model 3 (noted 2026-07-01).", "expected": "not_changes", "samples": [ { "contradiction": false, "slot": "different" }, { "contradiction": false, "slot": "changes" }, { "contradiction": false, "slot": "changes" } ] }, { "id": "had-been", "class": "past-ended", "older": "The user had been vegetarian for ten years until 2025.", "newer": "The user eats fish, per a note from 2026-03-09.", "expected": "not_changes", "samples": [ { "contradiction": false, "slot": "changes" }, { "contradiction": false, "slot": "different" }, { "contradiction": false, "slot": "different" } ] }, { "id": "restated-backups", "class": "past-ended", "older": "As of 2026-04-01, when the project used MySQL, backups ran nightly with mysqldump.", "newer": "On 2026-06-10 the project moved its database to PostgreSQL.", "expected": "not_changes", "samples": [ { "contradiction": false, "slot": "different" }, { "contradiction": false, "slot": "different" }, { "contradiction": false, "slot": "different" } ] }, { "id": "restated-rent", "class": "past-ended", "older": "As of 2026-01-20, when the user rented in Bremen, the user's rent was 780 euros a month.", "newer": "The user's rent is 910 euros a month (noted 2026-08-14).", "expected": "not_changes", "samples": [ { "contradiction": true, "slot": "changes" }, { "contradiction": false, "slot": "changes" }, { "contradiction": false, "slot": "changes" } ] }, { "id": "lives-dated", "class": "present-dated", "older": "The user lives in Bremen, as of 2026-02-10.", "newer": "Since May 2026 the user has lived in Hanover.", "expected": "changes", "samples": [ { "contradiction": true, "slot": "changes" }, { "contradiction": false, "slot": "changes" }, { "contradiction": false, "slot": "changes" } ] }, { "id": "works-noted", "class": "present-dated", "older": "The user is working at Lindqvist & Berg (noted 2026-01-05).", "newer": "The user joined Ostrava Robotics on 2026-05-20.", "expected": "changes", "samples": [ { "contradiction": false, "slot": "changes" }, { "contradiction": false, "slot": "changes" }, { "contradiction": false, "slot": "changes" } ] }, { "id": "course-to-home", "class": "present-dated", "older": "The user is in Lyon for a three-week course, as of 2026-03-02.", "newer": "The user signed a lease in Lyon on 2026-04-01 and lives there now.", "expected": "changes", "samples": [ { "contradiction": false, "slot": "changes" }, { "contradiction": false, "slot": "changes" }, { "contradiction": false, "slot": "changes" } ] }, { "id": "tracker-noted", "class": "present-dated", "older": "The team tracks its work in Jira, per a note from 2026-01-12.", "newer": "The team switched from Jira to Linear on 2026-04-18.", "expected": "changes", "samples": [ { "contradiction": false, "slot": "changes" }, { "contradiction": false, "slot": "changes" }, { "contradiction": false, "slot": "changes" } ] }, { "id": "phone-plan", "class": "present-dated", "older": "The user's phone plan is with Vodafone, as of 2026-02-01.", "newer": "The user moved their phone plan to Lebara on 2026-06-03.", "expected": "changes", "samples": [ { "contradiction": false, "slot": "changes" }, { "contradiction": false, "slot": "changes" }, { "contradiction": false, "slot": "changes" } ] }, { "id": "manager-history", "class": "past-then-past", "older": "The user's manager was previously Henrik.", "newer": "The user's manager was previously Sofia.", "expected": "not_changes", "samples": [ { "contradiction": true, "slot": "different" }, { "contradiction": true, "slot": "different" }, { "contradiction": true, "slot": "different" } ] }, { "id": "case-restated-flight", "class": "case", "older": "As of 2026-10-08, when the user was in Mumbai for a week for work, the user was due to fly back home to Delhi on Thursday 2026-10-15.", "newer": "The user moved to Mumbai, as stated in a session dated 2026-10-09.", "expected": "not_changes", "samples": [ { "contradiction": true, "slot": "changes" }, { "contradiction": true, "slot": "changes" }, { "contradiction": true, "slot": "changes" } ] }, { "id": "case-restated-hotel", "class": "case", "older": "As of 2026-10-08, when the user was in Mumbai for a week for work, the user was staying at a hotel in Bandra, Mumbai.", "newer": "The user moved to Mumbai, as stated in a session dated 2026-10-09.", "expected": "not_changes", "samples": [ { "contradiction": true, "slot": "changes" }, { "contradiction": true, "slot": "changes" }, { "contradiction": true, "slot": "changes" } ] } ], "bars": { "proposed": { "positives_retired": "5/5", "positive_misses": [], "false_retirements": "11/24", "false_rows": [ "worked-until", "before-2024", "back-in-2023", "had-been", "restated-rent" ], "ok": false }, "shipped": { "positives_retired": "0/5", "positive_misses": [ "lives-dated", "works-noted", "course-to-home", "tracker-noted", "phone-plan" ], "false_retirements": "1/24", "false_rows": [ "restated-rent" ], "ok": false }, "case": { "case-restated-flight": { "expected": "not_changes", "proposed": 3, "shipped": 3, "of": 3 }, "case-restated-hotel": { "expected": "not_changes", "proposed": 3, "shipped": 3, "of": 3 } } } } } } } }