{ "benchmark": "longmemeval", "selection": { "dataset_revision": "98d7416c24c778c2fee6e6f3006e7a073259d48f", "variant": "s", "sample_seed": 13, "question_limit": 150, "question_types": [], "questions_selected": 150, "questions_total": 500, "top_k": 10, "answer_model_id": "anthropic:claude-sonnet-5", "judge_model_id": "anthropic:claude-sonnet-5", "extraction_model_id": null, "embedding_model_id": "all-MiniLM-L6-v2", "context_budget_tokens": 500, "abstraction": false, "thresholds": { "extraction.similarity_threshold": 0.8, "confidence.uncalibrated_cap.enabled": 0.0, "confidence.uncalibrated_cap.cap_value": 0.7 }, "memory": "chunks" }, "retrieval_stage": { "questions": 143, "abstention_questions": 7, "mean_recall_at_k": 0.9122377622377622, "mean_precision_at_k": 0.6517482517482518, "recall_by_type": { "knowledge-update": 0.9761904761904762, "multi-session": 0.913157894736842, "single-session-assistant": 1.0, "single-session-preference": 0.8888888888888888, "single-session-user": 0.95, "temporal-reasoning": 0.8223684210526315 }, "per_question": [ { "question_id": "01493427", "question_type": "knowledge-update", "evidence_sessions": 2, "evidence_sessions_hit": 2, "particles_retrieved": 10, "recall_at_k": 1.0, "precision_at_k": 1.0, "abstention": false }, { "question_id": "08e075c7", "question_type": "knowledge-update", "evidence_sessions": 2, "evidence_sessions_hit": 2, "particles_retrieved": 10, "recall_at_k": 1.0, "precision_at_k": 0.2, "abstention": false }, { "question_id": "0977f2af", "question_type": "knowledge-update", "evidence_sessions": 2, "evidence_sessions_hit": 2, "particles_retrieved": 10, "recall_at_k": 1.0, "precision_at_k": 0.3, "abstention": false }, { "question_id": "10e09553", "question_type": "knowledge-update", "evidence_sessions": 2, "evidence_sessions_hit": 2, "particles_retrieved": 10, "recall_at_k": 1.0, "precision_at_k": 1.0, "abstention": false }, { "question_id": "26bdc477", "question_type": "knowledge-update", "evidence_sessions": 2, "evidence_sessions_hit": 2, "particles_retrieved": 10, "recall_at_k": 1.0, "precision_at_k": 0.8, "abstention": false }, { "question_id": "41698283", "question_type": "knowledge-update", "evidence_sessions": 2, "evidence_sessions_hit": 2, "particles_retrieved": 10, "recall_at_k": 1.0, "precision_at_k": 0.8, "abstention": false }, { "question_id": "603deb26", "question_type": "knowledge-update", "evidence_sessions": 2, "evidence_sessions_hit": 2, "particles_retrieved": 10, "recall_at_k": 1.0, "precision_at_k": 0.9, "abstention": false }, { "question_id": "6071bd76", "question_type": "knowledge-update", "evidence_sessions": 2, "evidence_sessions_hit": 2, "particles_retrieved": 10, "recall_at_k": 1.0, "precision_at_k": 1.0, "abstention": false }, { "question_id": "6a1eabeb", "question_type": "knowledge-update", "evidence_sessions": 2, "evidence_sessions_hit": 1, "particles_retrieved": 10, "recall_at_k": 0.5, "precision_at_k": 0.6, "abstention": false }, { "question_id": "6a27ffc2", "question_type": "knowledge-update", "evidence_sessions": 2, "evidence_sessions_hit": 2, "particles_retrieved": 10, "recall_at_k": 1.0, "precision_at_k": 0.8, "abstention": false }, { "question_id": "6aeb4375_abs", "question_type": "knowledge-update", "evidence_sessions": 0, "evidence_sessions_hit": 0, "particles_retrieved": 10, "recall_at_k": null, "precision_at_k": null, "abstention": true }, { "question_id": "7401057b", "question_type": "knowledge-update", "evidence_sessions": 2, "evidence_sessions_hit": 2, "particles_retrieved": 10, "recall_at_k": 1.0, "precision_at_k": 1.0, "abstention": false }, { "question_id": "7a87bd0c", "question_type": "knowledge-update", "evidence_sessions": 2, "evidence_sessions_hit": 2, "particles_retrieved": 10, "recall_at_k": 1.0, "precision_at_k": 0.9, "abstention": false }, { "question_id": "852ce960", "question_type": "knowledge-update", "evidence_sessions": 2, "evidence_sessions_hit": 2, "particles_retrieved": 10, "recall_at_k": 1.0, "precision_at_k": 1.0, "abstention": false }, { "question_id": "89941a94", "question_type": "knowledge-update", "evidence_sessions": 2, "evidence_sessions_hit": 2, "particles_retrieved": 10, "recall_at_k": 1.0, "precision_at_k": 1.0, "abstention": false }, { "question_id": "8fb83627", "question_type": "knowledge-update", "evidence_sessions": 2, "evidence_sessions_hit": 2, "particles_retrieved": 10, "recall_at_k": 1.0, "precision_at_k": 1.0, "abstention": false }, { "question_id": "9ea5eabc", "question_type": "knowledge-update", "evidence_sessions": 2, "evidence_sessions_hit": 2, "particles_retrieved": 10, "recall_at_k": 1.0, "precision_at_k": 0.9, "abstention": false }, { "question_id": "b01defab", "question_type": "knowledge-update", "evidence_sessions": 2, "evidence_sessions_hit": 2, "particles_retrieved": 10, "recall_at_k": 1.0, "precision_at_k": 1.0, "abstention": false }, { "question_id": "c4ea545c", "question_type": "knowledge-update", "evidence_sessions": 2, "evidence_sessions_hit": 2, "particles_retrieved": 10, "recall_at_k": 1.0, "precision_at_k": 0.5, "abstention": false }, { "question_id": "d7c942c3", "question_type": "knowledge-update", "evidence_sessions": 2, "evidence_sessions_hit": 2, "particles_retrieved": 10, "recall_at_k": 1.0, "precision_at_k": 0.2, "abstention": false }, { "question_id": "dfde3500", "question_type": "knowledge-update", "evidence_sessions": 2, "evidence_sessions_hit": 2, "particles_retrieved": 10, "recall_at_k": 1.0, "precision_at_k": 1.0, "abstention": false }, { "question_id": "f685340e_abs", "question_type": "knowledge-update", "evidence_sessions": 0, "evidence_sessions_hit": 0, "particles_retrieved": 10, "recall_at_k": null, "precision_at_k": null, "abstention": true }, { "question_id": "f9e8c073", "question_type": "knowledge-update", "evidence_sessions": 2, "evidence_sessions_hit": 2, "particles_retrieved": 10, "recall_at_k": 1.0, "precision_at_k": 1.0, "abstention": false }, { "question_id": "0100672e", "question_type": "multi-session", "evidence_sessions": 2, "evidence_sessions_hit": 2, "particles_retrieved": 10, "recall_at_k": 1.0, "precision_at_k": 1.0, "abstention": false }, { "question_id": "28dc39ac", "question_type": "multi-session", "evidence_sessions": 5, "evidence_sessions_hit": 5, "particles_retrieved": 10, "recall_at_k": 1.0, "precision_at_k": 1.0, "abstention": false }, { "question_id": "2e6d26dc", "question_type": "multi-session", "evidence_sessions": 4, "evidence_sessions_hit": 4, "particles_retrieved": 10, "recall_at_k": 1.0, "precision_at_k": 0.9, "abstention": false }, { "question_id": "37f165cf", "question_type": "multi-session", "evidence_sessions": 2, "evidence_sessions_hit": 2, "particles_retrieved": 10, "recall_at_k": 1.0, "precision_at_k": 0.7, "abstention": false }, { "question_id": "3fe836c9", "question_type": "multi-session", "evidence_sessions": 2, "evidence_sessions_hit": 2, "particles_retrieved": 10, "recall_at_k": 1.0, "precision_at_k": 1.0, "abstention": false }, { "question_id": "4adc0475", "question_type": "multi-session", "evidence_sessions": 2, "evidence_sessions_hit": 2, "particles_retrieved": 10, "recall_at_k": 1.0, "precision_at_k": 0.7, "abstention": false }, { "question_id": "55241a1f", "question_type": "multi-session", "evidence_sessions": 2, "evidence_sessions_hit": 2, "particles_retrieved": 10, "recall_at_k": 1.0, "precision_at_k": 1.0, "abstention": false }, { "question_id": "60036106", "question_type": "multi-session", "evidence_sessions": 2, "evidence_sessions_hit": 2, "particles_retrieved": 10, "recall_at_k": 1.0, "precision_at_k": 1.0, "abstention": false }, { "question_id": "60bf93ed", "question_type": "multi-session", "evidence_sessions": 2, "evidence_sessions_hit": 2, "particles_retrieved": 10, "recall_at_k": 1.0, "precision_at_k": 0.8, "abstention": false }, { "question_id": "61f8c8f8", "question_type": "multi-session", "evidence_sessions": 2, "evidence_sessions_hit": 1, "particles_retrieved": 10, "recall_at_k": 0.5, "precision_at_k": 1.0, "abstention": false }, { "question_id": "6c49646a", "question_type": "multi-session", "evidence_sessions": 2, "evidence_sessions_hit": 2, "particles_retrieved": 10, "recall_at_k": 1.0, "precision_at_k": 0.7, "abstention": false }, { "question_id": "77eafa52", "question_type": "multi-session", "evidence_sessions": 2, "evidence_sessions_hit": 2, "particles_retrieved": 10, "recall_at_k": 1.0, "precision_at_k": 1.0, "abstention": false }, { "question_id": "80ec1f4f_abs", "question_type": "multi-session", "evidence_sessions": 0, "evidence_sessions_hit": 0, "particles_retrieved": 10, "recall_at_k": null, "precision_at_k": null, "abstention": true }, { "question_id": "81507db6", "question_type": "multi-session", "evidence_sessions": 5, "evidence_sessions_hit": 2, "particles_retrieved": 10, "recall_at_k": 0.4, "precision_at_k": 0.8, "abstention": false }, { "question_id": "85fa3a3f", "question_type": "multi-session", "evidence_sessions": 2, "evidence_sessions_hit": 2, "particles_retrieved": 10, "recall_at_k": 1.0, "precision_at_k": 0.7, "abstention": false }, { "question_id": "87f22b4a", "question_type": "multi-session", "evidence_sessions": 2, "evidence_sessions_hit": 2, "particles_retrieved": 10, "recall_at_k": 1.0, "precision_at_k": 1.0, "abstention": false }, { "question_id": "88432d0a_abs", "question_type": "multi-session", "evidence_sessions": 0, "evidence_sessions_hit": 0, "particles_retrieved": 10, "recall_at_k": null, "precision_at_k": null, "abstention": true }, { "question_id": "91b15a6e", "question_type": "multi-session", "evidence_sessions": 2, "evidence_sessions_hit": 2, "particles_retrieved": 10, "recall_at_k": 1.0, "precision_at_k": 1.0, "abstention": false }, { "question_id": "92a0aa75", "question_type": "multi-session", "evidence_sessions": 2, "evidence_sessions_hit": 1, "particles_retrieved": 10, "recall_at_k": 0.5, "precision_at_k": 0.1, "abstention": false }, { "question_id": "9aaed6a3", "question_type": "multi-session", "evidence_sessions": 2, "evidence_sessions_hit": 1, "particles_retrieved": 10, "recall_at_k": 0.5, "precision_at_k": 0.2, "abstention": false }, { "question_id": "9ee3ecd6", "question_type": "multi-session", "evidence_sessions": 2, "evidence_sessions_hit": 2, "particles_retrieved": 10, "recall_at_k": 1.0, "precision_at_k": 0.5, "abstention": false }, { "question_id": "a11281a2", "question_type": "multi-session", "evidence_sessions": 2, "evidence_sessions_hit": 2, "particles_retrieved": 10, "recall_at_k": 1.0, "precision_at_k": 1.0, "abstention": false }, { "question_id": "a1cc6108", "question_type": "multi-session", "evidence_sessions": 2, "evidence_sessions_hit": 1, "particles_retrieved": 10, "recall_at_k": 0.5, "precision_at_k": 0.7, "abstention": false }, { "question_id": "a3332713", "question_type": "multi-session", "evidence_sessions": 2, "evidence_sessions_hit": 2, "particles_retrieved": 10, "recall_at_k": 1.0, "precision_at_k": 0.8, "abstention": false }, { "question_id": "a4996e51", "question_type": "multi-session", "evidence_sessions": 2, "evidence_sessions_hit": 2, "particles_retrieved": 10, "recall_at_k": 1.0, "precision_at_k": 1.0, "abstention": false }, { "question_id": "a9f6b44c", "question_type": "multi-session", "evidence_sessions": 3, "evidence_sessions_hit": 3, "particles_retrieved": 10, "recall_at_k": 1.0, "precision_at_k": 0.8, "abstention": false }, { "question_id": "bb7c3b45", "question_type": "multi-session", "evidence_sessions": 2, "evidence_sessions_hit": 2, "particles_retrieved": 10, "recall_at_k": 1.0, "precision_at_k": 0.6, "abstention": false }, { "question_id": "bc149d6b", "question_type": "multi-session", "evidence_sessions": 2, "evidence_sessions_hit": 1, "particles_retrieved": 10, "recall_at_k": 0.5, "precision_at_k": 0.6, "abstention": false }, { "question_id": "c18a7dc8", "question_type": "multi-session", "evidence_sessions": 2, "evidence_sessions_hit": 2, "particles_retrieved": 10, "recall_at_k": 1.0, "precision_at_k": 0.3, "abstention": false }, { "question_id": "d3ab962e", "question_type": "multi-session", "evidence_sessions": 2, "evidence_sessions_hit": 2, "particles_retrieved": 10, "recall_at_k": 1.0, "precision_at_k": 0.3, "abstention": false }, { "question_id": "d6062bb9", "question_type": "multi-session", "evidence_sessions": 2, "evidence_sessions_hit": 2, "particles_retrieved": 10, "recall_at_k": 1.0, "precision_at_k": 1.0, "abstention": false }, { "question_id": "d905b33f", "question_type": "multi-session", "evidence_sessions": 2, "evidence_sessions_hit": 2, "particles_retrieved": 10, "recall_at_k": 1.0, "precision_at_k": 0.3, "abstention": false }, { "question_id": "e6041065", "question_type": "multi-session", "evidence_sessions": 2, "evidence_sessions_hit": 2, "particles_retrieved": 10, "recall_at_k": 1.0, "precision_at_k": 1.0, "abstention": false }, { "question_id": "edced276", "question_type": "multi-session", "evidence_sessions": 2, "evidence_sessions_hit": 2, "particles_retrieved": 10, "recall_at_k": 1.0, "precision_at_k": 0.8, "abstention": false }, { "question_id": "eeda8a6d", "question_type": "multi-session", "evidence_sessions": 2, "evidence_sessions_hit": 2, "particles_retrieved": 10, "recall_at_k": 1.0, "precision_at_k": 1.0, "abstention": false }, { "question_id": "ef66a6e5", "question_type": "multi-session", "evidence_sessions": 2, "evidence_sessions_hit": 2, "particles_retrieved": 10, "recall_at_k": 1.0, "precision_at_k": 1.0, "abstention": false }, { "question_id": "gpt4_2f91af09", "question_type": "multi-session", "evidence_sessions": 3, "evidence_sessions_hit": 3, "particles_retrieved": 10, "recall_at_k": 1.0, "precision_at_k": 1.0, "abstention": false }, { "question_id": "gpt4_7fce9456", "question_type": "multi-session", "evidence_sessions": 5, "evidence_sessions_hit": 4, "particles_retrieved": 10, "recall_at_k": 0.8, "precision_at_k": 1.0, "abstention": false }, { "question_id": "gpt4_d12ceb0e", "question_type": "multi-session", "evidence_sessions": 3, "evidence_sessions_hit": 3, "particles_retrieved": 10, "recall_at_k": 1.0, "precision_at_k": 0.5, "abstention": false }, { "question_id": "gpt4_e05b82a6", "question_type": "multi-session", "evidence_sessions": 4, "evidence_sessions_hit": 4, "particles_retrieved": 10, "recall_at_k": 1.0, "precision_at_k": 1.0, "abstention": false }, { "question_id": "1d4da289", "question_type": "single-session-assistant", "evidence_sessions": 1, "evidence_sessions_hit": 1, "particles_retrieved": 10, "recall_at_k": 1.0, "precision_at_k": 0.4, "abstention": false }, { "question_id": "2bf43736", "question_type": "single-session-assistant", "evidence_sessions": 1, "evidence_sessions_hit": 1, "particles_retrieved": 10, "recall_at_k": 1.0, "precision_at_k": 0.9, "abstention": false }, { "question_id": "3249768e", "question_type": "single-session-assistant", "evidence_sessions": 1, "evidence_sessions_hit": 1, "particles_retrieved": 10, "recall_at_k": 1.0, "precision_at_k": 0.2, "abstention": false }, { "question_id": "352ab8bd", "question_type": "single-session-assistant", "evidence_sessions": 1, "evidence_sessions_hit": 1, "particles_retrieved": 10, "recall_at_k": 1.0, "precision_at_k": 0.8, "abstention": false }, { "question_id": "3e321797", "question_type": "single-session-assistant", "evidence_sessions": 1, "evidence_sessions_hit": 1, "particles_retrieved": 10, "recall_at_k": 1.0, "precision_at_k": 0.2, "abstention": false }, { "question_id": "488d3006", "question_type": "single-session-assistant", "evidence_sessions": 1, "evidence_sessions_hit": 1, "particles_retrieved": 10, "recall_at_k": 1.0, "precision_at_k": 0.6, "abstention": false }, { "question_id": "58470ed2", "question_type": "single-session-assistant", "evidence_sessions": 1, "evidence_sessions_hit": 1, "particles_retrieved": 10, "recall_at_k": 1.0, "precision_at_k": 0.4, "abstention": false }, { "question_id": "70b3e69b", "question_type": "single-session-assistant", "evidence_sessions": 1, "evidence_sessions_hit": 1, "particles_retrieved": 10, "recall_at_k": 1.0, "precision_at_k": 1.0, "abstention": false }, { "question_id": "8cf51dda", "question_type": "single-session-assistant", "evidence_sessions": 1, "evidence_sessions_hit": 1, "particles_retrieved": 10, "recall_at_k": 1.0, "precision_at_k": 0.3, "abstention": false }, { "question_id": "a40e080f", "question_type": "single-session-assistant", "evidence_sessions": 1, "evidence_sessions_hit": 1, "particles_retrieved": 10, "recall_at_k": 1.0, "precision_at_k": 0.5, "abstention": false }, { "question_id": "cc539528", "question_type": "single-session-assistant", "evidence_sessions": 1, "evidence_sessions_hit": 1, "particles_retrieved": 10, "recall_at_k": 1.0, "precision_at_k": 0.5, "abstention": false }, { "question_id": "ceb54acb", "question_type": "single-session-assistant", "evidence_sessions": 1, "evidence_sessions_hit": 1, "particles_retrieved": 10, "recall_at_k": 1.0, "precision_at_k": 0.1, "abstention": false }, { "question_id": "e3fc4d6e", "question_type": "single-session-assistant", "evidence_sessions": 1, "evidence_sessions_hit": 1, "particles_retrieved": 10, "recall_at_k": 1.0, "precision_at_k": 0.4, "abstention": false }, { "question_id": "e8a79c70", "question_type": "single-session-assistant", "evidence_sessions": 1, "evidence_sessions_hit": 1, "particles_retrieved": 10, "recall_at_k": 1.0, "precision_at_k": 0.4, "abstention": false }, { "question_id": "eaca4986", "question_type": "single-session-assistant", "evidence_sessions": 1, "evidence_sessions_hit": 1, "particles_retrieved": 10, "recall_at_k": 1.0, "precision_at_k": 0.3, "abstention": false }, { "question_id": "fca762bc", "question_type": "single-session-assistant", "evidence_sessions": 1, "evidence_sessions_hit": 1, "particles_retrieved": 10, "recall_at_k": 1.0, "precision_at_k": 0.3, "abstention": false }, { "question_id": "fea54f57", "question_type": "single-session-assistant", "evidence_sessions": 1, "evidence_sessions_hit": 1, "particles_retrieved": 10, "recall_at_k": 1.0, "precision_at_k": 0.7, "abstention": false }, { "question_id": "09d032c9", "question_type": "single-session-preference", "evidence_sessions": 1, "evidence_sessions_hit": 1, "particles_retrieved": 10, "recall_at_k": 1.0, "precision_at_k": 0.4, "abstention": false }, { "question_id": "195a1a1b", "question_type": "single-session-preference", "evidence_sessions": 1, "evidence_sessions_hit": 1, "particles_retrieved": 10, "recall_at_k": 1.0, "precision_at_k": 0.6, "abstention": false }, { "question_id": "35a27287", "question_type": "single-session-preference", "evidence_sessions": 1, "evidence_sessions_hit": 1, "particles_retrieved": 10, "recall_at_k": 1.0, "precision_at_k": 0.4, "abstention": false }, { "question_id": "505af2f5", "question_type": "single-session-preference", "evidence_sessions": 1, "evidence_sessions_hit": 1, "particles_retrieved": 10, "recall_at_k": 1.0, "precision_at_k": 0.3, "abstention": false }, { "question_id": "6b7dfb22", "question_type": "single-session-preference", "evidence_sessions": 1, "evidence_sessions_hit": 1, "particles_retrieved": 10, "recall_at_k": 1.0, "precision_at_k": 1.0, "abstention": false }, { "question_id": "75832dbd", "question_type": "single-session-preference", "evidence_sessions": 1, "evidence_sessions_hit": 1, "particles_retrieved": 10, "recall_at_k": 1.0, "precision_at_k": 0.2, "abstention": false }, { "question_id": "afdc33df", "question_type": "single-session-preference", "evidence_sessions": 1, "evidence_sessions_hit": 1, "particles_retrieved": 10, "recall_at_k": 1.0, "precision_at_k": 0.9, "abstention": false }, { "question_id": "d6233ab6", "question_type": "single-session-preference", "evidence_sessions": 1, "evidence_sessions_hit": 0, "particles_retrieved": 10, "recall_at_k": 0.0, "precision_at_k": 0.0, "abstention": false }, { "question_id": "fca70973", "question_type": "single-session-preference", "evidence_sessions": 1, "evidence_sessions_hit": 1, "particles_retrieved": 10, "recall_at_k": 1.0, "precision_at_k": 0.6, "abstention": false }, { "question_id": "001be529", "question_type": "single-session-user", "evidence_sessions": 1, "evidence_sessions_hit": 1, "particles_retrieved": 10, "recall_at_k": 1.0, "precision_at_k": 0.3, "abstention": false }, { "question_id": "0862e8bf", "question_type": "single-session-user", "evidence_sessions": 1, "evidence_sessions_hit": 1, "particles_retrieved": 10, "recall_at_k": 1.0, "precision_at_k": 1.0, "abstention": false }, { "question_id": "0862e8bf_abs", "question_type": "single-session-user", "evidence_sessions": 0, "evidence_sessions_hit": 0, "particles_retrieved": 10, "recall_at_k": null, "precision_at_k": null, "abstention": true }, { "question_id": "15745da0", "question_type": "single-session-user", "evidence_sessions": 1, "evidence_sessions_hit": 1, "particles_retrieved": 10, "recall_at_k": 1.0, "precision_at_k": 0.6, "abstention": false }, { "question_id": "19b5f2b3", "question_type": "single-session-user", "evidence_sessions": 1, "evidence_sessions_hit": 1, "particles_retrieved": 10, "recall_at_k": 1.0, "precision_at_k": 0.8, "abstention": false }, { "question_id": "29f2956b", "question_type": "single-session-user", "evidence_sessions": 1, "evidence_sessions_hit": 1, "particles_retrieved": 10, "recall_at_k": 1.0, "precision_at_k": 0.3, "abstention": false }, { "question_id": "311778f1", "question_type": "single-session-user", "evidence_sessions": 1, "evidence_sessions_hit": 1, "particles_retrieved": 10, "recall_at_k": 1.0, "precision_at_k": 0.6, "abstention": false }, { "question_id": "4fd1909e", "question_type": "single-session-user", "evidence_sessions": 1, "evidence_sessions_hit": 1, "particles_retrieved": 10, "recall_at_k": 1.0, "precision_at_k": 0.6, "abstention": false }, { "question_id": "545bd2b5", "question_type": "single-session-user", "evidence_sessions": 1, "evidence_sessions_hit": 1, "particles_retrieved": 10, "recall_at_k": 1.0, "precision_at_k": 0.5, "abstention": false }, { "question_id": "5d3d2817", "question_type": "single-session-user", "evidence_sessions": 1, "evidence_sessions_hit": 0, "particles_retrieved": 10, "recall_at_k": 0.0, "precision_at_k": 0.0, "abstention": false }, { "question_id": "66f24dbb", "question_type": "single-session-user", "evidence_sessions": 1, "evidence_sessions_hit": 1, "particles_retrieved": 10, "recall_at_k": 1.0, "precision_at_k": 0.4, "abstention": false }, { "question_id": "6b168ec8", "question_type": "single-session-user", "evidence_sessions": 1, "evidence_sessions_hit": 1, "particles_retrieved": 10, "recall_at_k": 1.0, "precision_at_k": 1.0, "abstention": false }, { "question_id": "86f00804", "question_type": "single-session-user", "evidence_sessions": 1, "evidence_sessions_hit": 1, "particles_retrieved": 10, "recall_at_k": 1.0, "precision_at_k": 0.9, "abstention": false }, { "question_id": "8a137a7f", "question_type": "single-session-user", "evidence_sessions": 1, "evidence_sessions_hit": 1, "particles_retrieved": 10, "recall_at_k": 1.0, "precision_at_k": 1.0, "abstention": false }, { "question_id": "a06e4cfe", "question_type": "single-session-user", "evidence_sessions": 1, "evidence_sessions_hit": 1, "particles_retrieved": 10, "recall_at_k": 1.0, "precision_at_k": 1.0, "abstention": false }, { "question_id": "af8d2e46", "question_type": "single-session-user", "evidence_sessions": 1, "evidence_sessions_hit": 1, "particles_retrieved": 10, "recall_at_k": 1.0, "precision_at_k": 0.8, "abstention": false }, { "question_id": "c19f7a0b", "question_type": "single-session-user", "evidence_sessions": 1, "evidence_sessions_hit": 1, "particles_retrieved": 10, "recall_at_k": 1.0, "precision_at_k": 0.4, "abstention": false }, { "question_id": "c5e8278d", "question_type": "single-session-user", "evidence_sessions": 1, "evidence_sessions_hit": 1, "particles_retrieved": 10, "recall_at_k": 1.0, "precision_at_k": 1.0, "abstention": false }, { "question_id": "d52b4f67", "question_type": "single-session-user", "evidence_sessions": 1, "evidence_sessions_hit": 1, "particles_retrieved": 10, "recall_at_k": 1.0, "precision_at_k": 0.3, "abstention": false }, { "question_id": "e47becba", "question_type": "single-session-user", "evidence_sessions": 1, "evidence_sessions_hit": 1, "particles_retrieved": 10, "recall_at_k": 1.0, "precision_at_k": 0.1, "abstention": false }, { "question_id": "f4f1d8a4", "question_type": "single-session-user", "evidence_sessions": 1, "evidence_sessions_hit": 1, "particles_retrieved": 10, "recall_at_k": 1.0, "precision_at_k": 0.1, "abstention": false }, { "question_id": "0bc8ad92", "question_type": "temporal-reasoning", "evidence_sessions": 3, "evidence_sessions_hit": 2, "particles_retrieved": 10, "recall_at_k": 0.6666666666666666, "precision_at_k": 0.6, "abstention": false }, { "question_id": "2c63a862", "question_type": "temporal-reasoning", "evidence_sessions": 2, "evidence_sessions_hit": 2, "particles_retrieved": 10, "recall_at_k": 1.0, "precision_at_k": 1.0, "abstention": false }, { "question_id": "2ebe6c90", "question_type": "temporal-reasoning", "evidence_sessions": 2, "evidence_sessions_hit": 2, "particles_retrieved": 10, "recall_at_k": 1.0, "precision_at_k": 0.3, "abstention": false }, { "question_id": "6e984302", "question_type": "temporal-reasoning", "evidence_sessions": 2, "evidence_sessions_hit": 1, "particles_retrieved": 10, "recall_at_k": 0.5, "precision_at_k": 0.1, "abstention": false }, { "question_id": "8077ef71", "question_type": "temporal-reasoning", "evidence_sessions": 1, "evidence_sessions_hit": 1, "particles_retrieved": 10, "recall_at_k": 1.0, "precision_at_k": 0.1, "abstention": false }, { "question_id": "993da5e2", "question_type": "temporal-reasoning", "evidence_sessions": 2, "evidence_sessions_hit": 2, "particles_retrieved": 10, "recall_at_k": 1.0, "precision_at_k": 0.6, "abstention": false }, { "question_id": "a3838d2b", "question_type": "temporal-reasoning", "evidence_sessions": 6, "evidence_sessions_hit": 3, "particles_retrieved": 10, "recall_at_k": 0.5, "precision_at_k": 1.0, "abstention": false }, { "question_id": "b29f3365", "question_type": "temporal-reasoning", "evidence_sessions": 2, "evidence_sessions_hit": 2, "particles_retrieved": 10, "recall_at_k": 1.0, "precision_at_k": 1.0, "abstention": false }, { "question_id": "b46e15ee", "question_type": "temporal-reasoning", "evidence_sessions": 4, "evidence_sessions_hit": 3, "particles_retrieved": 10, "recall_at_k": 0.75, "precision_at_k": 0.7, "abstention": false }, { "question_id": "b9cfe692", "question_type": "temporal-reasoning", "evidence_sessions": 3, "evidence_sessions_hit": 3, "particles_retrieved": 10, "recall_at_k": 1.0, "precision_at_k": 1.0, "abstention": false }, { "question_id": "bbf86515", "question_type": "temporal-reasoning", "evidence_sessions": 2, "evidence_sessions_hit": 2, "particles_retrieved": 10, "recall_at_k": 1.0, "precision_at_k": 0.4, "abstention": false }, { "question_id": "c8090214", "question_type": "temporal-reasoning", "evidence_sessions": 2, "evidence_sessions_hit": 2, "particles_retrieved": 10, "recall_at_k": 1.0, "precision_at_k": 0.6, "abstention": false }, { "question_id": "dcfa8644", "question_type": "temporal-reasoning", "evidence_sessions": 2, "evidence_sessions_hit": 2, "particles_retrieved": 10, "recall_at_k": 1.0, "precision_at_k": 1.0, "abstention": false }, { "question_id": "gpt4_1a1dc16d", "question_type": "temporal-reasoning", "evidence_sessions": 2, "evidence_sessions_hit": 2, "particles_retrieved": 10, "recall_at_k": 1.0, "precision_at_k": 0.8, "abstention": false }, { "question_id": "gpt4_1e4a8aeb", "question_type": "temporal-reasoning", "evidence_sessions": 2, "evidence_sessions_hit": 2, "particles_retrieved": 10, "recall_at_k": 1.0, "precision_at_k": 0.9, "abstention": false }, { "question_id": "gpt4_2312f94c", "question_type": "temporal-reasoning", "evidence_sessions": 2, "evidence_sessions_hit": 2, "particles_retrieved": 10, "recall_at_k": 1.0, "precision_at_k": 0.8, "abstention": false }, { "question_id": "gpt4_2487a7cb", "question_type": "temporal-reasoning", "evidence_sessions": 2, "evidence_sessions_hit": 2, "particles_retrieved": 10, "recall_at_k": 1.0, "precision_at_k": 0.5, "abstention": false }, { "question_id": "gpt4_468eb063", "question_type": "temporal-reasoning", "evidence_sessions": 1, "evidence_sessions_hit": 0, "particles_retrieved": 10, "recall_at_k": 0.0, "precision_at_k": 0.0, "abstention": false }, { "question_id": "gpt4_4fc4f797", "question_type": "temporal-reasoning", "evidence_sessions": 2, "evidence_sessions_hit": 2, "particles_retrieved": 10, "recall_at_k": 1.0, "precision_at_k": 1.0, "abstention": false }, { "question_id": "gpt4_5438fa52", "question_type": "temporal-reasoning", "evidence_sessions": 2, "evidence_sessions_hit": 2, "particles_retrieved": 10, "recall_at_k": 1.0, "precision_at_k": 0.2, "abstention": false }, { "question_id": "gpt4_59149c78", "question_type": "temporal-reasoning", "evidence_sessions": 2, "evidence_sessions_hit": 0, "particles_retrieved": 10, "recall_at_k": 0.0, "precision_at_k": 0.0, "abstention": false }, { "question_id": "gpt4_68e94288", "question_type": "temporal-reasoning", "evidence_sessions": 2, "evidence_sessions_hit": 1, "particles_retrieved": 10, "recall_at_k": 0.5, "precision_at_k": 0.1, "abstention": false }, { "question_id": "gpt4_70e84552", "question_type": "temporal-reasoning", "evidence_sessions": 2, "evidence_sessions_hit": 2, "particles_retrieved": 10, "recall_at_k": 1.0, "precision_at_k": 1.0, "abstention": false }, { "question_id": "gpt4_70e84552_abs", "question_type": "temporal-reasoning", "evidence_sessions": 0, "evidence_sessions_hit": 0, "particles_retrieved": 10, "recall_at_k": null, "precision_at_k": null, "abstention": true }, { "question_id": "gpt4_76048e76", "question_type": "temporal-reasoning", "evidence_sessions": 2, "evidence_sessions_hit": 2, "particles_retrieved": 10, "recall_at_k": 1.0, "precision_at_k": 0.9, "abstention": false }, { "question_id": "gpt4_7ca326fa", "question_type": "temporal-reasoning", "evidence_sessions": 3, "evidence_sessions_hit": 2, "particles_retrieved": 10, "recall_at_k": 0.6666666666666666, "precision_at_k": 0.9, "abstention": false }, { "question_id": "gpt4_7ddcf75f", "question_type": "temporal-reasoning", "evidence_sessions": 1, "evidence_sessions_hit": 1, "particles_retrieved": 10, "recall_at_k": 1.0, "precision_at_k": 0.4, "abstention": false }, { "question_id": "gpt4_7f6b06db", "question_type": "temporal-reasoning", "evidence_sessions": 3, "evidence_sessions_hit": 1, "particles_retrieved": 10, "recall_at_k": 0.3333333333333333, "precision_at_k": 0.1, "abstention": false }, { "question_id": "gpt4_8e165409", "question_type": "temporal-reasoning", "evidence_sessions": 2, "evidence_sessions_hit": 2, "particles_retrieved": 10, "recall_at_k": 1.0, "precision_at_k": 0.8, "abstention": false }, { "question_id": "gpt4_93159ced", "question_type": "temporal-reasoning", "evidence_sessions": 2, "evidence_sessions_hit": 1, "particles_retrieved": 10, "recall_at_k": 0.5, "precision_at_k": 0.5, "abstention": false }, { "question_id": "gpt4_93159ced_abs", "question_type": "temporal-reasoning", "evidence_sessions": 0, "evidence_sessions_hit": 0, "particles_retrieved": 10, "recall_at_k": null, "precision_at_k": null, "abstention": true }, { "question_id": "gpt4_9a159967", "question_type": "temporal-reasoning", "evidence_sessions": 3, "evidence_sessions_hit": 3, "particles_retrieved": 10, "recall_at_k": 1.0, "precision_at_k": 0.7, "abstention": false }, { "question_id": "gpt4_af6db32f", "question_type": "temporal-reasoning", "evidence_sessions": 1, "evidence_sessions_hit": 1, "particles_retrieved": 10, "recall_at_k": 1.0, "precision_at_k": 0.8, "abstention": false }, { "question_id": "gpt4_b5700ca0", "question_type": "temporal-reasoning", "evidence_sessions": 1, "evidence_sessions_hit": 1, "particles_retrieved": 10, "recall_at_k": 1.0, "precision_at_k": 0.2, "abstention": false }, { "question_id": "gpt4_b5700ca9", "question_type": "temporal-reasoning", "evidence_sessions": 1, "evidence_sessions_hit": 1, "particles_retrieved": 10, "recall_at_k": 1.0, "precision_at_k": 0.2, "abstention": false }, { "question_id": "gpt4_cd90e484", "question_type": "temporal-reasoning", "evidence_sessions": 2, "evidence_sessions_hit": 2, "particles_retrieved": 10, "recall_at_k": 1.0, "precision_at_k": 1.0, "abstention": false }, { "question_id": "gpt4_d31cdae3", "question_type": "temporal-reasoning", "evidence_sessions": 2, "evidence_sessions_hit": 1, "particles_retrieved": 10, "recall_at_k": 0.5, "precision_at_k": 0.1, "abstention": false }, { "question_id": "gpt4_e061b84g", "question_type": "temporal-reasoning", "evidence_sessions": 3, "evidence_sessions_hit": 1, "particles_retrieved": 10, "recall_at_k": 0.3333333333333333, "precision_at_k": 0.1, "abstention": false }, { "question_id": "gpt4_e072b769", "question_type": "temporal-reasoning", "evidence_sessions": 1, "evidence_sessions_hit": 1, "particles_retrieved": 10, "recall_at_k": 1.0, "precision_at_k": 1.0, "abstention": false }, { "question_id": "gpt4_fa19884c", "question_type": "temporal-reasoning", "evidence_sessions": 2, "evidence_sessions_hit": 2, "particles_retrieved": 10, "recall_at_k": 1.0, "precision_at_k": 1.0, "abstention": false } ] }, "qa_particles": { "condition": "qa_chunks", "model_id": "anthropic:claude-sonnet-5", "questions": 150, "accuracy": 0.29333333333333333, "accuracy_by_type": { "knowledge-update": 0.391304347826087, "multi-session": 0.2, "single-session-assistant": 0.4117647058823529, "single-session-preference": 0.3333333333333333, "single-session-user": 0.42857142857142855, "temporal-reasoning": 0.2 }, "per_question": [ { "question_id": "01493427", "question_type": "knowledge-update", "correct": false, "abstention": false }, { "question_id": "08e075c7", "question_type": "knowledge-update", "correct": true, "abstention": false }, { "question_id": "0977f2af", "question_type": "knowledge-update", "correct": false, "abstention": false }, { "question_id": "10e09553", "question_type": "knowledge-update", "correct": false, "abstention": false }, { "question_id": "26bdc477", "question_type": "knowledge-update", "correct": true, "abstention": false }, { "question_id": "41698283", "question_type": "knowledge-update", "correct": false, "abstention": false }, { "question_id": "603deb26", "question_type": "knowledge-update", "correct": true, "abstention": false }, { "question_id": "6071bd76", "question_type": "knowledge-update", "correct": false, "abstention": false }, { "question_id": "6a1eabeb", "question_type": "knowledge-update", "correct": true, "abstention": false }, { "question_id": "6a27ffc2", "question_type": "knowledge-update", "correct": true, "abstention": false }, { "question_id": "6aeb4375_abs", "question_type": "knowledge-update", "correct": true, "abstention": true }, { "question_id": "7401057b", "question_type": "knowledge-update", "correct": false, "abstention": false }, { "question_id": "7a87bd0c", "question_type": "knowledge-update", "correct": false, "abstention": false }, { "question_id": "852ce960", "question_type": "knowledge-update", "correct": false, "abstention": false }, { "question_id": "89941a94", "question_type": "knowledge-update", "correct": true, "abstention": false }, { "question_id": "8fb83627", "question_type": "knowledge-update", "correct": false, "abstention": false }, { "question_id": "9ea5eabc", "question_type": "knowledge-update", "correct": false, "abstention": false }, { "question_id": "b01defab", "question_type": "knowledge-update", "correct": false, "abstention": false }, { "question_id": "c4ea545c", "question_type": "knowledge-update", "correct": false, "abstention": false }, { "question_id": "d7c942c3", "question_type": "knowledge-update", "correct": true, "abstention": false }, { "question_id": "dfde3500", "question_type": "knowledge-update", "correct": true, "abstention": false }, { "question_id": "f685340e_abs", "question_type": "knowledge-update", "correct": false, "abstention": true }, { "question_id": "f9e8c073", "question_type": "knowledge-update", "correct": false, "abstention": false }, { "question_id": "0100672e", "question_type": "multi-session", "correct": false, "abstention": false }, { "question_id": "28dc39ac", "question_type": "multi-session", "correct": false, "abstention": false }, { "question_id": "2e6d26dc", "question_type": "multi-session", "correct": true, "abstention": false }, { "question_id": "37f165cf", "question_type": "multi-session", "correct": false, "abstention": false }, { "question_id": "3fe836c9", "question_type": "multi-session", "correct": true, "abstention": false }, { "question_id": "4adc0475", "question_type": "multi-session", "correct": false, "abstention": false }, { "question_id": "55241a1f", "question_type": "multi-session", "correct": false, "abstention": false }, { "question_id": "60036106", "question_type": "multi-session", "correct": false, "abstention": false }, { "question_id": "60bf93ed", "question_type": "multi-session", "correct": true, "abstention": false }, { "question_id": "61f8c8f8", "question_type": "multi-session", "correct": false, "abstention": false }, { "question_id": "6c49646a", "question_type": "multi-session", "correct": false, "abstention": false }, { "question_id": "77eafa52", "question_type": "multi-session", "correct": false, "abstention": false }, { "question_id": "80ec1f4f_abs", "question_type": "multi-session", "correct": true, "abstention": true }, { "question_id": "81507db6", "question_type": "multi-session", "correct": false, "abstention": false }, { "question_id": "85fa3a3f", "question_type": "multi-session", "correct": false, "abstention": false }, { "question_id": "87f22b4a", "question_type": "multi-session", "correct": false, "abstention": false }, { "question_id": "88432d0a_abs", "question_type": "multi-session", "correct": true, "abstention": true }, { "question_id": "91b15a6e", "question_type": "multi-session", "correct": true, "abstention": false }, { "question_id": "92a0aa75", "question_type": "multi-session", "correct": false, "abstention": false }, { "question_id": "9aaed6a3", "question_type": "multi-session", "correct": false, "abstention": false }, { "question_id": "9ee3ecd6", "question_type": "multi-session", "correct": false, "abstention": false }, { "question_id": "a11281a2", "question_type": "multi-session", "correct": false, "abstention": false }, { "question_id": "a1cc6108", "question_type": "multi-session", "correct": false, "abstention": false }, { "question_id": "a3332713", "question_type": "multi-session", "correct": true, "abstention": false }, { "question_id": "a4996e51", "question_type": "multi-session", "correct": true, "abstention": false }, { "question_id": "a9f6b44c", "question_type": "multi-session", "correct": false, "abstention": false }, { "question_id": "bb7c3b45", "question_type": "multi-session", "correct": false, "abstention": false }, { "question_id": "bc149d6b", "question_type": "multi-session", "correct": false, "abstention": false }, { "question_id": "c18a7dc8", "question_type": "multi-session", "correct": false, "abstention": false }, { "question_id": "d3ab962e", "question_type": "multi-session", "correct": false, "abstention": false }, { "question_id": "d6062bb9", "question_type": "multi-session", "correct": false, "abstention": false }, { "question_id": "d905b33f", "question_type": "multi-session", "correct": false, "abstention": false }, { "question_id": "e6041065", "question_type": "multi-session", "correct": false, "abstention": false }, { "question_id": "edced276", "question_type": "multi-session", "correct": false, "abstention": false }, { "question_id": "eeda8a6d", "question_type": "multi-session", "correct": false, "abstention": false }, { "question_id": "ef66a6e5", "question_type": "multi-session", "correct": false, "abstention": false }, { "question_id": "gpt4_2f91af09", "question_type": "multi-session", "correct": false, "abstention": false }, { "question_id": "gpt4_7fce9456", "question_type": "multi-session", "correct": false, "abstention": false }, { "question_id": "gpt4_d12ceb0e", "question_type": "multi-session", "correct": false, "abstention": false }, { "question_id": "gpt4_e05b82a6", "question_type": "multi-session", "correct": false, "abstention": false }, { "question_id": "1d4da289", "question_type": "single-session-assistant", "correct": true, "abstention": false }, { "question_id": "2bf43736", "question_type": "single-session-assistant", "correct": true, "abstention": false }, { "question_id": "3249768e", "question_type": "single-session-assistant", "correct": false, "abstention": false }, { "question_id": "352ab8bd", "question_type": "single-session-assistant", "correct": false, "abstention": false }, { "question_id": "3e321797", "question_type": "single-session-assistant", "correct": true, "abstention": false }, { "question_id": "488d3006", "question_type": "single-session-assistant", "correct": false, "abstention": false }, { "question_id": "58470ed2", "question_type": "single-session-assistant", "correct": false, "abstention": false }, { "question_id": "70b3e69b", "question_type": "single-session-assistant", "correct": false, "abstention": false }, { "question_id": "8cf51dda", "question_type": "single-session-assistant", "correct": false, "abstention": false }, { "question_id": "a40e080f", "question_type": "single-session-assistant", "correct": false, "abstention": false }, { "question_id": "cc539528", "question_type": "single-session-assistant", "correct": false, "abstention": false }, { "question_id": "ceb54acb", "question_type": "single-session-assistant", "correct": true, "abstention": false }, { "question_id": "e3fc4d6e", "question_type": "single-session-assistant", "correct": true, "abstention": false }, { "question_id": "e8a79c70", "question_type": "single-session-assistant", "correct": false, "abstention": false }, { "question_id": "eaca4986", "question_type": "single-session-assistant", "correct": false, "abstention": false }, { "question_id": "fca762bc", "question_type": "single-session-assistant", "correct": true, "abstention": false }, { "question_id": "fea54f57", "question_type": "single-session-assistant", "correct": true, "abstention": false }, { "question_id": "09d032c9", "question_type": "single-session-preference", "correct": true, "abstention": false }, { "question_id": "195a1a1b", "question_type": "single-session-preference", "correct": false, "abstention": false }, { "question_id": "35a27287", "question_type": "single-session-preference", "correct": false, "abstention": false }, { "question_id": "505af2f5", "question_type": "single-session-preference", "correct": true, "abstention": false }, { "question_id": "6b7dfb22", "question_type": "single-session-preference", "correct": false, "abstention": false }, { "question_id": "75832dbd", "question_type": "single-session-preference", "correct": false, "abstention": false }, { "question_id": "afdc33df", "question_type": "single-session-preference", "correct": false, "abstention": false }, { "question_id": "d6233ab6", "question_type": "single-session-preference", "correct": false, "abstention": false }, { "question_id": "fca70973", "question_type": "single-session-preference", "correct": true, "abstention": false }, { "question_id": "001be529", "question_type": "single-session-user", "correct": true, "abstention": false }, { "question_id": "0862e8bf", "question_type": "single-session-user", "correct": false, "abstention": false }, { "question_id": "0862e8bf_abs", "question_type": "single-session-user", "correct": true, "abstention": true }, { "question_id": "15745da0", "question_type": "single-session-user", "correct": false, "abstention": false }, { "question_id": "19b5f2b3", "question_type": "single-session-user", "correct": true, "abstention": false }, { "question_id": "29f2956b", "question_type": "single-session-user", "correct": true, "abstention": false }, { "question_id": "311778f1", "question_type": "single-session-user", "correct": true, "abstention": false }, { "question_id": "4fd1909e", "question_type": "single-session-user", "correct": true, "abstention": false }, { "question_id": "545bd2b5", "question_type": "single-session-user", "correct": true, "abstention": false }, { "question_id": "5d3d2817", "question_type": "single-session-user", "correct": false, "abstention": false }, { "question_id": "66f24dbb", "question_type": "single-session-user", "correct": false, "abstention": false }, { "question_id": "6b168ec8", "question_type": "single-session-user", "correct": false, "abstention": false }, { "question_id": "86f00804", "question_type": "single-session-user", "correct": false, "abstention": false }, { "question_id": "8a137a7f", "question_type": "single-session-user", "correct": false, "abstention": false }, { "question_id": "a06e4cfe", "question_type": "single-session-user", "correct": true, "abstention": false }, { "question_id": "af8d2e46", "question_type": "single-session-user", "correct": true, "abstention": false }, { "question_id": "c19f7a0b", "question_type": "single-session-user", "correct": false, "abstention": false }, { "question_id": "c5e8278d", "question_type": "single-session-user", "correct": false, "abstention": false }, { "question_id": "d52b4f67", "question_type": "single-session-user", "correct": false, "abstention": false }, { "question_id": "e47becba", "question_type": "single-session-user", "correct": false, "abstention": false }, { "question_id": "f4f1d8a4", "question_type": "single-session-user", "correct": false, "abstention": false }, { "question_id": "0bc8ad92", "question_type": "temporal-reasoning", "correct": true, "abstention": false }, { "question_id": "2c63a862", "question_type": "temporal-reasoning", "correct": false, "abstention": false }, { "question_id": "2ebe6c90", "question_type": "temporal-reasoning", "correct": true, "abstention": false }, { "question_id": "6e984302", "question_type": "temporal-reasoning", "correct": false, "abstention": false }, { "question_id": "8077ef71", "question_type": "temporal-reasoning", "correct": false, "abstention": false }, { "question_id": "993da5e2", "question_type": "temporal-reasoning", "correct": false, "abstention": false }, { "question_id": "a3838d2b", "question_type": "temporal-reasoning", "correct": false, "abstention": false }, { "question_id": "b29f3365", "question_type": "temporal-reasoning", "correct": false, "abstention": false }, { "question_id": "b46e15ee", "question_type": "temporal-reasoning", "correct": false, "abstention": false }, { "question_id": "b9cfe692", "question_type": "temporal-reasoning", "correct": false, "abstention": false }, { "question_id": "bbf86515", "question_type": "temporal-reasoning", "correct": false, "abstention": false }, { "question_id": "c8090214", "question_type": "temporal-reasoning", "correct": false, "abstention": false }, { "question_id": "dcfa8644", "question_type": "temporal-reasoning", "correct": true, "abstention": false }, { "question_id": "gpt4_1a1dc16d", "question_type": "temporal-reasoning", "correct": false, "abstention": false }, { "question_id": "gpt4_1e4a8aeb", "question_type": "temporal-reasoning", "correct": false, "abstention": false }, { "question_id": "gpt4_2312f94c", "question_type": "temporal-reasoning", "correct": false, "abstention": false }, { "question_id": "gpt4_2487a7cb", "question_type": "temporal-reasoning", "correct": false, "abstention": false }, { "question_id": "gpt4_468eb063", "question_type": "temporal-reasoning", "correct": false, "abstention": false }, { "question_id": "gpt4_4fc4f797", "question_type": "temporal-reasoning", "correct": false, "abstention": false }, { "question_id": "gpt4_5438fa52", "question_type": "temporal-reasoning", "correct": false, "abstention": false }, { "question_id": "gpt4_59149c78", "question_type": "temporal-reasoning", "correct": false, "abstention": false }, { "question_id": "gpt4_68e94288", "question_type": "temporal-reasoning", "correct": false, "abstention": false }, { "question_id": "gpt4_70e84552", "question_type": "temporal-reasoning", "correct": true, "abstention": false }, { "question_id": "gpt4_70e84552_abs", "question_type": "temporal-reasoning", "correct": true, "abstention": true }, { "question_id": "gpt4_76048e76", "question_type": "temporal-reasoning", "correct": false, "abstention": false }, { "question_id": "gpt4_7ca326fa", "question_type": "temporal-reasoning", "correct": false, "abstention": false }, { "question_id": "gpt4_7ddcf75f", "question_type": "temporal-reasoning", "correct": true, "abstention": false }, { "question_id": "gpt4_7f6b06db", "question_type": "temporal-reasoning", "correct": false, "abstention": false }, { "question_id": "gpt4_8e165409", "question_type": "temporal-reasoning", "correct": false, "abstention": false }, { "question_id": "gpt4_93159ced", "question_type": "temporal-reasoning", "correct": false, "abstention": false }, { "question_id": "gpt4_93159ced_abs", "question_type": "temporal-reasoning", "correct": true, "abstention": true }, { "question_id": "gpt4_9a159967", "question_type": "temporal-reasoning", "correct": false, "abstention": false }, { "question_id": "gpt4_af6db32f", "question_type": "temporal-reasoning", "correct": false, "abstention": false }, { "question_id": "gpt4_b5700ca0", "question_type": "temporal-reasoning", "correct": false, "abstention": false }, { "question_id": "gpt4_b5700ca9", "question_type": "temporal-reasoning", "correct": false, "abstention": false }, { "question_id": "gpt4_cd90e484", "question_type": "temporal-reasoning", "correct": true, "abstention": false }, { "question_id": "gpt4_d31cdae3", "question_type": "temporal-reasoning", "correct": false, "abstention": false }, { "question_id": "gpt4_e061b84g", "question_type": "temporal-reasoning", "correct": false, "abstention": false }, { "question_id": "gpt4_e072b769", "question_type": "temporal-reasoning", "correct": false, "abstention": false }, { "question_id": "gpt4_fa19884c", "question_type": "temporal-reasoning", "correct": false, "abstention": false } ] }, "qa_full_context": null, "qa_no_memory": null, "quality_notes": [ "Comparator run (memory=chunks): qa_full_context / qa_no_memory were not run — under an identical tuple they are the same calls as the particles run's; reuse those columns and say so." ], "generated_at": "2026-08-17T07:50:13.053599Z" }