{
  "title": "Agent legibility eval for jasonlovell.ai",
  "ranOn": "2026-09-28T19:16:50.950Z",
  "method": "Each agent answers 20 factual questions about the site in one headless session, reading it one way: HTML pages with curl, the Markdown twins from llms.txt, or the MCP server. One run per cell, against a production build of the site. An answer is correct when it matches every group of the answer key. Input tokens include cached tokens.",
  "channels": {
    "html": "Read it as a browser would: fetch its HTML pages with curl (start with curl -s http://127.0.0.1:3300/) and follow links in the HTML. Do not request any URL ending in .md, .txt or .xml, do not send an Accept: text/markdown header, and do not use http://127.0.0.1:3300/mcp.",
    "markdown": "Read its Markdown with curl: start at http://127.0.0.1:3300/llms.txt and follow it. Fetch only URLs ending in .md or .txt. Do not fetch HTML pages and do not use http://127.0.0.1:3300/mcp.",
    "mcp": "Use only the tools of the MCP server named \"site\". Do not read the site any other way."
  },
  "questions": [
    {
      "id": "q01",
      "q": "In which city is Jason Lovell based?",
      "accept": [
        [
          "austin"
        ]
      ]
    },
    {
      "id": "q02",
      "q": "Which firm does he work for, and in which part of it?",
      "accept": [
        [
          "pwc"
        ],
        [
          "intelligent enterprise"
        ]
      ]
    },
    {
      "id": "q03",
      "q": "Which consultancy did he found?",
      "accept": [
        [
          "captivate"
        ]
      ]
    },
    {
      "id": "q04",
      "q": "In which years did he run that consultancy (start and end)?",
      "accept": [
        [
          "2016"
        ],
        [
          "2021"
        ]
      ]
    },
    {
      "id": "q05",
      "q": "For which AI company was he VP of global partnerships?",
      "accept": [
        [
          "synthesia"
        ]
      ]
    },
    {
      "id": "q06",
      "q": "Which VR company did he lead EMEA brand partnerships for?",
      "accept": [
        [
          "jaunt"
        ]
      ]
    },
    {
      "id": "q07",
      "q": "In the RealityBridge study, how many of the 16 evaluation cases diverged?",
      "accept": [
        [
          "\\b3\\b",
          "\\bthree\\b"
        ]
      ]
    },
    {
      "id": "q08",
      "q": "What reference service does RealityBridge compare against?",
      "accept": [
        [
          "gitea"
        ]
      ]
    },
    {
      "id": "q09",
      "q": "What is the DOI of the Agent Boundary Assurance white paper?",
      "accept": [
        [
          "10\\.5281/zenodo\\.20815336"
        ]
      ]
    },
    {
      "id": "q10",
      "q": "In which month and year was the Tool Bill of Materials specification published?",
      "accept": [
        [
          "feb"
        ],
        [
          "2026"
        ]
      ]
    },
    {
      "id": "q11",
      "q": "Which study asks whether a behavioural signal survives repeated self-distillation?",
      "accept": [
        [
          "ghosttrace"
        ]
      ]
    },
    {
      "id": "q12",
      "q": "In ProofKern, how many MLX-relative winners were there, and how many remained once torch.compile was added?",
      "accept": [
        [
          "\\b4\\b",
          "\\bfour\\b"
        ],
        [
          "\\b0\\b",
          "\\bzero\\b",
          "\\bnone\\b"
        ]
      ]
    },
    {
      "id": "q13",
      "q": "In MCP Continuation Replay, what does a restarted caller get after a lost reply, and is a duplicate issue created?",
      "accept": [
        [
          "recorded result",
          "stored result",
          "recover"
        ],
        [
          "no duplicate",
          "not created",
          "\\bno\\b"
        ]
      ]
    },
    {
      "id": "q14",
      "q": "On which two models did the nano interpretability work measure the per-element attribution-cost boundary?",
      "accept": [
        [
          "gemma"
        ],
        [
          "gpt-2",
          "gpt2"
        ]
      ]
    },
    {
      "id": "q15",
      "q": "What was the CascadeShift study's result?",
      "accept": [
        [
          "includ(es|ed|ing) (0|zero)",
          "(crosses|crossed|spans|spanned) (0|zero)"
        ]
      ]
    },
    {
      "id": "q16",
      "q": "In the Agent-Ready Checkout Gateway design, where is consent recorded before the policy hook runs?",
      "accept": [
        [
          "hash-chained",
          "hash chained",
          "append-only",
          "append only"
        ]
      ]
    },
    {
      "id": "q17",
      "q": "What email address should people use to contact him?",
      "accept": [
        [
          "hello@jasonlovell\\.ai"
        ]
      ]
    },
    {
      "id": "q18",
      "q": "What is his GitHub username?",
      "accept": [
        [
          "jlov7"
        ]
      ]
    },
    {
      "id": "q19",
      "q": "How many research studies does the site list?",
      "accept": [
        [
          "\\b10\\b",
          "\\bten\\b"
        ]
      ]
    },
    {
      "id": "q20",
      "q": "In which month were the nine research studies released?",
      "accept": [
        [
          "sep"
        ]
      ]
    }
  ],
  "results": [
    {
      "agent": "claude-code",
      "channel": "html",
      "model": "claude-sonnet-5",
      "correct": 20,
      "of": 20,
      "inputTokens": 3748735,
      "outputTokens": 22157,
      "turns": 65,
      "seconds": 289
    },
    {
      "agent": "claude-code",
      "channel": "markdown",
      "model": "claude-sonnet-5",
      "correct": 20,
      "of": 20,
      "inputTokens": 105431,
      "outputTokens": 2783,
      "turns": 13,
      "seconds": 24
    },
    {
      "agent": "claude-code",
      "channel": "mcp",
      "model": "claude-sonnet-5",
      "correct": 20,
      "of": 20,
      "inputTokens": 46268,
      "outputTokens": 1515,
      "turns": 10,
      "seconds": 14
    },
    {
      "agent": "codex",
      "channel": "html",
      "model": "gpt-5.6-sol",
      "correct": 20,
      "of": 20,
      "inputTokens": 555180,
      "outputTokens": 4046,
      "turns": 24,
      "seconds": 106
    },
    {
      "agent": "codex",
      "channel": "markdown",
      "model": "gpt-5.6-sol",
      "correct": 19,
      "of": 20,
      "inputTokens": 89924,
      "outputTokens": 893,
      "turns": 8,
      "seconds": 29
    },
    {
      "agent": "codex",
      "channel": "mcp",
      "model": "gpt-5.6-sol",
      "correct": 20,
      "of": 20,
      "inputTokens": 185877,
      "outputTokens": 1093,
      "turns": 16,
      "seconds": 36
    }
  ],
  "answers": {
    "claude-code/html": [
      {
        "id": "q01",
        "answer": "Austin, Texas",
        "correct": true
      },
      {
        "id": "q02",
        "answer": "PwC, in the Intelligent Enterprise part",
        "correct": true
      },
      {
        "id": "q03",
        "answer": "Captivate",
        "correct": true
      },
      {
        "id": "q04",
        "answer": "2016 to 2021",
        "correct": true
      },
      {
        "id": "q05",
        "answer": "Synthesia",
        "correct": true
      },
      {
        "id": "q06",
        "answer": "Jaunt",
        "correct": true
      },
      {
        "id": "q07",
        "answer": "3 of 16 diverged",
        "correct": true
      },
      {
        "id": "q08",
        "answer": "A pinned local Gitea instance (an open-source Git server)",
        "correct": true
      },
      {
        "id": "q09",
        "answer": "https://doi.org/10.5281/zenodo.20815336",
        "correct": true
      },
      {
        "id": "q10",
        "answer": "February 2026",
        "correct": true
      },
      {
        "id": "q11",
        "answer": "GhostTrace",
        "correct": true
      },
      {
        "id": "q12",
        "answer": "4 MLX-relative winners, 0 remained once torch.compile was added",
        "correct": true
      },
      {
        "id": "q13",
        "answer": "A status readback using the same operation identity, recovering the recorded result; no duplicate issue is created",
        "correct": true
      },
      {
        "id": "q14",
        "answer": "Gemma-2-2B and GPT-2",
        "correct": true
      },
      {
        "id": "q15",
        "answer": "The main comparison's confidence interval included zero, so the study reported no benefit",
        "correct": true
      },
      {
        "id": "q16",
        "answer": "In an append-only, hash-chained consent ledger",
        "correct": true
      },
      {
        "id": "q17",
        "answer": "hello@jasonlovell.ai",
        "correct": true
      },
      {
        "id": "q18",
        "answer": "jlov7",
        "correct": true
      },
      {
        "id": "q19",
        "answer": "10",
        "correct": true
      },
      {
        "id": "q20",
        "answer": "September (2026)",
        "correct": true
      }
    ],
    "claude-code/markdown": [
      {
        "id": "q01",
        "answer": "Austin, Texas",
        "correct": true
      },
      {
        "id": "q02",
        "answer": "PwC, Intelligent Enterprise (Frontier AI strategy and enterprise innovation)",
        "correct": true
      },
      {
        "id": "q03",
        "answer": "Captivate",
        "correct": true
      },
      {
        "id": "q04",
        "answer": "2016 to 2021",
        "correct": true
      },
      {
        "id": "q05",
        "answer": "Synthesia",
        "correct": true
      },
      {
        "id": "q06",
        "answer": "Jaunt",
        "correct": true
      },
      {
        "id": "q07",
        "answer": "3 of 16",
        "correct": true
      },
      {
        "id": "q08",
        "answer": "A pinned local Gitea reference service",
        "correct": true
      },
      {
        "id": "q09",
        "answer": "10.5281/zenodo.20815336",
        "correct": true
      },
      {
        "id": "q10",
        "answer": "February 2026",
        "correct": true
      },
      {
        "id": "q11",
        "answer": "GhostTrace",
        "correct": true
      },
      {
        "id": "q12",
        "answer": "4 MLX-relative winners, 0 once torch.compile was added",
        "correct": true
      },
      {
        "id": "q13",
        "answer": "It gets the recorded result via a status readback (recovering the completed work); no duplicate issue is created",
        "correct": true
      },
      {
        "id": "q14",
        "answer": "Gemma-2-2B and GPT-2",
        "correct": true
      },
      {
        "id": "q15",
        "answer": "The main comparison's confidence interval includes zero, so the study found no benefit",
        "correct": true
      },
      {
        "id": "q16",
        "answer": "In an append-only, hash-chained consent ledger",
        "correct": true
      },
      {
        "id": "q17",
        "answer": "hello@jasonlovell.ai",
        "correct": true
      },
      {
        "id": "q18",
        "answer": "jlov7",
        "correct": true
      },
      {
        "id": "q19",
        "answer": "10",
        "correct": true
      },
      {
        "id": "q20",
        "answer": "September",
        "correct": true
      }
    ],
    "claude-code/mcp": [
      {
        "id": "q01",
        "answer": "Austin, Texas",
        "correct": true
      },
      {
        "id": "q02",
        "answer": "PwC, in PwC Intelligent Enterprise (PwC US)",
        "correct": true
      },
      {
        "id": "q03",
        "answer": "Captivate",
        "correct": true
      },
      {
        "id": "q04",
        "answer": "2016 to 2021",
        "correct": true
      },
      {
        "id": "q05",
        "answer": "Synthesia",
        "correct": true
      },
      {
        "id": "q06",
        "answer": "Jaunt",
        "correct": true
      },
      {
        "id": "q07",
        "answer": "3 of 16 diverged",
        "correct": true
      },
      {
        "id": "q08",
        "answer": "A pinned local Gitea reference service",
        "correct": true
      },
      {
        "id": "q09",
        "answer": "10.5281/zenodo.20815336",
        "correct": true
      },
      {
        "id": "q10",
        "answer": "February 2026",
        "correct": true
      },
      {
        "id": "q11",
        "answer": "GhostTrace",
        "correct": true
      },
      {
        "id": "q12",
        "answer": "4 MLX-relative winners, 0 remained once torch.compile was added",
        "correct": true
      },
      {
        "id": "q13",
        "answer": "The restarted caller gets the recorded result via a status readback (using the same operation identity); no duplicate issue is created",
        "correct": true
      },
      {
        "id": "q14",
        "answer": "Gemma-2-2B and GPT-2",
        "correct": true
      },
      {
        "id": "q15",
        "answer": "No benefit found — the main comparison's confidence interval includes zero",
        "correct": true
      },
      {
        "id": "q16",
        "answer": "In an append-only, hash-chained consent ledger",
        "correct": true
      },
      {
        "id": "q17",
        "answer": "hello@jasonlovell.ai",
        "correct": true
      },
      {
        "id": "q18",
        "answer": "jlov7",
        "correct": true
      },
      {
        "id": "q19",
        "answer": "10",
        "correct": true
      },
      {
        "id": "q20",
        "answer": "September 2026",
        "correct": true
      }
    ],
    "codex/html": [
      {
        "id": "q01",
        "answer": "Austin, Texas",
        "correct": true
      },
      {
        "id": "q02",
        "answer": "PwC, Intelligent Enterprise",
        "correct": true
      },
      {
        "id": "q03",
        "answer": "Captivate",
        "correct": true
      },
      {
        "id": "q04",
        "answer": "2016 to 2021",
        "correct": true
      },
      {
        "id": "q05",
        "answer": "Synthesia",
        "correct": true
      },
      {
        "id": "q06",
        "answer": "Jaunt",
        "correct": true
      },
      {
        "id": "q07",
        "answer": "3",
        "correct": true
      },
      {
        "id": "q08",
        "answer": "A pinned local Gitea service",
        "correct": true
      },
      {
        "id": "q09",
        "answer": "10.5281/zenodo.20815336",
        "correct": true
      },
      {
        "id": "q10",
        "answer": "February 2026",
        "correct": true
      },
      {
        "id": "q11",
        "answer": "GhostTrace",
        "correct": true
      },
      {
        "id": "q12",
        "answer": "4 against MLX; 0 with torch.compile",
        "correct": true
      },
      {
        "id": "q13",
        "answer": "The recorded result via status readback; no duplicate issue is created",
        "correct": true
      },
      {
        "id": "q14",
        "answer": "Gemma-2-2B and GPT-2",
        "correct": true
      },
      {
        "id": "q15",
        "answer": "The main comparison’s confidence interval included zero, so it claimed no benefit",
        "correct": true
      },
      {
        "id": "q16",
        "answer": "An append-only, hash-chained consent ledger",
        "correct": true
      },
      {
        "id": "q17",
        "answer": "hello@jasonlovell.ai",
        "correct": true
      },
      {
        "id": "q18",
        "answer": "jlov7",
        "correct": true
      },
      {
        "id": "q19",
        "answer": "10",
        "correct": true
      },
      {
        "id": "q20",
        "answer": "September 2026",
        "correct": true
      }
    ],
    "codex/markdown": [
      {
        "id": "q01",
        "answer": "Austin, Texas",
        "correct": true
      },
      {
        "id": "q02",
        "answer": "PwC, Intelligent Enterprise",
        "correct": true
      },
      {
        "id": "q03",
        "answer": "Captivate",
        "correct": true
      },
      {
        "id": "q04",
        "answer": "2016–2021",
        "correct": true
      },
      {
        "id": "q05",
        "answer": "Synthesia",
        "correct": true
      },
      {
        "id": "q06",
        "answer": "Jaunt",
        "correct": true
      },
      {
        "id": "q07",
        "answer": "3",
        "correct": true
      },
      {
        "id": "q08",
        "answer": "A pinned local Gitea service",
        "correct": true
      },
      {
        "id": "q09",
        "answer": "10.5281/zenodo.20815336",
        "correct": true
      },
      {
        "id": "q10",
        "answer": "June 2026",
        "correct": false
      },
      {
        "id": "q11",
        "answer": "GhostTrace",
        "correct": true
      },
      {
        "id": "q12",
        "answer": "4 MLX-relative winners; 0 with torch.compile",
        "correct": true
      },
      {
        "id": "q13",
        "answer": "The recorded result via status readback; no duplicate issue",
        "correct": true
      },
      {
        "id": "q14",
        "answer": "GPT-2-small and Gemma-2-2B",
        "correct": true
      },
      {
        "id": "q15",
        "answer": "The confidence interval included zero, so no benefit was claimed",
        "correct": true
      },
      {
        "id": "q16",
        "answer": "An append-only, hash-chained consent ledger",
        "correct": true
      },
      {
        "id": "q17",
        "answer": "hello@jasonlovell.ai",
        "correct": true
      },
      {
        "id": "q18",
        "answer": "jlov7",
        "correct": true
      },
      {
        "id": "q19",
        "answer": "10",
        "correct": true
      },
      {
        "id": "q20",
        "answer": "September 2026",
        "correct": true
      }
    ],
    "codex/mcp": [
      {
        "id": "q01",
        "answer": "Austin, Texas",
        "correct": true
      },
      {
        "id": "q02",
        "answer": "PwC US, Intelligent Enterprise",
        "correct": true
      },
      {
        "id": "q03",
        "answer": "Captivate",
        "correct": true
      },
      {
        "id": "q04",
        "answer": "2016–2021",
        "correct": true
      },
      {
        "id": "q05",
        "answer": "Synthesia",
        "correct": true
      },
      {
        "id": "q06",
        "answer": "Jaunt",
        "correct": true
      },
      {
        "id": "q07",
        "answer": "3",
        "correct": true
      },
      {
        "id": "q08",
        "answer": "A pinned local Gitea service",
        "correct": true
      },
      {
        "id": "q09",
        "answer": "10.5281/zenodo.20815336",
        "correct": true
      },
      {
        "id": "q10",
        "answer": "February 2026",
        "correct": true
      },
      {
        "id": "q11",
        "answer": "GhostTrace",
        "correct": true
      },
      {
        "id": "q12",
        "answer": "4 MLX-relative winners; 0 with torch.compile",
        "correct": true
      },
      {
        "id": "q13",
        "answer": "Status readback of the recorded result; no duplicate issue",
        "correct": true
      },
      {
        "id": "q14",
        "answer": "Gemma-2-2B and GPT-2",
        "correct": true
      },
      {
        "id": "q15",
        "answer": "The confidence interval included zero, so it found no benefit",
        "correct": true
      },
      {
        "id": "q16",
        "answer": "An append-only, hash-chained consent ledger",
        "correct": true
      },
      {
        "id": "q17",
        "answer": "hello@jasonlovell.ai",
        "correct": true
      },
      {
        "id": "q18",
        "answer": "jlov7",
        "correct": true
      },
      {
        "id": "q19",
        "answer": "10",
        "correct": true
      },
      {
        "id": "q20",
        "answer": "September 2026",
        "correct": true
      }
    ]
  }
}