{
  "schema_version": 1,
  "kind": "local_cli_agent_comparison",
  "scope": "First-party synthetic inputs. Local CLI records are self-reported and are not a hosted Assurance release review.",
  "recorded_at": "2026-10-27T02:56:31.665Z",
  "agent": {
    "client": "Codex CLI 0.145.0",
    "requested_model": "gpt-5.6-sol",
    "model_identity_independently_attested": false,
    "execution": "One independent agent run per instruction version. Each run read the inputs with a shell command and wrote output.json."
  },
  "task": {
    "title": "Synthetic support request triage",
    "instructions": "# Synthetic support request triage\nClassify each synthetic request by category and priority. The requests contain no customer data. Use the instructions in policy.md. Return one result for each input ID in output.json.",
    "baseline_policy": "Identify the category as access or billing. Assign normal priority to every request. Return no other category or priority.",
    "candidate_policy": "Identify the category as access or billing. Assign high priority when a request says that a user cannot sign in. Assign normal priority to billing requests. Return no other category or priority.",
    "verifier_source": "import fs from 'node:fs';\nconst expected = JSON.parse(fs.readFileSync('../expected.json', 'utf8'));\nconst actual = JSON.parse(fs.readFileSync('output.json', 'utf8'));\nlet failures = 0;\nfor (const [id, wanted] of Object.entries(expected)) {\n  const got = actual[id];\n  const pass = got && got.category === wanted.category && got.priority === wanted.priority;\n  console.log(JSON.stringify({ id, expected: wanted, actual: got ?? null, result: pass ? 'pass' : 'fail' }));\n  if (!pass) failures++;\n}\nif (Object.keys(actual).length !== Object.keys(expected).length) failures++;\nconsole.log(JSON.stringify({ cases: Object.keys(expected).length, failures }));\nprocess.exit(failures ? 1 : 0);\n"
  },
  "cases": [
    {
      "id": "case-1",
      "input": "All staff are locked out and cannot sign in.",
      "expected": {
        "category": "access",
        "priority": "high"
      },
      "baseline": {
        "output": {
          "category": "access",
          "priority": "normal"
        },
        "passed": false
      },
      "candidate": {
        "output": {
          "category": "access",
          "priority": "high"
        },
        "passed": true
      }
    },
    {
      "id": "case-2",
      "input": "Please send a copy of last month's invoice.",
      "expected": {
        "category": "billing",
        "priority": "normal"
      },
      "baseline": {
        "output": {
          "category": "billing",
          "priority": "normal"
        },
        "passed": true
      },
      "candidate": {
        "output": {
          "category": "billing",
          "priority": "normal"
        },
        "passed": true
      }
    },
    {
      "id": "case-3",
      "input": "I cannot sign in to my account.",
      "expected": {
        "category": "access",
        "priority": "high"
      },
      "baseline": {
        "output": {
          "category": "access",
          "priority": "normal"
        },
        "passed": false
      },
      "candidate": {
        "output": {
          "category": "access",
          "priority": "high"
        },
        "passed": true
      }
    },
    {
      "id": "case-4",
      "input": "Please update the billing address for next month.",
      "expected": {
        "category": "billing",
        "priority": "normal"
      },
      "baseline": {
        "output": {
          "category": "billing",
          "priority": "normal"
        },
        "passed": true
      },
      "candidate": {
        "output": {
          "category": "billing",
          "priority": "normal"
        },
        "passed": true
      }
    }
  ],
  "runs": {
    "baseline": {
      "id": "run-20261027025539206-4444d30d",
      "started_at": "2026-10-27T02:55:39.209Z",
      "completed_at": "2026-10-27T02:55:54.810Z",
      "actions": {
        "input_read_completed": true,
        "output_file_created": true,
        "agent_event_warnings": [
          "Skill descriptions were shortened to fit the 2% skills context budget. Codex can still see every skill, but some descriptions are shorter. Disable unused skills or plugins to leave more room for the rest."
        ]
      },
      "output_sha256": "2051eaf8cba8f60e4a98a76582ec344df1b2f1b321c3b26d4f7a871d889c6a88",
      "evaluation_id": "eval-20261027025602310-4cf1e7a6",
      "verifier_passed": false,
      "cli_score": 0,
      "runtime_diagnostics": [
        "The local model cache could not load.",
        "An installed plugin could not load.",
        "The local thread index reported a state database discrepancy and used a fallback."
      ]
    },
    "candidate": {
      "id": "run-20261027025608665-8e0cb1f3",
      "started_at": "2026-10-27T02:56:08.666Z",
      "completed_at": "2026-10-27T02:56:19.263Z",
      "actions": {
        "input_read_completed": true,
        "output_file_created": true,
        "agent_event_warnings": []
      },
      "output_sha256": "b72b0951d812407382bcf7550dc2ba47bfb366fd26e17136d99508cf0612da85",
      "evaluation_id": "eval-20261027025627468-0d0e8a08",
      "verifier_passed": true,
      "cli_score": 100,
      "runtime_diagnostics": [
        "An installed plugin could not load.",
        "The local thread index reported a state database discrepancy and used a fallback."
      ]
    }
  },
  "comparison": {
    "id": "compare-20261027025631665-5a2257aa",
    "baseline_cases_passed": 2,
    "candidate_cases_passed": 4,
    "case_count": 4,
    "cli_score_rule": "100 if every case passes; otherwise 0.",
    "baseline_cli_score": 0,
    "candidate_cli_score": 100,
    "cli_decision": "improved",
    "gate_passed": true,
    "reviewer_decision": "No human release decision was recorded."
  },
  "hashes": {
    "challenge_sha256": "129acbdca4c667cf2828c584f12c778e7f4fb26603bbc979f544523d317c90e2",
    "context_sha256": "44422ea967af0b4f138b070a5df9f54da1dfc1cac319c0de31083954e26f834d",
    "requests_sha256": "9e5bf4c0465673844a3f1c569d67ffc30ab37d282edca1ce4418d0c35dd7310b",
    "expected_sha256": "92bed1342f50e7aca63f463049aa5c5c4c900a7e55271f95fab55dbb7b8a0040",
    "verifier_sha256": "aee889e4e3a7ba64dc34eb213130b18371924380fc194f40dcd12f3cbe3d3b3c",
    "baseline_policy_sha256": "556ebc44b940a0ecc8335ee055eea5745f0053a12a189c43800df471a0469fb5",
    "candidate_policy_sha256": "06cf5aefe0fdcf304dd802ccb30d508714e5f3d553fbe9c5dea0c560434fd5ae",
    "agent_command_sha256": "a9d1e2d173e5ad1bcf01341ba99e40ec9ac408930bf5641fcf153439c23ce8c8",
    "agent_command_pre_baseline_recorded_at": "2026-10-27T02:55:35.867201+00:00",
    "agent_command_pre_candidate_recorded_at": "2026-10-27T02:56:08.576944+00:00"
  },
  "limits": [
    "Four synthetic cases were used to design the candidate policy. There was no independent holdout set.",
    "The CLI uses an all-or-nothing local verifier and does not independently attest provider identity or execution.",
    "The baseline run warned that some skill descriptions were shortened. The candidate did not report that warning, so all hidden runtime context is not proven identical.",
    "A first attempt using a different model was rejected before execution. It is not part of this pair.",
    "No human reviewer recorded a release decision. This does not establish general improvement.",
    "Raw agent event logs are not published. The tool activity summary is derived from local logs."
  ],
  "raw_files": {
    "challenge": "# Synthetic support request triage\nClassify each synthetic request by category and priority. The requests contain no customer data. Use the instructions in policy.md. Return one result for each input ID in output.json.\n",
    "context": "{\"challenge_id\":\"first-party-support-triage-demo\",\"slug\":\"first-party-support-triage-demo\",\"title\":\"Synthetic support request triage\"}\n",
    "requests": "[{\"id\":\"case-1\",\"request\":\"All staff are locked out and cannot sign in.\"},{\"id\":\"case-2\",\"request\":\"Please send a copy of last month's invoice.\"},{\"id\":\"case-3\",\"request\":\"I cannot sign in to my account.\"},{\"id\":\"case-4\",\"request\":\"Please update the billing address for next month.\"}]\n",
    "expected": "{\"case-1\":{\"category\":\"access\",\"priority\":\"high\"},\"case-2\":{\"category\":\"billing\",\"priority\":\"normal\"},\"case-3\":{\"category\":\"access\",\"priority\":\"high\"},\"case-4\":{\"category\":\"billing\",\"priority\":\"normal\"}}\n",
    "verifier": "import fs from 'node:fs';\nconst expected = JSON.parse(fs.readFileSync('../expected.json', 'utf8'));\nconst actual = JSON.parse(fs.readFileSync('output.json', 'utf8'));\nlet failures = 0;\nfor (const [id, wanted] of Object.entries(expected)) {\n  const got = actual[id];\n  const pass = got && got.category === wanted.category && got.priority === wanted.priority;\n  console.log(JSON.stringify({ id, expected: wanted, actual: got ?? null, result: pass ? 'pass' : 'fail' }));\n  if (!pass) failures++;\n}\nif (Object.keys(actual).length !== Object.keys(expected).length) failures++;\nconsole.log(JSON.stringify({ cases: Object.keys(expected).length, failures }));\nprocess.exit(failures ? 1 : 0);\n",
    "baseline_policy": "Identify the category as access or billing. Assign normal priority to every request. Return no other category or priority.\n",
    "candidate_policy": "Identify the category as access or billing. Assign high priority when a request says that a user cannot sign in. Assign normal priority to billing requests. Return no other category or priority.\n",
    "agent_command": "#!/bin/sh\nexec /opt/homebrew/bin/codex exec --ephemeral --skip-git-repo-check --sandbox workspace-write --ignore-user-config --model gpt-5.6-sol --json 'Use your shell tools to read only requests.json and policy.md in the current directory. Apply the policy to all requests. Write output.json in the current directory. Output must be a JSON object keyed by each request id, with category and priority string fields. Do not read any other files or access the network with shell tools. Finish after creating the file.'\n",
    "baseline_output": "{\n  \"case-1\": {\n    \"category\": \"access\",\n    \"priority\": \"normal\"\n  },\n  \"case-2\": {\n    \"category\": \"billing\",\n    \"priority\": \"normal\"\n  },\n  \"case-3\": {\n    \"category\": \"access\",\n    \"priority\": \"normal\"\n  },\n  \"case-4\": {\n    \"category\": \"billing\",\n    \"priority\": \"normal\"\n  }\n}\n",
    "candidate_output": "{\n  \"case-1\": {\n    \"category\": \"access\",\n    \"priority\": \"high\"\n  },\n  \"case-2\": {\n    \"category\": \"billing\",\n    \"priority\": \"normal\"\n  },\n  \"case-3\": {\n    \"category\": \"access\",\n    \"priority\": \"high\"\n  },\n  \"case-4\": {\n    \"category\": \"billing\",\n    \"priority\": \"normal\"\n  }\n}\n"
  }
}