V-14

extraction-prompts · run 2026-10-20-v6

What the broker wrote

What every version was given, as the suite chose to show it.

Broker: Variant desk

Hi,

Starting the file on Mill Run Flats LLC — 20-unit apartment building at 77 Mill Run Dr, Toledo, OH 43614 (Lucas County). Frame over block, built 2001.

The appraisal is ordered but not back yet, so I can't give you insurable values — for context the mortgage balance is $4,100,000, but don't rate off that. I'll send TIV the moment it lands.

Target premium in the low $20Ks. Clean loss runs.

Thanks.

What each version answered

Compare each answer with the expert's answer. Highlighted cells differ from the expert; marked rows show where the versions disagree.

Field	Expected	Current prompt v3	New prompt v6
insuredName	Mill Run Flats LLC	Mill Run Flats LLCMatches the expert	Mill Run Flats LLCMatches the expert
state	OH	OHMatches the expert	OHMatches the expert
county	Lucas	LucasMatches the expert	LucasMatches the expert
units	20	20Matches the expert	20Matches the expert
tiv	Not provided	Not providedMatches the expert	Not providedMatches the expert
yearBuilt	2001	2001Matches the expert	2001Matches the expert
Decision	follow up with the broker INCOMPLETE	follow up with the broker INCOMPLETEMatches the expert	follow up with the broker INCOMPLETEMatches the expert

For engineers

Checks are the suite's binary checks on each output; a judge that gave no verdict is unanswered and not counted.

v1

result: ok
prompt tokens: 4,166
output tokens: 790
cost: $0.0083
latency: 11,644 ms · 3 calls
at: 2026-10-21 01:10
stages: intake extraction grounding appetite
decision: INCOMPLETE
fitScore: 78
retries: 0
fieldsWrong: 0
modelErrors

check	verdict
extraction_schema	pass
fields_correct	pass
construction_canonical	not applicable
nulls_stay_null	pass
decision_correct	pass
attribution_jev	unanswered
attribution_claude	pass

{
 "decision": "INCOMPLETE",
 "expectedDecision": "INCOMPLETE",
 "decisionCorrect": true,
 "fitScore": 78,
 "confidence": 0.9499999999999998,
 "reasons": [
  "TIV (total insured value)"
 ],
 "extraction": {
  "insuredName": {
   "value": "Mill Run Flats LLC",
   "confidence": 0.98,
   "source": "email"
  },
  "propertyAddress": {
   "value": "77 Mill Run Dr, Toledo, OH 43614",
   "confidence": 0.97,
   "source": "email"
  },
  "state": {
   "value": "OH",
   "confidence": 0.98,
   "source": "email"
  },
  "county": {
   "value": "Lucas",
   "confidence": 0.97,
   "source": "email"
  },
  "units": {
   "value": 20,
   "confidence": 0.95,
   "source": "email"
  },
  "tiv": {
   "value": null,
   "confidence": 0.95,
   "source": null
  },
  "constructionType": {
   "value": "Frame over block",
   "confidence": 0.85,
   "source": "email"
  },
  "yearBuilt": {
   "value": 2001,
   "confidence": 0.97,
   "source": "email"
  },
  "targetPremium": {
   "value": null,
   "confidence": 0.6,
   "source": "email"
  },
  "subclass": {
   "value": null,
   "confidence": 0.4,
   "source": null
  },
  "priorLossSummary": {
   "value": "Clean loss runs",
   "confidence": 0.8,
   "source": "email"
  }
 },
 "extracted": [
  {
   "field": "insuredName",
   "value": "Mill Run Flats LLC",
   "source": "email"
  },
  {
   "field": "propertyAddress",
   "value": "77 Mill Run Dr, Toledo, OH 43614",
   "source": "email"
  },
  {
   "field": "state",
   "value": "OH",
   "source": "email"
  },
  {
   "field": "county",
   "value": "Lucas",
   "source": "email"
  },
  {
   "field": "units",
   "value": 20,
   "source": "email"
  },
  {
   "field": "constructionType",
   "value": "Frame over block",
   "source": "email"
  },
  {
   "field": "yearBuilt",
   "value": 2001,
   "source": "email"
  },
  {
   "field": "priorLossSummary",
   "value": "Clean loss runs",
   "source": "email"
  }
 ],
 "fields": [
  {
   "field": "insuredName",
   "expected": "Mill Run Flats LLC",
   "got": "Mill Run Flats LLC",
   "correct": true
  },
  {
   "field": "state",
   "expected": "OH",
   "got": "OH",
   "correct": true
  },
  {
   "field": "county",
   "expected": "Lucas",
   "got": "Lucas",
   "correct": true
  },
  {
   "field": "units",
   "expected": 20,
   "got": 20,
   "correct": true
  },
  {
   "field": "tiv",
   "expected": null,
   "got": null,
   "correct": true
  },
  {
   "field": "yearBuilt",
   "expected": 2001,
   "got": 2001,
   "correct": true
  }
 ],
 "retries": 0,
 "guardrails": [
  {
   "node": "extraction",
   "guardrail": "schema",
   "status": "pass"
  },
  {
   "node": "grounding",
   "guardrail": "grounding",
   "status": "pass"
  },
  {
   "node": "appetite",
   "guardrail": "missing_fields",
   "status": "warn"
  },
  {
   "node": "appetite",
   "guardrail": "confidence",
   "status": "pass"
  }
 ],
 "flags": [
  {
   "field": "tiv",
   "issue": "missing",
   "detail": "TIV (total insured value) is required by the appetite ruleset"
  }
 ]
}

Trace line (JSON)

{
  "run_id": "2026-10-20-v6",
  "suite": "extraction-prompts",
  "case_id": "V-14",
  "variant": "v1",
  "at": "2026-10-21T01:10:22.119Z",
  "ok": true,
  "usage": {
    "inputTokens": 6,
    "outputTokens": 790,
    "costUsd": 0.008328,
    "calls": 3,
    "cacheReadTokens": 4160,
    "cacheCreationTokens": 0,
    "latencyMs": 11644,
    "metered": true,
    "provider": "claude-cli"
  },
  "checks": {
    "extraction_schema": true,
    "fields_correct": true,
    "nulls_stay_null": true,
    "decision_correct": true,
    "attribution_claude": true
  },
  "output_hash": "4ee78bd741e74891",
  "unanswered": [
    "attribution_jev"
  ],
  "extra": {
    "decision": "INCOMPLETE",
    "fitScore": 78,
    "retries": 0,
    "fieldsWrong": 0,
    "modelErrors": ""
  },
  "stages": [
    {
      "name": "intake",
      "ok": true
    },
    {
      "name": "extraction",
      "ok": true
    },
    {
      "name": "grounding",
      "ok": true
    },
    {
      "name": "appetite",
      "ok": true
    }
  ],
  "cached": false
}

Full output record (JSON)

{
  "output": {
    "decision": "INCOMPLETE",
    "expectedDecision": "INCOMPLETE",
    "decisionCorrect": true,
    "fitScore": 78,
    "confidence": 0.9499999999999998,
    "reasons": [
      "TIV (total insured value)"
    ],
    "extraction": {
      "insuredName": {
        "value": "Mill Run Flats LLC",
        "confidence": 0.98,
        "source": "email"
      },
      "propertyAddress": {
        "value": "77 Mill Run Dr, Toledo, OH 43614",
        "confidence": 0.97,
        "source": "email"
      },
      "state": {
        "value": "OH",
        "confidence": 0.98,
        "source": "email"
      },
      "county": {
        "value": "Lucas",
        "confidence": 0.97,
        "source": "email"
      },
      "units": {
        "value": 20,
        "confidence": 0.95,
        "source": "email"
      },
      "tiv": {
        "value": null,
        "confidence": 0.95,
        "source": null
      },
      "constructionType": {
        "value": "Frame over block",
        "confidence": 0.85,
        "source": "email"
      },
      "yearBuilt": {
        "value": 2001,
        "confidence": 0.97,
        "source": "email"
      },
      "targetPremium": {
        "value": null,
        "confidence": 0.6,
        "source": "email"
      },
      "subclass": {
        "value": null,
        "confidence": 0.4,
        "source": null
      },
      "priorLossSummary": {
        "value": "Clean loss runs",
        "confidence": 0.8,
        "source": "email"
      }
    },
    "extracted": [
      {
        "field": "insuredName",
        "value": "Mill Run Flats LLC",
        "source": "email"
      },
      {
        "field": "propertyAddress",
        "value": "77 Mill Run Dr, Toledo, OH 43614",
        "source": "email"
      },
      {
        "field": "state",
        "value": "OH",
        "source": "email"
      },
      {
        "field": "county",
        "value": "Lucas",
        "source": "email"
      },
      {
        "field": "units",
        "value": 20,
        "source": "email"
      },
      {
        "field": "constructionType",
        "value": "Frame over block",
        "source": "email"
      },
      {
        "field": "yearBuilt",
        "value": 2001,
        "source": "email"
      },
      {
        "field": "priorLossSummary",
        "value": "Clean loss runs",
        "source": "email"
      }
    ],
    "fields": [
      {
        "field": "insuredName",
        "expected": "Mill Run Flats LLC",
        "got": "Mill Run Flats LLC",
        "correct": true
      },
      {
        "field": "state",
        "expected": "OH",
        "got": "OH",
        "correct": true
      },
      {
        "field": "county",
        "expected": "Lucas",
        "got": "Lucas",
        "correct": true
      },
      {
        "field": "units",
        "expected": 20,
        "got": 20,
        "correct": true
      },
      {
        "field": "tiv",
        "expected": null,
        "got": null,
        "correct": true
      },
      {
        "field": "yearBuilt",
        "expected": 2001,
        "got": 2001,
        "correct": true
      }
    ],
    "retries": 0,
    "guardrails": [
      {
        "node": "extraction",
        "guardrail": "schema",
        "status": "pass"
      },
      {
        "node": "grounding",
        "guardrail": "grounding",
        "status": "pass"
      },
      {
        "node": "appetite",
        "guardrail": "missing_fields",
        "status": "warn"
      },
      {
        "node": "appetite",
        "guardrail": "confidence",
        "status": "pass"
      }
    ],
    "flags": [
      {
        "field": "tiv",
        "issue": "missing",
        "detail": "TIV (total insured value) is required by the appetite ruleset"
      }
    ]
  },
  "usage": {
    "inputTokens": 6,
    "cacheReadTokens": 4160,
    "cacheCreationTokens": 0,
    "outputTokens": 790,
    "costUsd": 0.008328,
    "latencyMs": 11644,
    "calls": 3,
    "metered": true,
    "provider": "claude-cli"
  },
  "extra": {
    "decision": "INCOMPLETE",
    "fitScore": 78,
    "retries": 0,
    "fieldsWrong": 0,
    "modelErrors": ""
  },
  "stages": [
    {
      "name": "intake",
      "ok": true
    },
    {
      "name": "extraction",
      "ok": true
    },
    {
      "name": "grounding",
      "ok": true
    },
    {
      "name": "appetite",
      "ok": true
    }
  ]
}

v2

result: ok
prompt tokens: 4,594
output tokens: 804
cost: $0.0085
latency: 11,891 ms · 3 calls
at: 2026-10-21 01:09
stages: intake extraction grounding appetite
decision: INCOMPLETE
fitScore: 74
retries: 0
fieldsWrong: 0
modelErrors

check	verdict
extraction_schema	pass
fields_correct	pass
construction_canonical	not applicable
nulls_stay_null	pass
decision_correct	pass
attribution_jev	unanswered
attribution_claude	pass

{
 "decision": "INCOMPLETE",
 "expectedDecision": "INCOMPLETE",
 "decisionCorrect": true,
 "fitScore": 74,
 "confidence": 0.9483333333333333,
 "reasons": [
  "TIV (total insured value)"
 ],
 "extraction": {
  "insuredName": {
   "value": "Mill Run Flats LLC",
   "confidence": 0.97,
   "source": "email"
  },
  "propertyAddress": {
   "value": "77 Mill Run Dr, Toledo, OH 43614",
   "confidence": 0.97,
   "source": "email"
  },
  "state": {
   "value": "OH",
   "confidence": 0.98,
   "source": "email"
  },
  "county": {
   "value": "Lucas",
   "confidence": 0.95,
   "source": "email"
  },
  "units": {
   "value": 20,
   "confidence": 0.95,
   "source": "email"
  },
  "tiv": {
   "value": null,
   "confidence": 0.95,
   "source": null
  },
  "constructionType": {
   "value": "Frame",
   "confidence": 0.85,
   "source": "email"
  },
  "yearBuilt": {
   "value": 2001,
   "confidence": 0.97,
   "source": "email"
  },
  "targetPremium": {
   "value": null,
   "confidence": 0.6,
   "source": null
  },
  "subclass": {
   "value": null,
   "confidence": 0.5,
   "source": null
  },
  "priorLossSummary": {
   "value": "Clean loss runs",
   "confidence": 0.85,
   "source": "email"
  }
 },
 "extracted": [
  {
   "field": "insuredName",
   "value": "Mill Run Flats LLC",
   "source": "email"
  },
  {
   "field": "propertyAddress",
   "value": "77 Mill Run Dr, Toledo, OH 43614",
   "source": "email"
  },
  {
   "field": "state",
   "value": "OH",
   "source": "email"
  },
  {
   "field": "county",
   "value": "Lucas",
   "source": "email"
  },
  {
   "field": "units",
   "value": 20,
   "source": "email"
  },
  {
   "field": "constructionType",
   "value": "Frame",
   "source": "email"
  },
  {
   "field": "yearBuilt",
   "value": 2001,
   "source": "email"
  },
  {
   "field": "priorLossSummary",
   "value": "Clean loss runs",
   "source": "email"
  }
 ],
 "fields": [
  {
   "field": "insuredName",
   "expected": "Mill Run Flats LLC",
   "got": "Mill Run Flats LLC",
   "correct": true
  },
  {
   "field": "state",
   "expected": "OH",
   "got": "OH",
   "correct": true
  },
  {
   "field": "county",
   "expected": "Lucas",
   "got": "Lucas",
   "correct": true
  },
  {
   "field": "units",
   "expected": 20,
   "got": 20,
   "correct": true
  },
  {
   "field": "tiv",
   "expected": null,
   "got": null,
   "correct": true
  },
  {
   "field": "yearBuilt",
   "expected": 2001,
   "got": 2001,
   "correct": true
  }
 ],
 "retries": 0,
 "guardrails": [
  {
   "node": "extraction",
   "guardrail": "schema",
   "status": "pass"
  },
  {
   "node": "grounding",
   "guardrail": "grounding",
   "status": "pass"
  },
  {
   "node": "appetite",
   "guardrail": "missing_fields",
   "status": "warn"
  },
  {
   "node": "appetite",
   "guardrail": "confidence",
   "status": "pass"
  }
 ],
 "flags": [
  {
   "field": "tiv",
   "issue": "missing",
   "detail": "TIV (total insured value) is required by the appetite ruleset"
  }
 ]
}

Trace line (JSON)

{
  "run_id": "2026-10-20-v6",
  "suite": "extraction-prompts",
  "case_id": "V-14",
  "variant": "v2",
  "at": "2026-10-21T01:09:55.808Z",
  "ok": true,
  "usage": {
    "inputTokens": 6,
    "outputTokens": 804,
    "costUsd": 0.0085108,
    "calls": 3,
    "cacheReadTokens": 4588,
    "cacheCreationTokens": 0,
    "latencyMs": 11891,
    "metered": true,
    "provider": "claude-cli"
  },
  "checks": {
    "extraction_schema": true,
    "fields_correct": true,
    "nulls_stay_null": true,
    "decision_correct": true,
    "attribution_claude": true
  },
  "output_hash": "670d32cdcd028191",
  "unanswered": [
    "attribution_jev"
  ],
  "extra": {
    "decision": "INCOMPLETE",
    "fitScore": 74,
    "retries": 0,
    "fieldsWrong": 0,
    "modelErrors": ""
  },
  "stages": [
    {
      "name": "intake",
      "ok": true
    },
    {
      "name": "extraction",
      "ok": true
    },
    {
      "name": "grounding",
      "ok": true
    },
    {
      "name": "appetite",
      "ok": true
    }
  ],
  "cached": false
}

Full output record (JSON)

{
  "output": {
    "decision": "INCOMPLETE",
    "expectedDecision": "INCOMPLETE",
    "decisionCorrect": true,
    "fitScore": 74,
    "confidence": 0.9483333333333333,
    "reasons": [
      "TIV (total insured value)"
    ],
    "extraction": {
      "insuredName": {
        "value": "Mill Run Flats LLC",
        "confidence": 0.97,
        "source": "email"
      },
      "propertyAddress": {
        "value": "77 Mill Run Dr, Toledo, OH 43614",
        "confidence": 0.97,
        "source": "email"
      },
      "state": {
        "value": "OH",
        "confidence": 0.98,
        "source": "email"
      },
      "county": {
        "value": "Lucas",
        "confidence": 0.95,
        "source": "email"
      },
      "units": {
        "value": 20,
        "confidence": 0.95,
        "source": "email"
      },
      "tiv": {
        "value": null,
        "confidence": 0.95,
        "source": null
      },
      "constructionType": {
        "value": "Frame",
        "confidence": 0.85,
        "source": "email"
      },
      "yearBuilt": {
        "value": 2001,
        "confidence": 0.97,
        "source": "email"
      },
      "targetPremium": {
        "value": null,
        "confidence": 0.6,
        "source": null
      },
      "subclass": {
        "value": null,
        "confidence": 0.5,
        "source": null
      },
      "priorLossSummary": {
        "value": "Clean loss runs",
        "confidence": 0.85,
        "source": "email"
      }
    },
    "extracted": [
      {
        "field": "insuredName",
        "value": "Mill Run Flats LLC",
        "source": "email"
      },
      {
        "field": "propertyAddress",
        "value": "77 Mill Run Dr, Toledo, OH 43614",
        "source": "email"
      },
      {
        "field": "state",
        "value": "OH",
        "source": "email"
      },
      {
        "field": "county",
        "value": "Lucas",
        "source": "email"
      },
      {
        "field": "units",
        "value": 20,
        "source": "email"
      },
      {
        "field": "constructionType",
        "value": "Frame",
        "source": "email"
      },
      {
        "field": "yearBuilt",
        "value": 2001,
        "source": "email"
      },
      {
        "field": "priorLossSummary",
        "value": "Clean loss runs",
        "source": "email"
      }
    ],
    "fields": [
      {
        "field": "insuredName",
        "expected": "Mill Run Flats LLC",
        "got": "Mill Run Flats LLC",
        "correct": true
      },
      {
        "field": "state",
        "expected": "OH",
        "got": "OH",
        "correct": true
      },
      {
        "field": "county",
        "expected": "Lucas",
        "got": "Lucas",
        "correct": true
      },
      {
        "field": "units",
        "expected": 20,
        "got": 20,
        "correct": true
      },
      {
        "field": "tiv",
        "expected": null,
        "got": null,
        "correct": true
      },
      {
        "field": "yearBuilt",
        "expected": 2001,
        "got": 2001,
        "correct": true
      }
    ],
    "retries": 0,
    "guardrails": [
      {
        "node": "extraction",
        "guardrail": "schema",
        "status": "pass"
      },
      {
        "node": "grounding",
        "guardrail": "grounding",
        "status": "pass"
      },
      {
        "node": "appetite",
        "guardrail": "missing_fields",
        "status": "warn"
      },
      {
        "node": "appetite",
        "guardrail": "confidence",
        "status": "pass"
      }
    ],
    "flags": [
      {
        "field": "tiv",
        "issue": "missing",
        "detail": "TIV (total insured value) is required by the appetite ruleset"
      }
    ]
  },
  "usage": {
    "inputTokens": 6,
    "cacheReadTokens": 4588,
    "cacheCreationTokens": 0,
    "outputTokens": 804,
    "costUsd": 0.0085108,
    "latencyMs": 11891,
    "calls": 3,
    "metered": true,
    "provider": "claude-cli"
  },
  "extra": {
    "decision": "INCOMPLETE",
    "fitScore": 74,
    "retries": 0,
    "fieldsWrong": 0,
    "modelErrors": ""
  },
  "stages": [
    {
      "name": "intake",
      "ok": true
    },
    {
      "name": "extraction",
      "ok": true
    },
    {
      "name": "grounding",
      "ok": true
    },
    {
      "name": "appetite",
      "ok": true
    }
  ]
}

v3

result: ok
prompt tokens: 4,908
output tokens: 1,241
cost: $0.0129
latency: 14,747 ms · 3 calls
at: 2026-10-21 01:10
stages: intake extraction grounding appetite
decision: INCOMPLETE
fitScore: 74
retries: 0
fieldsWrong: 0
modelErrors

check	verdict
extraction_schema	pass
fields_correct	pass
construction_canonical	not applicable
nulls_stay_null	pass
decision_correct	pass
attribution_jev	unanswered
attribution_claude	pass

{
 "decision": "INCOMPLETE",
 "expectedDecision": "INCOMPLETE",
 "decisionCorrect": true,
 "fitScore": 74,
 "confidence": 0.9133333333333332,
 "reasons": [
  "TIV (total insured value)"
 ],
 "extraction": {
  "insuredName": {
   "value": "Mill Run Flats LLC",
   "confidence": 0.98,
   "source": "email"
  },
  "propertyAddress": {
   "value": "77 Mill Run Dr, Toledo, OH 43614",
   "confidence": 0.98,
   "source": "email"
  },
  "state": {
   "value": "OH",
   "confidence": 0.98,
   "source": "email"
  },
  "county": {
   "value": "Lucas",
   "confidence": 0.97,
   "source": "email"
  },
  "units": {
   "value": 20,
   "confidence": 0.97,
   "source": "email"
  },
  "tiv": {
   "value": null,
   "confidence": 0,
   "source": null
  },
  "constructionType": {
   "value": "Frame",
   "confidence": 0.6,
   "source": "email"
  },
  "yearBuilt": {
   "value": 2001,
   "confidence": 0.97,
   "source": "email"
  },
  "targetPremium": {
   "value": null,
   "confidence": 0,
   "source": null
  },
  "subclass": {
   "value": null,
   "confidence": 0,
   "source": null
  },
  "priorLossSummary": {
   "value": "Clean loss runs",
   "confidence": 0.8,
   "source": "email"
  }
 },
 "extracted": [
  {
   "field": "insuredName",
   "value": "Mill Run Flats LLC",
   "source": "email"
  },
  {
   "field": "propertyAddress",
   "value": "77 Mill Run Dr, Toledo, OH 43614",
   "source": "email"
  },
  {
   "field": "state",
   "value": "OH",
   "source": "email"
  },
  {
   "field": "county",
   "value": "Lucas",
   "source": "email"
  },
  {
   "field": "units",
   "value": 20,
   "source": "email"
  },
  {
   "field": "constructionType",
   "value": "Frame",
   "source": "email"
  },
  {
   "field": "yearBuilt",
   "value": 2001,
   "source": "email"
  },
  {
   "field": "priorLossSummary",
   "value": "Clean loss runs",
   "source": "email"
  }
 ],
 "fields": [
  {
   "field": "insuredName",
   "expected": "Mill Run Flats LLC",
   "got": "Mill Run Flats LLC",
   "correct": true
  },
  {
   "field": "state",
   "expected": "OH",
   "got": "OH",
   "correct": true
  },
  {
   "field": "county",
   "expected": "Lucas",
   "got": "Lucas",
   "correct": true
  },
  {
   "field": "units",
   "expected": 20,
   "got": 20,
   "correct": true
  },
  {
   "field": "tiv",
   "expected": null,
   "got": null,
   "correct": true
  },
  {
   "field": "yearBuilt",
   "expected": 2001,
   "got": 2001,
   "correct": true
  }
 ],
 "retries": 0,
 "guardrails": [
  {
   "node": "extraction",
   "guardrail": "schema",
   "status": "pass"
  },
  {
   "node": "grounding",
   "guardrail": "grounding",
   "status": "pass"
  },
  {
   "node": "appetite",
   "guardrail": "missing_fields",
   "status": "warn"
  },
  {
   "node": "appetite",
   "guardrail": "confidence",
   "status": "pass"
  }
 ],
 "flags": [
  {
   "field": "tiv",
   "issue": "missing",
   "detail": "TIV (total insured value) is required by the appetite ruleset"
  }
 ]
}

Trace line (JSON)

{
  "run_id": "2026-10-20-v6",
  "suite": "extraction-prompts",
  "case_id": "V-14",
  "variant": "v3",
  "at": "2026-10-21T01:10:00.991Z",
  "ok": true,
  "usage": {
    "inputTokens": 6,
    "outputTokens": 1241,
    "costUsd": 0.0129122,
    "calls": 3,
    "cacheReadTokens": 4902,
    "cacheCreationTokens": 0,
    "latencyMs": 14747,
    "metered": true,
    "provider": "claude-cli"
  },
  "checks": {
    "extraction_schema": true,
    "fields_correct": true,
    "nulls_stay_null": true,
    "decision_correct": true,
    "attribution_claude": true
  },
  "output_hash": "52b16f41e45f0057",
  "unanswered": [
    "attribution_jev"
  ],
  "extra": {
    "decision": "INCOMPLETE",
    "fitScore": 74,
    "retries": 0,
    "fieldsWrong": 0,
    "modelErrors": ""
  },
  "stages": [
    {
      "name": "intake",
      "ok": true
    },
    {
      "name": "extraction",
      "ok": true
    },
    {
      "name": "grounding",
      "ok": true
    },
    {
      "name": "appetite",
      "ok": true
    }
  ],
  "cached": false
}

Full output record (JSON)

{
  "output": {
    "decision": "INCOMPLETE",
    "expectedDecision": "INCOMPLETE",
    "decisionCorrect": true,
    "fitScore": 74,
    "confidence": 0.9133333333333332,
    "reasons": [
      "TIV (total insured value)"
    ],
    "extraction": {
      "insuredName": {
        "value": "Mill Run Flats LLC",
        "confidence": 0.98,
        "source": "email"
      },
      "propertyAddress": {
        "value": "77 Mill Run Dr, Toledo, OH 43614",
        "confidence": 0.98,
        "source": "email"
      },
      "state": {
        "value": "OH",
        "confidence": 0.98,
        "source": "email"
      },
      "county": {
        "value": "Lucas",
        "confidence": 0.97,
        "source": "email"
      },
      "units": {
        "value": 20,
        "confidence": 0.97,
        "source": "email"
      },
      "tiv": {
        "value": null,
        "confidence": 0,
        "source": null
      },
      "constructionType": {
        "value": "Frame",
        "confidence": 0.6,
        "source": "email"
      },
      "yearBuilt": {
        "value": 2001,
        "confidence": 0.97,
        "source": "email"
      },
      "targetPremium": {
        "value": null,
        "confidence": 0,
        "source": null
      },
      "subclass": {
        "value": null,
        "confidence": 0,
        "source": null
      },
      "priorLossSummary": {
        "value": "Clean loss runs",
        "confidence": 0.8,
        "source": "email"
      }
    },
    "extracted": [
      {
        "field": "insuredName",
        "value": "Mill Run Flats LLC",
        "source": "email"
      },
      {
        "field": "propertyAddress",
        "value": "77 Mill Run Dr, Toledo, OH 43614",
        "source": "email"
      },
      {
        "field": "state",
        "value": "OH",
        "source": "email"
      },
      {
        "field": "county",
        "value": "Lucas",
        "source": "email"
      },
      {
        "field": "units",
        "value": 20,
        "source": "email"
      },
      {
        "field": "constructionType",
        "value": "Frame",
        "source": "email"
      },
      {
        "field": "yearBuilt",
        "value": 2001,
        "source": "email"
      },
      {
        "field": "priorLossSummary",
        "value": "Clean loss runs",
        "source": "email"
      }
    ],
    "fields": [
      {
        "field": "insuredName",
        "expected": "Mill Run Flats LLC",
        "got": "Mill Run Flats LLC",
        "correct": true
      },
      {
        "field": "state",
        "expected": "OH",
        "got": "OH",
        "correct": true
      },
      {
        "field": "county",
        "expected": "Lucas",
        "got": "Lucas",
        "correct": true
      },
      {
        "field": "units",
        "expected": 20,
        "got": 20,
        "correct": true
      },
      {
        "field": "tiv",
        "expected": null,
        "got": null,
        "correct": true
      },
      {
        "field": "yearBuilt",
        "expected": 2001,
        "got": 2001,
        "correct": true
      }
    ],
    "retries": 0,
    "guardrails": [
      {
        "node": "extraction",
        "guardrail": "schema",
        "status": "pass"
      },
      {
        "node": "grounding",
        "guardrail": "grounding",
        "status": "pass"
      },
      {
        "node": "appetite",
        "guardrail": "missing_fields",
        "status": "warn"
      },
      {
        "node": "appetite",
        "guardrail": "confidence",
        "status": "pass"
      }
    ],
    "flags": [
      {
        "field": "tiv",
        "issue": "missing",
        "detail": "TIV (total insured value) is required by the appetite ruleset"
      }
    ]
  },
  "usage": {
    "inputTokens": 6,
    "cacheReadTokens": 4902,
    "cacheCreationTokens": 0,
    "outputTokens": 1241,
    "costUsd": 0.0129122,
    "latencyMs": 14747,
    "calls": 3,
    "metered": true,
    "provider": "claude-cli"
  },
  "extra": {
    "decision": "INCOMPLETE",
    "fitScore": 74,
    "retries": 0,
    "fieldsWrong": 0,
    "modelErrors": ""
  },
  "stages": [
    {
      "name": "intake",
      "ok": true
    },
    {
      "name": "extraction",
      "ok": true
    },
    {
      "name": "grounding",
      "ok": true
    },
    {
      "name": "appetite",
      "ok": true
    }
  ]
}

v3-repeat

result: ok
prompt tokens: 4,908
output tokens: 1,115
cost: $0.0117
latency: 14,176 ms · 3 calls
at: 2026-10-21 01:10
stages: intake extraction grounding appetite
decision: INCOMPLETE
fitScore: 74
retries: 0
fieldsWrong: 0
modelErrors

check	verdict
extraction_schema	pass
fields_correct	pass
construction_canonical	not applicable
nulls_stay_null	pass
decision_correct	pass
attribution_jev	unanswered
attribution_claude	pass

{
 "decision": "INCOMPLETE",
 "expectedDecision": "INCOMPLETE",
 "decisionCorrect": true,
 "fitScore": 74,
 "confidence": 0.9133333333333332,
 "reasons": [
  "TIV (total insured value)"
 ],
 "extraction": {
  "insuredName": {
   "value": "Mill Run Flats LLC",
   "confidence": 0.98,
   "source": "email"
  },
  "propertyAddress": {
   "value": "77 Mill Run Dr, Toledo, OH 43614",
   "confidence": 0.98,
   "source": "email"
  },
  "state": {
   "value": "OH",
   "confidence": 0.98,
   "source": "email"
  },
  "county": {
   "value": "Lucas",
   "confidence": 0.97,
   "source": "email"
  },
  "units": {
   "value": 20,
   "confidence": 0.97,
   "source": "email"
  },
  "tiv": {
   "value": null,
   "confidence": 0.95,
   "source": null
  },
  "constructionType": {
   "value": "Frame",
   "confidence": 0.6,
   "source": "email"
  },
  "yearBuilt": {
   "value": 2001,
   "confidence": 0.97,
   "source": "email"
  },
  "targetPremium": {
   "value": null,
   "confidence": 0.9,
   "source": null
  },
  "subclass": {
   "value": null,
   "confidence": 0.9,
   "source": null
  },
  "priorLossSummary": {
   "value": "Clean loss runs",
   "confidence": 0.8,
   "source": "email"
  }
 },
 "extracted": [
  {
   "field": "insuredName",
   "value": "Mill Run Flats LLC",
   "source": "email"
  },
  {
   "field": "propertyAddress",
   "value": "77 Mill Run Dr, Toledo, OH 43614",
   "source": "email"
  },
  {
   "field": "state",
   "value": "OH",
   "source": "email"
  },
  {
   "field": "county",
   "value": "Lucas",
   "source": "email"
  },
  {
   "field": "units",
   "value": 20,
   "source": "email"
  },
  {
   "field": "constructionType",
   "value": "Frame",
   "source": "email"
  },
  {
   "field": "yearBuilt",
   "value": 2001,
   "source": "email"
  },
  {
   "field": "priorLossSummary",
   "value": "Clean loss runs",
   "source": "email"
  }
 ],
 "fields": [
  {
   "field": "insuredName",
   "expected": "Mill Run Flats LLC",
   "got": "Mill Run Flats LLC",
   "correct": true
  },
  {
   "field": "state",
   "expected": "OH",
   "got": "OH",
   "correct": true
  },
  {
   "field": "county",
   "expected": "Lucas",
   "got": "Lucas",
   "correct": true
  },
  {
   "field": "units",
   "expected": 20,
   "got": 20,
   "correct": true
  },
  {
   "field": "tiv",
   "expected": null,
   "got": null,
   "correct": true
  },
  {
   "field": "yearBuilt",
   "expected": 2001,
   "got": 2001,
   "correct": true
  }
 ],
 "retries": 0,
 "guardrails": [
  {
   "node": "extraction",
   "guardrail": "schema",
   "status": "pass"
  },
  {
   "node": "grounding",
   "guardrail": "grounding",
   "status": "pass"
  },
  {
   "node": "appetite",
   "guardrail": "missing_fields",
   "status": "warn"
  },
  {
   "node": "appetite",
   "guardrail": "confidence",
   "status": "pass"
  }
 ],
 "flags": [
  {
   "field": "tiv",
   "issue": "missing",
   "detail": "TIV (total insured value) is required by the appetite ruleset"
  }
 ]
}

Trace line (JSON)

{
  "run_id": "2026-10-20-v6",
  "suite": "extraction-prompts",
  "case_id": "V-14",
  "variant": "v3-repeat",
  "at": "2026-10-21T01:10:09.972Z",
  "ok": true,
  "usage": {
    "inputTokens": 6,
    "outputTokens": 1115,
    "costUsd": 0.0116522,
    "calls": 3,
    "cacheReadTokens": 4902,
    "cacheCreationTokens": 0,
    "latencyMs": 14176,
    "metered": true,
    "provider": "claude-cli"
  },
  "checks": {
    "extraction_schema": true,
    "fields_correct": true,
    "nulls_stay_null": true,
    "decision_correct": true,
    "attribution_claude": true
  },
  "output_hash": "5daf76469a9803a8",
  "unanswered": [
    "attribution_jev"
  ],
  "extra": {
    "decision": "INCOMPLETE",
    "fitScore": 74,
    "retries": 0,
    "fieldsWrong": 0,
    "modelErrors": ""
  },
  "stages": [
    {
      "name": "intake",
      "ok": true
    },
    {
      "name": "extraction",
      "ok": true
    },
    {
      "name": "grounding",
      "ok": true
    },
    {
      "name": "appetite",
      "ok": true
    }
  ],
  "cached": false
}

Full output record (JSON)

{
  "output": {
    "decision": "INCOMPLETE",
    "expectedDecision": "INCOMPLETE",
    "decisionCorrect": true,
    "fitScore": 74,
    "confidence": 0.9133333333333332,
    "reasons": [
      "TIV (total insured value)"
    ],
    "extraction": {
      "insuredName": {
        "value": "Mill Run Flats LLC",
        "confidence": 0.98,
        "source": "email"
      },
      "propertyAddress": {
        "value": "77 Mill Run Dr, Toledo, OH 43614",
        "confidence": 0.98,
        "source": "email"
      },
      "state": {
        "value": "OH",
        "confidence": 0.98,
        "source": "email"
      },
      "county": {
        "value": "Lucas",
        "confidence": 0.97,
        "source": "email"
      },
      "units": {
        "value": 20,
        "confidence": 0.97,
        "source": "email"
      },
      "tiv": {
        "value": null,
        "confidence": 0.95,
        "source": null
      },
      "constructionType": {
        "value": "Frame",
        "confidence": 0.6,
        "source": "email"
      },
      "yearBuilt": {
        "value": 2001,
        "confidence": 0.97,
        "source": "email"
      },
      "targetPremium": {
        "value": null,
        "confidence": 0.9,
        "source": null
      },
      "subclass": {
        "value": null,
        "confidence": 0.9,
        "source": null
      },
      "priorLossSummary": {
        "value": "Clean loss runs",
        "confidence": 0.8,
        "source": "email"
      }
    },
    "extracted": [
      {
        "field": "insuredName",
        "value": "Mill Run Flats LLC",
        "source": "email"
      },
      {
        "field": "propertyAddress",
        "value": "77 Mill Run Dr, Toledo, OH 43614",
        "source": "email"
      },
      {
        "field": "state",
        "value": "OH",
        "source": "email"
      },
      {
        "field": "county",
        "value": "Lucas",
        "source": "email"
      },
      {
        "field": "units",
        "value": 20,
        "source": "email"
      },
      {
        "field": "constructionType",
        "value": "Frame",
        "source": "email"
      },
      {
        "field": "yearBuilt",
        "value": 2001,
        "source": "email"
      },
      {
        "field": "priorLossSummary",
        "value": "Clean loss runs",
        "source": "email"
      }
    ],
    "fields": [
      {
        "field": "insuredName",
        "expected": "Mill Run Flats LLC",
        "got": "Mill Run Flats LLC",
        "correct": true
      },
      {
        "field": "state",
        "expected": "OH",
        "got": "OH",
        "correct": true
      },
      {
        "field": "county",
        "expected": "Lucas",
        "got": "Lucas",
        "correct": true
      },
      {
        "field": "units",
        "expected": 20,
        "got": 20,
        "correct": true
      },
      {
        "field": "tiv",
        "expected": null,
        "got": null,
        "correct": true
      },
      {
        "field": "yearBuilt",
        "expected": 2001,
        "got": 2001,
        "correct": true
      }
    ],
    "retries": 0,
    "guardrails": [
      {
        "node": "extraction",
        "guardrail": "schema",
        "status": "pass"
      },
      {
        "node": "grounding",
        "guardrail": "grounding",
        "status": "pass"
      },
      {
        "node": "appetite",
        "guardrail": "missing_fields",
        "status": "warn"
      },
      {
        "node": "appetite",
        "guardrail": "confidence",
        "status": "pass"
      }
    ],
    "flags": [
      {
        "field": "tiv",
        "issue": "missing",
        "detail": "TIV (total insured value) is required by the appetite ruleset"
      }
    ]
  },
  "usage": {
    "inputTokens": 6,
    "cacheReadTokens": 4902,
    "cacheCreationTokens": 0,
    "outputTokens": 1115,
    "costUsd": 0.0116522,
    "latencyMs": 14176,
    "calls": 3,
    "metered": true,
    "provider": "claude-cli"
  },
  "extra": {
    "decision": "INCOMPLETE",
    "fitScore": 74,
    "retries": 0,
    "fieldsWrong": 0,
    "modelErrors": ""
  },
  "stages": [
    {
      "name": "intake",
      "ok": true
    },
    {
      "name": "extraction",
      "ok": true
    },
    {
      "name": "grounding",
      "ok": true
    },
    {
      "name": "appetite",
      "ok": true
    }
  ]
}

v4

result: ok
prompt tokens: 5,117
output tokens: 1,318
cost: $0.0137
latency: 15,798 ms · 3 calls
at: 2026-10-21 01:10
stages: intake extraction grounding appetite
decision: INCOMPLETE
fitScore: 68
retries: 0
fieldsWrong: 0
modelErrors

check	verdict
extraction_schema	pass
fields_correct	pass
construction_canonical	not applicable
nulls_stay_null	pass
decision_correct	pass
attribution_jev	unanswered
attribution_claude	pass

{
 "decision": "INCOMPLETE",
 "expectedDecision": "INCOMPLETE",
 "decisionCorrect": true,
 "fitScore": 68,
 "confidence": 0.9283333333333332,
 "reasons": [
  "TIV (total insured value)"
 ],
 "extraction": {
  "insuredName": {
   "value": "Mill Run Flats LLC",
   "confidence": 0.98,
   "source": "email"
  },
  "propertyAddress": {
   "value": "77 Mill Run Dr, Toledo, OH 43614",
   "confidence": 0.97,
   "source": "email"
  },
  "state": {
   "value": "OH",
   "confidence": 0.98,
   "source": "email"
  },
  "county": {
   "value": "Lucas",
   "confidence": 0.95,
   "source": "email"
  },
  "units": {
   "value": 20,
   "confidence": 0.97,
   "source": "email"
  },
  "tiv": {
   "value": null,
   "confidence": 0,
   "source": null
  },
  "constructionType": {
   "value": "Frame",
   "confidence": 0.7,
   "source": "email"
  },
  "yearBuilt": {
   "value": 2001,
   "confidence": 0.97,
   "source": "email"
  },
  "targetPremium": {
   "value": null,
   "confidence": 0,
   "source": null
  },
  "subclass": {
   "value": null,
   "confidence": 0,
   "source": null
  },
  "priorLossSummary": {
   "value": "Clean loss runs",
   "confidence": 0.85,
   "source": "email"
  }
 },
 "extracted": [
  {
   "field": "insuredName",
   "value": "Mill Run Flats LLC",
   "source": "email"
  },
  {
   "field": "propertyAddress",
   "value": "77 Mill Run Dr, Toledo, OH 43614",
   "source": "email"
  },
  {
   "field": "state",
   "value": "OH",
   "source": "email"
  },
  {
   "field": "county",
   "value": "Lucas",
   "source": "email"
  },
  {
   "field": "units",
   "value": 20,
   "source": "email"
  },
  {
   "field": "constructionType",
   "value": "Frame",
   "source": "email"
  },
  {
   "field": "yearBuilt",
   "value": 2001,
   "source": "email"
  },
  {
   "field": "priorLossSummary",
   "value": "Clean loss runs",
   "source": "email"
  }
 ],
 "fields": [
  {
   "field": "insuredName",
   "expected": "Mill Run Flats LLC",
   "got": "Mill Run Flats LLC",
   "correct": true
  },
  {
   "field": "state",
   "expected": "OH",
   "got": "OH",
   "correct": true
  },
  {
   "field": "county",
   "expected": "Lucas",
   "got": "Lucas",
   "correct": true
  },
  {
   "field": "units",
   "expected": 20,
   "got": 20,
   "correct": true
  },
  {
   "field": "tiv",
   "expected": null,
   "got": null,
   "correct": true
  },
  {
   "field": "yearBuilt",
   "expected": 2001,
   "got": 2001,
   "correct": true
  }
 ],
 "retries": 0,
 "guardrails": [
  {
   "node": "extraction",
   "guardrail": "schema",
   "status": "pass"
  },
  {
   "node": "grounding",
   "guardrail": "grounding",
   "status": "pass"
  },
  {
   "node": "appetite",
   "guardrail": "missing_fields",
   "status": "warn"
  },
  {
   "node": "appetite",
   "guardrail": "confidence",
   "status": "pass"
  }
 ],
 "flags": [
  {
   "field": "tiv",
   "issue": "missing",
   "detail": "TIV (total insured value) is required by the appetite ruleset"
  }
 ]
}

Trace line (JSON)

{
  "run_id": "2026-10-20-v6",
  "suite": "extraction-prompts",
  "case_id": "V-14",
  "variant": "v4",
  "at": "2026-10-21T01:10:11.613Z",
  "ok": true,
  "usage": {
    "inputTokens": 6,
    "outputTokens": 1318,
    "costUsd": 0.0137031,
    "calls": 3,
    "cacheReadTokens": 5111,
    "cacheCreationTokens": 0,
    "latencyMs": 15798,
    "metered": true,
    "provider": "claude-cli"
  },
  "checks": {
    "extraction_schema": true,
    "fields_correct": true,
    "nulls_stay_null": true,
    "decision_correct": true,
    "attribution_claude": true
  },
  "output_hash": "8d48a57c782eec70",
  "unanswered": [
    "attribution_jev"
  ],
  "extra": {
    "decision": "INCOMPLETE",
    "fitScore": 68,
    "retries": 0,
    "fieldsWrong": 0,
    "modelErrors": ""
  },
  "stages": [
    {
      "name": "intake",
      "ok": true
    },
    {
      "name": "extraction",
      "ok": true
    },
    {
      "name": "grounding",
      "ok": true
    },
    {
      "name": "appetite",
      "ok": true
    }
  ],
  "cached": false
}

Full output record (JSON)

{
  "output": {
    "decision": "INCOMPLETE",
    "expectedDecision": "INCOMPLETE",
    "decisionCorrect": true,
    "fitScore": 68,
    "confidence": 0.9283333333333332,
    "reasons": [
      "TIV (total insured value)"
    ],
    "extraction": {
      "insuredName": {
        "value": "Mill Run Flats LLC",
        "confidence": 0.98,
        "source": "email"
      },
      "propertyAddress": {
        "value": "77 Mill Run Dr, Toledo, OH 43614",
        "confidence": 0.97,
        "source": "email"
      },
      "state": {
        "value": "OH",
        "confidence": 0.98,
        "source": "email"
      },
      "county": {
        "value": "Lucas",
        "confidence": 0.95,
        "source": "email"
      },
      "units": {
        "value": 20,
        "confidence": 0.97,
        "source": "email"
      },
      "tiv": {
        "value": null,
        "confidence": 0,
        "source": null
      },
      "constructionType": {
        "value": "Frame",
        "confidence": 0.7,
        "source": "email"
      },
      "yearBuilt": {
        "value": 2001,
        "confidence": 0.97,
        "source": "email"
      },
      "targetPremium": {
        "value": null,
        "confidence": 0,
        "source": null
      },
      "subclass": {
        "value": null,
        "confidence": 0,
        "source": null
      },
      "priorLossSummary": {
        "value": "Clean loss runs",
        "confidence": 0.85,
        "source": "email"
      }
    },
    "extracted": [
      {
        "field": "insuredName",
        "value": "Mill Run Flats LLC",
        "source": "email"
      },
      {
        "field": "propertyAddress",
        "value": "77 Mill Run Dr, Toledo, OH 43614",
        "source": "email"
      },
      {
        "field": "state",
        "value": "OH",
        "source": "email"
      },
      {
        "field": "county",
        "value": "Lucas",
        "source": "email"
      },
      {
        "field": "units",
        "value": 20,
        "source": "email"
      },
      {
        "field": "constructionType",
        "value": "Frame",
        "source": "email"
      },
      {
        "field": "yearBuilt",
        "value": 2001,
        "source": "email"
      },
      {
        "field": "priorLossSummary",
        "value": "Clean loss runs",
        "source": "email"
      }
    ],
    "fields": [
      {
        "field": "insuredName",
        "expected": "Mill Run Flats LLC",
        "got": "Mill Run Flats LLC",
        "correct": true
      },
      {
        "field": "state",
        "expected": "OH",
        "got": "OH",
        "correct": true
      },
      {
        "field": "county",
        "expected": "Lucas",
        "got": "Lucas",
        "correct": true
      },
      {
        "field": "units",
        "expected": 20,
        "got": 20,
        "correct": true
      },
      {
        "field": "tiv",
        "expected": null,
        "got": null,
        "correct": true
      },
      {
        "field": "yearBuilt",
        "expected": 2001,
        "got": 2001,
        "correct": true
      }
    ],
    "retries": 0,
    "guardrails": [
      {
        "node": "extraction",
        "guardrail": "schema",
        "status": "pass"
      },
      {
        "node": "grounding",
        "guardrail": "grounding",
        "status": "pass"
      },
      {
        "node": "appetite",
        "guardrail": "missing_fields",
        "status": "warn"
      },
      {
        "node": "appetite",
        "guardrail": "confidence",
        "status": "pass"
      }
    ],
    "flags": [
      {
        "field": "tiv",
        "issue": "missing",
        "detail": "TIV (total insured value) is required by the appetite ruleset"
      }
    ]
  },
  "usage": {
    "inputTokens": 6,
    "cacheReadTokens": 5111,
    "cacheCreationTokens": 0,
    "outputTokens": 1318,
    "costUsd": 0.0137031,
    "latencyMs": 15798,
    "calls": 3,
    "metered": true,
    "provider": "claude-cli"
  },
  "extra": {
    "decision": "INCOMPLETE",
    "fitScore": 68,
    "retries": 0,
    "fieldsWrong": 0,
    "modelErrors": ""
  },
  "stages": [
    {
      "name": "intake",
      "ok": true
    },
    {
      "name": "extraction",
      "ok": true
    },
    {
      "name": "grounding",
      "ok": true
    },
    {
      "name": "appetite",
      "ok": true
    }
  ]
}

v5

result: ok
prompt tokens: 5,249
output tokens: 1,097
cost: $0.0115
latency: 14,515 ms · 3 calls
at: 2026-10-21 01:10
stages: intake extraction grounding appetite
decision: INCOMPLETE
fitScore: 74
retries: 0
fieldsWrong: 0
modelErrors

check	verdict
extraction_schema	pass
fields_correct	pass
construction_canonical	not applicable
nulls_stay_null	pass
decision_correct	pass
attribution_jev	unanswered
attribution_claude	pass

{
 "decision": "INCOMPLETE",
 "expectedDecision": "INCOMPLETE",
 "decisionCorrect": true,
 "fitScore": 74,
 "confidence": 0.93,
 "reasons": [
  "TIV (total insured value)"
 ],
 "extraction": {
  "insuredName": {
   "value": "Mill Run Flats LLC",
   "confidence": 0.98,
   "source": "email"
  },
  "propertyAddress": {
   "value": "77 Mill Run Dr, Toledo, OH 43614",
   "confidence": 0.98,
   "source": "email"
  },
  "state": {
   "value": "OH",
   "confidence": 0.98,
   "source": "email"
  },
  "county": {
   "value": "Lucas",
   "confidence": 0.97,
   "source": "email"
  },
  "units": {
   "value": 20,
   "confidence": 0.97,
   "source": "email"
  },
  "tiv": {
   "value": null,
   "confidence": 0,
   "source": null
  },
  "constructionType": {
   "value": "Frame",
   "confidence": 0.7,
   "source": "email"
  },
  "yearBuilt": {
   "value": 2001,
   "confidence": 0.97,
   "source": "email"
  },
  "targetPremium": {
   "value": null,
   "confidence": 0,
   "source": null
  },
  "subclass": {
   "value": null,
   "confidence": 0,
   "source": null
  },
  "priorLossSummary": {
   "value": "Clean loss runs",
   "confidence": 0.85,
   "source": "email"
  }
 },
 "extracted": [
  {
   "field": "insuredName",
   "value": "Mill Run Flats LLC",
   "source": "email"
  },
  {
   "field": "propertyAddress",
   "value": "77 Mill Run Dr, Toledo, OH 43614",
   "source": "email"
  },
  {
   "field": "state",
   "value": "OH",
   "source": "email"
  },
  {
   "field": "county",
   "value": "Lucas",
   "source": "email"
  },
  {
   "field": "units",
   "value": 20,
   "source": "email"
  },
  {
   "field": "constructionType",
   "value": "Frame",
   "source": "email"
  },
  {
   "field": "yearBuilt",
   "value": 2001,
   "source": "email"
  },
  {
   "field": "priorLossSummary",
   "value": "Clean loss runs",
   "source": "email"
  }
 ],
 "fields": [
  {
   "field": "insuredName",
   "expected": "Mill Run Flats LLC",
   "got": "Mill Run Flats LLC",
   "correct": true
  },
  {
   "field": "state",
   "expected": "OH",
   "got": "OH",
   "correct": true
  },
  {
   "field": "county",
   "expected": "Lucas",
   "got": "Lucas",
   "correct": true
  },
  {
   "field": "units",
   "expected": 20,
   "got": 20,
   "correct": true
  },
  {
   "field": "tiv",
   "expected": null,
   "got": null,
   "correct": true
  },
  {
   "field": "yearBuilt",
   "expected": 2001,
   "got": 2001,
   "correct": true
  }
 ],
 "retries": 0,
 "guardrails": [
  {
   "node": "extraction",
   "guardrail": "schema",
   "status": "pass"
  },
  {
   "node": "grounding",
   "guardrail": "grounding",
   "status": "pass"
  },
  {
   "node": "appetite",
   "guardrail": "missing_fields",
   "status": "warn"
  },
  {
   "node": "appetite",
   "guardrail": "confidence",
   "status": "pass"
  }
 ],
 "flags": [
  {
   "field": "tiv",
   "issue": "missing",
   "detail": "TIV (total insured value) is required by the appetite ruleset"
  }
 ]
}

Trace line (JSON)

{
  "run_id": "2026-10-20-v6",
  "suite": "extraction-prompts",
  "case_id": "V-14",
  "variant": "v5",
  "at": "2026-10-21T01:10:15.524Z",
  "ok": true,
  "usage": {
    "inputTokens": 6,
    "outputTokens": 1097,
    "costUsd": 0.0115063,
    "calls": 3,
    "cacheReadTokens": 5243,
    "cacheCreationTokens": 0,
    "latencyMs": 14515,
    "metered": true,
    "provider": "claude-cli"
  },
  "checks": {
    "extraction_schema": true,
    "fields_correct": true,
    "nulls_stay_null": true,
    "decision_correct": true,
    "attribution_claude": true
  },
  "output_hash": "65a9b97537667b85",
  "unanswered": [
    "attribution_jev"
  ],
  "extra": {
    "decision": "INCOMPLETE",
    "fitScore": 74,
    "retries": 0,
    "fieldsWrong": 0,
    "modelErrors": ""
  },
  "stages": [
    {
      "name": "intake",
      "ok": true
    },
    {
      "name": "extraction",
      "ok": true
    },
    {
      "name": "grounding",
      "ok": true
    },
    {
      "name": "appetite",
      "ok": true
    }
  ],
  "cached": false
}

Full output record (JSON)

{
  "output": {
    "decision": "INCOMPLETE",
    "expectedDecision": "INCOMPLETE",
    "decisionCorrect": true,
    "fitScore": 74,
    "confidence": 0.93,
    "reasons": [
      "TIV (total insured value)"
    ],
    "extraction": {
      "insuredName": {
        "value": "Mill Run Flats LLC",
        "confidence": 0.98,
        "source": "email"
      },
      "propertyAddress": {
        "value": "77 Mill Run Dr, Toledo, OH 43614",
        "confidence": 0.98,
        "source": "email"
      },
      "state": {
        "value": "OH",
        "confidence": 0.98,
        "source": "email"
      },
      "county": {
        "value": "Lucas",
        "confidence": 0.97,
        "source": "email"
      },
      "units": {
        "value": 20,
        "confidence": 0.97,
        "source": "email"
      },
      "tiv": {
        "value": null,
        "confidence": 0,
        "source": null
      },
      "constructionType": {
        "value": "Frame",
        "confidence": 0.7,
        "source": "email"
      },
      "yearBuilt": {
        "value": 2001,
        "confidence": 0.97,
        "source": "email"
      },
      "targetPremium": {
        "value": null,
        "confidence": 0,
        "source": null
      },
      "subclass": {
        "value": null,
        "confidence": 0,
        "source": null
      },
      "priorLossSummary": {
        "value": "Clean loss runs",
        "confidence": 0.85,
        "source": "email"
      }
    },
    "extracted": [
      {
        "field": "insuredName",
        "value": "Mill Run Flats LLC",
        "source": "email"
      },
      {
        "field": "propertyAddress",
        "value": "77 Mill Run Dr, Toledo, OH 43614",
        "source": "email"
      },
      {
        "field": "state",
        "value": "OH",
        "source": "email"
      },
      {
        "field": "county",
        "value": "Lucas",
        "source": "email"
      },
      {
        "field": "units",
        "value": 20,
        "source": "email"
      },
      {
        "field": "constructionType",
        "value": "Frame",
        "source": "email"
      },
      {
        "field": "yearBuilt",
        "value": 2001,
        "source": "email"
      },
      {
        "field": "priorLossSummary",
        "value": "Clean loss runs",
        "source": "email"
      }
    ],
    "fields": [
      {
        "field": "insuredName",
        "expected": "Mill Run Flats LLC",
        "got": "Mill Run Flats LLC",
        "correct": true
      },
      {
        "field": "state",
        "expected": "OH",
        "got": "OH",
        "correct": true
      },
      {
        "field": "county",
        "expected": "Lucas",
        "got": "Lucas",
        "correct": true
      },
      {
        "field": "units",
        "expected": 20,
        "got": 20,
        "correct": true
      },
      {
        "field": "tiv",
        "expected": null,
        "got": null,
        "correct": true
      },
      {
        "field": "yearBuilt",
        "expected": 2001,
        "got": 2001,
        "correct": true
      }
    ],
    "retries": 0,
    "guardrails": [
      {
        "node": "extraction",
        "guardrail": "schema",
        "status": "pass"
      },
      {
        "node": "grounding",
        "guardrail": "grounding",
        "status": "pass"
      },
      {
        "node": "appetite",
        "guardrail": "missing_fields",
        "status": "warn"
      },
      {
        "node": "appetite",
        "guardrail": "confidence",
        "status": "pass"
      }
    ],
    "flags": [
      {
        "field": "tiv",
        "issue": "missing",
        "detail": "TIV (total insured value) is required by the appetite ruleset"
      }
    ]
  },
  "usage": {
    "inputTokens": 6,
    "cacheReadTokens": 5243,
    "cacheCreationTokens": 0,
    "outputTokens": 1097,
    "costUsd": 0.0115063,
    "latencyMs": 14515,
    "calls": 3,
    "metered": true,
    "provider": "claude-cli"
  },
  "extra": {
    "decision": "INCOMPLETE",
    "fitScore": 74,
    "retries": 0,
    "fieldsWrong": 0,
    "modelErrors": ""
  },
  "stages": [
    {
      "name": "intake",
      "ok": true
    },
    {
      "name": "extraction",
      "ok": true
    },
    {
      "name": "grounding",
      "ok": true
    },
    {
      "name": "appetite",
      "ok": true
    }
  ]
}

v6

result: ok
prompt tokens: 5,284
output tokens: 1,147
cost: $0.0147
latency: 13,740 ms · 3 calls
at: 2026-10-21 01:10
stages: intake extraction grounding appetite
decision: INCOMPLETE
fitScore: 72
retries: 0
fieldsWrong: 0
modelErrors

check	verdict
extraction_schema	pass
fields_correct	pass
construction_canonical	not applicable
nulls_stay_null	pass
decision_correct	pass
attribution_jev	unanswered
attribution_claude	pass

{
 "decision": "INCOMPLETE",
 "expectedDecision": "INCOMPLETE",
 "decisionCorrect": true,
 "fitScore": 72,
 "confidence": 0.9083333333333332,
 "reasons": [
  "TIV (total insured value)"
 ],
 "extraction": {
  "insuredName": {
   "value": "Mill Run Flats LLC",
   "confidence": 0.97,
   "source": "email"
  },
  "propertyAddress": {
   "value": "77 Mill Run Dr, Toledo, OH 43614",
   "confidence": 0.97,
   "source": "email"
  },
  "state": {
   "value": "OH",
   "confidence": 0.97,
   "source": "email"
  },
  "county": {
   "value": "Lucas",
   "confidence": 0.95,
   "source": "email"
  },
  "units": {
   "value": 20,
   "confidence": 0.97,
   "source": "email"
  },
  "tiv": {
   "value": null,
   "confidence": 0.95,
   "source": null
  },
  "constructionType": {
   "value": "Frame",
   "confidence": 0.6,
   "source": "email"
  },
  "yearBuilt": {
   "value": 2001,
   "confidence": 0.97,
   "source": "email"
  },
  "targetPremium": {
   "value": null,
   "confidence": 0.9,
   "source": null
  },
  "subclass": {
   "value": null,
   "confidence": 0.9,
   "source": null
  },
  "priorLossSummary": {
   "value": "Clean loss runs",
   "confidence": 0.8,
   "source": "email"
  }
 },
 "extracted": [
  {
   "field": "insuredName",
   "value": "Mill Run Flats LLC",
   "source": "email"
  },
  {
   "field": "propertyAddress",
   "value": "77 Mill Run Dr, Toledo, OH 43614",
   "source": "email"
  },
  {
   "field": "state",
   "value": "OH",
   "source": "email"
  },
  {
   "field": "county",
   "value": "Lucas",
   "source": "email"
  },
  {
   "field": "units",
   "value": 20,
   "source": "email"
  },
  {
   "field": "constructionType",
   "value": "Frame",
   "source": "email"
  },
  {
   "field": "yearBuilt",
   "value": 2001,
   "source": "email"
  },
  {
   "field": "priorLossSummary",
   "value": "Clean loss runs",
   "source": "email"
  }
 ],
 "fields": [
  {
   "field": "insuredName",
   "expected": "Mill Run Flats LLC",
   "got": "Mill Run Flats LLC",
   "correct": true
  },
  {
   "field": "state",
   "expected": "OH",
   "got": "OH",
   "correct": true
  },
  {
   "field": "county",
   "expected": "Lucas",
   "got": "Lucas",
   "correct": true
  },
  {
   "field": "units",
   "expected": 20,
   "got": 20,
   "correct": true
  },
  {
   "field": "tiv",
   "expected": null,
   "got": null,
   "correct": true
  },
  {
   "field": "yearBuilt",
   "expected": 2001,
   "got": 2001,
   "correct": true
  }
 ],
 "retries": 0,
 "guardrails": [
  {
   "node": "extraction",
   "guardrail": "schema",
   "status": "pass"
  },
  {
   "node": "grounding",
   "guardrail": "grounding",
   "status": "pass"
  },
  {
   "node": "appetite",
   "guardrail": "missing_fields",
   "status": "warn"
  },
  {
   "node": "appetite",
   "guardrail": "confidence",
   "status": "pass"
  }
 ],
 "flags": [
  {
   "field": "tiv",
   "issue": "missing",
   "detail": "TIV (total insured value) is required by the appetite ruleset"
  }
 ]
}

Trace line (JSON)

{
  "run_id": "2026-10-20-v6",
  "suite": "extraction-prompts",
  "case_id": "V-14",
  "variant": "v6",
  "at": "2026-10-21T01:10:11.345Z",
  "ok": true,
  "usage": {
    "inputTokens": 6,
    "outputTokens": 1147,
    "costUsd": 0.0147086,
    "calls": 3,
    "cacheReadTokens": 4586,
    "cacheCreationTokens": 692,
    "latencyMs": 13740,
    "metered": true,
    "provider": "claude-cli"
  },
  "checks": {
    "extraction_schema": true,
    "fields_correct": true,
    "nulls_stay_null": true,
    "decision_correct": true,
    "attribution_claude": true
  },
  "output_hash": "67dbe7bb70cbf06f",
  "unanswered": [
    "attribution_jev"
  ],
  "extra": {
    "decision": "INCOMPLETE",
    "fitScore": 72,
    "retries": 0,
    "fieldsWrong": 0,
    "modelErrors": ""
  },
  "stages": [
    {
      "name": "intake",
      "ok": true
    },
    {
      "name": "extraction",
      "ok": true
    },
    {
      "name": "grounding",
      "ok": true
    },
    {
      "name": "appetite",
      "ok": true
    }
  ],
  "cached": false
}

Full output record (JSON)

{
  "output": {
    "decision": "INCOMPLETE",
    "expectedDecision": "INCOMPLETE",
    "decisionCorrect": true,
    "fitScore": 72,
    "confidence": 0.9083333333333332,
    "reasons": [
      "TIV (total insured value)"
    ],
    "extraction": {
      "insuredName": {
        "value": "Mill Run Flats LLC",
        "confidence": 0.97,
        "source": "email"
      },
      "propertyAddress": {
        "value": "77 Mill Run Dr, Toledo, OH 43614",
        "confidence": 0.97,
        "source": "email"
      },
      "state": {
        "value": "OH",
        "confidence": 0.97,
        "source": "email"
      },
      "county": {
        "value": "Lucas",
        "confidence": 0.95,
        "source": "email"
      },
      "units": {
        "value": 20,
        "confidence": 0.97,
        "source": "email"
      },
      "tiv": {
        "value": null,
        "confidence": 0.95,
        "source": null
      },
      "constructionType": {
        "value": "Frame",
        "confidence": 0.6,
        "source": "email"
      },
      "yearBuilt": {
        "value": 2001,
        "confidence": 0.97,
        "source": "email"
      },
      "targetPremium": {
        "value": null,
        "confidence": 0.9,
        "source": null
      },
      "subclass": {
        "value": null,
        "confidence": 0.9,
        "source": null
      },
      "priorLossSummary": {
        "value": "Clean loss runs",
        "confidence": 0.8,
        "source": "email"
      }
    },
    "extracted": [
      {
        "field": "insuredName",
        "value": "Mill Run Flats LLC",
        "source": "email"
      },
      {
        "field": "propertyAddress",
        "value": "77 Mill Run Dr, Toledo, OH 43614",
        "source": "email"
      },
      {
        "field": "state",
        "value": "OH",
        "source": "email"
      },
      {
        "field": "county",
        "value": "Lucas",
        "source": "email"
      },
      {
        "field": "units",
        "value": 20,
        "source": "email"
      },
      {
        "field": "constructionType",
        "value": "Frame",
        "source": "email"
      },
      {
        "field": "yearBuilt",
        "value": 2001,
        "source": "email"
      },
      {
        "field": "priorLossSummary",
        "value": "Clean loss runs",
        "source": "email"
      }
    ],
    "fields": [
      {
        "field": "insuredName",
        "expected": "Mill Run Flats LLC",
        "got": "Mill Run Flats LLC",
        "correct": true
      },
      {
        "field": "state",
        "expected": "OH",
        "got": "OH",
        "correct": true
      },
      {
        "field": "county",
        "expected": "Lucas",
        "got": "Lucas",
        "correct": true
      },
      {
        "field": "units",
        "expected": 20,
        "got": 20,
        "correct": true
      },
      {
        "field": "tiv",
        "expected": null,
        "got": null,
        "correct": true
      },
      {
        "field": "yearBuilt",
        "expected": 2001,
        "got": 2001,
        "correct": true
      }
    ],
    "retries": 0,
    "guardrails": [
      {
        "node": "extraction",
        "guardrail": "schema",
        "status": "pass"
      },
      {
        "node": "grounding",
        "guardrail": "grounding",
        "status": "pass"
      },
      {
        "node": "appetite",
        "guardrail": "missing_fields",
        "status": "warn"
      },
      {
        "node": "appetite",
        "guardrail": "confidence",
        "status": "pass"
      }
    ],
    "flags": [
      {
        "field": "tiv",
        "issue": "missing",
        "detail": "TIV (total insured value) is required by the appetite ruleset"
      }
    ]
  },
  "usage": {
    "inputTokens": 6,
    "cacheReadTokens": 4586,
    "cacheCreationTokens": 692,
    "outputTokens": 1147,
    "costUsd": 0.0147086,
    "latencyMs": 13740,
    "calls": 3,
    "metered": true,
    "provider": "claude-cli"
  },
  "extra": {
    "decision": "INCOMPLETE",
    "fitScore": 72,
    "retries": 0,
    "fieldsWrong": 0,
    "modelErrors": ""
  },
  "stages": [
    {
      "name": "intake",
      "ok": true
    },
    {
      "name": "extraction",
      "ok": true
    },
    {
      "name": "grounding",
      "ok": true
    },
    {
      "name": "appetite",
      "ok": true
    }
  ]
}