Weigh evals you can watch
2026-11-12 06:38 UTC

V-08

extraction-prompts · run 2026-10-20-v6

What the broker wrote

What every version was given, as the suite chose to show it.

Broker: Variant desk

Good afternoon,

Please consider Sonora Vista Homes LLC — 22 doors of single-family rentals around Tucson, AZ (Pima County), professionally managed. All frame, typical build years 2005-2007, call it 2006.

Combined TIV $3,300,000, $28K target on premium. One liability claim, closed.

Best.

What each version answered

Compare each answer with the expert's answer. Highlighted cells differ from the expert; marked rows show where the versions disagree.

FieldExpectedCurrent prompt v3New prompt v6
insuredNameSonora Vista Homes LLCSonora Vista Homes LLCMatches the expertSonora Vista Homes LLCMatches the expert
stateAZAZMatches the expertAZMatches the expert
countyPimaPimaMatches the expertPimaMatches the expert
units2222Matches the expert22Matches the expert
tiv$3,300,000$3,300,000Matches the expert$3,300,000Matches the expert
constructionTypeFrameFrameMatches the expertFrameMatches the expert
yearBuilt20062006Matches the expert2006Matches the expert
targetPremium$28,000$28,000Matches the expert$28,000Matches the expert
subclassSingle-family rental portfolioSingle-family rental portfolioMatches the expertSingle-family rental portfolioMatches the expert
Decisionfollow up with the broker INCOMPLETEfollow up with the broker INCOMPLETEMatches the expertfollow up with the broker INCOMPLETEMatches the expert
For engineers

Checks are the suite's binary checks on each output; a judge that gave no verdict is unanswered and not counted.

v1

result
ok
prompt tokens
4,176
output tokens
832
cost
$0.0087
latency
11,660 ms · 3 calls
at
2026-10-21 01:07
stages
intake extraction grounding appetite
decision
INCOMPLETE
fitScore
86
retries
0
fieldsWrong
0
modelErrors
checkverdict
extraction_schemapass
fields_correctpass
construction_canonicalpass
nulls_stay_nullnot applicable
decision_correctpass
attribution_jevunanswered
attribution_claudepass
{
 "decision": "INCOMPLETE",
 "expectedDecision": "INCOMPLETE",
 "decisionCorrect": true,
 "fitScore": 86,
 "confidence": 0.8783333333333333,
 "reasons": [
  "property address"
 ],
 "extraction": {
  "insuredName": {
   "value": "Sonora Vista Homes LLC",
   "confidence": 0.98,
   "source": "email"
  },
  "propertyAddress": {
   "value": null,
   "confidence": 0.95,
   "source": null
  },
  "state": {
   "value": "AZ",
   "confidence": 0.97,
   "source": "email"
  },
  "county": {
   "value": "Pima",
   "confidence": 0.95,
   "source": "email"
  },
  "units": {
   "value": 22,
   "confidence": 0.9,
   "source": "email"
  },
  "tiv": {
   "value": 3300000,
   "confidence": 0.97,
   "source": "email"
  },
  "constructionType": {
   "value": "Frame",
   "confidence": 0.85,
   "source": "email"
  },
  "yearBuilt": {
   "value": 2006,
   "confidence": 0.6,
   "source": "email"
  },
  "targetPremium": {
   "value": 28000,
   "confidence": 0.95,
   "source": "email"
  },
  "subclass": {
   "value": "Single-family rental portfolio",
   "confidence": 0.95,
   "source": "email"
  },
  "priorLossSummary": {
   "value": "One liability claim, closed.",
   "confidence": 0.9,
   "source": "email"
  }
 },
 "extracted": [
  {
   "field": "insuredName",
   "value": "Sonora Vista Homes LLC",
   "source": "email"
  },
  {
   "field": "state",
   "value": "AZ",
   "source": "email"
  },
  {
   "field": "county",
   "value": "Pima",
   "source": "email"
  },
  {
   "field": "units",
   "value": 22,
   "source": "email"
  },
  {
   "field": "tiv",
   "value": 3300000,
   "source": "email"
  },
  {
   "field": "constructionType",
   "value": "Frame",
   "source": "email"
  },
  {
   "field": "yearBuilt",
   "value": 2006,
   "source": "email"
  },
  {
   "field": "targetPremium",
   "value": 28000,
   "source": "email"
  },
  {
   "field": "subclass",
   "value": "Single-family rental portfolio",
   "source": "email"
  },
  {
   "field": "priorLossSummary",
   "value": "One liability claim, closed.",
   "source": "email"
  }
 ],
 "fields": [
  {
   "field": "insuredName",
   "expected": "Sonora Vista Homes LLC",
   "got": "Sonora Vista Homes LLC",
   "correct": true
  },
  {
   "field": "state",
   "expected": "AZ",
   "got": "AZ",
   "correct": true
  },
  {
   "field": "county",
   "expected": "Pima",
   "got": "Pima",
   "correct": true
  },
  {
   "field": "units",
   "expected": 22,
   "got": 22,
   "correct": true
  },
  {
   "field": "tiv",
   "expected": 3300000,
   "got": 3300000,
   "correct": true
  },
  {
   "field": "constructionType",
   "expected": "Frame",
   "got": "Frame",
   "correct": true
  },
  {
   "field": "yearBuilt",
   "expected": 2006,
   "got": 2006,
   "correct": true
  },
  {
   "field": "targetPremium",
   "expected": 28000,
   "got": 28000,
   "correct": true
  },
  {
   "field": "subclass",
   "expected": "Single-family rental portfolio",
   "got": "Single-family rental portfolio",
   "correct": true
  }
 ],
 "retries": 0,
 "guardrails": [
  {
   "node": "extraction",
   "guardrail": "schema",
   "status": "pass"
  },
  {
   "node": "grounding",
   "guardrail": "grounding",
   "status": "pass"
  },
  {
   "node": "appetite",
   "guardrail": "missing_fields",
   "status": "warn"
  },
  {
   "node": "appetite",
   "guardrail": "confidence",
   "status": "pass"
  }
 ],
 "flags": [
  {
   "field": "propertyAddress",
   "issue": "missing",
   "detail": "property address is required by the appetite ruleset"
  }
 ]
}
Trace line (JSON)
{
  "run_id": "2026-10-20-v6",
  "suite": "extraction-prompts",
  "case_id": "V-08",
  "variant": "v1",
  "at": "2026-10-21T01:07:47.111Z",
  "ok": true,
  "usage": {
    "inputTokens": 6,
    "outputTokens": 832,
    "costUsd": 0.008749,
    "calls": 3,
    "cacheReadTokens": 4170,
    "cacheCreationTokens": 0,
    "latencyMs": 11660,
    "metered": true,
    "provider": "claude-cli"
  },
  "checks": {
    "extraction_schema": true,
    "fields_correct": true,
    "construction_canonical": true,
    "decision_correct": true,
    "attribution_claude": true
  },
  "output_hash": "e09e78f699e01ffb",
  "unanswered": [
    "attribution_jev"
  ],
  "extra": {
    "decision": "INCOMPLETE",
    "fitScore": 86,
    "retries": 0,
    "fieldsWrong": 0,
    "modelErrors": ""
  },
  "stages": [
    {
      "name": "intake",
      "ok": true
    },
    {
      "name": "extraction",
      "ok": true
    },
    {
      "name": "grounding",
      "ok": true
    },
    {
      "name": "appetite",
      "ok": true
    }
  ],
  "cached": false
}
Full output record (JSON)
{
  "output": {
    "decision": "INCOMPLETE",
    "expectedDecision": "INCOMPLETE",
    "decisionCorrect": true,
    "fitScore": 86,
    "confidence": 0.8783333333333333,
    "reasons": [
      "property address"
    ],
    "extraction": {
      "insuredName": {
        "value": "Sonora Vista Homes LLC",
        "confidence": 0.98,
        "source": "email"
      },
      "propertyAddress": {
        "value": null,
        "confidence": 0.95,
        "source": null
      },
      "state": {
        "value": "AZ",
        "confidence": 0.97,
        "source": "email"
      },
      "county": {
        "value": "Pima",
        "confidence": 0.95,
        "source": "email"
      },
      "units": {
        "value": 22,
        "confidence": 0.9,
        "source": "email"
      },
      "tiv": {
        "value": 3300000,
        "confidence": 0.97,
        "source": "email"
      },
      "constructionType": {
        "value": "Frame",
        "confidence": 0.85,
        "source": "email"
      },
      "yearBuilt": {
        "value": 2006,
        "confidence": 0.6,
        "source": "email"
      },
      "targetPremium": {
        "value": 28000,
        "confidence": 0.95,
        "source": "email"
      },
      "subclass": {
        "value": "Single-family rental portfolio",
        "confidence": 0.95,
        "source": "email"
      },
      "priorLossSummary": {
        "value": "One liability claim, closed.",
        "confidence": 0.9,
        "source": "email"
      }
    },
    "extracted": [
      {
        "field": "insuredName",
        "value": "Sonora Vista Homes LLC",
        "source": "email"
      },
      {
        "field": "state",
        "value": "AZ",
        "source": "email"
      },
      {
        "field": "county",
        "value": "Pima",
        "source": "email"
      },
      {
        "field": "units",
        "value": 22,
        "source": "email"
      },
      {
        "field": "tiv",
        "value": 3300000,
        "source": "email"
      },
      {
        "field": "constructionType",
        "value": "Frame",
        "source": "email"
      },
      {
        "field": "yearBuilt",
        "value": 2006,
        "source": "email"
      },
      {
        "field": "targetPremium",
        "value": 28000,
        "source": "email"
      },
      {
        "field": "subclass",
        "value": "Single-family rental portfolio",
        "source": "email"
      },
      {
        "field": "priorLossSummary",
        "value": "One liability claim, closed.",
        "source": "email"
      }
    ],
    "fields": [
      {
        "field": "insuredName",
        "expected": "Sonora Vista Homes LLC",
        "got": "Sonora Vista Homes LLC",
        "correct": true
      },
      {
        "field": "state",
        "expected": "AZ",
        "got": "AZ",
        "correct": true
      },
      {
        "field": "county",
        "expected": "Pima",
        "got": "Pima",
        "correct": true
      },
      {
        "field": "units",
        "expected": 22,
        "got": 22,
        "correct": true
      },
      {
        "field": "tiv",
        "expected": 3300000,
        "got": 3300000,
        "correct": true
      },
      {
        "field": "constructionType",
        "expected": "Frame",
        "got": "Frame",
        "correct": true
      },
      {
        "field": "yearBuilt",
        "expected": 2006,
        "got": 2006,
        "correct": true
      },
      {
        "field": "targetPremium",
        "expected": 28000,
        "got": 28000,
        "correct": true
      },
      {
        "field": "subclass",
        "expected": "Single-family rental portfolio",
        "got": "Single-family rental portfolio",
        "correct": true
      }
    ],
    "retries": 0,
    "guardrails": [
      {
        "node": "extraction",
        "guardrail": "schema",
        "status": "pass"
      },
      {
        "node": "grounding",
        "guardrail": "grounding",
        "status": "pass"
      },
      {
        "node": "appetite",
        "guardrail": "missing_fields",
        "status": "warn"
      },
      {
        "node": "appetite",
        "guardrail": "confidence",
        "status": "pass"
      }
    ],
    "flags": [
      {
        "field": "propertyAddress",
        "issue": "missing",
        "detail": "property address is required by the appetite ruleset"
      }
    ]
  },
  "usage": {
    "inputTokens": 6,
    "cacheReadTokens": 4170,
    "cacheCreationTokens": 0,
    "outputTokens": 832,
    "costUsd": 0.008749,
    "latencyMs": 11660,
    "calls": 3,
    "metered": true,
    "provider": "claude-cli"
  },
  "extra": {
    "decision": "INCOMPLETE",
    "fitScore": 86,
    "retries": 0,
    "fieldsWrong": 0,
    "modelErrors": ""
  },
  "stages": [
    {
      "name": "intake",
      "ok": true
    },
    {
      "name": "extraction",
      "ok": true
    },
    {
      "name": "grounding",
      "ok": true
    },
    {
      "name": "appetite",
      "ok": true
    }
  ]
}

v2

result
ok
prompt tokens
4,610
output tokens
844
cost
$0.0089
latency
11,331 ms · 3 calls
at
2026-10-21 01:07
stages
intake extraction grounding appetite
decision
INCOMPLETE
fitScore
86
retries
0
fieldsWrong
0
modelErrors
checkverdict
extraction_schemapass
fields_correctpass
construction_canonicalpass
nulls_stay_nullnot applicable
decision_correctpass
attribution_jevunanswered
attribution_claudepass
{
 "decision": "INCOMPLETE",
 "expectedDecision": "INCOMPLETE",
 "decisionCorrect": true,
 "fitScore": 86,
 "confidence": 0.9366666666666666,
 "reasons": [
  "property address"
 ],
 "extraction": {
  "insuredName": {
   "value": "Sonora Vista Homes LLC",
   "confidence": 0.98,
   "source": "email"
  },
  "propertyAddress": {
   "value": null,
   "confidence": 0.9,
   "source": null
  },
  "state": {
   "value": "AZ",
   "confidence": 0.97,
   "source": "email"
  },
  "county": {
   "value": "Pima",
   "confidence": 0.95,
   "source": "email"
  },
  "units": {
   "value": 22,
   "confidence": 0.9,
   "source": "email"
  },
  "tiv": {
   "value": 3300000,
   "confidence": 0.97,
   "source": "email"
  },
  "constructionType": {
   "value": "Frame",
   "confidence": 0.95,
   "source": "email"
  },
  "yearBuilt": {
   "value": 2006,
   "confidence": 0.85,
   "source": "email"
  },
  "targetPremium": {
   "value": 28000,
   "confidence": 0.95,
   "source": "email"
  },
  "subclass": {
   "value": "Single-family rental portfolio",
   "confidence": 0.95,
   "source": "email"
  },
  "priorLossSummary": {
   "value": "One liability claim, closed.",
   "confidence": 0.85,
   "source": "email"
  }
 },
 "extracted": [
  {
   "field": "insuredName",
   "value": "Sonora Vista Homes LLC",
   "source": "email"
  },
  {
   "field": "state",
   "value": "AZ",
   "source": "email"
  },
  {
   "field": "county",
   "value": "Pima",
   "source": "email"
  },
  {
   "field": "units",
   "value": 22,
   "source": "email"
  },
  {
   "field": "tiv",
   "value": 3300000,
   "source": "email"
  },
  {
   "field": "constructionType",
   "value": "Frame",
   "source": "email"
  },
  {
   "field": "yearBuilt",
   "value": 2006,
   "source": "email"
  },
  {
   "field": "targetPremium",
   "value": 28000,
   "source": "email"
  },
  {
   "field": "subclass",
   "value": "Single-family rental portfolio",
   "source": "email"
  },
  {
   "field": "priorLossSummary",
   "value": "One liability claim, closed.",
   "source": "email"
  }
 ],
 "fields": [
  {
   "field": "insuredName",
   "expected": "Sonora Vista Homes LLC",
   "got": "Sonora Vista Homes LLC",
   "correct": true
  },
  {
   "field": "state",
   "expected": "AZ",
   "got": "AZ",
   "correct": true
  },
  {
   "field": "county",
   "expected": "Pima",
   "got": "Pima",
   "correct": true
  },
  {
   "field": "units",
   "expected": 22,
   "got": 22,
   "correct": true
  },
  {
   "field": "tiv",
   "expected": 3300000,
   "got": 3300000,
   "correct": true
  },
  {
   "field": "constructionType",
   "expected": "Frame",
   "got": "Frame",
   "correct": true
  },
  {
   "field": "yearBuilt",
   "expected": 2006,
   "got": 2006,
   "correct": true
  },
  {
   "field": "targetPremium",
   "expected": 28000,
   "got": 28000,
   "correct": true
  },
  {
   "field": "subclass",
   "expected": "Single-family rental portfolio",
   "got": "Single-family rental portfolio",
   "correct": true
  }
 ],
 "retries": 0,
 "guardrails": [
  {
   "node": "extraction",
   "guardrail": "schema",
   "status": "pass"
  },
  {
   "node": "grounding",
   "guardrail": "grounding",
   "status": "pass"
  },
  {
   "node": "appetite",
   "guardrail": "missing_fields",
   "status": "warn"
  },
  {
   "node": "appetite",
   "guardrail": "confidence",
   "status": "pass"
  }
 ],
 "flags": [
  {
   "field": "propertyAddress",
   "issue": "missing",
   "detail": "property address is required by the appetite ruleset"
  }
 ]
}
Trace line (JSON)
{
  "run_id": "2026-10-20-v6",
  "suite": "extraction-prompts",
  "case_id": "V-08",
  "variant": "v2",
  "at": "2026-10-21T01:07:24.352Z",
  "ok": true,
  "usage": {
    "inputTokens": 6,
    "outputTokens": 844,
    "costUsd": 0.008912400000000001,
    "calls": 3,
    "cacheReadTokens": 4604,
    "cacheCreationTokens": 0,
    "latencyMs": 11331,
    "metered": true,
    "provider": "claude-cli"
  },
  "checks": {
    "extraction_schema": true,
    "fields_correct": true,
    "construction_canonical": true,
    "decision_correct": true,
    "attribution_claude": true
  },
  "output_hash": "f6353e4215ccf9f2",
  "unanswered": [
    "attribution_jev"
  ],
  "extra": {
    "decision": "INCOMPLETE",
    "fitScore": 86,
    "retries": 0,
    "fieldsWrong": 0,
    "modelErrors": ""
  },
  "stages": [
    {
      "name": "intake",
      "ok": true
    },
    {
      "name": "extraction",
      "ok": true
    },
    {
      "name": "grounding",
      "ok": true
    },
    {
      "name": "appetite",
      "ok": true
    }
  ],
  "cached": false
}
Full output record (JSON)
{
  "output": {
    "decision": "INCOMPLETE",
    "expectedDecision": "INCOMPLETE",
    "decisionCorrect": true,
    "fitScore": 86,
    "confidence": 0.9366666666666666,
    "reasons": [
      "property address"
    ],
    "extraction": {
      "insuredName": {
        "value": "Sonora Vista Homes LLC",
        "confidence": 0.98,
        "source": "email"
      },
      "propertyAddress": {
        "value": null,
        "confidence": 0.9,
        "source": null
      },
      "state": {
        "value": "AZ",
        "confidence": 0.97,
        "source": "email"
      },
      "county": {
        "value": "Pima",
        "confidence": 0.95,
        "source": "email"
      },
      "units": {
        "value": 22,
        "confidence": 0.9,
        "source": "email"
      },
      "tiv": {
        "value": 3300000,
        "confidence": 0.97,
        "source": "email"
      },
      "constructionType": {
        "value": "Frame",
        "confidence": 0.95,
        "source": "email"
      },
      "yearBuilt": {
        "value": 2006,
        "confidence": 0.85,
        "source": "email"
      },
      "targetPremium": {
        "value": 28000,
        "confidence": 0.95,
        "source": "email"
      },
      "subclass": {
        "value": "Single-family rental portfolio",
        "confidence": 0.95,
        "source": "email"
      },
      "priorLossSummary": {
        "value": "One liability claim, closed.",
        "confidence": 0.85,
        "source": "email"
      }
    },
    "extracted": [
      {
        "field": "insuredName",
        "value": "Sonora Vista Homes LLC",
        "source": "email"
      },
      {
        "field": "state",
        "value": "AZ",
        "source": "email"
      },
      {
        "field": "county",
        "value": "Pima",
        "source": "email"
      },
      {
        "field": "units",
        "value": 22,
        "source": "email"
      },
      {
        "field": "tiv",
        "value": 3300000,
        "source": "email"
      },
      {
        "field": "constructionType",
        "value": "Frame",
        "source": "email"
      },
      {
        "field": "yearBuilt",
        "value": 2006,
        "source": "email"
      },
      {
        "field": "targetPremium",
        "value": 28000,
        "source": "email"
      },
      {
        "field": "subclass",
        "value": "Single-family rental portfolio",
        "source": "email"
      },
      {
        "field": "priorLossSummary",
        "value": "One liability claim, closed.",
        "source": "email"
      }
    ],
    "fields": [
      {
        "field": "insuredName",
        "expected": "Sonora Vista Homes LLC",
        "got": "Sonora Vista Homes LLC",
        "correct": true
      },
      {
        "field": "state",
        "expected": "AZ",
        "got": "AZ",
        "correct": true
      },
      {
        "field": "county",
        "expected": "Pima",
        "got": "Pima",
        "correct": true
      },
      {
        "field": "units",
        "expected": 22,
        "got": 22,
        "correct": true
      },
      {
        "field": "tiv",
        "expected": 3300000,
        "got": 3300000,
        "correct": true
      },
      {
        "field": "constructionType",
        "expected": "Frame",
        "got": "Frame",
        "correct": true
      },
      {
        "field": "yearBuilt",
        "expected": 2006,
        "got": 2006,
        "correct": true
      },
      {
        "field": "targetPremium",
        "expected": 28000,
        "got": 28000,
        "correct": true
      },
      {
        "field": "subclass",
        "expected": "Single-family rental portfolio",
        "got": "Single-family rental portfolio",
        "correct": true
      }
    ],
    "retries": 0,
    "guardrails": [
      {
        "node": "extraction",
        "guardrail": "schema",
        "status": "pass"
      },
      {
        "node": "grounding",
        "guardrail": "grounding",
        "status": "pass"
      },
      {
        "node": "appetite",
        "guardrail": "missing_fields",
        "status": "warn"
      },
      {
        "node": "appetite",
        "guardrail": "confidence",
        "status": "pass"
      }
    ],
    "flags": [
      {
        "field": "propertyAddress",
        "issue": "missing",
        "detail": "property address is required by the appetite ruleset"
      }
    ]
  },
  "usage": {
    "inputTokens": 6,
    "cacheReadTokens": 4604,
    "cacheCreationTokens": 0,
    "outputTokens": 844,
    "costUsd": 0.008912400000000001,
    "latencyMs": 11331,
    "calls": 3,
    "metered": true,
    "provider": "claude-cli"
  },
  "extra": {
    "decision": "INCOMPLETE",
    "fitScore": 86,
    "retries": 0,
    "fieldsWrong": 0,
    "modelErrors": ""
  },
  "stages": [
    {
      "name": "intake",
      "ok": true
    },
    {
      "name": "extraction",
      "ok": true
    },
    {
      "name": "grounding",
      "ok": true
    },
    {
      "name": "appetite",
      "ok": true
    }
  ]
}

v3

result
ok
prompt tokens
4,924
output tokens
845
cost
$0.0090
latency
12,677 ms · 3 calls
at
2026-10-21 01:07
stages
intake extraction grounding appetite
decision
INCOMPLETE
fitScore
86
retries
0
fieldsWrong
0
modelErrors
checkverdict
extraction_schemapass
fields_correctpass
construction_canonicalpass
nulls_stay_nullnot applicable
decision_correctpass
attribution_jevunanswered
attribution_claudepass
{
 "decision": "INCOMPLETE",
 "expectedDecision": "INCOMPLETE",
 "decisionCorrect": true,
 "fitScore": 86,
 "confidence": 0.9249999999999999,
 "reasons": [
  "property address"
 ],
 "extraction": {
  "insuredName": {
   "value": "Sonora Vista Homes LLC",
   "confidence": 0.98,
   "source": "email"
  },
  "propertyAddress": {
   "value": null,
   "confidence": 1,
   "source": null
  },
  "state": {
   "value": "AZ",
   "confidence": 0.97,
   "source": "email"
  },
  "county": {
   "value": "Pima",
   "confidence": 0.97,
   "source": "email"
  },
  "units": {
   "value": 22,
   "confidence": 0.9,
   "source": "email"
  },
  "tiv": {
   "value": 3300000,
   "confidence": 0.97,
   "source": "email"
  },
  "constructionType": {
   "value": "Frame",
   "confidence": 0.93,
   "source": "email"
  },
  "yearBuilt": {
   "value": 2006,
   "confidence": 0.8,
   "source": "email"
  },
  "targetPremium": {
   "value": 28000,
   "confidence": 0.95,
   "source": "email"
  },
  "subclass": {
   "value": "Single-family rental portfolio",
   "confidence": 0.93,
   "source": "email"
  },
  "priorLossSummary": {
   "value": "One liability claim, closed.",
   "confidence": 0.9,
   "source": "email"
  }
 },
 "extracted": [
  {
   "field": "insuredName",
   "value": "Sonora Vista Homes LLC",
   "source": "email"
  },
  {
   "field": "state",
   "value": "AZ",
   "source": "email"
  },
  {
   "field": "county",
   "value": "Pima",
   "source": "email"
  },
  {
   "field": "units",
   "value": 22,
   "source": "email"
  },
  {
   "field": "tiv",
   "value": 3300000,
   "source": "email"
  },
  {
   "field": "constructionType",
   "value": "Frame",
   "source": "email"
  },
  {
   "field": "yearBuilt",
   "value": 2006,
   "source": "email"
  },
  {
   "field": "targetPremium",
   "value": 28000,
   "source": "email"
  },
  {
   "field": "subclass",
   "value": "Single-family rental portfolio",
   "source": "email"
  },
  {
   "field": "priorLossSummary",
   "value": "One liability claim, closed.",
   "source": "email"
  }
 ],
 "fields": [
  {
   "field": "insuredName",
   "expected": "Sonora Vista Homes LLC",
   "got": "Sonora Vista Homes LLC",
   "correct": true
  },
  {
   "field": "state",
   "expected": "AZ",
   "got": "AZ",
   "correct": true
  },
  {
   "field": "county",
   "expected": "Pima",
   "got": "Pima",
   "correct": true
  },
  {
   "field": "units",
   "expected": 22,
   "got": 22,
   "correct": true
  },
  {
   "field": "tiv",
   "expected": 3300000,
   "got": 3300000,
   "correct": true
  },
  {
   "field": "constructionType",
   "expected": "Frame",
   "got": "Frame",
   "correct": true
  },
  {
   "field": "yearBuilt",
   "expected": 2006,
   "got": 2006,
   "correct": true
  },
  {
   "field": "targetPremium",
   "expected": 28000,
   "got": 28000,
   "correct": true
  },
  {
   "field": "subclass",
   "expected": "Single-family rental portfolio",
   "got": "Single-family rental portfolio",
   "correct": true
  }
 ],
 "retries": 0,
 "guardrails": [
  {
   "node": "extraction",
   "guardrail": "schema",
   "status": "pass"
  },
  {
   "node": "grounding",
   "guardrail": "grounding",
   "status": "pass"
  },
  {
   "node": "appetite",
   "guardrail": "missing_fields",
   "status": "warn"
  },
  {
   "node": "appetite",
   "guardrail": "confidence",
   "status": "pass"
  }
 ],
 "flags": [
  {
   "field": "propertyAddress",
   "issue": "missing",
   "detail": "property address is required by the appetite ruleset"
  }
 ]
}
Trace line (JSON)
{
  "run_id": "2026-10-20-v6",
  "suite": "extraction-prompts",
  "case_id": "V-08",
  "variant": "v3",
  "at": "2026-10-21T01:07:35.446Z",
  "ok": true,
  "usage": {
    "inputTokens": 6,
    "outputTokens": 845,
    "costUsd": 0.0089538,
    "calls": 3,
    "cacheReadTokens": 4918,
    "cacheCreationTokens": 0,
    "latencyMs": 12677,
    "metered": true,
    "provider": "claude-cli"
  },
  "checks": {
    "extraction_schema": true,
    "fields_correct": true,
    "construction_canonical": true,
    "decision_correct": true,
    "attribution_claude": true
  },
  "output_hash": "d0d4b1c882c02ca6",
  "unanswered": [
    "attribution_jev"
  ],
  "extra": {
    "decision": "INCOMPLETE",
    "fitScore": 86,
    "retries": 0,
    "fieldsWrong": 0,
    "modelErrors": ""
  },
  "stages": [
    {
      "name": "intake",
      "ok": true
    },
    {
      "name": "extraction",
      "ok": true
    },
    {
      "name": "grounding",
      "ok": true
    },
    {
      "name": "appetite",
      "ok": true
    }
  ],
  "cached": false
}
Full output record (JSON)
{
  "output": {
    "decision": "INCOMPLETE",
    "expectedDecision": "INCOMPLETE",
    "decisionCorrect": true,
    "fitScore": 86,
    "confidence": 0.9249999999999999,
    "reasons": [
      "property address"
    ],
    "extraction": {
      "insuredName": {
        "value": "Sonora Vista Homes LLC",
        "confidence": 0.98,
        "source": "email"
      },
      "propertyAddress": {
        "value": null,
        "confidence": 1,
        "source": null
      },
      "state": {
        "value": "AZ",
        "confidence": 0.97,
        "source": "email"
      },
      "county": {
        "value": "Pima",
        "confidence": 0.97,
        "source": "email"
      },
      "units": {
        "value": 22,
        "confidence": 0.9,
        "source": "email"
      },
      "tiv": {
        "value": 3300000,
        "confidence": 0.97,
        "source": "email"
      },
      "constructionType": {
        "value": "Frame",
        "confidence": 0.93,
        "source": "email"
      },
      "yearBuilt": {
        "value": 2006,
        "confidence": 0.8,
        "source": "email"
      },
      "targetPremium": {
        "value": 28000,
        "confidence": 0.95,
        "source": "email"
      },
      "subclass": {
        "value": "Single-family rental portfolio",
        "confidence": 0.93,
        "source": "email"
      },
      "priorLossSummary": {
        "value": "One liability claim, closed.",
        "confidence": 0.9,
        "source": "email"
      }
    },
    "extracted": [
      {
        "field": "insuredName",
        "value": "Sonora Vista Homes LLC",
        "source": "email"
      },
      {
        "field": "state",
        "value": "AZ",
        "source": "email"
      },
      {
        "field": "county",
        "value": "Pima",
        "source": "email"
      },
      {
        "field": "units",
        "value": 22,
        "source": "email"
      },
      {
        "field": "tiv",
        "value": 3300000,
        "source": "email"
      },
      {
        "field": "constructionType",
        "value": "Frame",
        "source": "email"
      },
      {
        "field": "yearBuilt",
        "value": 2006,
        "source": "email"
      },
      {
        "field": "targetPremium",
        "value": 28000,
        "source": "email"
      },
      {
        "field": "subclass",
        "value": "Single-family rental portfolio",
        "source": "email"
      },
      {
        "field": "priorLossSummary",
        "value": "One liability claim, closed.",
        "source": "email"
      }
    ],
    "fields": [
      {
        "field": "insuredName",
        "expected": "Sonora Vista Homes LLC",
        "got": "Sonora Vista Homes LLC",
        "correct": true
      },
      {
        "field": "state",
        "expected": "AZ",
        "got": "AZ",
        "correct": true
      },
      {
        "field": "county",
        "expected": "Pima",
        "got": "Pima",
        "correct": true
      },
      {
        "field": "units",
        "expected": 22,
        "got": 22,
        "correct": true
      },
      {
        "field": "tiv",
        "expected": 3300000,
        "got": 3300000,
        "correct": true
      },
      {
        "field": "constructionType",
        "expected": "Frame",
        "got": "Frame",
        "correct": true
      },
      {
        "field": "yearBuilt",
        "expected": 2006,
        "got": 2006,
        "correct": true
      },
      {
        "field": "targetPremium",
        "expected": 28000,
        "got": 28000,
        "correct": true
      },
      {
        "field": "subclass",
        "expected": "Single-family rental portfolio",
        "got": "Single-family rental portfolio",
        "correct": true
      }
    ],
    "retries": 0,
    "guardrails": [
      {
        "node": "extraction",
        "guardrail": "schema",
        "status": "pass"
      },
      {
        "node": "grounding",
        "guardrail": "grounding",
        "status": "pass"
      },
      {
        "node": "appetite",
        "guardrail": "missing_fields",
        "status": "warn"
      },
      {
        "node": "appetite",
        "guardrail": "confidence",
        "status": "pass"
      }
    ],
    "flags": [
      {
        "field": "propertyAddress",
        "issue": "missing",
        "detail": "property address is required by the appetite ruleset"
      }
    ]
  },
  "usage": {
    "inputTokens": 6,
    "cacheReadTokens": 4918,
    "cacheCreationTokens": 0,
    "outputTokens": 845,
    "costUsd": 0.0089538,
    "latencyMs": 12677,
    "calls": 3,
    "metered": true,
    "provider": "claude-cli"
  },
  "extra": {
    "decision": "INCOMPLETE",
    "fitScore": 86,
    "retries": 0,
    "fieldsWrong": 0,
    "modelErrors": ""
  },
  "stages": [
    {
      "name": "intake",
      "ok": true
    },
    {
      "name": "extraction",
      "ok": true
    },
    {
      "name": "grounding",
      "ok": true
    },
    {
      "name": "appetite",
      "ok": true
    }
  ]
}

v3-repeat

result
ok
prompt tokens
4,924
output tokens
837
cost
$0.0089
latency
11,375 ms · 3 calls
at
2026-10-21 01:07
stages
intake extraction grounding appetite
decision
INCOMPLETE
fitScore
88
retries
0
fieldsWrong
0
modelErrors
checkverdict
extraction_schemapass
fields_correctpass
construction_canonicalpass
nulls_stay_nullnot applicable
decision_correctpass
attribution_jevunanswered
attribution_claudepass
{
 "decision": "INCOMPLETE",
 "expectedDecision": "INCOMPLETE",
 "decisionCorrect": true,
 "fitScore": 88,
 "confidence": 0.9283333333333333,
 "reasons": [
  "property address"
 ],
 "extraction": {
  "insuredName": {
   "value": "Sonora Vista Homes LLC",
   "confidence": 0.98,
   "source": "email"
  },
  "propertyAddress": {
   "value": null,
   "confidence": 1,
   "source": null
  },
  "state": {
   "value": "AZ",
   "confidence": 0.97,
   "source": "email"
  },
  "county": {
   "value": "Pima",
   "confidence": 0.97,
   "source": "email"
  },
  "units": {
   "value": 22,
   "confidence": 0.9,
   "source": "email"
  },
  "tiv": {
   "value": 3300000,
   "confidence": 0.97,
   "source": "email"
  },
  "constructionType": {
   "value": "Frame",
   "confidence": 0.95,
   "source": "email"
  },
  "yearBuilt": {
   "value": 2006,
   "confidence": 0.8,
   "source": "email"
  },
  "targetPremium": {
   "value": 28000,
   "confidence": 0.95,
   "source": "email"
  },
  "subclass": {
   "value": "Single-family rental portfolio",
   "confidence": 0.93,
   "source": "email"
  },
  "priorLossSummary": {
   "value": "One liability claim, closed.",
   "confidence": 0.9,
   "source": "email"
  }
 },
 "extracted": [
  {
   "field": "insuredName",
   "value": "Sonora Vista Homes LLC",
   "source": "email"
  },
  {
   "field": "state",
   "value": "AZ",
   "source": "email"
  },
  {
   "field": "county",
   "value": "Pima",
   "source": "email"
  },
  {
   "field": "units",
   "value": 22,
   "source": "email"
  },
  {
   "field": "tiv",
   "value": 3300000,
   "source": "email"
  },
  {
   "field": "constructionType",
   "value": "Frame",
   "source": "email"
  },
  {
   "field": "yearBuilt",
   "value": 2006,
   "source": "email"
  },
  {
   "field": "targetPremium",
   "value": 28000,
   "source": "email"
  },
  {
   "field": "subclass",
   "value": "Single-family rental portfolio",
   "source": "email"
  },
  {
   "field": "priorLossSummary",
   "value": "One liability claim, closed.",
   "source": "email"
  }
 ],
 "fields": [
  {
   "field": "insuredName",
   "expected": "Sonora Vista Homes LLC",
   "got": "Sonora Vista Homes LLC",
   "correct": true
  },
  {
   "field": "state",
   "expected": "AZ",
   "got": "AZ",
   "correct": true
  },
  {
   "field": "county",
   "expected": "Pima",
   "got": "Pima",
   "correct": true
  },
  {
   "field": "units",
   "expected": 22,
   "got": 22,
   "correct": true
  },
  {
   "field": "tiv",
   "expected": 3300000,
   "got": 3300000,
   "correct": true
  },
  {
   "field": "constructionType",
   "expected": "Frame",
   "got": "Frame",
   "correct": true
  },
  {
   "field": "yearBuilt",
   "expected": 2006,
   "got": 2006,
   "correct": true
  },
  {
   "field": "targetPremium",
   "expected": 28000,
   "got": 28000,
   "correct": true
  },
  {
   "field": "subclass",
   "expected": "Single-family rental portfolio",
   "got": "Single-family rental portfolio",
   "correct": true
  }
 ],
 "retries": 0,
 "guardrails": [
  {
   "node": "extraction",
   "guardrail": "schema",
   "status": "pass"
  },
  {
   "node": "grounding",
   "guardrail": "grounding",
   "status": "pass"
  },
  {
   "node": "appetite",
   "guardrail": "missing_fields",
   "status": "warn"
  },
  {
   "node": "appetite",
   "guardrail": "confidence",
   "status": "pass"
  }
 ],
 "flags": [
  {
   "field": "propertyAddress",
   "issue": "missing",
   "detail": "property address is required by the appetite ruleset"
  }
 ]
}
Trace line (JSON)
{
  "run_id": "2026-10-20-v6",
  "suite": "extraction-prompts",
  "case_id": "V-08",
  "variant": "v3-repeat",
  "at": "2026-10-21T01:07:34.770Z",
  "ok": true,
  "usage": {
    "inputTokens": 6,
    "outputTokens": 837,
    "costUsd": 0.008873800000000001,
    "calls": 3,
    "cacheReadTokens": 4918,
    "cacheCreationTokens": 0,
    "latencyMs": 11375,
    "metered": true,
    "provider": "claude-cli"
  },
  "checks": {
    "extraction_schema": true,
    "fields_correct": true,
    "construction_canonical": true,
    "decision_correct": true,
    "attribution_claude": true
  },
  "output_hash": "9fcc0f2025f20ad7",
  "unanswered": [
    "attribution_jev"
  ],
  "extra": {
    "decision": "INCOMPLETE",
    "fitScore": 88,
    "retries": 0,
    "fieldsWrong": 0,
    "modelErrors": ""
  },
  "stages": [
    {
      "name": "intake",
      "ok": true
    },
    {
      "name": "extraction",
      "ok": true
    },
    {
      "name": "grounding",
      "ok": true
    },
    {
      "name": "appetite",
      "ok": true
    }
  ],
  "cached": false
}
Full output record (JSON)
{
  "output": {
    "decision": "INCOMPLETE",
    "expectedDecision": "INCOMPLETE",
    "decisionCorrect": true,
    "fitScore": 88,
    "confidence": 0.9283333333333333,
    "reasons": [
      "property address"
    ],
    "extraction": {
      "insuredName": {
        "value": "Sonora Vista Homes LLC",
        "confidence": 0.98,
        "source": "email"
      },
      "propertyAddress": {
        "value": null,
        "confidence": 1,
        "source": null
      },
      "state": {
        "value": "AZ",
        "confidence": 0.97,
        "source": "email"
      },
      "county": {
        "value": "Pima",
        "confidence": 0.97,
        "source": "email"
      },
      "units": {
        "value": 22,
        "confidence": 0.9,
        "source": "email"
      },
      "tiv": {
        "value": 3300000,
        "confidence": 0.97,
        "source": "email"
      },
      "constructionType": {
        "value": "Frame",
        "confidence": 0.95,
        "source": "email"
      },
      "yearBuilt": {
        "value": 2006,
        "confidence": 0.8,
        "source": "email"
      },
      "targetPremium": {
        "value": 28000,
        "confidence": 0.95,
        "source": "email"
      },
      "subclass": {
        "value": "Single-family rental portfolio",
        "confidence": 0.93,
        "source": "email"
      },
      "priorLossSummary": {
        "value": "One liability claim, closed.",
        "confidence": 0.9,
        "source": "email"
      }
    },
    "extracted": [
      {
        "field": "insuredName",
        "value": "Sonora Vista Homes LLC",
        "source": "email"
      },
      {
        "field": "state",
        "value": "AZ",
        "source": "email"
      },
      {
        "field": "county",
        "value": "Pima",
        "source": "email"
      },
      {
        "field": "units",
        "value": 22,
        "source": "email"
      },
      {
        "field": "tiv",
        "value": 3300000,
        "source": "email"
      },
      {
        "field": "constructionType",
        "value": "Frame",
        "source": "email"
      },
      {
        "field": "yearBuilt",
        "value": 2006,
        "source": "email"
      },
      {
        "field": "targetPremium",
        "value": 28000,
        "source": "email"
      },
      {
        "field": "subclass",
        "value": "Single-family rental portfolio",
        "source": "email"
      },
      {
        "field": "priorLossSummary",
        "value": "One liability claim, closed.",
        "source": "email"
      }
    ],
    "fields": [
      {
        "field": "insuredName",
        "expected": "Sonora Vista Homes LLC",
        "got": "Sonora Vista Homes LLC",
        "correct": true
      },
      {
        "field": "state",
        "expected": "AZ",
        "got": "AZ",
        "correct": true
      },
      {
        "field": "county",
        "expected": "Pima",
        "got": "Pima",
        "correct": true
      },
      {
        "field": "units",
        "expected": 22,
        "got": 22,
        "correct": true
      },
      {
        "field": "tiv",
        "expected": 3300000,
        "got": 3300000,
        "correct": true
      },
      {
        "field": "constructionType",
        "expected": "Frame",
        "got": "Frame",
        "correct": true
      },
      {
        "field": "yearBuilt",
        "expected": 2006,
        "got": 2006,
        "correct": true
      },
      {
        "field": "targetPremium",
        "expected": 28000,
        "got": 28000,
        "correct": true
      },
      {
        "field": "subclass",
        "expected": "Single-family rental portfolio",
        "got": "Single-family rental portfolio",
        "correct": true
      }
    ],
    "retries": 0,
    "guardrails": [
      {
        "node": "extraction",
        "guardrail": "schema",
        "status": "pass"
      },
      {
        "node": "grounding",
        "guardrail": "grounding",
        "status": "pass"
      },
      {
        "node": "appetite",
        "guardrail": "missing_fields",
        "status": "warn"
      },
      {
        "node": "appetite",
        "guardrail": "confidence",
        "status": "pass"
      }
    ],
    "flags": [
      {
        "field": "propertyAddress",
        "issue": "missing",
        "detail": "property address is required by the appetite ruleset"
      }
    ]
  },
  "usage": {
    "inputTokens": 6,
    "cacheReadTokens": 4918,
    "cacheCreationTokens": 0,
    "outputTokens": 837,
    "costUsd": 0.008873800000000001,
    "latencyMs": 11375,
    "calls": 3,
    "metered": true,
    "provider": "claude-cli"
  },
  "extra": {
    "decision": "INCOMPLETE",
    "fitScore": 88,
    "retries": 0,
    "fieldsWrong": 0,
    "modelErrors": ""
  },
  "stages": [
    {
      "name": "intake",
      "ok": true
    },
    {
      "name": "extraction",
      "ok": true
    },
    {
      "name": "grounding",
      "ok": true
    },
    {
      "name": "appetite",
      "ok": true
    }
  ]
}

v4

result
ok
prompt tokens
5,133
output tokens
876
cost
$0.0093
latency
12,165 ms · 3 calls
at
2026-10-21 01:07
stages
intake extraction grounding appetite
decision
INCOMPLETE
fitScore
86
retries
0
fieldsWrong
0
modelErrors
checkverdict
extraction_schemapass
fields_correctpass
construction_canonicalpass
nulls_stay_nullnot applicable
decision_correctpass
attribution_jevunanswered
attribution_claudepass
{
 "decision": "INCOMPLETE",
 "expectedDecision": "INCOMPLETE",
 "decisionCorrect": true,
 "fitScore": 86,
 "confidence": 0.9333333333333332,
 "reasons": [
  "property address"
 ],
 "extraction": {
  "insuredName": {
   "value": "Sonora Vista Homes LLC",
   "confidence": 0.98,
   "source": "email"
  },
  "propertyAddress": {
   "value": null,
   "confidence": 0,
   "source": null
  },
  "state": {
   "value": "AZ",
   "confidence": 0.97,
   "source": "email"
  },
  "county": {
   "value": "Pima",
   "confidence": 0.95,
   "source": "email"
  },
  "units": {
   "value": 22,
   "confidence": 0.95,
   "source": "email"
  },
  "tiv": {
   "value": 3300000,
   "confidence": 0.97,
   "source": "email"
  },
  "constructionType": {
   "value": "Frame",
   "confidence": 0.93,
   "source": "email"
  },
  "yearBuilt": {
   "value": 2006,
   "confidence": 0.8,
   "source": "email"
  },
  "targetPremium": {
   "value": 28000,
   "confidence": 0.95,
   "source": "email"
  },
  "subclass": {
   "value": "Single-family rental portfolio",
   "confidence": 0.93,
   "source": "email"
  },
  "priorLossSummary": {
   "value": "One liability claim, closed.",
   "confidence": 0.9,
   "source": "email"
  }
 },
 "extracted": [
  {
   "field": "insuredName",
   "value": "Sonora Vista Homes LLC",
   "source": "email"
  },
  {
   "field": "state",
   "value": "AZ",
   "source": "email"
  },
  {
   "field": "county",
   "value": "Pima",
   "source": "email"
  },
  {
   "field": "units",
   "value": 22,
   "source": "email"
  },
  {
   "field": "tiv",
   "value": 3300000,
   "source": "email"
  },
  {
   "field": "constructionType",
   "value": "Frame",
   "source": "email"
  },
  {
   "field": "yearBuilt",
   "value": 2006,
   "source": "email"
  },
  {
   "field": "targetPremium",
   "value": 28000,
   "source": "email"
  },
  {
   "field": "subclass",
   "value": "Single-family rental portfolio",
   "source": "email"
  },
  {
   "field": "priorLossSummary",
   "value": "One liability claim, closed.",
   "source": "email"
  }
 ],
 "fields": [
  {
   "field": "insuredName",
   "expected": "Sonora Vista Homes LLC",
   "got": "Sonora Vista Homes LLC",
   "correct": true
  },
  {
   "field": "state",
   "expected": "AZ",
   "got": "AZ",
   "correct": true
  },
  {
   "field": "county",
   "expected": "Pima",
   "got": "Pima",
   "correct": true
  },
  {
   "field": "units",
   "expected": 22,
   "got": 22,
   "correct": true
  },
  {
   "field": "tiv",
   "expected": 3300000,
   "got": 3300000,
   "correct": true
  },
  {
   "field": "constructionType",
   "expected": "Frame",
   "got": "Frame",
   "correct": true
  },
  {
   "field": "yearBuilt",
   "expected": 2006,
   "got": 2006,
   "correct": true
  },
  {
   "field": "targetPremium",
   "expected": 28000,
   "got": 28000,
   "correct": true
  },
  {
   "field": "subclass",
   "expected": "Single-family rental portfolio",
   "got": "Single-family rental portfolio",
   "correct": true
  }
 ],
 "retries": 0,
 "guardrails": [
  {
   "node": "extraction",
   "guardrail": "schema",
   "status": "pass"
  },
  {
   "node": "grounding",
   "guardrail": "grounding",
   "status": "pass"
  },
  {
   "node": "appetite",
   "guardrail": "missing_fields",
   "status": "warn"
  },
  {
   "node": "appetite",
   "guardrail": "confidence",
   "status": "pass"
  }
 ],
 "flags": [
  {
   "field": "propertyAddress",
   "issue": "missing",
   "detail": "property address is required by the appetite ruleset"
  }
 ]
}
Trace line (JSON)
{
  "run_id": "2026-10-20-v6",
  "suite": "extraction-prompts",
  "case_id": "V-08",
  "variant": "v4",
  "at": "2026-10-21T01:07:36.681Z",
  "ok": true,
  "usage": {
    "inputTokens": 6,
    "outputTokens": 876,
    "costUsd": 0.009284700000000002,
    "calls": 3,
    "cacheReadTokens": 5127,
    "cacheCreationTokens": 0,
    "latencyMs": 12165,
    "metered": true,
    "provider": "claude-cli"
  },
  "checks": {
    "extraction_schema": true,
    "fields_correct": true,
    "construction_canonical": true,
    "decision_correct": true,
    "attribution_claude": true
  },
  "output_hash": "38da2fe90e6a05bd",
  "unanswered": [
    "attribution_jev"
  ],
  "extra": {
    "decision": "INCOMPLETE",
    "fitScore": 86,
    "retries": 0,
    "fieldsWrong": 0,
    "modelErrors": ""
  },
  "stages": [
    {
      "name": "intake",
      "ok": true
    },
    {
      "name": "extraction",
      "ok": true
    },
    {
      "name": "grounding",
      "ok": true
    },
    {
      "name": "appetite",
      "ok": true
    }
  ],
  "cached": false
}
Full output record (JSON)
{
  "output": {
    "decision": "INCOMPLETE",
    "expectedDecision": "INCOMPLETE",
    "decisionCorrect": true,
    "fitScore": 86,
    "confidence": 0.9333333333333332,
    "reasons": [
      "property address"
    ],
    "extraction": {
      "insuredName": {
        "value": "Sonora Vista Homes LLC",
        "confidence": 0.98,
        "source": "email"
      },
      "propertyAddress": {
        "value": null,
        "confidence": 0,
        "source": null
      },
      "state": {
        "value": "AZ",
        "confidence": 0.97,
        "source": "email"
      },
      "county": {
        "value": "Pima",
        "confidence": 0.95,
        "source": "email"
      },
      "units": {
        "value": 22,
        "confidence": 0.95,
        "source": "email"
      },
      "tiv": {
        "value": 3300000,
        "confidence": 0.97,
        "source": "email"
      },
      "constructionType": {
        "value": "Frame",
        "confidence": 0.93,
        "source": "email"
      },
      "yearBuilt": {
        "value": 2006,
        "confidence": 0.8,
        "source": "email"
      },
      "targetPremium": {
        "value": 28000,
        "confidence": 0.95,
        "source": "email"
      },
      "subclass": {
        "value": "Single-family rental portfolio",
        "confidence": 0.93,
        "source": "email"
      },
      "priorLossSummary": {
        "value": "One liability claim, closed.",
        "confidence": 0.9,
        "source": "email"
      }
    },
    "extracted": [
      {
        "field": "insuredName",
        "value": "Sonora Vista Homes LLC",
        "source": "email"
      },
      {
        "field": "state",
        "value": "AZ",
        "source": "email"
      },
      {
        "field": "county",
        "value": "Pima",
        "source": "email"
      },
      {
        "field": "units",
        "value": 22,
        "source": "email"
      },
      {
        "field": "tiv",
        "value": 3300000,
        "source": "email"
      },
      {
        "field": "constructionType",
        "value": "Frame",
        "source": "email"
      },
      {
        "field": "yearBuilt",
        "value": 2006,
        "source": "email"
      },
      {
        "field": "targetPremium",
        "value": 28000,
        "source": "email"
      },
      {
        "field": "subclass",
        "value": "Single-family rental portfolio",
        "source": "email"
      },
      {
        "field": "priorLossSummary",
        "value": "One liability claim, closed.",
        "source": "email"
      }
    ],
    "fields": [
      {
        "field": "insuredName",
        "expected": "Sonora Vista Homes LLC",
        "got": "Sonora Vista Homes LLC",
        "correct": true
      },
      {
        "field": "state",
        "expected": "AZ",
        "got": "AZ",
        "correct": true
      },
      {
        "field": "county",
        "expected": "Pima",
        "got": "Pima",
        "correct": true
      },
      {
        "field": "units",
        "expected": 22,
        "got": 22,
        "correct": true
      },
      {
        "field": "tiv",
        "expected": 3300000,
        "got": 3300000,
        "correct": true
      },
      {
        "field": "constructionType",
        "expected": "Frame",
        "got": "Frame",
        "correct": true
      },
      {
        "field": "yearBuilt",
        "expected": 2006,
        "got": 2006,
        "correct": true
      },
      {
        "field": "targetPremium",
        "expected": 28000,
        "got": 28000,
        "correct": true
      },
      {
        "field": "subclass",
        "expected": "Single-family rental portfolio",
        "got": "Single-family rental portfolio",
        "correct": true
      }
    ],
    "retries": 0,
    "guardrails": [
      {
        "node": "extraction",
        "guardrail": "schema",
        "status": "pass"
      },
      {
        "node": "grounding",
        "guardrail": "grounding",
        "status": "pass"
      },
      {
        "node": "appetite",
        "guardrail": "missing_fields",
        "status": "warn"
      },
      {
        "node": "appetite",
        "guardrail": "confidence",
        "status": "pass"
      }
    ],
    "flags": [
      {
        "field": "propertyAddress",
        "issue": "missing",
        "detail": "property address is required by the appetite ruleset"
      }
    ]
  },
  "usage": {
    "inputTokens": 6,
    "cacheReadTokens": 5127,
    "cacheCreationTokens": 0,
    "outputTokens": 876,
    "costUsd": 0.009284700000000002,
    "latencyMs": 12165,
    "calls": 3,
    "metered": true,
    "provider": "claude-cli"
  },
  "extra": {
    "decision": "INCOMPLETE",
    "fitScore": 86,
    "retries": 0,
    "fieldsWrong": 0,
    "modelErrors": ""
  },
  "stages": [
    {
      "name": "intake",
      "ok": true
    },
    {
      "name": "extraction",
      "ok": true
    },
    {
      "name": "grounding",
      "ok": true
    },
    {
      "name": "appetite",
      "ok": true
    }
  ]
}

v5

result
ok
prompt tokens
5,265
output tokens
859
cost
$0.0091
latency
12,199 ms · 3 calls
at
2026-10-21 01:07
stages
intake extraction grounding appetite
decision
INCOMPLETE
fitScore
86
retries
0
fieldsWrong
0
modelErrors
checkverdict
extraction_schemapass
fields_correctpass
construction_canonicalpass
nulls_stay_nullnot applicable
decision_correctpass
attribution_jevunanswered
attribution_claudepass
{
 "decision": "INCOMPLETE",
 "expectedDecision": "INCOMPLETE",
 "decisionCorrect": true,
 "fitScore": 86,
 "confidence": 0.9333333333333332,
 "reasons": [
  "property address"
 ],
 "extraction": {
  "insuredName": {
   "value": "Sonora Vista Homes LLC",
   "confidence": 0.98,
   "source": "email"
  },
  "propertyAddress": {
   "value": null,
   "confidence": 0,
   "source": null
  },
  "state": {
   "value": "AZ",
   "confidence": 0.97,
   "source": "email"
  },
  "county": {
   "value": "Pima",
   "confidence": 0.95,
   "source": "email"
  },
  "units": {
   "value": 22,
   "confidence": 0.95,
   "source": "email"
  },
  "tiv": {
   "value": 3300000,
   "confidence": 0.97,
   "source": "email"
  },
  "constructionType": {
   "value": "Frame",
   "confidence": 0.93,
   "source": "email"
  },
  "yearBuilt": {
   "value": 2006,
   "confidence": 0.8,
   "source": "email"
  },
  "targetPremium": {
   "value": 28000,
   "confidence": 0.92,
   "source": "email"
  },
  "subclass": {
   "value": "Single-family rental portfolio",
   "confidence": 0.92,
   "source": "email"
  },
  "priorLossSummary": {
   "value": "One liability claim, closed.",
   "confidence": 0.85,
   "source": "email"
  }
 },
 "extracted": [
  {
   "field": "insuredName",
   "value": "Sonora Vista Homes LLC",
   "source": "email"
  },
  {
   "field": "state",
   "value": "AZ",
   "source": "email"
  },
  {
   "field": "county",
   "value": "Pima",
   "source": "email"
  },
  {
   "field": "units",
   "value": 22,
   "source": "email"
  },
  {
   "field": "tiv",
   "value": 3300000,
   "source": "email"
  },
  {
   "field": "constructionType",
   "value": "Frame",
   "source": "email"
  },
  {
   "field": "yearBuilt",
   "value": 2006,
   "source": "email"
  },
  {
   "field": "targetPremium",
   "value": 28000,
   "source": "email"
  },
  {
   "field": "subclass",
   "value": "Single-family rental portfolio",
   "source": "email"
  },
  {
   "field": "priorLossSummary",
   "value": "One liability claim, closed.",
   "source": "email"
  }
 ],
 "fields": [
  {
   "field": "insuredName",
   "expected": "Sonora Vista Homes LLC",
   "got": "Sonora Vista Homes LLC",
   "correct": true
  },
  {
   "field": "state",
   "expected": "AZ",
   "got": "AZ",
   "correct": true
  },
  {
   "field": "county",
   "expected": "Pima",
   "got": "Pima",
   "correct": true
  },
  {
   "field": "units",
   "expected": 22,
   "got": 22,
   "correct": true
  },
  {
   "field": "tiv",
   "expected": 3300000,
   "got": 3300000,
   "correct": true
  },
  {
   "field": "constructionType",
   "expected": "Frame",
   "got": "Frame",
   "correct": true
  },
  {
   "field": "yearBuilt",
   "expected": 2006,
   "got": 2006,
   "correct": true
  },
  {
   "field": "targetPremium",
   "expected": 28000,
   "got": 28000,
   "correct": true
  },
  {
   "field": "subclass",
   "expected": "Single-family rental portfolio",
   "got": "Single-family rental portfolio",
   "correct": true
  }
 ],
 "retries": 0,
 "guardrails": [
  {
   "node": "extraction",
   "guardrail": "schema",
   "status": "pass"
  },
  {
   "node": "grounding",
   "guardrail": "grounding",
   "status": "pass"
  },
  {
   "node": "appetite",
   "guardrail": "missing_fields",
   "status": "warn"
  },
  {
   "node": "appetite",
   "guardrail": "confidence",
   "status": "pass"
  }
 ],
 "flags": [
  {
   "field": "propertyAddress",
   "issue": "missing",
   "detail": "property address is required by the appetite ruleset"
  }
 ]
}
Trace line (JSON)
{
  "run_id": "2026-10-20-v6",
  "suite": "extraction-prompts",
  "case_id": "V-08",
  "variant": "v5",
  "at": "2026-10-21T01:07:47.105Z",
  "ok": true,
  "usage": {
    "inputTokens": 6,
    "outputTokens": 859,
    "costUsd": 0.009127900000000001,
    "calls": 3,
    "cacheReadTokens": 5259,
    "cacheCreationTokens": 0,
    "latencyMs": 12199,
    "metered": true,
    "provider": "claude-cli"
  },
  "checks": {
    "extraction_schema": true,
    "fields_correct": true,
    "construction_canonical": true,
    "decision_correct": true,
    "attribution_claude": true
  },
  "output_hash": "ffe73f9da2c38e5d",
  "unanswered": [
    "attribution_jev"
  ],
  "extra": {
    "decision": "INCOMPLETE",
    "fitScore": 86,
    "retries": 0,
    "fieldsWrong": 0,
    "modelErrors": ""
  },
  "stages": [
    {
      "name": "intake",
      "ok": true
    },
    {
      "name": "extraction",
      "ok": true
    },
    {
      "name": "grounding",
      "ok": true
    },
    {
      "name": "appetite",
      "ok": true
    }
  ],
  "cached": false
}
Full output record (JSON)
{
  "output": {
    "decision": "INCOMPLETE",
    "expectedDecision": "INCOMPLETE",
    "decisionCorrect": true,
    "fitScore": 86,
    "confidence": 0.9333333333333332,
    "reasons": [
      "property address"
    ],
    "extraction": {
      "insuredName": {
        "value": "Sonora Vista Homes LLC",
        "confidence": 0.98,
        "source": "email"
      },
      "propertyAddress": {
        "value": null,
        "confidence": 0,
        "source": null
      },
      "state": {
        "value": "AZ",
        "confidence": 0.97,
        "source": "email"
      },
      "county": {
        "value": "Pima",
        "confidence": 0.95,
        "source": "email"
      },
      "units": {
        "value": 22,
        "confidence": 0.95,
        "source": "email"
      },
      "tiv": {
        "value": 3300000,
        "confidence": 0.97,
        "source": "email"
      },
      "constructionType": {
        "value": "Frame",
        "confidence": 0.93,
        "source": "email"
      },
      "yearBuilt": {
        "value": 2006,
        "confidence": 0.8,
        "source": "email"
      },
      "targetPremium": {
        "value": 28000,
        "confidence": 0.92,
        "source": "email"
      },
      "subclass": {
        "value": "Single-family rental portfolio",
        "confidence": 0.92,
        "source": "email"
      },
      "priorLossSummary": {
        "value": "One liability claim, closed.",
        "confidence": 0.85,
        "source": "email"
      }
    },
    "extracted": [
      {
        "field": "insuredName",
        "value": "Sonora Vista Homes LLC",
        "source": "email"
      },
      {
        "field": "state",
        "value": "AZ",
        "source": "email"
      },
      {
        "field": "county",
        "value": "Pima",
        "source": "email"
      },
      {
        "field": "units",
        "value": 22,
        "source": "email"
      },
      {
        "field": "tiv",
        "value": 3300000,
        "source": "email"
      },
      {
        "field": "constructionType",
        "value": "Frame",
        "source": "email"
      },
      {
        "field": "yearBuilt",
        "value": 2006,
        "source": "email"
      },
      {
        "field": "targetPremium",
        "value": 28000,
        "source": "email"
      },
      {
        "field": "subclass",
        "value": "Single-family rental portfolio",
        "source": "email"
      },
      {
        "field": "priorLossSummary",
        "value": "One liability claim, closed.",
        "source": "email"
      }
    ],
    "fields": [
      {
        "field": "insuredName",
        "expected": "Sonora Vista Homes LLC",
        "got": "Sonora Vista Homes LLC",
        "correct": true
      },
      {
        "field": "state",
        "expected": "AZ",
        "got": "AZ",
        "correct": true
      },
      {
        "field": "county",
        "expected": "Pima",
        "got": "Pima",
        "correct": true
      },
      {
        "field": "units",
        "expected": 22,
        "got": 22,
        "correct": true
      },
      {
        "field": "tiv",
        "expected": 3300000,
        "got": 3300000,
        "correct": true
      },
      {
        "field": "constructionType",
        "expected": "Frame",
        "got": "Frame",
        "correct": true
      },
      {
        "field": "yearBuilt",
        "expected": 2006,
        "got": 2006,
        "correct": true
      },
      {
        "field": "targetPremium",
        "expected": 28000,
        "got": 28000,
        "correct": true
      },
      {
        "field": "subclass",
        "expected": "Single-family rental portfolio",
        "got": "Single-family rental portfolio",
        "correct": true
      }
    ],
    "retries": 0,
    "guardrails": [
      {
        "node": "extraction",
        "guardrail": "schema",
        "status": "pass"
      },
      {
        "node": "grounding",
        "guardrail": "grounding",
        "status": "pass"
      },
      {
        "node": "appetite",
        "guardrail": "missing_fields",
        "status": "warn"
      },
      {
        "node": "appetite",
        "guardrail": "confidence",
        "status": "pass"
      }
    ],
    "flags": [
      {
        "field": "propertyAddress",
        "issue": "missing",
        "detail": "property address is required by the appetite ruleset"
      }
    ]
  },
  "usage": {
    "inputTokens": 6,
    "cacheReadTokens": 5259,
    "cacheCreationTokens": 0,
    "outputTokens": 859,
    "costUsd": 0.009127900000000001,
    "latencyMs": 12199,
    "calls": 3,
    "metered": true,
    "provider": "claude-cli"
  },
  "extra": {
    "decision": "INCOMPLETE",
    "fitScore": 86,
    "retries": 0,
    "fieldsWrong": 0,
    "modelErrors": ""
  },
  "stages": [
    {
      "name": "intake",
      "ok": true
    },
    {
      "name": "extraction",
      "ok": true
    },
    {
      "name": "grounding",
      "ok": true
    },
    {
      "name": "appetite",
      "ok": true
    }
  ]
}

v6

result
ok
prompt tokens
5,300
output tokens
846
cost
$0.0115
latency
11,872 ms · 3 calls
at
2026-10-21 01:07
stages
intake extraction grounding appetite
decision
INCOMPLETE
fitScore
86
retries
0
fieldsWrong
0
modelErrors
checkverdict
extraction_schemapass
fields_correctpass
construction_canonicalpass
nulls_stay_nullnot applicable
decision_correctpass
attribution_jevunanswered
attribution_claudepass
{
 "decision": "INCOMPLETE",
 "expectedDecision": "INCOMPLETE",
 "decisionCorrect": true,
 "fitScore": 86,
 "confidence": 0.9416666666666665,
 "reasons": [
  "property address"
 ],
 "extraction": {
  "insuredName": {
   "value": "Sonora Vista Homes LLC",
   "confidence": 0.98,
   "source": "email"
  },
  "propertyAddress": {
   "value": null,
   "confidence": 0,
   "source": null
  },
  "state": {
   "value": "AZ",
   "confidence": 0.97,
   "source": "email"
  },
  "county": {
   "value": "Pima",
   "confidence": 0.95,
   "source": "email"
  },
  "units": {
   "value": 22,
   "confidence": 0.95,
   "source": "email"
  },
  "tiv": {
   "value": 3300000,
   "confidence": 0.97,
   "source": "email"
  },
  "constructionType": {
   "value": "Frame",
   "confidence": 0.93,
   "source": "email"
  },
  "yearBuilt": {
   "value": 2006,
   "confidence": 0.85,
   "source": "email"
  },
  "targetPremium": {
   "value": 28000,
   "confidence": 0.93,
   "source": "email"
  },
  "subclass": {
   "value": "Single-family rental portfolio",
   "confidence": 0.9,
   "source": "email"
  },
  "priorLossSummary": {
   "value": "One liability claim, closed.",
   "confidence": 0.9,
   "source": "email"
  }
 },
 "extracted": [
  {
   "field": "insuredName",
   "value": "Sonora Vista Homes LLC",
   "source": "email"
  },
  {
   "field": "state",
   "value": "AZ",
   "source": "email"
  },
  {
   "field": "county",
   "value": "Pima",
   "source": "email"
  },
  {
   "field": "units",
   "value": 22,
   "source": "email"
  },
  {
   "field": "tiv",
   "value": 3300000,
   "source": "email"
  },
  {
   "field": "constructionType",
   "value": "Frame",
   "source": "email"
  },
  {
   "field": "yearBuilt",
   "value": 2006,
   "source": "email"
  },
  {
   "field": "targetPremium",
   "value": 28000,
   "source": "email"
  },
  {
   "field": "subclass",
   "value": "Single-family rental portfolio",
   "source": "email"
  },
  {
   "field": "priorLossSummary",
   "value": "One liability claim, closed.",
   "source": "email"
  }
 ],
 "fields": [
  {
   "field": "insuredName",
   "expected": "Sonora Vista Homes LLC",
   "got": "Sonora Vista Homes LLC",
   "correct": true
  },
  {
   "field": "state",
   "expected": "AZ",
   "got": "AZ",
   "correct": true
  },
  {
   "field": "county",
   "expected": "Pima",
   "got": "Pima",
   "correct": true
  },
  {
   "field": "units",
   "expected": 22,
   "got": 22,
   "correct": true
  },
  {
   "field": "tiv",
   "expected": 3300000,
   "got": 3300000,
   "correct": true
  },
  {
   "field": "constructionType",
   "expected": "Frame",
   "got": "Frame",
   "correct": true
  },
  {
   "field": "yearBuilt",
   "expected": 2006,
   "got": 2006,
   "correct": true
  },
  {
   "field": "targetPremium",
   "expected": 28000,
   "got": 28000,
   "correct": true
  },
  {
   "field": "subclass",
   "expected": "Single-family rental portfolio",
   "got": "Single-family rental portfolio",
   "correct": true
  }
 ],
 "retries": 0,
 "guardrails": [
  {
   "node": "extraction",
   "guardrail": "schema",
   "status": "pass"
  },
  {
   "node": "grounding",
   "guardrail": "grounding",
   "status": "pass"
  },
  {
   "node": "appetite",
   "guardrail": "missing_fields",
   "status": "warn"
  },
  {
   "node": "appetite",
   "guardrail": "confidence",
   "status": "pass"
  }
 ],
 "flags": [
  {
   "field": "propertyAddress",
   "issue": "missing",
   "detail": "property address is required by the appetite ruleset"
  }
 ]
}
Trace line (JSON)
{
  "run_id": "2026-10-20-v6",
  "suite": "extraction-prompts",
  "case_id": "V-08",
  "variant": "v6",
  "at": "2026-10-21T01:07:44.647Z",
  "ok": true,
  "usage": {
    "inputTokens": 6,
    "outputTokens": 846,
    "costUsd": 0.011493500000000002,
    "calls": 3,
    "cacheReadTokens": 4655,
    "cacheCreationTokens": 639,
    "latencyMs": 11872,
    "metered": true,
    "provider": "claude-cli"
  },
  "checks": {
    "extraction_schema": true,
    "fields_correct": true,
    "construction_canonical": true,
    "decision_correct": true,
    "attribution_claude": true
  },
  "output_hash": "254bff042c90528a",
  "unanswered": [
    "attribution_jev"
  ],
  "extra": {
    "decision": "INCOMPLETE",
    "fitScore": 86,
    "retries": 0,
    "fieldsWrong": 0,
    "modelErrors": ""
  },
  "stages": [
    {
      "name": "intake",
      "ok": true
    },
    {
      "name": "extraction",
      "ok": true
    },
    {
      "name": "grounding",
      "ok": true
    },
    {
      "name": "appetite",
      "ok": true
    }
  ],
  "cached": false
}
Full output record (JSON)
{
  "output": {
    "decision": "INCOMPLETE",
    "expectedDecision": "INCOMPLETE",
    "decisionCorrect": true,
    "fitScore": 86,
    "confidence": 0.9416666666666665,
    "reasons": [
      "property address"
    ],
    "extraction": {
      "insuredName": {
        "value": "Sonora Vista Homes LLC",
        "confidence": 0.98,
        "source": "email"
      },
      "propertyAddress": {
        "value": null,
        "confidence": 0,
        "source": null
      },
      "state": {
        "value": "AZ",
        "confidence": 0.97,
        "source": "email"
      },
      "county": {
        "value": "Pima",
        "confidence": 0.95,
        "source": "email"
      },
      "units": {
        "value": 22,
        "confidence": 0.95,
        "source": "email"
      },
      "tiv": {
        "value": 3300000,
        "confidence": 0.97,
        "source": "email"
      },
      "constructionType": {
        "value": "Frame",
        "confidence": 0.93,
        "source": "email"
      },
      "yearBuilt": {
        "value": 2006,
        "confidence": 0.85,
        "source": "email"
      },
      "targetPremium": {
        "value": 28000,
        "confidence": 0.93,
        "source": "email"
      },
      "subclass": {
        "value": "Single-family rental portfolio",
        "confidence": 0.9,
        "source": "email"
      },
      "priorLossSummary": {
        "value": "One liability claim, closed.",
        "confidence": 0.9,
        "source": "email"
      }
    },
    "extracted": [
      {
        "field": "insuredName",
        "value": "Sonora Vista Homes LLC",
        "source": "email"
      },
      {
        "field": "state",
        "value": "AZ",
        "source": "email"
      },
      {
        "field": "county",
        "value": "Pima",
        "source": "email"
      },
      {
        "field": "units",
        "value": 22,
        "source": "email"
      },
      {
        "field": "tiv",
        "value": 3300000,
        "source": "email"
      },
      {
        "field": "constructionType",
        "value": "Frame",
        "source": "email"
      },
      {
        "field": "yearBuilt",
        "value": 2006,
        "source": "email"
      },
      {
        "field": "targetPremium",
        "value": 28000,
        "source": "email"
      },
      {
        "field": "subclass",
        "value": "Single-family rental portfolio",
        "source": "email"
      },
      {
        "field": "priorLossSummary",
        "value": "One liability claim, closed.",
        "source": "email"
      }
    ],
    "fields": [
      {
        "field": "insuredName",
        "expected": "Sonora Vista Homes LLC",
        "got": "Sonora Vista Homes LLC",
        "correct": true
      },
      {
        "field": "state",
        "expected": "AZ",
        "got": "AZ",
        "correct": true
      },
      {
        "field": "county",
        "expected": "Pima",
        "got": "Pima",
        "correct": true
      },
      {
        "field": "units",
        "expected": 22,
        "got": 22,
        "correct": true
      },
      {
        "field": "tiv",
        "expected": 3300000,
        "got": 3300000,
        "correct": true
      },
      {
        "field": "constructionType",
        "expected": "Frame",
        "got": "Frame",
        "correct": true
      },
      {
        "field": "yearBuilt",
        "expected": 2006,
        "got": 2006,
        "correct": true
      },
      {
        "field": "targetPremium",
        "expected": 28000,
        "got": 28000,
        "correct": true
      },
      {
        "field": "subclass",
        "expected": "Single-family rental portfolio",
        "got": "Single-family rental portfolio",
        "correct": true
      }
    ],
    "retries": 0,
    "guardrails": [
      {
        "node": "extraction",
        "guardrail": "schema",
        "status": "pass"
      },
      {
        "node": "grounding",
        "guardrail": "grounding",
        "status": "pass"
      },
      {
        "node": "appetite",
        "guardrail": "missing_fields",
        "status": "warn"
      },
      {
        "node": "appetite",
        "guardrail": "confidence",
        "status": "pass"
      }
    ],
    "flags": [
      {
        "field": "propertyAddress",
        "issue": "missing",
        "detail": "property address is required by the appetite ruleset"
      }
    ]
  },
  "usage": {
    "inputTokens": 6,
    "cacheReadTokens": 4655,
    "cacheCreationTokens": 639,
    "outputTokens": 846,
    "costUsd": 0.011493500000000002,
    "latencyMs": 11872,
    "calls": 3,
    "metered": true,
    "provider": "claude-cli"
  },
  "extra": {
    "decision": "INCOMPLETE",
    "fitScore": 86,
    "retries": 0,
    "fieldsWrong": 0,
    "modelErrors": ""
  },
  "stages": [
    {
      "name": "intake",
      "ok": true
    },
    {
      "name": "extraction",
      "ok": true
    },
    {
      "name": "grounding",
      "ok": true
    },
    {
      "name": "appetite",
      "ok": true
    }
  ]
}