Weigh evals you can watch
2026-11-12 06:38 UTC

V-13

extraction-prompts · run 2026-10-20-v6

What the broker wrote

What every version was given, as the suite chose to show it.

Broker: Variant desk

Team,

New one for the landlord program: Peachtree Walk Residential LLC, an 18-unit apartment building at 1490 Peachtree Walk NE, Atlanta, GA 30309 (Fulton County). Masonry, built 2009, sprinklered.

Heads up on values — the expiring SOV showed $2.9M, but the updated appraisal came in at $3.4M. Use the appraisal. For context the mortgage balance is about $2.6M.

Target premium $30,500. No losses in five years.

Thanks.

What each version answered

Compare each answer with the expert's answer. Highlighted cells differ from the expert; marked rows show where the versions disagree.

FieldExpectedCurrent prompt v3New prompt v6
insuredNamePeachtree Walk Residential LLCPeachtree Walk Residential LLCMatches the expertPeachtree Walk Residential LLCMatches the expert
stateGAGAMatches the expertGAMatches the expert
countyFultonFultonMatches the expertFultonMatches the expert
units1818Matches the expert18Matches the expert
tiv$3,400,000$3,400,000Matches the expert$3,400,000Matches the expert
yearBuilt20092009Matches the expert2009Matches the expert
targetPremium$30,500$30,500Matches the expert$30,500Matches the expert
DecisionVersions disagreequote it IN APPETITEfollow up with the broker INCOMPLETEDiffers from the expertquote it IN APPETITEMatches the expert
For engineers

Checks are the suite's binary checks on each output; a judge that gave no verdict is unanswered and not counted.

v1

result
ok
prompt tokens
4,336
output tokens
875
cost
$0.0217
latency
12,755 ms · 3 calls
at
2026-10-21 01:09
stages
intake extraction grounding appetite
decision
IN APPETITE
fitScore
93
retries
0
fieldsWrong
0
modelErrors
checkverdict
extraction_schemapass
fields_correctpass
construction_canonicalnot applicable
nulls_stay_nullnot applicable
decision_correctpass
attribution_jevunanswered
attribution_claudepass
{
 "decision": "IN APPETITE",
 "expectedDecision": "IN APPETITE",
 "decisionCorrect": true,
 "fitScore": 93,
 "confidence": 0.9242857142857143,
 "reasons": [
  "Peachtree Walk sits comfortably inside appetite: GA is an allowed state, Fulton County carries no coastal restriction, and 18 units is well under the 60-unit cap. TIV of $3.4M is mid-band, masonry built in 2009 is preferred construction, and the $30,500 target GWP falls inside the premium band. Five loss-free years leave no friction, so this risk is ready to bind."
 ],
 "extraction": {
  "insuredName": {
   "value": "Peachtree Walk Residential LLC",
   "confidence": 0.97,
   "source": "email"
  },
  "propertyAddress": {
   "value": "1490 Peachtree Walk NE, Atlanta, GA 30309",
   "confidence": 0.97,
   "source": "email"
  },
  "state": {
   "value": "GA",
   "confidence": 0.98,
   "source": "email"
  },
  "county": {
   "value": "Fulton",
   "confidence": 0.95,
   "source": "email"
  },
  "units": {
   "value": 18,
   "confidence": 0.95,
   "source": "email"
  },
  "tiv": {
   "value": 3400000,
   "confidence": 0.85,
   "source": "email"
  },
  "constructionType": {
   "value": "Masonry",
   "confidence": 0.8,
   "source": "email"
  },
  "yearBuilt": {
   "value": 2009,
   "confidence": 0.95,
   "source": "email"
  },
  "targetPremium": {
   "value": 30500,
   "confidence": 0.95,
   "source": "email"
  },
  "subclass": {
   "value": null,
   "confidence": 0.4,
   "source": null
  },
  "priorLossSummary": {
   "value": "No losses in five years",
   "confidence": 0.9,
   "source": "email"
  }
 },
 "extracted": [
  {
   "field": "insuredName",
   "value": "Peachtree Walk Residential LLC",
   "source": "email"
  },
  {
   "field": "propertyAddress",
   "value": "1490 Peachtree Walk NE, Atlanta, GA 30309",
   "source": "email"
  },
  {
   "field": "state",
   "value": "GA",
   "source": "email"
  },
  {
   "field": "county",
   "value": "Fulton",
   "source": "email"
  },
  {
   "field": "units",
   "value": 18,
   "source": "email"
  },
  {
   "field": "tiv",
   "value": 3400000,
   "source": "email"
  },
  {
   "field": "constructionType",
   "value": "Masonry",
   "source": "email"
  },
  {
   "field": "yearBuilt",
   "value": 2009,
   "source": "email"
  },
  {
   "field": "targetPremium",
   "value": 30500,
   "source": "email"
  },
  {
   "field": "priorLossSummary",
   "value": "No losses in five years",
   "source": "email"
  }
 ],
 "fields": [
  {
   "field": "insuredName",
   "expected": "Peachtree Walk Residential LLC",
   "got": "Peachtree Walk Residential LLC",
   "correct": true
  },
  {
   "field": "state",
   "expected": "GA",
   "got": "GA",
   "correct": true
  },
  {
   "field": "county",
   "expected": "Fulton",
   "got": "Fulton",
   "correct": true
  },
  {
   "field": "units",
   "expected": 18,
   "got": 18,
   "correct": true
  },
  {
   "field": "tiv",
   "expected": 3400000,
   "got": 3400000,
   "correct": true
  },
  {
   "field": "yearBuilt",
   "expected": 2009,
   "got": 2009,
   "correct": true
  },
  {
   "field": "targetPremium",
   "expected": 30500,
   "got": 30500,
   "correct": true
  }
 ],
 "retries": 0,
 "guardrails": [
  {
   "node": "extraction",
   "guardrail": "schema",
   "status": "pass"
  },
  {
   "node": "grounding",
   "guardrail": "grounding",
   "status": "pass"
  },
  {
   "node": "appetite",
   "guardrail": "missing_fields",
   "status": "pass"
  },
  {
   "node": "appetite",
   "guardrail": "confidence",
   "status": "pass"
  }
 ],
 "flags": []
}
Trace line (JSON)
{
  "run_id": "2026-10-20-v6",
  "suite": "extraction-prompts",
  "case_id": "V-13",
  "variant": "v1",
  "at": "2026-10-21T01:09:52.865Z",
  "ok": true,
  "usage": {
    "inputTokens": 6,
    "outputTokens": 875,
    "costUsd": 0.021675,
    "calls": 3,
    "cacheReadTokens": 1130,
    "cacheCreationTokens": 3200,
    "latencyMs": 12755,
    "metered": true,
    "provider": "claude-cli"
  },
  "checks": {
    "extraction_schema": true,
    "fields_correct": true,
    "decision_correct": true,
    "attribution_claude": true
  },
  "output_hash": "ee0ce874e9e26c59",
  "unanswered": [
    "attribution_jev"
  ],
  "extra": {
    "decision": "IN APPETITE",
    "fitScore": 93,
    "retries": 0,
    "fieldsWrong": 0,
    "modelErrors": ""
  },
  "stages": [
    {
      "name": "intake",
      "ok": true
    },
    {
      "name": "extraction",
      "ok": true
    },
    {
      "name": "grounding",
      "ok": true
    },
    {
      "name": "appetite",
      "ok": true
    }
  ],
  "cached": false
}
Full output record (JSON)
{
  "output": {
    "decision": "IN APPETITE",
    "expectedDecision": "IN APPETITE",
    "decisionCorrect": true,
    "fitScore": 93,
    "confidence": 0.9242857142857143,
    "reasons": [
      "Peachtree Walk sits comfortably inside appetite: GA is an allowed state, Fulton County carries no coastal restriction, and 18 units is well under the 60-unit cap. TIV of $3.4M is mid-band, masonry built in 2009 is preferred construction, and the $30,500 target GWP falls inside the premium band. Five loss-free years leave no friction, so this risk is ready to bind."
    ],
    "extraction": {
      "insuredName": {
        "value": "Peachtree Walk Residential LLC",
        "confidence": 0.97,
        "source": "email"
      },
      "propertyAddress": {
        "value": "1490 Peachtree Walk NE, Atlanta, GA 30309",
        "confidence": 0.97,
        "source": "email"
      },
      "state": {
        "value": "GA",
        "confidence": 0.98,
        "source": "email"
      },
      "county": {
        "value": "Fulton",
        "confidence": 0.95,
        "source": "email"
      },
      "units": {
        "value": 18,
        "confidence": 0.95,
        "source": "email"
      },
      "tiv": {
        "value": 3400000,
        "confidence": 0.85,
        "source": "email"
      },
      "constructionType": {
        "value": "Masonry",
        "confidence": 0.8,
        "source": "email"
      },
      "yearBuilt": {
        "value": 2009,
        "confidence": 0.95,
        "source": "email"
      },
      "targetPremium": {
        "value": 30500,
        "confidence": 0.95,
        "source": "email"
      },
      "subclass": {
        "value": null,
        "confidence": 0.4,
        "source": null
      },
      "priorLossSummary": {
        "value": "No losses in five years",
        "confidence": 0.9,
        "source": "email"
      }
    },
    "extracted": [
      {
        "field": "insuredName",
        "value": "Peachtree Walk Residential LLC",
        "source": "email"
      },
      {
        "field": "propertyAddress",
        "value": "1490 Peachtree Walk NE, Atlanta, GA 30309",
        "source": "email"
      },
      {
        "field": "state",
        "value": "GA",
        "source": "email"
      },
      {
        "field": "county",
        "value": "Fulton",
        "source": "email"
      },
      {
        "field": "units",
        "value": 18,
        "source": "email"
      },
      {
        "field": "tiv",
        "value": 3400000,
        "source": "email"
      },
      {
        "field": "constructionType",
        "value": "Masonry",
        "source": "email"
      },
      {
        "field": "yearBuilt",
        "value": 2009,
        "source": "email"
      },
      {
        "field": "targetPremium",
        "value": 30500,
        "source": "email"
      },
      {
        "field": "priorLossSummary",
        "value": "No losses in five years",
        "source": "email"
      }
    ],
    "fields": [
      {
        "field": "insuredName",
        "expected": "Peachtree Walk Residential LLC",
        "got": "Peachtree Walk Residential LLC",
        "correct": true
      },
      {
        "field": "state",
        "expected": "GA",
        "got": "GA",
        "correct": true
      },
      {
        "field": "county",
        "expected": "Fulton",
        "got": "Fulton",
        "correct": true
      },
      {
        "field": "units",
        "expected": 18,
        "got": 18,
        "correct": true
      },
      {
        "field": "tiv",
        "expected": 3400000,
        "got": 3400000,
        "correct": true
      },
      {
        "field": "yearBuilt",
        "expected": 2009,
        "got": 2009,
        "correct": true
      },
      {
        "field": "targetPremium",
        "expected": 30500,
        "got": 30500,
        "correct": true
      }
    ],
    "retries": 0,
    "guardrails": [
      {
        "node": "extraction",
        "guardrail": "schema",
        "status": "pass"
      },
      {
        "node": "grounding",
        "guardrail": "grounding",
        "status": "pass"
      },
      {
        "node": "appetite",
        "guardrail": "missing_fields",
        "status": "pass"
      },
      {
        "node": "appetite",
        "guardrail": "confidence",
        "status": "pass"
      }
    ],
    "flags": []
  },
  "usage": {
    "inputTokens": 6,
    "cacheReadTokens": 1130,
    "cacheCreationTokens": 3200,
    "outputTokens": 875,
    "costUsd": 0.021675,
    "latencyMs": 12755,
    "calls": 3,
    "metered": true,
    "provider": "claude-cli"
  },
  "extra": {
    "decision": "IN APPETITE",
    "fitScore": 93,
    "retries": 0,
    "fieldsWrong": 0,
    "modelErrors": ""
  },
  "stages": [
    {
      "name": "intake",
      "ok": true
    },
    {
      "name": "extraction",
      "ok": true
    },
    {
      "name": "grounding",
      "ok": true
    },
    {
      "name": "appetite",
      "ok": true
    }
  ]
}

v2

result
ok
prompt tokens
4,697
output tokens
1,041
cost
$0.0109
latency
14,170 ms · 3 calls
at
2026-10-21 01:09
stages
intake extraction grounding appetite
decision
INCOMPLETE
fitScore
86
retries
0
fieldsWrong
0
modelErrors
checkverdict
extraction_schemapass
fields_correctpass
construction_canonicalnot applicable
nulls_stay_nullnot applicable
decision_correctfail
attribution_jevunanswered
attribution_claudepass
{
 "decision": "INCOMPLETE",
 "expectedDecision": "IN APPETITE",
 "decisionCorrect": false,
 "fitScore": 86,
 "confidence": 0.9699999999999999,
 "reasons": [
  "construction type"
 ],
 "extraction": {
  "insuredName": {
   "value": "Peachtree Walk Residential LLC",
   "confidence": 0.98,
   "source": "email"
  },
  "propertyAddress": {
   "value": "1490 Peachtree Walk NE, Atlanta, GA 30309",
   "confidence": 0.98,
   "source": "email"
  },
  "state": {
   "value": "GA",
   "confidence": 0.99,
   "source": "email"
  },
  "county": {
   "value": "Fulton",
   "confidence": 0.97,
   "source": "email"
  },
  "units": {
   "value": 18,
   "confidence": 0.97,
   "source": "email"
  },
  "tiv": {
   "value": 3400000,
   "confidence": 0.93,
   "source": "email"
  },
  "constructionType": {
   "value": null,
   "confidence": 0.2,
   "source": null
  },
  "yearBuilt": {
   "value": 2009,
   "confidence": 0.97,
   "source": "email"
  },
  "targetPremium": {
   "value": 30500,
   "confidence": 0.98,
   "source": "email"
  },
  "subclass": {
   "value": null,
   "confidence": 0.3,
   "source": null
  },
  "priorLossSummary": {
   "value": "No losses in the past five years.",
   "confidence": 0.95,
   "source": "email"
  }
 },
 "extracted": [
  {
   "field": "insuredName",
   "value": "Peachtree Walk Residential LLC",
   "source": "email"
  },
  {
   "field": "propertyAddress",
   "value": "1490 Peachtree Walk NE, Atlanta, GA 30309",
   "source": "email"
  },
  {
   "field": "state",
   "value": "GA",
   "source": "email"
  },
  {
   "field": "county",
   "value": "Fulton",
   "source": "email"
  },
  {
   "field": "units",
   "value": 18,
   "source": "email"
  },
  {
   "field": "tiv",
   "value": 3400000,
   "source": "email"
  },
  {
   "field": "yearBuilt",
   "value": 2009,
   "source": "email"
  },
  {
   "field": "targetPremium",
   "value": 30500,
   "source": "email"
  },
  {
   "field": "priorLossSummary",
   "value": "No losses in the past five years.",
   "source": "email"
  }
 ],
 "fields": [
  {
   "field": "insuredName",
   "expected": "Peachtree Walk Residential LLC",
   "got": "Peachtree Walk Residential LLC",
   "correct": true
  },
  {
   "field": "state",
   "expected": "GA",
   "got": "GA",
   "correct": true
  },
  {
   "field": "county",
   "expected": "Fulton",
   "got": "Fulton",
   "correct": true
  },
  {
   "field": "units",
   "expected": 18,
   "got": 18,
   "correct": true
  },
  {
   "field": "tiv",
   "expected": 3400000,
   "got": 3400000,
   "correct": true
  },
  {
   "field": "yearBuilt",
   "expected": 2009,
   "got": 2009,
   "correct": true
  },
  {
   "field": "targetPremium",
   "expected": 30500,
   "got": 30500,
   "correct": true
  }
 ],
 "retries": 0,
 "guardrails": [
  {
   "node": "extraction",
   "guardrail": "schema",
   "status": "pass"
  },
  {
   "node": "grounding",
   "guardrail": "grounding",
   "status": "pass"
  },
  {
   "node": "appetite",
   "guardrail": "missing_fields",
   "status": "warn"
  },
  {
   "node": "appetite",
   "guardrail": "confidence",
   "status": "pass"
  }
 ],
 "flags": [
  {
   "field": "constructionType",
   "issue": "missing",
   "detail": "construction type is required by the appetite ruleset"
  }
 ]
}
Trace line (JSON)
{
  "run_id": "2026-10-20-v6",
  "suite": "extraction-prompts",
  "case_id": "V-13",
  "variant": "v2",
  "at": "2026-10-21T01:09:33.530Z",
  "ok": true,
  "usage": {
    "inputTokens": 6,
    "outputTokens": 1041,
    "costUsd": 0.010891099999999999,
    "calls": 3,
    "cacheReadTokens": 4691,
    "cacheCreationTokens": 0,
    "latencyMs": 14170,
    "metered": true,
    "provider": "claude-cli"
  },
  "checks": {
    "extraction_schema": true,
    "fields_correct": true,
    "decision_correct": false,
    "attribution_claude": true
  },
  "output_hash": "2693d378b357fb4b",
  "unanswered": [
    "attribution_jev"
  ],
  "extra": {
    "decision": "INCOMPLETE",
    "fitScore": 86,
    "retries": 0,
    "fieldsWrong": 0,
    "modelErrors": ""
  },
  "stages": [
    {
      "name": "intake",
      "ok": true
    },
    {
      "name": "extraction",
      "ok": true
    },
    {
      "name": "grounding",
      "ok": true
    },
    {
      "name": "appetite",
      "ok": false,
      "detail": "INCOMPLETE (want IN APPETITE)"
    }
  ],
  "cached": false
}
Full output record (JSON)
{
  "output": {
    "decision": "INCOMPLETE",
    "expectedDecision": "IN APPETITE",
    "decisionCorrect": false,
    "fitScore": 86,
    "confidence": 0.9699999999999999,
    "reasons": [
      "construction type"
    ],
    "extraction": {
      "insuredName": {
        "value": "Peachtree Walk Residential LLC",
        "confidence": 0.98,
        "source": "email"
      },
      "propertyAddress": {
        "value": "1490 Peachtree Walk NE, Atlanta, GA 30309",
        "confidence": 0.98,
        "source": "email"
      },
      "state": {
        "value": "GA",
        "confidence": 0.99,
        "source": "email"
      },
      "county": {
        "value": "Fulton",
        "confidence": 0.97,
        "source": "email"
      },
      "units": {
        "value": 18,
        "confidence": 0.97,
        "source": "email"
      },
      "tiv": {
        "value": 3400000,
        "confidence": 0.93,
        "source": "email"
      },
      "constructionType": {
        "value": null,
        "confidence": 0.2,
        "source": null
      },
      "yearBuilt": {
        "value": 2009,
        "confidence": 0.97,
        "source": "email"
      },
      "targetPremium": {
        "value": 30500,
        "confidence": 0.98,
        "source": "email"
      },
      "subclass": {
        "value": null,
        "confidence": 0.3,
        "source": null
      },
      "priorLossSummary": {
        "value": "No losses in the past five years.",
        "confidence": 0.95,
        "source": "email"
      }
    },
    "extracted": [
      {
        "field": "insuredName",
        "value": "Peachtree Walk Residential LLC",
        "source": "email"
      },
      {
        "field": "propertyAddress",
        "value": "1490 Peachtree Walk NE, Atlanta, GA 30309",
        "source": "email"
      },
      {
        "field": "state",
        "value": "GA",
        "source": "email"
      },
      {
        "field": "county",
        "value": "Fulton",
        "source": "email"
      },
      {
        "field": "units",
        "value": 18,
        "source": "email"
      },
      {
        "field": "tiv",
        "value": 3400000,
        "source": "email"
      },
      {
        "field": "yearBuilt",
        "value": 2009,
        "source": "email"
      },
      {
        "field": "targetPremium",
        "value": 30500,
        "source": "email"
      },
      {
        "field": "priorLossSummary",
        "value": "No losses in the past five years.",
        "source": "email"
      }
    ],
    "fields": [
      {
        "field": "insuredName",
        "expected": "Peachtree Walk Residential LLC",
        "got": "Peachtree Walk Residential LLC",
        "correct": true
      },
      {
        "field": "state",
        "expected": "GA",
        "got": "GA",
        "correct": true
      },
      {
        "field": "county",
        "expected": "Fulton",
        "got": "Fulton",
        "correct": true
      },
      {
        "field": "units",
        "expected": 18,
        "got": 18,
        "correct": true
      },
      {
        "field": "tiv",
        "expected": 3400000,
        "got": 3400000,
        "correct": true
      },
      {
        "field": "yearBuilt",
        "expected": 2009,
        "got": 2009,
        "correct": true
      },
      {
        "field": "targetPremium",
        "expected": 30500,
        "got": 30500,
        "correct": true
      }
    ],
    "retries": 0,
    "guardrails": [
      {
        "node": "extraction",
        "guardrail": "schema",
        "status": "pass"
      },
      {
        "node": "grounding",
        "guardrail": "grounding",
        "status": "pass"
      },
      {
        "node": "appetite",
        "guardrail": "missing_fields",
        "status": "warn"
      },
      {
        "node": "appetite",
        "guardrail": "confidence",
        "status": "pass"
      }
    ],
    "flags": [
      {
        "field": "constructionType",
        "issue": "missing",
        "detail": "construction type is required by the appetite ruleset"
      }
    ]
  },
  "usage": {
    "inputTokens": 6,
    "cacheReadTokens": 4691,
    "cacheCreationTokens": 0,
    "outputTokens": 1041,
    "costUsd": 0.010891099999999999,
    "latencyMs": 14170,
    "calls": 3,
    "metered": true,
    "provider": "claude-cli"
  },
  "extra": {
    "decision": "INCOMPLETE",
    "fitScore": 86,
    "retries": 0,
    "fieldsWrong": 0,
    "modelErrors": ""
  },
  "stages": [
    {
      "name": "intake",
      "ok": true
    },
    {
      "name": "extraction",
      "ok": true
    },
    {
      "name": "grounding",
      "ok": true
    },
    {
      "name": "appetite",
      "ok": false,
      "detail": "INCOMPLETE (want IN APPETITE)"
    }
  ]
}

v3

result
ok
prompt tokens
5,007
output tokens
869
cost
$0.0214
latency
13,520 ms · 3 calls
at
2026-10-21 01:09
stages
intake extraction grounding appetite
decision
INCOMPLETE
fitScore
88
retries
0
fieldsWrong
0
modelErrors
checkverdict
extraction_schemapass
fields_correctpass
construction_canonicalnot applicable
nulls_stay_nullnot applicable
decision_correctfail
attribution_jevunanswered
attribution_claudepass
{
 "decision": "INCOMPLETE",
 "expectedDecision": "IN APPETITE",
 "decisionCorrect": false,
 "fitScore": 88,
 "confidence": 0.9616666666666666,
 "reasons": [
  "construction type"
 ],
 "extraction": {
  "insuredName": {
   "value": "Peachtree Walk Residential LLC",
   "confidence": 0.98,
   "source": "email"
  },
  "propertyAddress": {
   "value": "1490 Peachtree Walk NE, Atlanta, GA 30309",
   "confidence": 0.97,
   "source": "email"
  },
  "state": {
   "value": "GA",
   "confidence": 0.98,
   "source": "email"
  },
  "county": {
   "value": "Fulton",
   "confidence": 0.97,
   "source": "email"
  },
  "units": {
   "value": 18,
   "confidence": 0.97,
   "source": "email"
  },
  "tiv": {
   "value": 3400000,
   "confidence": 0.9,
   "source": "email"
  },
  "constructionType": {
   "value": null,
   "confidence": 0.4,
   "source": null
  },
  "yearBuilt": {
   "value": 2009,
   "confidence": 0.97,
   "source": "email"
  },
  "targetPremium": {
   "value": 30500,
   "confidence": 0.97,
   "source": "email"
  },
  "subclass": {
   "value": null,
   "confidence": 0.9,
   "source": null
  },
  "priorLossSummary": {
   "value": "No losses in five years",
   "confidence": 0.95,
   "source": "email"
  }
 },
 "extracted": [
  {
   "field": "insuredName",
   "value": "Peachtree Walk Residential LLC",
   "source": "email"
  },
  {
   "field": "propertyAddress",
   "value": "1490 Peachtree Walk NE, Atlanta, GA 30309",
   "source": "email"
  },
  {
   "field": "state",
   "value": "GA",
   "source": "email"
  },
  {
   "field": "county",
   "value": "Fulton",
   "source": "email"
  },
  {
   "field": "units",
   "value": 18,
   "source": "email"
  },
  {
   "field": "tiv",
   "value": 3400000,
   "source": "email"
  },
  {
   "field": "yearBuilt",
   "value": 2009,
   "source": "email"
  },
  {
   "field": "targetPremium",
   "value": 30500,
   "source": "email"
  },
  {
   "field": "priorLossSummary",
   "value": "No losses in five years",
   "source": "email"
  }
 ],
 "fields": [
  {
   "field": "insuredName",
   "expected": "Peachtree Walk Residential LLC",
   "got": "Peachtree Walk Residential LLC",
   "correct": true
  },
  {
   "field": "state",
   "expected": "GA",
   "got": "GA",
   "correct": true
  },
  {
   "field": "county",
   "expected": "Fulton",
   "got": "Fulton",
   "correct": true
  },
  {
   "field": "units",
   "expected": 18,
   "got": 18,
   "correct": true
  },
  {
   "field": "tiv",
   "expected": 3400000,
   "got": 3400000,
   "correct": true
  },
  {
   "field": "yearBuilt",
   "expected": 2009,
   "got": 2009,
   "correct": true
  },
  {
   "field": "targetPremium",
   "expected": 30500,
   "got": 30500,
   "correct": true
  }
 ],
 "retries": 0,
 "guardrails": [
  {
   "node": "extraction",
   "guardrail": "schema",
   "status": "pass"
  },
  {
   "node": "grounding",
   "guardrail": "grounding",
   "status": "pass"
  },
  {
   "node": "appetite",
   "guardrail": "missing_fields",
   "status": "warn"
  },
  {
   "node": "appetite",
   "guardrail": "confidence",
   "status": "pass"
  }
 ],
 "flags": [
  {
   "field": "constructionType",
   "issue": "missing",
   "detail": "construction type is required by the appetite ruleset"
  }
 ]
}
Trace line (JSON)
{
  "run_id": "2026-10-20-v6",
  "suite": "extraction-prompts",
  "case_id": "V-13",
  "variant": "v3",
  "at": "2026-10-21T01:09:35.740Z",
  "ok": true,
  "usage": {
    "inputTokens": 6,
    "outputTokens": 869,
    "costUsd": 0.0213818,
    "calls": 3,
    "cacheReadTokens": 1878,
    "cacheCreationTokens": 3123,
    "latencyMs": 13520,
    "metered": true,
    "provider": "claude-cli"
  },
  "checks": {
    "extraction_schema": true,
    "fields_correct": true,
    "decision_correct": false,
    "attribution_claude": true
  },
  "output_hash": "7914f6509a981e1a",
  "unanswered": [
    "attribution_jev"
  ],
  "extra": {
    "decision": "INCOMPLETE",
    "fitScore": 88,
    "retries": 0,
    "fieldsWrong": 0,
    "modelErrors": ""
  },
  "stages": [
    {
      "name": "intake",
      "ok": true
    },
    {
      "name": "extraction",
      "ok": true
    },
    {
      "name": "grounding",
      "ok": true
    },
    {
      "name": "appetite",
      "ok": false,
      "detail": "INCOMPLETE (want IN APPETITE)"
    }
  ],
  "cached": false
}
Full output record (JSON)
{
  "output": {
    "decision": "INCOMPLETE",
    "expectedDecision": "IN APPETITE",
    "decisionCorrect": false,
    "fitScore": 88,
    "confidence": 0.9616666666666666,
    "reasons": [
      "construction type"
    ],
    "extraction": {
      "insuredName": {
        "value": "Peachtree Walk Residential LLC",
        "confidence": 0.98,
        "source": "email"
      },
      "propertyAddress": {
        "value": "1490 Peachtree Walk NE, Atlanta, GA 30309",
        "confidence": 0.97,
        "source": "email"
      },
      "state": {
        "value": "GA",
        "confidence": 0.98,
        "source": "email"
      },
      "county": {
        "value": "Fulton",
        "confidence": 0.97,
        "source": "email"
      },
      "units": {
        "value": 18,
        "confidence": 0.97,
        "source": "email"
      },
      "tiv": {
        "value": 3400000,
        "confidence": 0.9,
        "source": "email"
      },
      "constructionType": {
        "value": null,
        "confidence": 0.4,
        "source": null
      },
      "yearBuilt": {
        "value": 2009,
        "confidence": 0.97,
        "source": "email"
      },
      "targetPremium": {
        "value": 30500,
        "confidence": 0.97,
        "source": "email"
      },
      "subclass": {
        "value": null,
        "confidence": 0.9,
        "source": null
      },
      "priorLossSummary": {
        "value": "No losses in five years",
        "confidence": 0.95,
        "source": "email"
      }
    },
    "extracted": [
      {
        "field": "insuredName",
        "value": "Peachtree Walk Residential LLC",
        "source": "email"
      },
      {
        "field": "propertyAddress",
        "value": "1490 Peachtree Walk NE, Atlanta, GA 30309",
        "source": "email"
      },
      {
        "field": "state",
        "value": "GA",
        "source": "email"
      },
      {
        "field": "county",
        "value": "Fulton",
        "source": "email"
      },
      {
        "field": "units",
        "value": 18,
        "source": "email"
      },
      {
        "field": "tiv",
        "value": 3400000,
        "source": "email"
      },
      {
        "field": "yearBuilt",
        "value": 2009,
        "source": "email"
      },
      {
        "field": "targetPremium",
        "value": 30500,
        "source": "email"
      },
      {
        "field": "priorLossSummary",
        "value": "No losses in five years",
        "source": "email"
      }
    ],
    "fields": [
      {
        "field": "insuredName",
        "expected": "Peachtree Walk Residential LLC",
        "got": "Peachtree Walk Residential LLC",
        "correct": true
      },
      {
        "field": "state",
        "expected": "GA",
        "got": "GA",
        "correct": true
      },
      {
        "field": "county",
        "expected": "Fulton",
        "got": "Fulton",
        "correct": true
      },
      {
        "field": "units",
        "expected": 18,
        "got": 18,
        "correct": true
      },
      {
        "field": "tiv",
        "expected": 3400000,
        "got": 3400000,
        "correct": true
      },
      {
        "field": "yearBuilt",
        "expected": 2009,
        "got": 2009,
        "correct": true
      },
      {
        "field": "targetPremium",
        "expected": 30500,
        "got": 30500,
        "correct": true
      }
    ],
    "retries": 0,
    "guardrails": [
      {
        "node": "extraction",
        "guardrail": "schema",
        "status": "pass"
      },
      {
        "node": "grounding",
        "guardrail": "grounding",
        "status": "pass"
      },
      {
        "node": "appetite",
        "guardrail": "missing_fields",
        "status": "warn"
      },
      {
        "node": "appetite",
        "guardrail": "confidence",
        "status": "pass"
      }
    ],
    "flags": [
      {
        "field": "constructionType",
        "issue": "missing",
        "detail": "construction type is required by the appetite ruleset"
      }
    ]
  },
  "usage": {
    "inputTokens": 6,
    "cacheReadTokens": 1878,
    "cacheCreationTokens": 3123,
    "outputTokens": 869,
    "costUsd": 0.0213818,
    "latencyMs": 13520,
    "calls": 3,
    "metered": true,
    "provider": "claude-cli"
  },
  "extra": {
    "decision": "INCOMPLETE",
    "fitScore": 88,
    "retries": 0,
    "fieldsWrong": 0,
    "modelErrors": ""
  },
  "stages": [
    {
      "name": "intake",
      "ok": true
    },
    {
      "name": "extraction",
      "ok": true
    },
    {
      "name": "grounding",
      "ok": true
    },
    {
      "name": "appetite",
      "ok": false,
      "detail": "INCOMPLETE (want IN APPETITE)"
    }
  ]
}

v3-repeat

result
ok
prompt tokens
5,011
output tokens
861
cost
$0.0091
latency
12,466 ms · 3 calls
at
2026-10-21 01:09
stages
intake extraction grounding appetite
decision
INCOMPLETE
fitScore
86
retries
0
fieldsWrong
0
modelErrors
checkverdict
extraction_schemapass
fields_correctpass
construction_canonicalnot applicable
nulls_stay_nullnot applicable
decision_correctfail
attribution_jevunanswered
attribution_claudepass
{
 "decision": "INCOMPLETE",
 "expectedDecision": "IN APPETITE",
 "decisionCorrect": false,
 "fitScore": 86,
 "confidence": 0.9616666666666666,
 "reasons": [
  "construction type"
 ],
 "extraction": {
  "insuredName": {
   "value": "Peachtree Walk Residential LLC",
   "confidence": 0.98,
   "source": "email"
  },
  "propertyAddress": {
   "value": "1490 Peachtree Walk NE, Atlanta, GA 30309",
   "confidence": 0.97,
   "source": "email"
  },
  "state": {
   "value": "GA",
   "confidence": 0.98,
   "source": "email"
  },
  "county": {
   "value": "Fulton",
   "confidence": 0.97,
   "source": "email"
  },
  "units": {
   "value": 18,
   "confidence": 0.97,
   "source": "email"
  },
  "tiv": {
   "value": 3400000,
   "confidence": 0.9,
   "source": "email"
  },
  "constructionType": {
   "value": null,
   "confidence": 0.3,
   "source": null
  },
  "yearBuilt": {
   "value": 2009,
   "confidence": 0.97,
   "source": "email"
  },
  "targetPremium": {
   "value": 30500,
   "confidence": 0.97,
   "source": "email"
  },
  "subclass": {
   "value": null,
   "confidence": 0.9,
   "source": null
  },
  "priorLossSummary": {
   "value": "No losses in the past five years",
   "confidence": 0.93,
   "source": "email"
  }
 },
 "extracted": [
  {
   "field": "insuredName",
   "value": "Peachtree Walk Residential LLC",
   "source": "email"
  },
  {
   "field": "propertyAddress",
   "value": "1490 Peachtree Walk NE, Atlanta, GA 30309",
   "source": "email"
  },
  {
   "field": "state",
   "value": "GA",
   "source": "email"
  },
  {
   "field": "county",
   "value": "Fulton",
   "source": "email"
  },
  {
   "field": "units",
   "value": 18,
   "source": "email"
  },
  {
   "field": "tiv",
   "value": 3400000,
   "source": "email"
  },
  {
   "field": "yearBuilt",
   "value": 2009,
   "source": "email"
  },
  {
   "field": "targetPremium",
   "value": 30500,
   "source": "email"
  },
  {
   "field": "priorLossSummary",
   "value": "No losses in the past five years",
   "source": "email"
  }
 ],
 "fields": [
  {
   "field": "insuredName",
   "expected": "Peachtree Walk Residential LLC",
   "got": "Peachtree Walk Residential LLC",
   "correct": true
  },
  {
   "field": "state",
   "expected": "GA",
   "got": "GA",
   "correct": true
  },
  {
   "field": "county",
   "expected": "Fulton",
   "got": "Fulton",
   "correct": true
  },
  {
   "field": "units",
   "expected": 18,
   "got": 18,
   "correct": true
  },
  {
   "field": "tiv",
   "expected": 3400000,
   "got": 3400000,
   "correct": true
  },
  {
   "field": "yearBuilt",
   "expected": 2009,
   "got": 2009,
   "correct": true
  },
  {
   "field": "targetPremium",
   "expected": 30500,
   "got": 30500,
   "correct": true
  }
 ],
 "retries": 0,
 "guardrails": [
  {
   "node": "extraction",
   "guardrail": "schema",
   "status": "pass"
  },
  {
   "node": "grounding",
   "guardrail": "grounding",
   "status": "pass"
  },
  {
   "node": "appetite",
   "guardrail": "missing_fields",
   "status": "warn"
  },
  {
   "node": "appetite",
   "guardrail": "confidence",
   "status": "pass"
  }
 ],
 "flags": [
  {
   "field": "constructionType",
   "issue": "missing",
   "detail": "construction type is required by the appetite ruleset"
  }
 ]
}
Trace line (JSON)
{
  "run_id": "2026-10-20-v6",
  "suite": "extraction-prompts",
  "case_id": "V-13",
  "variant": "v3-repeat",
  "at": "2026-10-21T01:09:37.496Z",
  "ok": true,
  "usage": {
    "inputTokens": 6,
    "outputTokens": 861,
    "costUsd": 0.0091225,
    "calls": 3,
    "cacheReadTokens": 5005,
    "cacheCreationTokens": 0,
    "latencyMs": 12466,
    "metered": true,
    "provider": "claude-cli"
  },
  "checks": {
    "extraction_schema": true,
    "fields_correct": true,
    "decision_correct": false,
    "attribution_claude": true
  },
  "output_hash": "a91082016bb09e47",
  "unanswered": [
    "attribution_jev"
  ],
  "extra": {
    "decision": "INCOMPLETE",
    "fitScore": 86,
    "retries": 0,
    "fieldsWrong": 0,
    "modelErrors": ""
  },
  "stages": [
    {
      "name": "intake",
      "ok": true
    },
    {
      "name": "extraction",
      "ok": true
    },
    {
      "name": "grounding",
      "ok": true
    },
    {
      "name": "appetite",
      "ok": false,
      "detail": "INCOMPLETE (want IN APPETITE)"
    }
  ],
  "cached": false
}
Full output record (JSON)
{
  "output": {
    "decision": "INCOMPLETE",
    "expectedDecision": "IN APPETITE",
    "decisionCorrect": false,
    "fitScore": 86,
    "confidence": 0.9616666666666666,
    "reasons": [
      "construction type"
    ],
    "extraction": {
      "insuredName": {
        "value": "Peachtree Walk Residential LLC",
        "confidence": 0.98,
        "source": "email"
      },
      "propertyAddress": {
        "value": "1490 Peachtree Walk NE, Atlanta, GA 30309",
        "confidence": 0.97,
        "source": "email"
      },
      "state": {
        "value": "GA",
        "confidence": 0.98,
        "source": "email"
      },
      "county": {
        "value": "Fulton",
        "confidence": 0.97,
        "source": "email"
      },
      "units": {
        "value": 18,
        "confidence": 0.97,
        "source": "email"
      },
      "tiv": {
        "value": 3400000,
        "confidence": 0.9,
        "source": "email"
      },
      "constructionType": {
        "value": null,
        "confidence": 0.3,
        "source": null
      },
      "yearBuilt": {
        "value": 2009,
        "confidence": 0.97,
        "source": "email"
      },
      "targetPremium": {
        "value": 30500,
        "confidence": 0.97,
        "source": "email"
      },
      "subclass": {
        "value": null,
        "confidence": 0.9,
        "source": null
      },
      "priorLossSummary": {
        "value": "No losses in the past five years",
        "confidence": 0.93,
        "source": "email"
      }
    },
    "extracted": [
      {
        "field": "insuredName",
        "value": "Peachtree Walk Residential LLC",
        "source": "email"
      },
      {
        "field": "propertyAddress",
        "value": "1490 Peachtree Walk NE, Atlanta, GA 30309",
        "source": "email"
      },
      {
        "field": "state",
        "value": "GA",
        "source": "email"
      },
      {
        "field": "county",
        "value": "Fulton",
        "source": "email"
      },
      {
        "field": "units",
        "value": 18,
        "source": "email"
      },
      {
        "field": "tiv",
        "value": 3400000,
        "source": "email"
      },
      {
        "field": "yearBuilt",
        "value": 2009,
        "source": "email"
      },
      {
        "field": "targetPremium",
        "value": 30500,
        "source": "email"
      },
      {
        "field": "priorLossSummary",
        "value": "No losses in the past five years",
        "source": "email"
      }
    ],
    "fields": [
      {
        "field": "insuredName",
        "expected": "Peachtree Walk Residential LLC",
        "got": "Peachtree Walk Residential LLC",
        "correct": true
      },
      {
        "field": "state",
        "expected": "GA",
        "got": "GA",
        "correct": true
      },
      {
        "field": "county",
        "expected": "Fulton",
        "got": "Fulton",
        "correct": true
      },
      {
        "field": "units",
        "expected": 18,
        "got": 18,
        "correct": true
      },
      {
        "field": "tiv",
        "expected": 3400000,
        "got": 3400000,
        "correct": true
      },
      {
        "field": "yearBuilt",
        "expected": 2009,
        "got": 2009,
        "correct": true
      },
      {
        "field": "targetPremium",
        "expected": 30500,
        "got": 30500,
        "correct": true
      }
    ],
    "retries": 0,
    "guardrails": [
      {
        "node": "extraction",
        "guardrail": "schema",
        "status": "pass"
      },
      {
        "node": "grounding",
        "guardrail": "grounding",
        "status": "pass"
      },
      {
        "node": "appetite",
        "guardrail": "missing_fields",
        "status": "warn"
      },
      {
        "node": "appetite",
        "guardrail": "confidence",
        "status": "pass"
      }
    ],
    "flags": [
      {
        "field": "constructionType",
        "issue": "missing",
        "detail": "construction type is required by the appetite ruleset"
      }
    ]
  },
  "usage": {
    "inputTokens": 6,
    "cacheReadTokens": 5005,
    "cacheCreationTokens": 0,
    "outputTokens": 861,
    "costUsd": 0.0091225,
    "latencyMs": 12466,
    "calls": 3,
    "metered": true,
    "provider": "claude-cli"
  },
  "extra": {
    "decision": "INCOMPLETE",
    "fitScore": 86,
    "retries": 0,
    "fieldsWrong": 0,
    "modelErrors": ""
  },
  "stages": [
    {
      "name": "intake",
      "ok": true
    },
    {
      "name": "extraction",
      "ok": true
    },
    {
      "name": "grounding",
      "ok": true
    },
    {
      "name": "appetite",
      "ok": false,
      "detail": "INCOMPLETE (want IN APPETITE)"
    }
  ]
}

v4

result
ok
prompt tokens
5,309
output tokens
901
cost
$0.0096
latency
12,111 ms · 3 calls
at
2026-10-21 01:09
stages
intake extraction grounding appetite
decision
IN APPETITE
fitScore
92
retries
0
fieldsWrong
0
modelErrors
checkverdict
extraction_schemapass
fields_correctpass
construction_canonicalnot applicable
nulls_stay_nullnot applicable
decision_correctpass
attribution_jevunanswered
attribution_claudepass
{
 "decision": "IN APPETITE",
 "expectedDecision": "IN APPETITE",
 "decisionCorrect": true,
 "fitScore": 92,
 "confidence": 0.8928571428571427,
 "reasons": [
  "Peachtree Walk sits comfortably inside appetite: GA is an allowed state, Fulton County carries no coastal restriction, and 18 units fall well under the 60-unit cap. TIV of $3.4M is mid-band, the 2009 joisted masonry build is preferred construction, and $30,500 GWP sits inside the premium band with five clean loss years. Bind with standard terms."
 ],
 "extraction": {
  "insuredName": {
   "value": "Peachtree Walk Residential LLC",
   "confidence": 0.98,
   "source": "email"
  },
  "propertyAddress": {
   "value": "1490 Peachtree Walk NE, Atlanta, GA 30309",
   "confidence": 0.97,
   "source": "email"
  },
  "state": {
   "value": "GA",
   "confidence": 0.98,
   "source": "email"
  },
  "county": {
   "value": "Fulton",
   "confidence": 0.95,
   "source": "email"
  },
  "units": {
   "value": 18,
   "confidence": 0.97,
   "source": "email"
  },
  "tiv": {
   "value": 3400000,
   "confidence": 0.9,
   "source": "email"
  },
  "constructionType": {
   "value": "Joisted Masonry",
   "confidence": 0.8,
   "source": "email"
  },
  "yearBuilt": {
   "value": 2009,
   "confidence": 0.97,
   "source": "email"
  },
  "targetPremium": {
   "value": 30500,
   "confidence": 0.97,
   "source": "email"
  },
  "subclass": {
   "value": null,
   "confidence": 0.9,
   "source": null
  },
  "priorLossSummary": {
   "value": "No losses in the past five years",
   "confidence": 0.95,
   "source": "email"
  }
 },
 "extracted": [
  {
   "field": "insuredName",
   "value": "Peachtree Walk Residential LLC",
   "source": "email"
  },
  {
   "field": "propertyAddress",
   "value": "1490 Peachtree Walk NE, Atlanta, GA 30309",
   "source": "email"
  },
  {
   "field": "state",
   "value": "GA",
   "source": "email"
  },
  {
   "field": "county",
   "value": "Fulton",
   "source": "email"
  },
  {
   "field": "units",
   "value": 18,
   "source": "email"
  },
  {
   "field": "tiv",
   "value": 3400000,
   "source": "email"
  },
  {
   "field": "constructionType",
   "value": "Joisted Masonry",
   "source": "email"
  },
  {
   "field": "yearBuilt",
   "value": 2009,
   "source": "email"
  },
  {
   "field": "targetPremium",
   "value": 30500,
   "source": "email"
  },
  {
   "field": "priorLossSummary",
   "value": "No losses in the past five years",
   "source": "email"
  }
 ],
 "fields": [
  {
   "field": "insuredName",
   "expected": "Peachtree Walk Residential LLC",
   "got": "Peachtree Walk Residential LLC",
   "correct": true
  },
  {
   "field": "state",
   "expected": "GA",
   "got": "GA",
   "correct": true
  },
  {
   "field": "county",
   "expected": "Fulton",
   "got": "Fulton",
   "correct": true
  },
  {
   "field": "units",
   "expected": 18,
   "got": 18,
   "correct": true
  },
  {
   "field": "tiv",
   "expected": 3400000,
   "got": 3400000,
   "correct": true
  },
  {
   "field": "yearBuilt",
   "expected": 2009,
   "got": 2009,
   "correct": true
  },
  {
   "field": "targetPremium",
   "expected": 30500,
   "got": 30500,
   "correct": true
  }
 ],
 "retries": 0,
 "guardrails": [
  {
   "node": "extraction",
   "guardrail": "schema",
   "status": "pass"
  },
  {
   "node": "grounding",
   "guardrail": "grounding",
   "status": "warn"
  },
  {
   "node": "appetite",
   "guardrail": "missing_fields",
   "status": "pass"
  },
  {
   "node": "appetite",
   "guardrail": "confidence",
   "status": "pass"
  }
 ],
 "flags": [
  {
   "field": "constructionType",
   "issue": "ambiguous",
   "detail": "not clearly supported by source: Email says only 'Masonry'; 'Joisted Masonry' is a more specific classification that is inferred, not stated."
  }
 ]
}
Trace line (JSON)
{
  "run_id": "2026-10-20-v6",
  "suite": "extraction-prompts",
  "case_id": "V-13",
  "variant": "v4",
  "at": "2026-10-21T01:09:39.797Z",
  "ok": true,
  "usage": {
    "inputTokens": 6,
    "outputTokens": 901,
    "costUsd": 0.009552300000000001,
    "calls": 3,
    "cacheReadTokens": 5303,
    "cacheCreationTokens": 0,
    "latencyMs": 12111,
    "metered": true,
    "provider": "claude-cli"
  },
  "checks": {
    "extraction_schema": true,
    "fields_correct": true,
    "decision_correct": true,
    "attribution_claude": true
  },
  "output_hash": "d04a03a8a7375d0e",
  "unanswered": [
    "attribution_jev"
  ],
  "extra": {
    "decision": "IN APPETITE",
    "fitScore": 92,
    "retries": 0,
    "fieldsWrong": 0,
    "modelErrors": ""
  },
  "stages": [
    {
      "name": "intake",
      "ok": true
    },
    {
      "name": "extraction",
      "ok": true
    },
    {
      "name": "grounding",
      "ok": true
    },
    {
      "name": "appetite",
      "ok": true
    }
  ],
  "cached": false
}
Full output record (JSON)
{
  "output": {
    "decision": "IN APPETITE",
    "expectedDecision": "IN APPETITE",
    "decisionCorrect": true,
    "fitScore": 92,
    "confidence": 0.8928571428571427,
    "reasons": [
      "Peachtree Walk sits comfortably inside appetite: GA is an allowed state, Fulton County carries no coastal restriction, and 18 units fall well under the 60-unit cap. TIV of $3.4M is mid-band, the 2009 joisted masonry build is preferred construction, and $30,500 GWP sits inside the premium band with five clean loss years. Bind with standard terms."
    ],
    "extraction": {
      "insuredName": {
        "value": "Peachtree Walk Residential LLC",
        "confidence": 0.98,
        "source": "email"
      },
      "propertyAddress": {
        "value": "1490 Peachtree Walk NE, Atlanta, GA 30309",
        "confidence": 0.97,
        "source": "email"
      },
      "state": {
        "value": "GA",
        "confidence": 0.98,
        "source": "email"
      },
      "county": {
        "value": "Fulton",
        "confidence": 0.95,
        "source": "email"
      },
      "units": {
        "value": 18,
        "confidence": 0.97,
        "source": "email"
      },
      "tiv": {
        "value": 3400000,
        "confidence": 0.9,
        "source": "email"
      },
      "constructionType": {
        "value": "Joisted Masonry",
        "confidence": 0.8,
        "source": "email"
      },
      "yearBuilt": {
        "value": 2009,
        "confidence": 0.97,
        "source": "email"
      },
      "targetPremium": {
        "value": 30500,
        "confidence": 0.97,
        "source": "email"
      },
      "subclass": {
        "value": null,
        "confidence": 0.9,
        "source": null
      },
      "priorLossSummary": {
        "value": "No losses in the past five years",
        "confidence": 0.95,
        "source": "email"
      }
    },
    "extracted": [
      {
        "field": "insuredName",
        "value": "Peachtree Walk Residential LLC",
        "source": "email"
      },
      {
        "field": "propertyAddress",
        "value": "1490 Peachtree Walk NE, Atlanta, GA 30309",
        "source": "email"
      },
      {
        "field": "state",
        "value": "GA",
        "source": "email"
      },
      {
        "field": "county",
        "value": "Fulton",
        "source": "email"
      },
      {
        "field": "units",
        "value": 18,
        "source": "email"
      },
      {
        "field": "tiv",
        "value": 3400000,
        "source": "email"
      },
      {
        "field": "constructionType",
        "value": "Joisted Masonry",
        "source": "email"
      },
      {
        "field": "yearBuilt",
        "value": 2009,
        "source": "email"
      },
      {
        "field": "targetPremium",
        "value": 30500,
        "source": "email"
      },
      {
        "field": "priorLossSummary",
        "value": "No losses in the past five years",
        "source": "email"
      }
    ],
    "fields": [
      {
        "field": "insuredName",
        "expected": "Peachtree Walk Residential LLC",
        "got": "Peachtree Walk Residential LLC",
        "correct": true
      },
      {
        "field": "state",
        "expected": "GA",
        "got": "GA",
        "correct": true
      },
      {
        "field": "county",
        "expected": "Fulton",
        "got": "Fulton",
        "correct": true
      },
      {
        "field": "units",
        "expected": 18,
        "got": 18,
        "correct": true
      },
      {
        "field": "tiv",
        "expected": 3400000,
        "got": 3400000,
        "correct": true
      },
      {
        "field": "yearBuilt",
        "expected": 2009,
        "got": 2009,
        "correct": true
      },
      {
        "field": "targetPremium",
        "expected": 30500,
        "got": 30500,
        "correct": true
      }
    ],
    "retries": 0,
    "guardrails": [
      {
        "node": "extraction",
        "guardrail": "schema",
        "status": "pass"
      },
      {
        "node": "grounding",
        "guardrail": "grounding",
        "status": "warn"
      },
      {
        "node": "appetite",
        "guardrail": "missing_fields",
        "status": "pass"
      },
      {
        "node": "appetite",
        "guardrail": "confidence",
        "status": "pass"
      }
    ],
    "flags": [
      {
        "field": "constructionType",
        "issue": "ambiguous",
        "detail": "not clearly supported by source: Email says only 'Masonry'; 'Joisted Masonry' is a more specific classification that is inferred, not stated."
      }
    ]
  },
  "usage": {
    "inputTokens": 6,
    "cacheReadTokens": 5303,
    "cacheCreationTokens": 0,
    "outputTokens": 901,
    "costUsd": 0.009552300000000001,
    "latencyMs": 12111,
    "calls": 3,
    "metered": true,
    "provider": "claude-cli"
  },
  "extra": {
    "decision": "IN APPETITE",
    "fitScore": 92,
    "retries": 0,
    "fieldsWrong": 0,
    "modelErrors": ""
  },
  "stages": [
    {
      "name": "intake",
      "ok": true
    },
    {
      "name": "extraction",
      "ok": true
    },
    {
      "name": "grounding",
      "ok": true
    },
    {
      "name": "appetite",
      "ok": true
    }
  ]
}

v5

result
ok
prompt tokens
5,437
output tokens
905
cost
$0.0221
latency
12,614 ms · 3 calls
at
2026-10-21 01:09
stages
intake extraction grounding appetite
decision
IN APPETITE
fitScore
90
retries
0
fieldsWrong
0
modelErrors
checkverdict
extraction_schemapass
fields_correctpass
construction_canonicalnot applicable
nulls_stay_nullnot applicable
decision_correctpass
attribution_jevunanswered
attribution_claudepass
{
 "decision": "IN APPETITE",
 "expectedDecision": "IN APPETITE",
 "decisionCorrect": true,
 "fitScore": 90,
 "confidence": 0.8985714285714285,
 "reasons": [
  "Peachtree Walk sits comfortably inside appetite: GA is an allowed state, 18 units is well under the 60-unit cap, and $3.4M TIV falls mid-band. Joisted masonry built in 2009 is preferred construction, Fulton County has no coastal restriction, and the $30,500 GWP target sits within the premium band. Five years loss-free supports a clean bind."
 ],
 "extraction": {
  "insuredName": {
   "value": "Peachtree Walk Residential LLC",
   "confidence": 0.98,
   "source": "email"
  },
  "propertyAddress": {
   "value": "1490 Peachtree Walk NE, Atlanta, GA 30309",
   "confidence": 0.97,
   "source": "email"
  },
  "state": {
   "value": "GA",
   "confidence": 0.98,
   "source": "email"
  },
  "county": {
   "value": "Fulton",
   "confidence": 0.95,
   "source": "email"
  },
  "units": {
   "value": 18,
   "confidence": 0.98,
   "source": "email"
  },
  "tiv": {
   "value": 3400000,
   "confidence": 0.9,
   "source": "email"
  },
  "constructionType": {
   "value": "Joisted Masonry",
   "confidence": 0.85,
   "source": "email"
  },
  "yearBuilt": {
   "value": 2009,
   "confidence": 0.97,
   "source": "email"
  },
  "targetPremium": {
   "value": 30500,
   "confidence": 0.97,
   "source": "email"
  },
  "subclass": {
   "value": null,
   "confidence": 0.9,
   "source": null
  },
  "priorLossSummary": {
   "value": "No losses in five years",
   "confidence": 0.95,
   "source": "email"
  }
 },
 "extracted": [
  {
   "field": "insuredName",
   "value": "Peachtree Walk Residential LLC",
   "source": "email"
  },
  {
   "field": "propertyAddress",
   "value": "1490 Peachtree Walk NE, Atlanta, GA 30309",
   "source": "email"
  },
  {
   "field": "state",
   "value": "GA",
   "source": "email"
  },
  {
   "field": "county",
   "value": "Fulton",
   "source": "email"
  },
  {
   "field": "units",
   "value": 18,
   "source": "email"
  },
  {
   "field": "tiv",
   "value": 3400000,
   "source": "email"
  },
  {
   "field": "constructionType",
   "value": "Joisted Masonry",
   "source": "email"
  },
  {
   "field": "yearBuilt",
   "value": 2009,
   "source": "email"
  },
  {
   "field": "targetPremium",
   "value": 30500,
   "source": "email"
  },
  {
   "field": "priorLossSummary",
   "value": "No losses in five years",
   "source": "email"
  }
 ],
 "fields": [
  {
   "field": "insuredName",
   "expected": "Peachtree Walk Residential LLC",
   "got": "Peachtree Walk Residential LLC",
   "correct": true
  },
  {
   "field": "state",
   "expected": "GA",
   "got": "GA",
   "correct": true
  },
  {
   "field": "county",
   "expected": "Fulton",
   "got": "Fulton",
   "correct": true
  },
  {
   "field": "units",
   "expected": 18,
   "got": 18,
   "correct": true
  },
  {
   "field": "tiv",
   "expected": 3400000,
   "got": 3400000,
   "correct": true
  },
  {
   "field": "yearBuilt",
   "expected": 2009,
   "got": 2009,
   "correct": true
  },
  {
   "field": "targetPremium",
   "expected": 30500,
   "got": 30500,
   "correct": true
  }
 ],
 "retries": 0,
 "guardrails": [
  {
   "node": "extraction",
   "guardrail": "schema",
   "status": "pass"
  },
  {
   "node": "grounding",
   "guardrail": "grounding",
   "status": "warn"
  },
  {
   "node": "appetite",
   "guardrail": "missing_fields",
   "status": "pass"
  },
  {
   "node": "appetite",
   "guardrail": "confidence",
   "status": "pass"
  }
 ],
 "flags": [
  {
   "field": "constructionType",
   "issue": "ambiguous",
   "detail": "not clearly supported by source: Email says only 'Masonry'; 'Joisted Masonry' is a more specific classification not stated in the source."
  }
 ]
}
Trace line (JSON)
{
  "run_id": "2026-10-20-v6",
  "suite": "extraction-prompts",
  "case_id": "V-13",
  "variant": "v5",
  "at": "2026-10-21T01:09:50.170Z",
  "ok": true,
  "usage": {
    "inputTokens": 6,
    "outputTokens": 905,
    "costUsd": 0.0221319,
    "calls": 3,
    "cacheReadTokens": 2219,
    "cacheCreationTokens": 3212,
    "latencyMs": 12614,
    "metered": true,
    "provider": "claude-cli"
  },
  "checks": {
    "extraction_schema": true,
    "fields_correct": true,
    "decision_correct": true,
    "attribution_claude": true
  },
  "output_hash": "28faef38e8b8a15e",
  "unanswered": [
    "attribution_jev"
  ],
  "extra": {
    "decision": "IN APPETITE",
    "fitScore": 90,
    "retries": 0,
    "fieldsWrong": 0,
    "modelErrors": ""
  },
  "stages": [
    {
      "name": "intake",
      "ok": true
    },
    {
      "name": "extraction",
      "ok": true
    },
    {
      "name": "grounding",
      "ok": true
    },
    {
      "name": "appetite",
      "ok": true
    }
  ],
  "cached": false
}
Full output record (JSON)
{
  "output": {
    "decision": "IN APPETITE",
    "expectedDecision": "IN APPETITE",
    "decisionCorrect": true,
    "fitScore": 90,
    "confidence": 0.8985714285714285,
    "reasons": [
      "Peachtree Walk sits comfortably inside appetite: GA is an allowed state, 18 units is well under the 60-unit cap, and $3.4M TIV falls mid-band. Joisted masonry built in 2009 is preferred construction, Fulton County has no coastal restriction, and the $30,500 GWP target sits within the premium band. Five years loss-free supports a clean bind."
    ],
    "extraction": {
      "insuredName": {
        "value": "Peachtree Walk Residential LLC",
        "confidence": 0.98,
        "source": "email"
      },
      "propertyAddress": {
        "value": "1490 Peachtree Walk NE, Atlanta, GA 30309",
        "confidence": 0.97,
        "source": "email"
      },
      "state": {
        "value": "GA",
        "confidence": 0.98,
        "source": "email"
      },
      "county": {
        "value": "Fulton",
        "confidence": 0.95,
        "source": "email"
      },
      "units": {
        "value": 18,
        "confidence": 0.98,
        "source": "email"
      },
      "tiv": {
        "value": 3400000,
        "confidence": 0.9,
        "source": "email"
      },
      "constructionType": {
        "value": "Joisted Masonry",
        "confidence": 0.85,
        "source": "email"
      },
      "yearBuilt": {
        "value": 2009,
        "confidence": 0.97,
        "source": "email"
      },
      "targetPremium": {
        "value": 30500,
        "confidence": 0.97,
        "source": "email"
      },
      "subclass": {
        "value": null,
        "confidence": 0.9,
        "source": null
      },
      "priorLossSummary": {
        "value": "No losses in five years",
        "confidence": 0.95,
        "source": "email"
      }
    },
    "extracted": [
      {
        "field": "insuredName",
        "value": "Peachtree Walk Residential LLC",
        "source": "email"
      },
      {
        "field": "propertyAddress",
        "value": "1490 Peachtree Walk NE, Atlanta, GA 30309",
        "source": "email"
      },
      {
        "field": "state",
        "value": "GA",
        "source": "email"
      },
      {
        "field": "county",
        "value": "Fulton",
        "source": "email"
      },
      {
        "field": "units",
        "value": 18,
        "source": "email"
      },
      {
        "field": "tiv",
        "value": 3400000,
        "source": "email"
      },
      {
        "field": "constructionType",
        "value": "Joisted Masonry",
        "source": "email"
      },
      {
        "field": "yearBuilt",
        "value": 2009,
        "source": "email"
      },
      {
        "field": "targetPremium",
        "value": 30500,
        "source": "email"
      },
      {
        "field": "priorLossSummary",
        "value": "No losses in five years",
        "source": "email"
      }
    ],
    "fields": [
      {
        "field": "insuredName",
        "expected": "Peachtree Walk Residential LLC",
        "got": "Peachtree Walk Residential LLC",
        "correct": true
      },
      {
        "field": "state",
        "expected": "GA",
        "got": "GA",
        "correct": true
      },
      {
        "field": "county",
        "expected": "Fulton",
        "got": "Fulton",
        "correct": true
      },
      {
        "field": "units",
        "expected": 18,
        "got": 18,
        "correct": true
      },
      {
        "field": "tiv",
        "expected": 3400000,
        "got": 3400000,
        "correct": true
      },
      {
        "field": "yearBuilt",
        "expected": 2009,
        "got": 2009,
        "correct": true
      },
      {
        "field": "targetPremium",
        "expected": 30500,
        "got": 30500,
        "correct": true
      }
    ],
    "retries": 0,
    "guardrails": [
      {
        "node": "extraction",
        "guardrail": "schema",
        "status": "pass"
      },
      {
        "node": "grounding",
        "guardrail": "grounding",
        "status": "warn"
      },
      {
        "node": "appetite",
        "guardrail": "missing_fields",
        "status": "pass"
      },
      {
        "node": "appetite",
        "guardrail": "confidence",
        "status": "pass"
      }
    ],
    "flags": [
      {
        "field": "constructionType",
        "issue": "ambiguous",
        "detail": "not clearly supported by source: Email says only 'Masonry'; 'Joisted Masonry' is a more specific classification not stated in the source."
      }
    ]
  },
  "usage": {
    "inputTokens": 6,
    "cacheReadTokens": 2219,
    "cacheCreationTokens": 3212,
    "outputTokens": 905,
    "costUsd": 0.0221319,
    "latencyMs": 12614,
    "calls": 3,
    "metered": true,
    "provider": "claude-cli"
  },
  "extra": {
    "decision": "IN APPETITE",
    "fitScore": 90,
    "retries": 0,
    "fieldsWrong": 0,
    "modelErrors": ""
  },
  "stages": [
    {
      "name": "intake",
      "ok": true
    },
    {
      "name": "extraction",
      "ok": true
    },
    {
      "name": "grounding",
      "ok": true
    },
    {
      "name": "appetite",
      "ok": true
    }
  ]
}

v6

result
ok
prompt tokens
5,476
output tokens
908
cost
$0.0124
latency
12,555 ms · 3 calls
at
2026-10-21 01:09
stages
intake extraction grounding appetite
decision
IN APPETITE
fitScore
90
retries
0
fieldsWrong
0
modelErrors
checkverdict
extraction_schemapass
fields_correctpass
construction_canonicalnot applicable
nulls_stay_nullnot applicable
decision_correctpass
attribution_jevunanswered
attribution_claudepass
{
 "decision": "IN APPETITE",
 "expectedDecision": "IN APPETITE",
 "decisionCorrect": true,
 "fitScore": 90,
 "confidence": 0.8971428571428569,
 "reasons": [
  "Peachtree Walk sits comfortably inside appetite: GA is an approved state, 18 units is well under the 60-unit cap, and $3.4M TIV falls mid-band. Joisted masonry built in 2009 is preferred construction, Fulton County has no coastal restriction, the $30,500 GWP target sits inside the premium band, and five years of clean losses support a bind."
 ],
 "extraction": {
  "insuredName": {
   "value": "Peachtree Walk Residential LLC",
   "confidence": 0.98,
   "source": "email"
  },
  "propertyAddress": {
   "value": "1490 Peachtree Walk NE, Atlanta, GA 30309",
   "confidence": 0.97,
   "source": "email"
  },
  "state": {
   "value": "GA",
   "confidence": 0.98,
   "source": "email"
  },
  "county": {
   "value": "Fulton",
   "confidence": 0.95,
   "source": "email"
  },
  "units": {
   "value": 18,
   "confidence": 0.97,
   "source": "email"
  },
  "tiv": {
   "value": 3400000,
   "confidence": 0.93,
   "source": "email"
  },
  "constructionType": {
   "value": "Joisted Masonry",
   "confidence": 0.8,
   "source": "email"
  },
  "yearBuilt": {
   "value": 2009,
   "confidence": 0.97,
   "source": "email"
  },
  "targetPremium": {
   "value": 30500,
   "confidence": 0.97,
   "source": "email"
  },
  "subclass": {
   "value": null,
   "confidence": 0.9,
   "source": null
  },
  "priorLossSummary": {
   "value": "No losses in the past five years",
   "confidence": 0.95,
   "source": "email"
  }
 },
 "extracted": [
  {
   "field": "insuredName",
   "value": "Peachtree Walk Residential LLC",
   "source": "email"
  },
  {
   "field": "propertyAddress",
   "value": "1490 Peachtree Walk NE, Atlanta, GA 30309",
   "source": "email"
  },
  {
   "field": "state",
   "value": "GA",
   "source": "email"
  },
  {
   "field": "county",
   "value": "Fulton",
   "source": "email"
  },
  {
   "field": "units",
   "value": 18,
   "source": "email"
  },
  {
   "field": "tiv",
   "value": 3400000,
   "source": "email"
  },
  {
   "field": "constructionType",
   "value": "Joisted Masonry",
   "source": "email"
  },
  {
   "field": "yearBuilt",
   "value": 2009,
   "source": "email"
  },
  {
   "field": "targetPremium",
   "value": 30500,
   "source": "email"
  },
  {
   "field": "priorLossSummary",
   "value": "No losses in the past five years",
   "source": "email"
  }
 ],
 "fields": [
  {
   "field": "insuredName",
   "expected": "Peachtree Walk Residential LLC",
   "got": "Peachtree Walk Residential LLC",
   "correct": true
  },
  {
   "field": "state",
   "expected": "GA",
   "got": "GA",
   "correct": true
  },
  {
   "field": "county",
   "expected": "Fulton",
   "got": "Fulton",
   "correct": true
  },
  {
   "field": "units",
   "expected": 18,
   "got": 18,
   "correct": true
  },
  {
   "field": "tiv",
   "expected": 3400000,
   "got": 3400000,
   "correct": true
  },
  {
   "field": "yearBuilt",
   "expected": 2009,
   "got": 2009,
   "correct": true
  },
  {
   "field": "targetPremium",
   "expected": 30500,
   "got": 30500,
   "correct": true
  }
 ],
 "retries": 0,
 "guardrails": [
  {
   "node": "extraction",
   "guardrail": "schema",
   "status": "pass"
  },
  {
   "node": "grounding",
   "guardrail": "grounding",
   "status": "warn"
  },
  {
   "node": "appetite",
   "guardrail": "missing_fields",
   "status": "pass"
  },
  {
   "node": "appetite",
   "guardrail": "confidence",
   "status": "pass"
  }
 ],
 "flags": [
  {
   "field": "constructionType",
   "issue": "ambiguous",
   "detail": "not clearly supported by source: Email says only 'Masonry'; 'Joisted Masonry' is a more specific classification not stated."
  }
 ]
}
Trace line (JSON)
{
  "run_id": "2026-10-20-v6",
  "suite": "extraction-prompts",
  "case_id": "V-13",
  "variant": "v6",
  "at": "2026-10-21T01:09:46.195Z",
  "ok": true,
  "usage": {
    "inputTokens": 6,
    "outputTokens": 908,
    "costUsd": 0.012372900000000001,
    "calls": 3,
    "cacheReadTokens": 4769,
    "cacheCreationTokens": 701,
    "latencyMs": 12555,
    "metered": true,
    "provider": "claude-cli"
  },
  "checks": {
    "extraction_schema": true,
    "fields_correct": true,
    "decision_correct": true,
    "attribution_claude": true
  },
  "output_hash": "5fc889091c2db17c",
  "unanswered": [
    "attribution_jev"
  ],
  "extra": {
    "decision": "IN APPETITE",
    "fitScore": 90,
    "retries": 0,
    "fieldsWrong": 0,
    "modelErrors": ""
  },
  "stages": [
    {
      "name": "intake",
      "ok": true
    },
    {
      "name": "extraction",
      "ok": true
    },
    {
      "name": "grounding",
      "ok": true
    },
    {
      "name": "appetite",
      "ok": true
    }
  ],
  "cached": false
}
Full output record (JSON)
{
  "output": {
    "decision": "IN APPETITE",
    "expectedDecision": "IN APPETITE",
    "decisionCorrect": true,
    "fitScore": 90,
    "confidence": 0.8971428571428569,
    "reasons": [
      "Peachtree Walk sits comfortably inside appetite: GA is an approved state, 18 units is well under the 60-unit cap, and $3.4M TIV falls mid-band. Joisted masonry built in 2009 is preferred construction, Fulton County has no coastal restriction, the $30,500 GWP target sits inside the premium band, and five years of clean losses support a bind."
    ],
    "extraction": {
      "insuredName": {
        "value": "Peachtree Walk Residential LLC",
        "confidence": 0.98,
        "source": "email"
      },
      "propertyAddress": {
        "value": "1490 Peachtree Walk NE, Atlanta, GA 30309",
        "confidence": 0.97,
        "source": "email"
      },
      "state": {
        "value": "GA",
        "confidence": 0.98,
        "source": "email"
      },
      "county": {
        "value": "Fulton",
        "confidence": 0.95,
        "source": "email"
      },
      "units": {
        "value": 18,
        "confidence": 0.97,
        "source": "email"
      },
      "tiv": {
        "value": 3400000,
        "confidence": 0.93,
        "source": "email"
      },
      "constructionType": {
        "value": "Joisted Masonry",
        "confidence": 0.8,
        "source": "email"
      },
      "yearBuilt": {
        "value": 2009,
        "confidence": 0.97,
        "source": "email"
      },
      "targetPremium": {
        "value": 30500,
        "confidence": 0.97,
        "source": "email"
      },
      "subclass": {
        "value": null,
        "confidence": 0.9,
        "source": null
      },
      "priorLossSummary": {
        "value": "No losses in the past five years",
        "confidence": 0.95,
        "source": "email"
      }
    },
    "extracted": [
      {
        "field": "insuredName",
        "value": "Peachtree Walk Residential LLC",
        "source": "email"
      },
      {
        "field": "propertyAddress",
        "value": "1490 Peachtree Walk NE, Atlanta, GA 30309",
        "source": "email"
      },
      {
        "field": "state",
        "value": "GA",
        "source": "email"
      },
      {
        "field": "county",
        "value": "Fulton",
        "source": "email"
      },
      {
        "field": "units",
        "value": 18,
        "source": "email"
      },
      {
        "field": "tiv",
        "value": 3400000,
        "source": "email"
      },
      {
        "field": "constructionType",
        "value": "Joisted Masonry",
        "source": "email"
      },
      {
        "field": "yearBuilt",
        "value": 2009,
        "source": "email"
      },
      {
        "field": "targetPremium",
        "value": 30500,
        "source": "email"
      },
      {
        "field": "priorLossSummary",
        "value": "No losses in the past five years",
        "source": "email"
      }
    ],
    "fields": [
      {
        "field": "insuredName",
        "expected": "Peachtree Walk Residential LLC",
        "got": "Peachtree Walk Residential LLC",
        "correct": true
      },
      {
        "field": "state",
        "expected": "GA",
        "got": "GA",
        "correct": true
      },
      {
        "field": "county",
        "expected": "Fulton",
        "got": "Fulton",
        "correct": true
      },
      {
        "field": "units",
        "expected": 18,
        "got": 18,
        "correct": true
      },
      {
        "field": "tiv",
        "expected": 3400000,
        "got": 3400000,
        "correct": true
      },
      {
        "field": "yearBuilt",
        "expected": 2009,
        "got": 2009,
        "correct": true
      },
      {
        "field": "targetPremium",
        "expected": 30500,
        "got": 30500,
        "correct": true
      }
    ],
    "retries": 0,
    "guardrails": [
      {
        "node": "extraction",
        "guardrail": "schema",
        "status": "pass"
      },
      {
        "node": "grounding",
        "guardrail": "grounding",
        "status": "warn"
      },
      {
        "node": "appetite",
        "guardrail": "missing_fields",
        "status": "pass"
      },
      {
        "node": "appetite",
        "guardrail": "confidence",
        "status": "pass"
      }
    ],
    "flags": [
      {
        "field": "constructionType",
        "issue": "ambiguous",
        "detail": "not clearly supported by source: Email says only 'Masonry'; 'Joisted Masonry' is a more specific classification not stated."
      }
    ]
  },
  "usage": {
    "inputTokens": 6,
    "cacheReadTokens": 4769,
    "cacheCreationTokens": 701,
    "outputTokens": 908,
    "costUsd": 0.012372900000000001,
    "latencyMs": 12555,
    "calls": 3,
    "metered": true,
    "provider": "claude-cli"
  },
  "extra": {
    "decision": "IN APPETITE",
    "fitScore": 90,
    "retries": 0,
    "fieldsWrong": 0,
    "modelErrors": ""
  },
  "stages": [
    {
      "name": "intake",
      "ok": true
    },
    {
      "name": "extraction",
      "ok": true
    },
    {
      "name": "grounding",
      "ok": true
    },
    {
      "name": "appetite",
      "ok": true
    }
  ]
}