Weigh evals you can watch
2026-10-20 22:33 UTC

GOLD-08

exchange-triage · run 2026-10-19-prompts

Input

What every variant was given, as the suite chose to show it.

Broker: Dana Kirkland, Kirkland Coverage Solutions

Hi there,

Quick one for your landlord program — Redbud Court LLC, a 12-unit apartment court at 2308 E 6th St, Tulsa, OK 74104 (Tulsa County). Brick veneer over frame, built 1995, smoke detectors in every unit.

Owner is refinancing and the lender needs proof of property coverage by the end of the month. Clean losses per the insured — loss runs requested from the prior carrier.

I don't have the updated appraisal yet so I can't give you a firm TIV — can you indicate subject to values? Target premium around $14,000.

Dana Kirkland
Kirkland Coverage Solutions

Outputs

One column per variant. Checks are the suite's binary checks on this output; a judge that gave no verdict is unanswered and not counted.

v1

result
ok
prompt tokens
4,400
output tokens
960
cost
$0.0252
latency
194,168 ms · 3 calls
at
2026-10-19 22:09
stages
intake extraction grounding appetite
decision
INCOMPLETE
fitScore
78
retries
0
fieldsWrong
0
checkverdict
extraction_schemapass
fields_correctpass
construction_canonicalnot applicable
nulls_stay_nullpass
decision_correctpass
attribution_jevfail
attribution_claudefail
{
 "decision": "INCOMPLETE",
 "expectedDecision": "INCOMPLETE",
 "decisionCorrect": true,
 "fitScore": 78,
 "confidence": 0.9649999999999999,
 "reasons": [
  "TIV (total insured value)"
 ],
 "extraction": {
  "insuredName": {
   "value": "Redbud Court LLC",
   "confidence": 0.97,
   "source": "email"
  },
  "propertyAddress": {
   "value": "2308 E 6th St, Tulsa, OK 74104",
   "confidence": 0.97,
   "source": "email"
  },
  "state": {
   "value": "OK",
   "confidence": 0.98,
   "source": "email"
  },
  "county": {
   "value": "Tulsa",
   "confidence": 0.95,
   "source": "email"
  },
  "units": {
   "value": 12,
   "confidence": 0.97,
   "source": "email"
  },
  "tiv": {
   "value": null,
   "confidence": 0.95,
   "source": null
  },
  "constructionType": {
   "value": "Brick veneer over frame",
   "confidence": 0.93,
   "source": "email"
  },
  "yearBuilt": {
   "value": 1995,
   "confidence": 0.97,
   "source": "email"
  },
  "targetPremium": {
   "value": 14000,
   "confidence": 0.85,
   "source": "email"
  },
  "subclass": {
   "value": "Garden apartments",
   "confidence": 0.5,
   "source": "email"
  },
  "priorLossSummary": {
   "value": "Clean losses per the insured; loss runs requested from prior carrier but not yet received.",
   "confidence": 0.85,
   "source": "email"
  }
 },
 "extracted": [
  {
   "field": "insuredName",
   "value": "Redbud Court LLC",
   "source": "email"
  },
  {
   "field": "propertyAddress",
   "value": "2308 E 6th St, Tulsa, OK 74104",
   "source": "email"
  },
  {
   "field": "state",
   "value": "OK",
   "source": "email"
  },
  {
   "field": "county",
   "value": "Tulsa",
   "source": "email"
  },
  {
   "field": "units",
   "value": 12,
   "source": "email"
  },
  {
   "field": "constructionType",
   "value": "Brick veneer over frame",
   "source": "email"
  },
  {
   "field": "yearBuilt",
   "value": 1995,
   "source": "email"
  },
  {
   "field": "targetPremium",
   "value": 14000,
   "source": "email"
  },
  {
   "field": "subclass",
   "value": "Garden apartments",
   "source": "email"
  },
  {
   "field": "priorLossSummary",
   "value": "Clean losses per the insured; loss runs requested from prior carrier but not yet received.",
   "source": "email"
  }
 ],
 "fields": [
  {
   "field": "insuredName",
   "expected": "Redbud Court LLC",
   "got": "Redbud Court LLC",
   "correct": true
  },
  {
   "field": "state",
   "expected": "OK",
   "got": "OK",
   "correct": true
  },
  {
   "field": "county",
   "expected": "Tulsa",
   "got": "Tulsa",
   "correct": true
  },
  {
   "field": "units",
   "expected": 12,
   "got": 12,
   "correct": true
  },
  {
   "field": "tiv",
   "expected": null,
   "got": null,
   "correct": true
  },
  {
   "field": "yearBuilt",
   "expected": 1995,
   "got": 1995,
   "correct": true
  },
  {
   "field": "targetPremium",
   "expected": 14000,
   "got": 14000,
   "correct": true
  }
 ],
 "retries": 0,
 "guardrails": [
  {
   "node": "extraction",
   "guardrail": "schema",
   "status": "pass"
  },
  {
   "node": "grounding",
   "guardrail": "grounding",
   "status": "warn"
  },
  {
   "node": "appetite",
   "guardrail": "missing_fields",
   "status": "warn"
  },
  {
   "node": "appetite",
   "guardrail": "confidence",
   "status": "pass"
  }
 ],
 "flags": [
  {
   "field": "subclass",
   "issue": "ambiguous",
   "detail": "not clearly supported by source: Email says 'apartment court' but never states 'Garden apartments'; this is an inferred classification."
  },
  {
   "field": "tiv",
   "issue": "missing",
   "detail": "TIV (total insured value) is required by the appetite ruleset"
  }
 ]
}

v2

result
ok
prompt tokens
4,816
output tokens
983
cost
$0.0237
latency
232,886 ms · 3 calls
at
2026-10-19 22:08
stages
intake extraction grounding appetite
decision
INCOMPLETE
fitScore
74
retries
0
fieldsWrong
0
checkverdict
extraction_schemapass
fields_correctpass
construction_canonicalnot applicable
nulls_stay_nullpass
decision_correctpass
attribution_jevfail
attribution_claudefail
{
 "decision": "INCOMPLETE",
 "expectedDecision": "INCOMPLETE",
 "decisionCorrect": true,
 "fitScore": 74,
 "confidence": 0.8999999999999999,
 "reasons": [
  "TIV (total insured value)"
 ],
 "extraction": {
  "insuredName": {
   "value": "Redbud Court LLC",
   "confidence": 0.97,
   "source": "email"
  },
  "propertyAddress": {
   "value": "2308 E 6th St, Tulsa, OK 74104",
   "confidence": 0.97,
   "source": "email"
  },
  "state": {
   "value": "OK",
   "confidence": 0.98,
   "source": "email"
  },
  "county": {
   "value": "Tulsa",
   "confidence": 0.97,
   "source": "email"
  },
  "units": {
   "value": 12,
   "confidence": 0.97,
   "source": "email"
  },
  "tiv": {
   "value": null,
   "confidence": 0.95,
   "source": null
  },
  "constructionType": {
   "value": "Frame",
   "confidence": 0.9,
   "source": "email"
  },
  "yearBuilt": {
   "value": 1995,
   "confidence": 0.97,
   "source": "email"
  },
  "targetPremium": {
   "value": 14000,
   "confidence": 0.9,
   "source": "email"
  },
  "subclass": {
   "value": "Garden apartments",
   "confidence": 0.55,
   "source": "email"
  },
  "priorLossSummary": {
   "value": "Clean losses per the insured; loss runs requested from prior carrier but not yet received.",
   "confidence": 0.85,
   "source": "email"
  }
 },
 "extracted": [
  {
   "field": "insuredName",
   "value": "Redbud Court LLC",
   "source": "email"
  },
  {
   "field": "propertyAddress",
   "value": "2308 E 6th St, Tulsa, OK 74104",
   "source": "email"
  },
  {
   "field": "state",
   "value": "OK",
   "source": "email"
  },
  {
   "field": "county",
   "value": "Tulsa",
   "source": "email"
  },
  {
   "field": "units",
   "value": 12,
   "source": "email"
  },
  {
   "field": "constructionType",
   "value": "Frame",
   "source": "email"
  },
  {
   "field": "yearBuilt",
   "value": 1995,
   "source": "email"
  },
  {
   "field": "targetPremium",
   "value": 14000,
   "source": "email"
  },
  {
   "field": "subclass",
   "value": "Garden apartments",
   "source": "email"
  },
  {
   "field": "priorLossSummary",
   "value": "Clean losses per the insured; loss runs requested from prior carrier but not yet received.",
   "source": "email"
  }
 ],
 "fields": [
  {
   "field": "insuredName",
   "expected": "Redbud Court LLC",
   "got": "Redbud Court LLC",
   "correct": true
  },
  {
   "field": "state",
   "expected": "OK",
   "got": "OK",
   "correct": true
  },
  {
   "field": "county",
   "expected": "Tulsa",
   "got": "Tulsa",
   "correct": true
  },
  {
   "field": "units",
   "expected": 12,
   "got": 12,
   "correct": true
  },
  {
   "field": "tiv",
   "expected": null,
   "got": null,
   "correct": true
  },
  {
   "field": "yearBuilt",
   "expected": 1995,
   "got": 1995,
   "correct": true
  },
  {
   "field": "targetPremium",
   "expected": 14000,
   "got": 14000,
   "correct": true
  }
 ],
 "retries": 0,
 "guardrails": [
  {
   "node": "extraction",
   "guardrail": "schema",
   "status": "pass"
  },
  {
   "node": "grounding",
   "guardrail": "grounding",
   "status": "warn"
  },
  {
   "node": "appetite",
   "guardrail": "missing_fields",
   "status": "warn"
  },
  {
   "node": "appetite",
   "guardrail": "confidence",
   "status": "pass"
  }
 ],
 "flags": [
  {
   "field": "constructionType",
   "issue": "ambiguous",
   "detail": "not clearly supported by source: Email says brick veneer over frame, which is not simply 'Frame'; the value drops the brick veneer exterior detail (likely masonry veneer/frame)."
  },
  {
   "field": "subclass",
   "issue": "ambiguous",
   "detail": "not clearly supported by source: Email says 'apartment court' and never mentions 'Garden apartments'; subclass is inferred."
  },
  {
   "field": "tiv",
   "issue": "missing",
   "detail": "TIV (total insured value) is required by the appetite ruleset"
  }
 ]
}

v2-repeat

result
ok
prompt tokens
4,816
output tokens
1,007
cost
$0.0240
latency
13,085 ms · 3 calls
at
2026-10-20 04:58
stages
intake extraction grounding appetite
decision
INCOMPLETE
fitScore
74
retries
0
fieldsWrong
0
modelErrors
checkverdict
extraction_schemapass
fields_correctpass
construction_canonicalnot applicable
nulls_stay_nullpass
decision_correctpass
attribution_jevfail
attribution_claudefail
{
 "decision": "INCOMPLETE",
 "expectedDecision": "INCOMPLETE",
 "decisionCorrect": true,
 "fitScore": 74,
 "confidence": 0.8999999999999999,
 "reasons": [
  "TIV (total insured value)"
 ],
 "extraction": {
  "insuredName": {
   "value": "Redbud Court LLC",
   "confidence": 0.97,
   "source": "email"
  },
  "propertyAddress": {
   "value": "2308 E 6th St, Tulsa, OK 74104",
   "confidence": 0.97,
   "source": "email"
  },
  "state": {
   "value": "OK",
   "confidence": 0.98,
   "source": "email"
  },
  "county": {
   "value": "Tulsa",
   "confidence": 0.97,
   "source": "email"
  },
  "units": {
   "value": 12,
   "confidence": 0.97,
   "source": "email"
  },
  "tiv": {
   "value": null,
   "confidence": 0.95,
   "source": null
  },
  "constructionType": {
   "value": "Frame",
   "confidence": 0.9,
   "source": "email"
  },
  "yearBuilt": {
   "value": 1995,
   "confidence": 0.97,
   "source": "email"
  },
  "targetPremium": {
   "value": 14000,
   "confidence": 0.85,
   "source": "email"
  },
  "subclass": {
   "value": "Garden apartments",
   "confidence": 0.5,
   "source": "email"
  },
  "priorLossSummary": {
   "value": "Clean losses per the insured; loss runs requested from prior carrier but not yet received.",
   "confidence": 0.8,
   "source": "email"
  }
 },
 "extracted": [
  {
   "field": "insuredName",
   "value": "Redbud Court LLC",
   "source": "email"
  },
  {
   "field": "propertyAddress",
   "value": "2308 E 6th St, Tulsa, OK 74104",
   "source": "email"
  },
  {
   "field": "state",
   "value": "OK",
   "source": "email"
  },
  {
   "field": "county",
   "value": "Tulsa",
   "source": "email"
  },
  {
   "field": "units",
   "value": 12,
   "source": "email"
  },
  {
   "field": "constructionType",
   "value": "Frame",
   "source": "email"
  },
  {
   "field": "yearBuilt",
   "value": 1995,
   "source": "email"
  },
  {
   "field": "targetPremium",
   "value": 14000,
   "source": "email"
  },
  {
   "field": "subclass",
   "value": "Garden apartments",
   "source": "email"
  },
  {
   "field": "priorLossSummary",
   "value": "Clean losses per the insured; loss runs requested from prior carrier but not yet received.",
   "source": "email"
  }
 ],
 "fields": [
  {
   "field": "insuredName",
   "expected": "Redbud Court LLC",
   "got": "Redbud Court LLC",
   "correct": true
  },
  {
   "field": "state",
   "expected": "OK",
   "got": "OK",
   "correct": true
  },
  {
   "field": "county",
   "expected": "Tulsa",
   "got": "Tulsa",
   "correct": true
  },
  {
   "field": "units",
   "expected": 12,
   "got": 12,
   "correct": true
  },
  {
   "field": "tiv",
   "expected": null,
   "got": null,
   "correct": true
  },
  {
   "field": "yearBuilt",
   "expected": 1995,
   "got": 1995,
   "correct": true
  },
  {
   "field": "targetPremium",
   "expected": 14000,
   "got": 14000,
   "correct": true
  }
 ],
 "retries": 0,
 "guardrails": [
  {
   "node": "extraction",
   "guardrail": "schema",
   "status": "pass"
  },
  {
   "node": "grounding",
   "guardrail": "grounding",
   "status": "warn"
  },
  {
   "node": "appetite",
   "guardrail": "missing_fields",
   "status": "warn"
  },
  {
   "node": "appetite",
   "guardrail": "confidence",
   "status": "pass"
  }
 ],
 "flags": [
  {
   "field": "constructionType",
   "issue": "ambiguous",
   "detail": "not clearly supported by source: Email says brick veneer over frame; 'Frame' alone omits the brick veneer exterior and may misclassify construction (e.g., masonry veneer vs. frame)."
  },
  {
   "field": "subclass",
   "issue": "ambiguous",
   "detail": "not clearly supported by source: Email says 'apartment court'; 'Garden apartments' is an inference not stated in the sources."
  },
  {
   "field": "tiv",
   "issue": "missing",
   "detail": "TIV (total insured value) is required by the appetite ruleset"
  }
 ]
}

v3

result
ok
prompt tokens
5,100
output tokens
953
cost
$0.0233
latency
242,741 ms · 3 calls
at
2026-10-19 22:09
stages
intake extraction grounding appetite
decision
INCOMPLETE
fitScore
72
retries
0
fieldsWrong
0
checkverdict
extraction_schemapass
fields_correctpass
construction_canonicalnot applicable
nulls_stay_nullpass
decision_correctpass
attribution_jevpass
attribution_claudefail
{
 "decision": "INCOMPLETE",
 "expectedDecision": "INCOMPLETE",
 "decisionCorrect": true,
 "fitScore": 72,
 "confidence": 0.8491666666666666,
 "reasons": [
  "TIV (total insured value)"
 ],
 "extraction": {
  "insuredName": {
   "value": "Redbud Court LLC",
   "confidence": 0.97,
   "source": "email"
  },
  "propertyAddress": {
   "value": "2308 E 6th St, Tulsa, OK 74104",
   "confidence": 0.97,
   "source": "email"
  },
  "state": {
   "value": "OK",
   "confidence": 0.98,
   "source": "email"
  },
  "county": {
   "value": "Tulsa",
   "confidence": 0.95,
   "source": "email"
  },
  "units": {
   "value": 12,
   "confidence": 0.95,
   "source": "email"
  },
  "tiv": {
   "value": null,
   "confidence": 0.95,
   "source": null
  },
  "constructionType": {
   "value": "Frame",
   "confidence": 0.85,
   "source": "email"
  },
  "yearBuilt": {
   "value": 1995,
   "confidence": 0.97,
   "source": "email"
  },
  "targetPremium": {
   "value": 14000,
   "confidence": 0.85,
   "source": "email"
  },
  "subclass": {
   "value": null,
   "confidence": 0.9,
   "source": null
  },
  "priorLossSummary": {
   "value": "Clean losses per the insured; loss runs requested from prior carrier but not yet received.",
   "confidence": 0.8,
   "source": "email"
  }
 },
 "extracted": [
  {
   "field": "insuredName",
   "value": "Redbud Court LLC",
   "source": "email"
  },
  {
   "field": "propertyAddress",
   "value": "2308 E 6th St, Tulsa, OK 74104",
   "source": "email"
  },
  {
   "field": "state",
   "value": "OK",
   "source": "email"
  },
  {
   "field": "county",
   "value": "Tulsa",
   "source": "email"
  },
  {
   "field": "units",
   "value": 12,
   "source": "email"
  },
  {
   "field": "constructionType",
   "value": "Frame",
   "source": "email"
  },
  {
   "field": "yearBuilt",
   "value": 1995,
   "source": "email"
  },
  {
   "field": "targetPremium",
   "value": 14000,
   "source": "email"
  },
  {
   "field": "priorLossSummary",
   "value": "Clean losses per the insured; loss runs requested from prior carrier but not yet received.",
   "source": "email"
  }
 ],
 "fields": [
  {
   "field": "insuredName",
   "expected": "Redbud Court LLC",
   "got": "Redbud Court LLC",
   "correct": true
  },
  {
   "field": "state",
   "expected": "OK",
   "got": "OK",
   "correct": true
  },
  {
   "field": "county",
   "expected": "Tulsa",
   "got": "Tulsa",
   "correct": true
  },
  {
   "field": "units",
   "expected": 12,
   "got": 12,
   "correct": true
  },
  {
   "field": "tiv",
   "expected": null,
   "got": null,
   "correct": true
  },
  {
   "field": "yearBuilt",
   "expected": 1995,
   "got": 1995,
   "correct": true
  },
  {
   "field": "targetPremium",
   "expected": 14000,
   "got": 14000,
   "correct": true
  }
 ],
 "retries": 0,
 "guardrails": [
  {
   "node": "extraction",
   "guardrail": "schema",
   "status": "pass"
  },
  {
   "node": "grounding",
   "guardrail": "grounding",
   "status": "warn"
  },
  {
   "node": "appetite",
   "guardrail": "missing_fields",
   "status": "warn"
  },
  {
   "node": "appetite",
   "guardrail": "confidence",
   "status": "pass"
  }
 ],
 "flags": [
  {
   "field": "constructionType",
   "issue": "contradiction",
   "detail": "Email says brick veneer over frame, which is not simply 'Frame' (masonry veneer vs. frame), so the value loses the veneer detail and may misclassify construction."
  },
  {
   "field": "tiv",
   "issue": "missing",
   "detail": "TIV (total insured value) is required by the appetite ruleset"
  }
 ]
}