Weigh evals you can watch
2026-10-20 22:34 UTC

V-04

exchange-triage · run 2026-10-19-prompts

Input

What every variant was given, as the suite chose to show it.

Broker: Variant desk

Hello,

Please review Copper Creek Residential LP — 48 doors across 3 buildings on one campus at 1600 Copper Creek Rd, Marietta, GA 30060 (Cobb County). JM construction throughout, built 2013, fully sprinklered.

TIV $7,400,000. Target premium $63,000. Clean losses.

Best.

Outputs

One column per variant. Checks are the suite's binary checks on this output; a judge that gave no verdict is unanswered and not counted.

v1

result
ok
prompt tokens
4,246
output tokens
1,335
cost
$0.0283
latency
15,194 ms · 3 calls
at
2026-10-20 04:59
stages
intake extraction grounding appetite
decision
IN APPETITE
fitScore
88
retries
0
fieldsWrong
0
modelErrors
checkverdict
extraction_schemapass
fields_correctpass
construction_canonicalpass
nulls_stay_nullnot applicable
decision_correctpass
attribution_jevfail
attribution_claudefail
{
 "decision": "IN APPETITE",
 "expectedDecision": "IN APPETITE",
 "decisionCorrect": true,
 "fitScore": 88,
 "confidence": 0.947142857142857,
 "reasons": [
  "Copper Creek sits comfortably inside appetite: GA is an allowed state, 48 units is under the 60-unit cap, and $7.4M TIV falls mid-band. Joisted masonry built in 2013 is acceptable construction, Cobb County carries no coastal restriction, and the $63,000 GWP target is within the premium band. Clean losses support a bind, though the missing loss detail keeps this just short of top-tier."
 ],
 "extraction": {
  "insuredName": {
   "value": "Copper Creek Residential LP",
   "confidence": 0.97,
   "source": "email"
  },
  "propertyAddress": {
   "value": "1600 Copper Creek Rd, Marietta, GA 30060",
   "confidence": 0.97,
   "source": "email"
  },
  "state": {
   "value": "GA",
   "confidence": 0.99,
   "source": "email"
  },
  "county": {
   "value": "Cobb",
   "confidence": 0.97,
   "source": "email"
  },
  "units": {
   "value": 48,
   "confidence": 0.9,
   "source": "email"
  },
  "tiv": {
   "value": 7400000,
   "confidence": 0.98,
   "source": "email"
  },
  "constructionType": {
   "value": "Joisted masonry",
   "confidence": 0.85,
   "source": "email"
  },
  "yearBuilt": {
   "value": 2013,
   "confidence": 0.97,
   "source": "email"
  },
  "targetPremium": {
   "value": 63000,
   "confidence": 0.98,
   "source": "email"
  },
  "subclass": {
   "value": "Garden apartments",
   "confidence": 0.5,
   "source": "email"
  },
  "priorLossSummary": {
   "value": "Clean losses (no details provided)",
   "confidence": 0.7,
   "source": "email"
  }
 },
 "extracted": [
  {
   "field": "insuredName",
   "value": "Copper Creek Residential LP",
   "source": "email"
  },
  {
   "field": "propertyAddress",
   "value": "1600 Copper Creek Rd, Marietta, GA 30060",
   "source": "email"
  },
  {
   "field": "state",
   "value": "GA",
   "source": "email"
  },
  {
   "field": "county",
   "value": "Cobb",
   "source": "email"
  },
  {
   "field": "units",
   "value": 48,
   "source": "email"
  },
  {
   "field": "tiv",
   "value": 7400000,
   "source": "email"
  },
  {
   "field": "constructionType",
   "value": "Joisted masonry",
   "source": "email"
  },
  {
   "field": "yearBuilt",
   "value": 2013,
   "source": "email"
  },
  {
   "field": "targetPremium",
   "value": 63000,
   "source": "email"
  },
  {
   "field": "subclass",
   "value": "Garden apartments",
   "source": "email"
  },
  {
   "field": "priorLossSummary",
   "value": "Clean losses (no details provided)",
   "source": "email"
  }
 ],
 "fields": [
  {
   "field": "insuredName",
   "expected": "Copper Creek Residential LP",
   "got": "Copper Creek Residential LP",
   "correct": true
  },
  {
   "field": "state",
   "expected": "GA",
   "got": "GA",
   "correct": true
  },
  {
   "field": "county",
   "expected": "Cobb",
   "got": "Cobb",
   "correct": true
  },
  {
   "field": "units",
   "expected": 48,
   "got": 48,
   "correct": true
  },
  {
   "field": "tiv",
   "expected": 7400000,
   "got": 7400000,
   "correct": true
  },
  {
   "field": "constructionType",
   "expected": "joisted masonry",
   "got": "Joisted masonry",
   "correct": true
  },
  {
   "field": "yearBuilt",
   "expected": 2013,
   "got": 2013,
   "correct": true
  },
  {
   "field": "targetPremium",
   "expected": 63000,
   "got": 63000,
   "correct": true
  }
 ],
 "retries": 0,
 "guardrails": [
  {
   "node": "extraction",
   "guardrail": "schema",
   "status": "pass"
  },
  {
   "node": "grounding",
   "guardrail": "grounding",
   "status": "warn"
  },
  {
   "node": "appetite",
   "guardrail": "missing_fields",
   "status": "pass"
  },
  {
   "node": "appetite",
   "guardrail": "confidence",
   "status": "pass"
  }
 ],
 "flags": [
  {
   "field": "subclass",
   "issue": "ambiguous",
   "detail": "not clearly supported by source: Email never says 'garden apartments'; it only mentions 48 doors in 3 buildings on one campus, so this is inferred."
  }
 ]
}

v2

result
ok
prompt tokens
4,671
output tokens
971
cost
$0.0230
latency
276,377 ms · 3 calls
at
2026-10-19 22:17
stages
intake extraction grounding appetite
decision
IN APPETITE
fitScore
88
retries
0
fieldsWrong
0
checkverdict
extraction_schemapass
fields_correctpass
construction_canonicalpass
nulls_stay_nullnot applicable
decision_correctpass
attribution_jevfail
attribution_claudefail
{
 "decision": "IN APPETITE",
 "expectedDecision": "IN APPETITE",
 "decisionCorrect": true,
 "fitScore": 88,
 "confidence": 0.9642857142857143,
 "reasons": [
  "Copper Creek sits comfortably inside appetite: GA is an allowed state, Cobb County has no coastal restriction, and all six rule checks passed. TIV of $7.4M is mid-band, 48 units stays under the 60-unit cap, the 2013 joisted masonry build is preferred construction, and the $63K target GWP falls well within the premium band. Clean losses support a bind; the 48-unit count, which is near the upper end of the unit limit, keeps this just short of the top tier."
 ],
 "extraction": {
  "insuredName": {
   "value": "Copper Creek Residential LP",
   "confidence": 0.98,
   "source": "email"
  },
  "propertyAddress": {
   "value": "1600 Copper Creek Rd, Marietta, GA 30060",
   "confidence": 0.97,
   "source": "email"
  },
  "state": {
   "value": "GA",
   "confidence": 0.99,
   "source": "email"
  },
  "county": {
   "value": "Cobb",
   "confidence": 0.98,
   "source": "email"
  },
  "units": {
   "value": 48,
   "confidence": 0.9,
   "source": "email"
  },
  "tiv": {
   "value": 7400000,
   "confidence": 0.98,
   "source": "email"
  },
  "constructionType": {
   "value": "Joisted Masonry",
   "confidence": 0.95,
   "source": "email"
  },
  "yearBuilt": {
   "value": 2013,
   "confidence": 0.98,
   "source": "email"
  },
  "targetPremium": {
   "value": 63000,
   "confidence": 0.98,
   "source": "email"
  },
  "subclass": {
   "value": "Garden apartments",
   "confidence": 0.45,
   "source": "email"
  },
  "priorLossSummary": {
   "value": "Clean losses",
   "confidence": 0.85,
   "source": "email"
  }
 },
 "extracted": [
  {
   "field": "insuredName",
   "value": "Copper Creek Residential LP",
   "source": "email"
  },
  {
   "field": "propertyAddress",
   "value": "1600 Copper Creek Rd, Marietta, GA 30060",
   "source": "email"
  },
  {
   "field": "state",
   "value": "GA",
   "source": "email"
  },
  {
   "field": "county",
   "value": "Cobb",
   "source": "email"
  },
  {
   "field": "units",
   "value": 48,
   "source": "email"
  },
  {
   "field": "tiv",
   "value": 7400000,
   "source": "email"
  },
  {
   "field": "constructionType",
   "value": "Joisted Masonry",
   "source": "email"
  },
  {
   "field": "yearBuilt",
   "value": 2013,
   "source": "email"
  },
  {
   "field": "targetPremium",
   "value": 63000,
   "source": "email"
  },
  {
   "field": "subclass",
   "value": "Garden apartments",
   "source": "email"
  },
  {
   "field": "priorLossSummary",
   "value": "Clean losses",
   "source": "email"
  }
 ],
 "fields": [
  {
   "field": "insuredName",
   "expected": "Copper Creek Residential LP",
   "got": "Copper Creek Residential LP",
   "correct": true
  },
  {
   "field": "state",
   "expected": "GA",
   "got": "GA",
   "correct": true
  },
  {
   "field": "county",
   "expected": "Cobb",
   "got": "Cobb",
   "correct": true
  },
  {
   "field": "units",
   "expected": 48,
   "got": 48,
   "correct": true
  },
  {
   "field": "tiv",
   "expected": 7400000,
   "got": 7400000,
   "correct": true
  },
  {
   "field": "constructionType",
   "expected": "joisted masonry",
   "got": "Joisted Masonry",
   "correct": true
  },
  {
   "field": "yearBuilt",
   "expected": 2013,
   "got": 2013,
   "correct": true
  },
  {
   "field": "targetPremium",
   "expected": 63000,
   "got": 63000,
   "correct": true
  }
 ],
 "retries": 0,
 "guardrails": [
  {
   "node": "extraction",
   "guardrail": "schema",
   "status": "pass"
  },
  {
   "node": "grounding",
   "guardrail": "grounding",
   "status": "warn"
  },
  {
   "node": "appetite",
   "guardrail": "missing_fields",
   "status": "pass"
  },
  {
   "node": "appetite",
   "guardrail": "confidence",
   "status": "pass"
  }
 ],
 "flags": [
  {
   "field": "subclass",
   "issue": "ambiguous",
   "detail": "not clearly supported by source: Email does not say 'Garden apartments'; it only mentions 48 doors across 3 buildings on one campus, so the subclass is inferred."
  }
 ]
}

v2-repeat

result
ok
prompt tokens
4,671
output tokens
947
cost
$0.0119
latency
178,130 ms · 3 calls
at
2026-10-19 22:18
stages
intake extraction grounding appetite
decision
IN APPETITE
fitScore
88
retries
0
fieldsWrong
0
checkverdict
extraction_schemapass
fields_correctpass
construction_canonicalpass
nulls_stay_nullnot applicable
decision_correctpass
attribution_jevfail
attribution_claudefail
{
 "decision": "IN APPETITE",
 "expectedDecision": "IN APPETITE",
 "decisionCorrect": true,
 "fitScore": 88,
 "confidence": 0.9628571428571429,
 "reasons": [
  "Copper Creek sits comfortably inside appetite: GA is an allowed state, Cobb County has no coastal restrictions, and all six rule checks passed. TIV of $7.4M is mid-band, 48 units sits under the 60-unit cap, the 2013 joisted masonry build is preferred construction, and losses are clean. The $63,000 target GWP falls well within the premium band, so this risk is ready to bind-quote. Unit count at 80% of the cap is the only minor friction."
 ],
 "extraction": {
  "insuredName": {
   "value": "Copper Creek Residential LP",
   "confidence": 0.98,
   "source": "email"
  },
  "propertyAddress": {
   "value": "1600 Copper Creek Rd, Marietta, GA 30060",
   "confidence": 0.97,
   "source": "email"
  },
  "state": {
   "value": "GA",
   "confidence": 0.99,
   "source": "email"
  },
  "county": {
   "value": "Cobb",
   "confidence": 0.98,
   "source": "email"
  },
  "units": {
   "value": 48,
   "confidence": 0.9,
   "source": "email"
  },
  "tiv": {
   "value": 7400000,
   "confidence": 0.98,
   "source": "email"
  },
  "constructionType": {
   "value": "Joisted Masonry",
   "confidence": 0.95,
   "source": "email"
  },
  "yearBuilt": {
   "value": 2013,
   "confidence": 0.97,
   "source": "email"
  },
  "targetPremium": {
   "value": 63000,
   "confidence": 0.98,
   "source": "email"
  },
  "subclass": {
   "value": "Garden apartments",
   "confidence": 0.5,
   "source": "email"
  },
  "priorLossSummary": {
   "value": "Clean losses",
   "confidence": 0.85,
   "source": "email"
  }
 },
 "extracted": [
  {
   "field": "insuredName",
   "value": "Copper Creek Residential LP",
   "source": "email"
  },
  {
   "field": "propertyAddress",
   "value": "1600 Copper Creek Rd, Marietta, GA 30060",
   "source": "email"
  },
  {
   "field": "state",
   "value": "GA",
   "source": "email"
  },
  {
   "field": "county",
   "value": "Cobb",
   "source": "email"
  },
  {
   "field": "units",
   "value": 48,
   "source": "email"
  },
  {
   "field": "tiv",
   "value": 7400000,
   "source": "email"
  },
  {
   "field": "constructionType",
   "value": "Joisted Masonry",
   "source": "email"
  },
  {
   "field": "yearBuilt",
   "value": 2013,
   "source": "email"
  },
  {
   "field": "targetPremium",
   "value": 63000,
   "source": "email"
  },
  {
   "field": "subclass",
   "value": "Garden apartments",
   "source": "email"
  },
  {
   "field": "priorLossSummary",
   "value": "Clean losses",
   "source": "email"
  }
 ],
 "fields": [
  {
   "field": "insuredName",
   "expected": "Copper Creek Residential LP",
   "got": "Copper Creek Residential LP",
   "correct": true
  },
  {
   "field": "state",
   "expected": "GA",
   "got": "GA",
   "correct": true
  },
  {
   "field": "county",
   "expected": "Cobb",
   "got": "Cobb",
   "correct": true
  },
  {
   "field": "units",
   "expected": 48,
   "got": 48,
   "correct": true
  },
  {
   "field": "tiv",
   "expected": 7400000,
   "got": 7400000,
   "correct": true
  },
  {
   "field": "constructionType",
   "expected": "joisted masonry",
   "got": "Joisted Masonry",
   "correct": true
  },
  {
   "field": "yearBuilt",
   "expected": 2013,
   "got": 2013,
   "correct": true
  },
  {
   "field": "targetPremium",
   "expected": 63000,
   "got": 63000,
   "correct": true
  }
 ],
 "retries": 0,
 "guardrails": [
  {
   "node": "extraction",
   "guardrail": "schema",
   "status": "pass"
  },
  {
   "node": "grounding",
   "guardrail": "grounding",
   "status": "warn"
  },
  {
   "node": "appetite",
   "guardrail": "missing_fields",
   "status": "pass"
  },
  {
   "node": "appetite",
   "guardrail": "confidence",
   "status": "pass"
  }
 ],
 "flags": [
  {
   "field": "subclass",
   "issue": "ambiguous",
   "detail": "not clearly supported by source: Email never says 'garden apartments'; this is inferred from the 3-building campus description."
  }
 ]
}

v3

result
ok
prompt tokens
4,955
output tokens
874
cost
$0.0220
latency
191,770 ms · 3 calls
at
2026-10-19 22:18
stages
intake extraction grounding appetite
decision
IN APPETITE
fitScore
88
retries
0
fieldsWrong
0
checkverdict
extraction_schemapass
fields_correctpass
construction_canonicalpass
nulls_stay_nullnot applicable
decision_correctpass
attribution_jevpass
attribution_claudepass
{
 "decision": "IN APPETITE",
 "expectedDecision": "IN APPETITE",
 "decisionCorrect": true,
 "fitScore": 88,
 "confidence": 0.9614285714285714,
 "reasons": [
  "Copper Creek Residential LP sits comfortably inside appetite: GA is an allowed state, 48 units is under the 60-unit cap, and $7.4M TIV falls mid-band. Joisted masonry built in 2013, clean losses, a non-coastal county (Cobb), and $63,000 target GWP within the premium band make this a strong candidate to bind. The only friction is 48 units sitting in the upper range of the unit limit."
 ],
 "extraction": {
  "insuredName": {
   "value": "Copper Creek Residential LP",
   "confidence": 0.97,
   "source": "email"
  },
  "propertyAddress": {
   "value": "1600 Copper Creek Rd, Marietta, GA 30060",
   "confidence": 0.97,
   "source": "email"
  },
  "state": {
   "value": "GA",
   "confidence": 0.99,
   "source": "email"
  },
  "county": {
   "value": "Cobb",
   "confidence": 0.97,
   "source": "email"
  },
  "units": {
   "value": 48,
   "confidence": 0.9,
   "source": "email"
  },
  "tiv": {
   "value": 7400000,
   "confidence": 0.98,
   "source": "email"
  },
  "constructionType": {
   "value": "Joisted Masonry",
   "confidence": 0.95,
   "source": "email"
  },
  "yearBuilt": {
   "value": 2013,
   "confidence": 0.97,
   "source": "email"
  },
  "targetPremium": {
   "value": 63000,
   "confidence": 0.98,
   "source": "email"
  },
  "subclass": {
   "value": null,
   "confidence": 0.9,
   "source": null
  },
  "priorLossSummary": {
   "value": "Clean losses",
   "confidence": 0.85,
   "source": "email"
  }
 },
 "extracted": [
  {
   "field": "insuredName",
   "value": "Copper Creek Residential LP",
   "source": "email"
  },
  {
   "field": "propertyAddress",
   "value": "1600 Copper Creek Rd, Marietta, GA 30060",
   "source": "email"
  },
  {
   "field": "state",
   "value": "GA",
   "source": "email"
  },
  {
   "field": "county",
   "value": "Cobb",
   "source": "email"
  },
  {
   "field": "units",
   "value": 48,
   "source": "email"
  },
  {
   "field": "tiv",
   "value": 7400000,
   "source": "email"
  },
  {
   "field": "constructionType",
   "value": "Joisted Masonry",
   "source": "email"
  },
  {
   "field": "yearBuilt",
   "value": 2013,
   "source": "email"
  },
  {
   "field": "targetPremium",
   "value": 63000,
   "source": "email"
  },
  {
   "field": "priorLossSummary",
   "value": "Clean losses",
   "source": "email"
  }
 ],
 "fields": [
  {
   "field": "insuredName",
   "expected": "Copper Creek Residential LP",
   "got": "Copper Creek Residential LP",
   "correct": true
  },
  {
   "field": "state",
   "expected": "GA",
   "got": "GA",
   "correct": true
  },
  {
   "field": "county",
   "expected": "Cobb",
   "got": "Cobb",
   "correct": true
  },
  {
   "field": "units",
   "expected": 48,
   "got": 48,
   "correct": true
  },
  {
   "field": "tiv",
   "expected": 7400000,
   "got": 7400000,
   "correct": true
  },
  {
   "field": "constructionType",
   "expected": "joisted masonry",
   "got": "Joisted Masonry",
   "correct": true
  },
  {
   "field": "yearBuilt",
   "expected": 2013,
   "got": 2013,
   "correct": true
  },
  {
   "field": "targetPremium",
   "expected": 63000,
   "got": 63000,
   "correct": true
  }
 ],
 "retries": 0,
 "guardrails": [
  {
   "node": "extraction",
   "guardrail": "schema",
   "status": "pass"
  },
  {
   "node": "grounding",
   "guardrail": "grounding",
   "status": "pass"
  },
  {
   "node": "appetite",
   "guardrail": "missing_fields",
   "status": "pass"
  },
  {
   "node": "appetite",
   "guardrail": "confidence",
   "status": "pass"
  }
 ],
 "flags": []
}