Weigh evals you can watch
2026-10-20 22:33 UTC

GOLD-10

exchange-triage · run 2026-10-19-prompts

Input

What every variant was given, as the suite chose to show it.

Broker: Gary Novak, Novak & Associates

Team,

Rush submission — insured's coverage lapses Friday. Seabreeze Duplexes LLC, 12 units of duplex-style rentals on Ocean View Dr (the insured says the mailing address is the manager's office, schedule has the property address). Frame construction, built around 2005 per the insured, though the county record the manager pulled says 1988.

No current appraisal — values are being updated, appraiser on site next week. Prior carrier non-renewed after the storm season. The insured says losses were minor cosmetic damage only.

Schedule from the property manager attached. Can you get me anything bindable by Thursday?

Gary Novak
Novak & Associates

ATTACHMENT:
location,address,state,county,units,construction,year_built,tiv
1,118 Ocean View Dr,,,4,Frame,1988,
2,122 Ocean View Dr,,,4,Frame,1988,
note: manager schedule shows 8 units total across 2 locations; state/county fields left blank by manager; values pending appraisal

Outputs

One column per variant. Checks are the suite's binary checks on this output; a judge that gave no verdict is unanswered and not counted.

v1

result
ok
prompt tokens
4,445
output tokens
1,561
cost
$0.0314
latency
44,796 ms · 3 calls
at
2026-10-19 22:10
stages
intake extraction grounding appetite
decision
REFERRED
fitScore
45
retries
0
fieldsWrong
0
checkverdict
extraction_schemapass
fields_correctpass
construction_canonicalnot applicable
nulls_stay_nullpass
decision_correctpass
attribution_jevpass
attribution_claudefail
{
 "decision": "REFERRED",
 "expectedDecision": "REFERRED",
 "decisionCorrect": true,
 "fitScore": 45,
 "confidence": 0.6325,
 "reasons": [
  "aggregate confidence 63% below 70% threshold",
  "missing required fields: state, unit count, TIV (total insured value)"
 ],
 "extraction": {
  "insuredName": {
   "value": "Seabreeze Duplexes LLC",
   "confidence": 0.95,
   "source": "email"
  },
  "propertyAddress": {
   "value": "118 Ocean View Dr; 122 Ocean View Dr",
   "confidence": 0.75,
   "source": "both"
  },
  "state": {
   "value": null,
   "confidence": 0,
   "source": null
  },
  "county": {
   "value": null,
   "confidence": 0,
   "source": null
  },
  "units": {
   "value": null,
   "confidence": 0.2,
   "source": "both"
  },
  "tiv": {
   "value": null,
   "confidence": 0,
   "source": null
  },
  "constructionType": {
   "value": "Frame",
   "confidence": 0.95,
   "source": "both"
  },
  "yearBuilt": {
   "value": 1988,
   "confidence": 0.6,
   "source": "both"
  },
  "targetPremium": {
   "value": null,
   "confidence": 0,
   "source": null
  },
  "subclass": {
   "value": "Duplex/fourplex schedule",
   "confidence": 0.85,
   "source": "both"
  },
  "priorLossSummary": {
   "value": "Insured states losses were minor cosmetic damage only; prior carrier non-renewed after storm season. No loss runs provided.",
   "confidence": 0.5,
   "source": "email"
  }
 },
 "extracted": [
  {
   "field": "insuredName",
   "value": "Seabreeze Duplexes LLC",
   "source": "email"
  },
  {
   "field": "propertyAddress",
   "value": "118 Ocean View Dr; 122 Ocean View Dr",
   "source": "both"
  },
  {
   "field": "constructionType",
   "value": "Frame",
   "source": "both"
  },
  {
   "field": "yearBuilt",
   "value": 1988,
   "source": "both"
  },
  {
   "field": "subclass",
   "value": "Duplex/fourplex schedule",
   "source": "both"
  },
  {
   "field": "priorLossSummary",
   "value": "Insured states losses were minor cosmetic damage only; prior carrier non-renewed after storm season. No loss runs provided.",
   "source": "email"
  }
 ],
 "fields": [
  {
   "field": "state",
   "expected": null,
   "got": null,
   "correct": true
  },
  {
   "field": "tiv",
   "expected": null,
   "got": null,
   "correct": true
  }
 ],
 "retries": 0,
 "guardrails": [
  {
   "node": "extraction",
   "guardrail": "schema",
   "status": "pass"
  },
  {
   "node": "grounding",
   "guardrail": "grounding",
   "status": "warn"
  },
  {
   "node": "appetite",
   "guardrail": "missing_fields",
   "status": "warn"
  },
  {
   "node": "appetite",
   "guardrail": "confidence",
   "status": "trip"
  }
 ],
 "flags": [
  {
   "field": "propertyAddress",
   "issue": "ambiguous",
   "detail": "address has no resolvable state or county — cannot place the risk"
  },
  {
   "field": "yearBuilt",
   "issue": "contradiction",
   "detail": "Email says insured reports ~2005 while the county record and attachment show 1988; sources conflict, so 1988 is not uncontested."
  },
  {
   "field": "subclass",
   "issue": "ambiguous",
   "detail": "not clearly supported by source: Sources describe duplex-style rentals; 'Duplex/fourplex schedule' is an inference, and the schedule's 4-unit locations are not described as fourplexes."
  },
  {
   "field": "state",
   "issue": "missing",
   "detail": "state is required by the appetite ruleset"
  },
  {
   "field": "units",
   "issue": "missing",
   "detail": "unit count is required by the appetite ruleset"
  },
  {
   "field": "tiv",
   "issue": "missing",
   "detail": "TIV (total insured value) is required by the appetite ruleset"
  }
 ]
}

v2

result
ok
prompt tokens
4,939
output tokens
1,957
cost
$0.0339
latency
50,302 ms · 3 calls
at
2026-10-19 22:10
stages
intake extraction grounding appetite
decision
REFERRED
fitScore
42
retries
0
fieldsWrong
0
checkverdict
extraction_schemapass
fields_correctpass
construction_canonicalnot applicable
nulls_stay_nullpass
decision_correctpass
attribution_jevpass
attribution_claudefail
{
 "decision": "REFERRED",
 "expectedDecision": "REFERRED",
 "decisionCorrect": true,
 "fitScore": 42,
 "confidence": 0.5459999999999999,
 "reasons": [
  "aggregate confidence 55% below 70% threshold",
  "missing required fields: state, TIV (total insured value)"
 ],
 "extraction": {
  "insuredName": {
   "value": "Seabreeze Duplexes LLC",
   "confidence": 0.97,
   "source": "email"
  },
  "propertyAddress": {
   "value": "118 Ocean View Dr; 122 Ocean View Dr",
   "confidence": 0.8,
   "source": "both"
  },
  "state": {
   "value": null,
   "confidence": 0,
   "source": null
  },
  "county": {
   "value": null,
   "confidence": 0,
   "source": null
  },
  "units": {
   "value": 8,
   "confidence": 0.4,
   "source": "attachment"
  },
  "tiv": {
   "value": null,
   "confidence": 0,
   "source": null
  },
  "constructionType": {
   "value": "Frame",
   "confidence": 0.95,
   "source": "both"
  },
  "yearBuilt": {
   "value": 1988,
   "confidence": 0.7,
   "source": "both"
  },
  "targetPremium": {
   "value": null,
   "confidence": 0,
   "source": null
  },
  "subclass": {
   "value": "Duplex/fourplex schedule",
   "confidence": 0.8,
   "source": "both"
  },
  "priorLossSummary": {
   "value": "Insured states losses were minor cosmetic damage only; prior carrier non-renewed after storm season. Not verified by loss runs.",
   "confidence": 0.5,
   "source": "email"
  }
 },
 "extracted": [
  {
   "field": "insuredName",
   "value": "Seabreeze Duplexes LLC",
   "source": "email"
  },
  {
   "field": "propertyAddress",
   "value": "118 Ocean View Dr; 122 Ocean View Dr",
   "source": "both"
  },
  {
   "field": "units",
   "value": 8,
   "source": "attachment"
  },
  {
   "field": "constructionType",
   "value": "Frame",
   "source": "both"
  },
  {
   "field": "yearBuilt",
   "value": 1988,
   "source": "both"
  },
  {
   "field": "subclass",
   "value": "Duplex/fourplex schedule",
   "source": "both"
  },
  {
   "field": "priorLossSummary",
   "value": "Insured states losses were minor cosmetic damage only; prior carrier non-renewed after storm season. Not verified by loss runs.",
   "source": "email"
  }
 ],
 "fields": [
  {
   "field": "state",
   "expected": null,
   "got": null,
   "correct": true
  },
  {
   "field": "tiv",
   "expected": null,
   "got": null,
   "correct": true
  }
 ],
 "retries": 0,
 "guardrails": [
  {
   "node": "extraction",
   "guardrail": "schema",
   "status": "pass"
  },
  {
   "node": "grounding",
   "guardrail": "grounding",
   "status": "warn"
  },
  {
   "node": "appetite",
   "guardrail": "missing_fields",
   "status": "warn"
  },
  {
   "node": "appetite",
   "guardrail": "confidence",
   "status": "trip"
  }
 ],
 "flags": [
  {
   "field": "propertyAddress",
   "issue": "ambiguous",
   "detail": "address has no resolvable state or county — cannot place the risk"
  },
  {
   "field": "units",
   "issue": "contradiction",
   "detail": "Email says 12 units, while the attachment rows (4+4) and its note say 8 units across 2 locations; the sources conflict."
  },
  {
   "field": "yearBuilt",
   "issue": "contradiction",
   "detail": "Insured says around 2005 per the email, while the county record and schedule say 1988; the extracted value matches the county record but the sources conflict."
  },
  {
   "field": "subclass",
   "issue": "ambiguous",
   "detail": "not clearly supported by source: Sources describe duplex-style rentals; 'Duplex/fourplex schedule' is an inference, though the schedule's 4-unit locations loosely suggest fourplexes."
  },
  {
   "field": "state",
   "issue": "missing",
   "detail": "state is required by the appetite ruleset"
  },
  {
   "field": "tiv",
   "issue": "missing",
   "detail": "TIV (total insured value) is required by the appetite ruleset"
  }
 ]
}

v2-repeat

result
ok
prompt tokens
4,927
output tokens
1,869
cost
$0.0309
latency
51,518 ms · 3 calls
at
2026-10-19 22:10
stages
intake extraction grounding appetite
decision
REFERRED
fitScore
42
retries
0
fieldsWrong
0
checkverdict
extraction_schemapass
fields_correctpass
construction_canonicalnot applicable
nulls_stay_nullpass
decision_correctpass
attribution_jevpass
attribution_claudefail
{
 "decision": "REFERRED",
 "expectedDecision": "REFERRED",
 "decisionCorrect": true,
 "fitScore": 42,
 "confidence": 0.607,
 "reasons": [
  "email states 12 units, attachment schedule totals 8",
  "aggregate confidence 61% below 70% threshold",
  "missing required fields: state, TIV (total insured value)"
 ],
 "extraction": {
  "insuredName": {
   "value": "Seabreeze Duplexes LLC",
   "confidence": 0.95,
   "source": "email"
  },
  "propertyAddress": {
   "value": "118 Ocean View Dr; 122 Ocean View Dr",
   "confidence": 0.8,
   "source": "both"
  },
  "state": {
   "value": null,
   "confidence": 0,
   "source": null
  },
  "county": {
   "value": null,
   "confidence": 0,
   "source": null
  },
  "units": {
   "value": 12,
   "confidence": 0.35,
   "source": "email"
  },
  "tiv": {
   "value": null,
   "confidence": 0,
   "source": null
  },
  "constructionType": {
   "value": "Frame",
   "confidence": 0.95,
   "source": "both"
  },
  "yearBuilt": {
   "value": 1988,
   "confidence": 0.55,
   "source": "both"
  },
  "targetPremium": {
   "value": null,
   "confidence": 0,
   "source": null
  },
  "subclass": {
   "value": "Duplex/fourplex schedule",
   "confidence": 0.85,
   "source": "both"
  },
  "priorLossSummary": {
   "value": "Insured states losses were minor cosmetic damage only (unverified); prior carrier non-renewed after storm season.",
   "confidence": 0.4,
   "source": "email"
  }
 },
 "extracted": [
  {
   "field": "insuredName",
   "value": "Seabreeze Duplexes LLC",
   "source": "email"
  },
  {
   "field": "propertyAddress",
   "value": "118 Ocean View Dr; 122 Ocean View Dr",
   "source": "both"
  },
  {
   "field": "units",
   "value": 12,
   "source": "email"
  },
  {
   "field": "constructionType",
   "value": "Frame",
   "source": "both"
  },
  {
   "field": "yearBuilt",
   "value": 1988,
   "source": "both"
  },
  {
   "field": "subclass",
   "value": "Duplex/fourplex schedule",
   "source": "both"
  },
  {
   "field": "priorLossSummary",
   "value": "Insured states losses were minor cosmetic damage only (unverified); prior carrier non-renewed after storm season.",
   "source": "email"
  }
 ],
 "fields": [
  {
   "field": "state",
   "expected": null,
   "got": null,
   "correct": true
  },
  {
   "field": "tiv",
   "expected": null,
   "got": null,
   "correct": true
  }
 ],
 "retries": 0,
 "guardrails": [
  {
   "node": "extraction",
   "guardrail": "schema",
   "status": "pass"
  },
  {
   "node": "grounding",
   "guardrail": "grounding",
   "status": "trip"
  },
  {
   "node": "appetite",
   "guardrail": "missing_fields",
   "status": "warn"
  },
  {
   "node": "appetite",
   "guardrail": "confidence",
   "status": "trip"
  }
 ],
 "flags": [
  {
   "field": "units",
   "issue": "contradiction",
   "detail": "email states 12 units, attachment schedule totals 8"
  },
  {
   "field": "propertyAddress",
   "issue": "ambiguous",
   "detail": "address has no resolvable state or county — cannot place the risk"
  },
  {
   "field": "units",
   "issue": "contradiction",
   "detail": "Broker email says 12 units, but the attachment's rows sum to 8 and its note says 8 units total across 2 locations."
  },
  {
   "field": "subclass",
   "issue": "ambiguous",
   "detail": "not clearly supported by source: No source uses a 'Duplex/fourplex schedule' classification. The email says duplex-style rentals, and the schedule shows 4-unit locations, so the label is inferred."
  },
  {
   "field": "state",
   "issue": "missing",
   "detail": "state is required by the appetite ruleset"
  },
  {
   "field": "tiv",
   "issue": "missing",
   "detail": "TIV (total insured value) is required by the appetite ruleset"
  }
 ]
}

v3

result
ok
prompt tokens
5,187
output tokens
1,702
cost
$0.0312
latency
55,004 ms · 3 calls
at
2026-10-19 22:10
stages
intake extraction grounding appetite
decision
REFERRED
fitScore
32
retries
0
fieldsWrong
0
checkverdict
extraction_schemapass
fields_correctpass
construction_canonicalnot applicable
nulls_stay_nullpass
decision_correctpass
attribution_jevpass
attribution_claudefail
{
 "decision": "REFERRED",
 "expectedDecision": "REFERRED",
 "decisionCorrect": true,
 "fitScore": 32,
 "confidence": 0.645,
 "reasons": [
  "aggregate confidence 65% below 70% threshold",
  "missing required fields: state, unit count, TIV (total insured value)"
 ],
 "extraction": {
  "insuredName": {
   "value": "Seabreeze Duplexes LLC",
   "confidence": 0.97,
   "source": "email"
  },
  "propertyAddress": {
   "value": "118 Ocean View Dr; 122 Ocean View Dr",
   "confidence": 0.8,
   "source": "both"
  },
  "state": {
   "value": null,
   "confidence": 0,
   "source": null
  },
  "county": {
   "value": null,
   "confidence": 0,
   "source": null
  },
  "units": {
   "value": null,
   "confidence": 0,
   "source": null
  },
  "tiv": {
   "value": null,
   "confidence": 0,
   "source": null
  },
  "constructionType": {
   "value": "Frame",
   "confidence": 0.95,
   "source": "both"
  },
  "yearBuilt": {
   "value": 1988,
   "confidence": 0.6,
   "source": "both"
  },
  "targetPremium": {
   "value": null,
   "confidence": 0,
   "source": null
  },
  "subclass": {
   "value": "Duplex/fourplex schedule",
   "confidence": 0.7,
   "source": "email"
  },
  "priorLossSummary": {
   "value": "Insured states losses were minor cosmetic damage only (unverified). Prior carrier non-renewed after storm season.",
   "confidence": 0.6,
   "source": "email"
  }
 },
 "extracted": [
  {
   "field": "insuredName",
   "value": "Seabreeze Duplexes LLC",
   "source": "email"
  },
  {
   "field": "propertyAddress",
   "value": "118 Ocean View Dr; 122 Ocean View Dr",
   "source": "both"
  },
  {
   "field": "constructionType",
   "value": "Frame",
   "source": "both"
  },
  {
   "field": "yearBuilt",
   "value": 1988,
   "source": "both"
  },
  {
   "field": "subclass",
   "value": "Duplex/fourplex schedule",
   "source": "email"
  },
  {
   "field": "priorLossSummary",
   "value": "Insured states losses were minor cosmetic damage only (unverified). Prior carrier non-renewed after storm season.",
   "source": "email"
  }
 ],
 "fields": [
  {
   "field": "state",
   "expected": null,
   "got": null,
   "correct": true
  },
  {
   "field": "tiv",
   "expected": null,
   "got": null,
   "correct": true
  }
 ],
 "retries": 0,
 "guardrails": [
  {
   "node": "extraction",
   "guardrail": "schema",
   "status": "pass"
  },
  {
   "node": "grounding",
   "guardrail": "grounding",
   "status": "warn"
  },
  {
   "node": "appetite",
   "guardrail": "missing_fields",
   "status": "warn"
  },
  {
   "node": "appetite",
   "guardrail": "confidence",
   "status": "trip"
  }
 ],
 "flags": [
  {
   "field": "propertyAddress",
   "issue": "ambiguous",
   "detail": "address has no resolvable state or county — cannot place the risk"
  },
  {
   "field": "yearBuilt",
   "issue": "contradiction",
   "detail": "Insured says around 2005 per the email, while the county record and the schedule say 1988; the extraction took 1988 without flagging the conflict."
  },
  {
   "field": "state",
   "issue": "missing",
   "detail": "state is required by the appetite ruleset"
  },
  {
   "field": "units",
   "issue": "missing",
   "detail": "unit count is required by the appetite ruleset"
  },
  {
   "field": "tiv",
   "issue": "missing",
   "detail": "TIV (total insured value) is required by the appetite ruleset"
  }
 ]
}